client_side.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 33 Client-side Routines */
10
60#include "squid.h"
61#include "acl/FilledChecklist.h"
62#include "anyp/PortCfg.h"
63#include "base/AsyncCallbacks.h"
64#include "base/Subscription.h"
65#include "base/TextException.h"
66#include "CachePeer.h"
67#include "client_db.h"
68#include "client_side.h"
69#include "client_side_reply.h"
70#include "client_side_request.h"
72#include "clientStream.h"
73#include "comm.h"
74#include "comm/Connection.h"
75#include "comm/Loops.h"
76#include "comm/Read.h"
77#include "comm/TcpAcceptor.h"
78#include "comm/Write.h"
79#include "CommCalls.h"
80#include "debug/Messages.h"
82#include "errorpage.h"
83#include "fd.h"
84#include "fde.h"
85#include "fqdncache.h"
86#include "FwdState.h"
87#include "globals.h"
88#include "helper.h"
89#include "helper/Reply.h"
90#include "http.h"
93#include "http/Stream.h"
94#include "HttpHdrContRange.h"
95#include "HttpHeaderTools.h"
96#include "HttpReply.h"
97#include "HttpRequest.h"
98#include "ident/Config.h"
99#include "ident/Ident.h"
100#include "internal.h"
101#include "ipc/FdNotes.h"
102#include "ipc/StartListening.h"
103#include "log/access_log.h"
104#include "MemBuf.h"
105#include "MemObject.h"
106#include "mime_header.h"
107#include "parser/Tokenizer.h"
108#include "proxyp/Header.h"
109#include "proxyp/Parser.h"
110#include "sbuf/Stream.h"
111#include "security/Certificate.h"
113#include "security/Io.h"
114#include "security/KeyLog.h"
116#include "servers/forward.h"
117#include "SquidConfig.h"
118#include "StatCounters.h"
119#include "StatHist.h"
120#include "Store.h"
121#include "TimeOrTag.h"
122#include "tools.h"
123
124#if USE_AUTH
125#include "auth/UserRequest.h"
126#endif
127#if USE_DELAY_POOLS
128#include "ClientInfo.h"
129#include "MessageDelayPools.h"
130#endif
131#if USE_OPENSSL
132#include "ssl/bio.h"
133#include "ssl/context_storage.h"
134#include "ssl/gadgets.h"
135#include "ssl/helper.h"
136#include "ssl/ProxyCerts.h"
137#include "ssl/ServerBump.h"
138#include "ssl/support.h"
139#endif
140
141#include <climits>
142#include <cmath>
143#include <limits>
144
145#if HAVE_SYSTEMD_SD_DAEMON_H
146#include <systemd/sd-daemon.h>
147#endif
148
149// TODO: Remove this custom dialer and simplify by creating the TcpAcceptor
150// subscription later, inside clientListenerConnectionOpened() callback, just
151// like htcpOpenPorts(), icpOpenPorts(), and snmpPortOpened() do it.
154 public CallDialer,
155 public WithAnswer<Ipc::StartListeningAnswer>
156{
157public:
160 handler(aHandler), portCfg(aPortCfg), portTypeNote(note), sub(aSub) {}
161
162 /* CallDialer API */
163 void print(std::ostream &os) const override {
164 os << '(' << answer_ << ", " << FdNote(portTypeNote) << " port=" << (void*)&portCfg << ')';
165 }
166
167 virtual bool canDial(AsyncCall &) const { return true; }
168 virtual void dial(AsyncCall &) { (handler)(portCfg, portTypeNote, sub); }
169
170 /* WithAnswer API */
172
173public:
175
176private:
177 // answer_.conn (set/updated by IPC code) is portCfg.listenConn (used by us)
182};
183
184static void clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub);
185
187#if USE_IDENT
189#endif
190static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength);
191
192static void clientUpdateStatHistCounters(const LogTags &logType, int svc_time);
193static void clientUpdateStatCounters(const LogTags &logType);
195static bool clientPingHasFinished(ping_data const *aPing);
198
199char *skipLeadingSpace(char *aString);
200
201#if USE_IDENT
202static void
203clientIdentDone(const char *ident, void *data)
204{
206 xstrncpy(conn->clientConnection->rfc931, ident ? ident : dash_str, USER_IDENT_SZ);
207}
208#endif
209
210void
212{
214
215 if (logType.isTcpHit())
217
218 if (logType.oldType == LOG_TCP_HIT)
220 else if (logType.oldType == LOG_TCP_MEM_HIT)
222}
224void
225clientUpdateStatHistCounters(const LogTags &logType, int svc_time)
226{
235 switch (logType.oldType) {
236
239 break;
240
241 case LOG_TCP_INM_HIT:
242 case LOG_TCP_IMS_HIT:
244 break;
245
246 case LOG_TCP_HIT:
247
248 case LOG_TCP_MEM_HIT:
249
252 break;
253
254 case LOG_TCP_MISS:
255
258 break;
259
260 default:
261 /* make compiler warnings go away */
262 break;
263 }
264}
265
266bool
268{
269 if (0 != aPing->stop.tv_sec && 0 != aPing->start.tv_sec)
270 return true;
271
272 return false;
273}
274
275void
277{
278 ping_data *i;
279
280 switch (someEntry->code) {
281#if USE_CACHE_DIGESTS
282
283 case CD_PARENT_HIT:
284
285 case CD_SIBLING_HIT:
287 break;
288#endif
289
290 case SIBLING_HIT:
291
292 case PARENT_HIT:
293
295
298 i = &someEntry->ping;
299
302
303 if (i->timeout)
305
306 break;
307
308 case CLOSEST_PARENT:
309
310 case CLOSEST_DIRECT:
312
313 break;
314
315 default:
316 break;
317 }
318}
319
320void
322{
324
325 if (request->error)
327
330
332}
333
334void
336{
337 assert(request);
338 assert(aLogEntry != nullptr);
339
341 MemBuf mb;
342 mb.init();
343 request->header.packInto(&mb);
344 //This is the request after adaptation or redirection
345 aLogEntry->headers.adapted_request = xstrdup(mb.buf);
346
347 // the virgin request is saved to aLogEntry->request
348 if (aLogEntry->request) {
349 mb.reset();
350 aLogEntry->request->header.packInto(&mb);
351 aLogEntry->headers.request = xstrdup(mb.buf);
352 }
353
354#if USE_ADAPTATION
355 const Adaptation::History::Pointer ah = request->adaptLogHistory();
356 if (ah != nullptr) {
357 mb.reset();
358 ah->lastMeta.packInto(&mb);
359 aLogEntry->adapt.last_meta = xstrdup(mb.buf);
360 }
361#endif
362
363 mb.clean();
364 }
365
366#if ICAP_CLIENT
368 if (ih != nullptr)
369 ih->processingTime(aLogEntry->icap.processingTime);
370#endif
371
372 aLogEntry->http.method = request->method;
373 aLogEntry->http.version = request->http_ver;
374 aLogEntry->hier = request->hier;
375 aLogEntry->cache.extuser = request->extacl_user.termedBuf();
376
377 // Adapted request, if any, inherits and then collects all the stats, but
378 // the virgin request gets logged instead; copy the stats to log them.
379 // TODO: avoid losses by keeping these stats in a shared history object?
380 if (aLogEntry->request) {
381 aLogEntry->request->dnsWait = request->dnsWait;
382 aLogEntry->request->error = request->error;
383 }
384}
385
386void
388{
389 if (!out.size && loggingTags().oldType == LOG_TAG_NONE)
390 debugs(33, 5, "logging half-baked transaction: " << log_uri);
391
393 al->url = log_uri;
394 debugs(33, 9, "clientLogRequest: al.url='" << al->url << "'");
395
396 const auto findReply = [this]() -> const HttpReply * {
397 if (al->reply)
398 return al->reply.getRaw();
399 if (const auto le = loggingEntry())
400 return le->hasFreshestReply();
401 return nullptr;
402 };
403 if (const auto reply = findReply()) {
404 al->http.code = reply->sline.status();
405 al->http.content_type = reply->content_type.termedBuf();
406 }
407
408 debugs(33, 9, "clientLogRequest: http.code='" << al->http.code << "'");
409
410 if (loggingEntry() && loggingEntry()->mem_obj && loggingEntry()->objectLen() >= 0)
411 al->cache.objectSize = loggingEntry()->contentLen(); // payload duplicate ?? with or without TE ?
412
414 // the virgin request is saved to al->request
415 if (al->request && al->request->body_pipe)
418 // XXX: calculate without payload encoding or headers !!
419 al->http.clientReplySz.payloadData = out.size - out.headers_sz; // pretend its all un-encoded data for now.
420
422
424
425 if (request)
427
428#if USE_OPENSSL && 0
429
430 /* This is broken. Fails if the connection has been closed. Needs
431 * to snarf the ssl details some place earlier..
432 */
433 if (getConn() != NULL)
435
436#endif
437
438 /* Add notes (if we have a request to annotate) */
439 if (request) {
440 SBuf matched;
441 for (auto h: Config.notes) {
442 if (h->match(request, al->reply.getRaw(), al, matched)) {
443 request->notes()->add(h->key(), matched);
444 debugs(33, 3, h->key() << " " << matched);
445 }
446 }
447 // The al->notes and request->notes must point to the same object.
449 }
450
451 ACLFilledChecklist checklist(nullptr, request, nullptr);
452 if (al->reply) {
453 checklist.reply = al->reply.getRaw();
454 HTTPMSGLOCK(checklist.reply);
455 }
456
457 if (request) {
461 }
462 // no need checklist.syncAle(): already synced
463 checklist.al = al;
464 accessLogLog(al, &checklist);
465
466 bool updatePerformanceCounters = true;
469 statsCheck.al = al;
470 if (al->reply) {
471 statsCheck.reply = al->reply.getRaw();
472 HTTPMSGLOCK(statsCheck.reply);
473 }
474 updatePerformanceCounters = statsCheck.fastCheck().allowed();
475 }
476
477 if (updatePerformanceCounters) {
478 if (request)
480
481 if (getConn() != nullptr && getConn()->clientConnection != nullptr)
482 clientdbUpdate(getConn()->clientConnection->remote, loggingTags(), AnyP::PROTO_HTTP, out.size);
483 }
484}
485
486void
488{
489 safe_free(uri);
492 clearRequest();
493
496}
497
498void
500{
501 ClientHttpRequest *http = (ClientHttpRequest *)data;
502 assert(http != nullptr);
503 delete http;
504}
505
506/* This is a handler normally called by comm_close() */
508{
509 if (clientConnection) {
511 // keep closed clientConnection for logging, clientdb cleanup, etc.
512 }
513 deleteThis("ConnStateData::connStateClosed");
514}
515
516#if USE_AUTH
517void
519{
520 if (auth_ == nullptr) {
521 if (aur != nullptr) {
522 debugs(33, 2, "Adding connection-auth to " << clientConnection << " from " << by);
523 auth_ = aur;
524 }
525 return;
526 }
527
528 // clobered with self-pointer
529 // NP: something nasty is going on in Squid, but harmless.
530 if (aur == auth_) {
531 debugs(33, 2, "WARNING: Ignoring duplicate connection-auth for " << clientConnection << " from " << by);
532 return;
533 }
534
535 /*
536 * Connection-auth relies on a single set of credentials being preserved
537 * for all requests on a connection once they have been setup.
538 * There are several things which need to happen to preserve security
539 * when connection-auth credentials change unexpectedly or are unset.
540 *
541 * 1) auth helper released from any active state
542 *
543 * They can only be reserved by a handshake process which this
544 * connection can now never complete.
545 * This prevents helpers hanging when their connections close.
546 *
547 * 2) pinning is expected to be removed and server conn closed
548 *
549 * The upstream link is authenticated with the same credentials.
550 * Expecting the same level of consistency we should have received.
551 * This prevents upstream being faced with multiple or missing
552 * credentials after authentication.
553 * NP: un-pin is left to the cleanup in ConnStateData::swanSong()
554 * we just trigger that cleanup here via comm_reset_close() or
555 * ConnStateData::stopReceiving()
556 *
557 * 3) the connection needs to close.
558 *
559 * This prevents attackers injecting requests into a connection,
560 * or gateways wrongly multiplexing users into a single connection.
561 *
562 * When credentials are missing closure needs to follow an auth
563 * challenge for best recovery by the client.
564 *
565 * When credentials change there is nothing we can do but abort as
566 * fast as possible. Sending TCP RST instead of an HTTP response
567 * is the best-case action.
568 */
569
570 // clobbered with nul-pointer
571 if (aur == nullptr) {
572 debugs(33, 2, "WARNING: Graceful closure on " << clientConnection << " due to connection-auth erase from " << by);
574 auth_ = nullptr;
575 // XXX: need to test whether the connection re-auth challenge is sent. If not, how to trigger it from here.
576 // NP: the current situation seems to fix challenge loops in Safari without visible issues in others.
577 // we stop receiving more traffic but can leave the Job running to terminate after the error or challenge is delivered.
578 stopReceiving("connection-auth removed");
579 return;
580 }
581
582 // clobbered with alternative credentials
583 if (aur != auth_) {
584 debugs(33, 2, "ERROR: Closing " << clientConnection << " due to change of connection-auth from " << by);
586 auth_ = nullptr;
587 // this is a fatal type of problem.
588 // Close the connection immediately with TCP RST to abort all traffic flow
590 return;
591 }
592
593 /* NOT REACHABLE */
594}
595#endif
596
597void
599{
601 AsyncCall::Pointer callback = JobCallback(33, 5, TimeoutDialer, this, ConnStateData::requestTimeout);
602 commSetConnTimeout(clientConnection, timeout, callback);
603}
604
605void
607{
609 AsyncCall::Pointer callback = JobCallback(5, 4, TimeoutDialer, this, ConnStateData::lifetimeTimeout);
611}
612
613// cleans up before destructor is called
614void
616{
617 debugs(33, 2, clientConnection);
618
619 flags.readMore = false;
620 clientdbEstablished(clientConnection->remote, -1); /* decrement */
621
623 checkLogging();
624
625 // XXX: Closing pinned conn is too harsh: The Client may want to continue!
626 unpinConnection(true);
627
629
630#if USE_AUTH
631 // NP: do this bit after closing the connections to avoid side effects from unwanted TCP RST
632 setAuth(nullptr, "ConnStateData::SwanSong cleanup");
633#endif
634
635 flags.swanSang = true;
636}
637
638void
639ConnStateData::callException(const std::exception &ex)
640{
641 Server::callException(ex); // logs ex and stops the job
642
643 ErrorDetail::Pointer errorDetail;
644 if (const auto tex = dynamic_cast<const TextException*>(&ex))
645 errorDetail = new ExceptionErrorDetail(tex->id());
646 else
647 errorDetail = new ExceptionErrorDetail(Here().id());
648 updateError(ERR_GATEWAY_FAILURE, errorDetail);
649}
650
651void
653{
654 if (const auto context = pipeline.front()) {
655 const auto http = context->http;
656 assert(http);
657 http->updateError(error);
658 } else {
660 }
661}
662
663bool
665{
666 return cbdataReferenceValid(this) && // XXX: checking "this" in a method
668 !fd_table[clientConnection->fd].closing();
669}
670
672{
673 debugs(33, 3, clientConnection);
674
675 if (isOpen())
676 debugs(33, DBG_IMPORTANT, "ERROR: Squid BUG: ConnStateData did not close " << clientConnection);
677
678 if (!flags.swanSang)
679 debugs(33, DBG_IMPORTANT, "ERROR: Squid BUG: ConnStateData was not destroyed properly; " << clientConnection);
680
681 if (bodyPipe != nullptr)
683
684 delete bodyParser; // TODO: pool
685
686#if USE_OPENSSL
687 delete sslServerBump;
688#endif
689}
690
697void
699{
700 HttpRequest *request = http->request;
701
702 debugs(33, 3, "http_ver = " << request->http_ver);
703 debugs(33, 3, "method = " << request->method);
704
705 // TODO: move to HttpRequest::hdrCacheInit, just like HttpReply.
706 request->flags.proxyKeepalive = request->persistent();
707}
708
709int
711{
713 bodyLength > Config.maxRequestBodySize)
714 return 1; /* too large */
715
716 return 0;
717}
718
719bool
721{
723}
724
725void
727{
728 mb->appendf("\r\n--" SQUIDSTRINGPH "--\r\n", SQUIDSTRINGPRINT(boundary));
729 debugs(33, 6, "buf offset: " << mb->size);
730}
731
732void
733clientPackRangeHdr(const HttpReplyPointer &rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb)
734{
735 HttpHeader hdr(hoReply);
736 assert(rep);
737 assert(spec);
738
739 /* put boundary */
740 debugs(33, 5, "appending boundary: " << boundary);
741 /* rfc2046 requires to _prepend_ boundary with <crlf>! */
742 mb->appendf("\r\n--" SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(boundary));
743
744 /* stuff the header with required entries and pack it */
745
748
749 httpHeaderAddContRange(&hdr, *spec, rep->content_length);
750
751 hdr.packInto(mb);
752 hdr.clean();
753
754 /* append <crlf> (we packed a header, not a reply) */
755 mb->append("\r\n", 2);
756}
757
763int64_t
765{
766 int64_t clen = 0;
767 MemBuf mb;
768
769 assert(memObject());
770
771 mb.init();
773
774 while (pos != request->range->end()) {
775 /* account for headers for this range */
776 mb.reset();
777 clientPackRangeHdr(&storeEntry()->mem().freshestReply(),
778 *pos, range_iter.boundary, &mb);
779 clen += mb.size;
780
781 /* account for range content */
782 clen += (*pos)->length;
783
784 debugs(33, 6, "clientMRangeCLen: (clen += " << mb.size << " + " << (*pos)->length << ") == " << clen);
785 ++pos;
786 }
787
788 /* account for the terminating boundary */
789 mb.reset();
790
792
793 clen += mb.size;
794
795 mb.clean();
796
797 return clen;
798}
799
803String
805{
806 const char *key;
808 b.append(":",1);
809 key = storeEntry()->getMD5Text();
810 b.append(key, strlen(key));
811 return b;
812}
813
823void
825 HttpReply * rep, StoreIOBuffer receivedData)
826{
827 // do not try to deliver if client already ABORTED
828 if (!http->getConn() || !cbdataReferenceValid(http->getConn()) || !Comm::IsConnOpen(http->getConn()->clientConnection))
829 return;
830
831 /* Test preconditions */
832 assert(node != nullptr);
833 /* TODO: handle this rather than asserting
834 * - it should only ever happen if we cause an abort and
835 * the callback chain loops back to here, so we can simply return.
836 * However, that itself shouldn't happen, so it stays as an assert for now.
837 */
839 assert(node->node.next == nullptr);
840 Http::StreamPointer context = dynamic_cast<Http::Stream *>(node->data.getRaw());
841 assert(context != nullptr);
842
843 /* TODO: check offset is what we asked for */
844
845 // TODO: enforces HTTP/1 MUST on pipeline order, but is irrelevant to HTTP/2
846 if (context != http->getConn()->pipeline.front())
847 context->deferRecipientForLater(node, rep, receivedData);
848 else if (http->getConn()->cbControlMsgSent) // 1xx to the user is pending
849 context->deferRecipientForLater(node, rep, receivedData);
850 else
851 http->getConn()->handleReply(rep, receivedData);
852}
853
859void
861{
862 /* Test preconditions */
863 assert(node != nullptr);
864 /* TODO: handle this rather than asserting
865 * - it should only ever happen if we cause an abort and
866 * the callback chain loops back to here, so we can simply return.
867 * However, that itself shouldn't happen, so it stays as an assert for now.
868 */
870 /* Set null by ContextFree */
871 assert(node->node.next == nullptr);
872 /* this is the assert discussed above */
873 assert(nullptr == dynamic_cast<Http::Stream *>(node->data.getRaw()));
874 /* We are only called when the client socket shutsdown.
875 * Tell the prev pipeline member we're finished
876 */
878}
879
880void
882{
883 debugs(33, 5, clientConnection << " reading next req");
884
885 fd_note(clientConnection->fd, "Idle client: Waiting for next request");
890
891 readSomeData();
893}
894
895static void
897{
898 debugs(33, 2, conn->clientConnection << " Sending next");
899
902 if (deferredRequest->flags.deferred) {
904 assert(deferredRequest->http->out.size == 0);
906 clientSocketRecipient(deferredRequest->deferredparams.node,
907 deferredRequest->http,
908 deferredRequest->deferredparams.rep,
909 deferredRequest->deferredparams.queuedBuffer);
910 }
911
915}
916
917void
919{
921 debugs(33, 2, clientConnection << " Connection was closed");
922 return;
923 }
924
925 if (pinning.pinned && !Comm::IsConnOpen(pinning.serverConnection)) {
926 debugs(33, 2, clientConnection << " Connection was pinned but server side gone. Terminating client connection");
928 return;
929 }
930
945 if (const char *reason = stoppedReceiving()) {
946 debugs(33, 3, "closing for earlier request error: " << reason);
948 return;
949 }
950
958
959 if (!isOpen())
960 return;
961
969 Http::StreamPointer deferredRequest = pipeline.front();
970 if (deferredRequest != nullptr) {
971 debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded");
972 ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
973 } else if (flags.readMore) {
974 debugs(33, 3, clientConnection << ": calling readNextRequest()");
976 } else {
977 // XXX: Can this happen? CONNECT tunnels have deferredRequest set.
978 debugs(33, DBG_IMPORTANT, MYNAME << "abandoning " << clientConnection);
979 }
980}
981
982void
984{
985 debugs(33, 4, "sending error (" << clientConnection << "): " << error <<
986 "; old receiving error: " <<
987 (stoppedReceiving() ? stoppedReceiving_ : "none"));
988
989 if (const char *oldError = stoppedSending()) {
990 debugs(33, 3, "already stopped sending: " << oldError);
991 return; // nothing has changed as far as this connection is concerned
992 }
994
995 if (!stoppedReceiving()) {
996 if (const int64_t expecting = mayNeedToReadMoreBody()) {
997 debugs(33, 5, "must still read " << expecting <<
998 " request body bytes with " << inBuf.length() << " unused");
999 return; // wait for the request receiver to finish reading
1000 }
1001 }
1002
1004}
1005
1006void
1008{
1009 if (pipeline.empty())
1010 return;
1011
1012 auto ctx = pipeline.front();
1013 if (size) {
1015 if (ctx->http->loggingTags().isTcpHit())
1017 }
1018 ctx->writeComplete(size);
1019}
1020
1023{
1024 ClientHttpRequest *http = new ClientHttpRequest(this);
1025 http->req_sz = inBuf.length();
1026 http->setErrorUri(uri);
1027 auto *context = new Http::Stream(clientConnection, http);
1028 StoreIOBuffer tempBuffer;
1029 tempBuffer.data = context->reqbuf;
1030 tempBuffer.length = HTTP_REQBUF_SZ;
1033 clientSocketDetach, context, tempBuffer);
1034 return context;
1035}
1036
1037void
1039{
1040 // RegisteredRunner API callback - Squid has been shut down
1041
1042 // if connection is idle terminate it now,
1043 // otherwise wait for grace period to end
1044 if (pipeline.empty())
1046}
1047
1048void
1050{
1051 // RegisteredRunner API callback - Squid shutdown grace period is over
1052
1053 // force the client connection to close immediately
1054 // swanSong() in the close handler will cleanup.
1057}
1058
1059char *
1060skipLeadingSpace(char *aString)
1061{
1062 char *result = aString;
1063
1064 while (xisspace(*aString))
1065 ++aString;
1066
1067 return result;
1068}
1069
1075const char *
1076findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end)
1077{
1078 if (nullptr == end) {
1079 end = uriAndHTTPVersion + strcspn(uriAndHTTPVersion, "\r\n");
1080 assert(end);
1081 }
1082
1083 for (; end > uriAndHTTPVersion; --end) {
1084 if (*end == '\n' || *end == '\r')
1085 continue;
1086
1087 if (xisspace(*end)) {
1088 if (strncasecmp(end + 1, "HTTP/", 5) == 0)
1089 return end + 1;
1090 else
1091 break;
1092 }
1093 }
1094
1095 return nullptr;
1096}
1097
1098static char *
1100{
1101 int vhost = conn->port->vhost;
1102 int vport = conn->port->vport;
1103 static char ipbuf[MAX_IPSTRLEN];
1104
1105 /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1106
1107 // XXX: re-use proper URL parser for this
1108 SBuf url = hp->requestUri(); // use full provided URI if we abort
1109 do { // use a loop so we can break out of it
1111 if (tok.skip('/')) // origin-form URL already.
1112 break;
1113
1114 if (conn->port->vhost)
1115 return nullptr; /* already in good shape */
1116
1117 // skip the URI scheme
1118 static const CharacterSet uriScheme = CharacterSet("URI-scheme","+-.") + CharacterSet::ALPHA + CharacterSet::DIGIT;
1119 static const SBuf uriSchemeEnd("://");
1120 if (!tok.skipAll(uriScheme) || !tok.skip(uriSchemeEnd))
1121 break;
1122
1123 // skip the authority segment
1124 // RFC 3986 complex nested ABNF for "authority" boils down to this:
1125 static const CharacterSet authority = CharacterSet("authority","-._~%:@[]!$&'()*+,;=") +
1127 if (!tok.skipAll(authority))
1128 break;
1129
1130 static const SBuf slashUri("/");
1131 const SBuf t = tok.remaining();
1132 if (t.isEmpty())
1133 url = slashUri;
1134 else if (t[0]=='/') // looks like path
1135 url = t;
1136 else if (t[0]=='?' || t[0]=='#') { // looks like query or fragment. fix '/'
1137 url = slashUri;
1138 url.append(t);
1139 } // else do nothing. invalid path
1140
1141 } while(false);
1142
1143#if SHOULD_REJECT_UNKNOWN_URLS
1144 // reject URI which are not well-formed even after the processing above
1145 if (url.isEmpty() || url[0] != '/') {
1146 hp->parseStatusCode = Http::scBadRequest;
1147 return conn->abortRequestParsing("error:invalid-request");
1148 }
1149#endif
1150
1151 if (vport < 0)
1152 vport = conn->clientConnection->local.port();
1153
1154 char *receivedHost = nullptr;
1155 if (vhost && (receivedHost = hp->getHostHeaderField())) {
1156 SBuf host(receivedHost);
1157 debugs(33, 5, "ACCEL VHOST REWRITE: vhost=" << host << " + vport=" << vport);
1158 if (vport > 0) {
1159 // remove existing :port (if any), cope with IPv6+ without port
1160 const auto lastColonPos = host.rfind(':');
1161 if (lastColonPos != SBuf::npos && *host.rbegin() != ']') {
1162 host.chop(0, lastColonPos); // truncate until the last colon
1163 }
1164 host.appendf(":%d", vport);
1165 } // else nothing to alter port-wise.
1166 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1167 const auto url_sz = scheme.length() + host.length() + url.length() + 32;
1168 char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1169 snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH SQUIDSBUFPH, SQUIDSBUFPRINT(scheme), SQUIDSBUFPRINT(host), SQUIDSBUFPRINT(url));
1170 debugs(33, 5, "ACCEL VHOST REWRITE: " << uri);
1171 return uri;
1172 } else if (conn->port->defaultsite /* && !vhost */) {
1173 debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: defaultsite=" << conn->port->defaultsite << " + vport=" << vport);
1174 char vportStr[32];
1175 vportStr[0] = '\0';
1176 if (vport > 0) {
1177 snprintf(vportStr, sizeof(vportStr),":%d",vport);
1178 }
1179 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1180 const int url_sz = scheme.length() + strlen(conn->port->defaultsite) + sizeof(vportStr) + url.length() + 32;
1181 char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1182 snprintf(uri, url_sz, SQUIDSBUFPH "://%s%s" SQUIDSBUFPH,
1183 SQUIDSBUFPRINT(scheme), conn->port->defaultsite, vportStr, SQUIDSBUFPRINT(url));
1184 debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: " << uri);
1185 return uri;
1186 } else if (vport > 0 /* && (!vhost || no Host:) */) {
1187 debugs(33, 5, "ACCEL VPORT REWRITE: *_port IP + vport=" << vport);
1188 /* Put the local socket IP address as the hostname, with whatever vport we found */
1189 conn->clientConnection->local.toHostStr(ipbuf,MAX_IPSTRLEN);
1190 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1191 const int url_sz = scheme.length() + sizeof(ipbuf) + url.length() + 32;
1192 char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1193 snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1194 SQUIDSBUFPRINT(scheme), ipbuf, vport, SQUIDSBUFPRINT(url));
1195 debugs(33, 5, "ACCEL VPORT REWRITE: " << uri);
1196 return uri;
1197 }
1198
1199 return nullptr;
1200}
1201
1202static char *
1204{
1205 char *uri = nullptr;
1206 /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1207 if (const char *host = hp->getHostHeaderField()) {
1208 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1209 const int url_sz = scheme.length() + strlen(host) + hp->requestUri().length() + 32;
1210 uri = static_cast<char *>(xcalloc(url_sz, 1));
1211 snprintf(uri, url_sz, SQUIDSBUFPH "://%s" SQUIDSBUFPH,
1212 SQUIDSBUFPRINT(scheme),
1213 host,
1214 SQUIDSBUFPRINT(hp->requestUri()));
1215 }
1216 return uri;
1217}
1218
1219char *
1221{
1223
1224 if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1225 return nullptr; /* already in good shape */
1226
1227 char *uri = buildUrlFromHost(this, hp);
1228#if USE_OPENSSL
1229 if (!uri) {
1232 SBuf useHost;
1233 if (!tlsClientSni().isEmpty())
1234 useHost = tlsClientSni();
1235 else
1236 useHost = tlsConnectHostOrIp;
1237
1239 const int url_sz = scheme.length() + useHost.length() + hp->requestUri().length() + 32;
1240 uri = static_cast<char *>(xcalloc(url_sz, 1));
1241 snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH ":%hu" SQUIDSBUFPH,
1242 SQUIDSBUFPRINT(scheme),
1243 SQUIDSBUFPRINT(useHost),
1245 SQUIDSBUFPRINT(hp->requestUri()));
1246 }
1247#endif
1248 if (uri)
1249 debugs(33, 5, "TLS switching host rewrite: " << uri);
1250 return uri;
1251}
1252
1253static char *
1255{
1256 // TODO Must() on URI !empty when the parser supports throw. For now avoid assert().
1257 if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1258 return nullptr; /* already in good shape */
1259
1260 char *uri = buildUrlFromHost(conn, hp);
1261 if (!uri) {
1262 /* Put the local socket IP address as the hostname. */
1263 static char ipbuf[MAX_IPSTRLEN];
1264 conn->clientConnection->local.toHostStr(ipbuf,MAX_IPSTRLEN);
1265 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1266 const int url_sz = sizeof(ipbuf) + hp->requestUri().length() + 32;
1267 uri = static_cast<char *>(xcalloc(url_sz, 1));
1268 snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1269 SQUIDSBUFPRINT(scheme),
1270 ipbuf, conn->clientConnection->local.port(), SQUIDSBUFPRINT(hp->requestUri()));
1271 }
1272
1273 if (uri)
1274 debugs(33, 5, "TRANSPARENT REWRITE: " << uri);
1275 return uri;
1276}
1277
1280{
1281 /* Attempt to parse the first line; this will define where the method, url, version and header begin */
1282 {
1283 Must(hp);
1284
1287
1288 const bool parsedOk = hp->parse(inBuf);
1289
1290 // sync the buffers after parsing.
1291 inBuf = hp->remaining();
1292
1293 if (hp->needsMoreData()) {
1294 debugs(33, 5, "Incomplete request, waiting for end of request line");
1295 return nullptr;
1296 }
1297
1298 if (!parsedOk) {
1299 const bool tooBig =
1300 hp->parseStatusCode == Http::scRequestHeaderFieldsTooLarge ||
1301 hp->parseStatusCode == Http::scUriTooLong;
1302 auto result = abortRequestParsing(
1303 tooBig ? "error:request-too-large" : "error:invalid-request");
1304 // assume that remaining leftovers belong to this bad request
1305 if (!inBuf.isEmpty())
1307 return result;
1308 }
1309 }
1310
1311 /* We know the whole request is in parser now */
1312 debugs(11, 2, "HTTP Client " << clientConnection);
1313 debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
1314 hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol() << "\n" <<
1315 hp->mimeHeader() <<
1316 "\n----------");
1317
1318 /* deny CONNECT via accelerated ports */
1319 if (hp->method() == Http::METHOD_CONNECT && port != nullptr && port->flags.accelSurrogate) {
1320 debugs(33, DBG_IMPORTANT, "WARNING: CONNECT method received on " << transferProtocol << " Accelerator port " << port->s.port());
1321 debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1322 hp->parseStatusCode = Http::scMethodNotAllowed;
1323 return abortRequestParsing("error:method-not-allowed");
1324 }
1325
1326 /* HTTP/2 connection magic prefix starts with "PRI ".
1327 * Deny "PRI" method if used in HTTP/1.x or 0.9 versions.
1328 * If seen it signals a broken client or proxy has corrupted the traffic.
1329 */
1330 if (hp->method() == Http::METHOD_PRI && hp->messageProtocol() < Http::ProtocolVersion(2,0)) {
1331 debugs(33, DBG_IMPORTANT, "WARNING: PRI method received on " << transferProtocol << " port " << port->s.port());
1332 debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1333 hp->parseStatusCode = Http::scMethodNotAllowed;
1334 return abortRequestParsing("error:method-not-allowed");
1335 }
1336
1337 if (hp->method() == Http::METHOD_NONE) {
1338 debugs(33, DBG_IMPORTANT, "WARNING: Unsupported method: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1339 hp->parseStatusCode = Http::scMethodNotAllowed;
1340 return abortRequestParsing("error:unsupported-request-method");
1341 }
1342
1343 // Process headers after request line
1344 debugs(33, 3, "complete request received. " <<
1345 "prefix_sz = " << hp->messageHeaderSize() <<
1346 ", request-line-size=" << hp->firstLineSize() <<
1347 ", mime-header-size=" << hp->headerBlockSize() <<
1348 ", mime header block:\n" << hp->mimeHeader() << "\n----------");
1349
1350 /* Ok, all headers are received */
1351 ClientHttpRequest *http = new ClientHttpRequest(this);
1352
1353 http->req_sz = hp->messageHeaderSize();
1354 Http::Stream *result = new Http::Stream(clientConnection, http);
1355
1356 StoreIOBuffer tempBuffer;
1357 tempBuffer.data = result->reqbuf;
1358 tempBuffer.length = HTTP_REQBUF_SZ;
1359
1360 ClientStreamData newServer = new clientReplyContext(http);
1361 ClientStreamData newClient = result;
1364 clientSocketDetach, newClient, tempBuffer);
1365
1366 /* set url */
1367 debugs(33,5, "Prepare absolute URL from " <<
1368 (transparent()?"intercept":(port->flags.accelSurrogate ? "accel":"")));
1369 /* Rewrite the URL in transparent or accelerator mode */
1370 /* NP: there are several cases to traverse here:
1371 * - standard mode (forward proxy)
1372 * - transparent mode (TPROXY)
1373 * - transparent mode with failures
1374 * - intercept mode (NAT)
1375 * - intercept mode with failures
1376 * - accelerator mode (reverse proxy)
1377 * - internal relative-URL
1378 * - mixed combos of the above with internal URL
1379 * - remote interception with PROXY protocol
1380 * - remote reverse-proxy with PROXY protocol
1381 */
1382 if (switchedToHttps()) {
1383 http->uri = prepareTlsSwitchingURL(hp);
1384 } else if (transparent()) {
1385 /* intercept or transparent mode, properly working with no failures */
1386 http->uri = prepareTransparentURL(this, hp);
1387
1388 } else if (internalCheck(hp->requestUri())) { // NP: only matches relative-URI
1389 /* internal URL mode */
1390 // XXX: By prepending our name and port, we create an absolute URL
1391 // that may mismatch the (yet unparsed) Host header in the request.
1392 http->uri = xstrdup(internalLocalUri(nullptr, hp->requestUri()));
1393
1394 } else if (port->flags.accelSurrogate) {
1395 /* accelerator mode */
1396 http->uri = prepareAcceleratedURL(this, hp);
1397 http->flags.accel = true;
1398 }
1399
1400 if (!http->uri) {
1401 /* No special rewrites have been applied above, use the
1402 * requested url. may be rewritten later, so make extra room */
1403 int url_sz = hp->requestUri().length() + Config.appendDomainLen + 5;
1404 http->uri = (char *)xcalloc(url_sz, 1);
1405 SBufToCstring(http->uri, hp->requestUri());
1406 }
1407
1408 result->flags.parsed_ok = 1;
1409 return result;
1410}
1411
1412bool
1414{
1415 if (pipeline.empty() && inBuf.isEmpty()) {
1416 debugs(33, 4, "yes, without active requests and unparsed input");
1417 return true;
1418 }
1419
1421 debugs(33, 3, "yes, without half_closed_clients");
1422 return true;
1423 }
1424
1425 // Squid currently tries to parse (possibly again) a partially received
1426 // request after an EOF with half_closed_clients. To give that last parse in
1427 // afterClientRead() a chance, we ignore partially parsed requests here.
1428 debugs(33, 3, "no, honoring half_closed_clients");
1429 return false;
1430}
1431
1432void
1433ConnStateData::consumeInput(const size_t byteCount)
1434{
1435 assert(byteCount > 0 && byteCount <= inBuf.length());
1436 inBuf.consume(byteCount);
1437 debugs(33, 5, "inBuf has " << inBuf.length() << " unused bytes");
1438}
1439
1440void
1442{
1443 // Were we expecting to read more request body from half-closed connection?
1445 debugs(33, 3, "truncated body: closing half-closed " << clientConnection);
1447 return;
1448 }
1449
1450 if (flags.readMore)
1451 readSomeData();
1452}
1453
1454void
1456{
1457 // From HTTP p.o.v., we do not have to close after every error detected
1458 // at the client-side, but many such errors do require closure and the
1459 // client-side code is bad at handling errors so we play it safe.
1460 if (request)
1461 request->flags.proxyKeepalive = false;
1462 flags.readMore = false;
1463 debugs(33,4, "Will close after error: " << clientConnection);
1464}
1465
1466#if USE_OPENSSL
1468{
1469 ClientHttpRequest *http = context->http;
1470
1471 if (!sslServerBump)
1472 return false;
1473
1475 // Did we create an error entry while processing CONNECT?
1476 if (!sslServerBump->entry->isEmpty()) {
1477 quitAfterError(http->request);
1478
1479 // Get the saved error entry and send it to the client by replacing the
1480 // ClientHttpRequest store entry with it.
1482 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1483 assert(repContext);
1484 debugs(33, 5, "Responding with delated error for " << http->uri);
1485 repContext->setReplyToStoreEntry(sslServerBump->entry, "delayed SslBump error");
1486
1487 // Get error details from the fake certificate-peeking request.
1489 context->pullData();
1490 return true;
1491 }
1492
1493 // In bump-server-first mode, we have not necessarily seen the intended
1494 // server name at certificate-peeking time. Check for domain mismatch now,
1495 // when we can extract the intended name from the bumped HTTP request.
1496 if (const Security::CertPointer &srvCert = sslServerBump->serverCert) {
1497 HttpRequest *request = http->request;
1498 if (!Ssl::checkX509ServerValidity(srvCert.get(), request->url.host())) {
1499 debugs(33, 2, "SQUID_X509_V_ERR_DOMAIN_MISMATCH: Certificate " <<
1500 "does not match domainname " << request->url.host());
1501
1502 bool allowDomainMismatch = false;
1505 const auto sslErrors = std::make_unique<Security::CertErrors>(Security::CertError(SQUID_X509_V_ERR_DOMAIN_MISMATCH, srvCert));
1506 check.sslErrors = sslErrors.get();
1507 clientAclChecklistFill(check, http);
1508 allowDomainMismatch = check.fastCheck().allowed();
1509 }
1510
1511 if (!allowDomainMismatch) {
1512 quitAfterError(request);
1513
1515 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1516 assert (repContext);
1517
1518 request->hier = sslServerBump->request->hier;
1519
1520 // Create an error object and fill it
1521 const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request, http->al);
1522 err->src_addr = clientConnection->remote;
1525 srvCert, nullptr);
1527 repContext->setReplyToError(request->method, err);
1528 assert(context->http->out.offset == 0);
1529 context->pullData();
1530 return true;
1531 }
1532 }
1533 }
1534
1535 return false;
1536}
1537#endif // USE_OPENSSL
1538
1540bool
1542{
1544 debugs(33, 5, "disabled; send error: " << requestError);
1545 return false;
1546 }
1547
1548 if (!preservingClientData_) {
1549 debugs(33, 3, "may have forgotten client data; send error: " << requestError);
1550 return false;
1551 }
1552
1554 checklist.requestErrorType = requestError;
1555 fillChecklist(checklist);
1556 auto answer = checklist.fastCheck();
1557 if (answer.allowed() && answer.kind == 1) {
1558 debugs(33, 3, "Request will be tunneled to server");
1559 const auto context = pipeline.front();
1560 const auto http = context ? context->http : nullptr;
1561 const auto request = http ? http->request : nullptr;
1562 if (context)
1563 context->finished(); // Will remove from pipeline queue
1564 Comm::SetSelect(clientConnection->fd, COMM_SELECT_READ, nullptr, nullptr, 0);
1565 return initiateTunneledRequest(request, "unknown-protocol", preservedClientData);
1566 }
1567 debugs(33, 3, "denied; send error: " << requestError);
1568 return false;
1569}
1570
1571void
1573{
1574 /*
1575 * DPW 2007-05-18
1576 * Moved the TCP_RESET feature from clientReplyContext::sendMoreData
1577 * to here because calling comm_reset_close() causes http to
1578 * be freed before accessing.
1579 */
1580 if (request != nullptr && request->flags.resetTcp && Comm::IsConnOpen(conn->clientConnection)) {
1581 debugs(33, 3, "Sending TCP RST on " << conn->clientConnection);
1582 conn->flags.readMore = false;
1583 comm_reset_close(conn->clientConnection);
1584 }
1585}
1586
1587void
1589{
1590 ClientHttpRequest *http = context->http;
1591 bool mustReplyToOptions = false;
1592 bool expectBody = false;
1593
1594 // We already have the request parsed and checked, so we
1595 // only need to go through the final body/conn setup to doCallouts().
1596 assert(http->request);
1597 HttpRequest::Pointer request = http->request;
1598
1599 // temporary hack to avoid splitting this huge function with sensitive code
1600 const bool isFtp = !hp;
1601
1602 // Some blobs below are still HTTP-specific, but we would have to rewrite
1603 // this entire function to remove them from the FTP code path. Connection
1604 // setup and body_pipe preparation blobs are needed for FTP.
1605
1606 request->manager(conn, http->al);
1607
1608 request->flags.accelerated = http->flags.accel;
1609 request->flags.sslBumped=conn->switchedToHttps();
1610 // TODO: decouple http->flags.accel from request->flags.sslBumped
1611 request->flags.noDirect = (request->flags.accelerated && !request->flags.sslBumped) ?
1612 !conn->port->allow_direct : 0;
1613 request->sources |= isFtp ? Http::Message::srcFtp :
1614 ((request->flags.sslBumped || conn->port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
1615#if USE_AUTH
1616 if (request->flags.sslBumped) {
1617 if (conn->getAuth() != nullptr)
1618 request->auth_user_request = conn->getAuth();
1619 }
1620#endif
1621
1622 http->checkForInternalAccess();
1623
1624 if (!isFtp) {
1625 // XXX: for non-HTTP messages instantiate a different Http::Message child type
1626 // for now Squid only supports HTTP requests
1627 const AnyP::ProtocolVersion &http_ver = hp->messageProtocol();
1628 assert(request->http_ver.protocol == http_ver.protocol);
1629 request->http_ver.major = http_ver.major;
1630 request->http_ver.minor = http_ver.minor;
1631 }
1632
1633 mustReplyToOptions = (request->method == Http::METHOD_OPTIONS) &&
1635 if (!urlCheckRequest(request.getRaw()) || mustReplyToOptions) {
1637 conn->quitAfterError(request.getRaw());
1638 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1639 assert (repContext);
1641 conn, request.getRaw(), nullptr, nullptr);
1642 assert(context->http->out.offset == 0);
1643 context->pullData();
1645 return;
1646 }
1647
1648 const auto frameStatus = request->checkEntityFraming();
1649 if (frameStatus != Http::scNone) {
1651 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1652 assert (repContext);
1653 conn->quitAfterError(request.getRaw());
1654 repContext->setReplyToError(ERR_INVALID_REQ, frameStatus, nullptr, conn, request.getRaw(), nullptr, nullptr);
1655 assert(context->http->out.offset == 0);
1656 context->pullData();
1658 return;
1659 }
1660
1662 // Let tunneling code be fully responsible for CONNECT requests
1663 if (http->request->method == Http::METHOD_CONNECT) {
1664 context->mayUseConnection(true);
1665 conn->flags.readMore = false;
1666 }
1667
1668#if USE_OPENSSL
1669 if (conn->switchedToHttps() && conn->serveDelayedError(context)) {
1671 return;
1672 }
1673#endif
1674
1675 /* Do we expect a request-body? */
1676 const auto chunked = request->header.chunked();
1677 expectBody = chunked || request->content_length > 0;
1678 if (!context->mayUseConnection() && expectBody) {
1679 request->body_pipe = conn->expectRequestBody(
1680 chunked ? -1 : request->content_length);
1681
1682 /* Is it too large? */
1683 if (!chunked && // if chunked, we will check as we accumulate
1686 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1687 assert (repContext);
1688 conn->quitAfterError(request.getRaw());
1689 repContext->setReplyToError(ERR_TOO_BIG,
1690 Http::scContentTooLarge, nullptr,
1691 conn, http->request, nullptr, nullptr);
1692 assert(context->http->out.offset == 0);
1693 context->pullData();
1695 return;
1696 }
1697
1698 if (!isFtp) {
1699 // We may stop producing, comm_close, and/or call setReplyToError()
1700 // below, so quit on errors to avoid http->doCallouts()
1701 if (!conn->handleRequestBodyData()) {
1703 return;
1704 }
1705
1706 if (!request->body_pipe->productionEnded()) {
1707 debugs(33, 5, "need more request body");
1708 context->mayUseConnection(true);
1709 assert(conn->flags.readMore);
1710 }
1711 }
1712 }
1713
1714 http->calloutContext = new ClientRequestContext(http);
1715
1716 http->doCallouts();
1717
1719}
1720
1721void
1723{
1724 debugs(33, 3, context << " to " << pipeline.count() << '/' << pipeline.nrequests);
1725 if (bareError) {
1726 debugs(33, 5, "assigning " << bareError);
1727 assert(context);
1728 assert(context->http);
1729 context->http->updateError(bareError);
1730 bareError.clear();
1731 }
1732 pipeline.add(context);
1733}
1734
1735int
1737{
1738 // TODO: Support pipelined requests through pinned connections.
1739 if (pinning.pinned)
1740 return 0;
1742}
1743
1749bool
1751{
1752 const int existingRequestCount = pipeline.count();
1753
1754 // default to the configured pipeline size.
1755 // add 1 because the head of pipeline is counted in concurrent requests and not prefetch queue
1756#if USE_OPENSSL
1757 const int internalRequest = (transparent() && sslBumpMode == Ssl::bumpSplice) ? 1 : 0;
1758#else
1759 const int internalRequest = 0;
1760#endif
1761 const int concurrentRequestLimit = pipelinePrefetchMax() + 1 + internalRequest;
1762
1763 // when queue filled already we can't add more.
1764 if (existingRequestCount >= concurrentRequestLimit) {
1765 debugs(33, 3, clientConnection << " max concurrent requests reached (" << concurrentRequestLimit << ")");
1766 debugs(33, 5, clientConnection << " deferring new request until one is done");
1767 return true;
1768 }
1769
1770 return false;
1771}
1772
1778bool
1780{
1782 return proxyProtocolError("PROXY client not permitted by default ACL");
1783
1785 fillChecklist(ch);
1786 if (!ch.fastCheck().allowed())
1787 return proxyProtocolError("PROXY client not permitted by ACLs");
1788
1789 return true;
1790}
1791
1797bool
1799{
1800 if (msg) {
1801 // This is important to know, but maybe not so much that flooding the log is okay.
1802#if QUIET_PROXY_PROTOCOL
1803 // display the first of every 32 occurrences at level 1, the others at level 2.
1804 static uint8_t hide = 0;
1805 debugs(33, (hide++ % 32 == 0 ? DBG_IMPORTANT : 2), msg << " from " << clientConnection);
1806#else
1807 debugs(33, DBG_IMPORTANT, msg << " from " << clientConnection);
1808#endif
1809 mustStop(msg);
1810 }
1811 return false;
1812}
1813
1818bool
1820{
1821 try {
1822 const auto parsed = ProxyProtocol::Parse(inBuf);
1823 proxyProtocolHeader_ = parsed.header;
1825 inBuf.consume(parsed.size);
1827 if (proxyProtocolHeader_->hasForwardedAddresses()) {
1828 clientConnection->local = proxyProtocolHeader_->destinationAddress;
1831 clientConnection->flags ^= COMM_TRANSPARENT; // prevent TPROXY spoofing of this new IP.
1832 debugs(33, 5, "PROXY/" << proxyProtocolHeader_->version() << " upgrade: " << clientConnection);
1833 }
1835 debugs(33, 3, "PROXY protocol: waiting for more than " << inBuf.length() << " bytes");
1836 return false;
1837 } catch (const std::exception &e) {
1838 return proxyProtocolError(e.what());
1839 }
1840 return true;
1841}
1842
1843void
1845{
1847 return;
1848
1849 receivedFirstByte_ = true;
1851}
1852
1855void
1857{
1858 debugs(33, 5, clientConnection << ": attempting to parse");
1859
1860 // Loop while we have read bytes that are not needed for producing the body
1861 // On errors, bodyPipe may become nil, but readMore will be cleared
1862 while (!inBuf.isEmpty() && !bodyPipe && flags.readMore) {
1863
1864 // Prohibit concurrent requests when using a pinned to-server connection
1865 // because our Client classes do not support request pipelining.
1866 if (pinning.pinned && !pinning.readHandler) {
1867 debugs(33, 3, clientConnection << " waits for busy " << pinning.serverConnection);
1868 break;
1869 }
1870
1871 /* Limit the number of concurrent requests */
1873 break;
1874
1875 // try to parse the PROXY protocol header magic bytes
1878 break;
1879
1880 // we have been waiting for PROXY to provide client-IP
1881 // for some lookups, ie rDNS and IDENT.
1883
1884 // Done with PROXY protocol which has cleared preservingClientData_.
1885 // If the next protocol supports on_unsupported_protocol, then its
1886 // parseOneRequest() must reset preservingClientData_.
1888 }
1889
1890 if (Http::StreamPointer context = parseOneRequest()) {
1891 debugs(33, 5, clientConnection << ": done parsing a request");
1893 context->registerWithConn();
1894
1895#if USE_OPENSSL
1896 if (switchedToHttps())
1898#endif
1899
1900 processParsedRequest(context);
1901
1902 if (context->mayUseConnection()) {
1903 debugs(33, 3, "Not parsing new requests, as this request may need the connection");
1904 break;
1905 }
1906 } else {
1907 debugs(33, 5, clientConnection << ": not enough request data: " <<
1908 inBuf.length() << " < " << Config.maxRequestHeaderSize);
1910 break;
1911 }
1912 }
1913
1914 debugs(33, 7, "buffered leftovers: " << inBuf.length());
1915
1917 if (pipeline.empty()) {
1918 // we processed what we could parse, and no more data is coming
1919 debugs(33, 5, "closing half-closed without parsed requests: " << clientConnection);
1921 } else {
1922 // we parsed what we could, and no more data is coming
1923 debugs(33, 5, "monitoring half-closed while processing parsed requests: " << clientConnection);
1924 flags.readMore = false; // may already be false
1925 }
1926 }
1927}
1928
1929void
1931{
1932#if USE_OPENSSL
1933 if (parsingTlsHandshake) {
1935 return;
1936 }
1937#endif
1938
1939 /* Process next request */
1940 if (pipeline.empty())
1941 fd_note(clientConnection->fd, "Reading next request");
1942
1943 parseRequests();
1944
1945 if (!isOpen())
1946 return;
1947
1949}
1950
1957bool
1959{
1960 // if we are reading a body, stuff data into the body pipe
1961 if (bodyPipe != nullptr)
1962 return handleRequestBodyData();
1963 return true;
1964}
1965
1973bool
1975{
1976 assert(bodyPipe != nullptr);
1977
1978 if (bodyParser) { // chunked encoding
1979 if (const err_type error = handleChunkedRequestBody()) {
1981 return false;
1982 }
1983 } else { // identity encoding
1984 debugs(33,5, "handling plain request body for " << clientConnection);
1985 const size_t putSize = bodyPipe->putMoreData(inBuf.c_str(), inBuf.length());
1986 if (putSize > 0)
1987 consumeInput(putSize);
1988
1989 if (!bodyPipe->mayNeedMoreData()) {
1990 // BodyPipe will clear us automagically when we produced everything
1991 bodyPipe = nullptr;
1992 }
1993 }
1994
1995 if (!bodyPipe) {
1996 debugs(33,5, "produced entire request body for " << clientConnection);
1997
1998 if (const char *reason = stoppedSending()) {
1999 /* we've finished reading like good clients,
2000 * now do the close that initiateClose initiated.
2001 */
2002 debugs(33, 3, "closing for earlier sending error: " << reason);
2004 return false;
2005 }
2006 }
2007
2008 return true;
2009}
2010
2014{
2015 debugs(33, 7, "chunked from " << clientConnection << ": " << inBuf.length());
2016
2017 try { // the parser will throw on errors
2018
2019 if (inBuf.isEmpty()) // nothing to do
2020 return ERR_NONE;
2021
2024 const bool parsed = bodyParser->parse(inBuf);
2025 inBuf = bodyParser->remaining(); // sync buffers
2026 bpc.checkIn();
2027
2028 // dechunk then check: the size limit applies to _dechunked_ content
2030 return ERR_TOO_BIG;
2031
2032 if (parsed) {
2034 Must(!bodyPipe);
2035 return ERR_NONE; // nil bodyPipe implies body end for the caller
2036 }
2037
2038 // if chunk parser needs data, then the body pipe must need it too
2040
2041 // if parser needs more space and we can consume nothing, we will stall
2043 } catch (...) { // TODO: be more specific
2044 debugs(33, 3, "malformed chunks" << bodyPipe->status());
2045 return ERR_INVALID_REQ;
2046 }
2047
2048 debugs(33, 7, "need more chunked data" << *bodyPipe->status());
2049 return ERR_NONE;
2050}
2051
2053void
2055{
2057
2058 // XXX: The code below works if we fail during initial request parsing,
2059 // but if we fail when the server connection is used already, the server may send
2060 // us its response too, causing various assertions. How to prevent that?
2061#if WE_KNOW_HOW_TO_SEND_ERRORS
2063 if (context != NULL && !context->http->out.offset) { // output nothing yet
2064 clientStreamNode *node = context->getClientReplyContext();
2065 clientReplyContext *repContext = dynamic_cast<clientReplyContext*>(node->data.getRaw());
2066 assert(repContext);
2067 const Http::StatusCode scode = (error == ERR_TOO_BIG) ?
2068 Http::scContentTooLarge : HTTP_BAD_REQUEST;
2069 repContext->setReplyToError(error, scode,
2070 repContext->http->uri,
2071 CachePeer,
2072 repContext->http->request,
2073 inBuf, nullptr);
2074 context->pullData();
2075 } else {
2076 // close or otherwise we may get stuck as nobody will notice the error?
2078 }
2079#else
2080 debugs(33, 3, "aborting chunked request without error " << error);
2082#endif
2083 flags.readMore = false;
2084}
2085
2086void
2088{
2089 // request reader may get stuck waiting for space if nobody consumes body
2090 if (bodyPipe != nullptr)
2092
2093 // kids extend
2094}
2095
2097void
2099{
2100 if (!Comm::IsConnOpen(io.conn))
2101 return;
2102
2105 if (tunnelOnError(error))
2106 return;
2107
2108 /*
2109 * Just close the connection to not confuse browsers
2110 * using persistent connections. Some browsers open
2111 * a connection and then do not use it until much
2112 * later (presumeably because the request triggering
2113 * the open has already been completed on another
2114 * connection)
2115 */
2116 debugs(33, 3, "requestTimeout: FD " << io.fd << ": lifetime is expired.");
2117 io.conn->close();
2118}
2119
2120void
2122{
2123 debugs(33, DBG_IMPORTANT, "WARNING: Closing client connection due to lifetime timeout" <<
2124 Debug::Extra << "connection: " << io.conn);
2125
2126 LogTagsErrors lte;
2127 lte.timedout = true;
2129}
2130
2132 AsyncJob("ConnStateData"), // kids overwrite
2133 Server(xact)
2134#if USE_OPENSSL
2135 , tlsParser(Security::HandshakeParser::fromClient)
2136#endif
2137{
2138 // store the details required for creating more MasterXaction objects as new requests come in
2139 log_addr = xact->tcpClient->remote;
2141
2142 // register to receive notice of Squid signal events
2143 // which may affect long persisting client connections
2145}
2146
2147void
2149{
2152
2153 if (port->disable_pmtu_discovery != DISABLE_PMTU_OFF &&
2154 (transparent() || port->disable_pmtu_discovery == DISABLE_PMTU_ALWAYS)) {
2155#if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
2156 int i = IP_PMTUDISC_DONT;
2157 if (setsockopt(clientConnection->fd, SOL_IP, IP_MTU_DISCOVER, &i, sizeof(i)) < 0) {
2158 int xerrno = errno;
2159 debugs(33, 2, "WARNING: Path MTU discovery disabling failed on " << clientConnection << " : " << xstrerr(xerrno));
2160 }
2161#else
2162 static bool reported = false;
2163
2164 if (!reported) {
2165 debugs(33, DBG_IMPORTANT, "WARNING: Path MTU discovery disabling is not supported on your platform.");
2166 reported = true;
2167 }
2168#endif
2169 }
2170
2174
2175 needProxyProtocolHeader_ = port->flags.proxySurrogate;
2177 if (!proxyProtocolValidateClient()) // will close the connection on failure
2178 return;
2179 } else
2181
2182 // requires needProxyProtocolHeader_ which is initialized above
2184}
2185
2186void
2188{
2192
2193#if USE_IDENT
2194 if (Ident::TheConfig.identLookup) {
2195 ACLFilledChecklist identChecklist(Ident::TheConfig.identLookup, nullptr, nullptr);
2196 fillChecklist(identChecklist);
2197 if (identChecklist.fastCheck().allowed())
2199 }
2200#endif
2201
2203
2204#if USE_DELAY_POOLS
2205 fd_table[clientConnection->fd].clientInfo = nullptr;
2206
2207 if (!Config.onoff.client_db)
2208 return; // client delay pools require client_db
2209
2210 const auto &pools = ClientDelayPools::Instance()->pools;
2211 if (pools.size()) {
2212 ACLFilledChecklist ch(nullptr, nullptr, nullptr);
2213 fillChecklist(ch);
2214 // TODO: we check early to limit error response bandwidth but we
2215 // should recheck when we can honor delay_pool_uses_indirect
2216 for (unsigned int pool = 0; pool < pools.size(); ++pool) {
2217
2218 /* pools require explicit 'allow' to assign a client into them */
2219 if (pools[pool]->access) {
2220 ch.changeAcl(pools[pool]->access);
2221 auto answer = ch.fastCheck();
2222 if (answer.allowed()) {
2223
2224 /* request client information from db after we did all checks
2225 this will save hash lookup if client failed checks */
2227 assert(cli);
2228
2229 /* put client info in FDE */
2230 fd_table[clientConnection->fd].clientInfo = cli;
2231
2232 /* setup write limiter for this request */
2233 const double burst = floor(0.5 +
2234 (pools[pool]->highwatermark * Config.ClientDelay.initial)/100.0);
2235 cli->setWriteLimiter(pools[pool]->rate, burst, pools[pool]->highwatermark);
2236 break;
2237 } else {
2238 debugs(83, 4, "Delay pool " << pool << " skipped because ACL " << answer);
2239 }
2240 }
2241 }
2242 }
2243#endif
2244
2245 // kids must extend to actually start doing something (e.g., reading)
2246}
2247
2250{
2251 const auto handshakeResult = Security::Accept(*clientConnection);
2252
2253#if USE_OPENSSL
2254 // log ASAP, even if the handshake has not completed (or failed)
2255 const auto fd = clientConnection->fd;
2256 assert(fd >= 0);
2257 keyLogger.checkpoint(*fd_table[fd].ssl, *this);
2258#else
2259 // TODO: Support fd_table[fd].ssl dereference in other builds.
2260#endif
2261
2262 return handshakeResult;
2263}
2264
2266void
2268{
2269 Assure(params.port);
2270
2271 // NP: it is possible the port was reconfigured when the call or accept() was queued.
2272
2273 if (params.flag != Comm::OK) {
2274 // Its possible the call was still queued when the client disconnected
2275 debugs(33, 2, params.port->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2276 return;
2277 }
2278
2279 debugs(33, 4, params.conn << ": accepted");
2280 fd_note(params.conn->fd, "client http connect");
2281 const auto xact = MasterXaction::MakePortful(params.port);
2282 xact->tcpClient = params.conn;
2283
2284 // Socket is ready, setup the connection manager to start using it
2285 auto *srv = Http::NewServer(xact);
2286 // XXX: do not abandon the MasterXaction object
2287 AsyncJob::Start(srv); // usually async-calls readSomeData()
2288}
2289
2291static bool
2293{
2294 const auto conn = connState->clientConnection;
2295 if (Security::CreateServerSession(ctx, conn, connState->port->secure, "client https start")) {
2296 debugs(33, 5, "will negotiate TLS on " << conn);
2297 return true;
2298 }
2299
2300 debugs(33, DBG_IMPORTANT, "ERROR: could not create TLS server context for " << conn);
2301 conn->close();
2302 return false;
2303}
2304
2306static void
2307clientNegotiateSSL(int fd, void *data)
2308{
2309 ConnStateData *conn = (ConnStateData *)data;
2310
2311 const auto handshakeResult = conn->acceptTls();
2312 switch (handshakeResult.category) {
2314 break;
2315
2317 Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_READ, clientNegotiateSSL, conn, 0);
2318 return;
2319
2321 Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_WRITE, clientNegotiateSSL, conn, 0);
2322 return;
2323
2325 debugs(83, (handshakeResult.important ? Important(62) : 2), "ERROR: " << handshakeResult.errorDescription <<
2326 " while accepting a TLS connection on " << conn->clientConnection << ": " << handshakeResult.errorDetail);
2327 // TODO: No ConnStateData::tunnelOnError() on this forward-proxy code
2328 // path because we cannot know the intended connection target?
2329 conn->updateError(ERR_SECURE_ACCEPT_FAIL, handshakeResult.errorDetail);
2330 conn->clientConnection->close();
2331 return;
2332 }
2333
2334 Security::SessionPointer session(fd_table[fd].ssl);
2335
2336#if USE_OPENSSL
2337 if (Security::SessionIsResumed(session)) {
2338 debugs(83, 2, "Session " << SSL_get_session(session.get()) <<
2339 " reused on FD " << fd << " (" << fd_table[fd].ipaddr <<
2340 ":" << (int)fd_table[fd].remote_port << ")");
2341 } else {
2342 if (Debug::Enabled(83, 4)) {
2343 /* Write out the SSL session details.. actually the call below, but
2344 * OpenSSL headers do strange typecasts confusing GCC.. */
2345 /* PEM_write_SSL_SESSION(debug_log, SSL_get_session(ssl)); */
2346#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x00908000L
2347 PEM_ASN1_write(reinterpret_cast<i2d_of_void *>(i2d_SSL_SESSION),
2348 PEM_STRING_SSL_SESSION, debug_log,
2349 reinterpret_cast<char *>(SSL_get_session(session.get())),
2350 nullptr, nullptr, 0, nullptr, nullptr);
2351
2352#elif (ALLOW_ALWAYS_SSL_SESSION_DETAIL == 1)
2353
2354 /* When using gcc 3.3.x and OpenSSL 0.9.7x sometimes a compile error can occur here.
2355 * This is caused by an unpredicatble gcc behaviour on a cast of the first argument
2356 * of PEM_ASN1_write(). For this reason this code section is disabled. To enable it,
2357 * define ALLOW_ALWAYS_SSL_SESSION_DETAIL=1.
2358 * Because there are two possible usable cast, if you get an error here, try the other
2359 * commented line. */
2360
2361 PEM_ASN1_write((int(*)())i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2362 debug_log,
2363 reinterpret_cast<char *>(SSL_get_session(session.get())),
2364 nullptr, nullptr, 0, nullptr, nullptr);
2365 /* PEM_ASN1_write((int(*)(...))i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2366 debug_log,
2367 reinterpret_cast<char *>(SSL_get_session(session.get())),
2368 nullptr, nullptr, 0, nullptr, nullptr);
2369 */
2370#else
2371 debugs(83, 4, "With " OPENSSL_VERSION_TEXT ", session details are available only defining ALLOW_ALWAYS_SSL_SESSION_DETAIL=1 in the source.");
2372
2373#endif
2374 /* Note: This does not automatically fflush the log file.. */
2375 }
2376
2377 debugs(83, 2, "New session " << SSL_get_session(session.get()) <<
2378 " on FD " << fd << " (" << fd_table[fd].ipaddr << ":" <<
2379 fd_table[fd].remote_port << ")");
2380 }
2381#else
2382 debugs(83, 2, "TLS session reuse not yet implemented.");
2383#endif
2384
2385 // Connection established. Retrieve TLS connection parameters for logging.
2386 conn->clientConnection->tlsNegotiations()->retrieveNegotiatedInfo(session);
2387
2388#if USE_OPENSSL
2389 X509 *client_cert = SSL_get_peer_certificate(session.get());
2390
2391 if (client_cert) {
2392 debugs(83, 3, "FD " << fd << " client certificate: subject: " <<
2393 Security::SubjectName(*client_cert));
2394
2395 debugs(83, 3, "FD " << fd << " client certificate: issuer: " <<
2396 Security::IssuerName(*client_cert));
2397
2398 X509_free(client_cert);
2399 } else {
2400 debugs(83, 5, "FD " << fd << " has no client certificate.");
2401 }
2402#else
2403 debugs(83, 2, "Client certificate requesting not yet implemented.");
2404#endif
2405
2406 // If we are called, then bumped CONNECT has succeeded. Finalize it.
2407 if (auto xact = conn->pipeline.front()) {
2408 if (xact->http && xact->http->request && xact->http->request->method == Http::METHOD_CONNECT)
2409 xact->finished();
2410 // cannot proceed with encryption if requests wait for plain responses
2411 Must(conn->pipeline.empty());
2412 }
2413 /* careful: finished() above frees request, host, etc. */
2414
2415 conn->readSomeData();
2416}
2417
2422static void
2424{
2425 assert(connState);
2426 const Comm::ConnectionPointer &details = connState->clientConnection;
2427
2428 if (!ctx || !httpsCreate(connState, ctx))
2429 return;
2430
2432
2433 Comm::SetSelect(details->fd, COMM_SELECT_READ, clientNegotiateSSL, connState, 0);
2434}
2435
2436#if USE_OPENSSL
2440static void
2442{
2443 ConnStateData *connState = (ConnStateData *) data;
2444
2445 // if the connection is closed or closing, just return.
2446 if (!connState->isOpen())
2447 return;
2448
2449 if (answer.allowed()) {
2450 debugs(33, 2, "sslBump action " << Ssl::bumpMode(answer.kind) << "needed for " << connState->clientConnection);
2451 connState->sslBumpMode = static_cast<Ssl::BumpMode>(answer.kind);
2452 } else {
2453 debugs(33, 3, "sslBump not needed for " << connState->clientConnection);
2454 connState->sslBumpMode = Ssl::bumpSplice;
2455 }
2456
2457 if (connState->sslBumpMode == Ssl::bumpTerminate) {
2458 connState->clientConnection->close();
2459 return;
2460 }
2461
2462 if (!connState->fakeAConnectRequest("ssl-bump", connState->inBuf))
2463 connState->clientConnection->close();
2464}
2465#endif
2466
2468static void
2470{
2471 Assure(params.port);
2472
2473 // NP: it is possible the port was reconfigured when the call or accept() was queued.
2474
2475 if (params.flag != Comm::OK) {
2476 // Its possible the call was still queued when the client disconnected
2477 debugs(33, 2, "httpsAccept: " << params.port->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2478 return;
2479 }
2480
2481 const auto xact = MasterXaction::MakePortful(params.port);
2482 xact->tcpClient = params.conn;
2483
2484 debugs(33, 4, params.conn << " accepted, starting SSL negotiation.");
2485 fd_note(params.conn->fd, "client https connect");
2486
2487 // Socket is ready, setup the connection manager to start using it
2488 auto *srv = Https::NewServer(xact);
2489 // XXX: do not abandon the MasterXaction object
2490 AsyncJob::Start(srv); // usually async-calls postHttpsAccept()
2491}
2492
2493void
2495{
2496 if (port->flags.tunnelSslBumping) {
2497#if USE_OPENSSL
2498 debugs(33, 5, "accept transparent connection: " << clientConnection);
2499
2500 if (!Config.accessList.ssl_bump) {
2502 return;
2503 }
2504
2505 const auto mx = MasterXaction::MakePortful(port);
2506 mx->tcpClient = clientConnection;
2507 // Create a fake HTTP request and ALE for the ssl_bump ACL check,
2508 // using tproxy/intercept provided destination IP and port.
2509 // XXX: Merge with subsequent fakeAConnectRequest(), buildFakeRequest().
2510 // XXX: Do this earlier (e.g., in Http[s]::One::Server constructor).
2511 HttpRequest *request = new HttpRequest(mx);
2512 static char ip[MAX_IPSTRLEN];
2514 request->url.host(clientConnection->local.toStr(ip, sizeof(ip)));
2515 request->url.port(clientConnection->local.port());
2516 request->myportname = port->name;
2517 const AccessLogEntry::Pointer connectAle = new AccessLogEntry;
2518 CodeContext::Reset(connectAle);
2519 // TODO: Use these request/ALE when waiting for new bumped transactions.
2520
2521 ACLFilledChecklist *acl_checklist = new ACLFilledChecklist(Config.accessList.ssl_bump, request, nullptr);
2522 fillChecklist(*acl_checklist);
2523 // Build a local AccessLogEntry to allow requiresAle() acls work
2524 acl_checklist->al = connectAle;
2525 acl_checklist->al->cache.start_time = current_time;
2526 acl_checklist->al->tcpClient = clientConnection;
2527 acl_checklist->al->cache.port = port;
2528 acl_checklist->al->cache.caddr = log_addr;
2530 acl_checklist->al->updateError(bareError);
2531 HTTPMSGUNLOCK(acl_checklist->al->request);
2532 acl_checklist->al->request = request;
2533 HTTPMSGLOCK(acl_checklist->al->request);
2535 ClientHttpRequest *http = context ? context->http : nullptr;
2536 const char *log_uri = http ? http->log_uri : nullptr;
2537 acl_checklist->syncAle(request, log_uri);
2538 acl_checklist->nonBlockingCheck(httpsSslBumpAccessCheckDone, this);
2539#else
2540 fatal("FATAL: SSL-Bump requires --with-openssl");
2541#endif
2542 return;
2543 } else {
2544 httpsEstablish(this, port->secure.staticContext);
2545 }
2546}
2547
2548#if USE_OPENSSL
2549void
2551{
2552 ConnStateData * state_data = (ConnStateData *)(data);
2553 state_data->sslCrtdHandleReply(reply);
2554}
2555
2556void
2558{
2559 if (!isOpen()) {
2560 debugs(33, 3, "Connection gone while waiting for ssl_crtd helper reply; helper reply:" << reply);
2561 return;
2562 }
2563
2564 if (reply.result == Helper::BrokenHelper) {
2565 debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply);
2566 } else if (!reply.other().hasContent()) {
2567 debugs(1, DBG_IMPORTANT, "\"ssl_crtd\" helper returned <NULL> reply.");
2568 } else {
2570 if (reply_message.parse(reply.other().content(), reply.other().contentSize()) != Ssl::CrtdMessage::OK) {
2571 debugs(33, 5, "Reply from ssl_crtd for " << tlsConnectHostOrIp << " is incorrect");
2572 } else {
2573 if (reply.result != Helper::Okay) {
2574 debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply_message.getBody());
2575 } else {
2576 debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " was successfully received from ssl_crtd");
2579 auto ssl = fd_table[clientConnection->fd].ssl.get();
2580 bool ret = Ssl::configureSSLUsingPkeyAndCertFromMemory(ssl, reply_message.getBody().c_str(), *port);
2581 if (!ret)
2582 debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2583
2586 } else {
2588 if (ctx && !sslBumpCertKey.isEmpty())
2590 getSslContextDone(ctx);
2591 }
2592 return;
2593 }
2594 }
2595 }
2598}
2599
2601{
2603
2604 const bool connectedOk = sslServerBump && sslServerBump->connectedOk();
2605 if (connectedOk) {
2606 if (X509 *mimicCert = sslServerBump->serverCert.get())
2607 certProperties.mimicCert.resetAndLock(mimicCert);
2608
2609 ACLFilledChecklist checklist(nullptr, sslServerBump->request.getRaw());
2610 fillChecklist(checklist);
2611
2612 for (sslproxy_cert_adapt *ca = Config.ssl_client.cert_adapt; ca != nullptr; ca = ca->next) {
2613 // If the algorithm already set, then ignore it.
2614 if ((ca->alg == Ssl::algSetCommonName && certProperties.setCommonName) ||
2615 (ca->alg == Ssl::algSetValidAfter && certProperties.setValidAfter) ||
2616 (ca->alg == Ssl::algSetValidBefore && certProperties.setValidBefore) )
2617 continue;
2618
2619 if (ca->aclList && checklist.fastCheck(ca->aclList).allowed()) {
2620 const char *alg = Ssl::CertAdaptAlgorithmStr[ca->alg];
2621 const char *param = ca->param;
2622
2623 // For parameterless CN adaptation, use hostname from the
2624 // CONNECT request.
2625 if (ca->alg == Ssl::algSetCommonName) {
2626 if (!param)
2627 param = tlsConnectHostOrIp.c_str();
2628 certProperties.commonName = param;
2629 certProperties.setCommonName = true;
2630 } else if (ca->alg == Ssl::algSetValidAfter)
2631 certProperties.setValidAfter = true;
2632 else if (ca->alg == Ssl::algSetValidBefore)
2633 certProperties.setValidBefore = true;
2634
2635 debugs(33, 5, "Matches certificate adaptation aglorithm: " <<
2636 alg << " param: " << (param ? param : "-"));
2637 }
2638 }
2639
2640 certProperties.signAlgorithm = Ssl::algSignEnd;
2641 for (sslproxy_cert_sign *sg = Config.ssl_client.cert_sign; sg != nullptr; sg = sg->next) {
2642 if (sg->aclList && checklist.fastCheck(sg->aclList).allowed()) {
2643 certProperties.signAlgorithm = (Ssl::CertSignAlgorithm)sg->alg;
2644 break;
2645 }
2646 }
2647 } else {// did not try to connect (e.g. client-first) or failed to connect
2648 // In case of an error while connecting to the secure server, use a
2649 // trusted certificate, with no mimicked fields and no adaptation
2650 // algorithms. There is nothing we can mimic, so we want to minimize the
2651 // number of warnings the user will have to see to get to the error page.
2652 // We will close the connection, so that the trust is not extended to
2653 // non-Squid content.
2654 certProperties.signAlgorithm = Ssl::algSignTrusted;
2655 }
2656
2657 assert(certProperties.signAlgorithm != Ssl::algSignEnd);
2658
2659 if (certProperties.signAlgorithm == Ssl::algSignUntrusted) {
2660 assert(port->secure.untrustedSigningCa.cert);
2661 certProperties.signWithX509.resetAndLock(port->secure.untrustedSigningCa.cert.get());
2662 certProperties.signWithPkey.resetAndLock(port->secure.untrustedSigningCa.pkey.get());
2663 } else {
2664 assert(port->secure.signingCa.cert.get());
2665 certProperties.signWithX509.resetAndLock(port->secure.signingCa.cert.get());
2666
2667 if (port->secure.signingCa.pkey)
2668 certProperties.signWithPkey.resetAndLock(port->secure.signingCa.pkey.get());
2669 }
2670 signAlgorithm = certProperties.signAlgorithm;
2671
2672 certProperties.signHash = Ssl::DefaultSignHash;
2673}
2674
2677{
2678 debugs(33, 5, "Finding SSL certificate for " << cacheKey << " in cache");
2680 if (const auto ctx = ssl_ctx_cache ? ssl_ctx_cache->get(cacheKey) : nullptr) {
2681 if (Ssl::verifySslCertificate(*ctx, certProperties)) {
2682 debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is valid");
2683 return *ctx;
2684 } else {
2685 debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is out of date. Delete this certificate from cache");
2686 if (ssl_ctx_cache)
2687 ssl_ctx_cache->del(cacheKey);
2688 }
2689 }
2690 return Security::ContextPointer(nullptr);
2691}
2692
2693void
2695{
2697 if (!ssl_ctx_cache || !ssl_ctx_cache->add(cacheKey, ctx)) {
2698 // If it is not in storage delete after using. Else storage deleted it.
2699 fd_table[clientConnection->fd].dynamicTlsContext = ctx;
2700 }
2701}
2702
2703void
2705{
2706 if (port->secure.generateHostCertificates) {
2707 Ssl::CertificateProperties certProperties;
2708 buildSslCertGenerationParams(certProperties);
2709
2710 // Disable caching for bumpPeekAndSplice mode
2715
2717 if (ctx) {
2718 getSslContextDone(ctx);
2719 return;
2720 }
2721 }
2722
2723#if USE_SSL_CRTD
2724 try {
2725 debugs(33, 5, "Generating SSL certificate for " << certProperties.commonName << " using ssl_crtd.");
2728 request_message.composeRequest(certProperties);
2729 debugs(33, 5, "SSL crtd request: " << request_message.compose().c_str());
2730 Ssl::Helper::Submit(request_message, sslCrtdHandleReplyWrapper, this);
2731 return;
2732 } catch (const std::exception &e) {
2733 debugs(33, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtd " <<
2734 "request for " << certProperties.commonName <<
2735 " certificate: " << e.what() << "; will now block to " <<
2736 "generate that certificate.");
2737 // fall through to do blocking in-process generation.
2738 }
2739#endif // USE_SSL_CRTD
2740
2741 debugs(33, 5, "Generating SSL certificate for " << certProperties.commonName);
2744 auto ssl = fd_table[clientConnection->fd].ssl.get();
2745 if (!Ssl::configureSSL(ssl, certProperties, *port))
2746 debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2747
2750 } else {
2752 if (dynCtx && !sslBumpCertKey.isEmpty())
2754 getSslContextDone(dynCtx);
2755 }
2756 return;
2757 }
2758
2761}
2762
2763void
2765{
2766 if (port->secure.generateHostCertificates && !ctx) {
2767 debugs(33, 2, "Failed to generate TLS context for " << tlsConnectHostOrIp);
2768 }
2769
2770 // If generated ssl context = nullptr, try to use static ssl context.
2771 if (!ctx) {
2772 if (!port->secure.staticContext) {
2773 debugs(83, DBG_IMPORTANT, "Closing " << clientConnection->remote << " as lacking TLS context");
2775 return;
2776 } else {
2777 debugs(33, 5, "Using static TLS context.");
2778 ctx = port->secure.staticContext;
2779 }
2780 }
2781
2782 if (!httpsCreate(this, ctx))
2783 return;
2784
2785 // bumped intercepted conns should already have Config.Timeout.request set
2786 // but forwarded connections may only have Config.Timeout.lifetime. [Re]set
2787 // to make sure the connection does not get stuck on non-SSL clients.
2789
2790 switchedToHttps_ = true;
2791
2792 auto ssl = fd_table[clientConnection->fd].ssl.get();
2793 BIO *b = SSL_get_rbio(ssl);
2794 Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
2795 bio->setReadBufData(inBuf);
2796 inBuf.clear();
2798}
2799
2800void
2802{
2804 Must(http->request);
2805 auto &request = http->request;
2806
2807 // Depending on receivedFirstByte_, we are at the start of either an
2808 // established CONNECT tunnel with the client or an intercepted TCP (and
2809 // presumably TLS) connection from the client. Expect TLS Client Hello.
2810 const auto insideConnectTunnel = receivedFirstByte_;
2811 debugs(33, 5, (insideConnectTunnel ? "post-CONNECT " : "raw TLS ") << clientConnection);
2812
2813 tlsConnectHostOrIp = request->url.hostOrIp();
2814 tlsConnectPort = request->url.port();
2815 resetSslCommonName(request->url.host());
2816
2817 // We are going to read new request
2818 flags.readMore = true;
2819
2820 // keep version major.minor details the same.
2821 // but we are now performing the HTTPS handshake traffic
2823
2824 // If sslServerBump is set, then we have decided to deny CONNECT
2825 // and now want to switch to SSL to send the error to the client
2826 // without even peeking at the origin server certificate.
2827 if (bumpServerMode == Ssl::bumpServerFirst && !sslServerBump) {
2828 request->flags.sslPeek = true;
2829 sslServerBump = new Ssl::ServerBump(http);
2830 } else if (bumpServerMode == Ssl::bumpPeek || bumpServerMode == Ssl::bumpStare) {
2831 request->flags.sslPeek = true;
2832 sslServerBump = new Ssl::ServerBump(http, nullptr, bumpServerMode);
2833 }
2834
2835 // commSetConnTimeout() was called for this request before we switched.
2836 // Fix timeout to request_start_timeout
2838 // Also reset receivedFirstByte_ flag to allow this timeout work in the case we have
2839 // a bumbed "connect" request on non transparent port.
2840 receivedFirstByte_ = false;
2841 // Get more data to peek at Tls
2842 parsingTlsHandshake = true;
2843
2844 // If the protocol has changed, then reset preservingClientData_.
2845 // Otherwise, its value initially set in start() is still valid/fresh.
2846 // shouldPreserveClientData() uses parsingTlsHandshake which is reset above.
2847 if (insideConnectTunnel)
2849
2850 readSomeData();
2851}
2852
2853void
2855{
2857
2858 assert(!inBuf.isEmpty());
2860 fd_note(clientConnection->fd, "Parsing TLS handshake");
2861
2862 // stops being nil if we fail to parse the handshake
2863 ErrorDetail::Pointer parseErrorDetails;
2864
2865 try {
2866 if (!tlsParser.parseHello(inBuf)) {
2867 // need more data to finish parsing
2868 readSomeData();
2869 return;
2870 }
2871 }
2872 catch (const TextException &ex) {
2873 debugs(83, 2, "exception: " << ex);
2874 parseErrorDetails = new ExceptionErrorDetail(ex.id());
2875 }
2876 catch (...) {
2877 debugs(83, 2, "exception: " << CurrentException);
2878 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_PARSE");
2879 parseErrorDetails = d;
2880 }
2881
2882 parsingTlsHandshake = false;
2883
2884 // client data may be needed for splicing and for
2885 // tunneling unsupportedProtocol after an error
2887
2888 // Even if the parser failed, each TLS detail should either be set
2889 // correctly or still be "unknown"; copying unknown detail is a no-op.
2892 if (details && !details->serverName.isEmpty()) {
2893 resetSslCommonName(details->serverName.c_str());
2894 tlsClientSni_ = details->serverName;
2895 }
2896
2897 // We should disable read/write handlers
2899
2900 if (parseErrorDetails) {
2902 Must(context && context->http);
2903 HttpRequest::Pointer request = context->http->request;
2904 debugs(83, 5, "Got something other than TLS Client Hello. Cannot SslBump.");
2905 updateError(ERR_PROTOCOL_UNKNOWN, parseErrorDetails);
2908 return;
2909 }
2910
2911 if (!sslServerBump || sslServerBump->act.step1 == Ssl::bumpClientFirst) { // Either means client-first.
2913 return;
2915 debugs(83, 5, "server-first skips step2; start forwarding the request");
2918 ClientHttpRequest *http = context ? context->http : nullptr;
2919 // will call httpsPeeked() with certificate and connection, eventually
2921 } else {
2924 }
2925}
2926
2927static void
2929{
2930 ConnStateData *connState = (ConnStateData *) data;
2931
2932 // if the connection is closed or closing, just return.
2933 if (!connState->isOpen())
2934 return;
2935
2936 debugs(33, 5, "Answer: " << answer << " kind:" << answer.kind);
2937 assert(connState->serverBump());
2938 Ssl::BumpMode bumpAction;
2939 if (answer.allowed()) {
2940 bumpAction = (Ssl::BumpMode)answer.kind;
2941 } else
2942 bumpAction = Ssl::bumpSplice;
2943
2944 connState->serverBump()->act.step2 = bumpAction;
2945 connState->sslBumpMode = bumpAction;
2946 Http::StreamPointer context = connState->pipeline.front();
2947 if (ClientHttpRequest *http = (context ? context->http : nullptr))
2948 http->al->ssl.bumpMode = bumpAction;
2949
2950 if (bumpAction == Ssl::bumpTerminate) {
2951 connState->clientConnection->close();
2952 } else if (bumpAction != Ssl::bumpSplice) {
2953 connState->startPeekAndSplice();
2954 } else if (!connState->splice())
2955 connState->clientConnection->close();
2956}
2957
2958bool
2960{
2961 // normally we can splice here, because we just got client hello message
2962
2963 // fde::ssl/tls_read_method() probably reads from our own inBuf. If so, then
2964 // we should not lose any raw bytes when switching to raw I/O here.
2965 if (fd_table[clientConnection->fd].ssl.get())
2966 fd_table[clientConnection->fd].useDefaultIo();
2967
2968 // XXX: assuming that there was an HTTP/1.1 CONNECT to begin with...
2969 // reset the current protocol to HTTP/1.1 (was "HTTPS" for the bumping process)
2971 assert(!pipeline.empty());
2973 Must(context);
2974 Must(context->http);
2975 ClientHttpRequest *http = context->http;
2976 HttpRequest::Pointer request = http->request;
2977 context->finished();
2978 if (transparent()) {
2979 // For transparent connections, make a new fake CONNECT request, now
2980 // with SNI as target. doCallout() checks, adaptations may need that.
2981 return fakeAConnectRequest("splice", preservedClientData);
2982 } else {
2983 // For non transparent connections make a new tunneled CONNECT, which
2984 // also sets the HttpRequest::flags::forceTunnel flag to avoid
2985 // respond with "Connection Established" to the client.
2986 // This fake CONNECT request required to allow use of SNI in
2987 // doCallout() checks and adaptations.
2988 return initiateTunneledRequest(request, "splice", preservedClientData);
2989 }
2990}
2991
2992void
2994{
2995 // This is the Step2 of the SSL bumping
2998 ClientHttpRequest *http = context ? context->http : nullptr;
2999
3002 // Run a accessList check to check if want to splice or continue bumping
3003
3008 fillChecklist(*acl_checklist);
3010 return;
3011 }
3012
3013 // will call httpsPeeked() with certificate and connection, eventually
3014 Security::ContextPointer unConfiguredCTX(Ssl::createSSLContext(port->secure.signingCa.cert, port->secure.signingCa.pkey, port->secure));
3015 fd_table[clientConnection->fd].dynamicTlsContext = unConfiguredCTX;
3016
3017 if (!httpsCreate(this, unConfiguredCTX))
3018 return;
3019
3020 switchedToHttps_ = true;
3021
3022 auto ssl = fd_table[clientConnection->fd].ssl.get();
3023 BIO *b = SSL_get_rbio(ssl);
3024 Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3025 bio->setReadBufData(inBuf);
3026 bio->hold(true);
3027
3028 // We have successfully parsed client Hello, but our TLS handshake parser is
3029 // forgiving. Now we use a TLS library to parse the same bytes, so that we
3030 // can honor on_unsupported_protocol if needed. If there are no errors, we
3031 // expect Security::Accept() to ask us to write (our) TLS server Hello. We
3032 // also allow an ioWantRead result in case some fancy TLS extension that
3033 // Squid does not yet understand requires reading post-Hello client bytes.
3034 const auto handshakeResult = acceptTls();
3035 if (!handshakeResult.wantsIo())
3036 return handleSslBumpHandshakeError(handshakeResult);
3037
3038 // We need to reset inBuf here, to be used by incoming requests in the case
3039 // of SSL bump
3040 inBuf.clear();
3041
3042 debugs(83, 5, "Peek and splice at step2 done. Start forwarding the request!!! ");
3045}
3046
3048void
3050{
3051 auto errCategory = ERR_NONE;
3052
3053 switch (handshakeResult.category) {
3055 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_SUCCESS");
3056 updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3057 break;
3058 }
3059
3061 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_READ");
3062 updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3063 break;
3064 }
3065
3067 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_WRITE");
3068 updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3069 break;
3070 }
3071
3073 debugs(83, (handshakeResult.important ? DBG_IMPORTANT : 2), "ERROR: " << handshakeResult.errorDescription <<
3074 " while SslBump-accepting a TLS connection on " << clientConnection << ": " << handshakeResult.errorDetail);
3075 updateError(errCategory = ERR_SECURE_ACCEPT_FAIL, handshakeResult.errorDetail);
3076 break;
3077
3078 }
3079
3080 if (!tunnelOnError(errCategory))
3082}
3083
3084void
3086{
3087 auto ssl = fd_table[clientConnection->fd].ssl.get();
3088 BIO *b = SSL_get_rbio(ssl);
3089 assert(b);
3090 Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3091
3092 debugs(33, 5, "PeekAndSplice mode, proceed with client negotiation. Current state:" << SSL_state_string_long(ssl));
3093 bio->hold(false);
3094
3096 switchedToHttps_ = true;
3097}
3098
3099void
3101{
3102 Must(sslServerBump != nullptr);
3104 Must(pipeline.empty() || pipeline.front()->http == nullptr || pipeline.front()->http->request == pic.request.getRaw());
3105
3106 if (Comm::IsConnOpen(pic.connection)) {
3108 debugs(33, 5, "bumped HTTPS server: " << tlsConnectHostOrIp);
3109 } else
3110 debugs(33, 5, "Error while bumping: " << tlsConnectHostOrIp);
3111
3113}
3114
3115#endif /* USE_OPENSSL */
3116
3117bool
3118ConnStateData::initiateTunneledRequest(HttpRequest::Pointer const &cause, const char *reason, const SBuf &payload)
3119{
3120 // fake a CONNECT request to force connState to tunnel
3121 SBuf connectHost;
3122 AnyP::Port connectPort;
3123
3124 if (pinning.serverConnection != nullptr) {
3125 static char ip[MAX_IPSTRLEN];
3126 connectHost = pinning.serverConnection->remote.toStr(ip, sizeof(ip));
3127 if (const auto remotePort = pinning.serverConnection->remote.port())
3128 connectPort = remotePort;
3129 } else if (cause) {
3130 connectHost = cause->url.hostOrIp();
3131 connectPort = cause->url.port();
3132#if USE_OPENSSL
3133 } else if (!tlsConnectHostOrIp.isEmpty()) {
3134 connectHost = tlsConnectHostOrIp;
3135 connectPort = tlsConnectPort;
3136#endif
3137 } else if (transparent()) {
3138 static char ip[MAX_IPSTRLEN];
3139 connectHost = clientConnection->local.toStr(ip, sizeof(ip));
3140 connectPort = clientConnection->local.port();
3141 }
3142
3143 if (!connectPort) {
3144 // Typical cases are malformed HTTP requests on http_port and malformed
3145 // TLS handshakes on non-bumping https_port. TODO: Discover these
3146 // problems earlier so that they can be classified/detailed better.
3147 debugs(33, 2, "Not able to compute URL, abort request tunneling for " << reason);
3148 // TODO: throw when nonBlockingCheck() callbacks gain job protections
3149 static const auto d = MakeNamedErrorDetail("TUNNEL_TARGET");
3151 return false;
3152 }
3153
3154 debugs(33, 2, "Request tunneling for " << reason);
3155 const auto http = buildFakeRequest(connectHost, *connectPort, payload);
3156 HttpRequest::Pointer request = http->request;
3157 request->flags.forceTunnel = true;
3158 http->calloutContext = new ClientRequestContext(http);
3159 http->doCallouts();
3160 clientProcessRequestFinished(this, request);
3161 return true;
3162}
3163
3164bool
3165ConnStateData::fakeAConnectRequest(const char *reason, const SBuf &payload)
3166{
3167 debugs(33, 2, "fake a CONNECT request to force connState to tunnel for " << reason);
3168
3169 SBuf connectHost;
3171 const unsigned short connectPort = clientConnection->local.port();
3172
3173#if USE_OPENSSL
3174 if (!tlsClientSni_.isEmpty())
3175 connectHost.assign(tlsClientSni_);
3176 else
3177#endif
3178 {
3179 static char ip[MAX_IPSTRLEN];
3180 clientConnection->local.toHostStr(ip, sizeof(ip));
3181 connectHost.assign(ip);
3182 }
3183
3184 ClientHttpRequest *http = buildFakeRequest(connectHost, connectPort, payload);
3185
3186 http->calloutContext = new ClientRequestContext(http);
3187 HttpRequest::Pointer request = http->request;
3188 http->doCallouts();
3189 clientProcessRequestFinished(this, request);
3190 return true;
3191}
3192
3194ConnStateData::buildFakeRequest(SBuf &useHost, const AnyP::KnownPort usePort, const SBuf &payload)
3195{
3196 ClientHttpRequest *http = new ClientHttpRequest(this);
3197 Http::Stream *stream = new Http::Stream(clientConnection, http);
3198
3199 StoreIOBuffer tempBuffer;
3200 tempBuffer.data = stream->reqbuf;
3201 tempBuffer.length = HTTP_REQBUF_SZ;
3202
3203 ClientStreamData newServer = new clientReplyContext(http);
3204 ClientStreamData newClient = stream;
3207 clientSocketDetach, newClient, tempBuffer);
3208
3209 stream->flags.parsed_ok = 1; // Do we need it?
3210 stream->mayUseConnection(true);
3212 stream->registerWithConn();
3213
3214 const auto mx = MasterXaction::MakePortful(port);
3215 mx->tcpClient = clientConnection;
3216 // Setup Http::Request object. Maybe should be replaced by a call to (modified)
3217 // clientProcessRequest
3218 HttpRequest::Pointer request = new HttpRequest(mx);
3219 request->url.setScheme(AnyP::PROTO_AUTHORITY_FORM, nullptr);
3220 request->method = Http::METHOD_CONNECT;
3221 request->url.host(useHost.c_str());
3222 request->url.port(usePort);
3223
3224 http->uri = SBufToCstring(request->effectiveRequestUri());
3225 http->initRequest(request.getRaw());
3226
3227 request->manager(this, http->al);
3228
3229 request->header.putStr(Http::HOST, useHost.c_str());
3230
3231 request->sources |= ((switchedToHttps() || port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
3232#if USE_AUTH
3233 if (getAuth())
3234 request->auth_user_request = getAuth();
3235#endif
3236
3237 inBuf = payload;
3238 flags.readMore = false;
3239
3240 return http;
3241}
3242
3244static bool
3246{
3247 if (!Comm::IsConnOpen(c)) {
3248 Must(NHttpSockets > 0); // we tried to open some
3249 --NHttpSockets; // there will be fewer sockets than planned
3250 Must(HttpSockets[NHttpSockets] < 0); // no extra fds received
3251
3252 if (!NHttpSockets) // we could not open any listen sockets at all
3253 fatalf("Unable to open %s",FdNote(portType));
3254
3255 return false;
3256 }
3257 return true;
3258}
3259
3261static bool
3263{
3264 bool found = false;
3265 for (int i = 0; i < NHttpSockets && !found; ++i) {
3266 if ((found = HttpSockets[i] < 0))
3267 HttpSockets[i] = conn->fd;
3268 }
3269 return found;
3270}
3271
3272static void
3274{
3275 const auto savedContext = CodeContext::Current();
3276 for (AnyP::PortCfgPointer s = HttpPortList; s != nullptr; s = s->next) {
3278 const SBuf &scheme = AnyP::UriScheme(s->transport.protocol).image();
3279
3281 debugs(1, DBG_IMPORTANT, "WARNING: You have too many '" << scheme << "_port' lines." <<
3282 Debug::Extra << "The limit is " << MAXTCPLISTENPORTS << " HTTP ports.");
3283 continue;
3284 }
3285
3286#if USE_OPENSSL
3287 if (s->flags.tunnelSslBumping) {
3288 if (!Config.accessList.ssl_bump) {
3289 debugs(33, DBG_IMPORTANT, "WARNING: No ssl_bump configured. Disabling ssl-bump on " << scheme << "_port " << s->s);
3290 s->flags.tunnelSslBumping = false;
3291 }
3292 if (!s->secure.staticContext && !s->secure.generateHostCertificates) {
3293 debugs(1, DBG_IMPORTANT, "Will not bump SSL at " << scheme << "_port " << s->s << " due to TLS initialization failure.");
3294 s->flags.tunnelSslBumping = false;
3295 if (s->transport.protocol == AnyP::PROTO_HTTP)
3296 s->secure.encryptTransport = false;
3297 }
3298 if (s->flags.tunnelSslBumping) {
3299 // Create ssl_ctx cache for this port.
3300 Ssl::TheGlobalContextStorage.addLocalStorage(s->s, s->secure.dynamicCertMemCacheSize);
3301 }
3302 }
3303#endif
3304
3305 if (s->secure.encryptTransport && !s->secure.staticContext) {
3306 debugs(1, DBG_CRITICAL, "ERROR: Ignoring " << scheme << "_port " << s->s << " due to TLS context initialization failure.");
3307 continue;
3308 }
3309
3310 const auto protocol = s->transport.protocol;
3311 assert(protocol == AnyP::PROTO_HTTP || protocol == AnyP::PROTO_HTTPS);
3312 const auto isHttps = protocol == AnyP::PROTO_HTTPS;
3313 using AcceptCall = CommCbFunPtrCallT<CommAcceptCbPtrFun>;
3314 RefCount<AcceptCall> subCall = commCbCall(5, 5, isHttps ? "httpsAccept" : "httpAccept",
3317 }
3318 CodeContext::Reset(savedContext);
3319}
3320
3321void
3323{
3324 // Fill out a Comm::Connection which IPC will open as a listener for us
3325 port->listenConn = new Comm::Connection;
3326 port->listenConn->local = port->s;
3327 port->listenConn->flags =
3329 (port->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) |
3330 (port->flags.natIntercept ? COMM_INTERCEPTION : 0) |
3331 (port->workerQueues ? COMM_REUSEPORT : 0);
3332
3333 // route new connections to subCall
3334 typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
3336 const auto listenCall =
3337 asyncCall(33, 2, "clientListenerConnectionOpened",
3339 port, fdNote, sub));
3340 AsyncCallback<Ipc::StartListeningAnswer> callback(listenCall);
3341 Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, port->listenConn, fdNote, callback);
3342
3345 ++NHttpSockets;
3346}
3347
3349static void
3351{
3352 Must(s != nullptr);
3353
3354 if (!OpenedHttpSocket(s->listenConn, portTypeNote))
3355 return;
3356
3357 Must(Comm::IsConnOpen(s->listenConn));
3358
3359 // TCP: setup a job to handle accept() with subscribed handler
3360 AsyncJob::Start(new Comm::TcpAcceptor(s, FdNote(portTypeNote), sub));
3361
3362 debugs(1, Important(13), "Accepting " <<
3363 (s->flags.natIntercept ? "NAT intercepted " : "") <<
3364 (s->flags.tproxyIntercept ? "TPROXY intercepted " : "") <<
3365 (s->flags.tunnelSslBumping ? "SSL bumped " : "") <<
3366 (s->flags.accelSurrogate ? "reverse-proxy " : "")
3367 << FdNote(portTypeNote) << " connections at "
3368 << s->listenConn);
3369
3370 Must(AddOpenedHttpSocket(s->listenConn)); // otherwise, we have received a fd we did not ask for
3371
3372#if USE_SYSTEMD
3373 // When the very first port opens, tell systemd we are able to serve connections.
3374 // Subsequent sd_notify() calls, including calls during reconfiguration,
3375 // do nothing because the first call parameter is 1.
3376 // XXX: Send the notification only after opening all configured ports.
3378 const auto result = sd_notify(1, "READY=1");
3379 if (result < 0) {
3380 debugs(1, DBG_IMPORTANT, "WARNING: failed to send start-up notification to systemd" <<
3381 Debug::Extra << "sd_notify() error: " << xstrerr(-result));
3382 }
3383 }
3384#endif
3385}
3386
3387void
3389{
3392
3393 if (NHttpSockets < 1)
3394 fatal("No HTTP, HTTPS, or FTP ports configured");
3395}
3396
3397void
3399{
3400 const auto savedContext = CodeContext::Current();
3401 for (AnyP::PortCfgPointer s = HttpPortList; s != nullptr; s = s->next) {
3403 if (s->listenConn != nullptr) {
3404 debugs(1, Important(14), "Closing HTTP(S) port " << s->listenConn->local);
3405 s->listenConn->close();
3406 s->listenConn = nullptr;
3407 }
3408 }
3409 CodeContext::Reset(savedContext);
3410
3412
3413 // TODO see if we can drop HttpSockets array entirely */
3414 for (int i = 0; i < NHttpSockets; ++i) {
3415 HttpSockets[i] = -1;
3416 }
3417
3418 NHttpSockets = 0;
3419}
3420
3421int
3423{
3424 SBuf vary(request->vary_headers);
3425 const auto &reply = entry->mem().freshestReply();
3426 auto has_vary = reply.header.has(Http::HdrType::VARY);
3427#if X_ACCELERATOR_VARY
3428
3429 has_vary |=
3430 reply.header.has(Http::HdrType::HDR_X_ACCELERATOR_VARY);
3431#endif
3432
3433 if (!has_vary || entry->mem_obj->vary_headers.isEmpty()) {
3434 if (!vary.isEmpty()) {
3435 /* Oops... something odd is going on here.. */
3436 debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary object on second attempt, '" <<
3437 entry->mem_obj->urlXXX() << "' '" << vary << "'");
3438 request->vary_headers.clear();
3439 return VARY_CANCEL;
3440 }
3441
3442 if (!has_vary) {
3443 /* This is not a varying object */
3444 return VARY_NONE;
3445 }
3446
3447 /* virtual "vary" object found. Calculate the vary key and
3448 * continue the search
3449 */
3450 vary = httpMakeVaryMark(request, &reply);
3451
3452 if (!vary.isEmpty()) {
3453 request->vary_headers = vary;
3454 return VARY_OTHER;
3455 } else {
3456 /* Ouch.. we cannot handle this kind of variance */
3457 /* XXX This cannot really happen, but just to be complete */
3458 return VARY_CANCEL;
3459 }
3460 } else {
3461 if (vary.isEmpty()) {
3462 vary = httpMakeVaryMark(request, &reply);
3463
3464 if (!vary.isEmpty())
3465 request->vary_headers = vary;
3466 }
3467
3468 if (vary.isEmpty()) {
3469 /* Ouch.. we cannot handle this kind of variance */
3470 /* XXX This cannot really happen, but just to be complete */
3471 return VARY_CANCEL;
3472 } else if (vary.cmp(entry->mem_obj->vary_headers) == 0) {
3473 return VARY_MATCH;
3474 } else {
3475 /* Oops.. we have already been here and still haven't
3476 * found the requested variant. Bail out
3477 */
3478 debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary match on second attempt, '" <<
3479 entry->mem_obj->urlXXX() << "' '" << vary << "'");
3480 return VARY_CANCEL;
3481 }
3482 }
3483}
3484
3487{
3488 const auto checklist = new ACLFilledChecklist(acl, nullptr, nullptr);
3489 clientAclChecklistFill(*checklist, http);
3490 return checklist;
3491}
3492
3493void
3495{
3496 assert(http);
3497
3498 if (!checklist.request && http->request)
3499 checklist.setRequest(http->request);
3500
3501 if (!checklist.al && http->al) {
3502 checklist.al = http->al;
3503 checklist.syncAle(http->request, http->log_uri);
3504 if (!checklist.reply && http->al->reply) {
3505 checklist.reply = http->al->reply.getRaw();
3506 HTTPMSGLOCK(checklist.reply);
3507 }
3508 }
3509
3510 if (const auto conn = http->getConn())
3511 checklist.setConn(conn); // may already be set
3512}
3513
3514void
3516{
3517 const auto context = pipeline.front();
3518 if (const auto http = context ? context->http : nullptr)
3519 return clientAclChecklistFill(checklist, http); // calls checklist.setConn()
3520
3521 // no requests, but we always have connection-level details
3522 // TODO: ACL checks should not require a mutable ConnStateData. Adjust the
3523 // code that accidentally violates that principle to remove this const_cast!
3524 checklist.setConn(const_cast<ConnStateData*>(this));
3525
3526 // Set other checklist fields inside our fillConnectionLevelDetails() rather
3527 // than here because clientAclChecklistFill() code path calls that method
3528 // (via ACLFilledChecklist::setConn()) rather than calling us directly.
3529}
3530
3531void
3533{
3534 assert(checklist.conn() == this);
3536
3537 if (!checklist.request) { // preserve (better) addresses supplied by setRequest()
3538 checklist.src_addr = clientConnection->remote;
3539 checklist.my_addr = clientConnection->local; // TODO: or port->s?
3540 }
3541
3542#if USE_OPENSSL
3543 if (!checklist.sslErrors && sslServerBump)
3544 checklist.sslErrors = sslServerBump->sslErrors();
3545#endif
3546
3547 if (!checklist.rfc931[0]) // checklist creator may have supplied it already
3548 checklist.setIdent(clientConnection->rfc931);
3549
3550}
3551
3552bool
3554{
3556}
3557
3560{
3561 bodyPipe = new BodyPipe(this);
3562 if (size >= 0)
3564 else
3566 return bodyPipe;
3567}
3568
3569int64_t
3571{
3572 if (!bodyPipe)
3573 return 0; // request without a body or read/produced all body bytes
3574
3575 if (!bodyPipe->bodySizeKnown())
3576 return -1; // probably need to read more, but we cannot be sure
3577
3578 const int64_t needToProduce = bodyPipe->unproducedSize();
3579 const int64_t haveAvailable = static_cast<int64_t>(inBuf.length());
3580
3581 if (needToProduce <= haveAvailable)
3582 return 0; // we have read what we need (but are waiting for pipe space)
3583
3584 return needToProduce - haveAvailable;
3585}
3586
3587void
3589{
3590 debugs(33, 4, "receiving error (" << clientConnection << "): " << error <<
3591 "; old sending error: " <<
3592 (stoppedSending() ? stoppedSending_ : "none"));
3593
3594 if (const char *oldError = stoppedReceiving()) {
3595 debugs(33, 3, "already stopped receiving: " << oldError);
3596 return; // nothing has changed as far as this connection is concerned
3597 }
3598
3600
3601 if (const char *sendError = stoppedSending()) {
3602 debugs(33, 3, "closing because also stopped sending: " << sendError);
3604 }
3605}
3606
3607void
3609{
3610 if (bodyPipe != nullptr) {
3611 debugs(33, 4, "no consumer for virgin body " << bodyPipe->status());
3613 }
3614}
3615
3617void
3619{
3620 Must(bodyPipe != nullptr);
3621 debugs(33, 5, "start dechunking" << bodyPipe->status());
3624}
3625
3627void
3629{
3630 debugs(33, 5, "finish dechunking: " << withSuccess);
3631
3632 if (bodyPipe != nullptr) {
3633 debugs(33, 7, "dechunked tail: " << bodyPipe->status());
3634 BodyPipe::Pointer myPipe = bodyPipe;
3635 stopProducingFor(bodyPipe, withSuccess); // sets bodyPipe->bodySize()
3636 Must(!bodyPipe); // we rely on it being nil after we are done with body
3637 if (withSuccess) {
3638 Must(myPipe->bodySizeKnown());
3640 if (context != nullptr && context->http && context->http->request)
3641 context->http->request->setContentLength(myPipe->bodySize());
3642 }
3643 }
3644
3645 delete bodyParser;
3646 bodyParser = nullptr;
3647}
3648
3649// XXX: this is an HTTP/1-only operation
3650void
3652{
3653 if (const auto context = pipeline.front()) {
3654 if (context->http)
3655 context->http->al->reply = msg.reply;
3656 }
3657
3658 if (!isOpen()) {
3659 debugs(33, 3, "ignoring 1xx due to earlier closure");
3660 return;
3661 }
3662
3663 // HTTP/1 1xx status messages are only valid when there is a transaction to trigger them
3664 if (!pipeline.empty()) {
3665 HttpReply::Pointer rep(msg.reply);
3666 Must(rep);
3667 // remember the callback
3669
3672
3673 if (!writeControlMsgAndCall(rep.getRaw(), call)) {
3674 // but still inform the caller (so it may resume its operation)
3676 }
3677 return;
3678 }
3679
3680 debugs(33, 3, " closing due to missing context for 1xx");
3682}
3683
3684void
3686{
3688
3689 if (Http::StreamPointer deferredRequest = pipeline.front()) {
3690 debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded after control msg wrote");
3691 ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
3692 }
3693}
3694
3696void
3698{
3699 // FwdState might repin a failed connection sooner than this close
3700 // callback is called for the failed connection.
3701 assert(pinning.serverConnection == io.conn);
3702 pinning.closeHandler = nullptr; // Comm unregisters handlers before calling
3703 const bool sawZeroReply = pinning.zeroReply; // reset when unpinning
3704 pinning.serverConnection->noteClosure();
3705 unpinConnection(false);
3706
3707 if (sawZeroReply && clientConnection != nullptr) {
3708 debugs(33, 3, "Closing client connection on pinned zero reply.");
3710 }
3711
3712}
3713
3714void
3716{
3717 pinConnection(pinServer, *request);
3718}
3719
3720void
3722{
3723 Must(pic.connection);
3724 Must(pic.request);
3725 pinConnection(pic.connection, *pic.request);
3726
3727 // monitor pinned server connection for remote-end closures.
3729
3730 if (pipeline.empty())
3731 kick(); // in case parseRequests() was blocked by a busy pic.connection
3732}
3733
3735void
3737{
3738 if (Comm::IsConnOpen(pinning.serverConnection) &&
3739 pinning.serverConnection->fd == pinServer->fd) {
3740 debugs(33, 3, "already pinned" << pinServer);
3741 return;
3742 }
3743
3744 unpinConnection(true); // closes pinned connection, if any, and resets fields
3745
3746 pinning.serverConnection = pinServer;
3747
3748 debugs(33, 3, pinning.serverConnection);
3749
3750 Must(pinning.serverConnection != nullptr);
3751
3752 const char *pinnedHost = "[unknown]";
3753 pinning.host = xstrdup(request.url.host());
3754 pinning.port = request.url.port();
3755 pinnedHost = pinning.host;
3756 pinning.pinned = true;
3757 if (CachePeer *aPeer = pinServer->getPeer())
3758 pinning.peer = cbdataReference(aPeer);
3759 pinning.auth = request.flags.connectionAuth;
3760 char stmp[MAX_IPSTRLEN];
3761 char desc[FD_DESC_SZ];
3762 snprintf(desc, FD_DESC_SZ, "%s pinned connection for %s (%d)",
3763 (pinning.auth || !pinning.peer) ? pinnedHost : pinning.peer->name,
3766 fd_note(pinning.serverConnection->fd, desc);
3767
3769 pinning.closeHandler = JobCallback(33, 5,
3771 // remember the pinned connection so that cb does not unpin a fresher one
3772 typedef CommCloseCbParams Params;
3773 Params &params = GetCommParams<Params>(pinning.closeHandler);
3774 params.conn = pinning.serverConnection;
3775 comm_add_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3776}
3777
3780void
3782{
3783 if (pinning.readHandler != nullptr)
3784 return; // already monitoring
3785
3787 pinning.readHandler = JobCallback(33, 3,
3789 Comm::Read(pinning.serverConnection, pinning.readHandler);
3790}
3791
3792void
3794{
3795 if (pinning.readHandler != nullptr) {
3796 Comm::ReadCancel(pinning.serverConnection->fd, pinning.readHandler);
3797 pinning.readHandler = nullptr;
3798 }
3799}
3800
3801#if USE_OPENSSL
3802bool
3804{
3805 // A ready-for-reading connection means that the TLS server either closed
3806 // the connection, sent us some unexpected HTTP data, or started TLS
3807 // renegotiations. We should close the connection except for the last case.
3808
3809 Must(pinning.serverConnection != nullptr);
3810 auto ssl = fd_table[pinning.serverConnection->fd].ssl.get();
3811 if (!ssl)
3812 return false;
3813
3814 char buf[1];
3815 const int readResult = SSL_read(ssl, buf, sizeof(buf));
3816
3817 if (readResult > 0 || SSL_pending(ssl) > 0) {
3818 debugs(83, 2, pinning.serverConnection << " TLS application data read");
3819 return false;
3820 }
3821
3822 switch(const int error = SSL_get_error(ssl, readResult)) {
3823 case SSL_ERROR_WANT_WRITE:
3824 debugs(83, DBG_IMPORTANT, pinning.serverConnection << " TLS SSL_ERROR_WANT_WRITE request for idle pinned connection");
3825 [[fallthrough]]; // to restart monitoring, for now
3826
3827 case SSL_ERROR_NONE:
3828 case SSL_ERROR_WANT_READ:
3830 return true;
3831
3832 default:
3833 debugs(83, 2, pinning.serverConnection << " TLS error: " << error);
3834 return false;
3835 }
3836
3837 // not reached
3838 return true;
3839}
3840#endif
3841
3844void
3846{
3847 pinning.readHandler = nullptr; // Comm unregisters handlers before calling
3848
3849 if (io.flag == Comm::ERR_CLOSING)
3850 return; // close handler will clean up
3851
3852 Must(pinning.serverConnection == io.conn);
3853
3854#if USE_OPENSSL
3856 return;
3857#endif
3858
3859 const bool clientIsIdle = pipeline.empty();
3860
3861 debugs(33, 3, "idle pinned " << pinning.serverConnection << " read " <<
3862 io.size << (clientIsIdle ? " with idle client" : ""));
3863
3864 pinning.serverConnection->close();
3865
3866 // If we are still sending data to the client, do not close now. When we are done sending,
3867 // ConnStateData::kick() checks pinning.serverConnection and will close.
3868 // However, if we are idle, then we must close to inform the idle client and minimize races.
3869 if (clientIsIdle && clientConnection != nullptr)
3871}
3872
3875{
3876 debugs(33, 7, pinning.serverConnection);
3877 Must(request);
3878
3879 const auto pinningError = [&](const err_type type) {
3880 unpinConnection(true);
3881 HttpRequestPointer requestPointer = request;
3882 return ErrorState::NewForwarding(type, requestPointer, ale);
3883 };
3884
3885 if (!Comm::IsConnOpen(pinning.serverConnection))
3886 throw pinningError(ERR_ZERO_SIZE_OBJECT);
3887
3888 if (pinning.auth && pinning.host && strcasecmp(pinning.host, request->url.host()) != 0)
3889 throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_CONFLICT_HOST
3890
3891 if (pinning.port != request->url.port())
3892 throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_CONFLICT_HOST
3893
3894 if (pinning.peer && !cbdataReferenceValid(pinning.peer))
3895 throw pinningError(ERR_ZERO_SIZE_OBJECT);
3896
3897 if (pinning.peerAccessDenied)
3898 throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_FORWARDING_DENIED
3899
3901 return pinning.serverConnection;
3902}
3903
3906{
3907 if (const auto connManager = request ? request->pinnedConnection() : nullptr)
3908 return connManager->borrowPinnedConnection(request, ale);
3909
3910 // ERR_CANNOT_FORWARD is somewhat misleading here; we can still forward, but
3911 // there is no point since the client connection is now gone
3912 HttpRequestPointer requestPointer = request;
3913 throw ErrorState::NewForwarding(ERR_CANNOT_FORWARD, requestPointer, ale);
3914}
3915
3916void
3918{
3919 debugs(33, 3, pinning.serverConnection);
3920
3921 if (pinning.peer)
3923
3924 if (Comm::IsConnOpen(pinning.serverConnection)) {
3925 if (pinning.closeHandler != nullptr) {
3926 comm_remove_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3927 pinning.closeHandler = nullptr;
3928 }
3929
3931
3932 // close the server side socket if requested
3933 if (andClose)
3934 pinning.serverConnection->close();
3935 pinning.serverConnection = nullptr;
3936 }
3937
3938 safe_free(pinning.host);
3939
3940 pinning.zeroReply = false;
3941 pinning.peerAccessDenied = false;
3942
3943 /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host
3944 * connection has gone away */
3945}
3946
3947void
3949{
3950 auto error = rawError; // (cheap) copy so that we can detail
3951 // We detail even ERR_NONE: There should be no transactions left, and
3952 // detailed ERR_NONE will be unused. Otherwise, this detail helps in triage.
3953 if (error.details.empty()) {
3954 static const auto d = MakeNamedErrorDetail("WITH_CLIENT");
3955 error.details.push_back(d);
3956 }
3957
3958 debugs(33, 3, pipeline.count() << '/' << pipeline.nrequests << " after " << error);
3959
3960 if (pipeline.empty()) {
3961 bareError.update(error); // XXX: bareLogTagsErrors
3962 } else {
3963 // We terminate the current CONNECT/PUT/etc. context below, logging any
3964 // error details, but that context may leave unparsed bytes behind.
3965 // Consume them to stop checkLogging() from logging them again later.
3966 const auto intputToConsume =
3967#if USE_OPENSSL
3968 parsingTlsHandshake ? "TLS handshake" : // more specific than CONNECT
3969#endif
3970 bodyPipe ? "HTTP request body" :
3971 pipeline.back()->mayUseConnection() ? "HTTP CONNECT" :
3972 nullptr;
3973
3974 while (const auto context = pipeline.front()) {
3975 context->noteIoError(error, lte);
3976 context->finished(); // cleanup and self-deregister
3977 assert(context != pipeline.front());
3978 }
3979
3980 if (intputToConsume && !inBuf.isEmpty()) {
3981 debugs(83, 5, "forgetting client " << intputToConsume << " bytes: " << inBuf.length());
3982 inBuf.clear();
3983 }
3984 }
3985
3987}
3988
3990void
3992{
3993 // to simplify our logic, we assume that terminateAll() has been called
3995
3996 // do not log connections that closed after a transaction (it is normal)
3997 // TODO: access_log needs ACLs to match received-no-bytes connections
3999 return;
4000
4001 /* Create a temporary ClientHttpRequest object. Its destructor will log. */
4002 ClientHttpRequest http(this);
4003 http.req_sz = inBuf.length();
4004 // XXX: Or we died while waiting for the pinned connection to become idle.
4005 http.setErrorUri("error:transaction-end-before-headers");
4006 http.updateError(bareError);
4007}
4008
4009bool
4011{
4012 // PROXY protocol bytes are meant for us and, hence, cannot be tunneled
4014 return false;
4015
4016 // If our decision here is negative, configuration changes are irrelevant.
4017 // Otherwise, clientTunnelOnError() rechecks configuration before tunneling.
4019 return false;
4020
4021 // TODO: Figure out whether/how we can support FTP tunneling.
4022 if (port->transport.protocol == AnyP::PROTO_FTP)
4023 return false;
4024
4025#if USE_OPENSSL
4027 return true;
4028
4029 // the 1st HTTP request on a bumped connection
4031 return true;
4032#endif
4033
4034 // the 1st HTTP(S) request on a connection to an intercepting port
4035 if (!pipeline.nrequests && transparent())
4036 return true;
4037
4038 return false;
4039}
4040
4043{
4044 if (!theNotes)
4045 theNotes = new NotePairs;
4046 return theNotes;
4047}
4048
4049std::ostream &
4050operator <<(std::ostream &os, const ConnStateData::PinnedIdleContext &pic)
4051{
4052 return os << pic.connection << ", request=" << pic.request;
4053}
4054
4055std::ostream &
4056operator <<(std::ostream &os, const ConnStateData::ServerConnectionContext &scc)
4057{
4058 return os << scc.conn_ << ", srv_bytes=" << scc.preReadServerBytes.length();
4059}
4060
void accessLogLog(const AccessLogEntryPointer &, ACLChecklist *)
Definition: access_log.cc:136
#define Assure(condition)
Definition: Assure.h:35
RefCount< AsyncCallT< Dialer > > asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
CommCbFunPtrCallT< Dialer > * commCbCall(int debugSection, int debugLevel, const char *callName, const Dialer &dialer)
Definition: CommCalls.h:312
void IOACB(const CommAcceptCbParams &params)
Definition: CommCalls.h:31
#define COMM_TRANSPARENT
Definition: Connection.h:50
#define COMM_REUSEPORT
Definition: Connection.h:52
#define COMM_INTERCEPTION
Definition: Connection.h:51
#define COMM_NONBLOCKING
Definition: Connection.h:46
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
#define Here()
source code location of the caller
Definition: Here.h:15
void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, int64_t)
@ hoReply
Definition: HttpHeader.h:37
void IDCB(const char *ident, void *data)
Definition: Ident.h:17
@ LOG_TCP_OFFLINE_HIT
Definition: LogTags.h:54
@ LOG_TCP_HIT
Definition: LogTags.h:39
@ LOG_TCP_MISS
Definition: LogTags.h:40
@ LOG_TCP_MEM_HIT
Definition: LogTags.h:51
@ LOG_TCP_IMS_HIT
Definition: LogTags.h:47
@ LOG_TAG_NONE
Definition: LogTags.h:38
@ LOG_TCP_REFRESH_UNMODIFIED
Definition: LogTags.h:41
@ LOG_TCP_CLIENT_REFRESH_MISS
Definition: LogTags.h:46
@ LOG_TCP_INM_HIT
Definition: LogTags.h:48
int size
Definition: ModDevPoll.cc:75
int NHttpSockets
Definition: PortCfg.cc:25
int HttpSockets[MAXTCPLISTENPORTS]
Definition: PortCfg.cc:26
AnyP::PortCfgPointer HttpPortList
list of Squid http(s)_port configured
Definition: PortCfg.cc:22
#define MAXTCPLISTENPORTS
Definition: PortCfg.h:87
#define SQUIDSBUFPH
Definition: SBuf.h:31
void SBufToCstring(char *d, const SBuf &s)
Definition: SBuf.h:752
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
class SquidConfig Config
Definition: SquidConfig.cc:12
#define SQUIDSTRINGPH
Definition: SquidString.h:21
#define SQUIDSTRINGPRINT(s)
Definition: SquidString.h:22
StatCounters statCounter
Definition: StatCounters.cc:12
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
#define Must(condition)
Definition: TextException.h:75
int conn
the current server connection FD
Definition: Transport.cc:26
bool urlCheckRequest(const HttpRequest *r)
Definition: Uri.cc:919
void error(char *format,...)
#define acl_access
Definition: forward.h:45
#define assert(EX)
Definition: assert.h:17
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:265
#define cbdataReferenceDone(var)
Definition: cbdata.h:352
#define cbdataReference(var)
Definition: cbdata.h:343
void banAction(const Acl::Answer &action)
add action to the list of banned actions
Definition: Checklist.cc:395
const Acl::Tree * changeAcl(const Acl::Tree *t)
Definition: Checklist.h:176
Acl::Answer const & fastCheck()
Definition: Checklist.cc:332
void nonBlockingCheck(ACLCB *callback, void *callback_data)
Definition: Checklist.cc:237
void setIdent(const char *userIdentity)
configure rfc931 user identity for the first time
ConnStateData * conn() const
The client connection manager.
void setRequest(HttpRequest *)
configure client request-related fields for the first time
void setConn(ConnStateData *)
set either conn
char rfc931[USER_IDENT_SZ]
Ip::Address src_addr
CbcPointer< Security::CertErrors > sslErrors
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
void syncAle(HttpRequest *adaptedRequest, const char *logUri) const override
assigns uninitialized adapted_request and url ALE components
HttpRequest::Pointer request
char * last_meta
image of the last ICAP response header or eCAP meta received
AnyP::PortCfgPointer port
struct timeval start_time
The time the master transaction started.
struct timeval trTime
The response time.
MessageSizes clientReplySz
counters for the response sent to client
HttpRequestMethod method
MessageSizes clientRequestSz
counters for the original request received from client
AnyP::ProtocolVersion version
struct timeval processingTime
total ICAP processing time
HttpReplyPointer reply
class AccessLogEntry::CacheDetails cache
HierarchyLogEntry hier
void syncNotes(HttpRequest *request)
class AccessLogEntry::IcapLogEntry icap
class AccessLogEntry::Headers headers
Comm::ConnectionPointer tcpClient
TCP/IP level details about the client connection.
class AccessLogEntry::HttpDetails http
HttpRequest * adapted_request
HttpRequest * request
class AccessLogEntry::AdaptationDetails adapt
class AccessLogEntry::IcpDetails icp
ProxyProtocol::HeaderPointer proxyProtocolHeader
see ConnStateData::proxyProtocolHeader_
void updateError(const Error &)
sets (or updates the already stored) transaction error as needed
int kind
the matched custom access list verb (or zero)
Definition: Acl.h:170
bool allowed() const
Definition: Acl.h:156
HttpHeader lastMeta
Last received meta header (REQMOD or RESPMOD, whichever comes last).
Definition: History.h:61
unsigned int major
major version number
ProtocolType protocol
which protocol this version is for
unsigned int minor
minor version number
SBuf image() const
Definition: UriScheme.h:57
void setScheme(const AnyP::ProtocolType &p, const char *str)
convert the URL scheme to that given
Definition: Uri.h:70
void port(const Port p)
reset authority port subcomponent
Definition: Uri.h:95
void host(const char *src)
Definition: Uri.cc:100
SBuf hostOrIp() const
Definition: Uri.cc:115
a smart AsyncCall pointer for delivery of future results
static void Start(const Pointer &job)
Definition: AsyncJob.cc:37
virtual void start()
called by AsyncStart; do not call directly
Definition: AsyncJob.cc:59
void mustStop(const char *aReason)
Definition: AsyncJob.cc:85
void deleteThis(const char *aReason)
Definition: AsyncJob.cc:65
virtual void callException(const std::exception &e)
called when the job throws during an async call
Definition: AsyncJob.cc:143
virtual void releaseAuthServer()
Definition: UserRequest.cc:205
MemBuf & buf
Definition: BodyPipe.h:74
uint64_t producedSize() const
Definition: BodyPipe.h:112
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
size_t putMoreData(const char *buf, size_t size)
Definition: BodyPipe.cc:213
const MemBuf & buf() const
Definition: BodyPipe.h:137
bool bodySizeKnown() const
Definition: BodyPipe.h:109
uint64_t unproducedSize() const
Definition: BodyPipe.cc:179
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
const char * status() const
Definition: BodyPipe.cc:446
bool productionEnded() const
Definition: BodyPipe.h:113
bool mayNeedMoreData() const
Definition: BodyPipe.h:118
uint64_t bodySize() const
Definition: BodyPipe.cc:161
void enableAutoConsumption()
start or continue consuming when producing without consumer
Definition: BodyPipe.cc:316
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:18
static const CharacterSet DIGIT
Definition: CharacterSet.h:84
static const CharacterSet ALPHA
Definition: CharacterSet.h:76
static const CharacterSet HEXDIG
Definition: CharacterSet.h:88
unsigned short initial
std::vector< ClientDelayPool::Pointer > pools
static ClientDelayPools * Instance()
struct ClientHttpRequest::Out out
void clearRequest()
resets the current request and log_uri to nil
HttpRequest *const request
int64_t mRangeCLen() const
Definition: client_side.cc:764
ConnStateData * getConn() const
String rangeBoundaryStr() const
Definition: client_side.cc:804
void initRequest(HttpRequest *)
void checkForInternalAccess()
Checks whether the current request is internal and adjusts it accordingly.
void updateError(const Error &)
if necessary, stores new error information (if any)
MemObject * memObject() const
size_t req_sz
raw request size on input, not current request size
void setErrorUri(const char *)
HttpHdrRangeIter range_iter
struct ClientHttpRequest::Flags flags
bool multipartRangeRequest() const
Definition: client_side.cc:720
StoreEntry * storeEntry() const
ClientRequestContext * calloutContext
const LogTags & loggingTags() const
the processing tags associated with this request transaction.
const AccessLogEntry::Pointer al
access.log entry
StoreEntry * loggingEntry() const
struct ClientHttpRequest::Redirect redirect
void setWriteLimiter(const int aWriteSpeedLimit, const double anInitialBurst, const double aHighWatermark)
Definition: comm.cc:1376
Definition: ClpMap.h:41
const Value * get(const Key &)
Definition: ClpMap.h:188
void del(const Key &)
Remove the corresponding entry (if any)
Definition: ClpMap.h:268
bool add(const Key &, const Value &, Ttl)
Definition: ClpMap.h:220
static const Pointer & Current()
Definition: CodeContext.cc:33
static void Reset()
forgets the current context, setting it to nil/unknown
Definition: CodeContext.cc:77
AnyP::PortCfgPointer port
the configuration listening port this call relates to (may be nil)
Definition: CommCalls.h:100
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:83
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:85
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:82
Comm::ConnectionPointer conn
Definition: CommCalls.h:80
time_t timeLeft(const time_t idleTimeout) const
Definition: Connection.cc:146
CachePeer * getPeer() const
Definition: Connection.cc:124
Ip::Address remote
Definition: Connection.h:149
char rfc931[USER_IDENT_SZ]
Definition: Connection.h:176
Ip::Address local
Definition: Connection.h:146
Security::NegotiationHistory * tlsNegotiations()
Definition: Connection.cc:156
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:183
Comm::ConnectionPointer connection
to-server connection to be pinned
Definition: client_side.h:187
HttpRequest::Pointer request
to-server request that initiated serverConnection
Definition: client_side.h:188
noteTakeServerConnectionControl() callback parameter
Definition: client_side.h:213
Comm::ConnectionPointer conn_
to-server connection
Definition: client_side.h:224
SBuf preReadServerBytes
post-101 bytes received from the server
Definition: client_side.h:220
virtual int pipelinePrefetchMax() const
returning N allows a pipeline of 1+N requests (see pipeline_prefetch)
void postHttpsAccept()
the second part of old httpsAccept, waiting for future HttpsServer home
Ssl::ServerBump * serverBump()
Definition: client_side.h:285
void endingShutdown() override
bool fakeAConnectRequest(const char *reason, const SBuf &payload)
bool switchedToHttps() const
Definition: client_side.h:284
void readNextRequest()
Traffic parsing.
Definition: client_side.cc:881
virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io)
Our close handler called by Comm when the pinned connection is closed.
char * prepareTlsSwitchingURL(const Http1::RequestParserPointer &hp)
Security::KeyLogger keyLogger
managers logging of the being-accepted TLS connection secrets
Definition: client_side.h:384
void afterClientRead() override
processing to be done after a Comm::Read()
Ssl::ServerBump * sslServerBump
HTTPS server cert. fetching state for bump-ssl-server-first.
Definition: client_side.h:494
Http::Stream * parseHttpRequest(const Http1::RequestParserPointer &)
virtual bool writeControlMsgAndCall(HttpReply *rep, AsyncCall::Pointer &call)=0
handle a control message received by context from a peer and call back
void switchToHttps(ClientHttpRequest *, Ssl::BumpMode bumpServerMode)
void startPinnedConnectionMonitoring()
const char * stoppedReceiving() const
true if we stopped receiving the request
Definition: client_side.h:159
void consumeInput(const size_t byteCount)
remove no longer needed leading bytes from the input buffer
void clientAfterReadingRequests()
ClientHttpRequest * buildFakeRequest(SBuf &useHost, AnyP::KnownPort usePort, const SBuf &payload)
build a fake http request
virtual void handleReply(HttpReply *header, StoreIOBuffer receivedData)=0
void setAuth(const Auth::UserRequest::Pointer &aur, const char *cause)
Definition: client_side.cc:518
virtual Http::Stream * parseOneRequest()=0
BodyPipe::Pointer bodyPipe
set when we are reading request body
Definition: client_side.h:431
const SBuf & tlsClientSni() const
Definition: client_side.h:294
void sendControlMsg(HttpControlMsg) override
called to send the 1xx message and notify the Source
void kick()
try to make progress on a transaction or read more I/O
Definition: client_side.cc:918
const Auth::UserRequest::Pointer & getAuth() const
Definition: client_side.h:123
void getSslContextStart()
Start to create dynamic Security::ContextPointer for host or uses static port SSL context.
void httpsPeeked(PinnedIdleContext pic)
called by FwdState when it is done bumping the server
bool initiateTunneledRequest(HttpRequest::Pointer const &cause, const char *reason, const SBuf &payload)
generates and sends to tunnel.cc a fake request with a given payload
void swanSong() override
Definition: client_side.cc:615
bool splice()
Splice a bumped client connection on peek-and-splice mode.
void start() override
called by AsyncStart; do not call directly
void parseRequests()
SBuf tlsClientSni_
TLS client delivered SNI value. Empty string if none has been received.
Definition: client_side.h:490
Error bareError
a problem that occurred without a request (e.g., while parsing headers)
Definition: client_side.h:381
bool handleReadData() override
void expectNoForwarding()
cleans up virgin request [body] forwarding state
Http::Stream * abortRequestParsing(const char *const errUri)
stop parsing the request and create context for relaying error info
void startShutdown() override
void sslCrtdHandleReply(const Helper::Reply &reply)
Process response from ssl_crtd.
err_type handleChunkedRequestBody()
parses available chunked encoded body bytes, checks size, returns errors
SBuf sslCommonName_
CN name for SSL certificate generation.
Definition: client_side.h:487
void resetSslCommonName(const char *name)
Definition: client_side.h:293
void afterClientWrite(size_t) override
processing to sync state after a Comm::Write()
void doneWithControlMsg() override
void terminateAll(const Error &, const LogTagsErrors &) override
abort any pending transactions and prevent new ones (by closing)
void abortChunkedRequestBody(const err_type error)
quit on errors related to chunked request body handling
void pinConnection(const Comm::ConnectionPointer &pinServerConn, const HttpRequest &request)
Forward future client requests using the given server connection.
bool preservingClientData_
whether preservedClientData is valid and should be kept up to date
Definition: client_side.h:434
void callException(const std::exception &) override
called when the job throws during an async call
Definition: client_side.cc:639
static void sslCrtdHandleReplyWrapper(void *data, const Helper::Reply &reply)
Callback function. It is called when squid receive message from ssl_crtd.
void lifetimeTimeout(const CommTimeoutCbParams &params)
Ssl::CertSignAlgorithm signAlgorithm
The signing algorithm to use.
Definition: client_side.h:495
Security::ContextPointer getTlsContextFromCache(const SBuf &cacheKey, const Ssl::CertificateProperties &certProperties)
AnyP::Port tlsConnectPort
The TLS server port number as passed in the CONNECT request.
Definition: client_side.h:486
Auth::UserRequest::Pointer auth_
some user details that can be used to perform authentication on this connection
Definition: client_side.h:474
Comm::ConnectionPointer borrowPinnedConnection(HttpRequest *, const AccessLogEntryPointer &)
ConnStateData-specific part of BorrowPinnedConnection()
bool shouldPreserveClientData() const
void parseTlsHandshake()
void notePinnedConnectionBecameIdle(PinnedIdleContext pic)
Called when a pinned connection becomes available for forwarding the next request.
void stopPinnedConnectionMonitoring()
The caller assumes responsibility for connection closure detection.
void pinBusyConnection(const Comm::ConnectionPointer &pinServerConn, const HttpRequest::Pointer &request)
bool handleIdleClientPinnedTlsRead()
bool proxyProtocolError(const char *reason)
bool serveDelayedError(Http::Stream *)
SBuf preservedClientData
Definition: client_side.h:349
struct ConnStateData::@37 pinning
const char * stoppedSending_
the reason why we no longer write the response or nil
Definition: client_side.h:499
void resetReadTimeout(time_t timeout)
(re)sets timeout for receiving more bytes from the client
Definition: client_side.cc:598
void quitAfterError(HttpRequest *request)
SBuf tlsConnectHostOrIp
The TLS server host name appears in CONNECT request or the server ip address for the intercepted requ...
Definition: client_side.h:485
struct ConnStateData::@36 flags
~ConnStateData() override
Definition: client_side.cc:671
void receivedFirstByte() override
Update flags and timeout after the first byte received.
uint64_t parsedBumpedRequestCount
The number of parsed HTTP requests headers on a bumped client connection.
Definition: client_side.h:481
Security::IoResult acceptTls()
BodyPipe::Pointer expectRequestBody(int64_t size)
bool switchedToHttps_
Definition: client_side.h:478
bool isOpen() const
Definition: client_side.cc:664
void clientPinnedConnectionRead(const CommIoCbParams &io)
static Comm::ConnectionPointer BorrowPinnedConnection(HttpRequest *, const AccessLogEntryPointer &)
void stopReceiving(const char *error)
note request receiving error and close as soon as we write the response
Ip::Address log_addr
Definition: client_side.h:136
void checkLogging()
log the last (attempt at) transaction if nobody else did
void unpinConnection(const bool andClose)
Undo pinConnection() and, optionally, close the pinned connection.
void handleSslBumpHandshakeError(const Security::IoResult &)
process a problematic Security::Accept() result on the SslBump code path
void startDechunkingRequest()
initialize dechunking state
void startPeekAndSplice()
Initializes and starts a peek-and-splice negotiation with the SSL client.
Ssl::BumpMode sslBumpMode
ssl_bump decision (Ssl::bumpEnd if n/a).
Definition: client_side.h:304
bool concurrentRequestQueueFilled() const
ConnStateData(const MasterXactionPointer &xact)
void extendLifetime()
(re)sets client_lifetime timeout
Definition: client_side.cc:606
ProxyProtocol::HeaderPointer proxyProtocolHeader_
the parsed PROXY protocol header
Definition: client_side.h:470
bool shouldCloseOnEof() const override
whether to stop serving our client after reading EOF on its connection
void fillChecklist(ACLFilledChecklist &) const override
configure the given checklist (to reflect the current transaction state)
void requestTimeout(const CommTimeoutCbParams &params)
void whenClientIpKnown()
virtual time_t idleTimeout() const =0
timeout to use when waiting for the next request
void finishDechunkingRequest(bool withSuccess)
put parsed content into input buffer and clean up
virtual void processParsedRequest(Http::StreamPointer &)=0
start processing a freshly parsed request
void getSslContextDone(Security::ContextPointer &)
finish configuring the newly created SSL context"
bool transparent() const
bool parsingTlsHandshake
Definition: client_side.h:479
bool parseProxyProtocolHeader()
AnyP::Port port
destination port of the request that caused serverConnection
Definition: client_side.h:145
bool handleRequestBodyData()
void updateError(const Error &)
if necessary, stores new error information (if any)
Definition: client_side.cc:652
void doPeekAndSpliceStep()
const char * stoppedReceiving_
the reason why we no longer read the request or nil
Definition: client_side.h:501
bool needProxyProtocolHeader_
whether PROXY protocol header is still expected
Definition: client_side.h:467
void noteBodyConsumerAborted(BodyPipe::Pointer) override=0
bool tunnelOnError(const err_type)
initiate tunneling if possible or return false otherwise
void stopSending(const char *error)
note response sending error and close as soon as we read the request
Definition: client_side.cc:983
int64_t mayNeedToReadMoreBody() const
Security::HandshakeParser tlsParser
Definition: client_side.h:308
void storeTlsContextToCache(const SBuf &cacheKey, Security::ContextPointer &ctx)
const char * stoppedSending() const
true if we stopped sending the response
Definition: client_side.h:161
SBuf sslBumpCertKey
Key to use to store/retrieve generated certificate.
Definition: client_side.h:491
NotePairs::Pointer notes()
Http1::TeChunkedParser * bodyParser
parses HT