client_side.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 33 Client-side Routines */
10
60#include "squid.h"
61#include "acl/FilledChecklist.h"
62#include "anyp/PortCfg.h"
63#include "base/AsyncCallbacks.h"
64#include "base/Subscription.h"
65#include "base/TextException.h"
66#include "CachePeer.h"
67#include "client_db.h"
68#include "client_side.h"
69#include "client_side_reply.h"
70#include "client_side_request.h"
72#include "clientStream.h"
73#include "comm.h"
74#include "comm/Connection.h"
75#include "comm/Loops.h"
76#include "comm/Read.h"
77#include "comm/TcpAcceptor.h"
78#include "comm/Write.h"
79#include "CommCalls.h"
80#include "debug/Messages.h"
82#include "errorpage.h"
83#include "fd.h"
84#include "fde.h"
85#include "fqdncache.h"
86#include "FwdState.h"
87#include "globals.h"
88#include "helper.h"
89#include "helper/Reply.h"
90#include "http.h"
93#include "http/Stream.h"
94#include "HttpHdrContRange.h"
95#include "HttpHeaderTools.h"
96#include "HttpReply.h"
97#include "HttpRequest.h"
98#include "ident/Config.h"
99#include "ident/Ident.h"
100#include "internal.h"
101#include "ipc/FdNotes.h"
102#include "ipc/StartListening.h"
103#include "log/access_log.h"
104#include "MemBuf.h"
105#include "MemObject.h"
106#include "mime_header.h"
107#include "parser/Tokenizer.h"
108#include "proxyp/Header.h"
109#include "proxyp/Parser.h"
110#include "sbuf/Stream.h"
111#include "security/Certificate.h"
113#include "security/Io.h"
114#include "security/KeyLog.h"
116#include "servers/forward.h"
117#include "SquidConfig.h"
118#include "StatCounters.h"
119#include "StatHist.h"
120#include "Store.h"
121#include "TimeOrTag.h"
122#include "tools.h"
123
124#if USE_AUTH
125#include "auth/UserRequest.h"
126#endif
127#if USE_DELAY_POOLS
128#include "ClientInfo.h"
129#include "MessageDelayPools.h"
130#endif
131#if USE_OPENSSL
132#include "ssl/bio.h"
133#include "ssl/context_storage.h"
134#include "ssl/gadgets.h"
135#include "ssl/helper.h"
136#include "ssl/ProxyCerts.h"
137#include "ssl/ServerBump.h"
138#include "ssl/support.h"
139#endif
140
141#include <climits>
142#include <cmath>
143#include <limits>
144
145#if HAVE_SYSTEMD_SD_DAEMON_H
146#include <systemd/sd-daemon.h>
147#endif
148
149// TODO: Remove this custom dialer and simplify by creating the TcpAcceptor
150// subscription later, inside clientListenerConnectionOpened() callback, just
151// like htcpOpenPorts(), icpOpenPorts(), and snmpPortOpened() do it.
154 public CallDialer,
155 public WithAnswer<Ipc::StartListeningAnswer>
156{
157public:
160 handler(aHandler), portCfg(aPortCfg), portTypeNote(note), sub(aSub) {}
161
162 /* CallDialer API */
163 void print(std::ostream &os) const override {
164 os << '(' << answer_ << ", " << FdNote(portTypeNote) << " port=" << (void*)&portCfg << ')';
165 }
166
167 virtual bool canDial(AsyncCall &) const { return true; }
168 virtual void dial(AsyncCall &) { (handler)(portCfg, portTypeNote, sub); }
169
170 /* WithAnswer API */
172
173public:
175
176private:
177 // answer_.conn (set/updated by IPC code) is portCfg.listenConn (used by us)
182};
183
184static void clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub);
185
187#if USE_IDENT
189#endif
190static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength);
191
192static void clientUpdateStatHistCounters(const LogTags &logType, int svc_time);
193static void clientUpdateStatCounters(const LogTags &logType);
195static bool clientPingHasFinished(ping_data const *aPing);
198
199char *skipLeadingSpace(char *aString);
200
201#if USE_IDENT
202static void
203clientIdentDone(const char *ident, void *data)
204{
206 xstrncpy(conn->clientConnection->rfc931, ident ? ident : dash_str, USER_IDENT_SZ);
207}
208#endif
209
210void
212{
214
215 if (logType.isTcpHit())
217
218 if (logType.oldType == LOG_TCP_HIT)
220 else if (logType.oldType == LOG_TCP_MEM_HIT)
222}
223
224void
225clientUpdateStatHistCounters(const LogTags &logType, int svc_time)
226{
235 switch (logType.oldType) {
236
239 break;
240
241 case LOG_TCP_INM_HIT:
242 case LOG_TCP_IMS_HIT:
244 break;
245
246 case LOG_TCP_HIT:
247
248 case LOG_TCP_MEM_HIT:
249
252 break;
253
254 case LOG_TCP_MISS:
255
258 break;
259
260 default:
261 /* make compiler warnings go away */
262 break;
263 }
264}
265
266bool
268{
269 if (0 != aPing->stop.tv_sec && 0 != aPing->start.tv_sec)
270 return true;
271
272 return false;
273}
274
275void
277{
278 ping_data *i;
279
280 switch (someEntry->code) {
281#if USE_CACHE_DIGESTS
282
283 case CD_PARENT_HIT:
284
285 case CD_SIBLING_HIT:
287 break;
288#endif
289
290 case SIBLING_HIT:
291
292 case PARENT_HIT:
293
295
298 i = &someEntry->ping;
299
302
303 if (i->timeout)
305
306 break;
307
308 case CLOSEST_PARENT:
309
310 case CLOSEST_DIRECT:
312
313 break;
314
315 default:
316 break;
317 }
318}
319
320void
322{
324
325 if (request->error)
327
330
332}
333
334void
336{
337 assert(request);
338 assert(aLogEntry != nullptr);
339
341 MemBuf mb;
342 mb.init();
343 request->header.packInto(&mb);
344 //This is the request after adaptation or redirection
345 aLogEntry->headers.adapted_request = xstrdup(mb.buf);
346
347 // the virgin request is saved to aLogEntry->request
348 if (aLogEntry->request) {
349 mb.reset();
350 aLogEntry->request->header.packInto(&mb);
351 aLogEntry->headers.request = xstrdup(mb.buf);
352 }
353
354#if USE_ADAPTATION
355 const Adaptation::History::Pointer ah = request->adaptLogHistory();
356 if (ah != nullptr) {
357 mb.reset();
358 ah->lastMeta.packInto(&mb);
359 aLogEntry->adapt.last_meta = xstrdup(mb.buf);
360 }
361#endif
362
363 mb.clean();
364 }
365
366#if ICAP_CLIENT
368 if (ih != nullptr)
369 ih->processingTime(aLogEntry->icap.processingTime);
370#endif
371
372 aLogEntry->http.method = request->method;
373 aLogEntry->http.version = request->http_ver;
374 aLogEntry->hier = request->hier;
375 aLogEntry->cache.extuser = request->extacl_user.termedBuf();
376
377 // Adapted request, if any, inherits and then collects all the stats, but
378 // the virgin request gets logged instead; copy the stats to log them.
379 // TODO: avoid losses by keeping these stats in a shared history object?
380 if (aLogEntry->request) {
381 aLogEntry->request->dnsWait = request->dnsWait;
382 aLogEntry->request->error = request->error;
383 }
384}
385
386void
388{
389 if (!out.size && loggingTags().oldType == LOG_TAG_NONE)
390 debugs(33, 5, "logging half-baked transaction: " << log_uri);
391
393 al->url = log_uri;
394 debugs(33, 9, "clientLogRequest: al.url='" << al->url << "'");
395
396 const auto findReply = [this]() -> const HttpReply * {
397 if (al->reply)
398 return al->reply.getRaw();
399 if (const auto le = loggingEntry())
400 return le->hasFreshestReply();
401 return nullptr;
402 };
403 if (const auto reply = findReply()) {
404 al->http.code = reply->sline.status();
405 al->http.content_type = reply->content_type.termedBuf();
406 }
407
408 debugs(33, 9, "clientLogRequest: http.code='" << al->http.code << "'");
409
410 if (loggingEntry() && loggingEntry()->mem_obj && loggingEntry()->objectLen() >= 0)
411 al->cache.objectSize = loggingEntry()->contentLen(); // payload duplicate ?? with or without TE ?
412
414 // the virgin request is saved to al->request
415 if (al->request && al->request->body_pipe)
418 // XXX: calculate without payload encoding or headers !!
419 al->http.clientReplySz.payloadData = out.size - out.headers_sz; // pretend its all un-encoded data for now.
420
422
424
425 if (request)
427
428#if USE_OPENSSL && 0
429
430 /* This is broken. Fails if the connection has been closed. Needs
431 * to snarf the ssl details some place earlier..
432 */
433 if (getConn() != NULL)
435
436#endif
437
438 /* Add notes (if we have a request to annotate) */
439 if (request) {
440 SBuf matched;
441 for (auto h: Config.notes) {
442 if (h->match(request, al->reply.getRaw(), al, matched)) {
443 request->notes()->add(h->key(), matched);
444 debugs(33, 3, h->key() << " " << matched);
445 }
446 }
447 // The al->notes and request->notes must point to the same object.
449 }
450
451 ACLFilledChecklist checklist(nullptr, request, nullptr);
452 if (al->reply) {
453 checklist.reply = al->reply.getRaw();
454 HTTPMSGLOCK(checklist.reply);
455 }
456
457 if (request) {
461 }
462 // no need checklist.syncAle(): already synced
463 checklist.al = al;
464 accessLogLog(al, &checklist);
465
466 bool updatePerformanceCounters = true;
469 statsCheck.al = al;
470 if (al->reply) {
471 statsCheck.reply = al->reply.getRaw();
472 HTTPMSGLOCK(statsCheck.reply);
473 }
474 updatePerformanceCounters = statsCheck.fastCheck().allowed();
475 }
476
477 if (updatePerformanceCounters) {
478 if (request)
480
481 if (getConn() != nullptr && getConn()->clientConnection != nullptr)
482 clientdbUpdate(getConn()->clientConnection->remote, loggingTags(), AnyP::PROTO_HTTP, out.size);
483 }
484}
485
486void
488{
489 safe_free(uri);
492 clearRequest();
493
496}
497
498void
500{
501 ClientHttpRequest *http = (ClientHttpRequest *)data;
502 assert(http != nullptr);
503 delete http;
504}
505
506/* This is a handler normally called by comm_close() */
508{
509 if (clientConnection) {
511 // keep closed clientConnection for logging, clientdb cleanup, etc.
512 }
513 deleteThis("ConnStateData::connStateClosed");
514}
515
516#if USE_AUTH
517void
519{
520 if (auth_ == nullptr) {
521 if (aur != nullptr) {
522 debugs(33, 2, "Adding connection-auth to " << clientConnection << " from " << by);
523 auth_ = aur;
524 }
525 return;
526 }
527
528 // clobered with self-pointer
529 // NP: something nasty is going on in Squid, but harmless.
530 if (aur == auth_) {
531 debugs(33, 2, "WARNING: Ignoring duplicate connection-auth for " << clientConnection << " from " << by);
532 return;
533 }
534
535 /*
536 * Connection-auth relies on a single set of credentials being preserved
537 * for all requests on a connection once they have been setup.
538 * There are several things which need to happen to preserve security
539 * when connection-auth credentials change unexpectedly or are unset.
540 *
541 * 1) auth helper released from any active state
542 *
543 * They can only be reserved by a handshake process which this
544 * connection can now never complete.
545 * This prevents helpers hanging when their connections close.
546 *
547 * 2) pinning is expected to be removed and server conn closed
548 *
549 * The upstream link is authenticated with the same credentials.
550 * Expecting the same level of consistency we should have received.
551 * This prevents upstream being faced with multiple or missing
552 * credentials after authentication.
553 * NP: un-pin is left to the cleanup in ConnStateData::swanSong()
554 * we just trigger that cleanup here via comm_reset_close() or
555 * ConnStateData::stopReceiving()
556 *
557 * 3) the connection needs to close.
558 *
559 * This prevents attackers injecting requests into a connection,
560 * or gateways wrongly multiplexing users into a single connection.
561 *
562 * When credentials are missing closure needs to follow an auth
563 * challenge for best recovery by the client.
564 *
565 * When credentials change there is nothing we can do but abort as
566 * fast as possible. Sending TCP RST instead of an HTTP response
567 * is the best-case action.
568 */
569
570 // clobbered with nul-pointer
571 if (aur == nullptr) {
572 debugs(33, 2, "WARNING: Graceful closure on " << clientConnection << " due to connection-auth erase from " << by);
574 auth_ = nullptr;
575 // XXX: need to test whether the connection re-auth challenge is sent. If not, how to trigger it from here.
576 // NP: the current situation seems to fix challenge loops in Safari without visible issues in others.
577 // we stop receiving more traffic but can leave the Job running to terminate after the error or challenge is delivered.
578 stopReceiving("connection-auth removed");
579 return;
580 }
581
582 // clobbered with alternative credentials
583 if (aur != auth_) {
584 debugs(33, 2, "ERROR: Closing " << clientConnection << " due to change of connection-auth from " << by);
586 auth_ = nullptr;
587 // this is a fatal type of problem.
588 // Close the connection immediately with TCP RST to abort all traffic flow
590 return;
591 }
592
593 /* NOT REACHABLE */
594}
595#endif
596
597void
599{
601 AsyncCall::Pointer callback = JobCallback(33, 5, TimeoutDialer, this, ConnStateData::requestTimeout);
602 commSetConnTimeout(clientConnection, timeout, callback);
603}
604
605void
607{
609 AsyncCall::Pointer callback = JobCallback(5, 4, TimeoutDialer, this, ConnStateData::lifetimeTimeout);
611}
612
613// cleans up before destructor is called
614void
616{
617 debugs(33, 2, clientConnection);
618
619 flags.readMore = false;
620 clientdbEstablished(clientConnection->remote, -1); /* decrement */
621
623 checkLogging();
624
625 // XXX: Closing pinned conn is too harsh: The Client may want to continue!
626 unpinConnection(true);
627
629
630#if USE_AUTH
631 // NP: do this bit after closing the connections to avoid side effects from unwanted TCP RST
632 setAuth(nullptr, "ConnStateData::SwanSong cleanup");
633#endif
634
635 flags.swanSang = true;
636}
637
638void
639ConnStateData::callException(const std::exception &ex)
640{
641 Server::callException(ex); // logs ex and stops the job
642
643 ErrorDetail::Pointer errorDetail;
644 if (const auto tex = dynamic_cast<const TextException*>(&ex))
645 errorDetail = new ExceptionErrorDetail(tex->id());
646 else
647 errorDetail = new ExceptionErrorDetail(Here().id());
648 updateError(ERR_GATEWAY_FAILURE, errorDetail);
649}
650
651void
653{
654 if (const auto context = pipeline.front()) {
655 const auto http = context->http;
656 assert(http);
657 http->updateError(error);
658 } else {
660 }
661}
662
663bool
665{
666 return cbdataReferenceValid(this) && // XXX: checking "this" in a method
668 !fd_table[clientConnection->fd].closing();
669}
670
672{
673 debugs(33, 3, clientConnection);
674
675 if (isOpen())
676 debugs(33, DBG_IMPORTANT, "ERROR: Squid BUG: ConnStateData did not close " << clientConnection);
677
678 if (!flags.swanSang)
679 debugs(33, DBG_IMPORTANT, "ERROR: Squid BUG: ConnStateData was not destroyed properly; " << clientConnection);
680
681 if (bodyPipe != nullptr)
683
684 delete bodyParser; // TODO: pool
685
686#if USE_OPENSSL
687 delete sslServerBump;
688#endif
689}
690
697void
699{
700 HttpRequest *request = http->request;
701
702 debugs(33, 3, "http_ver = " << request->http_ver);
703 debugs(33, 3, "method = " << request->method);
704
705 // TODO: move to HttpRequest::hdrCacheInit, just like HttpReply.
706 request->flags.proxyKeepalive = request->persistent();
707}
708
709int
711{
713 bodyLength > Config.maxRequestBodySize)
714 return 1; /* too large */
715
716 return 0;
717}
718
719bool
721{
723}
724
725void
727{
728 mb->appendf("\r\n--" SQUIDSTRINGPH "--\r\n", SQUIDSTRINGPRINT(boundary));
729 debugs(33, 6, "buf offset: " << mb->size);
730}
731
732void
733clientPackRangeHdr(const HttpReplyPointer &rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb)
734{
735 HttpHeader hdr(hoReply);
736 assert(rep);
737 assert(spec);
738
739 /* put boundary */
740 debugs(33, 5, "appending boundary: " << boundary);
741 /* rfc2046 requires to _prepend_ boundary with <crlf>! */
742 mb->appendf("\r\n--" SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(boundary));
743
744 /* stuff the header with required entries and pack it */
745
748
749 httpHeaderAddContRange(&hdr, *spec, rep->content_length);
750
751 hdr.packInto(mb);
752 hdr.clean();
753
754 /* append <crlf> (we packed a header, not a reply) */
755 mb->append("\r\n", 2);
756}
757
763int64_t
765{
766 int64_t clen = 0;
767 MemBuf mb;
768
769 assert(memObject());
770
771 mb.init();
773
774 while (pos != request->range->end()) {
775 /* account for headers for this range */
776 mb.reset();
777 clientPackRangeHdr(&storeEntry()->mem().freshestReply(),
778 *pos, range_iter.boundary, &mb);
779 clen += mb.size;
780
781 /* account for range content */
782 clen += (*pos)->length;
783
784 debugs(33, 6, "clientMRangeCLen: (clen += " << mb.size << " + " << (*pos)->length << ") == " << clen);
785 ++pos;
786 }
787
788 /* account for the terminating boundary */
789 mb.reset();
790
792
793 clen += mb.size;
794
795 mb.clean();
796
797 return clen;
798}
799
803String
805{
806 const char *key;
808 b.append(":",1);
809 key = storeEntry()->getMD5Text();
810 b.append(key, strlen(key));
811 return b;
812}
813
823void
825 HttpReply * rep, StoreIOBuffer receivedData)
826{
827 // do not try to deliver if client already ABORTED
828 if (!http->getConn() || !cbdataReferenceValid(http->getConn()) || !Comm::IsConnOpen(http->getConn()->clientConnection))
829 return;
830
831 /* Test preconditions */
832 assert(node != nullptr);
833 /* TODO: handle this rather than asserting
834 * - it should only ever happen if we cause an abort and
835 * the callback chain loops back to here, so we can simply return.
836 * However, that itself shouldn't happen, so it stays as an assert for now.
837 */
839 assert(node->node.next == nullptr);
840 Http::StreamPointer context = dynamic_cast<Http::Stream *>(node->data.getRaw());
841 assert(context != nullptr);
842
843 /* TODO: check offset is what we asked for */
844
845 // TODO: enforces HTTP/1 MUST on pipeline order, but is irrelevant to HTTP/2
846 if (context != http->getConn()->pipeline.front())
847 context->deferRecipientForLater(node, rep, receivedData);
848 else if (http->getConn()->cbControlMsgSent) // 1xx to the user is pending
849 context->deferRecipientForLater(node, rep, receivedData);
850 else
851 http->getConn()->handleReply(rep, receivedData);
852}
853
859void
861{
862 /* Test preconditions */
863 assert(node != nullptr);
864 /* TODO: handle this rather than asserting
865 * - it should only ever happen if we cause an abort and
866 * the callback chain loops back to here, so we can simply return.
867 * However, that itself shouldn't happen, so it stays as an assert for now.
868 */
870 /* Set null by ContextFree */
871 assert(node->node.next == nullptr);
872 /* this is the assert discussed above */
873 assert(nullptr == dynamic_cast<Http::Stream *>(node->data.getRaw()));
874 /* We are only called when the client socket shutsdown.
875 * Tell the prev pipeline member we're finished
876 */
878}
879
880void
882{
883 debugs(33, 5, clientConnection << " reading next req");
884
885 fd_note(clientConnection->fd, "Idle client: Waiting for next request");
890
891 readSomeData();
893}
894
895static void
897{
898 debugs(33, 2, conn->clientConnection << " Sending next");
899
902 if (deferredRequest->flags.deferred) {
904 assert(deferredRequest->http->out.size == 0);
906 clientSocketRecipient(deferredRequest->deferredparams.node,
907 deferredRequest->http,
908 deferredRequest->deferredparams.rep,
909 deferredRequest->deferredparams.queuedBuffer);
910 }
911
915}
916
917void
919{
921 debugs(33, 2, clientConnection << " Connection was closed");
922 return;
923 }
924
925 if (pinning.pinned && !Comm::IsConnOpen(pinning.serverConnection)) {
926 debugs(33, 2, clientConnection << " Connection was pinned but server side gone. Terminating client connection");
928 return;
929 }
930
945 if (const char *reason = stoppedReceiving()) {
946 debugs(33, 3, "closing for earlier request error: " << reason);
948 return;
949 }
950
961 if (clientParseRequests()) {
962 debugs(33, 3, clientConnection << ": parsed next request from buffer");
963 }
964
973 debugs(33, 3, "half-closed client with no pending requests, closing");
975 return;
976 }
977
985 Http::StreamPointer deferredRequest = pipeline.front();
986 if (deferredRequest != nullptr) {
987 debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded");
988 ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
989 } else if (flags.readMore) {
990 debugs(33, 3, clientConnection << ": calling readNextRequest()");
992 } else {
993 // XXX: Can this happen? CONNECT tunnels have deferredRequest set.
994 debugs(33, DBG_IMPORTANT, MYNAME << "abandoning " << clientConnection);
995 }
996}
997
998void
1000{
1001 debugs(33, 4, "sending error (" << clientConnection << "): " << error <<
1002 "; old receiving error: " <<
1003 (stoppedReceiving() ? stoppedReceiving_ : "none"));
1004
1005 if (const char *oldError = stoppedSending()) {
1006 debugs(33, 3, "already stopped sending: " << oldError);
1007 return; // nothing has changed as far as this connection is concerned
1008 }
1010
1011 if (!stoppedReceiving()) {
1012 if (const int64_t expecting = mayNeedToReadMoreBody()) {
1013 debugs(33, 5, "must still read " << expecting <<
1014 " request body bytes with " << inBuf.length() << " unused");
1015 return; // wait for the request receiver to finish reading
1016 }
1017 }
1018
1020}
1021
1022void
1024{
1025 if (pipeline.empty())
1026 return;
1027
1028 auto ctx = pipeline.front();
1029 if (size) {
1031 if (ctx->http->loggingTags().isTcpHit())
1033 }
1034 ctx->writeComplete(size);
1035}
1036
1039{
1040 ClientHttpRequest *http = new ClientHttpRequest(this);
1041 http->req_sz = inBuf.length();
1042 http->setErrorUri(uri);
1043 auto *context = new Http::Stream(clientConnection, http);
1044 StoreIOBuffer tempBuffer;
1045 tempBuffer.data = context->reqbuf;
1046 tempBuffer.length = HTTP_REQBUF_SZ;
1049 clientSocketDetach, context, tempBuffer);
1050 return context;
1051}
1052
1053void
1055{
1056 // RegisteredRunner API callback - Squid has been shut down
1057
1058 // if connection is idle terminate it now,
1059 // otherwise wait for grace period to end
1060 if (pipeline.empty())
1062}
1063
1064void
1066{
1067 // RegisteredRunner API callback - Squid shutdown grace period is over
1068
1069 // force the client connection to close immediately
1070 // swanSong() in the close handler will cleanup.
1073}
1074
1075char *
1076skipLeadingSpace(char *aString)
1077{
1078 char *result = aString;
1079
1080 while (xisspace(*aString))
1081 ++aString;
1082
1083 return result;
1084}
1085
1091const char *
1092findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end)
1093{
1094 if (nullptr == end) {
1095 end = uriAndHTTPVersion + strcspn(uriAndHTTPVersion, "\r\n");
1096 assert(end);
1097 }
1098
1099 for (; end > uriAndHTTPVersion; --end) {
1100 if (*end == '\n' || *end == '\r')
1101 continue;
1102
1103 if (xisspace(*end)) {
1104 if (strncasecmp(end + 1, "HTTP/", 5) == 0)
1105 return end + 1;
1106 else
1107 break;
1108 }
1109 }
1110
1111 return nullptr;
1112}
1113
1114static char *
1116{
1117 int vhost = conn->port->vhost;
1118 int vport = conn->port->vport;
1119 static char ipbuf[MAX_IPSTRLEN];
1120
1121 /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1122
1123 static const SBuf cache_object("cache_object://");
1124 if (hp->requestUri().startsWith(cache_object))
1125 return nullptr; /* already in good shape */
1126
1127 // XXX: re-use proper URL parser for this
1128 SBuf url = hp->requestUri(); // use full provided URI if we abort
1129 do { // use a loop so we can break out of it
1131 if (tok.skip('/')) // origin-form URL already.
1132 break;
1133
1134 if (conn->port->vhost)
1135 return nullptr; /* already in good shape */
1136
1137 // skip the URI scheme
1138 static const CharacterSet uriScheme = CharacterSet("URI-scheme","+-.") + CharacterSet::ALPHA + CharacterSet::DIGIT;
1139 static const SBuf uriSchemeEnd("://");
1140 if (!tok.skipAll(uriScheme) || !tok.skip(uriSchemeEnd))
1141 break;
1142
1143 // skip the authority segment
1144 // RFC 3986 complex nested ABNF for "authority" boils down to this:
1145 static const CharacterSet authority = CharacterSet("authority","-._~%:@[]!$&'()*+,;=") +
1147 if (!tok.skipAll(authority))
1148 break;
1149
1150 static const SBuf slashUri("/");
1151 const SBuf t = tok.remaining();
1152 if (t.isEmpty())
1153 url = slashUri;
1154 else if (t[0]=='/') // looks like path
1155 url = t;
1156 else if (t[0]=='?' || t[0]=='#') { // looks like query or fragment. fix '/'
1157 url = slashUri;
1158 url.append(t);
1159 } // else do nothing. invalid path
1160
1161 } while(false);
1162
1163#if SHOULD_REJECT_UNKNOWN_URLS
1164 // reject URI which are not well-formed even after the processing above
1165 if (url.isEmpty() || url[0] != '/') {
1166 hp->parseStatusCode = Http::scBadRequest;
1167 return conn->abortRequestParsing("error:invalid-request");
1168 }
1169#endif
1170
1171 if (vport < 0)
1172 vport = conn->clientConnection->local.port();
1173
1174 char *receivedHost = nullptr;
1175 if (vhost && (receivedHost = hp->getHostHeaderField())) {
1176 SBuf host(receivedHost);
1177 debugs(33, 5, "ACCEL VHOST REWRITE: vhost=" << host << " + vport=" << vport);
1178 if (vport > 0) {
1179 // remove existing :port (if any), cope with IPv6+ without port
1180 const auto lastColonPos = host.rfind(':');
1181 if (lastColonPos != SBuf::npos && *host.rbegin() != ']') {
1182 host.chop(0, lastColonPos); // truncate until the last colon
1183 }
1184 host.appendf(":%d", vport);
1185 } // else nothing to alter port-wise.
1186 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1187 const auto url_sz = scheme.length() + host.length() + url.length() + 32;
1188 char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1189 snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH SQUIDSBUFPH, SQUIDSBUFPRINT(scheme), SQUIDSBUFPRINT(host), SQUIDSBUFPRINT(url));
1190 debugs(33, 5, "ACCEL VHOST REWRITE: " << uri);
1191 return uri;
1192 } else if (conn->port->defaultsite /* && !vhost */) {
1193 debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: defaultsite=" << conn->port->defaultsite << " + vport=" << vport);
1194 char vportStr[32];
1195 vportStr[0] = '\0';
1196 if (vport > 0) {
1197 snprintf(vportStr, sizeof(vportStr),":%d",vport);
1198 }
1199 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1200 const int url_sz = scheme.length() + strlen(conn->port->defaultsite) + sizeof(vportStr) + url.length() + 32;
1201 char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1202 snprintf(uri, url_sz, SQUIDSBUFPH "://%s%s" SQUIDSBUFPH,
1203 SQUIDSBUFPRINT(scheme), conn->port->defaultsite, vportStr, SQUIDSBUFPRINT(url));
1204 debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: " << uri);
1205 return uri;
1206 } else if (vport > 0 /* && (!vhost || no Host:) */) {
1207 debugs(33, 5, "ACCEL VPORT REWRITE: *_port IP + vport=" << vport);
1208 /* Put the local socket IP address as the hostname, with whatever vport we found */
1209 conn->clientConnection->local.toHostStr(ipbuf,MAX_IPSTRLEN);
1210 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1211 const int url_sz = scheme.length() + sizeof(ipbuf) + url.length() + 32;
1212 char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1213 snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1214 SQUIDSBUFPRINT(scheme), ipbuf, vport, SQUIDSBUFPRINT(url));
1215 debugs(33, 5, "ACCEL VPORT REWRITE: " << uri);
1216 return uri;
1217 }
1218
1219 return nullptr;
1220}
1221
1222static char *
1224{
1225 char *uri = nullptr;
1226 /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1227 if (const char *host = hp->getHostHeaderField()) {
1228 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1229 const int url_sz = scheme.length() + strlen(host) + hp->requestUri().length() + 32;
1230 uri = static_cast<char *>(xcalloc(url_sz, 1));
1231 snprintf(uri, url_sz, SQUIDSBUFPH "://%s" SQUIDSBUFPH,
1232 SQUIDSBUFPRINT(scheme),
1233 host,
1234 SQUIDSBUFPRINT(hp->requestUri()));
1235 }
1236 return uri;
1237}
1238
1239char *
1241{
1243
1244 if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1245 return nullptr; /* already in good shape */
1246
1247 char *uri = buildUrlFromHost(this, hp);
1248#if USE_OPENSSL
1249 if (!uri) {
1252 SBuf useHost;
1253 if (!tlsClientSni().isEmpty())
1254 useHost = tlsClientSni();
1255 else
1256 useHost = tlsConnectHostOrIp;
1257
1259 const int url_sz = scheme.length() + useHost.length() + hp->requestUri().length() + 32;
1260 uri = static_cast<char *>(xcalloc(url_sz, 1));
1261 snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH ":%hu" SQUIDSBUFPH,
1262 SQUIDSBUFPRINT(scheme),
1263 SQUIDSBUFPRINT(useHost),
1265 SQUIDSBUFPRINT(hp->requestUri()));
1266 }
1267#endif
1268 if (uri)
1269 debugs(33, 5, "TLS switching host rewrite: " << uri);
1270 return uri;
1271}
1272
1273static char *
1275{
1276 // TODO Must() on URI !empty when the parser supports throw. For now avoid assert().
1277 if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1278 return nullptr; /* already in good shape */
1279
1280 char *uri = buildUrlFromHost(conn, hp);
1281 if (!uri) {
1282 /* Put the local socket IP address as the hostname. */
1283 static char ipbuf[MAX_IPSTRLEN];
1284 conn->clientConnection->local.toHostStr(ipbuf,MAX_IPSTRLEN);
1285 const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1286 const int url_sz = sizeof(ipbuf) + hp->requestUri().length() + 32;
1287 uri = static_cast<char *>(xcalloc(url_sz, 1));
1288 snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1289 SQUIDSBUFPRINT(scheme),
1290 ipbuf, conn->clientConnection->local.port(), SQUIDSBUFPRINT(hp->requestUri()));
1291 }
1292
1293 if (uri)
1294 debugs(33, 5, "TRANSPARENT REWRITE: " << uri);
1295 return uri;
1296}
1297
1300{
1301 /* Attempt to parse the first line; this will define where the method, url, version and header begin */
1302 {
1303 Must(hp);
1304
1307
1308 const bool parsedOk = hp->parse(inBuf);
1309
1310 // sync the buffers after parsing.
1311 inBuf = hp->remaining();
1312
1313 if (hp->needsMoreData()) {
1314 debugs(33, 5, "Incomplete request, waiting for end of request line");
1315 return nullptr;
1316 }
1317
1318 if (!parsedOk) {
1319 const bool tooBig =
1320 hp->parseStatusCode == Http::scRequestHeaderFieldsTooLarge ||
1321 hp->parseStatusCode == Http::scUriTooLong;
1322 auto result = abortRequestParsing(
1323 tooBig ? "error:request-too-large" : "error:invalid-request");
1324 // assume that remaining leftovers belong to this bad request
1325 if (!inBuf.isEmpty())
1327 return result;
1328 }
1329 }
1330
1331 /* We know the whole request is in parser now */
1332 debugs(11, 2, "HTTP Client " << clientConnection);
1333 debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
1334 hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol() << "\n" <<
1335 hp->mimeHeader() <<
1336 "\n----------");
1337
1338 /* deny CONNECT via accelerated ports */
1339 if (hp->method() == Http::METHOD_CONNECT && port != nullptr && port->flags.accelSurrogate) {
1340 debugs(33, DBG_IMPORTANT, "WARNING: CONNECT method received on " << transferProtocol << " Accelerator port " << port->s.port());
1341 debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1342 hp->parseStatusCode = Http::scMethodNotAllowed;
1343 return abortRequestParsing("error:method-not-allowed");
1344 }
1345
1346 /* HTTP/2 connection magic prefix starts with "PRI ".
1347 * Deny "PRI" method if used in HTTP/1.x or 0.9 versions.
1348 * If seen it signals a broken client or proxy has corrupted the traffic.
1349 */
1350 if (hp->method() == Http::METHOD_PRI && hp->messageProtocol() < Http::ProtocolVersion(2,0)) {
1351 debugs(33, DBG_IMPORTANT, "WARNING: PRI method received on " << transferProtocol << " port " << port->s.port());
1352 debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1353 hp->parseStatusCode = Http::scMethodNotAllowed;
1354 return abortRequestParsing("error:method-not-allowed");
1355 }
1356
1357 if (hp->method() == Http::METHOD_NONE) {
1358 debugs(33, DBG_IMPORTANT, "WARNING: Unsupported method: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1359 hp->parseStatusCode = Http::scMethodNotAllowed;
1360 return abortRequestParsing("error:unsupported-request-method");
1361 }
1362
1363 // Process headers after request line
1364 debugs(33, 3, "complete request received. " <<
1365 "prefix_sz = " << hp->messageHeaderSize() <<
1366 ", request-line-size=" << hp->firstLineSize() <<
1367 ", mime-header-size=" << hp->headerBlockSize() <<
1368 ", mime header block:\n" << hp->mimeHeader() << "\n----------");
1369
1370 /* Ok, all headers are received */
1371 ClientHttpRequest *http = new ClientHttpRequest(this);
1372
1373 http->req_sz = hp->messageHeaderSize();
1374 Http::Stream *result = new Http::Stream(clientConnection, http);
1375
1376 StoreIOBuffer tempBuffer;
1377 tempBuffer.data = result->reqbuf;
1378 tempBuffer.length = HTTP_REQBUF_SZ;
1379
1380 ClientStreamData newServer = new clientReplyContext(http);
1381 ClientStreamData newClient = result;
1384 clientSocketDetach, newClient, tempBuffer);
1385
1386 /* set url */
1387 debugs(33,5, "Prepare absolute URL from " <<
1388 (transparent()?"intercept":(port->flags.accelSurrogate ? "accel":"")));
1389 /* Rewrite the URL in transparent or accelerator mode */
1390 /* NP: there are several cases to traverse here:
1391 * - standard mode (forward proxy)
1392 * - transparent mode (TPROXY)
1393 * - transparent mode with failures
1394 * - intercept mode (NAT)
1395 * - intercept mode with failures
1396 * - accelerator mode (reverse proxy)
1397 * - internal relative-URL
1398 * - mixed combos of the above with internal URL
1399 * - remote interception with PROXY protocol
1400 * - remote reverse-proxy with PROXY protocol
1401 */
1402 if (switchedToHttps()) {
1403 http->uri = prepareTlsSwitchingURL(hp);
1404 } else if (transparent()) {
1405 /* intercept or transparent mode, properly working with no failures */
1406 http->uri = prepareTransparentURL(this, hp);
1407
1408 } else if (internalCheck(hp->requestUri())) { // NP: only matches relative-URI
1409 /* internal URL mode */
1410 /* prepend our name & port */
1411 http->uri = xstrdup(internalLocalUri(nullptr, hp->requestUri()));
1412 // We just re-wrote the URL. Must replace the Host: header.
1413 // But have not parsed there yet!! flag for local-only handling.
1414 http->flags.internal = true;
1415
1416 } else if (port->flags.accelSurrogate) {
1417 /* accelerator mode */
1418 http->uri = prepareAcceleratedURL(this, hp);
1419 http->flags.accel = true;
1420 }
1421
1422 if (!http->uri) {
1423 /* No special rewrites have been applied above, use the
1424 * requested url. may be rewritten later, so make extra room */
1425 int url_sz = hp->requestUri().length() + Config.appendDomainLen + 5;
1426 http->uri = (char *)xcalloc(url_sz, 1);
1427 SBufToCstring(http->uri, hp->requestUri());
1428 }
1429
1430 result->flags.parsed_ok = 1;
1431 return result;
1432}
1433
1434bool
1436{
1437 if (pipeline.empty() && inBuf.isEmpty()) {
1438 debugs(33, 4, "yes, without active requests and unparsed input");
1439 return true;
1440 }
1441
1443 debugs(33, 3, "yes, without half_closed_clients");
1444 return true;
1445 }
1446
1447 // Squid currently tries to parse (possibly again) a partially received
1448 // request after an EOF with half_closed_clients. To give that last parse in
1449 // afterClientRead() a chance, we ignore partially parsed requests here.
1450 debugs(33, 3, "no, honoring half_closed_clients");
1451 return false;
1452}
1453
1454void
1455ConnStateData::consumeInput(const size_t byteCount)
1456{
1457 assert(byteCount > 0 && byteCount <= inBuf.length());
1458 inBuf.consume(byteCount);
1459 debugs(33, 5, "inBuf has " << inBuf.length() << " unused bytes");
1460}
1461
1462void
1464{
1465 // Were we expecting to read more request body from half-closed connection?
1467 debugs(33, 3, "truncated body: closing half-closed " << clientConnection);
1469 return;
1470 }
1471
1472 if (flags.readMore)
1473 readSomeData();
1474}
1475
1476void
1478{
1479 // From HTTP p.o.v., we do not have to close after every error detected
1480 // at the client-side, but many such errors do require closure and the
1481 // client-side code is bad at handling errors so we play it safe.
1482 if (request)
1483 request->flags.proxyKeepalive = false;
1484 flags.readMore = false;
1485 debugs(33,4, "Will close after error: " << clientConnection);
1486}
1487
1488#if USE_OPENSSL
1490{
1491 ClientHttpRequest *http = context->http;
1492
1493 if (!sslServerBump)
1494 return false;
1495
1497 // Did we create an error entry while processing CONNECT?
1498 if (!sslServerBump->entry->isEmpty()) {
1499 quitAfterError(http->request);
1500
1501 // Get the saved error entry and send it to the client by replacing the
1502 // ClientHttpRequest store entry with it.
1504 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1505 assert(repContext);
1506 debugs(33, 5, "Responding with delated error for " << http->uri);
1507 repContext->setReplyToStoreEntry(sslServerBump->entry, "delayed SslBump error");
1508
1509 // Get error details from the fake certificate-peeking request.
1511 context->pullData();
1512 return true;
1513 }
1514
1515 // In bump-server-first mode, we have not necessarily seen the intended
1516 // server name at certificate-peeking time. Check for domain mismatch now,
1517 // when we can extract the intended name from the bumped HTTP request.
1518 if (const Security::CertPointer &srvCert = sslServerBump->serverCert) {
1519 HttpRequest *request = http->request;
1520 if (!Ssl::checkX509ServerValidity(srvCert.get(), request->url.host())) {
1521 debugs(33, 2, "SQUID_X509_V_ERR_DOMAIN_MISMATCH: Certificate " <<
1522 "does not match domainname " << request->url.host());
1523
1524 bool allowDomainMismatch = false;
1528 clientAclChecklistFill(check, http);
1529 allowDomainMismatch = check.fastCheck().allowed();
1530 delete check.sslErrors;
1531 check.sslErrors = nullptr;
1532 }
1533
1534 if (!allowDomainMismatch) {
1535 quitAfterError(request);
1536
1538 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1539 assert (repContext);
1540
1541 request->hier = sslServerBump->request->hier;
1542
1543 // Create an error object and fill it
1544 const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request, http->al);
1545 err->src_addr = clientConnection->remote;
1548 srvCert, nullptr);
1550 repContext->setReplyToError(request->method, err);
1551 assert(context->http->out.offset == 0);
1552 context->pullData();
1553 return true;
1554 }
1555 }
1556 }
1557
1558 return false;
1559}
1560#endif // USE_OPENSSL
1561
1563bool
1565{
1567 debugs(33, 5, "disabled; send error: " << requestError);
1568 return false;
1569 }
1570
1571 if (!preservingClientData_) {
1572 debugs(33, 3, "may have forgotten client data; send error: " << requestError);
1573 return false;
1574 }
1575
1577 checklist.requestErrorType = requestError;
1578 fillChecklist(checklist);
1579 auto answer = checklist.fastCheck();
1580 if (answer.allowed() && answer.kind == 1) {
1581 debugs(33, 3, "Request will be tunneled to server");
1582 const auto context = pipeline.front();
1583 const auto http = context ? context->http : nullptr;
1584 const auto request = http ? http->request : nullptr;
1585 if (context)
1586 context->finished(); // Will remove from pipeline queue
1587 Comm::SetSelect(clientConnection->fd, COMM_SELECT_READ, nullptr, nullptr, 0);
1588 return initiateTunneledRequest(request, "unknown-protocol", preservedClientData);
1589 }
1590 debugs(33, 3, "denied; send error: " << requestError);
1591 return false;
1592}
1593
1594void
1596{
1597 /*
1598 * DPW 2007-05-18
1599 * Moved the TCP_RESET feature from clientReplyContext::sendMoreData
1600 * to here because calling comm_reset_close() causes http to
1601 * be freed before accessing.
1602 */
1603 if (request != nullptr && request->flags.resetTcp && Comm::IsConnOpen(conn->clientConnection)) {
1604 debugs(33, 3, "Sending TCP RST on " << conn->clientConnection);
1605 conn->flags.readMore = false;
1606 comm_reset_close(conn->clientConnection);
1607 }
1608}
1609
1610void
1612{
1613 ClientHttpRequest *http = context->http;
1614 bool mustReplyToOptions = false;
1615 bool expectBody = false;
1616
1617 // We already have the request parsed and checked, so we
1618 // only need to go through the final body/conn setup to doCallouts().
1619 assert(http->request);
1620 HttpRequest::Pointer request = http->request;
1621
1622 // temporary hack to avoid splitting this huge function with sensitive code
1623 const bool isFtp = !hp;
1624
1625 // Some blobs below are still HTTP-specific, but we would have to rewrite
1626 // this entire function to remove them from the FTP code path. Connection
1627 // setup and body_pipe preparation blobs are needed for FTP.
1628
1629 request->manager(conn, http->al);
1630
1631 request->flags.accelerated = http->flags.accel;
1632 request->flags.sslBumped=conn->switchedToHttps();
1633 // TODO: decouple http->flags.accel from request->flags.sslBumped
1634 request->flags.noDirect = (request->flags.accelerated && !request->flags.sslBumped) ?
1635 !conn->port->allow_direct : 0;
1636 request->sources |= isFtp ? Http::Message::srcFtp :
1637 ((request->flags.sslBumped || conn->port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
1638#if USE_AUTH
1639 if (request->flags.sslBumped) {
1640 if (conn->getAuth() != nullptr)
1641 request->auth_user_request = conn->getAuth();
1642 }
1643#endif
1644
1645 if (internalCheck(request->url.path())) {
1646 if (internalHostnameIs(request->url.host()) && request->url.port() == getMyPort()) {
1647 debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true));
1648 http->flags.internal = true;
1649 } else if (Config.onoff.global_internal_static && internalStaticCheck(request->url.path())) {
1650 debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true) << " (global_internal_static on)");
1651 request->url.setScheme(AnyP::PROTO_HTTP, "http");
1652 request->url.host(internalHostname());
1653 request->url.port(getMyPort());
1654 http->flags.internal = true;
1655 http->setLogUriToRequestUri();
1656 } else
1657 debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true) << " (not this proxy)");
1658 }
1659
1660 request->flags.internal = http->flags.internal;
1661
1662 if (!isFtp) {
1663 // XXX: for non-HTTP messages instantiate a different Http::Message child type
1664 // for now Squid only supports HTTP requests
1665 const AnyP::ProtocolVersion &http_ver = hp->messageProtocol();
1666 assert(request->http_ver.protocol == http_ver.protocol);
1667 request->http_ver.major = http_ver.major;
1668 request->http_ver.minor = http_ver.minor;
1669 }
1670
1671 mustReplyToOptions = (request->method == Http::METHOD_OPTIONS) &&
1673 if (!urlCheckRequest(request.getRaw()) || mustReplyToOptions) {
1675 conn->quitAfterError(request.getRaw());
1676 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1677 assert (repContext);
1679 conn, request.getRaw(), nullptr, nullptr);
1680 assert(context->http->out.offset == 0);
1681 context->pullData();
1683 return;
1684 }
1685
1686 const auto frameStatus = request->checkEntityFraming();
1687 if (frameStatus != Http::scNone) {
1689 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1690 assert (repContext);
1691 conn->quitAfterError(request.getRaw());
1692 repContext->setReplyToError(ERR_INVALID_REQ, frameStatus, nullptr, conn, request.getRaw(), nullptr, nullptr);
1693 assert(context->http->out.offset == 0);
1694 context->pullData();
1696 return;
1697 }
1698
1700 // Let tunneling code be fully responsible for CONNECT requests
1701 if (http->request->method == Http::METHOD_CONNECT) {
1702 context->mayUseConnection(true);
1703 conn->flags.readMore = false;
1704 }
1705
1706#if USE_OPENSSL
1707 if (conn->switchedToHttps() && conn->serveDelayedError(context)) {
1709 return;
1710 }
1711#endif
1712
1713 /* Do we expect a request-body? */
1714 const auto chunked = request->header.chunked();
1715 expectBody = chunked || request->content_length > 0;
1716 if (!context->mayUseConnection() && expectBody) {
1717 request->body_pipe = conn->expectRequestBody(
1718 chunked ? -1 : request->content_length);
1719
1720 /* Is it too large? */
1721 if (!chunked && // if chunked, we will check as we accumulate
1724 clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1725 assert (repContext);
1726 conn->quitAfterError(request.getRaw());
1727 repContext->setReplyToError(ERR_TOO_BIG,
1728 Http::scContentTooLarge, nullptr,
1729 conn, http->request, nullptr, nullptr);
1730 assert(context->http->out.offset == 0);
1731 context->pullData();
1733 return;
1734 }
1735
1736 if (!isFtp) {
1737 // We may stop producing, comm_close, and/or call setReplyToError()
1738 // below, so quit on errors to avoid http->doCallouts()
1739 if (!conn->handleRequestBodyData()) {
1741 return;
1742 }
1743
1744 if (!request->body_pipe->productionEnded()) {
1745 debugs(33, 5, "need more request body");
1746 context->mayUseConnection(true);
1747 assert(conn->flags.readMore);
1748 }
1749 }
1750 }
1751
1752 http->calloutContext = new ClientRequestContext(http);
1753
1754 http->doCallouts();
1755
1757}
1758
1759void
1761{
1762 debugs(33, 3, context << " to " << pipeline.count() << '/' << pipeline.nrequests);
1763 if (bareError) {
1764 debugs(33, 5, "assigning " << bareError);
1765 assert(context);
1766 assert(context->http);
1767 context->http->updateError(bareError);
1768 bareError.clear();
1769 }
1770 pipeline.add(context);
1771}
1772
1773int
1775{
1776 // TODO: Support pipelined requests through pinned connections.
1777 if (pinning.pinned)
1778 return 0;
1780}
1781
1787bool
1789{
1790 const int existingRequestCount = pipeline.count();
1791
1792 // default to the configured pipeline size.
1793 // add 1 because the head of pipeline is counted in concurrent requests and not prefetch queue
1794#if USE_OPENSSL
1795 const int internalRequest = (transparent() && sslBumpMode == Ssl::bumpSplice) ? 1 : 0;
1796#else
1797 const int internalRequest = 0;
1798#endif
1799 const int concurrentRequestLimit = pipelinePrefetchMax() + 1 + internalRequest;
1800
1801 // when queue filled already we can't add more.
1802 if (existingRequestCount >= concurrentRequestLimit) {
1803 debugs(33, 3, clientConnection << " max concurrent requests reached (" << concurrentRequestLimit << ")");
1804 debugs(33, 5, clientConnection << " deferring new request until one is done");
1805 return true;
1806 }
1807
1808 return false;
1809}
1810
1816bool
1818{
1820 return proxyProtocolError("PROXY client not permitted by default ACL");
1821
1823 fillChecklist(ch);
1824 if (!ch.fastCheck().allowed())
1825 return proxyProtocolError("PROXY client not permitted by ACLs");
1826
1827 return true;
1828}
1829
1835bool
1837{
1838 if (msg) {
1839 // This is important to know, but maybe not so much that flooding the log is okay.
1840#if QUIET_PROXY_PROTOCOL
1841 // display the first of every 32 occurrences at level 1, the others at level 2.
1842 static uint8_t hide = 0;
1843 debugs(33, (hide++ % 32 == 0 ? DBG_IMPORTANT : 2), msg << " from " << clientConnection);
1844#else
1845 debugs(33, DBG_IMPORTANT, msg << " from " << clientConnection);
1846#endif
1847 mustStop(msg);
1848 }
1849 return false;
1850}
1851
1856bool
1858{
1859 try {
1860 const auto parsed = ProxyProtocol::Parse(inBuf);
1861 proxyProtocolHeader_ = parsed.header;
1863 inBuf.consume(parsed.size);
1865 if (proxyProtocolHeader_->hasForwardedAddresses()) {
1866 clientConnection->local = proxyProtocolHeader_->destinationAddress;
1869 clientConnection->flags ^= COMM_TRANSPARENT; // prevent TPROXY spoofing of this new IP.
1870 debugs(33, 5, "PROXY/" << proxyProtocolHeader_->version() << " upgrade: " << clientConnection);
1871 }
1873 debugs(33, 3, "PROXY protocol: waiting for more than " << inBuf.length() << " bytes");
1874 return false;
1875 } catch (const std::exception &e) {
1876 return proxyProtocolError(e.what());
1877 }
1878 return true;
1879}
1880
1881void
1883{
1885 return;
1886
1887 receivedFirstByte_ = true;
1889}
1890
1896bool
1898{
1899 bool parsed_req = false;
1900
1901 debugs(33, 5, clientConnection << ": attempting to parse");
1902
1903 // Loop while we have read bytes that are not needed for producing the body
1904 // On errors, bodyPipe may become nil, but readMore will be cleared
1905 while (!inBuf.isEmpty() && !bodyPipe && flags.readMore) {
1906
1907 // Prohibit concurrent requests when using a pinned to-server connection
1908 // because our Client classes do not support request pipelining.
1909 if (pinning.pinned && !pinning.readHandler) {
1910 debugs(33, 3, clientConnection << " waits for busy " << pinning.serverConnection);
1911 break;
1912 }
1913
1914 /* Limit the number of concurrent requests */
1916 break;
1917
1918 // try to parse the PROXY protocol header magic bytes
1921 break;
1922
1923 // we have been waiting for PROXY to provide client-IP
1924 // for some lookups, ie rDNS and IDENT.
1926
1927 // Done with PROXY protocol which has cleared preservingClientData_.
1928 // If the next protocol supports on_unsupported_protocol, then its
1929 // parseOneRequest() must reset preservingClientData_.
1931 }
1932
1933 if (Http::StreamPointer context = parseOneRequest()) {
1934 debugs(33, 5, clientConnection << ": done parsing a request");
1936 context->registerWithConn();
1937
1938#if USE_OPENSSL
1939 if (switchedToHttps())
1941#endif
1942
1943 processParsedRequest(context);
1944
1945 parsed_req = true; // XXX: do we really need to parse everything right NOW ?
1946
1947 if (context->mayUseConnection()) {
1948 debugs(33, 3, "Not parsing new requests, as this request may need the connection");
1949 break;
1950 }
1951 } else {
1952 debugs(33, 5, clientConnection << ": not enough request data: " <<
1953 inBuf.length() << " < " << Config.maxRequestHeaderSize);
1955 break;
1956 }
1957 }
1958
1959 /* XXX where to 'finish' the parsing pass? */
1960 return parsed_req;
1961}
1962
1963void
1965{
1966#if USE_OPENSSL
1967 if (parsingTlsHandshake) {
1969 return;
1970 }
1971#endif
1972
1973 /* Process next request */
1974 if (pipeline.empty())
1975 fd_note(clientConnection->fd, "Reading next request");
1976
1977 if (!clientParseRequests()) {
1978 if (!isOpen())
1979 return;
1980 // We may get here if the client half-closed after sending a partial
1981 // request. See doClientRead() and shouldCloseOnEof().
1982 // XXX: This partially duplicates ConnStateData::kick().
1984 debugs(33, 5, clientConnection << ": half-closed connection, no completed request parsed, connection closing.");
1986 return;
1987 }
1988 }
1989
1990 if (!isOpen())
1991 return;
1992
1994}
1995
2002bool
2004{
2005 // if we are reading a body, stuff data into the body pipe
2006 if (bodyPipe != nullptr)
2007 return handleRequestBodyData();
2008 return true;
2009}
2010
2018bool
2020{
2021 assert(bodyPipe != nullptr);
2022
2023 if (bodyParser) { // chunked encoding
2024 if (const err_type error = handleChunkedRequestBody()) {
2026 return false;
2027 }
2028 } else { // identity encoding
2029 debugs(33,5, "handling plain request body for " << clientConnection);
2030 const size_t putSize = bodyPipe->putMoreData(inBuf.c_str(), inBuf.length());
2031 if (putSize > 0)
2032 consumeInput(putSize);
2033
2034 if (!bodyPipe->mayNeedMoreData()) {
2035 // BodyPipe will clear us automagically when we produced everything
2036 bodyPipe = nullptr;
2037 }
2038 }
2039
2040 if (!bodyPipe) {
2041 debugs(33,5, "produced entire request body for " << clientConnection);
2042
2043 if (const char *reason = stoppedSending()) {
2044 /* we've finished reading like good clients,
2045 * now do the close that initiateClose initiated.
2046 */
2047 debugs(33, 3, "closing for earlier sending error: " << reason);
2049 return false;
2050 }
2051 }
2052
2053 return true;
2054}
2055
2059{
2060 debugs(33, 7, "chunked from " << clientConnection << ": " << inBuf.length());
2061
2062 try { // the parser will throw on errors
2063
2064 if (inBuf.isEmpty()) // nothing to do
2065 return ERR_NONE;
2066
2069 const bool parsed = bodyParser->parse(inBuf);
2070 inBuf = bodyParser->remaining(); // sync buffers
2071 bpc.checkIn();
2072
2073 // dechunk then check: the size limit applies to _dechunked_ content
2075 return ERR_TOO_BIG;
2076
2077 if (parsed) {
2079 Must(!bodyPipe);
2080 return ERR_NONE; // nil bodyPipe implies body end for the caller
2081 }
2082
2083 // if chunk parser needs data, then the body pipe must need it too
2085
2086 // if parser needs more space and we can consume nothing, we will stall
2088 } catch (...) { // TODO: be more specific
2089 debugs(33, 3, "malformed chunks" << bodyPipe->status());
2090 return ERR_INVALID_REQ;
2091 }
2092
2093 debugs(33, 7, "need more chunked data" << *bodyPipe->status());
2094 return ERR_NONE;
2095}
2096
2098void
2100{
2102
2103 // XXX: The code below works if we fail during initial request parsing,
2104 // but if we fail when the server connection is used already, the server may send
2105 // us its response too, causing various assertions. How to prevent that?
2106#if WE_KNOW_HOW_TO_SEND_ERRORS
2108 if (context != NULL && !context->http->out.offset) { // output nothing yet
2109 clientStreamNode *node = context->getClientReplyContext();
2110 clientReplyContext *repContext = dynamic_cast<clientReplyContext*>(node->data.getRaw());
2111 assert(repContext);
2112 const Http::StatusCode scode = (error == ERR_TOO_BIG) ?
2113 Http::scContentTooLarge : HTTP_BAD_REQUEST;
2114 repContext->setReplyToError(error, scode,
2115 repContext->http->uri,
2116 CachePeer,
2117 repContext->http->request,
2118 inBuf, NULL);
2119 context->pullData();
2120 } else {
2121 // close or otherwise we may get stuck as nobody will notice the error?
2123 }
2124#else
2125 debugs(33, 3, "aborting chunked request without error " << error);
2127#endif
2128 flags.readMore = false;
2129}
2130
2131void
2133{
2134 // request reader may get stuck waiting for space if nobody consumes body
2135 if (bodyPipe != nullptr)
2137
2138 // kids extend
2139}
2140
2142void
2144{
2145 if (!Comm::IsConnOpen(io.conn))
2146 return;
2147
2150 if (tunnelOnError(error))
2151 return;
2152
2153 /*
2154 * Just close the connection to not confuse browsers
2155 * using persistent connections. Some browsers open
2156 * a connection and then do not use it until much
2157 * later (presumeably because the request triggering
2158 * the open has already been completed on another
2159 * connection)
2160 */
2161 debugs(33, 3, "requestTimeout: FD " << io.fd << ": lifetime is expired.");
2162 io.conn->close();
2163}
2164
2165void
2167{
2168 debugs(33, DBG_IMPORTANT, "WARNING: Closing client connection due to lifetime timeout" <<
2169 Debug::Extra << "connection: " << io.conn);
2170
2171 LogTagsErrors lte;
2172 lte.timedout = true;
2174}
2175
2177 AsyncJob("ConnStateData"), // kids overwrite
2178 Server(xact)
2179#if USE_OPENSSL
2180 , tlsParser(Security::HandshakeParser::fromClient)
2181#endif
2182{
2183 // store the details required for creating more MasterXaction objects as new requests come in
2184 log_addr = xact->tcpClient->remote;
2186
2187 // register to receive notice of Squid signal events
2188 // which may affect long persisting client connections
2190}
2191
2192void
2194{
2197
2198 if (port->disable_pmtu_discovery != DISABLE_PMTU_OFF &&
2199 (transparent() || port->disable_pmtu_discovery == DISABLE_PMTU_ALWAYS)) {
2200#if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
2201 int i = IP_PMTUDISC_DONT;
2202 if (setsockopt(clientConnection->fd, SOL_IP, IP_MTU_DISCOVER, &i, sizeof(i)) < 0) {
2203 int xerrno = errno;
2204 debugs(33, 2, "WARNING: Path MTU discovery disabling failed on " << clientConnection << " : " << xstrerr(xerrno));
2205 }
2206#else
2207 static bool reported = false;
2208
2209 if (!reported) {
2210 debugs(33, DBG_IMPORTANT, "WARNING: Path MTU discovery disabling is not supported on your platform.");
2211 reported = true;
2212 }
2213#endif
2214 }
2215
2219
2220 needProxyProtocolHeader_ = port->flags.proxySurrogate;
2222 if (!proxyProtocolValidateClient()) // will close the connection on failure
2223 return;
2224 } else
2226
2227 // requires needProxyProtocolHeader_ which is initialized above
2229}
2230
2231void
2233{
2237
2238#if USE_IDENT
2239 if (Ident::TheConfig.identLookup) {
2240 ACLFilledChecklist identChecklist(Ident::TheConfig.identLookup, nullptr, nullptr);
2241 fillChecklist(identChecklist);
2242 if (identChecklist.fastCheck().allowed())
2244 }
2245#endif
2246
2248
2249#if USE_DELAY_POOLS
2250 fd_table[clientConnection->fd].clientInfo = nullptr;
2251
2252 if (!Config.onoff.client_db)
2253 return; // client delay pools require client_db
2254
2255 const auto &pools = ClientDelayPools::Instance()->pools;
2256 if (pools.size()) {
2257 ACLFilledChecklist ch(nullptr, nullptr, nullptr);
2258 fillChecklist(ch);
2259 // TODO: we check early to limit error response bandwidth but we
2260 // should recheck when we can honor delay_pool_uses_indirect
2261 for (unsigned int pool = 0; pool < pools.size(); ++pool) {
2262
2263 /* pools require explicit 'allow' to assign a client into them */
2264 if (pools[pool]->access) {
2265 ch.changeAcl(pools[pool]->access);
2266 auto answer = ch.fastCheck();
2267 if (answer.allowed()) {
2268
2269 /* request client information from db after we did all checks
2270 this will save hash lookup if client failed checks */
2272 assert(cli);
2273
2274 /* put client info in FDE */
2275 fd_table[clientConnection->fd].clientInfo = cli;
2276
2277 /* setup write limiter for this request */
2278 const double burst = floor(0.5 +
2279 (pools[pool]->highwatermark * Config.ClientDelay.initial)/100.0);
2280 cli->setWriteLimiter(pools[pool]->rate, burst, pools[pool]->highwatermark);
2281 break;
2282 } else {
2283 debugs(83, 4, "Delay pool " << pool << " skipped because ACL " << answer);
2284 }
2285 }
2286 }
2287 }
2288#endif
2289
2290 // kids must extend to actually start doing something (e.g., reading)
2291}
2292
2295{
2296 const auto handshakeResult = Security::Accept(*clientConnection);
2297
2298#if USE_OPENSSL
2299 // log ASAP, even if the handshake has not completed (or failed)
2300 const auto fd = clientConnection->fd;
2301 assert(fd >= 0);
2302 keyLogger.checkpoint(*fd_table[fd].ssl, *this);
2303#else
2304 // TODO: Support fd_table[fd].ssl dereference in other builds.
2305#endif
2306
2307 return handshakeResult;
2308}
2309
2311void
2313{
2314 Assure(params.port);
2315
2316 // NP: it is possible the port was reconfigured when the call or accept() was queued.
2317
2318 if (params.flag != Comm::OK) {
2319 // Its possible the call was still queued when the client disconnected
2320 debugs(33, 2, params.port->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2321 return;
2322 }
2323
2324 debugs(33, 4, params.conn << ": accepted");
2325 fd_note(params.conn->fd, "client http connect");
2326 const auto xact = MasterXaction::MakePortful(params.port);
2327 xact->tcpClient = params.conn;
2328
2329 // Socket is ready, setup the connection manager to start using it
2330 auto *srv = Http::NewServer(xact);
2331 // XXX: do not abandon the MasterXaction object
2332 AsyncJob::Start(srv); // usually async-calls readSomeData()
2333}
2334
2336static bool
2338{
2339 const auto conn = connState->clientConnection;
2340 if (Security::CreateServerSession(ctx, conn, connState->port->secure, "client https start")) {
2341 debugs(33, 5, "will negotiate TLS on " << conn);
2342 return true;
2343 }
2344
2345 debugs(33, DBG_IMPORTANT, "ERROR: could not create TLS server context for " << conn);
2346 conn->close();
2347 return false;
2348}
2349
2351static void
2352clientNegotiateSSL(int fd, void *data)
2353{
2354 ConnStateData *conn = (ConnStateData *)data;
2355
2356 const auto handshakeResult = conn->acceptTls();
2357 switch (handshakeResult.category) {
2359 break;
2360
2362 Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_READ, clientNegotiateSSL, conn, 0);
2363 return;
2364
2366 Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_WRITE, clientNegotiateSSL, conn, 0);
2367 return;
2368
2370 debugs(83, (handshakeResult.important ? Important(62) : 2), "ERROR: " << handshakeResult.errorDescription <<
2371 " while accepting a TLS connection on " << conn->clientConnection << ": " << handshakeResult.errorDetail);
2372 // TODO: No ConnStateData::tunnelOnError() on this forward-proxy code
2373 // path because we cannot know the intended connection target?
2374 conn->updateError(ERR_SECURE_ACCEPT_FAIL, handshakeResult.errorDetail);
2375 conn->clientConnection->close();
2376 return;
2377 }
2378
2379 Security::SessionPointer session(fd_table[fd].ssl);
2380
2381#if USE_OPENSSL
2382 if (Security::SessionIsResumed(session)) {
2383 debugs(83, 2, "Session " << SSL_get_session(session.get()) <<
2384 " reused on FD " << fd << " (" << fd_table[fd].ipaddr <<
2385 ":" << (int)fd_table[fd].remote_port << ")");
2386 } else {
2387 if (Debug::Enabled(83, 4)) {
2388 /* Write out the SSL session details.. actually the call below, but
2389 * OpenSSL headers do strange typecasts confusing GCC.. */
2390 /* PEM_write_SSL_SESSION(debug_log, SSL_get_session(ssl)); */
2391#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x00908000L
2392 PEM_ASN1_write(reinterpret_cast<i2d_of_void *>(i2d_SSL_SESSION),
2393 PEM_STRING_SSL_SESSION, debug_log,
2394 reinterpret_cast<char *>(SSL_get_session(session.get())),
2395 nullptr, nullptr, 0, nullptr, nullptr);
2396
2397#elif (ALLOW_ALWAYS_SSL_SESSION_DETAIL == 1)
2398
2399 /* When using gcc 3.3.x and OpenSSL 0.9.7x sometimes a compile error can occur here.
2400 * This is caused by an unpredicatble gcc behaviour on a cast of the first argument
2401 * of PEM_ASN1_write(). For this reason this code section is disabled. To enable it,
2402 * define ALLOW_ALWAYS_SSL_SESSION_DETAIL=1.
2403 * Because there are two possible usable cast, if you get an error here, try the other
2404 * commented line. */
2405
2406 PEM_ASN1_write((int(*)())i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2407 debug_log,
2408 reinterpret_cast<char *>(SSL_get_session(session.get())),
2409 nullptr, nullptr, 0, nullptr, nullptr);
2410 /* PEM_ASN1_write((int(*)(...))i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2411 debug_log,
2412 reinterpret_cast<char *>(SSL_get_session(session.get())),
2413 nullptr, nullptr, 0, nullptr, nullptr);
2414 */
2415#else
2416 debugs(83, 4, "With " OPENSSL_VERSION_TEXT ", session details are available only defining ALLOW_ALWAYS_SSL_SESSION_DETAIL=1 in the source.");
2417
2418#endif
2419 /* Note: This does not automatically fflush the log file.. */
2420 }
2421
2422 debugs(83, 2, "New session " << SSL_get_session(session.get()) <<
2423 " on FD " << fd << " (" << fd_table[fd].ipaddr << ":" <<
2424 fd_table[fd].remote_port << ")");
2425 }
2426#else
2427 debugs(83, 2, "TLS session reuse not yet implemented.");
2428#endif
2429
2430 // Connection established. Retrieve TLS connection parameters for logging.
2431 conn->clientConnection->tlsNegotiations()->retrieveNegotiatedInfo(session);
2432
2433#if USE_OPENSSL
2434 X509 *client_cert = SSL_get_peer_certificate(session.get());
2435
2436 if (client_cert) {
2437 debugs(83, 3, "FD " << fd << " client certificate: subject: " <<
2438 Security::SubjectName(*client_cert));
2439
2440 debugs(83, 3, "FD " << fd << " client certificate: issuer: " <<
2441 Security::IssuerName(*client_cert));
2442
2443 X509_free(client_cert);
2444 } else {
2445 debugs(83, 5, "FD " << fd << " has no client certificate.");
2446 }
2447#else
2448 debugs(83, 2, "Client certificate requesting not yet implemented.");
2449#endif
2450
2451 // If we are called, then bumped CONNECT has succeeded. Finalize it.
2452 if (auto xact = conn->pipeline.front()) {
2453 if (xact->http && xact->http->request && xact->http->request->method == Http::METHOD_CONNECT)
2454 xact->finished();
2455 // cannot proceed with encryption if requests wait for plain responses
2456 Must(conn->pipeline.empty());
2457 }
2458 /* careful: finished() above frees request, host, etc. */
2459
2460 conn->readSomeData();
2461}
2462
2467static void
2469{
2470 assert(connState);
2471 const Comm::ConnectionPointer &details = connState->clientConnection;
2472
2473 if (!ctx || !httpsCreate(connState, ctx))
2474 return;
2475
2477
2478 Comm::SetSelect(details->fd, COMM_SELECT_READ, clientNegotiateSSL, connState, 0);
2479}
2480
2481#if USE_OPENSSL
2485static void
2487{
2488 ConnStateData *connState = (ConnStateData *) data;
2489
2490 // if the connection is closed or closing, just return.
2491 if (!connState->isOpen())
2492 return;
2493
2494 if (answer.allowed()) {
2495 debugs(33, 2, "sslBump action " << Ssl::bumpMode(answer.kind) << "needed for " << connState->clientConnection);
2496 connState->sslBumpMode = static_cast<Ssl::BumpMode>(answer.kind);
2497 } else {
2498 debugs(33, 3, "sslBump not needed for " << connState->clientConnection);
2499 connState->sslBumpMode = Ssl::bumpSplice;
2500 }
2501
2502 if (connState->sslBumpMode == Ssl::bumpTerminate) {
2503 connState->clientConnection->close();
2504 return;
2505 }
2506
2507 if (!connState->fakeAConnectRequest("ssl-bump", connState->inBuf))
2508 connState->clientConnection->close();
2509}
2510#endif
2511
2513static void
2515{
2516 Assure(params.port);
2517
2518 // NP: it is possible the port was reconfigured when the call or accept() was queued.
2519
2520 if (params.flag != Comm::OK) {
2521 // Its possible the call was still queued when the client disconnected
2522 debugs(33, 2, "httpsAccept: " << params.port->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2523 return;
2524 }
2525
2526 const auto xact = MasterXaction::MakePortful(params.port);
2527 xact->tcpClient = params.conn;
2528
2529 debugs(33, 4, params.conn << " accepted, starting SSL negotiation.");
2530 fd_note(params.conn->fd, "client https connect");
2531
2532 // Socket is ready, setup the connection manager to start using it
2533 auto *srv = Https::NewServer(xact);
2534 // XXX: do not abandon the MasterXaction object
2535 AsyncJob::Start(srv); // usually async-calls postHttpsAccept()
2536}
2537
2538void
2540{
2541 if (port->flags.tunnelSslBumping) {
2542#if USE_OPENSSL
2543 debugs(33, 5, "accept transparent connection: " << clientConnection);
2544
2545 if (!Config.accessList.ssl_bump) {
2547 return;
2548 }
2549
2550 const auto mx = MasterXaction::MakePortful(port);
2551 mx->tcpClient = clientConnection;
2552 // Create a fake HTTP request and ALE for the ssl_bump ACL check,
2553 // using tproxy/intercept provided destination IP and port.
2554 // XXX: Merge with subsequent fakeAConnectRequest(), buildFakeRequest().
2555 // XXX: Do this earlier (e.g., in Http[s]::One::Server constructor).
2556 HttpRequest *request = new HttpRequest(mx);
2557 static char ip[MAX_IPSTRLEN];
2559 request->url.host(clientConnection->local.toStr(ip, sizeof(ip)));
2560 request->url.port(clientConnection->local.port());
2561 request->myportname = port->name;
2562 const AccessLogEntry::Pointer connectAle = new AccessLogEntry;
2563 CodeContext::Reset(connectAle);
2564 // TODO: Use these request/ALE when waiting for new bumped transactions.
2565
2566 ACLFilledChecklist *acl_checklist = new ACLFilledChecklist(Config.accessList.ssl_bump, request, nullptr);
2567 fillChecklist(*acl_checklist);
2568 // Build a local AccessLogEntry to allow requiresAle() acls work
2569 acl_checklist->al = connectAle;
2570 acl_checklist->al->cache.start_time = current_time;
2571 acl_checklist->al->tcpClient = clientConnection;
2572 acl_checklist->al->cache.port = port;
2573 acl_checklist->al->cache.caddr = log_addr;
2575 acl_checklist->al->updateError(bareError);
2576 HTTPMSGUNLOCK(acl_checklist->al->request);
2577 acl_checklist->al->request = request;
2578 HTTPMSGLOCK(acl_checklist->al->request);
2580 ClientHttpRequest *http = context ? context->http : nullptr;
2581 const char *log_uri = http ? http->log_uri : nullptr;
2582 acl_checklist->syncAle(request, log_uri);
2583 acl_checklist->nonBlockingCheck(httpsSslBumpAccessCheckDone, this);
2584#else
2585 fatal("FATAL: SSL-Bump requires --with-openssl");
2586#endif
2587 return;
2588 } else {
2589 httpsEstablish(this, port->secure.staticContext);
2590 }
2591}
2592
2593#if USE_OPENSSL
2594void
2596{
2597 ConnStateData * state_data = (ConnStateData *)(data);
2598 state_data->sslCrtdHandleReply(reply);
2599}
2600
2601void
2603{
2604 if (!isOpen()) {
2605 debugs(33, 3, "Connection gone while waiting for ssl_crtd helper reply; helper reply:" << reply);
2606 return;
2607 }
2608
2609 if (reply.result == Helper::BrokenHelper) {
2610 debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply);
2611 } else if (!reply.other().hasContent()) {
2612 debugs(1, DBG_IMPORTANT, "\"ssl_crtd\" helper returned <NULL> reply.");
2613 } else {
2615 if (reply_message.parse(reply.other().content(), reply.other().contentSize()) != Ssl::CrtdMessage::OK) {
2616 debugs(33, 5, "Reply from ssl_crtd for " << tlsConnectHostOrIp << " is incorrect");
2617 } else {
2618 if (reply.result != Helper::Okay) {
2619 debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply_message.getBody());
2620 } else {
2621 debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " was successfully received from ssl_crtd");
2624 auto ssl = fd_table[clientConnection->fd].ssl.get();
2625 bool ret = Ssl::configureSSLUsingPkeyAndCertFromMemory(ssl, reply_message.getBody().c_str(), *port);
2626 if (!ret)
2627 debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2628
2631 } else {
2633 if (ctx && !sslBumpCertKey.isEmpty())
2635 getSslContextDone(ctx);
2636 }
2637 return;
2638 }
2639 }
2640 }
2643}
2644
2646{
2648
2649 const bool connectedOk = sslServerBump && sslServerBump->connectedOk();
2650 if (connectedOk) {
2651 if (X509 *mimicCert = sslServerBump->serverCert.get())
2652 certProperties.mimicCert.resetAndLock(mimicCert);
2653
2654 ACLFilledChecklist checklist(nullptr, sslServerBump->request.getRaw());
2655 fillChecklist(checklist);
2656
2657 for (sslproxy_cert_adapt *ca = Config.ssl_client.cert_adapt; ca != nullptr; ca = ca->next) {
2658 // If the algorithm already set, then ignore it.
2659 if ((ca->alg == Ssl::algSetCommonName && certProperties.setCommonName) ||
2660 (ca->alg == Ssl::algSetValidAfter && certProperties.setValidAfter) ||
2661 (ca->alg == Ssl::algSetValidBefore && certProperties.setValidBefore) )
2662 continue;
2663
2664 if (ca->aclList && checklist.fastCheck(ca->aclList).allowed()) {
2665 const char *alg = Ssl::CertAdaptAlgorithmStr[ca->alg];
2666 const char *param = ca->param;
2667
2668 // For parameterless CN adaptation, use hostname from the
2669 // CONNECT request.
2670 if (ca->alg == Ssl::algSetCommonName) {
2671 if (!param)
2672 param = tlsConnectHostOrIp.c_str();
2673 certProperties.commonName = param;
2674 certProperties.setCommonName = true;
2675 } else if (ca->alg == Ssl::algSetValidAfter)
2676 certProperties.setValidAfter = true;
2677 else if (ca->alg == Ssl::algSetValidBefore)
2678 certProperties.setValidBefore = true;
2679
2680 debugs(33, 5, "Matches certificate adaptation aglorithm: " <<
2681 alg << " param: " << (param ? param : "-"));
2682 }
2683 }
2684
2685 certProperties.signAlgorithm = Ssl::algSignEnd;
2686 for (sslproxy_cert_sign *sg = Config.ssl_client.cert_sign; sg != nullptr; sg = sg->next) {
2687 if (sg->aclList && checklist.fastCheck(sg->aclList).allowed()) {
2688 certProperties.signAlgorithm = (Ssl::CertSignAlgorithm)sg->alg;
2689 break;
2690 }
2691 }
2692 } else {// did not try to connect (e.g. client-first) or failed to connect
2693 // In case of an error while connecting to the secure server, use a
2694 // trusted certificate, with no mimicked fields and no adaptation
2695 // algorithms. There is nothing we can mimic, so we want to minimize the
2696 // number of warnings the user will have to see to get to the error page.
2697 // We will close the connection, so that the trust is not extended to
2698 // non-Squid content.
2699 certProperties.signAlgorithm = Ssl::algSignTrusted;
2700 }
2701
2702 assert(certProperties.signAlgorithm != Ssl::algSignEnd);
2703
2704 if (certProperties.signAlgorithm == Ssl::algSignUntrusted) {
2705 assert(port->secure.untrustedSigningCa.cert);
2706 certProperties.signWithX509.resetAndLock(port->secure.untrustedSigningCa.cert.get());
2707 certProperties.signWithPkey.resetAndLock(port->secure.untrustedSigningCa.pkey.get());
2708 } else {
2709 assert(port->secure.signingCa.cert.get());
2710 certProperties.signWithX509.resetAndLock(port->secure.signingCa.cert.get());
2711
2712 if (port->secure.signingCa.pkey)
2713 certProperties.signWithPkey.resetAndLock(port->secure.signingCa.pkey.get());
2714 }
2715 signAlgorithm = certProperties.signAlgorithm;
2716
2717 certProperties.signHash = Ssl::DefaultSignHash;
2718}
2719
2722{
2723 debugs(33, 5, "Finding SSL certificate for " << cacheKey << " in cache");
2725 if (const auto ctx = ssl_ctx_cache ? ssl_ctx_cache->get(cacheKey) : nullptr) {
2726 if (Ssl::verifySslCertificate(*ctx, certProperties)) {
2727 debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is valid");
2728 return *ctx;
2729 } else {
2730 debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is out of date. Delete this certificate from cache");
2731 if (ssl_ctx_cache)
2732 ssl_ctx_cache->del(cacheKey);
2733 }
2734 }
2735 return Security::ContextPointer(nullptr);
2736}
2737
2738void
2740{
2742 if (!ssl_ctx_cache || !ssl_ctx_cache->add(cacheKey, ctx)) {
2743 // If it is not in storage delete after using. Else storage deleted it.
2744 fd_table[clientConnection->fd].html?amicTlsContext = ctx;
2745 }
2746}
2747
2748void
2750{
2751 if (port->secure.generateHostCertificates) {
2752 Ssl::CertificateProperties certProperties;
2753 buildSslCertGenerationParams(certProperties);
2754
2755 // Disable caching for bumpPeekAndSplice mode
2760
2762 if (ctx) {
2763 getSslContextDone(ctx);
2764 return;
2765 }
2766 }
2767
2768#if USE_SSL_CRTD
2769 try {
2770 debugs(33, 5, "Generating SSL certificate for " << certProperties.commonName << " using ssl_crtd.");
2773 request_message.composeRequest(certProperties);
2774 debugs(33, 5, "SSL crtd request: " << request_message.compose().c_str());
2775 Ssl::Helper::Submit(request_message, sslCrtdHandleReplyWrapper, this);
2776 return;
2777 } catch (const std::exception &e) {
2778 debugs(33, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtd " <<
2779 "request for " << certProperties.commonName <<
2780 " certificate: " << e.what() << "; will now block to " <<
2781 "generate that certificate.");
2782 // fall through to do blocking in-process generation.
2783 }
2784#endif // USE_SSL_CRTD
2785
2786 debugs(33, 5, "Generating SSL certificate for " << certProperties.commonName);
2789 auto ssl = fd_table[clientConnection->fd].ssl.get();
2790 if (!Ssl::configureSSL(ssl, certProperties, *port))
2791 debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2792
2795 } else {
2797 if (dynCtx && !sslBumpCertKey.isEmpty())
2799 getSslContextDone(dynCtx);
2800 }
2801 return;
2802 }
2803
2806}
2807
2808void
2810{
2811 if (port->secure.generateHostCertificates && !ctx) {
2812 debugs(33, 2, "Failed to generate TLS context for " << tlsConnectHostOrIp);
2813 }
2814
2815 // If generated ssl context = NULL, try to use static ssl context.
2816 if (!ctx) {
2817 if (!port->secure.staticContext) {
2818 debugs(83, DBG_IMPORTANT, "Closing " << clientConnection->remote << " as lacking TLS context");
2820 return;
2821 } else {
2822 debugs(33, 5, "Using static TLS context.");
2823 ctx = port->secure.staticContext;
2824 }
2825 }
2826
2827 if (!httpsCreate(this, ctx))
2828 return;
2829
2830 // bumped intercepted conns should already have Config.Timeout.request set
2831 // but forwarded connections may only have Config.Timeout.lifetime. [Re]set
2832 // to make sure the connection does not get stuck on non-SSL clients.
2834
2835 switchedToHttps_ = true;
2836
2837 auto ssl = fd_table[clientConnection->fd].ssl.get();
2838 BIO *b = SSL_get_rbio(ssl);
2839 Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
2840 bio->setReadBufData(inBuf);
2841 inBuf.clear();
2843}
2844
2845void
2847{
2849 Must(http->request);
2850 auto &request = http->request;
2851
2852 // Depending on receivedFirstByte_, we are at the start of either an
2853 // established CONNECT tunnel with the client or an intercepted TCP (and
2854 // presumably TLS) connection from the client. Expect TLS Client Hello.
2855 const auto insideConnectTunnel = receivedFirstByte_;
2856 debugs(33, 5, (insideConnectTunnel ? "post-CONNECT " : "raw TLS ") << clientConnection);
2857
2858 tlsConnectHostOrIp = request->url.hostOrIp();
2859 tlsConnectPort = request->url.port();
2860 resetSslCommonName(request->url.host());
2861
2862 // We are going to read new request
2863 flags.readMore = true;
2864
2865 // keep version major.minor details the same.
2866 // but we are now performing the HTTPS handshake traffic
2868
2869 // If sslServerBump is set, then we have decided to deny CONNECT
2870 // and now want to switch to SSL to send the error to the client
2871 // without even peeking at the origin server certificate.
2872 if (bumpServerMode == Ssl::bumpServerFirst && !sslServerBump) {
2873 request->flags.sslPeek = true;
2874 sslServerBump = new Ssl::ServerBump(http);
2875 } else if (bumpServerMode == Ssl::bumpPeek || bumpServerMode == Ssl::bumpStare) {
2876 request->flags.sslPeek = true;
2877 sslServerBump = new Ssl::ServerBump(http, nullptr, bumpServerMode);
2878 }
2879
2880 // commSetConnTimeout() was called for this request before we switched.
2881 // Fix timeout to request_start_timeout
2883 // Also reset receivedFirstByte_ flag to allow this timeout work in the case we have
2884 // a bumbed "connect" request on non transparent port.
2885 receivedFirstByte_ = false;
2886 // Get more data to peek at Tls
2887 parsingTlsHandshake = true;
2888
2889 // If the protocol has changed, then reset preservingClientData_.
2890 // Otherwise, its value initially set in start() is still valid/fresh.
2891 // shouldPreserveClientData() uses parsingTlsHandshake which is reset above.
2892 if (insideConnectTunnel)
2894
2895 readSomeData();
2896}
2897
2898void
2900{
2902
2903 assert(!inBuf.isEmpty());
2905 fd_note(clientConnection->fd, "Parsing TLS handshake");
2906
2907 // stops being nil if we fail to parse the handshake
2908 ErrorDetail::Pointer parseErrorDetails;
2909
2910 try {
2911 if (!tlsParser.parseHello(inBuf)) {
2912 // need more data to finish parsing
2913 readSomeData();
2914 return;
2915 }
2916 }
2917 catch (const TextException &ex) {
2918 debugs(83, 2, "exception: " << ex);
2919 parseErrorDetails = new ExceptionErrorDetail(ex.id());
2920 }
2921 catch (...) {
2922 debugs(83, 2, "exception: " << CurrentException);
2923 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_PARSE");
2924 parseErrorDetails = d;
2925 }
2926
2927 parsingTlsHandshake = false;
2928
2929 // client data may be needed for splicing and for
2930 // tunneling unsupportedProtocol after an error
2932
2933 // Even if the parser failed, each TLS detail should either be set
2934 // correctly or still be "unknown"; copying unknown detail is a no-op.
2937 if (details && !details->serverName.isEmpty()) {
2938 resetSslCommonName(details->serverName.c_str());
2939 tlsClientSni_ = details->serverName;
2940 }
2941
2942 // We should disable read/write handlers
2944
2945 if (parseErrorDetails) {
2947 Must(context && context->http);
2948 HttpRequest::Pointer request = context->http->request;
2949 debugs(83, 5, "Got something other than TLS Client Hello. Cannot SslBump.");
2950 updateError(ERR_PROTOCOL_UNKNOWN, parseErrorDetails);
2953 return;
2954 }
2955
2956 if (!sslServerBump || sslServerBump->act.step1 == Ssl::bumpClientFirst) { // Either means client-first.
2958 return;
2960 debugs(83, 5, "server-first skips step2; start forwarding the request");
2963 ClientHttpRequest *http = context ? context->http : nullptr;
2964 // will call httpsPeeked() with certificate and connection, eventually
2966 } else {
2969 }
2970}
2971
2972static void
2974{
2975 ConnStateData *connState = (ConnStateData *) data;
2976
2977 // if the connection is closed or closing, just return.
2978 if (!connState->isOpen())
2979 return;
2980
2981 debugs(33, 5, "Answer: " << answer << " kind:" << answer.kind);
2982 assert(connState->serverBump());
2983 Ssl::BumpMode bumpAction;
2984 if (answer.allowed()) {
2985 bumpAction = (Ssl::BumpMode)answer.kind;
2986 } else
2987 bumpAction = Ssl::bumpSplice;
2988
2989 connState->serverBump()->act.step2 = bumpAction;
2990 connState->sslBumpMode = bumpAction;
2991 Http::StreamPointer context = connState->pipeline.front();
2992 if (ClientHttpRequest *http = (context ? context->http : nullptr))
2993 http->al->ssl.bumpMode = bumpAction;
2994
2995 if (bumpAction == Ssl::bumpTerminate) {
2996 connState->clientConnection->close();
2997 } else if (bumpAction != Ssl::bumpSplice) {
2998 connState->startPeekAndSplice();
2999 } else if (!connState->splice())
3000 connState->clientConnection->close();
3001}
3002
3003bool
3005{
3006 // normally we can splice here, because we just got client hello message
3007
3008 // fde::ssl/tls_read_method() probably reads from our own inBuf. If so, then
3009 // we should not lose any raw bytes when switching to raw I/O here.
3010 if (fd_table[clientConnection->fd].ssl.get())
3011 fd_table[clientConnection->fd].useDefaultIo();
3012
3013 // XXX: assuming that there was an HTTP/1.1 CONNECT to begin with...
3014 // reset the current protocol to HTTP/1.1 (was "HTTPS" for the bumping process)
3016 assert(!pipeline.empty());
3018 Must(context);
3019 Must(context->http);
3020 ClientHttpRequest *http = context->http;
3021 HttpRequest::Pointer request = http->request;
3022 context->finished();
3023 if (transparent()) {
3024 // For transparent connections, make a new fake CONNECT request, now
3025 // with SNI as target. doCallout() checks, adaptations may need that.
3026 return fakeAConnectRequest("splice", preservedClientData);
3027 } else {
3028 // For non transparent connections make a new tunneled CONNECT, which
3029 // also sets the HttpRequest::flags::forceTunnel flag to avoid
3030 // respond with "Connection Established" to the client.
3031 // This fake CONNECT request required to allow use of SNI in
3032 // doCallout() checks and adaptations.
3033 return initiateTunneledRequest(request, "splice", preservedClientData);
3034 }
3035}
3036
3037void
3039{
3040 // This is the Step2 of the SSL bumping
3043 ClientHttpRequest *http = context ? context->http : nullptr;
3044
3047 // Run a accessList check to check if want to splice or continue bumping
3048
3053 fillChecklist(*acl_checklist);
3055 return;
3056 }
3057
3058 // will call httpsPeeked() with certificate and connection, eventually
3059 Security::ContextPointer unConfiguredCTX(Ssl::createSSLContext(port->secure.signingCa.cert, port->secure.signingCa.pkey, port->secure));
3060 fd_table[clientConnection->fd].html?amicTlsContext = unConfiguredCTX;
3061
3062 if (!httpsCreate(this, unConfiguredCTX))
3063 return;
3064
3065 switchedToHttps_ = true;
3066
3067 auto ssl = fd_table[clientConnection->fd].ssl.get();
3068 BIO *b = SSL_get_rbio(ssl);
3069 Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3070 bio->setReadBufData(inBuf);
3071 bio->hold(true);
3072
3073 // We have successfully parsed client Hello, but our TLS handshake parser is
3074 // forgiving. Now we use a TLS library to parse the same bytes, so that we
3075 // can honor on_unsupported_protocol if needed. If there are no errors, we
3076 // expect Security::Accept() to ask us to write (our) TLS server Hello. We
3077 // also allow an ioWantRead result in case some fancy TLS extension that
3078 // Squid does not yet understand requires reading post-Hello client bytes.
3079 const auto handshakeResult = acceptTls();
3080 if (!handshakeResult.wantsIo())
3081 return handleSslBumpHandshakeError(handshakeResult);
3082
3083 // We need to reset inBuf here, to be used by incoming requests in the case
3084 // of SSL bump
3085 inBuf.clear();
3086
3087 debugs(83, 5, "Peek and splice at step2 done. Start forwarding the request!!! ");
3090}
3091
3093void
3095{
3096 auto errCategory = ERR_NONE;
3097
3098 switch (handshakeResult.category) {
3100 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_SUCCESS");
3101 updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3102 break;
3103 }
3104
3106 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_READ");
3107 updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3108 break;
3109 }
3110
3112 static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_WRITE");
3113 updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3114 break;
3115 }
3116
3118 debugs(83, (handshakeResult.important ? DBG_IMPORTANT : 2), "ERROR: " << handshakeResult.errorDescription <<
3119 " while SslBump-accepting a TLS connection on " << clientConnection << ": " << handshakeResult.errorDetail);
3120 updateError(errCategory = ERR_SECURE_ACCEPT_FAIL, handshakeResult.errorDetail);
3121 break;
3122
3123 }
3124
3125 if (!tunnelOnError(errCategory))
3127}
3128
3129void
3131{
3132 auto ssl = fd_table[clientConnection->fd].ssl.get();
3133 BIO *b = SSL_get_rbio(ssl);
3134 assert(b);
3135 Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3136
3137 debugs(33, 5, "PeekAndSplice mode, proceed with client negotiation. Current state:" << SSL_state_string_long(ssl));
3138 bio->hold(false);
3139
3141 switchedToHttps_ = true;
3142}
3143
3144void
3146{
3147 Must(sslServerBump != nullptr);
3149 Must(pipeline.empty() || pipeline.front()->http == nullptr || pipeline.front()->http->request == pic.request.getRaw());
3150
3151 if (Comm::IsConnOpen(pic.connection)) {
3153 debugs(33, 5, "bumped HTTPS server: " << tlsConnectHostOrIp);
3154 } else
3155 debugs(33, 5, "Error while bumping: " << tlsConnectHostOrIp);
3156
3158}
3159
3160#endif /* USE_OPENSSL */
3161
3162bool
3163ConnStateData::initiateTunneledRequest(HttpRequest::Pointer const &cause, const char *reason, const SBuf &payload)
3164{
3165 // fake a CONNECT request to force connState to tunnel
3166 SBuf connectHost;
3167 AnyP::Port connectPort;
3168
3169 if (pinning.serverConnection != nullptr) {
3170 static char ip[MAX_IPSTRLEN];
3171 connectHost = pinning.serverConnection->remote.toStr(ip, sizeof(ip));
3172 if (const auto remotePort = pinning.serverConnection->remote.port())
3173 connectPort = remotePort;
3174 } else if (cause) {
3175 connectHost = cause->url.hostOrIp();
3176 connectPort = cause->url.port();
3177#if USE_OPENSSL
3178 } else if (!tlsConnectHostOrIp.isEmpty()) {
3179 connectHost = tlsConnectHostOrIp;
3180 connectPort = tlsConnectPort;
3181#endif
3182 } else if (transparent()) {
3183 static char ip[MAX_IPSTRLEN];
3184 connectHost = clientConnection->local.toStr(ip, sizeof(ip));
3185 connectPort = clientConnection->local.port();
3186 }
3187
3188 if (!connectPort) {
3189 // Typical cases are malformed HTTP requests on http_port and malformed
3190 // TLS handshakes on non-bumping https_port. TODO: Discover these
3191 // problems earlier so that they can be classified/detailed better.
3192 debugs(33, 2, "Not able to compute URL, abort request tunneling for " << reason);
3193 // TODO: throw when nonBlockingCheck() callbacks gain job protections
3194 static const auto d = MakeNamedErrorDetail("TUNNEL_TARGET");
3196 return false;
3197 }
3198
3199 debugs(33, 2, "Request tunneling for " << reason);
3200 const auto http = buildFakeRequest(connectHost, *connectPort, payload);
3201 HttpRequest::Pointer request = http->request;
3202 request->flags.forceTunnel = true;
3203 http->calloutContext = new ClientRequestContext(http);
3204 http->doCallouts();
3205 clientProcessRequestFinished(this, request);
3206 return true;
3207}
3208
3209bool
3210ConnStateData::fakeAConnectRequest(const char *reason, const SBuf &payload)
3211{
3212 debugs(33, 2, "fake a CONNECT request to force connState to tunnel for " << reason);
3213
3214 SBuf connectHost;
3216 const unsigned short connectPort = clientConnection->local.port();
3217
3218#if USE_OPENSSL
3219 if (!tlsClientSni_.isEmpty())
3220 connectHost.assign(tlsClientSni_);
3221 else
3222#endif
3223 {
3224 static char ip[MAX_IPSTRLEN];
3225 clientConnection->local.toHostStr(ip, sizeof(ip));
3226 connectHost.assign(ip);
3227 }
3228
3229 ClientHttpRequest *http = buildFakeRequest(connectHost, connectPort, payload);
3230
3231 http->calloutContext = new ClientRequestContext(http);
3232 HttpRequest::Pointer request = http->request;
3233 http->doCallouts();
3234 clientProcessRequestFinished(this, request);
3235 return true;
3236}
3237
3239ConnStateData::buildFakeRequest(SBuf &useHost, const AnyP::KnownPort usePort, const SBuf &payload)
3240{
3241 ClientHttpRequest *http = new ClientHttpRequest(this);
3242 Http::Stream *stream = new Http::Stream(clientConnection, http);
3243
3244 StoreIOBuffer tempBuffer;
3245 tempBuffer.data = stream->reqbuf;
3246 tempBuffer.length = HTTP_REQBUF_SZ;
3247
3248 ClientStreamData newServer = new clientReplyContext(http);
3249 ClientStreamData newClient = stream;
3252 clientSocketDetach, newClient, tempBuffer);
3253
3254 stream->flags.parsed_ok = 1; // Do we need it?
3255 stream->mayUseConnection(true);
3257 stream->registerWithConn();
3258
3259 const auto mx = MasterXaction::MakePortful(port);
3260 mx->tcpClient = clientConnection;
3261 // Setup Http::Request object. Maybe should be replaced by a call to (modified)
3262 // clientProcessRequest
3263 HttpRequest::Pointer request = new HttpRequest(mx);
3264 request->url.setScheme(AnyP::PROTO_AUTHORITY_FORM, nullptr);
3265 request->method = Http::METHOD_CONNECT;
3266 request->url.host(useHost.c_str());
3267 request->url.port(usePort);
3268
3269 http->uri = SBufToCstring(request->effectiveRequestUri());
3270 http->initRequest(request.getRaw());
3271
3272 request->manager(this, http->al);
3273
3274 request->header.putStr(Http::HOST, useHost.c_str());
3275
3276 request->sources |= ((switchedToHttps() || port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
3277#if USE_AUTH
3278 if (getAuth())
3279 request->auth_user_request = getAuth();
3280#endif
3281
3282 inBuf = payload;
3283 flags.readMore = false;
3284
3285 return http;
3286}
3287
3289static bool
3291{
3292 if (!Comm::IsConnOpen(c)) {
3293 Must(NHttpSockets > 0); // we tried to open some
3294 --NHttpSockets; // there will be fewer sockets than planned
3295 Must(HttpSockets[NHttpSockets] < 0); // no extra fds received
3296
3297 if (!NHttpSockets) // we could not open any listen sockets at all
3298 fatalf("Unable to open %s",FdNote(portType));
3299
3300 return false;
3301 }
3302 return true;
3303}
3304
3306static bool
3308{
3309 bool found = false;
3310 for (int i = 0; i < NHttpSockets && !found; ++i) {
3311 if ((found = HttpSockets[i] < 0))
3312 HttpSockets[i] = conn->fd;
3313 }
3314 return found;
3315}
3316
3317static void
3319{
3320 const auto savedContext = CodeContext::Current();
3321 for (AnyP::PortCfgPointer s = HttpPortList; s != nullptr; s = s->next) {
3323 const SBuf &scheme = AnyP::UriScheme(s->transport.protocol).image();
3324
3326 debugs(1, DBG_IMPORTANT, "WARNING: You have too many '" << scheme << "_port' lines." <<
3327 Debug::Extra << "The limit is " << MAXTCPLISTENPORTS << " HTTP ports.");
3328 continue;
3329 }
3330
3331#if USE_OPENSSL
3332 if (s->flags.tunnelSslBumping) {
3333 if (!Config.accessList.ssl_bump) {
3334 debugs(33, DBG_IMPORTANT, "WARNING: No ssl_bump configured. Disabling ssl-bump on " << scheme << "_port " << s->s);
3335 s->flags.tunnelSslBumping = false;
3336 }
3337 if (!s->secure.staticContext && !s->secure.generateHostCertificates) {
3338 debugs(1, DBG_IMPORTANT, "Will not bump SSL at " << scheme << "_port " << s->s << " due to TLS initialization failure.");
3339 s->flags.tunnelSslBumping = false;
3340 if (s->transport.protocol == AnyP::PROTO_HTTP)
3341 s->secure.encryptTransport = false;
3342 }
3343 if (s->flags.tunnelSslBumping) {
3344 // Create ssl_ctx cache for this port.
3345 Ssl::TheGlobalContextStorage.addLocalStorage(s->s, s->secure.html?amicCertMemCacheSize);
3346 }
3347 }
3348#endif
3349
3350 if (s->secure.encryptTransport && !s->secure.staticContext) {
3351 debugs(1, DBG_CRITICAL, "ERROR: Ignoring " << scheme << "_port " << s->s << " due to TLS context initialization failure.");
3352 continue;
3353 }
3354
3355 const auto protocol = s->transport.protocol;
3356 assert(protocol == AnyP::PROTO_HTTP || protocol == AnyP::PROTO_HTTPS);
3357 const auto isHttps = protocol == AnyP::PROTO_HTTPS;
3358 using AcceptCall = CommCbFunPtrCallT<CommAcceptCbPtrFun>;
3359 RefCount<AcceptCall> subCall = commCbCall(5, 5, isHttps ? "httpsAccept" : "httpAccept",
3362 }
3363 CodeContext::Reset(savedContext);
3364}
3365
3366void
3368{
3369 // Fill out a Comm::Connection which IPC will open as a listener for us
3370 port->listenConn = new Comm::Connection;
3371 port->listenConn->local = port->s;
3372 port->listenConn->flags =
3374 (port->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) |
3375 (port->flags.natIntercept ? COMM_INTERCEPTION : 0) |
3376 (port->workerQueues ? COMM_REUSEPORT : 0);
3377
3378 // route new connections to subCall
3379 typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
3381 const auto listenCall =
3382 asyncCall(33, 2, "clientListenerConnectionOpened",
3384 port, fdNote, sub));
3385 AsyncCallback<Ipc::StartListeningAnswer> callback(listenCall);
3386 Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, port->listenConn, fdNote, callback);
3387
3390 ++NHttpSockets;
3391}
3392
3394static void
3396{
3397 Must(s != nullptr);
3398
3399 if (!OpenedHttpSocket(s->listenConn, portTypeNote))
3400 return;
3401
3402 Must(Comm::IsConnOpen(s->listenConn));
3403
3404 // TCP: setup a job to handle accept() with subscribed handler
3405 AsyncJob::Start(new Comm::TcpAcceptor(s, FdNote(portTypeNote), sub));
3406
3407 debugs(1, Important(13), "Accepting " <<
3408 (s->flags.natIntercept ? "NAT intercepted " : "") <<
3409 (s->flags.tproxyIntercept ? "TPROXY intercepted " : "") <<
3410 (s->flags.tunnelSslBumping ? "SSL bumped " : "") <<
3411 (s->flags.accelSurrogate ? "reverse-proxy " : "")
3412 << FdNote(portTypeNote) << " connections at "
3413 << s->listenConn);
3414
3415 Must(AddOpenedHttpSocket(s->listenConn)); // otherwise, we have received a fd we did not ask for
3416
3417#if USE_SYSTEMD
3418 // When the very first port opens, tell systemd we are able to serve connections.
3419 // Subsequent sd_notify() calls, including calls during reconfiguration,
3420 // do nothing because the first call parameter is 1.
3421 // XXX: Send the notification only after opening all configured ports.
3423 const auto result = sd_notify(1, "READY=1");
3424 if (result < 0) {
3425 debugs(1, DBG_IMPORTANT, "WARNING: failed to send start-up notification to systemd" <<
3426 Debug::Extra << "sd_notify() error: " << xstrerr(-result));
3427 }
3428 }
3429#endif
3430}
3431
3432void
3434{
3437
3438 if (NHttpSockets < 1)
3439 fatal("No HTTP, HTTPS, or FTP ports configured");
3440}
3441
3442void
3444{
3445 const auto savedContext = CodeContext::Current();
3446 for (AnyP::PortCfgPointer s = HttpPortList; s != nullptr; s = s->next) {
3448 if (s->listenConn != nullptr) {
3449 debugs(1, Important(14), "Closing HTTP(S) port " << s->listenConn->local);
3450 s->listenConn->close();
3451 s->listenConn = nullptr;
3452 }
3453 }
3454 CodeContext::Reset(savedContext);
3455
3457
3458 // TODO see if we can drop HttpSockets array entirely */
3459 for (int i = 0; i < NHttpSockets; ++i) {
3460 HttpSockets[i] = -1;
3461 }
3462
3463 NHttpSockets = 0;
3464}
3465
3466int
3468{
3469 SBuf vary(request->vary_headers);
3470 const auto &reply = entry->mem().freshestReply();
3471 auto has_vary = reply.header.has(Http::HdrType::VARY);
3472#if X_ACCELERATOR_VARY
3473
3474 has_vary |=
3475 reply.header.has(Http::HdrType::HDR_X_ACCELERATOR_VARY);
3476#endif
3477
3478 if (!has_vary || entry->mem_obj->vary_headers.isEmpty()) {
3479 if (!vary.isEmpty()) {
3480 /* Oops... something odd is going on here.. */
3481 debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary object on second attempt, '" <<
3482 entry->mem_obj->urlXXX() << "' '" << vary << "'");
3483 request->vary_headers.clear();
3484 return VARY_CANCEL;
3485 }
3486
3487 if (!has_vary) {
3488 /* This is not a varying object */
3489 return VARY_NONE;
3490 }
3491
3492 /* virtual "vary" object found. Calculate the vary key and
3493 * continue the search
3494 */
3495 vary = httpMakeVaryMark(request, &reply);
3496
3497 if (!vary.isEmpty()) {
3498 request->vary_headers = vary;
3499 return VARY_OTHER;
3500 } else {
3501 /* Ouch.. we cannot handle this kind of variance */
3502 /* XXX This cannot really happen, but just to be complete */
3503 return VARY_CANCEL;
3504 }
3505 } else {
3506 if (vary.isEmpty()) {
3507 vary = httpMakeVaryMark(request, &reply);
3508
3509 if (!vary.isEmpty())
3510 request->vary_headers = vary;
3511 }
3512
3513 if (vary.isEmpty()) {
3514 /* Ouch.. we cannot handle this kind of variance */
3515 /* XXX This cannot really happen, but just to be complete */
3516 return VARY_CANCEL;
3517 } else if (vary.cmp(entry->mem_obj->vary_headers) == 0) {
3518 return VARY_MATCH;
3519 } else {
3520 /* Oops.. we have already been here and still haven't
3521 * found the requested variant. Bail out
3522 */
3523 debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary match on second attempt, '" <<
3524 entry->mem_obj->urlXXX() << "' '" << vary << "'");
3525 return VARY_CANCEL;
3526 }
3527 }
3528}
3529
3532{
3533 const auto checklist = new ACLFilledChecklist(acl, nullptr, nullptr);
3534 clientAclChecklistFill(*checklist, http);
3535 return checklist;
3536}
3537
3538void
3540{
3541 assert(http);
3542
3543 if (!checklist.request && http->request)
3544 checklist.setRequest(http->request);
3545
3546 if (!checklist.al && http->al) {
3547 checklist.al = http->al;
3548 checklist.syncAle(http->request, http->log_uri);
3549 if (!checklist.reply && http->al->reply) {
3550 checklist.reply = http->al->reply.getRaw();
3551 HTTPMSGLOCK(checklist.reply);
3552 }
3553 }
3554
3555 if (const auto conn = http->getConn())
3556 checklist.setConn(conn); // may already be set
3557}
3558
3559void
3561{
3562 const auto context = pipeline.front();
3563 if (const auto http = context ? context->http : nullptr)
3564 return clientAclChecklistFill(checklist, http); // calls checklist.setConn()
3565
3566 // no requests, but we always have connection-level details
3567 // TODO: ACL checks should not require a mutable ConnStateData. Adjust the
3568 // code that accidentally violates that principle to remove this const_cast!
3569 checklist.setConn(const_cast<ConnStateData*>(this));
3570
3571 // Set other checklist fields inside our fillConnectionLevelDetails() rather
3572 // than here because clientAclChecklistFill() code path calls that method
3573 // (via ACLFilledChecklist::setConn()) rather than calling us directly.
3574}
3575
3576void
3578{
3579 assert(checklist.conn() == this);
3581
3582 if (!checklist.request) { // preserve (better) addresses supplied by setRequest()
3583 checklist.src_addr = clientConnection->remote;
3584 checklist.my_addr = clientConnection->local; // TODO: or port->s?
3585 }
3586
3587#if USE_OPENSSL
3588 if (!checklist.sslErrors && sslServerBump)
3590#endif
3591
3592 if (!checklist.rfc931[0]) // checklist creator may have supplied it already
3593 checklist.setIdent(clientConnection->rfc931);
3594
3595}
3596
3597bool
3599{
3601}
3602
3605{
3606 bodyPipe = new BodyPipe(this);
3607 if (size >= 0)
3609 else
3611 return bodyPipe;
3612}
3613
3614int64_t
3616{
3617 if (!bodyPipe)
3618 return 0; // request without a body or read/produced all body bytes
3619
3620 if (!bodyPipe->bodySizeKnown())
3621 return -1; // probably need to read more, but we cannot be sure
3622
3623 const int64_t needToProduce = bodyPipe->unproducedSize();
3624 const int64_t haveAvailable = static_cast<int64_t>(inBuf.length());
3625
3626 if (needToProduce <= haveAvailable)
3627 return 0; // we have read what we need (but are waiting for pipe space)
3628
3629 return needToProduce - haveAvailable;
3630}
3631
3632void
3634{
3635 debugs(33, 4, "receiving error (" << clientConnection << "): " << error <<
3636 "; old sending error: " <<
3637 (stoppedSending() ? stoppedSending_ : "none"));
3638
3639 if (const char *oldError = stoppedReceiving()) {
3640 debugs(33, 3, "already stopped receiving: " << oldError);
3641 return; // nothing has changed as far as this connection is concerned
3642 }
3643
3645
3646 if (const char *sendError = stoppedSending()) {
3647 debugs(33, 3, "closing because also stopped sending: " << sendError);
3649 }
3650}
3651
3652void
3654{
3655 if (bodyPipe != nullptr) {
3656 debugs(33, 4, "no consumer for virgin body " << bodyPipe->status());
3658 }
3659}
3660
3662void
3664{
3665 Must(bodyPipe != nullptr);
3666 debugs(33, 5, "start dechunking" << bodyPipe->status());
3669}
3670
3672void
3674{
3675 debugs(33, 5, "finish dechunking: " << withSuccess);
3676
3677 if (bodyPipe != nullptr) {
3678 debugs(33, 7, "dechunked tail: " << bodyPipe->status());
3679 BodyPipe::Pointer myPipe = bodyPipe;
3680 stopProducingFor(bodyPipe, withSuccess); // sets bodyPipe->bodySize()
3681 Must(!bodyPipe); // we rely on it being nil after we are done with body
3682 if (withSuccess) {
3683 Must(myPipe->bodySizeKnown());
3685 if (context != nullptr && context->http && context->http->request)
3686 context->http->request->setContentLength(myPipe->bodySize());
3687 }
3688 }
3689
3690 delete bodyParser;
3691 bodyParser = nullptr;
3692}
3693
3694// XXX: this is an HTTP/1-only operation
3695void
3697{
3698 if (const auto context = pipeline.front()) {
3699 if (context->http)
3700 context->http->al->reply = msg.reply;
3701 }
3702
3703 if (!isOpen()) {
3704 debugs(33, 3, "ignoring 1xx due to earlier closure");
3705 return;
3706 }
3707
3708 // HTTP/1 1xx status messages are only valid when there is a transaction to trigger them
3709 if (!pipeline.empty()) {
3710 HttpReply::Pointer rep(msg.reply);
3711 Must(rep);
3712 // remember the callback
3714
3717
3718 if (!writeControlMsgAndCall(rep.getRaw(), call)) {
3719 // but still inform the caller (so it may resume its operation)
3721 }
3722 return;
3723 }
3724
3725 debugs(33, 3, " closing due to missing context for 1xx");
3727}
3728
3729void
3731{
3733
3734 if (Http::StreamPointer deferredRequest = pipeline.front()) {
3735 debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded after control msg wrote");
3736 ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
3737 }
3738}
3739
3741void
3743{
3744 // FwdState might repin a failed connection sooner than this close
3745 // callback is called for the failed connection.
3746 assert(pinning.serverConnection == io.conn);
3747 pinning.closeHandler = nullptr; // Comm unregisters handlers before calling
3748 const bool sawZeroReply = pinning.zeroReply; // reset when unpinning
3749 pinning.serverConnection->noteClosure();
3750 unpinConnection(false);
3751
3752 if (sawZeroReply && clientConnection != nullptr) {
3753 debugs(33, 3, "Closing client connection on pinned zero reply.");
3755 }
3756
3757}
3758
3759void
3761{
3762 pinConnection(pinServer, *request);
3763}
3764
3765void
3767{
3768 Must(pic.connection);
3769 Must(pic.request);
3770 pinConnection(pic.connection, *pic.request);
3771
3772 // monitor pinned server connection for remote-end closures.
3774
3775 if (pipeline.empty())
3776 kick(); // in case clientParseRequests() was blocked by a busy pic.connection
3777}
3778
3780void
3782{
3783 if (Comm::IsConnOpen(pinning.serverConnection) &&
3784 pinning.serverConnection->fd == pinServer->fd) {
3785 debugs(33, 3, "already pinned" << pinServer);
3786 return;
3787 }
3788
3789 unpinConnection(true); // closes pinned connection, if any, and resets fields
3790
3791 pinning.serverConnection = pinServer;
3792
3793 debugs(33, 3, pinning.serverConnection);
3794
3795 Must(pinning.serverConnection != nullptr);
3796
3797 const char *pinnedHost = "[unknown]";
3798 pinning.host = xstrdup(request.url.host());
3799 pinning.port = request.url.port();
3800 pinnedHost = pinning.host;
3801 pinning.pinned = true;
3802 if (CachePeer *aPeer = pinServer->getPeer())
3803 pinning.peer = cbdataReference(aPeer);
3804 pinning.auth = request.flags.connectionAuth;
3805 char stmp[MAX_IPSTRLEN];
3806 char desc[FD_DESC_SZ];
3807 snprintf(desc, FD_DESC_SZ, "%s pinned connection for %s (%d)",
3808 (pinning.auth || !pinning.peer) ? pinnedHost : pinning.peer->name,
3811 fd_note(pinning.serverConnection->fd, desc);
3812
3814 pinning.closeHandler = JobCallback(33, 5,
3816 // remember the pinned connection so that cb does not unpin a fresher one
3817 typedef CommCloseCbParams Params;
3818 Params &params = GetCommParams<Params>(pinning.closeHandler);
3819 params.conn = pinning.serverConnection;
3820 comm_add_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3821}
3822
3825void
3827{
3828 if (pinning.readHandler != nullptr)
3829 return; // already monitoring
3830
3832 pinning.readHandler = JobCallback(33, 3,
3834 Comm::Read(pinning.serverConnection, pinning.readHandler);
3835}
3836
3837void
3839{
3840 if (pinning.readHandler != nullptr) {
3841 Comm::ReadCancel(pinning.serverConnection->fd, pinning.readHandler);
3842 pinning.readHandler = nullptr;
3843 }
3844}
3845
3846#if USE_OPENSSL
3847bool
3849{
3850 // A ready-for-reading connection means that the TLS server either closed
3851 // the connection, sent us some unexpected HTTP data, or started TLS
3852 // renegotiations. We should close the connection except for the last case.
3853
3854 Must(pinning.serverConnection != nullptr);
3855 auto ssl = fd_table[pinning.serverConnection->fd].ssl.get();
3856 if (!ssl)
3857 return false;
3858
3859 char buf[1];
3860 const int readResult = SSL_read(ssl, buf, sizeof(buf));
3861
3862 if (readResult > 0 || SSL_pending(ssl) > 0) {
3863 debugs(83, 2, pinning.serverConnection << " TLS application data read");
3864 return false;
3865 }
3866
3867 switch(const int error = SSL_get_error(ssl, readResult)) {
3868 case SSL_ERROR_WANT_WRITE:
3869 debugs(83, DBG_IMPORTANT, pinning.serverConnection << " TLS SSL_ERROR_WANT_WRITE request for idle pinned connection");
3870 [[fallthrough]]; // to restart monitoring, for now
3871
3872 case SSL_ERROR_NONE:
3873 case SSL_ERROR_WANT_READ:
3875 return true;
3876
3877 default:
3878 debugs(83, 2, pinning.serverConnection << " TLS error: " << error);
3879 return false;
3880 }
3881
3882 // not reached
3883 return true;
3884}
3885#endif
3886
3889void
3891{
3892 pinning.readHandler = nullptr; // Comm unregisters handlers before calling
3893
3894 if (io.flag == Comm::ERR_CLOSING)
3895 return; // close handler will clean up
3896
3897 Must(pinning.serverConnection == io.conn);
3898
3899#if USE_OPENSSL
3901 return;
3902#endif
3903
3904 const bool clientIsIdle = pipeline.empty();
3905
3906 debugs(33, 3, "idle pinned " << pinning.serverConnection << " read " <<
3907 io.size << (clientIsIdle ? " with idle client" : ""));
3908
3909 pinning.serverConnection->close();
3910
3911 // If we are still sending data to the client, do not close now. When we are done sending,
3912 // ConnStateData::kick() checks pinning.serverConnection and will close.
3913 // However, if we are idle, then we must close to inform the idle client and minimize races.
3914 if (clientIsIdle && clientConnection != nullptr)
3916}
3917
3920{
3921 debugs(33, 7, pinning.serverConnection);
3922 Must(request);
3923
3924 const auto pinningError = [&](const err_type type) {
3925 unpinConnection(true);
3926 HttpRequestPointer requestPointer = request;
3927 return ErrorState::NewForwarding(type, requestPointer, ale);
3928 };
3929
3930 if (!Comm::IsConnOpen(pinning.serverConnection))
3931 throw pinningError(ERR_ZERO_SIZE_OBJECT);
3932
3933 if (pinning.auth && pinning.host && strcasecmp(pinning.host, request->url.host()) != 0)
3934 throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_CONFLICT_HOST
3935
3936 if (pinning.port != request->url.port())
3937 throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_CONFLICT_HOST
3938
3939 if (pinning.peer && !cbdataReferenceValid(pinning.peer))
3940 throw pinningError(ERR_ZERO_SIZE_OBJECT);
3941
3942 if (pinning.peerAccessDenied)
3943 throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_FORWARDING_DENIED
3944
3946 return pinning.serverConnection;
3947}
3948
3951{
3952 if (const auto connManager = request ? request->pinnedConnection() : nullptr)
3953 return connManager->borrowPinnedConnection(request, ale);
3954
3955 // ERR_CANNOT_FORWARD is somewhat misleading here; we can still forward, but
3956 // there is no point since the client connection is now gone
3957 HttpRequestPointer requestPointer = request;
3958 throw ErrorState::NewForwarding(ERR_CANNOT_FORWARD, requestPointer, ale);
3959}
3960
3961void
3963{
3964 debugs(33, 3, pinning.serverConnection);
3965
3966 if (pinning.peer)
3968
3969 if (Comm::IsConnOpen(pinning.serverConnection)) {
3970 if (pinning.closeHandler != nullptr) {
3971 comm_remove_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3972 pinning.closeHandler = nullptr;
3973 }
3974
3976
3977 // close the server side socket if requested
3978 if (andClose)
3979 pinning.serverConnection->close();
3980 pinning.serverConnection = nullptr;
3981 }
3982
3983 safe_free(pinning.host);
3984
3985 pinning.zeroReply = false;
3986 pinning.peerAccessDenied = false;
3987
3988 /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host
3989 * connection has gone away */
3990}
3991
3992void
3994{
3995 auto error = rawError; // (cheap) copy so that we can detail
3996 // We detail even ERR_NONE: There should be no transactions left, and
3997 // detailed ERR_NONE will be unused. Otherwise, this detail helps in triage.
3998 if (!error.detail) {
3999 static const auto d = MakeNamedErrorDetail("WITH_CLIENT");
4000 error.detail = d;
4001 }
4002
4003 debugs(33, 3, pipeline.count() << '/' << pipeline.nrequests << " after " << error);
4004
4005 if (pipeline.empty()) {
4006 bareError.update(error); // XXX: bareLogTagsErrors
4007 } else {
4008 // We terminate the current CONNECT/PUT/etc. context below, logging any
4009 // error details, but that context may leave unparsed bytes behind.
4010 // Consume them to stop checkLogging() from logging them again later.
4011 const auto intputToConsume =
4012#if USE_OPENSSL
4013 parsingTlsHandshake ? "TLS handshake" : // more specific than CONNECT
4014#endif
4015 bodyPipe ? "HTTP request body" :
4016 pipeline.back()->mayUseConnection() ? "HTTP CONNECT" :
4017 nullptr;
4018
4019 while (const auto context = pipeline.front()) {
4020 context->noteIoError(error, lte);
4021 context->finished(); // cleanup and self-deregister
4022 assert(context != pipeline.front());
4023 }
4024
4025 if (intputToConsume && !inBuf.isEmpty()) {
4026 debugs(83, 5, "forgetting client " << intputToConsume << " bytes: " << inBuf.length());
4027 inBuf.clear();
4028 }
4029 }
4030
4032}
4033
4035void
4037{
4038 // to simplify our logic, we assume that terminateAll() has been called
4040
4041 // do not log connections that closed after a transaction (it is normal)
4042 // TODO: access_log needs ACLs to match received-no-bytes connections
4044 return;
4045
4046 /* Create a temporary ClientHttpRequest object. Its destructor will log. */
4047 ClientHttpRequest http(this);
4048 http.req_sz = inBuf.length();
4049 // XXX: Or we died while waiting for the pinned connection to become idle.
4050 http.setErrorUri("error:transaction-end-before-headers");
4051 http.updateError(bareError);
4052}
4053
4054bool
4056{
4057 // PROXY protocol bytes are meant for us and, hence, cannot be tunneled
4059 return false;
4060
4061 // If our decision here is negative, configuration changes are irrelevant.
4062 // Otherwise, clientTunnelOnError() rechecks configuration before tunneling.
4064 return false;
4065
4066 // TODO: Figure out whether/how we can support FTP tunneling.
4067 if (port->transport.protocol == AnyP::PROTO_FTP)
4068 return false;
4069
4070#if USE_OPENSSL
4072 return true;
4073
4074 // the 1st HTTP request on a bumped connection
4076 return true;
4077#endif
4078
4079 // the 1st HTTP(S) request on a connection to an intercepting port
4080 if (!pipeline.nrequests && transparent())
4081 return true;
4082
4083 return false;
4084}
4085
4088{
4089 if (!theNotes)
4090 theNotes = new NotePairs;
4091 return theNotes;
4092}
4093
4094std::ostream &
4095operator <<(std::ostream &os, const ConnStateData::PinnedIdleContext &pic)
4096{
4097 return os << pic.connection << ", request=" << pic.request;
4098}
4099
4100std::ostream &
4101operator <<(std::ostream &os, const ConnStateData::ServerConnectionContext &scc)
4102{
4103 return os << scc.conn_ << ", srv_bytes=" << scc.preReadServerBytes.length();
4104}
4105
void accessLogLog(const AccessLogEntryPointer &, ACLChecklist *)
Definition: access_log.cc:147
#define Assure(condition)
Definition: Assure.h:35
RefCount< AsyncCallT< Dialer > > asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:155
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
CommCbFunPtrCallT< Dialer > * commCbCall(int debugSection, int debugLevel, const char *callName, const Dialer &dialer)
Definition: CommCalls.h:312
void IOACB(const CommAcceptCbParams &params)
Definition: CommCalls.h:31
#define COMM_TRANSPARENT
Definition: Connection.h:50
#define COMM_REUSEPORT
Definition: Connection.h:52
#define COMM_INTERCEPTION
Definition: Connection.h:51
#define COMM_NONBLOCKING
Definition: Connection.h:46
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
#define Here()
source code location of the caller
Definition: Here.h:15
void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, int64_t)
@ hoReply
Definition: HttpHeader.h:37
void IDCB(const char *ident, void *data)
Definition: Ident.h:17
@ LOG_TCP_OFFLINE_HIT
Definition: LogTags.h:54
@ LOG_TCP_HIT
Definition: LogTags.h:39
@ LOG_TCP_MISS
Definition: LogTags.h:40
@ LOG_TCP_MEM_HIT
Definition: LogTags.h:51
@ LOG_TCP_IMS_HIT
Definition: LogTags.h:47
@ LOG_TAG_NONE
Definition: LogTags.h:38
@ LOG_TCP_REFRESH_UNMODIFIED
Definition: LogTags.h:41
@ LOG_TCP_CLIENT_REFRESH_MISS
Definition: LogTags.h:46
@ LOG_TCP_INM_HIT
Definition: LogTags.h:48
int size
Definition: ModDevPoll.cc:75
int NHttpSockets
Definition: PortCfg.cc:25
int HttpSockets[MAXTCPLISTENPORTS]
Definition: PortCfg.cc:26
AnyP::PortCfgPointer HttpPortList
list of Squid http(s)_port configured
Definition: PortCfg.cc:22
#define MAXTCPLISTENPORTS
Definition: PortCfg.h:87
#define SQUIDSBUFPH
Definition: SBuf.h:31
void SBufToCstring(char *d, const SBuf &s)
Definition: SBuf.h:752
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
class SquidConfig Config
Definition: SquidConfig.cc:12
#define SQUIDSTRINGPH
Definition: SquidString.h:21
#define SQUIDSTRINGPRINT(s)
Definition: SquidString.h:22
StatCounters statCounter
Definition: StatCounters.cc:12
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
#define Must(condition)
Definition: TextException.h:71
int conn
the current server connection FD
Definition: Transport.cc:26
bool urlCheckRequest(const HttpRequest *r)
Definition: Uri.cc:841
void error(char *format,...)
#define acl_access
Definition: forward.h:45
#define assert(EX)
Definition: assert.h:17
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:265
#define cbdataReferenceDone(var)
Definition: cbdata.h:352
#define cbdataReference(var)
Definition: cbdata.h:343
void banAction(const Acl::Answer &action)
add action to the list of banned actions
Definition: Checklist.cc:395
const Acl::Tree * changeAcl(const Acl::Tree *t)
Definition: Checklist.h:176
Acl::Answer const & fastCheck()
Definition: Checklist.cc:332
void nonBlockingCheck(ACLCB *callback, void *callback_data)
Definition: Checklist.cc:237
void setIdent(const char *userIdentity)
configure rfc931 user identity for the first time
HttpRequest * request
ConnStateData * conn() const
The client connection manager.
const Security::CertErrors * sslErrors
void setRequest(HttpRequest *)
configure client request-related fields for the first time
void setConn(ConnStateData *)
set either conn
char rfc931[USER_IDENT_SZ]
Ip::Address src_addr
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
void syncAle(HttpRequest *adaptedRequest, const char *logUri) const override
assigns uninitialized adapted_request and url ALE components
char * last_meta
image of the last ICAP response header or eCAP meta received
AnyP::PortCfgPointer port
struct timeval start_time
The time the master transaction started.
struct timeval trTime
The response time.
MessageSizes clientReplySz
counters for the response sent to client
HttpRequestMethod method
MessageSizes clientRequestSz
counters for the original request received from client
AnyP::ProtocolVersion version
struct timeval processingTime
total ICAP processing time
HttpReplyPointer reply
class AccessLogEntry::CacheDetails cache
HierarchyLogEntry hier
void syncNotes(HttpRequest *request)
class AccessLogEntry::IcapLogEntry icap
class AccessLogEntry::Headers headers
Comm::ConnectionPointer tcpClient
TCP/IP level details about the client connection.
class AccessLogEntry::HttpDetails http
HttpRequest * adapted_request
HttpRequest * request
class AccessLogEntry::AdaptationDetails adapt
class AccessLogEntry::IcpDetails icp
ProxyProtocol::HeaderPointer proxyProtocolHeader
see ConnStateData::proxyProtocolHeader_
void updateError(const Error &)
sets (or updates the already stored) transaction error as needed
int kind
the matched custom access list verb (or zero)
Definition: Acl.h:170
bool allowed() const
Definition: Acl.h:156
HttpHeader lastMeta
Last received meta header (REQMOD or RESPMOD, whichever comes last).
Definition: History.h:61
unsigned int major
major version number
ProtocolType protocol
which protocol this version is for
unsigned int minor
minor version number
SBuf image() const
Definition: UriScheme.h:57
AnyP::UriScheme const & getScheme() const
Definition: Uri.h:67
SBuf & authority(bool requirePort=false) const
Definition: Uri.cc:570
void setScheme(const AnyP::ProtocolType &p, const char *str)
convert the URL scheme to that given
Definition: Uri.h:70
void path(const char *p)
Definition: Uri.h:101
void port(const Port p)
reset authority port subcomponent
Definition: Uri.h:95
void host(const char *src)
Definition: Uri.cc:99
SBuf hostOrIp() const
Definition: Uri.cc:114
a smart AsyncCall pointer for delivery of future results
static void Start(const Pointer &job)
Definition: AsyncJob.cc:24
virtual void start()
called by AsyncStart; do not call directly
Definition: AsyncJob.cc:44
void mustStop(const char *aReason)
Definition: AsyncJob.cc:70
void deleteThis(const char *aReason)
Definition: AsyncJob.cc:50
virtual void callException(const std::exception &e)
called when the job throws during an async call
Definition: AsyncJob.cc:128
virtual void releaseAuthServer()
Definition: UserRequest.cc:205
MemBuf & buf
Definition: BodyPipe.h:74
uint64_t producedSize() const
Definition: BodyPipe.h:112
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
size_t putMoreData(const char *buf, size_t size)
Definition: BodyPipe.cc:213
const MemBuf & buf() const
Definition: BodyPipe.h:137
bool bodySizeKnown() const
Definition: BodyPipe.h:109
uint64_t unproducedSize() const
Definition: BodyPipe.cc:179
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
const char * status() const
Definition: BodyPipe.cc:446
bool productionEnded() const
Definition: BodyPipe.h:113
bool mayNeedMoreData() const
Definition: BodyPipe.h:118
uint64_t bodySize() const
Definition: BodyPipe.cc:161
void enableAutoConsumption()
start or continue consuming when producing without consumer
Definition: BodyPipe.cc:316
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:18
static const CharacterSet DIGIT
Definition: CharacterSet.h:84
static const CharacterSet ALPHA
Definition: CharacterSet.h:76
static const CharacterSet HEXDIG
Definition: CharacterSet.h:88
unsigned short initial
std::vector< ClientDelayPool::Pointer > pools
static ClientDelayPools * Instance()
struct ClientHttpRequest::Out out
void clearRequest()
resets the current request and log_uri to nil
HttpRequest *const request
int64_t mRangeCLen() const
Definition: client_side.cc:764
ConnStateData * getConn() const
String rangeBoundaryStr() const
Definition: client_side.cc:804
void initRequest(HttpRequest *)
void setLogUriToRequestUri()
sets log_uri when we know the current request
void updateError(const Error &)
if necessary, stores new error information (if any)
MemObject * memObject() const
size_t req_sz
raw request size on input, not current request size
void setErrorUri(const char *)
HttpHdrRangeIter range_iter
struct ClientHttpRequest::Flags flags
bool multipartRangeRequest() const
Definition: client_side.cc:720
StoreEntry * storeEntry() const
ClientRequestContext * calloutContext
const LogTags & loggingTags() const
the processing tags associated with this request transaction.
const AccessLogEntry::Pointer al
access.log entry
StoreEntry * loggingEntry() const
struct ClientHttpRequest::Redirect redirect
void setWriteLimiter(const int aWriteSpeedLimit, const double anInitialBurst, const double aHighWatermark)
Definition: comm.cc:1376
Definition: ClpMap.h:41
const Value * get(const Key &)
Definition: ClpMap.h:178
void del(const Key &)
Remove the corresponding entry (if any)
Definition: ClpMap.h:258
bool add(const Key &, const Value &, Ttl)
Definition: ClpMap.h:210
static const Pointer & Current()
Definition: CodeContext.cc:33
static void Reset()
forgets the current context, setting it to nil/unknown
Definition: CodeContext.cc:77
AnyP::PortCfgPointer port
the configuration listening port this call relates to (may be nil)
Definition: CommCalls.h:100
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:83
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:85
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:82
Comm::ConnectionPointer conn
Definition: CommCalls.h:80
time_t timeLeft(const time_t idleTimeout) const
Definition: Connection.cc:146
CachePeer * getPeer() const
Definition: Connection.cc:124
Ip::Address remote
Definition: Connection.h:149
char rfc931[USER_IDENT_SZ]
Definition: Connection.h:176
Ip::Address local
Definition: Connection.h:146
Security::NegotiationHistory * tlsNegotiations()
Definition: Connection.cc:156
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:184
Comm::ConnectionPointer connection
to-server connection to be pinned
Definition: client_side.h:188
HttpRequest::Pointer request
to-server request that initiated serverConnection
Definition: client_side.h:189
noteTakeServerConnectionControl() callback parameter
Definition: client_side.h:214
Comm::ConnectionPointer conn_
to-server connection
Definition: client_side.h:225
SBuf preReadServerBytes
post-101 bytes received from the server
Definition: client_side.h:221
virtual int pipelinePrefetchMax() const
returning N allows a pipeline of 1+N requests (see pipeline_prefetch)
void postHttpsAccept()
the second part of old httpsAccept, waiting for future HttpsServer home
Ssl::ServerBump * serverBump()
Definition: client_side.h:286
void endingShutdown() override
bool fakeAConnectRequest(const char *reason, const SBuf &payload)
bool switchedToHttps() const
Definition: client_side.h:285
void readNextRequest()
Definition: client_side.cc:881
virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io)
Our close handler called by Comm when the pinned connection is closed.
char * prepareTlsSwitchingURL(const Http1::RequestParserPointer &hp)
Security::KeyLogger keyLogger
managers logging of the being-accepted TLS connection secrets
Definition: client_side.h:385
void afterClientRead() override
processing to be done after a Comm::Read()
Ssl::ServerBump * sslServerBump
HTTPS server cert. fetching state for bump-ssl-server-first.
Definition: client_side.h:494
Http::Stream * parseHttpRequest(const Http1::RequestParserPointer &)
virtual bool writeControlMsgAndCall(HttpReply *rep, AsyncCall::Pointer &call)=0
handle a control message received by context from a peer and call back
void switchToHttps(ClientHttpRequest *, Ssl::BumpMode bumpServerMode)
void startPinnedConnectionMonitoring()
const char * stoppedReceiving() const
true if we stopped receiving the request
Definition: client_side.h:160
void consumeInput(const size_t byteCount)
remove no longer needed leading bytes from the input buffer
void clientAfterReadingRequests()
ClientHttpRequest * buildFakeRequest(SBuf &useHost, AnyP::KnownPort usePort, const SBuf &payload)
build a fake http request
virtual void handleReply(HttpReply *header, StoreIOBuffer receivedData)=0
void setAuth(const Auth::UserRequest::Pointer &aur, const char *cause)
Definition: client_side.cc:518
virtual Http::Stream * parseOneRequest()=0
BodyPipe::Pointer bodyPipe
set when we are reading request body
Definition: client_side.h:432
const SBuf & tlsClientSni() const
Definition: client_side.h:295
void sendControlMsg(HttpControlMsg) override
called to send the 1xx message and notify the Source
void kick()
try to make progress on a transaction or read more I/O
Definition: client_side.cc:918
const Auth::UserRequest::Pointer & getAuth() const
Definition: client_side.h:124
void getSslContextStart()
Start to create dynamic Security::ContextPointer for host or uses static port SSL context.
void httpsPeeked(PinnedIdleContext pic)
called by FwdState when it is done bumping the server
bool initiateTunneledRequest(HttpRequest::Pointer const &cause, const char *reason, const SBuf &payload)
generates and sends to tunnel.cc a fake request with a given payload
void swanSong() override
Definition: client_side.cc:615
bool splice()
Splice a bumped client connection on peek-and-splice mode.
void start() override
called by AsyncStart; do not call directly
SBuf tlsClientSni_
TLS client delivered SNI value. Empty string if none has been received.
Definition: client_side.h:490
Error bareError
a problem that occurred without a request (e.g., while parsing headers)
Definition: client_side.h:382
bool handleReadData() override
void expectNoForwarding()
cleans up virgin request [body] forwarding state
Http::Stream * abortRequestParsing(const char *const errUri)
stop parsing the request and create context for relaying error info
void startShutdown() override
void sslCrtdHandleReply(const Helper::Reply &reply)
Process response from ssl_crtd.
err_type handleChunkedRequestBody()
parses available chunked encoded body bytes, checks size, returns errors
SBuf sslCommonName_
CN name for SSL certificate generation.
Definition: client_side.h:487
void resetSslCommonName(const char *name)
Definition: client_side.h:294
void afterClientWrite(size_t) override
processing to sync state after a Comm::Write()
void doneWithControlMsg() override
void terminateAll(const Error &, const LogTagsErrors &) override
abort any pending transactions and prevent new ones (by closing)
void abortChunkedRequestBody(const err_type error)
quit on errors related to chunked request body handling
void pinConnection(const Comm::ConnectionPointer &pinServerConn, const HttpRequest &request)
Forward future client requests using the given server connection.
bool preservingClientData_
whether preservedClientData is valid and should be kept up to date
Definition: client_side.h:435
void callException(const std::exception &) override
called when the job throws during an async call
Definition: client_side.cc:639
static void sslCrtdHandleReplyWrapper(void *data, const Helper::Reply &reply)
Callback function. It is called when squid receive message from ssl_crtd.
void lifetimeTimeout(const CommTimeoutCbParams &params)
Ssl::CertSignAlgorithm signAlgorithm
The signing algorithm to use.
Definition: client_side.h:495
Security::ContextPointer getTlsContextFromCache(const SBuf &cacheKey, const Ssl::CertificateProperties &certProperties)
AnyP::Port tlsConnectPort
The TLS server port number as passed in the CONNECT request.
Definition: client_side.h:486
Auth::UserRequest::Pointer auth_
some user details that can be used to perform authentication on this connection
Definition: client_side.h:474
Comm::ConnectionPointer borrowPinnedConnection(HttpRequest *, const AccessLogEntryPointer &)
ConnStateData-specific part of BorrowPinnedConnection()
struct ConnStateData::@37 flags
bool shouldPreserveClientData() const
void parseTlsHandshake()
void notePinnedConnectionBecameIdle(PinnedIdleContext pic)
Called when a pinned connection becomes available for forwarding the next request.
void stopPinnedConnectionMonitoring()
The caller assumes responsibility for connection closure detection.
void pinBusyConnection(const Comm::ConnectionPointer &pinServerConn, const HttpRequest::Pointer &request)
bool handleIdleClientPinnedTlsRead()
bool proxyProtocolError(const char *reason)
bool serveDelayedError(Http::Stream *)
SBuf preservedClientData
Definition: client_side.h:350
const char * stoppedSending_
the reason why we no longer write the response or nil
Definition: client_side.h:499
void resetReadTimeout(time_t timeout)
(re)sets timeout for receiving more bytes from the client
Definition: client_side.cc:598
void quitAfterError(HttpRequest *request)
SBuf tlsConnectHostOrIp
The TLS server host name appears in CONNECT request or the server ip address for the intercepted requ...
Definition: client_side.h:485
~ConnStateData() override
Definition: client_side.cc:671
void receivedFirstByte() override
Update flags and timeout after the first byte received.