client_side.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 33 Client-side Routines */
10 
60 #include "squid.h"
61 #include "acl/FilledChecklist.h"
62 #include "anyp/PortCfg.h"
63 #include "base/Subscription.h"
64 #include "base/TextException.h"
65 #include "CachePeer.h"
66 #include "client_db.h"
67 #include "client_side.h"
68 #include "client_side_reply.h"
69 #include "client_side_request.h"
70 #include "ClientRequestContext.h"
71 #include "clientStream.h"
72 #include "comm.h"
73 #include "comm/Connection.h"
74 #include "comm/Loops.h"
75 #include "comm/Read.h"
76 #include "comm/TcpAcceptor.h"
77 #include "comm/Write.h"
78 #include "CommCalls.h"
79 #include "DebugMessages.h"
81 #include "errorpage.h"
82 #include "fd.h"
83 #include "fde.h"
84 #include "fqdncache.h"
85 #include "FwdState.h"
86 #include "globals.h"
87 #include "helper.h"
88 #include "helper/Reply.h"
89 #include "http.h"
90 #include "http/one/RequestParser.h"
92 #include "http/Stream.h"
93 #include "HttpHdrContRange.h"
94 #include "HttpHeaderTools.h"
95 #include "HttpReply.h"
96 #include "HttpRequest.h"
97 #include "ident/Config.h"
98 #include "ident/Ident.h"
99 #include "internal.h"
100 #include "ipc/FdNotes.h"
101 #include "ipc/StartListening.h"
102 #include "log/access_log.h"
103 #include "MemBuf.h"
104 #include "MemObject.h"
105 #include "mime_header.h"
106 #include "parser/Tokenizer.h"
107 #include "proxyp/Header.h"
108 #include "proxyp/Parser.h"
109 #include "sbuf/Stream.h"
111 #include "security/Io.h"
112 #include "security/KeyLog.h"
114 #include "servers/forward.h"
115 #include "SquidConfig.h"
116 #include "SquidTime.h"
117 #include "StatCounters.h"
118 #include "StatHist.h"
119 #include "Store.h"
120 #include "TimeOrTag.h"
121 #include "tools.h"
122 
123 #if USE_AUTH
124 #include "auth/UserRequest.h"
125 #endif
126 #if USE_DELAY_POOLS
127 #include "ClientInfo.h"
128 #include "MessageDelayPools.h"
129 #endif
130 #if USE_OPENSSL
131 #include "ssl/bio.h"
132 #include "ssl/context_storage.h"
133 #include "ssl/gadgets.h"
134 #include "ssl/helper.h"
135 #include "ssl/ProxyCerts.h"
136 #include "ssl/ServerBump.h"
137 #include "ssl/support.h"
138 #endif
139 
140 // for tvSubUsec() which should be in SquidTime.h
141 #include "util.h"
142 
143 #include <climits>
144 #include <cmath>
145 #include <limits>
146 
147 #if HAVE_SYSTEMD_SD_DAEMON_H
148 #include <systemd/sd-daemon.h>
149 #endif
150 
153 {
154 public:
157  handler(aHandler), portCfg(aPortCfg), portTypeNote(note), sub(aSub) {}
158 
159  virtual void print(std::ostream &os) const {
160  startPrint(os) <<
161  ", " << FdNote(portTypeNote) << " port=" << (void*)&portCfg << ')';
162  }
163 
164  virtual bool canDial(AsyncCall &) const { return true; }
165  virtual void dial(AsyncCall &) { (handler)(portCfg, portTypeNote, sub); }
166 
167 public:
169 
170 private:
174 };
175 
176 static void clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub);
177 
179 #if USE_IDENT
181 #endif
182 static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength);
183 
184 static void clientUpdateStatHistCounters(const LogTags &logType, int svc_time);
185 static void clientUpdateStatCounters(const LogTags &logType);
187 static bool clientPingHasFinished(ping_data const *aPing);
190 
191 char *skipLeadingSpace(char *aString);
192 
193 #if USE_IDENT
194 static void
195 clientIdentDone(const char *ident, void *data)
196 {
197  ConnStateData *conn = (ConnStateData *)data;
198  xstrncpy(conn->clientConnection->rfc931, ident ? ident : dash_str, USER_IDENT_SZ);
199 }
200 #endif
201 
202 void
204 {
206 
207  if (logType.isTcpHit())
209 
210  if (logType.oldType == LOG_TCP_HIT)
212  else if (logType.oldType == LOG_TCP_MEM_HIT)
214 }
215 
216 void
217 clientUpdateStatHistCounters(const LogTags &logType, int svc_time)
218 {
227  switch (logType.oldType) {
228 
231  break;
232 
233  case LOG_TCP_INM_HIT:
234  case LOG_TCP_IMS_HIT:
236  break;
237 
238  case LOG_TCP_HIT:
239 
240  case LOG_TCP_MEM_HIT:
241 
242  case LOG_TCP_OFFLINE_HIT:
244  break;
245 
246  case LOG_TCP_MISS:
247 
250  break;
251 
252  default:
253  /* make compiler warnings go away */
254  break;
255  }
256 }
257 
258 bool
260 {
261  if (0 != aPing->stop.tv_sec && 0 != aPing->start.tv_sec)
262  return true;
263 
264  return false;
265 }
266 
267 void
269 {
270  ping_data *i;
271 
272  switch (someEntry->code) {
273 #if USE_CACHE_DIGESTS
274 
275  case CD_PARENT_HIT:
276 
277  case CD_SIBLING_HIT:
279  break;
280 #endif
281 
282  case SIBLING_HIT:
283 
284  case PARENT_HIT:
285 
286  case FIRST_PARENT_MISS:
287 
288  case CLOSEST_PARENT_MISS:
290  i = &someEntry->ping;
291 
292  if (clientPingHasFinished(i))
294 
295  if (i->timeout)
297 
298  break;
299 
300  case CLOSEST_PARENT:
301 
302  case CLOSEST_DIRECT:
304 
305  break;
306 
307  default:
308  break;
309  }
310 }
311 
312 void
314 {
316 
317  if (request->error)
319 
321  tvSubMsec(al->cache.start_time, current_time));
322 
324 }
325 
326 void
328 {
329  assert(request);
330  assert(aLogEntry != NULL);
331 
332  if (Config.onoff.log_mime_hdrs) {
333  MemBuf mb;
334  mb.init();
335  request->header.packInto(&mb);
336  //This is the request after adaptation or redirection
337  aLogEntry->headers.adapted_request = xstrdup(mb.buf);
338 
339  // the virgin request is saved to aLogEntry->request
340  if (aLogEntry->request) {
341  mb.reset();
342  aLogEntry->request->header.packInto(&mb);
343  aLogEntry->headers.request = xstrdup(mb.buf);
344  }
345 
346 #if USE_ADAPTATION
347  const Adaptation::History::Pointer ah = request->adaptLogHistory();
348  if (ah != NULL) {
349  mb.reset();
350  ah->lastMeta.packInto(&mb);
351  aLogEntry->adapt.last_meta = xstrdup(mb.buf);
352  }
353 #endif
354 
355  mb.clean();
356  }
357 
358 #if ICAP_CLIENT
359  const Adaptation::Icap::History::Pointer ih = request->icapHistory();
360  if (ih != NULL)
361  ih->processingTime(aLogEntry->icap.processingTime);
362 #endif
363 
364  aLogEntry->http.method = request->method;
365  aLogEntry->http.version = request->http_ver;
366  aLogEntry->hier = request->hier;
367  aLogEntry->cache.extuser = request->extacl_user.termedBuf();
368 
369  // Adapted request, if any, inherits and then collects all the stats, but
370  // the virgin request gets logged instead; copy the stats to log them.
371  // TODO: avoid losses by keeping these stats in a shared history object?
372  if (aLogEntry->request) {
373  aLogEntry->request->dnsWait = request->dnsWait;
374  aLogEntry->request->error = request->error;
375  }
376 }
377 
378 void
380 {
381  if (!out.size && loggingTags().oldType == LOG_TAG_NONE)
382  debugs(33, 5, "logging half-baked transaction: " << log_uri);
383 
384  al->icp.opcode = ICP_INVALID;
385  al->url = log_uri;
386  debugs(33, 9, "clientLogRequest: al.url='" << al->url << "'");
387 
388  const auto findReply = [this]() -> const HttpReply * {
389  if (al->reply)
390  return al->reply.getRaw();
391  if (const auto le = loggingEntry())
392  return le->hasFreshestReply();
393  return nullptr;
394  };
395  if (const auto reply = findReply()) {
396  al->http.code = reply->sline.status();
397  al->http.content_type = reply->content_type.termedBuf();
398  }
399 
400  debugs(33, 9, "clientLogRequest: http.code='" << al->http.code << "'");
401 
402  if (loggingEntry() && loggingEntry()->mem_obj && loggingEntry()->objectLen() >= 0)
403  al->cache.objectSize = loggingEntry()->contentLen(); // payload duplicate ?? with or without TE ?
404 
405  al->http.clientRequestSz.header = req_sz;
406  // the virgin request is saved to al->request
407  if (al->request && al->request->body_pipe)
408  al->http.clientRequestSz.payloadData = al->request->body_pipe->producedSize();
409  al->http.clientReplySz.header = out.headers_sz;
410  // XXX: calculate without payload encoding or headers !!
411  al->http.clientReplySz.payloadData = out.size - out.headers_sz; // pretend its all un-encoded data for now.
412 
413  al->cache.highOffset = out.offset;
414 
415  tvSub(al->cache.trTime, al->cache.start_time, current_time);
416 
417  if (request)
419 
420 #if USE_OPENSSL && 0
421 
422  /* This is broken. Fails if the connection has been closed. Needs
423  * to snarf the ssl details some place earlier..
424  */
425  if (getConn() != NULL)
426  al->cache.ssluser = sslGetUserEmail(fd_table[getConn()->fd].ssl);
427 
428 #endif
429 
430  /* Add notes (if we have a request to annotate) */
431  if (request) {
432  SBuf matched;
433  for (auto h: Config.notes) {
434  if (h->match(request, al->reply.getRaw(), al, matched)) {
435  request->notes()->add(h->key(), matched);
436  debugs(33, 3, h->key() << " " << matched);
437  }
438  }
439  // The al->notes and request->notes must point to the same object.
440  al->syncNotes(request);
441  }
442 
443  ACLFilledChecklist checklist(NULL, request, NULL);
444  if (al->reply) {
445  checklist.reply = al->reply.getRaw();
446  HTTPMSGLOCK(checklist.reply);
447  }
448 
449  if (request) {
450  HTTPMSGUNLOCK(al->adapted_request);
451  al->adapted_request = request;
452  HTTPMSGLOCK(al->adapted_request);
453  }
454  // no need checklist.syncAle(): already synced
455  checklist.al = al;
456  accessLogLog(al, &checklist);
457 
458  bool updatePerformanceCounters = true;
461  statsCheck.al = al;
462  if (al->reply) {
463  statsCheck.reply = al->reply.getRaw();
464  HTTPMSGLOCK(statsCheck.reply);
465  }
466  updatePerformanceCounters = statsCheck.fastCheck().allowed();
467  }
468 
469  if (updatePerformanceCounters) {
470  if (request)
471  updateCounters();
472 
473  if (getConn() != NULL && getConn()->clientConnection != NULL)
475  }
476 }
477 
478 void
480 {
481  safe_free(uri);
484  clearRequest();
485 
486  if (client_stream.tail)
488 }
489 
490 void
491 httpRequestFree(void *data)
492 {
493  ClientHttpRequest *http = (ClientHttpRequest *)data;
494  assert(http != NULL);
495  delete http;
496 }
497 
498 /* This is a handler normally called by comm_close() */
500 {
501  if (clientConnection) {
503  // keep closed clientConnection for logging, clientdb cleanup, etc.
504  }
505  deleteThis("ConnStateData::connStateClosed");
506 }
507 
508 #if USE_AUTH
509 void
511 {
512  if (auth_ == NULL) {
513  if (aur != NULL) {
514  debugs(33, 2, "Adding connection-auth to " << clientConnection << " from " << by);
515  auth_ = aur;
516  }
517  return;
518  }
519 
520  // clobered with self-pointer
521  // NP: something nasty is going on in Squid, but harmless.
522  if (aur == auth_) {
523  debugs(33, 2, "WARNING: Ignoring duplicate connection-auth for " << clientConnection << " from " << by);
524  return;
525  }
526 
527  /*
528  * Connection-auth relies on a single set of credentials being preserved
529  * for all requests on a connection once they have been setup.
530  * There are several things which need to happen to preserve security
531  * when connection-auth credentials change unexpectedly or are unset.
532  *
533  * 1) auth helper released from any active state
534  *
535  * They can only be reserved by a handshake process which this
536  * connection can now never complete.
537  * This prevents helpers hanging when their connections close.
538  *
539  * 2) pinning is expected to be removed and server conn closed
540  *
541  * The upstream link is authenticated with the same credentials.
542  * Expecting the same level of consistency we should have received.
543  * This prevents upstream being faced with multiple or missing
544  * credentials after authentication.
545  * NP: un-pin is left to the cleanup in ConnStateData::swanSong()
546  * we just trigger that cleanup here via comm_reset_close() or
547  * ConnStateData::stopReceiving()
548  *
549  * 3) the connection needs to close.
550  *
551  * This prevents attackers injecting requests into a connection,
552  * or gateways wrongly multiplexing users into a single connection.
553  *
554  * When credentials are missing closure needs to follow an auth
555  * challenge for best recovery by the client.
556  *
557  * When credentials change there is nothing we can do but abort as
558  * fast as possible. Sending TCP RST instead of an HTTP response
559  * is the best-case action.
560  */
561 
562  // clobbered with nul-pointer
563  if (aur == NULL) {
564  debugs(33, 2, "WARNING: Graceful closure on " << clientConnection << " due to connection-auth erase from " << by);
566  auth_ = NULL;
567  // XXX: need to test whether the connection re-auth challenge is sent. If not, how to trigger it from here.
568  // NP: the current situation seems to fix challenge loops in Safari without visible issues in others.
569  // we stop receiving more traffic but can leave the Job running to terminate after the error or challenge is delivered.
570  stopReceiving("connection-auth removed");
571  return;
572  }
573 
574  // clobbered with alternative credentials
575  if (aur != auth_) {
576  debugs(33, 2, "ERROR: Closing " << clientConnection << " due to change of connection-auth from " << by);
578  auth_ = NULL;
579  // this is a fatal type of problem.
580  // Close the connection immediately with TCP RST to abort all traffic flow
582  return;
583  }
584 
585  /* NOT REACHABLE */
586 }
587 #endif
588 
589 void
590 ConnStateData::resetReadTimeout(const time_t timeout)
591 {
593  AsyncCall::Pointer callback = JobCallback(33, 5, TimeoutDialer, this, ConnStateData::requestTimeout);
594  commSetConnTimeout(clientConnection, timeout, callback);
595 }
596 
597 void
599 {
601  AsyncCall::Pointer callback = JobCallback(5, 4, TimeoutDialer, this, ConnStateData::lifetimeTimeout);
603 }
604 
605 // cleans up before destructor is called
606 void
608 {
609  debugs(33, 2, HERE << clientConnection);
610 
611  flags.readMore = false;
612  clientdbEstablished(clientConnection->remote, -1); /* decrement */
613 
615  checkLogging();
616 
617  // XXX: Closing pinned conn is too harsh: The Client may want to continue!
618  unpinConnection(true);
619 
621 
622 #if USE_AUTH
623  // NP: do this bit after closing the connections to avoid side effects from unwanted TCP RST
624  setAuth(NULL, "ConnStateData::SwanSong cleanup");
625 #endif
626 
627  flags.swanSang = true;
628 }
629 
630 void
631 ConnStateData::callException(const std::exception &ex)
632 {
633  Server::callException(ex); // logs ex and stops the job
634 
635  ErrorDetail::Pointer errorDetail;
636  if (const auto tex = dynamic_cast<const TextException*>(&ex))
637  errorDetail = new ExceptionErrorDetail(tex->id());
638  else
639  errorDetail = new ExceptionErrorDetail(Here().id());
640  updateError(ERR_GATEWAY_FAILURE, errorDetail);
641 }
642 
643 void
645 {
646  if (const auto context = pipeline.front()) {
647  const auto http = context->http;
648  assert(http);
649  http->updateError(error);
650  } else {
652  }
653 }
654 
655 bool
657 {
658  return cbdataReferenceValid(this) && // XXX: checking "this" in a method
660  !fd_table[clientConnection->fd].closing();
661 }
662 
664 {
665  debugs(33, 3, HERE << clientConnection);
666 
667  if (isOpen())
668  debugs(33, DBG_IMPORTANT, "BUG: ConnStateData did not close " << clientConnection);
669 
670  if (!flags.swanSang)
671  debugs(33, DBG_IMPORTANT, "BUG: ConnStateData was not destroyed properly; " << clientConnection);
672 
673  if (bodyPipe != NULL)
674  stopProducingFor(bodyPipe, false);
675 
676  delete bodyParser; // TODO: pool
677 
678 #if USE_OPENSSL
679  delete sslServerBump;
680 #endif
681 }
682 
689 void
691 {
692  HttpRequest *request = http->request;
693 
694  debugs(33, 3, "http_ver = " << request->http_ver);
695  debugs(33, 3, "method = " << request->method);
696 
697  // TODO: move to HttpRequest::hdrCacheInit, just like HttpReply.
698  request->flags.proxyKeepalive = request->persistent();
699 }
700 
701 int
703 {
705  bodyLength > Config.maxRequestBodySize)
706  return 1; /* too large */
707 
708  return 0;
709 }
710 
711 bool
713 {
714  return request->multipartRangeRequest();
715 }
716 
717 void
719 {
720  mb->appendf("\r\n--" SQUIDSTRINGPH "--\r\n", SQUIDSTRINGPRINT(boundary));
721  debugs(33, 6, "buf offset: " << mb->size);
722 }
723 
724 void
725 clientPackRangeHdr(const HttpReplyPointer &rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb)
726 {
727  HttpHeader hdr(hoReply);
728  assert(rep);
729  assert(spec);
730 
731  /* put boundary */
732  debugs(33, 5, "appending boundary: " << boundary);
733  /* rfc2046 requires to _prepend_ boundary with <crlf>! */
734  mb->appendf("\r\n--" SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(boundary));
735 
736  /* stuff the header with required entries and pack it */
737 
738  if (rep->header.has(Http::HdrType::CONTENT_TYPE))
740 
741  httpHeaderAddContRange(&hdr, *spec, rep->content_length);
742 
743  hdr.packInto(mb);
744  hdr.clean();
745 
746  /* append <crlf> (we packed a header, not a reply) */
747  mb->append("\r\n", 2);
748 }
749 
755 int64_t
757 {
758  int64_t clen = 0;
759  MemBuf mb;
760 
761  assert(memObject());
762 
763  mb.init();
765 
766  while (pos != request->range->end()) {
767  /* account for headers for this range */
768  mb.reset();
769  clientPackRangeHdr(&storeEntry()->mem().freshestReply(),
770  *pos, range_iter.boundary, &mb);
771  clen += mb.size;
772 
773  /* account for range content */
774  clen += (*pos)->length;
775 
776  debugs(33, 6, "clientMRangeCLen: (clen += " << mb.size << " + " << (*pos)->length << ") == " << clen);
777  ++pos;
778  }
779 
780  /* account for the terminating boundary */
781  mb.reset();
782 
784 
785  clen += mb.size;
786 
787  mb.clean();
788 
789  return clen;
790 }
791 
795 String
797 {
798  const char *key;
799  String b(APP_FULLNAME);
800  b.append(":",1);
801  key = storeEntry()->getMD5Text();
802  b.append(key, strlen(key));
803  return b;
804 }
805 
815 void
817  HttpReply * rep, StoreIOBuffer receivedData)
818 {
819  // do not try to deliver if client already ABORTED
820  if (!http->getConn() || !cbdataReferenceValid(http->getConn()) || !Comm::IsConnOpen(http->getConn()->clientConnection))
821  return;
822 
823  /* Test preconditions */
824  assert(node != NULL);
825  /* TODO: handle this rather than asserting
826  * - it should only ever happen if we cause an abort and
827  * the callback chain loops back to here, so we can simply return.
828  * However, that itself shouldn't happen, so it stays as an assert for now.
829  */
831  assert(node->node.next == NULL);
832  Http::StreamPointer context = dynamic_cast<Http::Stream *>(node->data.getRaw());
833  assert(context != NULL);
834 
835  /* TODO: check offset is what we asked for */
836 
837  // TODO: enforces HTTP/1 MUST on pipeline order, but is irrelevant to HTTP/2
838  if (context != http->getConn()->pipeline.front())
839  context->deferRecipientForLater(node, rep, receivedData);
840  else if (http->getConn()->cbControlMsgSent) // 1xx to the user is pending
841  context->deferRecipientForLater(node, rep, receivedData);
842  else
843  http->getConn()->handleReply(rep, receivedData);
844 }
845 
851 void
853 {
854  /* Test preconditions */
855  assert(node != NULL);
856  /* TODO: handle this rather than asserting
857  * - it should only ever happen if we cause an abort and
858  * the callback chain loops back to here, so we can simply return.
859  * However, that itself shouldn't happen, so it stays as an assert for now.
860  */
862  /* Set null by ContextFree */
863  assert(node->node.next == NULL);
864  /* this is the assert discussed above */
865  assert(NULL == dynamic_cast<Http::Stream *>(node->data.getRaw()));
866  /* We are only called when the client socket shutsdown.
867  * Tell the prev pipeline member we're finished
868  */
869  clientStreamDetach(node, http);
870 }
871 
872 void
874 {
875  debugs(33, 5, HERE << clientConnection << " reading next req");
876 
877  fd_note(clientConnection->fd, "Idle client: Waiting for next request");
882 
883  readSomeData();
885 }
886 
887 static void
889 {
890  debugs(33, 2, HERE << conn->clientConnection << " Sending next");
891 
894  if (deferredRequest->flags.deferred) {
896  assert(deferredRequest->http->out.size == 0);
898  clientSocketRecipient(deferredRequest->deferredparams.node,
899  deferredRequest->http,
900  deferredRequest->deferredparams.rep,
901  deferredRequest->deferredparams.queuedBuffer);
902  }
903 
907 }
908 
909 void
911 {
913  debugs(33, 2, clientConnection << " Connection was closed");
914  return;
915  }
916 
917  if (pinning.pinned && !Comm::IsConnOpen(pinning.serverConnection)) {
918  debugs(33, 2, clientConnection << " Connection was pinned but server side gone. Terminating client connection");
920  return;
921  }
922 
937  if (const char *reason = stoppedReceiving()) {
938  debugs(33, 3, "closing for earlier request error: " << reason);
940  return;
941  }
942 
953  if (clientParseRequests()) {
954  debugs(33, 3, clientConnection << ": parsed next request from buffer");
955  }
956 
965  debugs(33, 3, "half-closed client with no pending requests, closing");
967  return;
968  }
969 
977  Http::StreamPointer deferredRequest = pipeline.front();
978  if (deferredRequest != nullptr) {
979  debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded");
980  ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
981  } else if (flags.readMore) {
982  debugs(33, 3, clientConnection << ": calling readNextRequest()");
983  readNextRequest();
984  } else {
985  // XXX: Can this happen? CONNECT tunnels have deferredRequest set.
986  debugs(33, DBG_IMPORTANT, MYNAME << "abandoning " << clientConnection);
987  }
988 }
989 
990 void
992 {
993  debugs(33, 4, HERE << "sending error (" << clientConnection << "): " << error <<
994  "; old receiving error: " <<
995  (stoppedReceiving() ? stoppedReceiving_ : "none"));
996 
997  if (const char *oldError = stoppedSending()) {
998  debugs(33, 3, HERE << "already stopped sending: " << oldError);
999  return; // nothing has changed as far as this connection is concerned
1000  }
1002 
1003  if (!stoppedReceiving()) {
1004  if (const int64_t expecting = mayNeedToReadMoreBody()) {
1005  debugs(33, 5, HERE << "must still read " << expecting <<
1006  " request body bytes with " << inBuf.length() << " unused");
1007  return; // wait for the request receiver to finish reading
1008  }
1009  }
1010 
1012 }
1013 
1014 void
1016 {
1017  if (pipeline.empty())
1018  return;
1019 
1020  auto ctx = pipeline.front();
1021  if (size) {
1023  if (ctx->http->loggingTags().isTcpHit())
1025  }
1026  ctx->writeComplete(size);
1027 }
1028 
1029 Http::Stream *
1031 {
1032  ClientHttpRequest *http = new ClientHttpRequest(this);
1033  http->req_sz = inBuf.length();
1034  http->setErrorUri(uri);
1035  auto *context = new Http::Stream(clientConnection, http);
1036  StoreIOBuffer tempBuffer;
1037  tempBuffer.data = context->reqbuf;
1038  tempBuffer.length = HTTP_REQBUF_SZ;
1041  clientSocketDetach, context, tempBuffer);
1042  return context;
1043 }
1044 
1045 void
1047 {
1048  // RegisteredRunner API callback - Squid has been shut down
1049 
1050  // if connection is idle terminate it now,
1051  // otherwise wait for grace period to end
1052  if (pipeline.empty())
1053  endingShutdown();
1054 }
1055 
1056 void
1058 {
1059  // RegisteredRunner API callback - Squid shutdown grace period is over
1060 
1061  // force the client connection to close immediately
1062  // swanSong() in the close handler will cleanup.
1065 }
1066 
1067 char *
1068 skipLeadingSpace(char *aString)
1069 {
1070  char *result = aString;
1071 
1072  while (xisspace(*aString))
1073  ++aString;
1074 
1075  return result;
1076 }
1077 
1083 const char *
1084 findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end)
1085 {
1086  if (NULL == end) {
1087  end = uriAndHTTPVersion + strcspn(uriAndHTTPVersion, "\r\n");
1088  assert(end);
1089  }
1090 
1091  for (; end > uriAndHTTPVersion; --end) {
1092  if (*end == '\n' || *end == '\r')
1093  continue;
1094 
1095  if (xisspace(*end)) {
1096  if (strncasecmp(end + 1, "HTTP/", 5) == 0)
1097  return end + 1;
1098  else
1099  break;
1100  }
1101  }
1102 
1103  return NULL;
1104 }
1105 
1106 static char *
1108 {
1109  int vhost = conn->port->vhost;
1110  int vport = conn->port->vport;
1111  static char ipbuf[MAX_IPSTRLEN];
1112 
1113  /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1114 
1115  static const SBuf cache_object("cache_object://");
1116  if (hp->requestUri().startsWith(cache_object))
1117  return nullptr; /* already in good shape */
1118 
1119  // XXX: re-use proper URL parser for this
1120  SBuf url = hp->requestUri(); // use full provided URI if we abort
1121  do { // use a loop so we can break out of it
1122  ::Parser::Tokenizer tok(url);
1123  if (tok.skip('/')) // origin-form URL already.
1124  break;
1125 
1126  if (conn->port->vhost)
1127  return nullptr; /* already in good shape */
1128 
1129  // skip the URI scheme
1130  static const CharacterSet uriScheme = CharacterSet("URI-scheme","+-.") + CharacterSet::ALPHA + CharacterSet::DIGIT;
1131  static const SBuf uriSchemeEnd("://");
1132  if (!tok.skipAll(uriScheme) || !tok.skip(uriSchemeEnd))
1133  break;
1134 
1135  // skip the authority segment
1136  // RFC 3986 complex nested ABNF for "authority" boils down to this:
1137  static const CharacterSet authority = CharacterSet("authority","-._~%:@[]!$&'()*+,;=") +
1139  if (!tok.skipAll(authority))
1140  break;
1141 
1142  static const SBuf slashUri("/");
1143  const SBuf t = tok.remaining();
1144  if (t.isEmpty())
1145  url = slashUri;
1146  else if (t[0]=='/') // looks like path
1147  url = t;
1148  else if (t[0]=='?' || t[0]=='#') { // looks like query or fragment. fix '/'
1149  url = slashUri;
1150  url.append(t);
1151  } // else do nothing. invalid path
1152 
1153  } while(false);
1154 
1155 #if SHOULD_REJECT_UNKNOWN_URLS
1156  // reject URI which are not well-formed even after the processing above
1157  if (url.isEmpty() || url[0] != '/') {
1158  hp->parseStatusCode = Http::scBadRequest;
1159  return conn->abortRequestParsing("error:invalid-request");
1160  }
1161 #endif
1162 
1163  if (vport < 0)
1164  vport = conn->clientConnection->local.port();
1165 
1166  char *receivedHost = nullptr;
1167  if (vhost && (receivedHost = hp->getHostHeaderField())) {
1168  SBuf host(receivedHost);
1169  debugs(33, 5, "ACCEL VHOST REWRITE: vhost=" << host << " + vport=" << vport);
1170  if (vport > 0) {
1171  // remove existing :port (if any), cope with IPv6+ without port
1172  const auto lastColonPos = host.rfind(':');
1173  if (lastColonPos != SBuf::npos && *host.rbegin() != ']') {
1174  host.chop(0, lastColonPos); // truncate until the last colon
1175  }
1176  host.appendf(":%d", vport);
1177  } // else nothing to alter port-wise.
1178  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1179  const auto url_sz = scheme.length() + host.length() + url.length() + 32;
1180  char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1181  snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH SQUIDSBUFPH, SQUIDSBUFPRINT(scheme), SQUIDSBUFPRINT(host), SQUIDSBUFPRINT(url));
1182  debugs(33, 5, "ACCEL VHOST REWRITE: " << uri);
1183  return uri;
1184  } else if (conn->port->defaultsite /* && !vhost */) {
1185  debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: defaultsite=" << conn->port->defaultsite << " + vport=" << vport);
1186  char vportStr[32];
1187  vportStr[0] = '\0';
1188  if (vport > 0) {
1189  snprintf(vportStr, sizeof(vportStr),":%d",vport);
1190  }
1191  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1192  const int url_sz = scheme.length() + strlen(conn->port->defaultsite) + sizeof(vportStr) + url.length() + 32;
1193  char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1194  snprintf(uri, url_sz, SQUIDSBUFPH "://%s%s" SQUIDSBUFPH,
1195  SQUIDSBUFPRINT(scheme), conn->port->defaultsite, vportStr, SQUIDSBUFPRINT(url));
1196  debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: " << uri);
1197  return uri;
1198  } else if (vport > 0 /* && (!vhost || no Host:) */) {
1199  debugs(33, 5, "ACCEL VPORT REWRITE: *_port IP + vport=" << vport);
1200  /* Put the local socket IP address as the hostname, with whatever vport we found */
1201  conn->clientConnection->local.toHostStr(ipbuf,MAX_IPSTRLEN);
1202  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1203  const int url_sz = scheme.length() + sizeof(ipbuf) + url.length() + 32;
1204  char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1205  snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1206  SQUIDSBUFPRINT(scheme), ipbuf, vport, SQUIDSBUFPRINT(url));
1207  debugs(33, 5, "ACCEL VPORT REWRITE: " << uri);
1208  return uri;
1209  }
1210 
1211  return nullptr;
1212 }
1213 
1214 static char *
1216 {
1217  char *uri = nullptr;
1218  /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1219  if (const char *host = hp->getHostHeaderField()) {
1220  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1221  const int url_sz = scheme.length() + strlen(host) + hp->requestUri().length() + 32;
1222  uri = static_cast<char *>(xcalloc(url_sz, 1));
1223  snprintf(uri, url_sz, SQUIDSBUFPH "://%s" SQUIDSBUFPH,
1224  SQUIDSBUFPRINT(scheme),
1225  host,
1226  SQUIDSBUFPRINT(hp->requestUri()));
1227  }
1228  return uri;
1229 }
1230 
1231 char *
1233 {
1234  Must(switchedToHttps());
1235 
1236  if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1237  return nullptr; /* already in good shape */
1238 
1239  char *uri = buildUrlFromHost(this, hp);
1240 #if USE_OPENSSL
1241  if (!uri) {
1244  SBuf useHost;
1245  if (!tlsClientSni().isEmpty())
1246  useHost = tlsClientSni();
1247  else
1248  useHost = tlsConnectHostOrIp;
1249 
1250  const SBuf &scheme = AnyP::UriScheme(transferProtocol.protocol).image();
1251  const int url_sz = scheme.length() + useHost.length() + hp->requestUri().length() + 32;
1252  uri = static_cast<char *>(xcalloc(url_sz, 1));
1253  snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH ":%d" SQUIDSBUFPH,
1254  SQUIDSBUFPRINT(scheme),
1255  SQUIDSBUFPRINT(useHost),
1257  SQUIDSBUFPRINT(hp->requestUri()));
1258  }
1259 #endif
1260  if (uri)
1261  debugs(33, 5, "TLS switching host rewrite: " << uri);
1262  return uri;
1263 }
1264 
1265 static char *
1267 {
1268  // TODO Must() on URI !empty when the parser supports throw. For now avoid assert().
1269  if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1270  return nullptr; /* already in good shape */
1271 
1272  char *uri = buildUrlFromHost(conn, hp);
1273  if (!uri) {
1274  /* Put the local socket IP address as the hostname. */
1275  static char ipbuf[MAX_IPSTRLEN];
1276  conn->clientConnection->local.toHostStr(ipbuf,MAX_IPSTRLEN);
1277  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1278  const int url_sz = sizeof(ipbuf) + hp->requestUri().length() + 32;
1279  uri = static_cast<char *>(xcalloc(url_sz, 1));
1280  snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1281  SQUIDSBUFPRINT(scheme),
1282  ipbuf, conn->clientConnection->local.port(), SQUIDSBUFPRINT(hp->requestUri()));
1283  }
1284 
1285  if (uri)
1286  debugs(33, 5, "TRANSPARENT REWRITE: " << uri);
1287  return uri;
1288 }
1289 
1290 Http::Stream *
1292 {
1293  /* Attempt to parse the first line; this will define where the method, url, version and header begin */
1294  {
1295  Must(hp);
1296 
1299 
1300  const bool parsedOk = hp->parse(inBuf);
1301 
1302  // sync the buffers after parsing.
1303  inBuf = hp->remaining();
1304 
1305  if (hp->needsMoreData()) {
1306  debugs(33, 5, "Incomplete request, waiting for end of request line");
1307  return NULL;
1308  }
1309 
1310  if (!parsedOk) {
1311  const bool tooBig =
1312  hp->parseStatusCode == Http::scRequestHeaderFieldsTooLarge ||
1313  hp->parseStatusCode == Http::scUriTooLong;
1314  auto result = abortRequestParsing(
1315  tooBig ? "error:request-too-large" : "error:invalid-request");
1316  // assume that remaining leftovers belong to this bad request
1317  if (!inBuf.isEmpty())
1319  return result;
1320  }
1321  }
1322 
1323  /* We know the whole request is in parser now */
1324  debugs(11, 2, "HTTP Client " << clientConnection);
1325  debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
1326  hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol() << "\n" <<
1327  hp->mimeHeader() <<
1328  "\n----------");
1329 
1330  /* deny CONNECT via accelerated ports */
1331  if (hp->method() == Http::METHOD_CONNECT && port != NULL && port->flags.accelSurrogate) {
1332  debugs(33, DBG_IMPORTANT, "WARNING: CONNECT method received on " << transferProtocol << " Accelerator port " << port->s.port());
1333  debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1334  hp->parseStatusCode = Http::scMethodNotAllowed;
1335  return abortRequestParsing("error:method-not-allowed");
1336  }
1337 
1338  /* RFC 7540 section 11.6 registers the method PRI as HTTP/2 specific
1339  * Deny "PRI" method if used in HTTP/1.x or 0.9 versions.
1340  * If seen it signals a broken client or proxy has corrupted the traffic.
1341  */
1342  if (hp->method() == Http::METHOD_PRI && hp->messageProtocol() < Http::ProtocolVersion(2,0)) {
1343  debugs(33, DBG_IMPORTANT, "WARNING: PRI method received on " << transferProtocol << " port " << port->s.port());
1344  debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1345  hp->parseStatusCode = Http::scMethodNotAllowed;
1346  return abortRequestParsing("error:method-not-allowed");
1347  }
1348 
1349  if (hp->method() == Http::METHOD_NONE) {
1350  debugs(33, DBG_IMPORTANT, "WARNING: Unsupported method: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1351  hp->parseStatusCode = Http::scMethodNotAllowed;
1352  return abortRequestParsing("error:unsupported-request-method");
1353  }
1354 
1355  // Process headers after request line
1356  debugs(33, 3, "complete request received. " <<
1357  "prefix_sz = " << hp->messageHeaderSize() <<
1358  ", request-line-size=" << hp->firstLineSize() <<
1359  ", mime-header-size=" << hp->headerBlockSize() <<
1360  ", mime header block:\n" << hp->mimeHeader() << "\n----------");
1361 
1362  /* Ok, all headers are received */
1363  ClientHttpRequest *http = new ClientHttpRequest(this);
1364 
1365  http->req_sz = hp->messageHeaderSize();
1366  Http::Stream *result = new Http::Stream(clientConnection, http);
1367 
1368  StoreIOBuffer tempBuffer;
1369  tempBuffer.data = result->reqbuf;
1370  tempBuffer.length = HTTP_REQBUF_SZ;
1371 
1372  ClientStreamData newServer = new clientReplyContext(http);
1373  ClientStreamData newClient = result;
1376  clientSocketDetach, newClient, tempBuffer);
1377 
1378  /* set url */
1379  debugs(33,5, "Prepare absolute URL from " <<
1380  (transparent()?"intercept":(port->flags.accelSurrogate ? "accel":"")));
1381  /* Rewrite the URL in transparent or accelerator mode */
1382  /* NP: there are several cases to traverse here:
1383  * - standard mode (forward proxy)
1384  * - transparent mode (TPROXY)
1385  * - transparent mode with failures
1386  * - intercept mode (NAT)
1387  * - intercept mode with failures
1388  * - accelerator mode (reverse proxy)
1389  * - internal relative-URL
1390  * - mixed combos of the above with internal URL
1391  * - remote interception with PROXY protocol
1392  * - remote reverse-proxy with PROXY protocol
1393  */
1394  if (switchedToHttps()) {
1395  http->uri = prepareTlsSwitchingURL(hp);
1396  } else if (transparent()) {
1397  /* intercept or transparent mode, properly working with no failures */
1398  http->uri = prepareTransparentURL(this, hp);
1399 
1400  } else if (internalCheck(hp->requestUri())) { // NP: only matches relative-URI
1401  /* internal URL mode */
1402  /* prepend our name & port */
1403  http->uri = xstrdup(internalLocalUri(NULL, hp->requestUri()));
1404  // We just re-wrote the URL. Must replace the Host: header.
1405  // But have not parsed there yet!! flag for local-only handling.
1406  http->flags.internal = true;
1407 
1408  } else if (port->flags.accelSurrogate) {
1409  /* accelerator mode */
1410  http->uri = prepareAcceleratedURL(this, hp);
1411  http->flags.accel = true;
1412  }
1413 
1414  if (!http->uri) {
1415  /* No special rewrites have been applied above, use the
1416  * requested url. may be rewritten later, so make extra room */
1417  int url_sz = hp->requestUri().length() + Config.appendDomainLen + 5;
1418  http->uri = (char *)xcalloc(url_sz, 1);
1419  SBufToCstring(http->uri, hp->requestUri());
1420  }
1421 
1422  result->flags.parsed_ok = 1;
1423  return result;
1424 }
1425 
1426 bool
1428 {
1429  if (pipeline.empty() && inBuf.isEmpty()) {
1430  debugs(33, 4, "yes, without active requests and unparsed input");
1431  return true;
1432  }
1433 
1435  debugs(33, 3, "yes, without half_closed_clients");
1436  return true;
1437  }
1438 
1439  // Squid currently tries to parse (possibly again) a partially received
1440  // request after an EOF with half_closed_clients. To give that last parse in
1441  // afterClientRead() a chance, we ignore partially parsed requests here.
1442  debugs(33, 3, "no, honoring half_closed_clients");
1443  return false;
1444 }
1445 
1446 void
1447 ConnStateData::consumeInput(const size_t byteCount)
1448 {
1449  assert(byteCount > 0 && byteCount <= inBuf.length());
1450  inBuf.consume(byteCount);
1451  debugs(33, 5, "inBuf has " << inBuf.length() << " unused bytes");
1452 }
1453 
1454 void
1456 {
1457  // Were we expecting to read more request body from half-closed connection?
1459  debugs(33, 3, HERE << "truncated body: closing half-closed " << clientConnection);
1461  return;
1462  }
1463 
1464  if (flags.readMore)
1465  readSomeData();
1466 }
1467 
1468 void
1470 {
1471  // From HTTP p.o.v., we do not have to close after every error detected
1472  // at the client-side, but many such errors do require closure and the
1473  // client-side code is bad at handling errors so we play it safe.
1474  if (request)
1475  request->flags.proxyKeepalive = false;
1476  flags.readMore = false;
1477  debugs(33,4, HERE << "Will close after error: " << clientConnection);
1478 }
1479 
1480 #if USE_OPENSSL
1482 {
1483  ClientHttpRequest *http = context->http;
1484 
1485  if (!sslServerBump)
1486  return false;
1487 
1489  // Did we create an error entry while processing CONNECT?
1490  if (!sslServerBump->entry->isEmpty()) {
1491  quitAfterError(http->request);
1492 
1493  // Get the saved error entry and send it to the client by replacing the
1494  // ClientHttpRequest store entry with it.
1496  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1497  assert(repContext);
1498  debugs(33, 5, "Responding with delated error for " << http->uri);
1499  repContext->setReplyToStoreEntry(sslServerBump->entry, "delayed SslBump error");
1500 
1501  // Get error details from the fake certificate-peeking request.
1503  context->pullData();
1504  return true;
1505  }
1506 
1507  // In bump-server-first mode, we have not necessarily seen the intended
1508  // server name at certificate-peeking time. Check for domain mismatch now,
1509  // when we can extract the intended name from the bumped HTTP request.
1510  if (const Security::CertPointer &srvCert = sslServerBump->serverCert) {
1511  HttpRequest *request = http->request;
1512  if (!Ssl::checkX509ServerValidity(srvCert.get(), request->url.host())) {
1513  debugs(33, 2, "SQUID_X509_V_ERR_DOMAIN_MISMATCH: Certificate " <<
1514  "does not match domainname " << request->url.host());
1515 
1516  bool allowDomainMismatch = false;
1520  clientAclChecklistFill(check, http);
1521  allowDomainMismatch = check.fastCheck().allowed();
1522  delete check.sslErrors;
1523  check.sslErrors = NULL;
1524  }
1525 
1526  if (!allowDomainMismatch) {
1528 
1530  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1531  assert (repContext);
1532 
1533  request->hier = sslServerBump->request->hier;
1534 
1535  // Create an error object and fill it
1537  err->src_addr = clientConnection->remote;
1540  srvCert, nullptr);
1542  repContext->setReplyToError(request->method, err);
1543  assert(context->http->out.offset == 0);
1544  context->pullData();
1545  return true;
1546  }
1547  }
1548  }
1549 
1550  return false;
1551 }
1552 #endif // USE_OPENSSL
1553 
1555 bool
1557 {
1559  debugs(33, 5, "disabled; send error: " << requestError);
1560  return false;
1561  }
1562 
1563  if (!preservingClientData_) {
1564  debugs(33, 3, "may have forgotten client data; send error: " << requestError);
1565  return false;
1566  }
1567 
1569  checklist.requestErrorType = requestError;
1570  fillChecklist(checklist);
1571  auto answer = checklist.fastCheck();
1572  if (answer.allowed() && answer.kind == 1) {
1573  debugs(33, 3, "Request will be tunneled to server");
1574  const auto context = pipeline.front();
1575  const auto http = context ? context->http : nullptr;
1576  const auto request = http ? http->request : nullptr;
1577  if (context)
1578  context->finished(); // Will remove from pipeline queue
1581  }
1582  debugs(33, 3, "denied; send error: " << requestError);
1583  return false;
1584 }
1585 
1586 void
1588 {
1589  /*
1590  * DPW 2007-05-18
1591  * Moved the TCP_RESET feature from clientReplyContext::sendMoreData
1592  * to here because calling comm_reset_close() causes http to
1593  * be freed before accessing.
1594  */
1595  if (request != NULL && request->flags.resetTcp && Comm::IsConnOpen(conn->clientConnection)) {
1596  debugs(33, 3, HERE << "Sending TCP RST on " << conn->clientConnection);
1597  conn->flags.readMore = false;
1598  comm_reset_close(conn->clientConnection);
1599  }
1600 }
1601 
1602 void
1604 {
1605  ClientHttpRequest *http = context->http;
1606  bool mustReplyToOptions = false;
1607  bool expectBody = false;
1608 
1609  // We already have the request parsed and checked, so we
1610  // only need to go through the final body/conn setup to doCallouts().
1611  assert(http->request);
1613 
1614  // temporary hack to avoid splitting this huge function with sensitive code
1615  const bool isFtp = !hp;
1616 
1617  // Some blobs below are still HTTP-specific, but we would have to rewrite
1618  // this entire function to remove them from the FTP code path. Connection
1619  // setup and body_pipe preparation blobs are needed for FTP.
1620 
1621  request->manager(conn, http->al);
1622 
1623  request->flags.accelerated = http->flags.accel;
1624  request->flags.sslBumped=conn->switchedToHttps();
1625  // TODO: decouple http->flags.accel from request->flags.sslBumped
1626  request->flags.noDirect = (request->flags.accelerated && !request->flags.sslBumped) ?
1627  !conn->port->allow_direct : 0;
1628  request->sources |= isFtp ? Http::Message::srcFtp :
1629  ((request->flags.sslBumped || conn->port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
1630 #if USE_AUTH
1631  if (request->flags.sslBumped) {
1632  if (conn->getAuth() != NULL)
1633  request->auth_user_request = conn->getAuth();
1634  }
1635 #endif
1636 
1637  if (internalCheck(request->url.path())) {
1638  if (internalHostnameIs(request->url.host()) && request->url.port() == getMyPort()) {
1639  debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true));
1640  http->flags.internal = true;
1642  debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true) << " (global_internal_static on)");
1643  request->url.setScheme(AnyP::PROTO_HTTP, "http");
1644  request->url.host(internalHostname());
1645  request->url.port(getMyPort());
1646  http->flags.internal = true;
1647  http->setLogUriToRequestUri();
1648  } else
1649  debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true) << " (not this proxy)");
1650  }
1651 
1652  request->flags.internal = http->flags.internal;
1653 
1654  if (!isFtp) {
1655  // XXX: for non-HTTP messages instantiate a different Http::Message child type
1656  // for now Squid only supports HTTP requests
1657  const AnyP::ProtocolVersion &http_ver = hp->messageProtocol();
1658  assert(request->http_ver.protocol == http_ver.protocol);
1659  request->http_ver.major = http_ver.major;
1660  request->http_ver.minor = http_ver.minor;
1661  }
1662 
1663  mustReplyToOptions = (request->method == Http::METHOD_OPTIONS) &&
1664  (request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0);
1665  if (!urlCheckRequest(request.getRaw()) || mustReplyToOptions) {
1667  conn->quitAfterError(request.getRaw());
1668  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1669  assert (repContext);
1671  conn, request.getRaw(), nullptr, nullptr);
1672  assert(context->http->out.offset == 0);
1673  context->pullData();
1675  return;
1676  }
1677 
1678  const auto frameStatus = request->checkEntityFraming();
1679  if (frameStatus != Http::scNone) {
1681  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1682  assert (repContext);
1683  conn->quitAfterError(request.getRaw());
1684  repContext->setReplyToError(ERR_INVALID_REQ, frameStatus, request->method, nullptr, conn, request.getRaw(), nullptr, nullptr);
1685  assert(context->http->out.offset == 0);
1686  context->pullData();
1688  return;
1689  }
1690 
1691  clientSetKeepaliveFlag(http);
1692  // Let tunneling code be fully responsible for CONNECT requests
1693  if (http->request->method == Http::METHOD_CONNECT) {
1694  context->mayUseConnection(true);
1695  conn->flags.readMore = false;
1696  }
1697 
1698 #if USE_OPENSSL
1699  if (conn->switchedToHttps() && conn->serveDelayedError(context)) {
1701  return;
1702  }
1703 #endif
1704 
1705  /* Do we expect a request-body? */
1706  const auto chunked = request->header.chunked();
1707  expectBody = chunked || request->content_length > 0;
1708  if (!context->mayUseConnection() && expectBody) {
1709  request->body_pipe = conn->expectRequestBody(
1710  chunked ? -1 : request->content_length);
1711 
1712  /* Is it too large? */
1713  if (!chunked && // if chunked, we will check as we accumulate
1716  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1717  assert (repContext);
1718  conn->quitAfterError(request.getRaw());
1719  repContext->setReplyToError(ERR_TOO_BIG,
1721  conn, http->request, nullptr, nullptr);
1722  assert(context->http->out.offset == 0);
1723  context->pullData();
1725  return;
1726  }
1727 
1728  if (!isFtp) {
1729  // We may stop producing, comm_close, and/or call setReplyToError()
1730  // below, so quit on errors to avoid http->doCallouts()
1731  if (!conn->handleRequestBodyData()) {
1733  return;
1734  }
1735 
1736  if (!request->body_pipe->productionEnded()) {
1737  debugs(33, 5, "need more request body");
1738  context->mayUseConnection(true);
1739  assert(conn->flags.readMore);
1740  }
1741  }
1742  }
1743 
1744  http->calloutContext = new ClientRequestContext(http);
1745 
1746  http->doCallouts();
1747 
1749 }
1750 
1751 void
1753 {
1754  debugs(33, 3, context << " to " << pipeline.count() << '/' << pipeline.nrequests);
1755  if (bareError) {
1756  debugs(33, 5, "assigning " << bareError);
1757  assert(context);
1758  assert(context->http);
1759  context->http->updateError(bareError);
1760  bareError.clear();
1761  }
1762  pipeline.add(context);
1763 }
1764 
1765 int
1767 {
1768  // TODO: Support pipelined requests through pinned connections.
1769  if (pinning.pinned)
1770  return 0;
1772 }
1773 
1779 bool
1781 {
1782  const int existingRequestCount = pipeline.count();
1783 
1784  // default to the configured pipeline size.
1785  // add 1 because the head of pipeline is counted in concurrent requests and not prefetch queue
1786 #if USE_OPENSSL
1787  const int internalRequest = (transparent() && sslBumpMode == Ssl::bumpSplice) ? 1 : 0;
1788 #else
1789  const int internalRequest = 0;
1790 #endif
1791  const int concurrentRequestLimit = pipelinePrefetchMax() + 1 + internalRequest;
1792 
1793  // when queue filled already we can't add more.
1794  if (existingRequestCount >= concurrentRequestLimit) {
1795  debugs(33, 3, clientConnection << " max concurrent requests reached (" << concurrentRequestLimit << ")");
1796  debugs(33, 5, clientConnection << " deferring new request until one is done");
1797  return true;
1798  }
1799 
1800  return false;
1801 }
1802 
1808 bool
1810 {
1812  return proxyProtocolError("PROXY client not permitted by default ACL");
1813 
1815  fillChecklist(ch);
1816  if (!ch.fastCheck().allowed())
1817  return proxyProtocolError("PROXY client not permitted by ACLs");
1818 
1819  return true;
1820 }
1821 
1827 bool
1829 {
1830  if (msg) {
1831  // This is important to know, but maybe not so much that flooding the log is okay.
1832 #if QUIET_PROXY_PROTOCOL
1833  // display the first of every 32 occurrences at level 1, the others at level 2.
1834  static uint8_t hide = 0;
1835  debugs(33, (hide++ % 32 == 0 ? DBG_IMPORTANT : 2), msg << " from " << clientConnection);
1836 #else
1837  debugs(33, DBG_IMPORTANT, msg << " from " << clientConnection);
1838 #endif
1839  mustStop(msg);
1840  }
1841  return false;
1842 }
1843 
1848 bool
1850 {
1851  try {
1852  const auto parsed = ProxyProtocol::Parse(inBuf);
1853  proxyProtocolHeader_ = parsed.header;
1855  inBuf.consume(parsed.size);
1856  needProxyProtocolHeader_ = false;
1857  if (proxyProtocolHeader_->hasForwardedAddresses()) {
1858  clientConnection->local = proxyProtocolHeader_->destinationAddress;
1859  clientConnection->remote = proxyProtocolHeader_->sourceAddress;
1861  clientConnection->flags ^= COMM_TRANSPARENT; // prevent TPROXY spoofing of this new IP.
1862  debugs(33, 5, "PROXY/" << proxyProtocolHeader_->version() << " upgrade: " << clientConnection);
1863  }
1864  } catch (const Parser::BinaryTokenizer::InsufficientInput &) {
1865  debugs(33, 3, "PROXY protocol: waiting for more than " << inBuf.length() << " bytes");
1866  return false;
1867  } catch (const std::exception &e) {
1868  return proxyProtocolError(e.what());
1869  }
1870  return true;
1871 }
1872 
1873 void
1875 {
1876  if (receivedFirstByte_)
1877  return;
1878 
1879  receivedFirstByte_ = true;
1881 }
1882 
1888 bool
1890 {
1891  bool parsed_req = false;
1892 
1893  debugs(33, 5, HERE << clientConnection << ": attempting to parse");
1894 
1895  // Loop while we have read bytes that are not needed for producing the body
1896  // On errors, bodyPipe may become nil, but readMore will be cleared
1897  while (!inBuf.isEmpty() && !bodyPipe && flags.readMore) {
1898 
1899  // Prohibit concurrent requests when using a pinned to-server connection
1900  // because our Client classes do not support request pipelining.
1901  if (pinning.pinned && !pinning.readHandler) {
1902  debugs(33, 3, clientConnection << " waits for busy " << pinning.serverConnection);
1903  break;
1904  }
1905 
1906  /* Limit the number of concurrent requests */
1908  break;
1909 
1910  // try to parse the PROXY protocol header magic bytes
1912  if (!parseProxyProtocolHeader())
1913  break;
1914 
1915  // we have been waiting for PROXY to provide client-IP
1916  // for some lookups, ie rDNS and IDENT.
1918 
1919  // Done with PROXY protocol which has cleared preservingClientData_.
1920  // If the next protocol supports on_unsupported_protocol, then its
1921  // parseOneRequest() must reset preservingClientData_.
1923  }
1924 
1925  if (Http::StreamPointer context = parseOneRequest()) {
1926  debugs(33, 5, clientConnection << ": done parsing a request");
1927  extendLifetime();
1928  context->registerWithConn();
1929 
1930 #if USE_OPENSSL
1931  if (switchedToHttps())
1933 #endif
1934 
1935  processParsedRequest(context);
1936 
1937  parsed_req = true; // XXX: do we really need to parse everything right NOW ?
1938 
1939  if (context->mayUseConnection()) {
1940  debugs(33, 3, HERE << "Not parsing new requests, as this request may need the connection");
1941  break;
1942  }
1943  } else {
1944  debugs(33, 5, clientConnection << ": not enough request data: " <<
1945  inBuf.length() << " < " << Config.maxRequestHeaderSize);
1947  break;
1948  }
1949  }
1950 
1951  /* XXX where to 'finish' the parsing pass? */
1952  return parsed_req;
1953 }
1954 
1955 void
1957 {
1958 #if USE_OPENSSL
1959  if (parsingTlsHandshake) {
1961  return;
1962  }
1963 #endif
1964 
1965  /* Process next request */
1966  if (pipeline.empty())
1967  fd_note(clientConnection->fd, "Reading next request");
1968 
1969  if (!clientParseRequests()) {
1970  if (!isOpen())
1971  return;
1972  // We may get here if the client half-closed after sending a partial
1973  // request. See doClientRead() and shouldCloseOnEof().
1974  // XXX: This partially duplicates ConnStateData::kick().
1976  debugs(33, 5, clientConnection << ": half-closed connection, no completed request parsed, connection closing.");
1978  return;
1979  }
1980  }
1981 
1982  if (!isOpen())
1983  return;
1984 
1986 }
1987 
1994 bool
1996 {
1997  // if we are reading a body, stuff data into the body pipe
1998  if (bodyPipe != NULL)
1999  return handleRequestBodyData();
2000  return true;
2001 }
2002 
2010 bool
2012 {
2013  assert(bodyPipe != NULL);
2014 
2015  if (bodyParser) { // chunked encoding
2016  if (const err_type error = handleChunkedRequestBody()) {
2018  return false;
2019  }
2020  } else { // identity encoding
2021  debugs(33,5, HERE << "handling plain request body for " << clientConnection);
2022  const size_t putSize = bodyPipe->putMoreData(inBuf.c_str(), inBuf.length());
2023  if (putSize > 0)
2024  consumeInput(putSize);
2025 
2026  if (!bodyPipe->mayNeedMoreData()) {
2027  // BodyPipe will clear us automagically when we produced everything
2028  bodyPipe = NULL;
2029  }
2030  }
2031 
2032  if (!bodyPipe) {
2033  debugs(33,5, HERE << "produced entire request body for " << clientConnection);
2034 
2035  if (const char *reason = stoppedSending()) {
2036  /* we've finished reading like good clients,
2037  * now do the close that initiateClose initiated.
2038  */
2039  debugs(33, 3, HERE << "closing for earlier sending error: " << reason);
2041  return false;
2042  }
2043  }
2044 
2045  return true;
2046 }
2047 
2049 err_type
2051 {
2052  debugs(33, 7, "chunked from " << clientConnection << ": " << inBuf.length());
2053 
2054  try { // the parser will throw on errors
2055 
2056  if (inBuf.isEmpty()) // nothing to do
2057  return ERR_NONE;
2058 
2059  BodyPipeCheckout bpc(*bodyPipe);
2061  const bool parsed = bodyParser->parse(inBuf);
2062  inBuf = bodyParser->remaining(); // sync buffers
2063  bpc.checkIn();
2064 
2065  // dechunk then check: the size limit applies to _dechunked_ content
2067  return ERR_TOO_BIG;
2068 
2069  if (parsed) {
2071  Must(!bodyPipe);
2072  return ERR_NONE; // nil bodyPipe implies body end for the caller
2073  }
2074 
2075  // if chunk parser needs data, then the body pipe must need it too
2077 
2078  // if parser needs more space and we can consume nothing, we will stall
2080  } catch (...) { // TODO: be more specific
2081  debugs(33, 3, HERE << "malformed chunks" << bodyPipe->status());
2082  return ERR_INVALID_REQ;
2083  }
2084 
2085  debugs(33, 7, HERE << "need more chunked data" << *bodyPipe->status());
2086  return ERR_NONE;
2087 }
2088 
2090 void
2092 {
2093  finishDechunkingRequest(false);
2094 
2095  // XXX: The code below works if we fail during initial request parsing,
2096  // but if we fail when the server connection is used already, the server may send
2097  // us its response too, causing various assertions. How to prevent that?
2098 #if WE_KNOW_HOW_TO_SEND_ERRORS
2099  Http::StreamPointer context = pipeline.front();
2100  if (context != NULL && !context->http->out.offset) { // output nothing yet
2101  clientStreamNode *node = context->getClientReplyContext();
2102  clientReplyContext *repContext = dynamic_cast<clientReplyContext*>(node->data.getRaw());
2103  assert(repContext);
2104  const Http::StatusCode scode = (error == ERR_TOO_BIG) ?
2105  Http::scPayloadTooLarge : HTTP_BAD_REQUEST;
2106  repContext->setReplyToError(error, scode,
2107  repContext->http->request->method,
2108  repContext->http->uri,
2109  CachePeer,
2110  repContext->http->request,
2111  inBuf, NULL);
2112  context->pullData();
2113  } else {
2114  // close or otherwise we may get stuck as nobody will notice the error?
2116  }
2117 #else
2118  debugs(33, 3, HERE << "aborting chunked request without error " << error);
2120 #endif
2121  flags.readMore = false;
2122 }
2123 
2124 void
2126 {
2127  // request reader may get stuck waiting for space if nobody consumes body
2128  if (bodyPipe != NULL)
2130 
2131  // kids extend
2132 }
2133 
2135 void
2137 {
2138  if (!Comm::IsConnOpen(io.conn))
2139  return;
2140 
2142  updateError(error);
2144  return;
2145 
2146  /*
2147  * Just close the connection to not confuse browsers
2148  * using persistent connections. Some browsers open
2149  * a connection and then do not use it until much
2150  * later (presumeably because the request triggering
2151  * the open has already been completed on another
2152  * connection)
2153  */
2154  debugs(33, 3, "requestTimeout: FD " << io.fd << ": lifetime is expired.");
2155  io.conn->close();
2156 }
2157 
2158 void
2160 {
2161  debugs(33, DBG_IMPORTANT, "WARNING: Closing client connection due to lifetime timeout" <<
2162  Debug::Extra << "connection: " << io.conn);
2163 
2164  LogTagsErrors lte;
2165  lte.timedout = true;
2167 }
2168 
2170  AsyncJob("ConnStateData"), // kids overwrite
2171  Server(xact)
2172 #if USE_OPENSSL
2173  , tlsParser(Security::HandshakeParser::fromClient)
2174 #endif
2175 {
2176  // store the details required for creating more MasterXaction objects as new requests come in
2177  log_addr = xact->tcpClient->remote;
2179 
2180  // register to receive notice of Squid signal events
2181  // which may affect long persisting client connections
2182  registerRunner();
2183 }
2184 
2185 void
2187 {
2190 
2191  if (port->disable_pmtu_discovery != DISABLE_PMTU_OFF &&
2192  (transparent() || port->disable_pmtu_discovery == DISABLE_PMTU_ALWAYS)) {
2193 #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
2194  int i = IP_PMTUDISC_DONT;
2195  if (setsockopt(clientConnection->fd, SOL_IP, IP_MTU_DISCOVER, &i, sizeof(i)) < 0) {
2196  int xerrno = errno;
2197  debugs(33, 2, "WARNING: Path MTU discovery disabling failed on " << clientConnection << " : " << xstrerr(xerrno));
2198  }
2199 #else
2200  static bool reported = false;
2201 
2202  if (!reported) {
2203  debugs(33, DBG_IMPORTANT, "NOTICE: Path MTU discovery disabling is not supported on your platform.");
2204  reported = true;
2205  }
2206 #endif
2207  }
2208 
2210  AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, ConnStateData::connStateClosed);
2212 
2213  needProxyProtocolHeader_ = port->flags.proxySurrogate;
2215  if (!proxyProtocolValidateClient()) // will close the connection on failure
2216  return;
2217  } else
2219 
2220  // requires needProxyProtocolHeader_ which is initialized above
2222 }
2223 
2224 void
2226 {
2227  if (Config.onoff.log_fqdn)
2229 
2230 #if USE_IDENT
2231  if (Ident::TheConfig.identLookup) {
2232  ACLFilledChecklist identChecklist(Ident::TheConfig.identLookup, NULL, NULL);
2233  fillChecklist(identChecklist);
2234  if (identChecklist.fastCheck().allowed())
2236  }
2237 #endif
2238 
2240 
2241 #if USE_DELAY_POOLS
2242  fd_table[clientConnection->fd].clientInfo = NULL;
2243 
2244  if (!Config.onoff.client_db)
2245  return; // client delay pools require client_db
2246 
2247  const auto &pools = ClientDelayPools::Instance()->pools;
2248  if (pools.size()) {
2250  fillChecklist(ch);
2251  // TODO: we check early to limit error response bandwidth but we
2252  // should recheck when we can honor delay_pool_uses_indirect
2253  for (unsigned int pool = 0; pool < pools.size(); ++pool) {
2254 
2255  /* pools require explicit 'allow' to assign a client into them */
2256  if (pools[pool]->access) {
2257  ch.changeAcl(pools[pool]->access);
2258  auto answer = ch.fastCheck();
2259  if (answer.allowed()) {
2260 
2261  /* request client information from db after we did all checks
2262  this will save hash lookup if client failed checks */
2264  assert(cli);
2265 
2266  /* put client info in FDE */
2267  fd_table[clientConnection->fd].clientInfo = cli;
2268 
2269  /* setup write limiter for this request */
2270  const double burst = floor(0.5 +
2271  (pools[pool]->highwatermark * Config.ClientDelay.initial)/100.0);
2272  cli->setWriteLimiter(pools[pool]->rate, burst, pools[pool]->highwatermark);
2273  break;
2274  } else {
2275  debugs(83, 4, HERE << "Delay pool " << pool << " skipped because ACL " << answer);
2276  }
2277  }
2278  }
2279  }
2280 #endif
2281 
2282  // kids must extend to actually start doing something (e.g., reading)
2283 }
2284 
2287 {
2288  const auto handshakeResult = Security::Accept(*clientConnection);
2289 
2290 #if USE_OPENSSL
2291  // log ASAP, even if the handshake has not completed (or failed)
2292  const auto fd = clientConnection->fd;
2293  assert(fd >= 0);
2294  keyLogger.checkpoint(*fd_table[fd].ssl, *this);
2295 #else
2296  // TODO: Support fd_table[fd].ssl dereference in other builds.
2297 #endif
2298 
2299  return handshakeResult;
2300 }
2301 
2303 void
2305 {
2306  MasterXaction::Pointer xact = params.xaction;
2307  AnyP::PortCfgPointer s = xact->squidPort;
2308 
2309  // NP: it is possible the port was reconfigured when the call or accept() was queued.
2310 
2311  if (params.flag != Comm::OK) {
2312  // Its possible the call was still queued when the client disconnected
2313  debugs(33, 2, s->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2314  return;
2315  }
2316 
2317  debugs(33, 4, params.conn << ": accepted");
2318  fd_note(params.conn->fd, "client http connect");
2319 
2320  // Socket is ready, setup the connection manager to start using it
2321  auto *srv = Http::NewServer(xact);
2322  AsyncJob::Start(srv); // usually async-calls readSomeData()
2323 }
2324 
2326 static bool
2328 {
2329  const auto conn = connState->clientConnection;
2330  if (Security::CreateServerSession(ctx, conn, connState->port->secure, "client https start")) {
2331  debugs(33, 5, "will negotiate TLS on " << conn);
2332  return true;
2333  }
2334 
2335  debugs(33, DBG_IMPORTANT, "ERROR: could not create TLS server context for " << conn);
2336  conn->close();
2337  return false;
2338 }
2339 
2341 static void
2342 clientNegotiateSSL(int fd, void *data)
2343 {
2344  ConnStateData *conn = (ConnStateData *)data;
2345 
2346  const auto handshakeResult = conn->acceptTls();
2347  switch (handshakeResult.category) {
2349  break;
2350 
2352  Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_READ, clientNegotiateSSL, conn, 0);
2353  return;
2354 
2356  Comm::SetSelect(conn->clientConnection->fd, COMM_SELECT_WRITE, clientNegotiateSSL, conn, 0);
2357  return;
2358 
2360  debugs(83, (handshakeResult.important ? Important(62) : 2), "ERROR: " << handshakeResult.errorDescription <<
2361  " while accepting a TLS connection on " << conn->clientConnection << ": " << handshakeResult.errorDetail);
2362  // TODO: No ConnStateData::tunnelOnError() on this forward-proxy code
2363  // path because we cannot know the intended connection target?
2364  conn->updateError(ERR_SECURE_ACCEPT_FAIL, handshakeResult.errorDetail);
2365  conn->clientConnection->close();
2366  return;
2367  }
2368 
2369  Security::SessionPointer session(fd_table[fd].ssl);
2370 
2371 #if USE_OPENSSL
2372  if (Security::SessionIsResumed(session)) {
2373  debugs(83, 2, "Session " << SSL_get_session(session.get()) <<
2374  " reused on FD " << fd << " (" << fd_table[fd].ipaddr <<
2375  ":" << (int)fd_table[fd].remote_port << ")");
2376  } else {
2377  if (Debug::Enabled(83, 4)) {
2378  /* Write out the SSL session details.. actually the call below, but
2379  * OpenSSL headers do strange typecasts confusing GCC.. */
2380  /* PEM_write_SSL_SESSION(debug_log, SSL_get_session(ssl)); */
2381 #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x00908000L
2382  PEM_ASN1_write(reinterpret_cast<i2d_of_void *>(i2d_SSL_SESSION),
2383  PEM_STRING_SSL_SESSION, debug_log,
2384  reinterpret_cast<char *>(SSL_get_session(session.get())),
2385  nullptr, nullptr, 0, nullptr, nullptr);
2386 
2387 #elif (ALLOW_ALWAYS_SSL_SESSION_DETAIL == 1)
2388 
2389  /* When using gcc 3.3.x and OpenSSL 0.9.7x sometimes a compile error can occur here.
2390  * This is caused by an unpredicatble gcc behaviour on a cast of the first argument
2391  * of PEM_ASN1_write(). For this reason this code section is disabled. To enable it,
2392  * define ALLOW_ALWAYS_SSL_SESSION_DETAIL=1.
2393  * Because there are two possible usable cast, if you get an error here, try the other
2394  * commented line. */
2395 
2396  PEM_ASN1_write((int(*)())i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2397  debug_log,
2398  reinterpret_cast<char *>(SSL_get_session(session.get())),
2399  nullptr, nullptr, 0, nullptr, nullptr);
2400  /* PEM_ASN1_write((int(*)(...))i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2401  debug_log,
2402  reinterpret_cast<char *>(SSL_get_session(session.get())),
2403  nullptr, nullptr, 0, nullptr, nullptr);
2404  */
2405 #else
2406  debugs(83, 4, "With " OPENSSL_VERSION_TEXT ", session details are available only defining ALLOW_ALWAYS_SSL_SESSION_DETAIL=1 in the source.");
2407 
2408 #endif
2409  /* Note: This does not automatically fflush the log file.. */
2410  }
2411 
2412  debugs(83, 2, "New session " << SSL_get_session(session.get()) <<
2413  " on FD " << fd << " (" << fd_table[fd].ipaddr << ":" <<
2414  fd_table[fd].remote_port << ")");
2415  }
2416 #else
2417  debugs(83, 2, "TLS session reuse not yet implemented.");
2418 #endif
2419 
2420  // Connection established. Retrieve TLS connection parameters for logging.
2421  conn->clientConnection->tlsNegotiations()->retrieveNegotiatedInfo(session);
2422 
2423 #if USE_OPENSSL
2424  X509 *client_cert = SSL_get_peer_certificate(session.get());
2425 
2426  if (client_cert) {
2427  debugs(83, 3, "FD " << fd << " client certificate: subject: " <<
2428  X509_NAME_oneline(X509_get_subject_name(client_cert), 0, 0));
2429 
2430  debugs(83, 3, "FD " << fd << " client certificate: issuer: " <<
2431  X509_NAME_oneline(X509_get_issuer_name(client_cert), 0, 0));
2432 
2433  X509_free(client_cert);
2434  } else {
2435  debugs(83, 5, "FD " << fd << " has no client certificate.");
2436  }
2437 #else
2438  debugs(83, 2, "Client certificate requesting not yet implemented.");
2439 #endif
2440 
2441  // If we are called, then bumped CONNECT has succeeded. Finalize it.
2442  if (auto xact = conn->pipeline.front()) {
2443  if (xact->http && xact->http->request && xact->http->request->method == Http::METHOD_CONNECT)
2444  xact->finished();
2445  // cannot proceed with encryption if requests wait for plain responses
2446  Must(conn->pipeline.empty());
2447  }
2448  /* careful: finished() above frees request, host, etc. */
2449 
2450  conn->readSomeData();
2451 }
2452 
2457 static void
2459 {
2460  assert(connState);
2461  const Comm::ConnectionPointer &details = connState->clientConnection;
2462 
2463  if (!ctx || !httpsCreate(connState, ctx))
2464  return;
2465 
2466  connState->resetReadTimeout(Config.Timeout.request);
2467 
2468  Comm::SetSelect(details->fd, COMM_SELECT_READ, clientNegotiateSSL, connState, 0);
2469 }
2470 
2471 #if USE_OPENSSL
2472 
2475 static void
2477 {
2478  ConnStateData *connState = (ConnStateData *) data;
2479 
2480  // if the connection is closed or closing, just return.
2481  if (!connState->isOpen())
2482  return;
2483 
2484  if (answer.allowed()) {
2485  debugs(33, 2, "sslBump action " << Ssl::bumpMode(answer.kind) << "needed for " << connState->clientConnection);
2486  connState->sslBumpMode = static_cast<Ssl::BumpMode>(answer.kind);
2487  } else {
2488  debugs(33, 3, "sslBump not needed for " << connState->clientConnection);
2489  connState->sslBumpMode = Ssl::bumpSplice;
2490  }
2491 
2492  if (connState->sslBumpMode == Ssl::bumpTerminate) {
2493  connState->clientConnection->close();
2494  return;
2495  }
2496 
2497  if (!connState->fakeAConnectRequest("ssl-bump", connState->inBuf))
2498  connState->clientConnection->close();
2499 }
2500 #endif
2501 
2503 static void
2505 {
2506  MasterXaction::Pointer xact = params.xaction;
2507  const AnyP::PortCfgPointer s = xact->squidPort;
2508 
2509  // NP: it is possible the port was reconfigured when the call or accept() was queued.
2510 
2511  if (params.flag != Comm::OK) {
2512  // Its possible the call was still queued when the client disconnected
2513  debugs(33, 2, "httpsAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2514  return;
2515  }
2516 
2517  debugs(33, 4, HERE << params.conn << " accepted, starting SSL negotiation.");
2518  fd_note(params.conn->fd, "client https connect");
2519 
2520  // Socket is ready, setup the connection manager to start using it
2521  auto *srv = Https::NewServer(xact);
2522  AsyncJob::Start(srv); // usually async-calls postHttpsAccept()
2523 }
2524 
2525 void
2527 {
2528  if (port->flags.tunnelSslBumping) {
2529 #if USE_OPENSSL
2530  debugs(33, 5, "accept transparent connection: " << clientConnection);
2531 
2532  if (!Config.accessList.ssl_bump) {
2534  return;
2535  }
2536 
2539  // Create a fake HTTP request and ALE for the ssl_bump ACL check,
2540  // using tproxy/intercept provided destination IP and port.
2541  // XXX: Merge with subsequent fakeAConnectRequest(), buildFakeRequest().
2542  // XXX: Do this earlier (e.g., in Http[s]::One::Server constructor).
2543  HttpRequest *request = new HttpRequest(mx);
2544  static char ip[MAX_IPSTRLEN];
2546  request->url.host(clientConnection->local.toStr(ip, sizeof(ip)));
2548  request->myportname = port->name;
2549  const AccessLogEntry::Pointer connectAle = new AccessLogEntry;
2550  CodeContext::Reset(connectAle);
2551  // TODO: Use these request/ALE when waiting for new bumped transactions.
2552 
2554  fillChecklist(*acl_checklist);
2555  // Build a local AccessLogEntry to allow requiresAle() acls work
2556  acl_checklist->al = connectAle;
2557  acl_checklist->al->cache.start_time = current_time;
2558  acl_checklist->al->tcpClient = clientConnection;
2559  acl_checklist->al->cache.port = port;
2560  acl_checklist->al->cache.caddr = log_addr;
2561  acl_checklist->al->proxyProtocolHeader = proxyProtocolHeader_;
2562  acl_checklist->al->updateError(bareError);
2563  HTTPMSGUNLOCK(acl_checklist->al->request);
2564  acl_checklist->al->request = request;
2565  HTTPMSGLOCK(acl_checklist->al->request);
2566  Http::StreamPointer context = pipeline.front();
2567  ClientHttpRequest *http = context ? context->http : nullptr;
2568  const char *log_uri = http ? http->log_uri : nullptr;
2569  acl_checklist->syncAle(request, log_uri);
2570  acl_checklist->nonBlockingCheck(httpsSslBumpAccessCheckDone, this);
2571 #else
2572  fatal("FATAL: SSL-Bump requires --with-openssl");
2573 #endif
2574  return;
2575  } else {
2576  httpsEstablish(this, port->secure.staticContext);
2577  }
2578 }
2579 
2580 #if USE_OPENSSL
2581 void
2583 {
2584  ConnStateData * state_data = (ConnStateData *)(data);
2585  state_data->sslCrtdHandleReply(reply);
2586 }
2587 
2588 void
2590 {
2591  if (!isOpen()) {
2592  debugs(33, 3, "Connection gone while waiting for ssl_crtd helper reply; helper reply:" << reply);
2593  return;
2594  }
2595 
2596  if (reply.result == Helper::BrokenHelper) {
2597  debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply);
2598  } else if (!reply.other().hasContent()) {
2599  debugs(1, DBG_IMPORTANT, HERE << "\"ssl_crtd\" helper returned <NULL> reply.");
2600  } else {
2602  if (reply_message.parse(reply.other().content(), reply.other().contentSize()) != Ssl::CrtdMessage::OK) {
2603  debugs(33, 5, "Reply from ssl_crtd for " << tlsConnectHostOrIp << " is incorrect");
2604  } else {
2605  if (reply.result != Helper::Okay) {
2606  debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply_message.getBody());
2607  } else {
2608  debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " was successfully received from ssl_crtd");
2611  auto ssl = fd_table[clientConnection->fd].ssl.get();
2612  bool ret = Ssl::configureSSLUsingPkeyAndCertFromMemory(ssl, reply_message.getBody().c_str(), *port);
2613  if (!ret)
2614  debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2615 
2618  } else {
2620  if (ctx && !sslBumpCertKey.isEmpty())
2622  getSslContextDone(ctx);
2623  }
2624  return;
2625  }
2626  }
2627  }
2630 }
2631 
2633 {
2635 
2636  const bool connectedOk = sslServerBump && sslServerBump->connectedOk();
2637  if (connectedOk) {
2638  if (X509 *mimicCert = sslServerBump->serverCert.get())
2639  certProperties.mimicCert.resetAndLock(mimicCert);
2640 
2641  ACLFilledChecklist checklist(nullptr, sslServerBump->request.getRaw());
2642  fillChecklist(checklist);
2643 
2644  for (sslproxy_cert_adapt *ca = Config.ssl_client.cert_adapt; ca != NULL; ca = ca->next) {
2645  // If the algorithm already set, then ignore it.
2646  if ((ca->alg == Ssl::algSetCommonName && certProperties.setCommonName) ||
2647  (ca->alg == Ssl::algSetValidAfter && certProperties.setValidAfter) ||
2648  (ca->alg == Ssl::algSetValidBefore && certProperties.setValidBefore) )
2649  continue;
2650 
2651  if (ca->aclList && checklist.fastCheck(ca->aclList).allowed()) {
2652  const char *alg = Ssl::CertAdaptAlgorithmStr[ca->alg];
2653  const char *param = ca->param;
2654 
2655  // For parameterless CN adaptation, use hostname from the
2656  // CONNECT request.
2657  if (ca->alg == Ssl::algSetCommonName) {
2658  if (!param)
2659  param = tlsConnectHostOrIp.c_str();
2660  certProperties.commonName = param;
2661  certProperties.setCommonName = true;
2662  } else if (ca->alg == Ssl::algSetValidAfter)
2663  certProperties.setValidAfter = true;
2664  else if (ca->alg == Ssl::algSetValidBefore)
2665  certProperties.setValidBefore = true;
2666 
2667  debugs(33, 5, HERE << "Matches certificate adaptation aglorithm: " <<
2668  alg << " param: " << (param ? param : "-"));
2669  }
2670  }
2671 
2672  certProperties.signAlgorithm = Ssl::algSignEnd;
2673  for (sslproxy_cert_sign *sg = Config.ssl_client.cert_sign; sg != NULL; sg = sg->next) {
2674  if (sg->aclList && checklist.fastCheck(sg->aclList).allowed()) {
2675  certProperties.signAlgorithm = (Ssl::CertSignAlgorithm)sg->alg;
2676  break;
2677  }
2678  }
2679  } else {// did not try to connect (e.g. client-first) or failed to connect
2680  // In case of an error while connecting to the secure server, use a
2681  // trusted certificate, with no mimicked fields and no adaptation
2682  // algorithms. There is nothing we can mimic, so we want to minimize the
2683  // number of warnings the user will have to see to get to the error page.
2684  // We will close the connection, so that the trust is not extended to
2685  // non-Squid content.
2686  certProperties.signAlgorithm = Ssl::algSignTrusted;
2687  }
2688 
2689  assert(certProperties.signAlgorithm != Ssl::algSignEnd);
2690 
2691  if (certProperties.signAlgorithm == Ssl::algSignUntrusted) {
2692  assert(port->secure.untrustedSigningCa.cert);
2693  certProperties.signWithX509.resetAndLock(port->secure.untrustedSigningCa.cert.get());
2694  certProperties.signWithPkey.resetAndLock(port->secure.untrustedSigningCa.pkey.get());
2695  } else {
2696  assert(port->secure.signingCa.cert.get());
2697  certProperties.signWithX509.resetAndLock(port->secure.signingCa.cert.get());
2698 
2699  if (port->secure.signingCa.pkey)
2700  certProperties.signWithPkey.resetAndLock(port->secure.signingCa.pkey.get());
2701  }
2702  signAlgorithm = certProperties.signAlgorithm;
2703 
2704  certProperties.signHash = Ssl::DefaultSignHash;
2705 }
2706 
2709 {
2710  debugs(33, 5, "Finding SSL certificate for " << cacheKey << " in cache");
2712  if (const auto ctx = ssl_ctx_cache ? ssl_ctx_cache->get(cacheKey) : nullptr) {
2713  if (Ssl::verifySslCertificate(*ctx, certProperties)) {
2714  debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is valid");
2715  return *ctx;
2716  } else {
2717  debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is out of date. Delete this certificate from cache");
2718  if (ssl_ctx_cache)
2719  ssl_ctx_cache->del(cacheKey);
2720  }
2721  }
2722  return Security::ContextPointer(nullptr);
2723 }
2724 
2725 void
2727 {
2729  if (!ssl_ctx_cache || !ssl_ctx_cache->add(cacheKey, ctx)) {
2730  // If it is not in storage delete after using. Else storage deleted it.
2731  fd_table[clientConnection->fd].html?amicTlsContext = ctx;
2732  }
2733 }
2734 
2735 void
2737 {
2738  if (port->secure.generateHostCertificates) {
2739  Ssl::CertificateProperties certProperties;
2740  buildSslCertGenerationParams(certProperties);
2741 
2742  // Disable caching for bumpPeekAndSplice mode
2745  Ssl::InRamCertificateDbKey(certProperties, sslBumpCertKey);
2747 
2749  if (ctx) {
2750  getSslContextDone(ctx);
2751  return;
2752  }
2753  }
2754 
2755 #if USE_SSL_CRTD
2756  try {
2757  debugs(33, 5, HERE << "Generating SSL certificate for " << certProperties.commonName << " using ssl_crtd.");
2760  request_message.composeRequest(certProperties);
2761  debugs(33, 5, HERE << "SSL crtd request: " << request_message.compose().c_str());
2762  Ssl::Helper::Submit(request_message, sslCrtdHandleReplyWrapper, this);
2763  return;
2764  } catch (const std::exception &e) {
2765  debugs(33, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtd " <<
2766  "request for " << certProperties.commonName <<
2767  " certificate: " << e.what() << "; will now block to " <<
2768  "generate that certificate.");
2769  // fall through to do blocking in-process generation.
2770  }
2771 #endif // USE_SSL_CRTD
2772 
2773  debugs(33, 5, HERE << "Generating SSL certificate for " << certProperties.commonName);
2776  auto ssl = fd_table[clientConnection->fd].ssl.get();
2777  if (!Ssl::configureSSL(ssl, certProperties, *port))
2778  debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2779 
2782  } else {
2784  if (dynCtx && !sslBumpCertKey.isEmpty())
2786  getSslContextDone(dynCtx);
2787  }
2788  return;
2789  }
2790 
2793 }
2794 
2795 void
2797 {
2798  if (port->secure.generateHostCertificates && !ctx) {
2799  debugs(33, 2, "Failed to generate TLS context for " << tlsConnectHostOrIp);
2800  }
2801 
2802  // If generated ssl context = NULL, try to use static ssl context.
2803  if (!ctx) {
2804  if (!port->secure.staticContext) {
2805  debugs(83, DBG_IMPORTANT, "Closing " << clientConnection->remote << " as lacking TLS context");
2807  return;
2808  } else {
2809  debugs(33, 5, "Using static TLS context.");
2810  ctx = port->secure.staticContext;
2811  }
2812  }
2813 
2814  if (!httpsCreate(this, ctx))
2815  return;
2816 
2817  // bumped intercepted conns should already have Config.Timeout.request set
2818  // but forwarded connections may only have Config.Timeout.lifetime. [Re]set
2819  // to make sure the connection does not get stuck on non-SSL clients.
2821 
2822  switchedToHttps_ = true;
2823 
2824  auto ssl = fd_table[clientConnection->fd].ssl.get();
2825  BIO *b = SSL_get_rbio(ssl);
2826  Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
2827  bio->setReadBufData(inBuf);
2828  inBuf.clear();
2830 }
2831 
2832 void
2834 {
2836  Must(http->request);
2837  auto &request = http->request;
2838 
2839  // Depending on receivedFirstByte_, we are at the start of either an
2840  // established CONNECT tunnel with the client or an intercepted TCP (and
2841  // presumably TLS) connection from the client. Expect TLS Client Hello.
2842  const auto insideConnectTunnel = receivedFirstByte_;
2843  debugs(33, 5, (insideConnectTunnel ? "post-CONNECT " : "raw TLS ") << clientConnection);
2844 
2845  tlsConnectHostOrIp = request->url.hostOrIp();
2846  tlsConnectPort = request->url.port();
2847  resetSslCommonName(request->url.host());
2848 
2849  // We are going to read new request
2850  flags.readMore = true;
2851 
2852  // keep version major.minor details the same.
2853  // but we are now performing the HTTPS handshake traffic
2855 
2856  // If sslServerBump is set, then we have decided to deny CONNECT
2857  // and now want to switch to SSL to send the error to the client
2858  // without even peeking at the origin server certificate.
2859  if (bumpServerMode == Ssl::bumpServerFirst && !sslServerBump) {
2860  request->flags.sslPeek = true;
2861  sslServerBump = new Ssl::ServerBump(http);
2862  } else if (bumpServerMode == Ssl::bumpPeek || bumpServerMode == Ssl::bumpStare) {
2863  request->flags.sslPeek = true;
2864  sslServerBump = new Ssl::ServerBump(http, nullptr, bumpServerMode);
2865  }
2866 
2867  // commSetConnTimeout() was called for this request before we switched.
2868  // Fix timeout to request_start_timeout
2870  // Also reset receivedFirstByte_ flag to allow this timeout work in the case we have
2871  // a bumbed "connect" request on non transparent port.
2872  receivedFirstByte_ = false;
2873  // Get more data to peek at Tls
2874  parsingTlsHandshake = true;
2875 
2876  // If the protocol has changed, then reset preservingClientData_.
2877  // Otherwise, its value initially set in start() is still valid/fresh.
2878  // shouldPreserveClientData() uses parsingTlsHandshake which is reset above.
2879  if (insideConnectTunnel)
2881 
2882  readSomeData();
2883 }
2884 
2885 void
2887 {
2889 
2890  assert(!inBuf.isEmpty());
2892  fd_note(clientConnection->fd, "Parsing TLS handshake");
2893 
2894  // stops being nil if we fail to parse the handshake
2895  ErrorDetail::Pointer parseErrorDetails;
2896 
2897  try {
2898  if (!tlsParser.parseHello(inBuf)) {
2899  // need more data to finish parsing
2900  readSomeData();
2901  return;
2902  }
2903  }
2904  catch (const TextException &ex) {
2905  debugs(83, 2, "exception: " << ex);
2906  parseErrorDetails = new ExceptionErrorDetail(ex.id());
2907  }
2908  catch (...) {
2909  debugs(83, 2, "exception: " << CurrentException);
2910  static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_PARSE");
2911  parseErrorDetails = d;
2912  }
2913 
2914  parsingTlsHandshake = false;
2915 
2916  // client data may be needed for splicing and for
2917  // tunneling unsupportedProtocol after an error
2919 
2920  // Even if the parser failed, each TLS detail should either be set
2921  // correctly or still be "unknown"; copying unknown detail is a no-op.
2924  if (details && !details->serverName.isEmpty()) {
2925  resetSslCommonName(details->serverName.c_str());
2926  tlsClientSni_ = details->serverName;
2927  }
2928 
2929  // We should disable read/write handlers
2931 
2932  if (parseErrorDetails) {
2933  Http::StreamPointer context = pipeline.front();
2934  Must(context && context->http);
2935  HttpRequest::Pointer request = context->http->request;
2936  debugs(83, 5, "Got something other than TLS Client Hello. Cannot SslBump.");
2937  updateError(ERR_PROTOCOL_UNKNOWN, parseErrorDetails);
2940  return;
2941  }
2942 
2943  if (!sslServerBump || sslServerBump->act.step1 == Ssl::bumpClientFirst) { // Either means client-first.
2945  return;
2946  } else if (sslServerBump->act.step1 == Ssl::bumpServerFirst) {
2947  debugs(83, 5, "server-first skips step2; start forwarding the request");
2948  sslServerBump->step = XactionStep::tlsBump3;
2949  Http::StreamPointer context = pipeline.front();
2950  ClientHttpRequest *http = context ? context->http : nullptr;
2951  // will call httpsPeeked() with certificate and connection, eventually
2953  } else {
2956  }
2957 }
2958 
2959 static void
2961 {
2962  ConnStateData *connState = (ConnStateData *) data;
2963 
2964  // if the connection is closed or closing, just return.
2965  if (!connState->isOpen())
2966  return;
2967 
2968  debugs(33, 5, "Answer: " << answer << " kind:" << answer.kind);
2969  assert(connState->serverBump());
2970  Ssl::BumpMode bumpAction;
2971  if (answer.allowed()) {
2972  bumpAction = (Ssl::BumpMode)answer.kind;
2973  } else
2974  bumpAction = Ssl::bumpSplice;
2975 
2976  connState->serverBump()->act.step2 = bumpAction;
2977  connState->sslBumpMode = bumpAction;
2978  Http::StreamPointer context = connState->pipeline.front();
2979  if (ClientHttpRequest *http = (context ? context->http : nullptr))
2980  http->al->ssl.bumpMode = bumpAction;
2981 
2982  if (bumpAction == Ssl::bumpTerminate) {
2983  connState->clientConnection->close();
2984  } else if (bumpAction != Ssl::bumpSplice) {
2985  connState->startPeekAndSplice();
2986  } else if (!connState->splice())
2987  connState->clientConnection->close();
2988 }
2989 
2990 bool
2992 {
2993  // normally we can splice here, because we just got client hello message
2994 
2995  // fde::ssl/tls_read_method() probably reads from our own inBuf. If so, then
2996  // we should not lose any raw bytes when switching to raw I/O here.
2997  if (fd_table[clientConnection->fd].ssl.get())
2998  fd_table[clientConnection->fd].useDefaultIo();
2999 
3000  // XXX: assuming that there was an HTTP/1.1 CONNECT to begin with...
3001  // reset the current protocol to HTTP/1.1 (was "HTTPS" for the bumping process)
3003  assert(!pipeline.empty());
3004  Http::StreamPointer context = pipeline.front();
3005  Must(context);
3006  Must(context->http);
3007  ClientHttpRequest *http = context->http;
3009  context->finished();
3010  if (transparent()) {
3011  // For transparent connections, make a new fake CONNECT request, now
3012  // with SNI as target. doCallout() checks, adaptations may need that.
3013  return fakeAConnectRequest("splice", preservedClientData);
3014  } else {
3015  // For non transparent connections make a new tunneled CONNECT, which
3016  // also sets the HttpRequest::flags::forceTunnel flag to avoid
3017  // respond with "Connection Established" to the client.
3018  // This fake CONNECT request required to allow use of SNI in
3019  // doCallout() checks and adaptations.
3021  }
3022 }
3023 
3024 void
3026 {
3027  // This is the Step2 of the SSL bumping
3029  Http::StreamPointer context = pipeline.front();
3030  ClientHttpRequest *http = context ? context->http : nullptr;
3031 
3032  if (sslServerBump->at(XactionStep::tlsBump1)) {
3033  sslServerBump->step = XactionStep::tlsBump2;
3034  // Run a accessList check to check if want to splice or continue bumping
3035 
3040  fillChecklist(*acl_checklist);
3041  acl_checklist->nonBlockingCheck(httpsSslBumpStep2AccessCheckDone, this);
3042  return;
3043  }
3044 
3045  // will call httpsPeeked() with certificate and connection, eventually
3046  Security::ContextPointer unConfiguredCTX(Ssl::createSSLContext(port->secure.signingCa.cert, port->secure.signingCa.pkey, port->secure));
3047  fd_table[clientConnection->fd].html?amicTlsContext = unConfiguredCTX;
3048 
3049  if (!httpsCreate(this, unConfiguredCTX))
3050  return;
3051 
3052  switchedToHttps_ = true;
3053 
3054  auto ssl = fd_table[clientConnection->fd].ssl.get();
3055  BIO *b = SSL_get_rbio(ssl);
3056  Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3057  bio->setReadBufData(inBuf);
3058  bio->hold(true);
3059 
3060  // We have successfully parsed client Hello, but our TLS handshake parser is
3061  // forgiving. Now we use a TLS library to parse the same bytes, so that we
3062  // can honor on_unsupported_protocol if needed. If there are no errors, we
3063  // expect Security::Accept() to ask us to write (our) TLS server Hello. We
3064  // also allow an ioWantRead result in case some fancy TLS extension that
3065  // Squid does not yet understand requires reading post-Hello client bytes.
3066  const auto handshakeResult = acceptTls();
3067  if (!handshakeResult.wantsIo())
3068  return handleSslBumpHandshakeError(handshakeResult);
3069 
3070  // We need to reset inBuf here, to be used by incoming requests in the case
3071  // of SSL bump
3072  inBuf.clear();
3073 
3074  debugs(83, 5, "Peek and splice at step2 done. Start forwarding the request!!! ");
3075  sslServerBump->step = XactionStep::tlsBump3;
3077 }
3078 
3080 void
3082 {
3083  auto errCategory = ERR_NONE;
3084 
3085  switch (handshakeResult.category) {
3087  static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_SUCCESS");
3088  updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3089  break;
3090  }
3091 
3093  static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_READ");
3094  updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3095  break;
3096  }
3097 
3099  static const auto d = MakeNamedErrorDetail("TLS_ACCEPT_UNEXPECTED_WRITE");
3100  updateError(errCategory = ERR_GATEWAY_FAILURE, d);
3101  break;
3102  }
3103 
3105  debugs(83, (handshakeResult.important ? DBG_IMPORTANT : 2), "ERROR: " << handshakeResult.errorDescription <<
3106  " while SslBump-accepting a TLS connection on " << clientConnection << ": " << handshakeResult.errorDetail);
3107  updateError(errCategory = ERR_SECURE_ACCEPT_FAIL, handshakeResult.errorDetail);
3108  break;
3109 
3110  }
3111 
3112  if (!tunnelOnError(HttpRequestMethod(), errCategory))
3114 }
3115 
3116 void
3118 {
3119  auto ssl = fd_table[clientConnection->fd].ssl.get();
3120  BIO *b = SSL_get_rbio(ssl);
3121  assert(b);
3122  Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3123 
3124  debugs(33, 5, "PeekAndSplice mode, proceed with client negotiation. Current state:" << SSL_state_string_long(ssl));
3125  bio->hold(false);
3126 
3128  switchedToHttps_ = true;
3129 }
3130 
3131 void
3133 {
3134  Must(sslServerBump != NULL);
3135  Must(sslServerBump->request == pic.request);
3136  Must(pipeline.empty() || pipeline.front()->http == nullptr || pipeline.front()->http->request == pic.request.getRaw());
3137 
3138  if (Comm::IsConnOpen(pic.connection)) {
3140  debugs(33, 5, "bumped HTTPS server: " << tlsConnectHostOrIp);
3141  } else
3142  debugs(33, 5, "Error while bumping: " << tlsConnectHostOrIp);
3143 
3145 }
3146 
3147 #endif /* USE_OPENSSL */
3148 
3149 bool
3150 ConnStateData::initiateTunneledRequest(HttpRequest::Pointer const &cause, Http::MethodType const method, const char *reason, const SBuf &payload)
3151 {
3152  // fake a CONNECT request to force connState to tunnel
3153  SBuf connectHost;
3154  unsigned short connectPort = 0;
3155 
3156  if (pinning.serverConnection != nullptr) {
3157  static char ip[MAX_IPSTRLEN];
3158  connectHost = pinning.serverConnection->remote.toStr(ip, sizeof(ip));
3159  connectPort = pinning.serverConnection->remote.port();
3160  } else if (cause) {
3161  connectHost = cause->url.hostOrIp();
3162  connectPort = cause->url.port();
3163 #if USE_OPENSSL
3164  } else if (!tlsConnectHostOrIp.isEmpty()) {
3165  connectHost = tlsConnectHostOrIp;
3166  connectPort = tlsConnectPort;
3167 #endif
3168  } else if (transparent()) {
3169  static char ip[MAX_IPSTRLEN];
3170  connectHost = clientConnection->local.toStr(ip, sizeof(ip));
3171  connectPort = clientConnection->local.port();
3172  } else {
3173  // Typical cases are malformed HTTP requests on http_port and malformed
3174  // TLS handshakes on non-bumping https_port. TODO: Discover these
3175  // problems earlier so that they can be classified/detailed better.
3176  debugs(33, 2, "Not able to compute URL, abort request tunneling for " << reason);
3177  // TODO: throw when nonBlockingCheck() callbacks gain job protections
3178  static const auto d = MakeNamedErrorDetail("TUNNEL_TARGET");
3180  return false;
3181  }
3182 
3183  debugs(33, 2, "Request tunneling for " << reason);
3184  ClientHttpRequest *http = buildFakeRequest(method, connectHost, connectPort, payload);
3186  request->flags.forceTunnel = true;
3187  http->calloutContext = new ClientRequestContext(http);
3188  http->doCallouts();
3190  return true;
3191 }
3192 
3193 bool
3194 ConnStateData::fakeAConnectRequest(const char *reason, const SBuf &payload)
3195 {
3196  debugs(33, 2, "fake a CONNECT request to force connState to tunnel for " << reason);
3197 
3198  SBuf connectHost;
3199  assert(transparent());
3200  const unsigned short connectPort = clientConnection->local.port();
3201 
3202 #if USE_OPENSSL
3203  if (!tlsClientSni_.isEmpty())
3204  connectHost.assign(tlsClientSni_);
3205  else
3206 #endif
3207  {
3208  static char ip[MAX_IPSTRLEN];
3209  clientConnection->local.toHostStr(ip, sizeof(ip));
3210  connectHost.assign(ip);
3211  }
3212 
3213  ClientHttpRequest *http = buildFakeRequest(Http::METHOD_CONNECT, connectHost, connectPort, payload);
3214 
3215  http->calloutContext = new ClientRequestContext(http);
3217  http->doCallouts();
3219  return true;
3220 }
3221 
3223 ConnStateData::buildFakeRequest(Http::MethodType const method, SBuf &useHost, unsigned short usePort, const SBuf &payload)
3224 {
3225  ClientHttpRequest *http = new ClientHttpRequest(this);
3226  Http::Stream *stream = new Http::Stream(clientConnection, http);
3227 
3228  StoreIOBuffer tempBuffer;
3229  tempBuffer.data = stream->reqbuf;
3230  tempBuffer.length = HTTP_REQBUF_SZ;
3231 
3232  ClientStreamData newServer = new clientReplyContext(http);
3233  ClientStreamData newClient = stream;
3236  clientSocketDetach, newClient, tempBuffer);
3237 
3238  stream->flags.parsed_ok = 1; // Do we need it?
3239  stream->mayUseConnection(true);
3240  extendLifetime();
3241  stream->registerWithConn();
3242 
3245  // Setup Http::Request object. Maybe should be replaced by a call to (modified)
3246  // clientProcessRequest
3249  request->url.setScheme(proto, nullptr);
3250  request->method = method;
3251  request->url.host(useHost.c_str());
3252  request->url.port(usePort);
3253 
3254  http->uri = SBufToCstring(request->effectiveRequestUri());
3255  http->initRequest(request.getRaw());
3256 
3257  request->manager(this, http->al);
3258 
3259  if (proto == AnyP::PROTO_HTTP)
3260  request->header.putStr(Http::HOST, useHost.c_str());
3261 
3262  request->sources |= ((switchedToHttps() || port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
3263 #if USE_AUTH
3264  if (getAuth())
3265  request->auth_user_request = getAuth();
3266 #endif
3267 
3268  inBuf = payload;
3269  flags.readMore = false;
3270 
3271  return http;
3272 }
3273 
3275 static bool
3277 {
3278  if (!Comm::IsConnOpen(c)) {
3279  Must(NHttpSockets > 0); // we tried to open some
3280  --NHttpSockets; // there will be fewer sockets than planned
3281  Must(HttpSockets[NHttpSockets] < 0); // no extra fds received
3282 
3283  if (!NHttpSockets) // we could not open any listen sockets at all
3284  fatalf("Unable to open %s",FdNote(portType));
3285 
3286  return false;
3287  }
3288  return true;
3289 }
3290 
3292 static bool
3294 {
3295  bool found = false;
3296  for (int i = 0; i < NHttpSockets && !found; ++i) {
3297  if ((found = HttpSockets[i] < 0))
3298  HttpSockets[i] = conn->fd;
3299  }
3300  return found;
3301 }
3302 
3303 static void
3305 {
3306  for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) {
3307  const SBuf &scheme = AnyP::UriScheme(s->transport.protocol).image();
3308 
3310  debugs(1, DBG_IMPORTANT, "WARNING: You have too many '" << scheme << "_port' lines." <<
3311  Debug::Extra << "The limit is " << MAXTCPLISTENPORTS << " HTTP ports.");
3312  continue;
3313  }
3314 
3315 #if USE_OPENSSL
3316  if (s->flags.tunnelSslBumping) {
3317  if (!Config.accessList.ssl_bump) {
3318  debugs(33, DBG_IMPORTANT, "WARNING: No ssl_bump configured. Disabling ssl-bump on " << scheme << "_port " << s->s);
3319  s->flags.tunnelSslBumping = false;
3320  }
3321  if (!s->secure.staticContext && !s->secure.generateHostCertificates) {
3322  debugs(1, DBG_IMPORTANT, "Will not bump SSL at " << scheme << "_port " << s->s << " due to TLS initialization failure.");
3323  s->flags.tunnelSslBumping = false;
3324  if (s->transport.protocol == AnyP::PROTO_HTTP)
3325  s->secure.encryptTransport = false;
3326  }
3327  if (s->flags.tunnelSslBumping) {
3328  // Create ssl_ctx cache for this port.
3329  Ssl::TheGlobalContextStorage.addLocalStorage(s->s, s->secure.html?amicCertMemCacheSize);
3330  }
3331  }
3332 #endif
3333 
3334  if (s->secure.encryptTransport && !s->secure.staticContext) {
3335  debugs(1, DBG_CRITICAL, "ERROR: Ignoring " << scheme << "_port " << s->s << " due to TLS context initialization failure.");
3336  continue;
3337  }
3338 
3339  // Fill out a Comm::Connection which IPC will open as a listener for us
3340  // then pass back when active so we can start a TcpAcceptor subscription.
3341  s->listenConn = new Comm::Connection;
3342  s->listenConn->local = s->s;
3343 
3344  s->listenConn->flags = COMM_NONBLOCKING | (s->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) |
3345  (s->flags.natIntercept ? COMM_INTERCEPTION : 0) |
3346  (s->workerQueues ? COMM_REUSEPORT : 0);
3347 
3348  typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
3349  if (s->transport.protocol == AnyP::PROTO_HTTP) {
3350  // setup the subscriptions such that new connections accepted by listenConn are handled by HTTP
3353 
3354  AsyncCall::Pointer listenCall = asyncCall(33,2, "clientListenerConnectionOpened",
3356  Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnHttpSocket, listenCall);
3357 
3358  } else if (s->transport.protocol == AnyP::PROTO_HTTPS) {
3359  // setup the subscriptions such that new connections accepted by listenConn are handled by HTTPS
3362 
3363  AsyncCall::Pointer listenCall = asyncCall(33, 2, "clientListenerConnectionOpened",
3365  s, Ipc::fdnHttpsSocket, sub));
3366  Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnHttpsSocket, listenCall);
3367  }
3368 
3369  HttpSockets[NHttpSockets] = -1; // set in clientListenerConnectionOpened
3370  ++NHttpSockets;
3371  }
3372 }
3373 
3374 void
3376 {
3377  // Fill out a Comm::Connection which IPC will open as a listener for us
3378  port->listenConn = new Comm::Connection;
3379  port->listenConn->local = port->s;
3380  port->listenConn->flags =
3382  (port->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) |
3383  (port->flags.natIntercept ? COMM_INTERCEPTION : 0);
3384 
3385  // route new connections to subCall
3386  typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
3388  AsyncCall::Pointer listenCall =
3389  asyncCall(33, 2, "clientListenerConnectionOpened",
3391  port, fdNote, sub));
3392  Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, port->listenConn, fdNote, listenCall);
3393 
3395  HttpSockets[NHttpSockets] = -1;
3396  ++NHttpSockets;
3397 }
3398 
3400 static void
3402 {
3403  Must(s != NULL);
3404 
3405  if (!OpenedHttpSocket(s->listenConn, portTypeNote))
3406  return;
3407 
3408  Must(Comm::IsConnOpen(s->listenConn));
3409 
3410  // TCP: setup a job to handle accept() with subscribed handler
3411  AsyncJob::Start(new Comm::TcpAcceptor(s, FdNote(portTypeNote), sub));
3412 
3413  debugs(1, Important(13), "Accepting " <<
3414  (s->flags.natIntercept ? "NAT intercepted " : "") <<
3415  (s->flags.tproxyIntercept ? "TPROXY intercepted " : "") <<
3416  (s->flags.tunnelSslBumping ? "SSL bumped " : "") <<
3417  (s->flags.accelSurrogate ? "reverse-proxy " : "")
3418  << FdNote(portTypeNote) << " connections at "
3419  << s->listenConn);
3420 
3421  Must(AddOpenedHttpSocket(s->listenConn)); // otherwise, we have received a fd we did not ask for
3422 
3423 #if USE_SYSTEMD
3424  // When the very first port opens, tell systemd we are able to serve connections.
3425  // Subsequent sd_notify() calls, including calls during reconfiguration,
3426  // do nothing because the first call parameter is 1.
3427  // XXX: Send the notification only after opening all configured ports.
3428  if (opt_foreground || opt_no_daemon) {
3429  const auto result = sd_notify(1, "READY=1");
3430  if (result < 0) {
3431  debugs(1, DBG_IMPORTANT, "WARNING: failed to send start-up notification to systemd" <<
3432  Debug::Extra << "sd_notify() error: " << xstrerr(-result));
3433  }
3434  }
3435 #endif
3436 }
3437 
3438 void
3440 {
3443 
3444  if (NHttpSockets < 1)
3445  fatal("No HTTP, HTTPS, or FTP ports configured");
3446 }
3447 
3448 void
3450 {
3451  for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) {
3452  if (s->listenConn != NULL) {
3453  debugs(1, Important(14), "Closing HTTP(S) port " << s->listenConn->local);
3454  s->listenConn->close();
3455  s->listenConn = NULL;
3456  }
3457  }
3458 
3460 
3461  // TODO see if we can drop HttpSockets array entirely */
3462  for (int i = 0; i < NHttpSockets; ++i) {
3463  HttpSockets[i] = -1;
3464  }
3465 
3466  NHttpSockets = 0;
3467 }
3468 
3469 int
3471 {
3472  SBuf vary(request->vary_headers);
3473  const auto &reply = entry->mem().freshestReply();
3474  auto has_vary = reply.header.has(Http::HdrType::VARY);
3475 #if X_ACCELERATOR_VARY
3476 
3477  has_vary |=
3478  reply.header.has(Http::HdrType::HDR_X_ACCELERATOR_VARY);
3479 #endif
3480 
3481  if (!has_vary || entry->mem_obj->vary_headers.isEmpty()) {
3482  if (!vary.isEmpty()) {
3483  /* Oops... something odd is going on here.. */
3484  debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary object on second attempt, '" <<
3485  entry->mem_obj->urlXXX() << "' '" << vary << "'");
3486  request->vary_headers.clear();
3487  return VARY_CANCEL;
3488  }
3489 
3490  if (!has_vary) {
3491  /* This is not a varying object */
3492  return VARY_NONE;
3493  }
3494 
3495  /* virtual "vary" object found. Calculate the vary key and
3496  * continue the search
3497  */
3498  vary = httpMakeVaryMark(request, &reply);
3499 
3500  if (!vary.isEmpty()) {
3501  request->vary_headers = vary;
3502  return VARY_OTHER;
3503  } else {
3504  /* Ouch.. we cannot handle this kind of variance */
3505  /* XXX This cannot really happen, but just to be complete */
3506  return VARY_CANCEL;
3507  }
3508  } else {
3509  if (vary.isEmpty()) {
3510  vary = httpMakeVaryMark(request, &reply);
3511 
3512  if (!vary.isEmpty())
3513  request->vary_headers = vary;
3514  }
3515 
3516  if (vary.isEmpty()) {
3517  /* Ouch.. we cannot handle this kind of variance */
3518  /* XXX This cannot really happen, but just to be complete */
3519  return VARY_CANCEL;
3520  } else if (vary.cmp(entry->mem_obj->vary_headers) == 0) {
3521  return VARY_MATCH;
3522  } else {
3523  /* Oops.. we have already been here and still haven't
3524  * found the requested variant. Bail out
3525  */
3526  debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary match on second attempt, '" <<
3527  entry->mem_obj->urlXXX() << "' '" << vary << "'");
3528  return VARY_CANCEL;
3529  }
3530  }
3531 }
3532 
3535 {
3536  const auto checklist = new ACLFilledChecklist(acl, nullptr, nullptr);
3537  clientAclChecklistFill(*checklist, http);
3538  return checklist;
3539 }
3540 
3541 void
3543 {
3544  assert(http);
3545 
3546  if (!checklist.request && http->request)
3547  checklist.setRequest(http->request);
3548 
3549  if (!checklist.al && http->al) {
3550  checklist.al = http->al;
3551  checklist.syncAle(http->request, http->log_uri);
3552  if (!checklist.reply && http->al->reply) {
3553  checklist.reply = http->al->reply.getRaw();
3554  HTTPMSGLOCK(checklist.reply);
3555  }
3556  }
3557 
3558  if (const auto conn = http->getConn())
3559  checklist.setConn(conn); // may already be set
3560 }
3561 
3562 void
3564 {
3565  const auto context = pipeline.front();
3566  if (const auto http = context ? context->http : nullptr)
3567  return clientAclChecklistFill(checklist, http); // calls checklist.setConn()
3568 
3569  // no requests, but we always have connection-level details
3570  // TODO: ACL checks should not require a mutable ConnStateData. Adjust the
3571  // code that accidentally violates that principle to remove this const_cast!
3572  checklist.setConn(const_cast<ConnStateData*>(this));
3573 
3574  // Set other checklist fields inside our fillConnectionLevelDetails() rather
3575  // than here because clientAclChecklistFill() code path calls that method
3576  // (via ACLFilledChecklist::setConn()) rather than calling us directly.
3577 }
3578 
3579 void
3581 {
3582  assert(checklist.conn() == this);
3584 
3585  if (!checklist.request) { // preserve (better) addresses supplied by setRequest()
3586  checklist.src_addr = clientConnection->remote;
3587  checklist.my_addr = clientConnection->local; // TODO: or port->s?
3588  }
3589 
3590 #if USE_OPENSSL
3591  if (!checklist.sslErrors && sslServerBump)
3593 #endif
3594 
3595  if (!checklist.rfc931[0]) // checklist creator may have supplied it already
3596  checklist.setIdent(clientConnection->rfc931);
3597 
3598 }
3599 
3600 bool
3602 {
3604 }
3605 
3608 {
3609  bodyPipe = new BodyPipe(this);
3610  if (size >= 0)
3612  else
3614  return bodyPipe;
3615 }
3616 
3617 int64_t
3619 {
3620  if (!bodyPipe)
3621  return 0; // request without a body or read/produced all body bytes
3622 
3623  if (!bodyPipe->bodySizeKnown())
3624  return -1; // probably need to read more, but we cannot be sure
3625 
3626  const int64_t needToProduce = bodyPipe->unproducedSize();
3627  const int64_t haveAvailable = static_cast<int64_t>(inBuf.length());
3628 
3629  if (needToProduce <= haveAvailable)
3630  return 0; // we have read what we need (but are waiting for pipe space)
3631 
3632  return needToProduce - haveAvailable;
3633 }
3634 
3635 void
3637 {
3638  debugs(33, 4, HERE << "receiving error (" << clientConnection << "): " << error <<
3639  "; old sending error: " <<
3640  (stoppedSending() ? stoppedSending_ : "none"));
3641 
3642  if (const char *oldError = stoppedReceiving()) {
3643  debugs(33, 3, HERE << "already stopped receiving: " << oldError);
3644  return; // nothing has changed as far as this connection is concerned
3645  }
3646 
3648 
3649  if (const char *sendError = stoppedSending()) {
3650  debugs(33, 3, HERE << "closing because also stopped sending: " << sendError);
3652  }
3653 }
3654 
3655 void
3657 {
3658  if (bodyPipe != NULL) {
3659  debugs(33, 4, HERE << "no consumer for virgin body " << bodyPipe->status());
3661  }
3662 }
3663 
3665 void
3667 {
3668  Must(bodyPipe != NULL);
3669  debugs(33, 5, HERE << "start dechunking" << bodyPipe->status());
3670  assert(!bodyParser);
3672 }
3673 
3675 void
3677 {
3678  debugs(33, 5, HERE << "finish dechunking: " << withSuccess);
3679 
3680  if (bodyPipe != NULL) {
3681  debugs(33, 7, HERE << "dechunked tail: " << bodyPipe->status());
3682  BodyPipe::Pointer myPipe = bodyPipe;
3683  stopProducingFor(bodyPipe, withSuccess); // sets bodyPipe->bodySize()
3684  Must(!bodyPipe); // we rely on it being nil after we are done with body
3685  if (withSuccess) {
3686  Must(myPipe->bodySizeKnown());
3687  Http::StreamPointer context = pipeline.front();
3688  if (context != NULL && context->http && context->http->request)
3689  context->http->request->setContentLength(myPipe->bodySize());
3690  }
3691  }
3692 
3693  delete bodyParser;
3694  bodyParser = NULL;
3695 }
3696 
3697 // XXX: this is an HTTP/1-only operation
3698 void
3700 {
3701  if (const auto context = pipeline.front()) {
3702  if (context->http)
3703  context->http->al->reply = msg.reply;
3704  }
3705 
3706  if (!isOpen()) {
3707  debugs(33, 3, HERE << "ignoring 1xx due to earlier closure");
3708  return;
3709  }
3710 
3711  // HTTP/1 1xx status messages are only valid when there is a transaction to trigger them
3712  if (!pipeline.empty()) {
3713  HttpReply::Pointer rep(msg.reply);
3714  Must(rep);
3715  // remember the callback
3717 
3720 
3721  if (!writeControlMsgAndCall(rep.getRaw(), call)) {
3722  // but still inform the caller (so it may resume its operation)
3724  }
3725  return;
3726  }
3727 
3728  debugs(33, 3, HERE << " closing due to missing context for 1xx");
3730 }
3731 
3732 void
3734 {
3736 
3737  if (Http::StreamPointer deferredRequest = pipeline.front()) {
3738  debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded after control msg wrote");
3739  ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
3740  }
3741 }
3742 
3744 void
3746 {
3747  // FwdState might repin a failed connection sooner than this close
3748  // callback is called for the failed connection.
3749  assert(pinning.serverConnection == io.conn);
3750  pinning.closeHandler = NULL; // Comm unregisters handlers before calling
3751  const bool sawZeroReply = pinning.zeroReply; // reset when unpinning
3752  pinning.serverConnection->noteClosure();
3753  unpinConnection(false);
3754 
3755  if (sawZeroReply && clientConnection != NULL) {
3756  debugs(33, 3, "Closing client connection on pinned zero reply.");
3758  }
3759 
3760 }
3761 
3762 void
3764 {
3765  pinConnection(pinServer, *request);
3766 }
3767 
3768 void
3770 {
3771  Must(pic.connection);
3772  Must(pic.request);
3773  pinConnection(pic.connection, *pic.request);
3774 
3775  // monitor pinned server connection for remote-end closures.
3777 
3778  if (pipeline.empty())
3779  kick(); // in case clientParseRequests() was blocked by a busy pic.connection
3780 }
3781 
3783 void
3785 {
3786  if (Comm::IsConnOpen(pinning.serverConnection) &&
3787  pinning.serverConnection->fd == pinServer->fd) {
3788  debugs(33, 3, "already pinned" << pinServer);
3789  return;
3790  }
3791 
3792  unpinConnection(true); // closes pinned connection, if any, and resets fields
3793 
3794  pinning.serverConnection = pinServer;
3795 
3796  debugs(33, 3, HERE << pinning.serverConnection);
3797 
3798  Must(pinning.serverConnection != NULL);
3799 
3800  const char *pinnedHost = "[unknown]";
3801  pinning.host = xstrdup(request.url.host());
3802  pinning.port = request.url.port();
3803  pinnedHost = pinning.host;
3804  pinning.pinned = true;
3805  if (CachePeer *aPeer = pinServer->getPeer())
3806  pinning.peer = cbdataReference(aPeer);
3807  pinning.auth = request.flags.connectionAuth;
3808  char stmp[MAX_IPSTRLEN];
3809  char desc[FD_DESC_SZ];
3810  snprintf(desc, FD_DESC_SZ, "%s pinned connection for %s (%d)",
3811  (pinning.auth || !pinning.peer) ? pinnedHost : pinning.peer->name,
3813  clientConnection->fd);
3814  fd_note(pinning.serverConnection->fd, desc);
3815 
3817  pinning.closeHandler = JobCallback(33, 5,
3819  // remember the pinned connection so that cb does not unpin a fresher one
3820  typedef CommCloseCbParams Params;
3821  Params &params = GetCommParams<Params>(pinning.closeHandler);
3822  params.conn = pinning.serverConnection;
3823  comm_add_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3824 }
3825 
3828 void
3830 {
3831  if (pinning.readHandler != NULL)
3832  return; // already monitoring
3833 
3835  pinning.readHandler = JobCallback(33, 3,
3837  Comm::Read(pinning.serverConnection, pinning.readHandler);
3838 }
3839 
3840 void
3842 {
3843  if (pinning.readHandler != NULL) {
3844  Comm::ReadCancel(pinning.serverConnection->fd, pinning.readHandler);
3845  pinning.readHandler = NULL;
3846  }
3847 }
3848 
3849 #if USE_OPENSSL
3850 bool
3852 {
3853  // A ready-for-reading connection means that the TLS server either closed
3854  // the connection, sent us some unexpected HTTP data, or started TLS
3855  // renegotiations. We should close the connection except for the last case.
3856 
3857  Must(pinning.serverConnection != nullptr);
3858  auto ssl = fd_table[pinning.serverConnection->fd].ssl.get();
3859  if (!ssl)
3860  return false;
3861 
3862  char buf[1];
3863  const int readResult = SSL_read(ssl, buf, sizeof(buf));
3864 
3865  if (readResult > 0 || SSL_pending(ssl) > 0) {
3866  debugs(83, 2, pinning.serverConnection << " TLS application data read");
3867  return false;
3868  }
3869 
3870  switch(const int error = SSL_get_error(ssl, readResult)) {
3871  case SSL_ERROR_WANT_WRITE:
3872  debugs(83, DBG_IMPORTANT, pinning.serverConnection << " TLS SSL_ERROR_WANT_WRITE request for idle pinned connection");
3873  // fall through to restart monitoring, for now
3874  case SSL_ERROR_NONE:
3875  case SSL_ERROR_WANT_READ:
3877  return true;
3878 
3879  default:
3880  debugs(83, 2, pinning.serverConnection << " TLS error: " << error);
3881  return false;
3882  }
3883 
3884  // not reached
3885  return true;
3886 }
3887 #endif
3888 
3891 void
3893 {
3894  pinning.readHandler = NULL; // Comm unregisters handlers before calling
3895 
3896  if (io.flag == Comm::ERR_CLOSING)
3897  return; // close handler will clean up
3898 
3899  Must(pinning.serverConnection == io.conn);
3900 
3901 #if USE_OPENSSL
3903  return;
3904 #endif
3905 
3906  const bool clientIsIdle = pipeline.empty();
3907 
3908  debugs(33, 3, "idle pinned " << pinning.serverConnection << " read " <<
3909  io.size << (clientIsIdle ? " with idle client" : ""));
3910 
3911  pinning.serverConnection->close();
3912 
3913  // If we are still sending data to the client, do not close now. When we are done sending,
3914  // ConnStateData::kick() checks pinning.serverConnection and will close.
3915  // However, if we are idle, then we must close to inform the idle client and minimize races.
3916  if (clientIsIdle && clientConnection != NULL)
3918 }
3919 
3922 {
3923  debugs(33, 7, pinning.serverConnection);
3924  Must(request);
3925 
3926  const auto pinningError = [&](const err_type type) {
3927  unpinConnection(true);
3928  HttpRequestPointer requestPointer = request;
3929  return ErrorState::NewForwarding(type, requestPointer, ale);
3930  };
3931 
3932  if (!Comm::IsConnOpen(pinning.serverConnection))
3933  throw pinningError(ERR_ZERO_SIZE_OBJECT);
3934 
3935  if (pinning.auth && pinning.host && strcasecmp(pinning.host, request->url.host()) != 0)
3936  throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_CONFLICT_HOST
3937 
3938  if (pinning.port != request->url.port())
3939  throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_CONFLICT_HOST
3940 
3941  if (pinning.peer && !cbdataReferenceValid(pinning.peer))
3942  throw pinningError(ERR_ZERO_SIZE_OBJECT);
3943 
3944  if (pinning.peerAccessDenied)
3945  throw pinningError(ERR_CANNOT_FORWARD); // or generalize ERR_FORWARDING_DENIED
3946 
3948  return pinning.serverConnection;
3949 }
3950 
3953 {
3954  if (const auto connManager = request ? request->pinnedConnection() : nullptr)
3955  return connManager->borrowPinnedConnection(request, ale);
3956 
3957  // ERR_CANNOT_FORWARD is somewhat misleading here; we can still forward, but
3958  // there is no point since the client connection is now gone
3959  HttpRequestPointer requestPointer = request;
3960  throw ErrorState::NewForwarding(ERR_CANNOT_FORWARD, requestPointer, ale);
3961 }
3962 
3963 void
3965 {
3966  debugs(33, 3, HERE << pinning.serverConnection);
3967 
3968  if (pinning.peer)
3970 
3971  if (Comm::IsConnOpen(pinning.serverConnection)) {
3972  if (pinning.closeHandler != NULL) {
3973  comm_remove_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3974  pinning.closeHandler = NULL;
3975  }
3976 
3978 
3979  // close the server side socket if requested
3980  if (andClose)
3981  pinning.serverConnection->close();
3982  pinning.serverConnection = NULL;
3983  }
3984 
3985  safe_free(pinning.host);
3986 
3987  pinning.zeroReply = false;
3988  pinning.peerAccessDenied = false;
3989 
3990  /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host
3991  * connection has gone away */
3992 }
3993 
3994 void
3996 {
3997  auto error = rawError; // (cheap) copy so that we can detail
3998  // We detail even ERR_NONE: There should be no transactions left, and
3999  // detailed ERR_NONE will be unused. Otherwise, this detail helps in triage.
4000  if (!error.detail) {
4001  static const auto d = MakeNamedErrorDetail("WITH_CLIENT");
4002  error.detail = d;
4003  }
4004 
4005  debugs(33, 3, pipeline.count() << '/' << pipeline.nrequests << " after " << error);
4006 
4007  if (pipeline.empty()) {
4008  bareError.update(error); // XXX: bareLogTagsErrors
4009  } else {
4010  // We terminate the current CONNECT/PUT/etc. context below, logging any
4011  // error details, but that context may leave unparsed bytes behind.
4012  // Consume them to stop checkLogging() from logging them again later.
4013  const auto intputToConsume =
4014 #if USE_OPENSSL
4015  parsingTlsHandshake ? "TLS handshake" : // more specific than CONNECT
4016 #endif
4017  bodyPipe ? "HTTP request body" :
4018  pipeline.back()->mayUseConnection() ? "HTTP CONNECT" :
4019  nullptr;
4020 
4021  while (const auto context = pipeline.front()) {
4022  context->noteIoError(error, lte);
4023  context->finished(); // cleanup and self-deregister
4024  assert(context != pipeline.front());
4025  }
4026 
4027  if (intputToConsume && !inBuf.isEmpty()) {
4028  debugs(83, 5, "forgetting client " << intputToConsume << " bytes: " << inBuf.length());
4029  inBuf.clear();
4030  }
4031  }
4032 
4034 }
4035 
4037 void
4039 {
4040  // to simplify our logic, we assume that terminateAll() has been called
4041  assert(pipeline.empty());
4042 
4043  // do not log connections that closed after a transaction (it is normal)
4044  // TODO: access_log needs ACLs to match received-no-bytes connections
4045  if (pipeline.nrequests && inBuf.isEmpty())
4046  return;
4047 
4048  /* Create a temporary ClientHttpRequest object. Its destructor will log. */
4049  ClientHttpRequest http(this);
4050  http.req_sz = inBuf.length();
4051  // XXX: Or we died while waiting for the pinned connection to become idle.
4052  http.setErrorUri("error:transaction-end-before-headers");
4053  http.updateError(bareError);
4054 }
4055 
4056 bool
4058 {
4059  // PROXY protocol bytes are meant for us and, hence, cannot be tunneled
4061  return false;
4062 
4063  // If our decision here is negative, configuration changes are irrelevant.
4064  // Otherwise, clientTunnelOnError() rechecks configuration before tunneling.
4066  return false;
4067 
4068  // TODO: Figure out whether/how we can support FTP tunneling.
4069  if (port->transport.protocol == AnyP::PROTO_FTP)
4070  return false;
4071 
4072 #if USE_OPENSSL
4073  if (parsingTlsHandshake)
4074  return true;
4075 
4076  // the 1st HTTP request on a bumped connection
4078  return true;
4079 #endif
4080 
4081  // the 1st HTTP(S) request on a connection to an intercepting port
4082  if (!pipeline.nrequests && transparent())
4083  return true;
4084 
4085  return false;
4086 }
4087 
4090 {
4091  if (!theNotes)
4092  theNotes = new NotePairs;
4093  return theNotes;
4094 }
4095 
4096 std::ostream &
4097 operator <<(std::ostream &os, const ConnStateData::PinnedIdleContext &pic)
4098 {
4099  return os << pic.connection << ", request=" << pic.request;
4100 }
4101 
4102 std::ostream &
4103 operator <<(std::ostream &os, const ConnStateData::ServerConnectionContext &scc)
4104 {
4105  return os << scc.conn_ << ", srv_bytes=" << scc.preReadServerBytes.length();
4106 }
4107 
const MemBuf & buf() const
Definition: BodyPipe.h:137
void Read(const Comm::ConnectionPointer &conn, AsyncCall::Pointer &callback)
Definition: Read.cc:40
@ initClient
HTTP or FTP client.
#define FD_DESC_SZ
Definition: defines.h:32
bool shouldPreserveClientData() const
void fatal(const char *message)
Definition: fatal.cc:28
void whenClientIpKnown()
const char * xstrerr(int error)
Definition: xstrerror.cc:83
Definition: parse.c:104
#define USER_IDENT_SZ
Definition: defines.h:37
virtual void noteBodyConsumerAborted(BodyPipe::Pointer)=0
@ bumpPeek
Definition: support.h:126
#define FQDN_LOOKUP_IF_MISS
Definition: defines.h:34
@ LOG_TCP_IMS_HIT
Definition: LogTags.h:47
AnyP::ProtocolVersion transferProtocol
Definition: Server.h:105
char method[16]
Definition: tcp-banger2.c:115
@ ERR_SECURE_CONNECT_FAIL
Definition: forward.h:31
char * buf
Definition: MemBuf.h:134
void checkpoint(const Connection &, const Acl::ChecklistFiller &)
Definition: KeyLogger.h:50
virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io)
Our close handler called by Comm when the pinned connection is closed.
AsyncCall::Pointer comm_add_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:922
void clientAfterReadingRequests()
SBuf tlsClientSni_
TLS client delivered SNI value. Empty string if none has been received.
Definition: client_side.h:489
#define MYNAME
Definition: Debug.h:165
int half_closed_clients
Definition: SquidConfig.h:296
void pinConnection(const Comm::ConnectionPointer &pinServerConn, const HttpRequest &request)
Forward future client requests using the given server connection.
virtual void processParsedRequest(Http::StreamPointer &)=0
start processing a freshly parsed request
void fillConnectionLevelDetails(ACLFilledChecklist &) const
SQUIDCEXTERN CSR clientGetMoreData
Definition: client_side.h:528
void IOACB(const CommAcceptCbParams &params)
Definition: CommCalls.h:33
@ hoReply
Definition: HttpHeader.h:37
const char * findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end)
SourceLocationId id() const
same-location exceptions have the same ID
Definition: TextException.h:39
void * xcalloc(size_t n, size_t sz)
Definition: xalloc.cc:71
unsigned int major
major version number
#define Here()
source code location of the caller
Definition: Here.h:15
int getMyPort(void)
Definition: tools.cc:1008
@ scMethodNotAllowed
Definition: StatusCode.h:49
void pinBusyConnection(const Comm::ConnectionPointer &pinServerConn, const HttpRequest::Pointer &request)
void clientProcessRequestFinished(ConnStateData *conn, const HttpRequest::Pointer &request)
void InRamCertificateDbKey(const Ssl::CertificateProperties &certProperties, SBuf &key)
Definition: support.cc:1388
@ HDR_X_ACCELERATOR_VARY
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
struct SquidConfig::@97 Timeout
@ scPayloadTooLarge
Definition: StatusCode.h:57
SBuf & assign(const SBuf &S)
Definition: SBuf.cc:83
@ CLOSEST_PARENT_MISS
Definition: hier_code.h:21
void port(unsigned short p)
Definition: Uri.h:94
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
struct Http::Stream::@78 flags
void stopPinnedConnectionMonitoring()
The caller assumes responsibility for connection closure detection.
SQUIDCEXTERN CSS clientReplyStatus
Definition: client_side.h:529
@ VARY_NONE
Definition: enums.h:193
CertSignAlgorithm
Definition: gadgets.h:150
void lifetimeTimeout(const CommTimeoutCbParams &params)
ClientHttpRequest * http
std::shared_ptr< SSL_CTX > ContextPointer
Definition: Context.h:29
Comm::ConnectionPointer connection
to-server connection to be pinned
Definition: client_side.h:188
void clientConnectionsClose()
virtual bool handleReadData()
const_reverse_iterator rbegin() const
Definition: SBuf.h:584
const char * stoppedSending() const
true if we stopped sending the response
Definition: client_side.h:162
@ ERR_GATEWAY_FAILURE
Definition: forward.h:67
Ip::Address src_addr
@ scBadRequest
Definition: StatusCode.h:44
ACLFilledChecklist * clientAclChecklistCreate(const acl_access *acl, ClientHttpRequest *http)
Security::CertPointer mimicCert
Certificate to mimic.
Definition: gadgets.h:216
static std::ostream & Extra(std::ostream &os)
prefixes each grouped debugs() line after the first one in the group
Definition: Debug.h:99
@ scNone
Definition: StatusCode.h:21
Security::CertPointer signWithX509
Certificate to sign the generated request.
Definition: gadgets.h:217
void getSslContextStart()
Start to create dynamic Security::ContextPointer for host or uses static port SSL context.
SBuf inBuf
read I/O buffer for the client connection
Definition: Server.h:111
void setAuth(const Auth::UserRequest::Pointer &aur, const char *cause)
Definition: client_side.cc:510
struct ClientHttpRequest::Redirect redirect
mb_size_t size
Definition: MemBuf.h:135
void retrieveParsedInfo(Security::TlsDetails::Pointer const &details)
Extract information from parser stored in TlsDetails object.
AnyP::PortCfgPointer HttpPortList
list of Squid http(s)_port configured
Definition: PortCfg.cc:21
void fd_note(int fd, const char *s)
Definition: fd.cc:217
HttpHeader header
Definition: Message.h:75
void resetSslCommonName(const char *name)
Definition: client_side.h:294
AsyncCall::Pointer cbControlMsgSent
Call to schedule when the control msg has been sent.
static void clientUpdateHierCounters(HierarchyLogEntry *)
Definition: client_side.cc:268
ByteCounter hit_kbytes_out
Definition: StatCounters.h:47
uint64_t unproducedSize() const
Definition: BodyPipe.cc:179
Security::ContextPointer GetFrom(Security::SessionPointer &s)
Helper function to retrieve a (non-locked) ContextPointer from a SessionPointer.
Definition: Session.h:105
@ PARENT_HIT
Definition: hier_code.h:16
acl_access * on_unsupported_protocol
Definition: SquidConfig.h:407
bool urlCheckRequest(const HttpRequest *r)
Definition: Uri.cc:832
bool isEmpty() const
Definition: SBuf.h:424
void initRequest(HttpRequest *)
MemObject * mem_obj
Definition: Store.h:222
StatHist nearHitSvcTime
Definition: StatCounters.h:50
void registerWithConn()
register this stream with the Server
Definition: Stream.cc:52
struct ConnStateData::@39 pinning
int64_t maxRequestBodySize
Definition: SquidConfig.h:133
bool proxyProtocolError(const char *reason)
int varyEvaluateMatch(StoreEntry *entry, HttpRequest *request)
void sslCrtdHandleReply(const Helper::Reply &reply)
Process response from ssl_crtd.
const EVP_MD * signHash
The signing hash to use.
Definition: gadgets.h:224
@ SQUID_X509_V_ERR_DOMAIN_MISMATCH
Definition: forward.h:216
bool empty() const
whether there are none or any requests currently pipelined
Definition: Pipeline.h:56
bool isTcpHit() const
determine if the log tag code indicates a cache HIT
Definition: LogTags.cc:101
NotePairs::Pointer theNotes
Definition: client_side.h:504
ErrorDetailPointer errorDetail
ioError case details (or nil)
Definition: Io.h:38
#define debug_log
change-avoidance macro; new code should call DebugStream() instead
Definition: Debug.h:109
struct node * next
Definition: parse.c:105
int log_mime_hdrs
Definition: SquidConfig.h:291
ConnStateData * getConn() const
char * skipLeadingSpace(char *aString)
Callback cbSuccess
called after successfully writing the 1xx message
void updateError(const Error &)
sets (or updates the already stored) transaction error as needed
void setErrorUri(const char *errorUri)
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
void extendLifetime()
(re)sets client_lifetime timeout
Definition: client_side.cc:598
acl_access * cert_error
Definition: SquidConfig.h:526
MemObject & mem()
Definition: Store.h:52
void banAction(const Acl::Answer &action)
add action to the list of banned actions
Definition: Checklist.cc:395
Security::ContextPointer createSSLContext(Security::CertPointer &x509, Security::PrivateKeyPointer &pkey, Security::ServerOptions &)
Create SSL context and apply ssl certificate and private key to it.
Definition: support.cc:917
void doPeekAndSpliceStep()
Comm::ConnectionPointer borrowPinnedConnection(HttpRequest *, const AccessLogEntryPointer &)
ConnStateData-specific part of BorrowPinnedConnection()
void clientStartListeningOn(AnyP::PortCfgPointer &port, const RefCount< CommCbFunPtrCallT< CommAcceptCbPtrFun > > &subCall, const Ipc::FdNoteId fdNote)
accept requests to a given port and inform subCall about them
dials clientListenerConnectionOpened call
Definition: client_side.cc:153
char reqbuf[HTTP_REQBUF_SZ]
Definition: Stream.h:134
void error(char *format,...)
static void Submit(CrtdMessage const &message, HLPCB *callback, void *data)
Submit crtd message to external crtd server.
Definition: helper.cc:125
unsigned int minor
minor version number
Ip::Address log_addr
Definition: client_side.h:137
SBuf hostOrIp() const
Definition: Uri.cc:113
void setWriteLimiter(const int aWriteSpeedLimit, const double anInitialBurst, const double aHighWatermark)
Definition: comm.cc:1347
void readNextRequest()
Definition: client_side.cc:873
bool hasContent() const
Definition: MemBuf.h:54
std::vector< ClientDelayPool::Pointer > pools
#define SQUIDSTRINGPRINT(s)
Definition: SquidString.h:22
void hold(bool h)
Prevents or allow writing on socket.
Definition: bio.h:86
@ ERR_UNSUP_REQ
Definition: forward.h:44
void httpRequestFree(void *data)
Definition: client_side.cc:491
bool clientParseRequests()
Traffic parsing.
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:95
Definition: SBuf.h:87
virtual void sendControlMsg(HttpControlMsg)
called to send the 1xx message and notify the Source
BumpMode
Definition: support.h:126
struct timeval stop
Definition: PingData.h:37
Ip::Address client_netmask
Definition: SquidConfig.h:247
BodyPipe::Pointer bodyPipe
set when we are reading request body
Definition: client_side.h:432
bool needsMoreData() const
Definition: Parser.h:66
static void httpsSslBumpStep2AccessCheckDone(Acl::Answer answer, void *data)
@ ERR_REQUEST_PARSE_TIMEOUT
Definition: forward.h:82
bool initiateTunneledRequest(HttpRequest::Pointer const &cause, Http::MethodType const method, const char *reason, const SBuf &payload)
generates and sends to tunnel.cc a fake request with a given payload
int commSetConnTimeout(const Comm::ConnectionPointer &conn, int timeout, AsyncCall::Pointer &callback)
Definition: comm.cc:564
void SBufToCstring(char *d, const SBuf &s)
Definition: SBuf.h:745
bool splice()
Splice a bumped client connection on peek-and-splice mode.
const char * stoppedReceiving_
the reason why we no longer read the request or nil
Definition: client_side.h:500
#define xstrdup
void enableAutoConsumption()
start or continue consuming when producing without consumer
Definition: BodyPipe.cc:316
void setReplyToError(err_type, Http::StatusCode, const HttpRequestMethod &, char const *, const ConnStateData *, HttpRequest *, const char *, Auth::UserRequest::Pointer)
builds error using clientBuildError() and calls setReplyToError() below
int64_t mayNeedToReadMoreBody() const
@ bumpTerminate
Definition: support.h:126
struct ClientHttpRequest::Flags flags
void clientdbUpdate(const Ip::Address &addr, const LogTags &ltype, AnyP::ProtocolType p, size_t size)
Definition: client_db.cc:139
Definition: Server.h:30
int clientdbEstablished(const Ip::Address &addr, int delta)
Definition: client_db.cc:183
@ LOG_TCP_CLIENT_REFRESH_MISS
Definition: LogTags.h:46
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
virtual void terminateAll(const Error &, const LogTagsErrors &)
abort any pending transactions and prevent new ones (by closing)
time_t request_start_timeout
Definition: SquidConfig.h:123
Security::KeyLogger keyLogger
managers logging of the being-accepted TLS connection secrets
Definition: client_side.h:385
@ METHOD_PRI
Definition: MethodType.h:89
int content_length
Definition: tcp-banger2.c:121
int type
Definition: errorpage.cc:153
struct ClientHttpRequest::Out out
C * getRaw() const
Definition: RefCount.h:80
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:211
char * xstrncpy(char *dst, const char *src, size_t n)
Definition: xstring.cc:37
virtual void dial(AsyncCall &)
Definition: client_side.cc:165
struct timeval start_time
The time the master transaction started.
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:398
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
Acl::Answer const & fastCheck()
Definition: Checklist.cc:332
std::string const & getBody() const
Current body. If parsing is not finished the method returns incompleted body.
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:27
Http1::TeChunkedParser * bodyParser
parses HTTP/1.1 chunked request body
Definition: client_side.h:109
@ OK
Definition: Flag.h:16
const char * sslGetUserEmail(SSL *ssl)
Definition: support.cc:877
int global_internal_static
Definition: SquidConfig.h:329
void add(const Http::StreamPointer &)
register a new request context to the pipeline
Definition: Pipeline.cc:20
std::vector< HttpHdrRangeSpec * >::iterator iterator
struct ConnStateData::@38 flags
ClientHttpRequest * buildFakeRequest(Http::MethodType const method, SBuf &useHost, unsigned short usePort, const SBuf &payload)
build a fake http request
void accessLogLog(const AccessLogEntryPointer &, ACLChecklist *)
Definition: access_log.cc:148
a summary a TLS I/O operation outcome
Definition: Io.h:19
void StopListening()
reject new connections to any configured ftp_port
Definition: FtpServer.cc:280
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:150
bool commIsHalfClosed(int fd)
Definition: comm.h:97
char * toUrl(char *buf, unsigned int len) const
Definition: Address.cc:874
@ ERR_NONE
Definition: forward.h:15
@ CLOSEST_PARENT
Definition: hier_code.h:22
#define DBG_CRITICAL
Definition: Debug.h:40
void handleSslBumpHandshakeError(const Security::IoResult &)
process a problematic Security::Accept() result on the SslBump code path
StatusCode
Definition: StatusCode.h:20
virtual void fillChecklist(ACLFilledChecklist &) const
configure the given checklist (to reflect the current transaction state)
struct StatCounters::@135 netdb
err_type
Definition: forward.h:14
#define SQUIDSTRINGPH
Definition: SquidString.h:21
bool multipartRangeRequest() const
Definition: HttpRequest.cc:435
SBuf sslBumpCertKey
Key to use to store/retrieve generated certificate.
Definition: client_side.h:490
const char * FdNote(int fdNodeId)
converts FdNoteId into a string
Definition: FdNotes.cc:16
Notes notes
note
Definition: SquidConfig.h:488
void StartListening()
accept connections on all configured ftp_ports
Definition: FtpServer.cc:261
@ ERR_CLOSING
Definition: Flag.h:25
virtual bool shouldCloseOnEof() const
whether to stop serving our client after reading EOF on its connection
virtual void afterClientRead()
processing to be done after a Comm::Read()
#define cbdataReference(var)
Definition: cbdata.h:341
void * BIO_get_data(BIO *table)
Definition: openssl.h:62
Ssl::BumpStep step
The SSL bumping step.
Definition: ServerBump.h:63
static void Start(const Comm::ConnectionPointer &client, StoreEntry *, HttpRequest *, const AccessLogEntryPointer &alp)
Initiates request forwarding to a peer or origin server.
Definition: FwdState.cc:362
@ VARY_CANCEL
Definition: enums.h:196
@ LOG_TCP_HIT
Definition: LogTags.h:39
static void sslCrtdHandleReplyWrapper(void *data, const Helper::Reply &reply)
Callback function. It is called when squid receive message from ssl_crtd.
#define DBG_IMPORTANT
Definition: Debug.h:41
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
void startDechunkingRequest()
initialize dechunking state
static void clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub)
process clientHttpConnectionsOpen result
ProtocolType protocol
which protocol this version is for
@ LOG_TCP_MISS
Definition: LogTags.h:40
static int port
Definition: ldap_backend.cc:69
Error error
the first transaction problem encountered (or falsy)
Definition: HttpRequest.h:161
bool checkX509ServerValidity(X509 *cert, const char *server)
Definition: support.cc:254
unsigned short tlsConnectPort
The TLS server port number as passed in the CONNECT request.
Definition: client_side.h:485
Pipeline pipeline
set of requests waiting to be serviced
Definition: Server.h:116
void clear()
Definition: SBuf.cc:175
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:592
#define COMM_NONBLOCKING
Definition: Connection.h:46
StoreEntry * entry
Definition: ServerBump.h:54
static const CharacterSet ALPHA
Definition: CharacterSet.h:76
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:792
const Value * get(const Key &)
Definition: ClpMap.h:175
Security::IoResult acceptTls()
int opt_no_daemon
acl_access * stats_collection
Definition: SquidConfig.h:376
virtual void endingShutdown()
@ METHOD_OPTIONS
Definition: MethodType.h:31
void startPinnedConnectionMonitoring()
void finishDechunkingRequest(bool withSuccess)
put parsed content into input buffer and clean up
void stopReceiving(const char *error)
note request receiving error and close as soon as we write the response
static bool Enabled(const int section, const int level)
whether debugging the given section and the given level produces output
Definition: Debug.h:70
SBuf & chop(size_type pos, size_type n=npos)
Definition: SBuf.cc:530
void clientStreamDetach(clientStreamNode *thisObject, ClientHttpRequest *http)
int HttpSockets[MAXTCPLISTENPORTS]
Definition: PortCfg.cc:25
iterator end()
HttpHdrRangeIter range_iter
a transaction problem
Definition: Error.h:18
@ DISABLE_PMTU_ALWAYS
Definition: enums.h:233
void(* Handler)(AnyP::PortCfgPointer &portCfg, const Ipc::FdNoteId note, const Subscription::Pointer &sub)
Definition: client_side.cc:155
Http::Stream * parseHttpRequest(const Http1::RequestParserPointer &)
const char * CertAdaptAlgorithmStr[]
Definition: gadgets.cc:198
#define MAX_IPSTRLEN
Length of buffer that needs to be allocated to old a null-terminated IP-string.
Definition: forward.h:25
static void Reset()
forgets the current context, setting it to nil/unknown
Definition: CodeContext.cc:75
HttpRequest::Pointer request
faked, minimal request; required by Client API
Definition: ServerBump.h:53
#define COMM_INTERCEPTION
Definition: Connection.h:51
void ReadCancel(int fd, AsyncCall::Pointer &callback)
Cancel the read pending on FD. No action if none pending.
Definition: Read.cc:219
IdentConfig TheConfig
Definition: Ident.cc:81
void setIdent(const char *userIdentity)
configure rfc931 user identity for the first time
size_type rfind(char c, size_type endPos=npos) const
Definition: SBuf.cc:692
bool bodySizeKnown() const
Definition: BodyPipe.h:109
ProtocolType
Definition: ProtocolType.h:23
@ SIBLING_HIT
Definition: hier_code.h:15
HttpRequest * request
MemBuf & buf
Definition: BodyPipe.h:74
ConnStateData * conn() const
The client connection manager.
#define COMM_TRANSPARENT
Definition: Connection.h:50
void postHttpsAccept()
the second part of old httpsAccept, waiting for future HttpsServer home
Ssl::ServerBump * sslServerBump
HTTPS server cert. fetching state for bump-ssl-server-first.
Definition: client_side.h:493
bool multipartRangeRequest() const
Definition: client_side.cc:712
bool SessionIsResumed(const Security::SessionPointer &)
whether the session is a resumed one
Definition: Session.cc:212
acl_access * proxyProtocol
acceptable PROXY protocol clients
Definition: SquidConfig.h:402
bool mayUseConnection() const
Definition: Stream.h:140
const char * stoppedSending_
the reason why we no longer write the response or nil
Definition: client_side.h:498
void connStateClosed(const CommCloseCbParams &io)
Definition: client_side.cc:499
@ ERR_REQUEST_START_TIMEOUT
Definition: forward.h:81
unsigned int toHostStr(char *buf, const unsigned int len) const
Definition: Address.cc:842
virtual Http::Stream * parseOneRequest()=0
void applyClientMask(const Address &mask)
Definition: Address.cc:105
@ VARY_MATCH
Definition: enums.h:194
virtual void doneWithControlMsg()
@ ICP_INVALID
Definition: icp_opcode.h:15
Ssl::CertSignAlgorithm signAlgorithm
The signing algorithm to use.
Definition: client_side.h:494
unsigned short initial
@ bumpServerFirst
Definition: support.h:126
bool needProxyProtocolHeader_
whether PROXY protocol header is still expected
Definition: client_side.h:467
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
static const std::string code_new_certificate
String code for "new_certificate" messages.
Definition: crtd_message.h:76
virtual void swanSong()
Definition: Server.cc:51
void clearRequest()
resets the current request and log_uri to nil
char rfc931[USER_IDENT_SZ]
Definition: Connection.h:174
uint64_t bodySize() const
Definition: BodyPipe.cc:161
StatHist nearMissSvcTime
Definition: StatCounters.h:49
int size
Definition: ModDevPoll.cc:76
Comm::ConnectionPointer tcpClient
the client TCP connection which originated this transaction
Definition: MasterXaction.h:53
const MemBuf & other() const
Definition: Reply.h:42
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:184
struct SquidConfig::@106 Addrs
const char * fqdncache_gethostbyaddr(const Ip::Address &addr, int flags)
Definition: fqdncache.cc:481
#define NULL
Definition: types.h:166
static void httpsAccept(const CommAcceptCbParams &params)
@ algSignTrusted
Definition: gadgets.h:150
SBuf consume(size_type n=npos)
Definition: SBuf.cc:481
@ ERR_PROTOCOL_UNKNOWN
Definition: forward.h:73
void StartListening(int sock_type, int proto, const Comm::ConnectionPointer &listenConn, FdNoteId fdNote, AsyncCall::Pointer &callback)
virtual void handleReply(HttpReply *header, StoreIOBuffer receivedData)=0
Ssl::BumpMode sslBumpMode
ssl_bump decision (Ssl::bumpEnd if n/a).
Definition: client_side.h:305
ICP probing of cache_peers during peer selection.
Definition: PingData.h:26
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
void mustStop(const char *aReason)
Definition: AsyncJob.cc:70
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
bundles HTTP 1xx reply and the "successfully forwarded" callback
time_t timeLeft(const time_t idleTimeout) const
Definition: Connection.cc:146
SBuf vary_headers
Definition: MemObject.h:204
sslproxy_cert_adapt * cert_adapt
Definition: SquidConfig.h:528
const EVP_MD * DefaultSignHash
Definition: support.cc:44
void updateError(const Error &)
if necessary, stores new error information (if any)
Definition: client_side.cc:644
void clientSocketRecipient(clientStreamNode *node, ClientHttpRequest *http, HttpReply *rep, StoreIOBuffer receivedData)
Definition: client_side.cc:816
@ scNotImplemented
Definition: StatusCode.h:72
static void httpsSslBumpAccessCheckDone(Acl::Answer answer, void *data)
struct StatCounters::@130 icp
@ ERR_SECURE_ACCEPT_FAIL
Definition: forward.h:80
Http::StreamPointer front() const
get the first request context in the pipeline
Definition: Pipeline.cc:28
LocalContextStorage * getLocalStorage(Ip::Address const &address)
Return the local storage for the given listening address/port.
uint64_t producedSize() const
Definition: BodyPipe.h:112
Category category
primary outcome classification
Definition: Io.h:40
bool parsingTlsHandshake
Definition: client_side.h:479
void append(char const *buf, int len)
Definition: String.cc:151
Subscription::Pointer sub
The handler to be subscribed for this connection listener.
Definition: client_side.cc:173
virtual void receivedFirstByte()
Update flags and timeout after the first byte received.
Helper::ResultCode result
The helper response 'result' field.
Definition: Reply.h:59
size_t count() const
how many requests are currently pipelined
Definition: Pipeline.h:53
@ algSignEnd
Definition: gadgets.h:150
@ LOG_TAG_NONE
Definition: LogTags.h:38
@ srcHttp
http_port or HTTP server
Definition: Message.h:39
std::string compose() const
unsigned short port() const
Definition: Address.cc:778
Definition: MemBuf.h:24
SQUIDCEXTERN int tvSubUsec(struct timeval, struct timeval)
Definition: util.c:39
void setReplyToStoreEntry(StoreEntry *e, const char *reason)
replaces current response store entry with the given one
Ip::Address local
Definition: Connection.h:144
clientStreamNode * getClientReplyContext() const
Definition: Stream.cc:513
CachePeer * getPeer() const
Definition: Connection.cc:124
static char * prepareAcceleratedURL(ConnStateData *conn, const Http1::RequestParserPointer &hp)
static const CharacterSet HEXDIG
Definition: CharacterSet.h:88
int64_t contentLen() const
Definition: Store.h:259
void configureUnconfiguredSslContext(Security::ContextPointer &, Ssl::CertSignAlgorithm signAlgorithm, AnyP::PortCfg &)
Definition: support.cc:987
void clean()
Definition: MemBuf.cc:112
StatHist hitSvcTime
Definition: StatCounters.h:51
virtual void doneWithControlMsg()
static void httpsEstablish(ConnStateData *connState, const Security::ContextPointer &ctx)
Comm::ConnectionPointer clientConnection
bool concurrentRequestQueueFilled() const
Definition: ClpMap.h:41
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:152
acl_access * ssl_bump
Definition: SquidConfig.h:395
@ FIRST_PARENT_MISS
Definition: hier_code.h:20
void httpsPeeked(PinnedIdleContext pic)
called by FwdState when it is done bumping the server
bool preservingClientData_
whether preservedClientData is valid and should be kept up to date
Definition: client_side.h:435
@ METHOD_CONNECT
Definition: MethodType.h:29
virtual void print(std::ostream &os) const
Definition: client_side.cc:159
Comm::ConnectionPointer tcpClient
TCP/IP level details about the client connection.
void resetReadTimeout(time_t timeout)
(re)sets timeout for receiving more bytes from the client
Definition: client_side.cc:590
FdNoteId
We cannot send char* FD notes to other processes. Pass int IDs and convert.
Definition: FdNotes.h:20
void stopSending(const char *error)
note response sending error and close as soon as we read the request
Definition: client_side.cc:991
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
const char * internalHostname(void)
Definition: internal.cc:141
bool handleIdleClientPinnedTlsRead()
#define safe_free(x)
Definition: xalloc.h:73
@ LOG_TCP_REFRESH_UNMODIFIED
Definition: LogTags.h:41
@ srcFtp
ftp_port or FTP server
Definition: Message.h:40
virtual int pipelinePrefetchMax() const
returning N allows a pipeline of 1+N requests (see pipeline_prefetch)
int tvSubMsec(struct timeval, struct timeval)
Definition: time.cc:37
Ip::Address remote
Definition: Connection.h:147
SBuf tlsConnectHostOrIp
The TLS server host name appears in CONNECT request or the server ip address for the intercepted requ...
Definition: client_side.h:484
NotePairs::Pointer notes()
struct SquidConfig::@112 accessList
@ scRequestHeaderFieldsTooLarge
Definition: StatusCode.h:69
TlsDetails::Pointer details
TLS handshake meta info. Never nil.
Definition: Handshake.h:77
ClientHttpRequest * http
Definition: Stream.h:132
void parseTlsHandshake()
Http::StreamPointer back() const
get the last request context in the pipeline
Definition: Pipeline.cc:40
void count(double val)
Definition: StatHist.cc:57
CommCbFunPtrCallT< Dialer > * commCbCall(int debugSection, int debugLevel, const char *callName, const Dialer &dialer)
Definition: CommCalls.h:342
Http::Stream * abortRequestParsing(const char *const errUri)
stop parsing the request and create context for relaying error info
iterator begin()
void clientOpenListenSockets(void)
void comm_reset_close(const Comm::ConnectionPointer &conn)
Definition: comm.cc:754