client_side.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 33 Client-side Routines */
10 
60 #include "squid.h"
61 #include "acl/FilledChecklist.h"
62 #include "anyp/PortCfg.h"
63 #include "base/Subscription.h"
64 #include "base/TextException.h"
65 #include "CachePeer.h"
66 #include "client_db.h"
67 #include "client_side.h"
68 #include "client_side_reply.h"
69 #include "client_side_request.h"
70 #include "ClientRequestContext.h"
71 #include "clientStream.h"
72 #include "comm.h"
73 #include "comm/Connection.h"
74 #include "comm/Loops.h"
75 #include "comm/Read.h"
76 #include "comm/TcpAcceptor.h"
77 #include "comm/Write.h"
78 #include "CommCalls.h"
79 #include "errorpage.h"
80 #include "fd.h"
81 #include "fde.h"
82 #include "fqdncache.h"
83 #include "FwdState.h"
84 #include "globals.h"
85 #include "helper.h"
86 #include "helper/Reply.h"
87 #include "http.h"
88 #include "http/one/RequestParser.h"
90 #include "http/Stream.h"
91 #include "HttpHdrContRange.h"
92 #include "HttpHeaderTools.h"
93 #include "HttpReply.h"
94 #include "HttpRequest.h"
95 #include "ident/Config.h"
96 #include "ident/Ident.h"
97 #include "internal.h"
98 #include "ipc/FdNotes.h"
99 #include "ipc/StartListening.h"
100 #include "log/access_log.h"
101 #include "MemBuf.h"
102 #include "MemObject.h"
103 #include "mime_header.h"
104 #include "parser/Tokenizer.h"
105 #include "profiler/Profiler.h"
106 #include "proxyp/Header.h"
107 #include "proxyp/Parser.h"
109 #include "servers/forward.h"
110 #include "SquidConfig.h"
111 #include "SquidTime.h"
112 #include "StatCounters.h"
113 #include "StatHist.h"
114 #include "Store.h"
115 #include "TimeOrTag.h"
116 #include "tools.h"
117 
118 #if USE_AUTH
119 #include "auth/UserRequest.h"
120 #endif
121 #if USE_DELAY_POOLS
122 #include "ClientInfo.h"
123 #include "MessageDelayPools.h"
124 #endif
125 #if USE_OPENSSL
126 #include "ssl/bio.h"
127 #include "ssl/context_storage.h"
128 #include "ssl/gadgets.h"
129 #include "ssl/helper.h"
130 #include "ssl/ProxyCerts.h"
131 #include "ssl/ServerBump.h"
132 #include "ssl/support.h"
133 #endif
134 
135 // for tvSubUsec() which should be in SquidTime.h
136 #include "util.h"
137 
138 #include <climits>
139 #include <cmath>
140 #include <limits>
141 
142 #if LINGERING_CLOSE
143 #define comm_close comm_lingering_close
144 #endif
145 
148 {
149 public:
152  handler(aHandler), portCfg(aPortCfg), portTypeNote(note), sub(aSub) {}
153 
154  virtual void print(std::ostream &os) const {
155  startPrint(os) <<
156  ", " << FdNote(portTypeNote) << " port=" << (void*)&portCfg << ')';
157  }
158 
159  virtual bool canDial(AsyncCall &) const { return true; }
160  virtual void dial(AsyncCall &) { (handler)(portCfg, portTypeNote, sub); }
161 
162 public:
164 
165 private:
169 };
170 
172 
175 #if USE_IDENT
177 #endif
179 static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength);
180 
181 static void clientUpdateStatHistCounters(const LogTags &logType, int svc_time);
182 static void clientUpdateStatCounters(const LogTags &logType);
184 static bool clientPingHasFinished(ping_data const *aPing);
187 
188 char *skipLeadingSpace(char *aString);
189 
190 #if USE_IDENT
191 static void
192 clientIdentDone(const char *ident, void *data)
193 {
194  ConnStateData *conn = (ConnStateData *)data;
195  xstrncpy(conn->clientConnection->rfc931, ident ? ident : dash_str, USER_IDENT_SZ);
196 }
197 #endif
198 
199 void
201 {
203 
204  if (logType.isTcpHit())
206 
207  if (logType.oldType == LOG_TCP_HIT)
209  else if (logType.oldType == LOG_TCP_MEM_HIT)
211 }
212 
213 void
214 clientUpdateStatHistCounters(const LogTags &logType, int svc_time)
215 {
224  switch (logType.oldType) {
225 
228  break;
229 
230  case LOG_TCP_INM_HIT:
231  case LOG_TCP_IMS_HIT:
233  break;
234 
235  case LOG_TCP_HIT:
236 
237  case LOG_TCP_MEM_HIT:
238 
239  case LOG_TCP_OFFLINE_HIT:
241  break;
242 
243  case LOG_TCP_MISS:
244 
247  break;
248 
249  default:
250  /* make compiler warnings go away */
251  break;
252  }
253 }
254 
255 bool
257 {
258  if (0 != aPing->stop.tv_sec && 0 != aPing->start.tv_sec)
259  return true;
260 
261  return false;
262 }
263 
264 void
266 {
267  ping_data *i;
268 
269  switch (someEntry->code) {
270 #if USE_CACHE_DIGESTS
271 
272  case CD_PARENT_HIT:
273 
274  case CD_SIBLING_HIT:
276  break;
277 #endif
278 
279  case SIBLING_HIT:
280 
281  case PARENT_HIT:
282 
283  case FIRST_PARENT_MISS:
284 
285  case CLOSEST_PARENT_MISS:
287  i = &someEntry->ping;
288 
289  if (clientPingHasFinished(i))
291 
292  if (i->timeout)
294 
295  break;
296 
297  case CLOSEST_PARENT:
298 
299  case CLOSEST_DIRECT:
301 
302  break;
303 
304  default:
305  break;
306  }
307 }
308 
309 void
311 {
312  clientUpdateStatCounters(logType);
313 
314  if (request->errType != ERR_NONE)
316 
318  tvSubMsec(al->cache.start_time, current_time));
319 
321 }
322 
323 void
325 {
326  assert(request);
327  assert(aLogEntry != NULL);
328 
329  if (Config.onoff.log_mime_hdrs) {
330  MemBuf mb;
331  mb.init();
332  request->header.packInto(&mb);
333  //This is the request after adaptation or redirection
334  aLogEntry->headers.adapted_request = xstrdup(mb.buf);
335 
336  // the virgin request is saved to aLogEntry->request
337  if (aLogEntry->request) {
338  mb.reset();
339  aLogEntry->request->header.packInto(&mb);
340  aLogEntry->headers.request = xstrdup(mb.buf);
341  }
342 
343 #if USE_ADAPTATION
344  const Adaptation::History::Pointer ah = request->adaptLogHistory();
345  if (ah != NULL) {
346  mb.reset();
347  ah->lastMeta.packInto(&mb);
348  aLogEntry->adapt.last_meta = xstrdup(mb.buf);
349  }
350 #endif
351 
352  mb.clean();
353  }
354 
355 #if ICAP_CLIENT
356  const Adaptation::Icap::History::Pointer ih = request->icapHistory();
357  if (ih != NULL)
358  ih->processingTime(aLogEntry->icap.processingTime);
359 #endif
360 
361  aLogEntry->http.method = request->method;
362  aLogEntry->http.version = request->http_ver;
363  aLogEntry->hier = request->hier;
364  aLogEntry->cache.extuser = request->extacl_user.termedBuf();
365 
366  // Adapted request, if any, inherits and then collects all the stats, but
367  // the virgin request gets logged instead; copy the stats to log them.
368  // TODO: avoid losses by keeping these stats in a shared history object?
369  if (aLogEntry->request) {
370  aLogEntry->request->dnsWait = request->dnsWait;
371  aLogEntry->request->errType = request->errType;
372  aLogEntry->request->errDetail = request->errDetail;
373  }
374 }
375 
376 void
378 {
379  if (!out.size && logType.oldType == LOG_TAG_NONE)
380  debugs(33, 5, "logging half-baked transaction: " << log_uri);
381 
382  al->icp.opcode = ICP_INVALID;
383  al->url = log_uri;
384  debugs(33, 9, "clientLogRequest: al.url='" << al->url << "'");
385 
386  if (al->reply) {
387  al->http.code = al->reply->sline.status();
388  al->http.content_type = al->reply->content_type.termedBuf();
389  } else if (loggingEntry() && loggingEntry()->mem_obj) {
390  al->http.code = loggingEntry()->mem_obj->getReply()->sline.status();
391  al->http.content_type = loggingEntry()->mem_obj->getReply()->content_type.termedBuf();
392  }
393 
394  debugs(33, 9, "clientLogRequest: http.code='" << al->http.code << "'");
395 
396  if (loggingEntry() && loggingEntry()->mem_obj && loggingEntry()->objectLen() >= 0)
397  al->cache.objectSize = loggingEntry()->contentLen(); // payload duplicate ?? with or without TE ?
398 
399  al->http.clientRequestSz.header = req_sz;
400  // the virgin request is saved to al->request
401  if (al->request && al->request->body_pipe)
402  al->http.clientRequestSz.payloadData = al->request->body_pipe->producedSize();
403  al->http.clientReplySz.header = out.headers_sz;
404  // XXX: calculate without payload encoding or headers !!
405  al->http.clientReplySz.payloadData = out.size - out.headers_sz; // pretend its all un-encoded data for now.
406 
407  al->cache.highOffset = out.offset;
408 
409  al->cache.code = logType;
410 
411  tvSub(al->cache.trTime, al->cache.start_time, current_time);
412 
413  if (request)
415 
416 #if USE_OPENSSL && 0
417 
418  /* This is broken. Fails if the connection has been closed. Needs
419  * to snarf the ssl details some place earlier..
420  */
421  if (getConn() != NULL)
422  al->cache.ssluser = sslGetUserEmail(fd_table[getConn()->fd].ssl);
423 
424 #endif
425 
426  /* Add notes (if we have a request to annotate) */
427  if (request) {
428  SBuf matched;
429  for (auto h: Config.notes) {
430  if (h->match(request, al->reply, NULL, matched)) {
431  request->notes()->add(h->key(), matched);
432  debugs(33, 3, h->key() << " " << matched);
433  }
434  }
435  // The al->notes and request->notes must point to the same object.
436  al->syncNotes(request);
437  }
438 
439  ACLFilledChecklist checklist(NULL, request, NULL);
440  if (al->reply) {
441  checklist.reply = al->reply;
442  HTTPMSGLOCK(checklist.reply);
443  }
444 
445  if (request) {
446  HTTPMSGUNLOCK(al->adapted_request);
447  al->adapted_request = request;
448  HTTPMSGLOCK(al->adapted_request);
449  }
450  // no need checklist.syncAle(): already synced
451  checklist.al = al;
452  accessLogLog(al, &checklist);
453 
454  bool updatePerformanceCounters = true;
457  statsCheck.al = al;
458  if (al->reply) {
459  statsCheck.reply = al->reply;
460  HTTPMSGLOCK(statsCheck.reply);
461  }
462  updatePerformanceCounters = statsCheck.fastCheck().allowed();
463  }
464 
465  if (updatePerformanceCounters) {
466  if (request)
467  updateCounters();
468 
469  if (getConn() != NULL && getConn()->clientConnection != NULL)
470  clientdbUpdate(getConn()->clientConnection->remote, logType, AnyP::PROTO_HTTP, out.size);
471  }
472 }
473 
474 void
476 {
477  safe_free(uri);
478  safe_free(redirect.location);
479  range_iter.boundary.clean();
480  clearRequest();
481 
482  if (client_stream.tail)
483  clientStreamAbort((clientStreamNode *)client_stream.tail->data, this);
484 }
485 
486 void
488 {
489  ClientHttpRequest *http = (ClientHttpRequest *)data;
490  assert(http != NULL);
491  delete http;
492 }
493 
494 /* This is a handler normally called by comm_close() */
496 {
497  deleteThis("ConnStateData::connStateClosed");
498 }
499 
500 #if USE_AUTH
501 void
503 {
504  if (auth_ == NULL) {
505  if (aur != NULL) {
506  debugs(33, 2, "Adding connection-auth to " << clientConnection << " from " << by);
507  auth_ = aur;
508  }
509  return;
510  }
511 
512  // clobered with self-pointer
513  // NP: something nasty is going on in Squid, but harmless.
514  if (aur == auth_) {
515  debugs(33, 2, "WARNING: Ignoring duplicate connection-auth for " << clientConnection << " from " << by);
516  return;
517  }
518 
519  /*
520  * Connection-auth relies on a single set of credentials being preserved
521  * for all requests on a connection once they have been setup.
522  * There are several things which need to happen to preserve security
523  * when connection-auth credentials change unexpectedly or are unset.
524  *
525  * 1) auth helper released from any active state
526  *
527  * They can only be reserved by a handshake process which this
528  * connection can now never complete.
529  * This prevents helpers hanging when their connections close.
530  *
531  * 2) pinning is expected to be removed and server conn closed
532  *
533  * The upstream link is authenticated with the same credentials.
534  * Expecting the same level of consistency we should have received.
535  * This prevents upstream being faced with multiple or missing
536  * credentials after authentication.
537  * NP: un-pin is left to the cleanup in ConnStateData::swanSong()
538  * we just trigger that cleanup here via comm_reset_close() or
539  * ConnStateData::stopReceiving()
540  *
541  * 3) the connection needs to close.
542  *
543  * This prevents attackers injecting requests into a connection,
544  * or gateways wrongly multiplexing users into a single connection.
545  *
546  * When credentials are missing closure needs to follow an auth
547  * challenge for best recovery by the client.
548  *
549  * When credentials change there is nothing we can do but abort as
550  * fast as possible. Sending TCP RST instead of an HTTP response
551  * is the best-case action.
552  */
553 
554  // clobbered with nul-pointer
555  if (aur == NULL) {
556  debugs(33, 2, "WARNING: Graceful closure on " << clientConnection << " due to connection-auth erase from " << by);
557  auth_->releaseAuthServer();
558  auth_ = NULL;
559  // XXX: need to test whether the connection re-auth challenge is sent. If not, how to trigger it from here.
560  // NP: the current situation seems to fix challenge loops in Safari without visible issues in others.
561  // we stop receiving more traffic but can leave the Job running to terminate after the error or challenge is delivered.
562  stopReceiving("connection-auth removed");
563  return;
564  }
565 
566  // clobbered with alternative credentials
567  if (aur != auth_) {
568  debugs(33, 2, "ERROR: Closing " << clientConnection << " due to change of connection-auth from " << by);
569  auth_->releaseAuthServer();
570  auth_ = NULL;
571  // this is a fatal type of problem.
572  // Close the connection immediately with TCP RST to abort all traffic flow
573  comm_reset_close(clientConnection);
574  return;
575  }
576 
577  /* NOT REACHABLE */
578 }
579 #endif
580 
581 // cleans up before destructor is called
582 void
584 {
585  debugs(33, 2, HERE << clientConnection);
586  checkLogging();
587 
588  flags.readMore = false;
589  clientdbEstablished(clientConnection->remote, -1); /* decrement */
590  pipeline.terminateAll(0);
591 
592  // XXX: Closing pinned conn is too harsh: The Client may want to continue!
593  unpinConnection(true);
594 
595  Server::swanSong(); // closes the client connection
596 
597 #if USE_AUTH
598  // NP: do this bit after closing the connections to avoid side effects from unwanted TCP RST
599  setAuth(NULL, "ConnStateData::SwanSong cleanup");
600 #endif
601 
602  flags.swanSang = true;
603 }
604 
605 bool
607 {
608  return cbdataReferenceValid(this) && // XXX: checking "this" in a method
609  Comm::IsConnOpen(clientConnection) &&
610  !fd_table[clientConnection->fd].closing();
611 }
612 
614 {
615  debugs(33, 3, HERE << clientConnection);
616 
617  if (isOpen())
618  debugs(33, DBG_IMPORTANT, "BUG: ConnStateData did not close " << clientConnection);
619 
620  if (!flags.swanSang)
621  debugs(33, DBG_IMPORTANT, "BUG: ConnStateData was not destroyed properly; " << clientConnection);
622 
623  if (bodyPipe != NULL)
624  stopProducingFor(bodyPipe, false);
625 
626  delete bodyParser; // TODO: pool
627 
628 #if USE_OPENSSL
629  delete sslServerBump;
630 #endif
631 }
632 
639 void
641 {
642  HttpRequest *request = http->request;
643 
644  debugs(33, 3, "http_ver = " << request->http_ver);
645  debugs(33, 3, "method = " << request->method);
646 
647  // TODO: move to HttpRequest::hdrCacheInit, just like HttpReply.
648  request->flags.proxyKeepalive = request->persistent();
649 }
650 
652 static int
654 {
655  // No Content-Length means this request just has no body, but conflicting
656  // Content-Lengths mean a message framing error (RFC 7230 Section 3.3.3 #4).
658  return 0;
659 
660  switch (r->method.id()) {
661 
662  case Http::METHOD_GET:
663 
664  case Http::METHOD_HEAD:
665  /* We do not want to see a request entity on GET/HEAD requests */
666  return (r->content_length <= 0 || Config.onoff.request_entities);
667 
668  default:
669  /* For other types of requests we don't care */
670  return 1;
671  }
672 
673  /* NOT REACHED */
674 }
675 
676 int
678 {
680  bodyLength > Config.maxRequestBodySize)
681  return 1; /* too large */
682 
683  return 0;
684 }
685 
686 bool
688 {
689  return request->multipartRangeRequest();
690 }
691 
692 void
694 {
695  mb->appendf("\r\n--" SQUIDSTRINGPH "--\r\n", SQUIDSTRINGPRINT(boundary));
696  debugs(33, 6, "buf offset: " << mb->size);
697 }
698 
699 void
700 clientPackRangeHdr(const HttpReplyPointer &rep, const HttpHdrRangeSpec * spec, String boundary, MemBuf * mb)
701 {
702  HttpHeader hdr(hoReply);
703  assert(rep);
704  assert(spec);
705 
706  /* put boundary */
707  debugs(33, 5, "appending boundary: " << boundary);
708  /* rfc2046 requires to _prepend_ boundary with <crlf>! */
709  mb->appendf("\r\n--" SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(boundary));
710 
711  /* stuff the header with required entries and pack it */
712 
713  if (rep->header.has(Http::HdrType::CONTENT_TYPE))
715 
716  httpHeaderAddContRange(&hdr, *spec, rep->content_length);
717 
718  hdr.packInto(mb);
719  hdr.clean();
720 
721  /* append <crlf> (we packed a header, not a reply) */
722  mb->append("\r\n", 2);
723 }
724 
730 int
732 {
733  int64_t clen = 0;
734  MemBuf mb;
735 
736  assert(memObject());
737 
738  mb.init();
739  HttpHdrRange::iterator pos = request->range->begin();
740 
741  while (pos != request->range->end()) {
742  /* account for headers for this range */
743  mb.reset();
744  clientPackRangeHdr(memObject()->getReply(),
745  *pos, range_iter.boundary, &mb);
746  clen += mb.size;
747 
748  /* account for range content */
749  clen += (*pos)->length;
750 
751  debugs(33, 6, "clientMRangeCLen: (clen += " << mb.size << " + " << (*pos)->length << ") == " << clen);
752  ++pos;
753  }
754 
755  /* account for the terminating boundary */
756  mb.reset();
757 
758  clientPackTermBound(range_iter.boundary, &mb);
759 
760  clen += mb.size;
761 
762  mb.clean();
763 
764  return clen;
765 }
766 
770 String
772 {
773  const char *key;
774  String b(APP_FULLNAME);
775  b.append(":",1);
776  key = storeEntry()->getMD5Text();
777  b.append(key, strlen(key));
778  return b;
779 }
780 
790 void
792  HttpReply * rep, StoreIOBuffer receivedData)
793 {
794  // do not try to deliver if client already ABORTED
795  if (!http->getConn() || !cbdataReferenceValid(http->getConn()) || !Comm::IsConnOpen(http->getConn()->clientConnection))
796  return;
797 
798  /* Test preconditions */
799  assert(node != NULL);
801  /* TODO: handle this rather than asserting
802  * - it should only ever happen if we cause an abort and
803  * the callback chain loops back to here, so we can simply return.
804  * However, that itself shouldn't happen, so it stays as an assert for now.
805  */
807  assert(node->node.next == NULL);
808  Http::StreamPointer context = dynamic_cast<Http::Stream *>(node->data.getRaw());
809  assert(context != NULL);
810 
811  /* TODO: check offset is what we asked for */
812 
813  // TODO: enforces HTTP/1 MUST on pipeline order, but is irrelevant to HTTP/2
814  if (context != http->getConn()->pipeline.front())
815  context->deferRecipientForLater(node, rep, receivedData);
816  else if (http->getConn()->cbControlMsgSent) // 1xx to the user is pending
817  context->deferRecipientForLater(node, rep, receivedData);
818  else
819  http->getConn()->handleReply(rep, receivedData);
820 
822 }
823 
829 void
831 {
832  /* Test preconditions */
833  assert(node != NULL);
834  /* TODO: handle this rather than asserting
835  * - it should only ever happen if we cause an abort and
836  * the callback chain loops back to here, so we can simply return.
837  * However, that itself shouldn't happen, so it stays as an assert for now.
838  */
840  /* Set null by ContextFree */
841  assert(node->node.next == NULL);
842  /* this is the assert discussed above */
843  assert(NULL == dynamic_cast<Http::Stream *>(node->data.getRaw()));
844  /* We are only called when the client socket shutsdown.
845  * Tell the prev pipeline member we're finished
846  */
847  clientStreamDetach(node, http);
848 }
849 
850 void
852 {
853  debugs(33, 5, HERE << clientConnection << " reading next req");
854 
855  fd_note(clientConnection->fd, "Idle client: Waiting for next request");
860  AsyncCall::Pointer timeoutCall = JobCallback(33, 5,
861  TimeoutDialer, this, ConnStateData::requestTimeout);
862  commSetConnTimeout(clientConnection, clientConnection->timeLeft(idleTimeout()), timeoutCall);
863 
864  readSomeData();
866 }
867 
868 static void
870 {
871  debugs(33, 2, HERE << conn->clientConnection << " Sending next");
872 
875  if (deferredRequest->flags.deferred) {
877  assert(deferredRequest->http->out.size == 0);
879  clientSocketRecipient(deferredRequest->deferredparams.node,
880  deferredRequest->http,
881  deferredRequest->deferredparams.rep,
882  deferredRequest->deferredparams.queuedBuffer);
883  }
884 
888 }
889 
890 void
892 {
893  if (!Comm::IsConnOpen(clientConnection)) {
894  debugs(33, 2, clientConnection << " Connection was closed");
895  return;
896  }
897 
898  if (pinning.pinned && !Comm::IsConnOpen(pinning.serverConnection)) {
899  debugs(33, 2, clientConnection << " Connection was pinned but server side gone. Terminating client connection");
900  clientConnection->close();
901  return;
902  }
903 
918  if (const char *reason = stoppedReceiving()) {
919  debugs(33, 3, "closing for earlier request error: " << reason);
920  clientConnection->close();
921  return;
922  }
923 
934  if (clientParseRequests()) {
935  debugs(33, 3, clientConnection << ": parsed next request from buffer");
936  }
937 
945  if (commIsHalfClosed(clientConnection->fd) && pipeline.empty()) {
946  debugs(33, 3, "half-closed client with no pending requests, closing");
947  clientConnection->close();
948  return;
949  }
950 
958  Http::StreamPointer deferredRequest = pipeline.front();
959  if (deferredRequest != nullptr) {
960  debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded");
961  ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
962  } else if (flags.readMore) {
963  debugs(33, 3, clientConnection << ": calling readNextRequest()");
964  readNextRequest();
965  } else {
966  // XXX: Can this happen? CONNECT tunnels have deferredRequest set.
967  debugs(33, DBG_IMPORTANT, MYNAME << "abandoning " << clientConnection);
968  }
969 }
970 
971 void
973 {
974  debugs(33, 4, HERE << "sending error (" << clientConnection << "): " << error <<
975  "; old receiving error: " <<
976  (stoppedReceiving() ? stoppedReceiving_ : "none"));
977 
978  if (const char *oldError = stoppedSending()) {
979  debugs(33, 3, HERE << "already stopped sending: " << oldError);
980  return; // nothing has changed as far as this connection is concerned
981  }
982  stoppedSending_ = error;
983 
984  if (!stoppedReceiving()) {
985  if (const int64_t expecting = mayNeedToReadMoreBody()) {
986  debugs(33, 5, HERE << "must still read " << expecting <<
987  " request body bytes with " << inBuf.length() << " unused");
988  return; // wait for the request receiver to finish reading
989  }
990  }
991 
992  clientConnection->close();
993 }
994 
995 void
997 {
998  if (pipeline.empty())
999  return;
1000 
1001  auto ctx = pipeline.front();
1002  if (size) {
1004  if (ctx->http->logType.isTcpHit())
1006  }
1007  ctx->writeComplete(size);
1008 }
1009 
1010 Http::Stream *
1012 {
1013  ClientHttpRequest *http = new ClientHttpRequest(this);
1014  http->req_sz = inBuf.length();
1015  http->setErrorUri(uri);
1016  auto *context = new Http::Stream(clientConnection, http);
1017  StoreIOBuffer tempBuffer;
1018  tempBuffer.data = context->reqbuf;
1019  tempBuffer.length = HTTP_REQBUF_SZ;
1022  clientSocketDetach, context, tempBuffer);
1023  return context;
1024 }
1025 
1026 void
1028 {
1029  // RegisteredRunner API callback - Squid has been shut down
1030 
1031  // if connection is idle terminate it now,
1032  // otherwise wait for grace period to end
1033  if (pipeline.empty())
1034  endingShutdown();
1035 }
1036 
1037 void
1039 {
1040  // RegisteredRunner API callback - Squid shutdown grace period is over
1041 
1042  // force the client connection to close immediately
1043  // swanSong() in the close handler will cleanup.
1044  if (Comm::IsConnOpen(clientConnection))
1045  clientConnection->close();
1046 }
1047 
1048 char *
1049 skipLeadingSpace(char *aString)
1050 {
1051  char *result = aString;
1052 
1053  while (xisspace(*aString))
1054  ++aString;
1055 
1056  return result;
1057 }
1058 
1064 const char *
1065 findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end)
1066 {
1067  if (NULL == end) {
1068  end = uriAndHTTPVersion + strcspn(uriAndHTTPVersion, "\r\n");
1069  assert(end);
1070  }
1071 
1072  for (; end > uriAndHTTPVersion; --end) {
1073  if (*end == '\n' || *end == '\r')
1074  continue;
1075 
1076  if (xisspace(*end)) {
1077  if (strncasecmp(end + 1, "HTTP/", 5) == 0)
1078  return end + 1;
1079  else
1080  break;
1081  }
1082  }
1083 
1084  return NULL;
1085 }
1086 
1087 static char *
1089 {
1090  int vhost = conn->port->vhost;
1091  int vport = conn->port->vport;
1092  static char ipbuf[MAX_IPSTRLEN];
1093 
1094  /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1095 
1096  static const SBuf cache_object("cache_object://");
1097  if (hp->requestUri().startsWith(cache_object))
1098  return nullptr; /* already in good shape */
1099 
1100  // XXX: re-use proper URL parser for this
1101  SBuf url = hp->requestUri(); // use full provided URI if we abort
1102  do { // use a loop so we can break out of it
1103  ::Parser::Tokenizer tok(url);
1104  if (tok.skip('/')) // origin-form URL already.
1105  break;
1106 
1107  if (conn->port->vhost)
1108  return nullptr; /* already in good shape */
1109 
1110  // skip the URI scheme
1111  static const CharacterSet uriScheme = CharacterSet("URI-scheme","+-.") + CharacterSet::ALPHA + CharacterSet::DIGIT;
1112  static const SBuf uriSchemeEnd("://");
1113  if (!tok.skipAll(uriScheme) || !tok.skip(uriSchemeEnd))
1114  break;
1115 
1116  // skip the authority segment
1117  // RFC 3986 complex nested ABNF for "authority" boils down to this:
1118  static const CharacterSet authority = CharacterSet("authority","-._~%:@[]!$&'()*+,;=") +
1120  if (!tok.skipAll(authority))
1121  break;
1122 
1123  static const SBuf slashUri("/");
1124  const SBuf t = tok.remaining();
1125  if (t.isEmpty())
1126  url = slashUri;
1127  else if (t[0]=='/') // looks like path
1128  url = t;
1129  else if (t[0]=='?' || t[0]=='#') { // looks like query or fragment. fix '/'
1130  url = slashUri;
1131  url.append(t);
1132  } // else do nothing. invalid path
1133 
1134  } while(false);
1135 
1136 #if SHOULD_REJECT_UNKNOWN_URLS
1137  // reject URI which are not well-formed even after the processing above
1138  if (url.isEmpty() || url[0] != '/') {
1139  hp->parseStatusCode = Http::scBadRequest;
1140  return conn->abortRequestParsing("error:invalid-request");
1141  }
1142 #endif
1143 
1144  if (vport < 0)
1145  vport = conn->clientConnection->local.port();
1146 
1147  char *host = NULL;
1148  if (vhost && (host = hp->getHostHeaderField())) {
1149  debugs(33, 5, "ACCEL VHOST REWRITE: vhost=" << host << " + vport=" << vport);
1150  char thost[256];
1151  if (vport > 0) {
1152  thost[0] = '\0';
1153  char *t = NULL;
1154  if (host[strlen(host) - 1] != ']' && (t = strrchr(host,':')) != nullptr) {
1155  strncpy(thost, host, (t-host));
1156  snprintf(thost+(t-host), sizeof(thost)-(t-host), ":%d", vport);
1157  host = thost;
1158  } else if (!t) {
1159  snprintf(thost, sizeof(thost), "%s:%d",host, vport);
1160  host = thost;
1161  }
1162  } // else nothing to alter port-wise.
1163  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1164  const int url_sz = scheme.length() + strlen(host) + url.length() + 32;
1165  char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1166  snprintf(uri, url_sz, SQUIDSBUFPH "://%s" SQUIDSBUFPH, SQUIDSBUFPRINT(scheme), host, SQUIDSBUFPRINT(url));
1167  debugs(33, 5, "ACCEL VHOST REWRITE: " << uri);
1168  return uri;
1169  } else if (conn->port->defaultsite /* && !vhost */) {
1170  debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: defaultsite=" << conn->port->defaultsite << " + vport=" << vport);
1171  char vportStr[32];
1172  vportStr[0] = '\0';
1173  if (vport > 0) {
1174  snprintf(vportStr, sizeof(vportStr),":%d",vport);
1175  }
1176  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1177  const int url_sz = scheme.length() + strlen(conn->port->defaultsite) + sizeof(vportStr) + url.length() + 32;
1178  char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1179  snprintf(uri, url_sz, SQUIDSBUFPH "://%s%s" SQUIDSBUFPH,
1180  SQUIDSBUFPRINT(scheme), conn->port->defaultsite, vportStr, SQUIDSBUFPRINT(url));
1181  debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: " << uri);
1182  return uri;
1183  } else if (vport > 0 /* && (!vhost || no Host:) */) {
1184  debugs(33, 5, "ACCEL VPORT REWRITE: *_port IP + vport=" << vport);
1185  /* Put the local socket IP address as the hostname, with whatever vport we found */
1187  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1188  const int url_sz = scheme.length() + sizeof(ipbuf) + url.length() + 32;
1189  char *uri = static_cast<char *>(xcalloc(url_sz, 1));
1190  snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1191  SQUIDSBUFPRINT(scheme), ipbuf, vport, SQUIDSBUFPRINT(url));
1192  debugs(33, 5, "ACCEL VPORT REWRITE: " << uri);
1193  return uri;
1194  }
1195 
1196  return nullptr;
1197 }
1198 
1199 static char *
1201 {
1202  char *uri = nullptr;
1203  /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1204  if (const char *host = hp->getHostHeaderField()) {
1205  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1206  const int url_sz = scheme.length() + strlen(host) + hp->requestUri().length() + 32;
1207  uri = static_cast<char *>(xcalloc(url_sz, 1));
1208  snprintf(uri, url_sz, SQUIDSBUFPH "://%s" SQUIDSBUFPH,
1209  SQUIDSBUFPRINT(scheme),
1210  host,
1211  SQUIDSBUFPRINT(hp->requestUri()));
1212  }
1213  return uri;
1214 }
1215 
1216 char *
1218 {
1219  Must(switchedToHttps());
1220 
1221  if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1222  return nullptr; /* already in good shape */
1223 
1224  char *uri = buildUrlFromHost(this, hp);
1225 #if USE_OPENSSL
1226  if (!uri) {
1227  Must(tlsConnectPort);
1228  Must(!tlsConnectHostOrIp.isEmpty());
1229  SBuf useHost;
1230  if (!tlsClientSni().isEmpty())
1231  useHost = tlsClientSni();
1232  else
1233  useHost = tlsConnectHostOrIp;
1234 
1235  const SBuf &scheme = AnyP::UriScheme(transferProtocol.protocol).image();
1236  const int url_sz = scheme.length() + useHost.length() + hp->requestUri().length() + 32;
1237  uri = static_cast<char *>(xcalloc(url_sz, 1));
1238  snprintf(uri, url_sz, SQUIDSBUFPH "://" SQUIDSBUFPH ":%d" SQUIDSBUFPH,
1239  SQUIDSBUFPRINT(scheme),
1240  SQUIDSBUFPRINT(useHost),
1241  tlsConnectPort,
1242  SQUIDSBUFPRINT(hp->requestUri()));
1243  }
1244 #endif
1245  if (uri)
1246  debugs(33, 5, "TLS switching host rewrite: " << uri);
1247  return uri;
1248 }
1249 
1250 static char *
1252 {
1253  // TODO Must() on URI !empty when the parser supports throw. For now avoid assert().
1254  if (!hp->requestUri().isEmpty() && hp->requestUri()[0] != '/')
1255  return nullptr; /* already in good shape */
1256 
1257  char *uri = buildUrlFromHost(conn, hp);
1258  if (!uri) {
1259  /* Put the local socket IP address as the hostname. */
1260  static char ipbuf[MAX_IPSTRLEN];
1262  const SBuf &scheme = AnyP::UriScheme(conn->transferProtocol.protocol).image();
1263  const int url_sz = sizeof(ipbuf) + hp->requestUri().length() + 32;
1264  uri = static_cast<char *>(xcalloc(url_sz, 1));
1265  snprintf(uri, url_sz, SQUIDSBUFPH "://%s:%d" SQUIDSBUFPH,
1266  SQUIDSBUFPRINT(scheme),
1267  ipbuf, conn->clientConnection->local.port(), SQUIDSBUFPRINT(hp->requestUri()));
1268  }
1269 
1270  if (uri)
1271  debugs(33, 5, "TRANSPARENT REWRITE: " << uri);
1272  return uri;
1273 }
1274 
1275 Http::Stream *
1277 {
1278  /* Attempt to parse the first line; this will define where the method, url, version and header begin */
1279  {
1280  Must(hp);
1281 
1282  if (preservingClientData_)
1283  preservedClientData = inBuf;
1284 
1285  const bool parsedOk = hp->parse(inBuf);
1286 
1287  // sync the buffers after parsing.
1288  inBuf = hp->remaining();
1289 
1290  if (hp->needsMoreData()) {
1291  debugs(33, 5, "Incomplete request, waiting for end of request line");
1292  return NULL;
1293  }
1294 
1295  if (!parsedOk) {
1296  const bool tooBig =
1297  hp->parseStatusCode == Http::scRequestHeaderFieldsTooLarge ||
1298  hp->parseStatusCode == Http::scUriTooLong;
1299  auto result = abortRequestParsing(
1300  tooBig ? "error:request-too-large" : "error:invalid-request");
1301  // assume that remaining leftovers belong to this bad request
1302  if (!inBuf.isEmpty())
1303  consumeInput(inBuf.length());
1304  return result;
1305  }
1306  }
1307 
1308  /* We know the whole request is in parser now */
1309  debugs(11, 2, "HTTP Client " << clientConnection);
1310  debugs(11, 2, "HTTP Client REQUEST:\n---------\n" <<
1311  hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol() << "\n" <<
1312  hp->mimeHeader() <<
1313  "\n----------");
1314 
1315  /* deny CONNECT via accelerated ports */
1316  if (hp->method() == Http::METHOD_CONNECT && port != NULL && port->flags.accelSurrogate) {
1317  debugs(33, DBG_IMPORTANT, "WARNING: CONNECT method received on " << transferProtocol << " Accelerator port " << port->s.port());
1318  debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1319  hp->parseStatusCode = Http::scMethodNotAllowed;
1320  return abortRequestParsing("error:method-not-allowed");
1321  }
1322 
1323  /* RFC 7540 section 11.6 registers the method PRI as HTTP/2 specific
1324  * Deny "PRI" method if used in HTTP/1.x or 0.9 versions.
1325  * If seen it signals a broken client or proxy has corrupted the traffic.
1326  */
1327  if (hp->method() == Http::METHOD_PRI && hp->messageProtocol() < Http::ProtocolVersion(2,0)) {
1328  debugs(33, DBG_IMPORTANT, "WARNING: PRI method received on " << transferProtocol << " port " << port->s.port());
1329  debugs(33, DBG_IMPORTANT, "WARNING: for request: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1330  hp->parseStatusCode = Http::scMethodNotAllowed;
1331  return abortRequestParsing("error:method-not-allowed");
1332  }
1333 
1334  if (hp->method() == Http::METHOD_NONE) {
1335  debugs(33, DBG_IMPORTANT, "WARNING: Unsupported method: " << hp->method() << " " << hp->requestUri() << " " << hp->messageProtocol());
1336  hp->parseStatusCode = Http::scMethodNotAllowed;
1337  return abortRequestParsing("error:unsupported-request-method");
1338  }
1339 
1340  // Process headers after request line
1341  debugs(33, 3, "complete request received. " <<
1342  "prefix_sz = " << hp->messageHeaderSize() <<
1343  ", request-line-size=" << hp->firstLineSize() <<
1344  ", mime-header-size=" << hp->headerBlockSize() <<
1345  ", mime header block:\n" << hp->mimeHeader() << "\n----------");
1346 
1347  /* Ok, all headers are received */
1348  ClientHttpRequest *http = new ClientHttpRequest(this);
1349 
1350  http->req_sz = hp->messageHeaderSize();
1351  Http::Stream *result = new Http::Stream(clientConnection, http);
1352 
1353  StoreIOBuffer tempBuffer;
1354  tempBuffer.data = result->reqbuf;
1355  tempBuffer.length = HTTP_REQBUF_SZ;
1356 
1357  ClientStreamData newServer = new clientReplyContext(http);
1358  ClientStreamData newClient = result;
1359  clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
1361  clientSocketDetach, newClient, tempBuffer);
1362 
1363  /* set url */
1364  debugs(33,5, "Prepare absolute URL from " <<
1365  (transparent()?"intercept":(port->flags.accelSurrogate ? "accel":"")));
1366  /* Rewrite the URL in transparent or accelerator mode */
1367  /* NP: there are several cases to traverse here:
1368  * - standard mode (forward proxy)
1369  * - transparent mode (TPROXY)
1370  * - transparent mode with failures
1371  * - intercept mode (NAT)
1372  * - intercept mode with failures
1373  * - accelerator mode (reverse proxy)
1374  * - internal relative-URL
1375  * - mixed combos of the above with internal URL
1376  * - remote interception with PROXY protocol
1377  * - remote reverse-proxy with PROXY protocol
1378  */
1379  if (switchedToHttps()) {
1380  http->uri = prepareTlsSwitchingURL(hp);
1381  } else if (transparent()) {
1382  /* intercept or transparent mode, properly working with no failures */
1383  http->uri = prepareTransparentURL(this, hp);
1384 
1385  } else if (internalCheck(hp->requestUri())) { // NP: only matches relative-URI
1386  /* internal URL mode */
1387  /* prepend our name & port */
1388  http->uri = xstrdup(internalLocalUri(NULL, hp->requestUri()));
1389  // We just re-wrote the URL. Must replace the Host: header.
1390  // But have not parsed there yet!! flag for local-only handling.
1391  http->flags.internal = true;
1392 
1393  } else if (port->flags.accelSurrogate) {
1394  /* accelerator mode */
1395  http->uri = prepareAcceleratedURL(this, hp);
1396  http->flags.accel = true;
1397  }
1398 
1399  if (!http->uri) {
1400  /* No special rewrites have been applied above, use the
1401  * requested url. may be rewritten later, so make extra room */
1402  int url_sz = hp->requestUri().length() + Config.appendDomainLen + 5;
1403  http->uri = (char *)xcalloc(url_sz, 1);
1404  SBufToCstring(http->uri, hp->requestUri());
1405  }
1406 
1407  result->flags.parsed_ok = 1;
1408  return result;
1409 }
1410 
1411 bool
1413 {
1414  if (size == 0) {
1415  if (pipeline.empty() && inBuf.isEmpty()) {
1416  /* no current or pending requests */
1417  debugs(33, 4, HERE << clientConnection << " closed");
1418  return true;
1419  } else if (!Config.onoff.half_closed_clients) {
1420  /* admin doesn't want to support half-closed client sockets */
1421  debugs(33, 3, HERE << clientConnection << " aborted (half_closed_clients disabled)");
1422  pipeline.terminateAll(0);
1423  return true;
1424  }
1425  }
1426 
1427  return false;
1428 }
1429 
1430 void
1431 ConnStateData::consumeInput(const size_t byteCount)
1432 {
1433  assert(byteCount > 0 && byteCount <= inBuf.length());
1434  inBuf.consume(byteCount);
1435  debugs(33, 5, "inBuf has " << inBuf.length() << " unused bytes");
1436 }
1437 
1438 void
1440 {
1441  // Were we expecting to read more request body from half-closed connection?
1442  if (mayNeedToReadMoreBody() && commIsHalfClosed(clientConnection->fd)) {
1443  debugs(33, 3, HERE << "truncated body: closing half-closed " << clientConnection);
1444  clientConnection->close();
1445  return;
1446  }
1447 
1448  if (flags.readMore)
1449  readSomeData();
1450 }
1451 
1452 void
1454 {
1455  // From HTTP p.o.v., we do not have to close after every error detected
1456  // at the client-side, but many such errors do require closure and the
1457  // client-side code is bad at handling errors so we play it safe.
1458  if (request)
1459  request->flags.proxyKeepalive = false;
1460  flags.readMore = false;
1461  debugs(33,4, HERE << "Will close after error: " << clientConnection);
1462 }
1463 
1464 #if USE_OPENSSL
1466 {
1467  ClientHttpRequest *http = context->http;
1468 
1469  if (!sslServerBump)
1470  return false;
1471 
1472  assert(sslServerBump->entry);
1473  // Did we create an error entry while processing CONNECT?
1474  if (!sslServerBump->entry->isEmpty()) {
1475  quitAfterError(http->request);
1476 
1477  // Get the saved error entry and send it to the client by replacing the
1478  // ClientHttpRequest store entry with it.
1480  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1481  assert(repContext);
1482  debugs(33, 5, "Responding with delated error for " << http->uri);
1483  repContext->setReplyToStoreEntry(sslServerBump->entry, "delayed SslBump error");
1484 
1485  // Get error details from the fake certificate-peeking request.
1486  http->request->detailError(sslServerBump->request->errType, sslServerBump->request->errDetail);
1487  context->pullData();
1488  return true;
1489  }
1490 
1491  // In bump-server-first mode, we have not necessarily seen the intended
1492  // server name at certificate-peeking time. Check for domain mismatch now,
1493  // when we can extract the intended name from the bumped HTTP request.
1494  if (const Security::CertPointer &srvCert = sslServerBump->serverCert) {
1495  HttpRequest *request = http->request;
1496  if (!Ssl::checkX509ServerValidity(srvCert.get(), request->url.host())) {
1497  debugs(33, 2, "SQUID_X509_V_ERR_DOMAIN_MISMATCH: Certificate " <<
1498  "does not match domainname " << request->url.host());
1499 
1500  bool allowDomainMismatch = false;
1503  check.al = http->al;
1505  check.syncAle(request, http->log_uri);
1506  allowDomainMismatch = check.fastCheck().allowed();
1507  delete check.sslErrors;
1508  check.sslErrors = NULL;
1509  }
1510 
1511  if (!allowDomainMismatch) {
1512  quitAfterError(request);
1513 
1515  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1516  assert (repContext);
1517 
1518  request->hier = sslServerBump->request->hier;
1519 
1520  // Create an error object and fill it
1521  const auto err = new ErrorState(ERR_SECURE_CONNECT_FAIL, Http::scServiceUnavailable, request, http->al);
1522  err->src_addr = clientConnection->remote;
1523  Ssl::ErrorDetail *errDetail = new Ssl::ErrorDetail(
1525  srvCert.get(), nullptr);
1526  err->detail = errDetail;
1527  repContext->setReplyToError(request->method, err);
1528  assert(context->http->out.offset == 0);
1529  context->pullData();
1530  return true;
1531  }
1532  }
1533  }
1534 
1535  return false;
1536 }
1537 #endif // USE_OPENSSL
1538 
1540 bool
1542 {
1543  assert(conn);
1544  assert(conn->pipeline.front() == context);
1545  return conn->tunnelOnError(method, requestError);
1546 }
1547 
1549 bool
1551 {
1553  debugs(33, 5, "disabled; send error: " << requestError);
1554  return false;
1555  }
1556 
1557  if (!preservingClientData_) {
1558  debugs(33, 3, "may have forgotten client data; send error: " << requestError);
1559  return false;
1560  }
1561 
1562  const auto context = pipeline.front();
1563  const auto http = context ? context->http : nullptr;
1564  const auto request = http ? http->request : nullptr;
1565 
1567  checklist.al = http ? http->al : nullptr;
1568  checklist.requestErrorType = requestError;
1569  checklist.src_addr = clientConnection->remote;
1570  checklist.my_addr = clientConnection->local;
1571  checklist.conn(this);
1572  const char *log_uri = http ? http->log_uri : nullptr;
1573  checklist.syncAle(request, log_uri);
1574  auto answer = checklist.fastCheck();
1575  if (answer.allowed() && answer.kind == 1) {
1576  debugs(33, 3, "Request will be tunneled to server");
1577  if (context)
1578  context->finished(); // Will remove from pipeline queue
1579  Comm::SetSelect(clientConnection->fd, COMM_SELECT_READ, NULL, NULL, 0);
1580  return initiateTunneledRequest(request, Http::METHOD_NONE, "unknown-protocol", preservedClientData);
1581  }
1582  debugs(33, 3, "denied; send error: " << requestError);
1583  return false;
1584 }
1585 
1586 void
1588 {
1589  /*
1590  * DPW 2007-05-18
1591  * Moved the TCP_RESET feature from clientReplyContext::sendMoreData
1592  * to here because calling comm_reset_close() causes http to
1593  * be freed before accessing.
1594  */
1595  if (request != NULL && request->flags.resetTcp && Comm::IsConnOpen(conn->clientConnection)) {
1596  debugs(33, 3, HERE << "Sending TCP RST on " << conn->clientConnection);
1597  conn->flags.readMore = false;
1599  }
1600 }
1601 
1602 void
1604 {
1605  ClientHttpRequest *http = context->http;
1606  bool chunked = false;
1607  bool mustReplyToOptions = false;
1608  bool unsupportedTe = false;
1609  bool expectBody = false;
1610 
1611  // We already have the request parsed and checked, so we
1612  // only need to go through the final body/conn setup to doCallouts().
1613  assert(http->request);
1615 
1616  // temporary hack to avoid splitting this huge function with sensitive code
1617  const bool isFtp = !hp;
1618 
1619  // Some blobs below are still HTTP-specific, but we would have to rewrite
1620  // this entire function to remove them from the FTP code path. Connection
1621  // setup and body_pipe preparation blobs are needed for FTP.
1622 
1623  request->manager(conn, http->al);
1624 
1625  request->flags.accelerated = http->flags.accel;
1626  request->flags.sslBumped=conn->switchedToHttps();
1627  // TODO: decouple http->flags.accel from request->flags.sslBumped
1628  request->flags.noDirect = (request->flags.accelerated && !request->flags.sslBumped) ?
1629  !conn->port->allow_direct : 0;
1630  request->sources |= isFtp ? Http::Message::srcFtp :
1631  ((request->flags.sslBumped || conn->port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
1632 #if USE_AUTH
1633  if (request->flags.sslBumped) {
1634  if (conn->getAuth() != NULL)
1635  request->auth_user_request = conn->getAuth();
1636  }
1637 #endif
1638 
1639  if (internalCheck(request->url.path())) {
1640  if (internalHostnameIs(request->url.host()) && request->url.port() == getMyPort()) {
1641  debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true));
1642  http->flags.internal = true;
1643  } else if (Config.onoff.global_internal_static && internalStaticCheck(request->url.path())) {
1644  debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true) << " (global_internal_static on)");
1645  request->url.setScheme(AnyP::PROTO_HTTP, "http");
1646  request->url.host(internalHostname());
1647  request->url.port(getMyPort());
1648  http->flags.internal = true;
1649  http->setLogUriToRequestUri();
1650  } else
1651  debugs(33, 2, "internal URL found: " << request->url.getScheme() << "://" << request->url.authority(true) << " (not this proxy)");
1652  }
1653 
1654  request->flags.internal = http->flags.internal;
1655 
1656  if (!isFtp) {
1657  // XXX: for non-HTTP messages instantiate a different Http::Message child type
1658  // for now Squid only supports HTTP requests
1659  const AnyP::ProtocolVersion &http_ver = hp->messageProtocol();
1660  assert(request->http_ver.protocol == http_ver.protocol);
1661  request->http_ver.major = http_ver.major;
1662  request->http_ver.minor = http_ver.minor;
1663  }
1664 
1665  if (request->header.chunked()) {
1666  chunked = true;
1667  } else if (request->header.has(Http::HdrType::TRANSFER_ENCODING)) {
1669  // HTTP/1.1 requires chunking to be the last encoding if there is one
1670  unsupportedTe = te.size() && te != "identity";
1671  } // else implied identity coding
1672 
1673  mustReplyToOptions = (request->method == Http::METHOD_OPTIONS) &&
1674  (request->header.getInt64(Http::HdrType::MAX_FORWARDS) == 0);
1675  if (!urlCheckRequest(request.getRaw()) || mustReplyToOptions || unsupportedTe) {
1677  conn->quitAfterError(request.getRaw());
1678  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1679  assert (repContext);
1681  conn->clientConnection->remote, request.getRaw(), NULL, NULL);
1682  assert(context->http->out.offset == 0);
1683  context->pullData();
1684  clientProcessRequestFinished(conn, request);
1685  return;
1686  }
1687 
1688  if (!chunked && !clientIsContentLengthValid(request.getRaw())) {
1690  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1691  assert (repContext);
1692  conn->quitAfterError(request.getRaw());
1693  repContext->setReplyToError(ERR_INVALID_REQ,
1694  Http::scLengthRequired, request->method, NULL,
1695  conn->clientConnection->remote, request.getRaw(), NULL, NULL);
1696  assert(context->http->out.offset == 0);
1697  context->pullData();
1698  clientProcessRequestFinished(conn, request);
1699  return;
1700  }
1701 
1702  clientSetKeepaliveFlag(http);
1703  // Let tunneling code be fully responsible for CONNECT requests
1704  if (http->request->method == Http::METHOD_CONNECT) {
1705  context->mayUseConnection(true);
1706  conn->flags.readMore = false;
1707  }
1708 
1709 #if USE_OPENSSL
1710  if (conn->switchedToHttps() && conn->serveDelayedError(context)) {
1711  clientProcessRequestFinished(conn, request);
1712  return;
1713  }
1714 #endif
1715 
1716  /* Do we expect a request-body? */
1717  expectBody = chunked || request->content_length > 0;
1718  if (!context->mayUseConnection() && expectBody) {
1719  request->body_pipe = conn->expectRequestBody(
1720  chunked ? -1 : request->content_length);
1721 
1722  /* Is it too large? */
1723  if (!chunked && // if chunked, we will check as we accumulate
1726  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1727  assert (repContext);
1728  conn->quitAfterError(request.getRaw());
1729  repContext->setReplyToError(ERR_TOO_BIG,
1731  conn->clientConnection->remote, http->request, NULL, NULL);
1732  assert(context->http->out.offset == 0);
1733  context->pullData();
1734  clientProcessRequestFinished(conn, request);
1735  return;
1736  }
1737 
1738  if (!isFtp) {
1739  // We may stop producing, comm_close, and/or call setReplyToError()
1740  // below, so quit on errors to avoid http->doCallouts()
1741  if (!conn->handleRequestBodyData()) {
1742  clientProcessRequestFinished(conn, request);
1743  return;
1744  }
1745 
1746  if (!request->body_pipe->productionEnded()) {
1747  debugs(33, 5, "need more request body");
1748  context->mayUseConnection(true);
1749  assert(conn->flags.readMore);
1750  }
1751  }
1752  }
1753 
1754  http->calloutContext = new ClientRequestContext(http);
1755 
1756  http->doCallouts();
1757 
1758  clientProcessRequestFinished(conn, request);
1759 }
1760 
1761 int
1763 {
1764  // TODO: Support pipelined requests through pinned connections.
1765  if (pinning.pinned)
1766  return 0;
1768 }
1769 
1775 bool
1777 {
1778  const int existingRequestCount = pipeline.count();
1779 
1780  // default to the configured pipeline size.
1781  // add 1 because the head of pipeline is counted in concurrent requests and not prefetch queue
1782 #if USE_OPENSSL
1783  const int internalRequest = (transparent() && sslBumpMode == Ssl::bumpSplice) ? 1 : 0;
1784 #else
1785  const int internalRequest = 0;
1786 #endif
1787  const int concurrentRequestLimit = pipelinePrefetchMax() + 1 + internalRequest;
1788 
1789  // when queue filled already we cant add more.
1790  if (existingRequestCount >= concurrentRequestLimit) {
1791  debugs(33, 3, clientConnection << " max concurrent requests reached (" << concurrentRequestLimit << ")");
1792  debugs(33, 5, clientConnection << " deferring new request until one is done");
1793  return true;
1794  }
1795 
1796  return false;
1797 }
1798 
1804 bool
1806 {
1808  return proxyProtocolError("PROXY client not permitted by default ACL");
1809 
1810  ACLFilledChecklist ch(Config.accessList.proxyProtocol, NULL, clientConnection->rfc931);
1811  ch.src_addr = clientConnection->remote;
1812  ch.my_addr = clientConnection->local;
1813  ch.conn(this);
1814 
1815  if (!ch.fastCheck().allowed())
1816  return proxyProtocolError("PROXY client not permitted by ACLs");
1817 
1818  return true;
1819 }
1820 
1826 bool
1828 {
1829  if (msg) {
1830  // This is important to know, but maybe not so much that flooding the log is okay.
1831 #if QUIET_PROXY_PROTOCOL
1832  // display the first of every 32 occurances at level 1, the others at level 2.
1833  static uint8_t hide = 0;
1834  debugs(33, (hide++ % 32 == 0 ? DBG_IMPORTANT : 2), msg << " from " << clientConnection);
1835 #else
1836  debugs(33, DBG_IMPORTANT, msg << " from " << clientConnection);
1837 #endif
1838  mustStop(msg);
1839  }
1840  return false;
1841 }
1842 
1847 bool
1849 {
1850  try {
1851  const auto parsed = ProxyProtocol::Parse(inBuf);
1852  proxyProtocolHeader_ = parsed.header;
1853  assert(bool(proxyProtocolHeader_));
1854  inBuf.consume(parsed.size);
1855  needProxyProtocolHeader_ = false;
1856  if (proxyProtocolHeader_->hasForwardedAddresses()) {
1857  clientConnection->local = proxyProtocolHeader_->destinationAddress;
1858  clientConnection->remote = proxyProtocolHeader_->sourceAddress;
1859  if ((clientConnection->flags & COMM_TRANSPARENT))
1860  clientConnection->flags ^= COMM_TRANSPARENT; // prevent TPROXY spoofing of this new IP.
1861  debugs(33, 5, "PROXY/" << proxyProtocolHeader_->version() << " upgrade: " << clientConnection);
1862  }
1863  } catch (const Parser::BinaryTokenizer::InsufficientInput &) {
1864  debugs(33, 3, "PROXY protocol: waiting for more than " << inBuf.length() << " bytes");
1865  return false;
1866  } catch (const std::exception &e) {
1867  return proxyProtocolError(e.what());
1868  }
1869  return true;
1870 }
1871 
1872 void
1874 {
1875  if (receivedFirstByte_)
1876  return;
1877 
1878  receivedFirstByte_ = true;
1879  // Set timeout to Config.Timeout.request
1881  AsyncCall::Pointer timeoutCall = JobCallback(33, 5,
1882  TimeoutDialer, this, ConnStateData::requestTimeout);
1883  commSetConnTimeout(clientConnection, Config.Timeout.request, timeoutCall);
1884 }
1885 
1891 bool
1893 {
1894  bool parsed_req = false;
1895 
1896  debugs(33, 5, HERE << clientConnection << ": attempting to parse");
1897 
1898  // Loop while we have read bytes that are not needed for producing the body
1899  // On errors, bodyPipe may become nil, but readMore will be cleared
1900  while (!inBuf.isEmpty() && !bodyPipe && flags.readMore) {
1901 
1902  // Prohibit concurrent requests when using a pinned to-server connection
1903  // because our Client classes do not support request pipelining.
1904  if (pinning.pinned && !pinning.readHandler) {
1905  debugs(33, 3, clientConnection << " waits for busy " << pinning.serverConnection);
1906  break;
1907  }
1908 
1909  /* Limit the number of concurrent requests */
1910  if (concurrentRequestQueueFilled())
1911  break;
1912 
1913  // try to parse the PROXY protocol header magic bytes
1914  if (needProxyProtocolHeader_) {
1915  if (!parseProxyProtocolHeader())
1916  break;
1917 
1918  // we have been waiting for PROXY to provide client-IP
1919  // for some lookups, ie rDNS and IDENT.
1920  whenClientIpKnown();
1921 
1922  // Done with PROXY protocol which has cleared preservingClientData_.
1923  // If the next protocol supports on_unsupported_protocol, then its
1924  // parseOneRequest() must reset preservingClientData_.
1925  assert(!preservingClientData_);
1926  }
1927 
1928  if (Http::StreamPointer context = parseOneRequest()) {
1929  debugs(33, 5, clientConnection << ": done parsing a request");
1930 
1931  AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "clientLifetimeTimeout",
1932  CommTimeoutCbPtrFun(clientLifetimeTimeout, context->http));
1933  commSetConnTimeout(clientConnection, Config.Timeout.lifetime, timeoutCall);
1934 
1935  context->registerWithConn();
1936 
1937 #if USE_OPENSSL
1938  if (switchedToHttps())
1939  parsedBumpedRequestCount++;
1940 #endif
1941 
1942  processParsedRequest(context);
1943 
1944  parsed_req = true; // XXX: do we really need to parse everything right NOW ?
1945 
1946  if (context->mayUseConnection()) {
1947  debugs(33, 3, HERE << "Not parsing new requests, as this request may need the connection");
1948  break;
1949  }
1950  } else {
1951  debugs(33, 5, clientConnection << ": not enough request data: " <<
1952  inBuf.length() << " < " << Config.maxRequestHeaderSize);
1953  Must(inBuf.length() < Config.maxRequestHeaderSize);
1954  break;
1955  }
1956  }
1957 
1958  /* XXX where to 'finish' the parsing pass? */
1959  return parsed_req;
1960 }
1961 
1962 void
1964 {
1965 #if USE_OPENSSL
1966  if (parsingTlsHandshake) {
1967  parseTlsHandshake();
1968  return;
1969  }
1970 #endif
1971 
1972  /* Process next request */
1973  if (pipeline.empty())
1974  fd_note(clientConnection->fd, "Reading next request");
1975 
1976  if (!clientParseRequests()) {
1977  if (!isOpen())
1978  return;
1979  /*
1980  * If the client here is half closed and we failed
1981  * to parse a request, close the connection.
1982  * The above check with connFinishedWithConn() only
1983  * succeeds _if_ the buffer is empty which it won't
1984  * be if we have an incomplete request.
1985  * XXX: This duplicates ConnStateData::kick
1986  */
1987  if (pipeline.empty() && commIsHalfClosed(clientConnection->fd)) {
1988  debugs(33, 5, clientConnection << ": half-closed connection, no completed request parsed, connection closing.");
1989  clientConnection->close();
1990  return;
1991  }
1992  }
1993 
1994  if (!isOpen())
1995  return;
1996 
1997  clientAfterReadingRequests();
1998 }
1999 
2006 bool
2008 {
2009  // if we are reading a body, stuff data into the body pipe
2010  if (bodyPipe != NULL)
2011  return handleRequestBodyData();
2012  return true;
2013 }
2014 
2022 bool
2024 {
2025  assert(bodyPipe != NULL);
2026 
2027  if (bodyParser) { // chunked encoding
2028  if (const err_type error = handleChunkedRequestBody()) {
2029  abortChunkedRequestBody(error);
2030  return false;
2031  }
2032  } else { // identity encoding
2033  debugs(33,5, HERE << "handling plain request body for " << clientConnection);
2034  const size_t putSize = bodyPipe->putMoreData(inBuf.c_str(), inBuf.length());
2035  if (putSize > 0)
2036  consumeInput(putSize);
2037 
2038  if (!bodyPipe->mayNeedMoreData()) {
2039  // BodyPipe will clear us automagically when we produced everything
2040  bodyPipe = NULL;
2041  }
2042  }
2043 
2044  if (!bodyPipe) {
2045  debugs(33,5, HERE << "produced entire request body for " << clientConnection);
2046 
2047  if (const char *reason = stoppedSending()) {
2048  /* we've finished reading like good clients,
2049  * now do the close that initiateClose initiated.
2050  */
2051  debugs(33, 3, HERE << "closing for earlier sending error: " << reason);
2052  clientConnection->close();
2053  return false;
2054  }
2055  }
2056 
2057  return true;
2058 }
2059 
2061 err_type
2063 {
2064  debugs(33, 7, "chunked from " << clientConnection << ": " << inBuf.length());
2065 
2066  try { // the parser will throw on errors
2067 
2068  if (inBuf.isEmpty()) // nothing to do
2069  return ERR_NONE;
2070 
2071  BodyPipeCheckout bpc(*bodyPipe);
2072  bodyParser->setPayloadBuffer(&bpc.buf);
2073  const bool parsed = bodyParser->parse(inBuf);
2074  inBuf = bodyParser->remaining(); // sync buffers
2075  bpc.checkIn();
2076 
2077  // dechunk then check: the size limit applies to _dechunked_ content
2078  if (clientIsRequestBodyTooLargeForPolicy(bodyPipe->producedSize()))
2079  return ERR_TOO_BIG;
2080 
2081  if (parsed) {
2082  finishDechunkingRequest(true);
2083  Must(!bodyPipe);
2084  return ERR_NONE; // nil bodyPipe implies body end for the caller
2085  }
2086 
2087  // if chunk parser needs data, then the body pipe must need it too
2088  Must(!bodyParser->needsMoreData() || bodyPipe->mayNeedMoreData());
2089 
2090  // if parser needs more space and we can consume nothing, we will stall
2091  Must(!bodyParser->needsMoreSpace() || bodyPipe->buf().hasContent());
2092  } catch (...) { // TODO: be more specific
2093  debugs(33, 3, HERE << "malformed chunks" << bodyPipe->status());
2094  return ERR_INVALID_REQ;
2095  }
2096 
2097  debugs(33, 7, HERE << "need more chunked data" << *bodyPipe->status());
2098  return ERR_NONE;
2099 }
2100 
2102 void
2104 {
2105  finishDechunkingRequest(false);
2106 
2107  // XXX: The code below works if we fail during initial request parsing,
2108  // but if we fail when the server connection is used already, the server may send
2109  // us its response too, causing various assertions. How to prevent that?
2110 #if WE_KNOW_HOW_TO_SEND_ERRORS
2111  Http::StreamPointer context = pipeline.front();
2112  if (context != NULL && !context->http->out.offset) { // output nothing yet
2113  clientStreamNode *node = context->getClientReplyContext();
2114  clientReplyContext *repContext = dynamic_cast<clientReplyContext*>(node->data.getRaw());
2115  assert(repContext);
2116  const Http::StatusCode scode = (error == ERR_TOO_BIG) ?
2117  Http::scPayloadTooLarge : HTTP_BAD_REQUEST;
2118  repContext->setReplyToError(error, scode,
2119  repContext->http->request->method,
2120  repContext->http->uri,
2121  CachePeer,
2122  repContext->http->request,
2123  inBuf, NULL);
2124  context->pullData();
2125  } else {
2126  // close or otherwise we may get stuck as nobody will notice the error?
2127  comm_reset_close(clientConnection);
2128  }
2129 #else
2130  debugs(33, 3, HERE << "aborting chunked request without error " << error);
2131  comm_reset_close(clientConnection);
2132 #endif
2133  flags.readMore = false;
2134 }
2135 
2136 void
2138 {
2139  // request reader may get stuck waiting for space if nobody consumes body
2140  if (bodyPipe != NULL)
2141  bodyPipe->enableAutoConsumption();
2142 
2143  // kids extend
2144 }
2145 
2147 void
2149 {
2150  if (!Comm::IsConnOpen(io.conn))
2151  return;
2152 
2153  const err_type error = receivedFirstByte_ ? ERR_REQUEST_PARSE_TIMEOUT : ERR_REQUEST_START_TIMEOUT;
2154  if (tunnelOnError(HttpRequestMethod(), error))
2155  return;
2156 
2157  /*
2158  * Just close the connection to not confuse browsers
2159  * using persistent connections. Some browsers open
2160  * a connection and then do not use it until much
2161  * later (presumeably because the request triggering
2162  * the open has already been completed on another
2163  * connection)
2164  */
2165  debugs(33, 3, "requestTimeout: FD " << io.fd << ": lifetime is expired.");
2166  io.conn->close();
2167 }
2168 
2169 static void
2171 {
2172  ClientHttpRequest *http = static_cast<ClientHttpRequest *>(io.data);
2173  debugs(33, DBG_IMPORTANT, "WARNING: Closing client connection due to lifetime timeout");
2174  debugs(33, DBG_IMPORTANT, "\t" << http->uri);
2175  http->logType.err.timedout = true;
2176  if (Comm::IsConnOpen(io.conn))
2177  io.conn->close();
2178 }
2179 
2181  AsyncJob("ConnStateData"), // kids overwrite
2182  Server(xact),
2183  bodyParser(nullptr),
2184 #if USE_OPENSSL
2185  sslBumpMode(Ssl::bumpEnd),
2186 #endif
2187  needProxyProtocolHeader_(false),
2188 #if USE_OPENSSL
2189  switchedToHttps_(false),
2190  parsingTlsHandshake(false),
2191  parsedBumpedRequestCount(0),
2192  tlsConnectPort(0),
2193  sslServerBump(NULL),
2194  signAlgorithm(Ssl::algSignTrusted),
2195 #endif
2196  stoppedSending_(NULL),
2197  stoppedReceiving_(NULL)
2198 {
2199  flags.readMore = true; // kids may overwrite
2200  flags.swanSang = false;
2201 
2202  pinning.host = NULL;
2203  pinning.port = -1;
2204  pinning.pinned = false;
2205  pinning.auth = false;
2206  pinning.zeroReply = false;
2207  pinning.peer = NULL;
2208 
2209  // store the details required for creating more MasterXaction objects as new requests come in
2210  log_addr = xact->tcpClient->remote;
2212 
2213  // register to receive notice of Squid signal events
2214  // which may affect long persisting client connections
2215  registerRunner();
2216 }
2217 
2218 void
2220 {
2223 
2224  if (port->disable_pmtu_discovery != DISABLE_PMTU_OFF &&
2225  (transparent() || port->disable_pmtu_discovery == DISABLE_PMTU_ALWAYS)) {
2226 #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
2227  int i = IP_PMTUDISC_DONT;
2228  if (setsockopt(clientConnection->fd, SOL_IP, IP_MTU_DISCOVER, &i, sizeof(i)) < 0) {
2229  int xerrno = errno;
2230  debugs(33, 2, "WARNING: Path MTU discovery disabling failed on " << clientConnection << " : " << xstrerr(xerrno));
2231  }
2232 #else
2233  static bool reported = false;
2234 
2235  if (!reported) {
2236  debugs(33, DBG_IMPORTANT, "NOTICE: Path MTU discovery disabling is not supported on your platform.");
2237  reported = true;
2238  }
2239 #endif
2240  }
2241 
2243  AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, ConnStateData::connStateClosed);
2245 
2246  needProxyProtocolHeader_ = port->flags.proxySurrogate;
2248  if (!proxyProtocolValidateClient()) // will close the connection on failure
2249  return;
2250  } else
2252 
2253  // requires needProxyProtocolHeader_ which is initialized above
2255 }
2256 
2257 void
2259 {
2260  if (Config.onoff.log_fqdn)
2262 
2263 #if USE_IDENT
2264  if (Ident::TheConfig.identLookup) {
2265  ACLFilledChecklist identChecklist(Ident::TheConfig.identLookup, NULL, NULL);
2266  identChecklist.src_addr = clientConnection->remote;
2267  identChecklist.my_addr = clientConnection->local;
2268  if (identChecklist.fastCheck().allowed())
2270  }
2271 #endif
2272 
2274 
2275 #if USE_DELAY_POOLS
2276  fd_table[clientConnection->fd].clientInfo = NULL;
2277 
2278  if (!Config.onoff.client_db)
2279  return; // client delay pools require client_db
2280 
2281  const auto &pools = ClientDelayPools::Instance()->pools;
2282  if (pools.size()) {
2284 
2285  // TODO: we check early to limit error response bandwith but we
2286  // should recheck when we can honor delay_pool_uses_indirect
2287  // TODO: we should also pass the port details for myportname here.
2290 
2291  for (unsigned int pool = 0; pool < pools.size(); ++pool) {
2292 
2293  /* pools require explicit 'allow' to assign a client into them */
2294  if (pools[pool]->access) {
2295  ch.changeAcl(pools[pool]->access);
2296  auto answer = ch.fastCheck();
2297  if (answer.allowed()) {
2298 
2299  /* request client information from db after we did all checks
2300  this will save hash lookup if client failed checks */
2302  assert(cli);
2303 
2304  /* put client info in FDE */
2305  fd_table[clientConnection->fd].clientInfo = cli;
2306 
2307  /* setup write limiter for this request */
2308  const double burst = floor(0.5 +
2309  (pools[pool]->highwatermark * Config.ClientDelay.initial)/100.0);
2310  cli->setWriteLimiter(pools[pool]->rate, burst, pools[pool]->highwatermark);
2311  break;
2312  } else {
2313  debugs(83, 4, HERE << "Delay pool " << pool << " skipped because ACL " << answer);
2314  }
2315  }
2316  }
2317  }
2318 #endif
2319 
2320  // kids must extend to actually start doing something (e.g., reading)
2321 }
2322 
2324 void
2326 {
2327  MasterXaction::Pointer xact = params.xaction;
2328  AnyP::PortCfgPointer s = xact->squidPort;
2329 
2330  // NP: it is possible the port was reconfigured when the call or accept() was queued.
2331 
2332  if (params.flag != Comm::OK) {
2333  // Its possible the call was still queued when the client disconnected
2334  debugs(33, 2, s->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2335  return;
2336  }
2337 
2338  debugs(33, 4, params.conn << ": accepted");
2339  fd_note(params.conn->fd, "client http connect");
2340 
2341  if (s->tcp_keepalive.enabled)
2342  commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
2343 
2345 
2346  // Socket is ready, setup the connection manager to start using it
2347  auto *srv = Http::NewServer(xact);
2348  AsyncJob::Start(srv); // usually async-calls readSomeData()
2349 }
2350 
2352 static bool
2354 {
2355  const auto conn = connState->clientConnection;
2356  if (Security::CreateServerSession(ctx, conn, connState->port->secure, "client https start")) {
2357  debugs(33, 5, "will negotiate TLS on " << conn);
2358  return true;
2359  }
2360 
2361  debugs(33, DBG_IMPORTANT, "ERROR: could not create TLS server context for " << conn);
2362  conn->close();
2363  return false;
2364 }
2365 
2372 static int
2374 {
2375  // TODO: maybe throw instead of returning -1
2376  // see https://github.com/squid-cache/squid/pull/81#discussion_r153053278
2377  int fd = conn->clientConnection->fd;
2378  auto session = fd_table[fd].ssl.get();
2379 
2380  errno = 0;
2381 
2382 #if USE_OPENSSL
2383  const auto ret = SSL_accept(session);
2384  if (ret > 0)
2385  return 1;
2386 
2387  const int xerrno = errno;
2388  const auto ssl_error = SSL_get_error(session, ret);
2389 
2390  switch (ssl_error) {
2391 
2392  case SSL_ERROR_WANT_READ:
2393  Comm::SetSelect(fd, COMM_SELECT_READ, callback, (callback ? conn : nullptr), 0);
2394  return 0;
2395 
2396  case SSL_ERROR_WANT_WRITE:
2397  Comm::SetSelect(fd, COMM_SELECT_WRITE, callback, (callback ? conn : nullptr), 0);
2398  return 0;
2399 
2400  case SSL_ERROR_SYSCALL:
2401  if (ret == 0) {
2402  debugs(83, 2, "Error negotiating SSL connection on FD " << fd << ": Aborted by client: " << ssl_error);
2403  } else {
2404  debugs(83, (xerrno == ECONNRESET) ? 1 : 2, "Error negotiating SSL connection on FD " << fd << ": " <<
2405  (xerrno == 0 ? Security::ErrorString(ssl_error) : xstrerr(xerrno)));
2406  }
2407  break;
2408 
2409  case SSL_ERROR_ZERO_RETURN:
2410  debugs(83, DBG_IMPORTANT, "Error negotiating SSL connection on FD " << fd << ": Closed by client");
2411  break;
2412 
2413  default:
2414  debugs(83, DBG_IMPORTANT, "Error negotiating SSL connection on FD " <<
2415  fd << ": " << Security::ErrorString(ssl_error) <<
2416  " (" << ssl_error << "/" << ret << ")");
2417  }
2418 
2419 #elif USE_GNUTLS
2420 
2421  const auto x = gnutls_handshake(session);
2422  if (x == GNUTLS_E_SUCCESS)
2423  return 1;
2424 
2425  if (gnutls_error_is_fatal(x)) {
2426  debugs(83, 2, "Error negotiating TLS on " << conn->clientConnection << ": Aborted by client: " << Security::ErrorString(x));
2427 
2428  } else if (x == GNUTLS_E_INTERRUPTED || x == GNUTLS_E_AGAIN) {
2429  const auto ioAction = (gnutls_record_get_direction(session)==0 ? COMM_SELECT_READ : COMM_SELECT_WRITE);
2430  Comm::SetSelect(fd, ioAction, callback, (callback ? conn : nullptr), 0);
2431  return 0;
2432  }
2433 
2434 #else
2435  // Performing TLS handshake should never be reachable without a TLS/SSL library.
2436  (void)session; // avoid compiler and static analysis complaints
2437  fatal("FATAL: HTTPS not supported by this Squid.");
2438 #endif
2439 
2440  return -1;
2441 }
2442 
2444 static void
2445 clientNegotiateSSL(int fd, void *data)
2446 {
2447  ConnStateData *conn = (ConnStateData *)data;
2448 
2449  const int ret = tlsAttemptHandshake(conn, clientNegotiateSSL);
2450  if (ret <= 0) {
2451  if (ret < 0) // An error
2452  conn->clientConnection->close();
2453  return;
2454  }
2455 
2456  Security::SessionPointer session(fd_table[fd].ssl);
2457 
2458 #if USE_OPENSSL
2459  if (Security::SessionIsResumed(session)) {
2460  debugs(83, 2, "Session " << SSL_get_session(session.get()) <<
2461  " reused on FD " << fd << " (" << fd_table[fd].ipaddr <<
2462  ":" << (int)fd_table[fd].remote_port << ")");
2463  } else {
2464  if (Debug::Enabled(83, 4)) {
2465  /* Write out the SSL session details.. actually the call below, but
2466  * OpenSSL headers do strange typecasts confusing GCC.. */
2467  /* PEM_write_SSL_SESSION(debug_log, SSL_get_session(ssl)); */
2468 #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x00908000L
2469  PEM_ASN1_write(reinterpret_cast<i2d_of_void *>(i2d_SSL_SESSION),
2470  PEM_STRING_SSL_SESSION, debug_log,
2471  reinterpret_cast<char *>(SSL_get_session(session.get())),
2472  nullptr, nullptr, 0, nullptr, nullptr);
2473 
2474 #elif (ALLOW_ALWAYS_SSL_SESSION_DETAIL == 1)
2475 
2476  /* When using gcc 3.3.x and OpenSSL 0.9.7x sometimes a compile error can occur here.
2477  * This is caused by an unpredicatble gcc behaviour on a cast of the first argument
2478  * of PEM_ASN1_write(). For this reason this code section is disabled. To enable it,
2479  * define ALLOW_ALWAYS_SSL_SESSION_DETAIL=1.
2480  * Because there are two possible usable cast, if you get an error here, try the other
2481  * commented line. */
2482 
2483  PEM_ASN1_write((int(*)())i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2484  debug_log,
2485  reinterpret_cast<char *>(SSL_get_session(session.get())),
2486  nullptr, nullptr, 0, nullptr, nullptr);
2487  /* PEM_ASN1_write((int(*)(...))i2d_SSL_SESSION, PEM_STRING_SSL_SESSION,
2488  debug_log,
2489  reinterpret_cast<char *>(SSL_get_session(session.get())),
2490  nullptr, nullptr, 0, nullptr, nullptr);
2491  */
2492 #else
2493  debugs(83, 4, "With " OPENSSL_VERSION_TEXT ", session details are available only defining ALLOW_ALWAYS_SSL_SESSION_DETAIL=1 in the source.");
2494 
2495 #endif
2496  /* Note: This does not automatically fflush the log file.. */
2497  }
2498 
2499  debugs(83, 2, "New session " << SSL_get_session(session.get()) <<
2500  " on FD " << fd << " (" << fd_table[fd].ipaddr << ":" <<
2501  fd_table[fd].remote_port << ")");
2502  }
2503 #else
2504  debugs(83, 2, "TLS session reuse not yet implemented.");
2505 #endif
2506 
2507  // Connection established. Retrieve TLS connection parameters for logging.
2509 
2510 #if USE_OPENSSL
2511  X509 *client_cert = SSL_get_peer_certificate(session.get());
2512 
2513  if (client_cert) {
2514  debugs(83, 3, "FD " << fd << " client certificate: subject: " <<
2515  X509_NAME_oneline(X509_get_subject_name(client_cert), 0, 0));
2516 
2517  debugs(83, 3, "FD " << fd << " client certificate: issuer: " <<
2518  X509_NAME_oneline(X509_get_issuer_name(client_cert), 0, 0));
2519 
2520  X509_free(client_cert);
2521  } else {
2522  debugs(83, 5, "FD " << fd << " has no client certificate.");
2523  }
2524 #else
2525  debugs(83, 2, "Client certificate requesting not yet implemented.");
2526 #endif
2527 
2528  conn->readSomeData();
2529 }
2530 
2535 static void
2537 {
2538  assert(connState);
2539  const Comm::ConnectionPointer &details = connState->clientConnection;
2540 
2541  if (!ctx || !httpsCreate(connState, ctx))
2542  return;
2543 
2545  AsyncCall::Pointer timeoutCall = JobCallback(33, 5, TimeoutDialer,
2546  connState, ConnStateData::requestTimeout);
2547  commSetConnTimeout(details, Config.Timeout.request, timeoutCall);
2548 
2549  Comm::SetSelect(details->fd, COMM_SELECT_READ, clientNegotiateSSL, connState, 0);
2550 }
2551 
2552 #if USE_OPENSSL
2553 
2556 static void
2558 {
2559  ConnStateData *connState = (ConnStateData *) data;
2560 
2561  // if the connection is closed or closing, just return.
2562  if (!connState->isOpen())
2563  return;
2564 
2565  if (answer.allowed()) {
2566  debugs(33, 2, "sslBump action " << Ssl::bumpMode(answer.kind) << "needed for " << connState->clientConnection);
2567  connState->sslBumpMode = static_cast<Ssl::BumpMode>(answer.kind);
2568  } else {
2569  debugs(33, 3, "sslBump not needed for " << connState->clientConnection);
2570  connState->sslBumpMode = Ssl::bumpSplice;
2571  }
2572 
2573  if (connState->sslBumpMode == Ssl::bumpTerminate) {
2574  connState->clientConnection->close();
2575  return;
2576  }
2577 
2578  if (!connState->fakeAConnectRequest("ssl-bump", connState->inBuf))
2579  connState->clientConnection->close();
2580 }
2581 #endif
2582 
2584 static void
2586 {
2587  MasterXaction::Pointer xact = params.xaction;
2588  const AnyP::PortCfgPointer s = xact->squidPort;
2589 
2590  // NP: it is possible the port was reconfigured when the call or accept() was queued.
2591 
2592  if (params.flag != Comm::OK) {
2593  // Its possible the call was still queued when the client disconnected
2594  debugs(33, 2, "httpsAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno));
2595  return;
2596  }
2597 
2598  debugs(33, 4, HERE << params.conn << " accepted, starting SSL negotiation.");
2599  fd_note(params.conn->fd, "client https connect");
2600 
2601  if (s->tcp_keepalive.enabled) {
2602  commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
2603  }
2605 
2606  // Socket is ready, setup the connection manager to start using it
2607  auto *srv = Https::NewServer(xact);
2608  AsyncJob::Start(srv); // usually async-calls postHttpsAccept()
2609 }
2610 
2611 void
2613 {
2614  if (port->flags.tunnelSslBumping) {
2615 #if USE_OPENSSL
2616  debugs(33, 5, "accept transparent connection: " << clientConnection);
2617 
2618  if (!Config.accessList.ssl_bump) {
2620  return;
2621  }
2622 
2625  // Create a fake HTTP request for ssl_bump ACL check,
2626  // using tproxy/intercept provided destination IP and port.
2627  HttpRequest *request = new HttpRequest(mx);
2628  static char ip[MAX_IPSTRLEN];
2630  request->url.host(clientConnection->local.toStr(ip, sizeof(ip)));
2631  request->url.port(clientConnection->local.port());
2632  request->myportname = port->name;
2633 
2634  ACLFilledChecklist *acl_checklist = new ACLFilledChecklist(Config.accessList.ssl_bump, request, NULL);
2635  acl_checklist->src_addr = clientConnection->remote;
2636  acl_checklist->my_addr = port->s;
2637  // Build a local AccessLogEntry to allow requiresAle() acls work
2638  acl_checklist->al = new AccessLogEntry;
2639  acl_checklist->al->cache.start_time = current_time;
2640  acl_checklist->al->tcpClient = clientConnection;
2641  acl_checklist->al->cache.port = port;
2642  acl_checklist->al->cache.caddr = log_addr;
2643  acl_checklist->al->proxyProtocolHeader = proxyProtocolHeader_;
2644  HTTPMSGUNLOCK(acl_checklist->al->request);
2645  acl_checklist->al->request = request;
2646  HTTPMSGLOCK(acl_checklist->al->request);
2647  Http::StreamPointer context = pipeline.front();
2648  ClientHttpRequest *http = context ? context->http : nullptr;
2649  const char *log_uri = http ? http->log_uri : nullptr;
2650  acl_checklist->syncAle(request, log_uri);
2651  acl_checklist->nonBlockingCheck(httpsSslBumpAccessCheckDone, this);
2652 #else
2653  fatal("FATAL: SSL-Bump requires --with-openssl");
2654 #endif
2655  return;
2656  } else {
2657  httpsEstablish(this, port->secure.staticContext);
2658  }
2659 }
2660 
2661 #if USE_OPENSSL
2662 void
2664 {
2665  ConnStateData * state_data = (ConnStateData *)(data);
2666  state_data->sslCrtdHandleReply(reply);
2667 }
2668 
2669 void
2671 {
2672  if (!isOpen()) {
2673  debugs(33, 3, "Connection gone while waiting for ssl_crtd helper reply; helper reply:" << reply);
2674  return;
2675  }
2676 
2677  if (reply.result == Helper::BrokenHelper) {
2678  debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply);
2679  } else if (!reply.other().hasContent()) {
2680  debugs(1, DBG_IMPORTANT, HERE << "\"ssl_crtd\" helper returned <NULL> reply.");
2681  } else {
2683  if (reply_message.parse(reply.other().content(), reply.other().contentSize()) != Ssl::CrtdMessage::OK) {
2684  debugs(33, 5, "Reply from ssl_crtd for " << tlsConnectHostOrIp << " is incorrect");
2685  } else {
2686  if (reply.result != Helper::Okay) {
2687  debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " cannot be generated. ssl_crtd response: " << reply_message.getBody());
2688  } else {
2689  debugs(33, 5, "Certificate for " << tlsConnectHostOrIp << " was successfully recieved from ssl_crtd");
2692  auto ssl = fd_table[clientConnection->fd].ssl.get();
2693  bool ret = Ssl::configureSSLUsingPkeyAndCertFromMemory(ssl, reply_message.getBody().c_str(), *port);
2694  if (!ret)
2695  debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2696 
2699  } else {
2701  if (ctx && !sslBumpCertKey.isEmpty())
2703  getSslContextDone(ctx);
2704  }
2705  return;
2706  }
2707  }
2708  }
2710  getSslContextDone(nil);
2711 }
2712 
2714 {
2716 
2717  const bool connectedOk = sslServerBump && sslServerBump->connectedOk();
2718  if (connectedOk) {
2719  if (X509 *mimicCert = sslServerBump->serverCert.get())
2720  certProperties.mimicCert.resetAndLock(mimicCert);
2721 
2725 
2726  for (sslproxy_cert_adapt *ca = Config.ssl_client.cert_adapt; ca != NULL; ca = ca->next) {
2727  // If the algorithm already set, then ignore it.
2728  if ((ca->alg == Ssl::algSetCommonName && certProperties.setCommonName) ||
2729  (ca->alg == Ssl::algSetValidAfter && certProperties.setValidAfter) ||
2730  (ca->alg == Ssl::algSetValidBefore && certProperties.setValidBefore) )
2731  continue;
2732 
2733  if (ca->aclList && checklist.fastCheck(ca->aclList).allowed()) {
2734  const char *alg = Ssl::CertAdaptAlgorithmStr[ca->alg];
2735  const char *param = ca->param;
2736 
2737  // For parameterless CN adaptation, use hostname from the
2738  // CONNECT request.
2739  if (ca->alg == Ssl::algSetCommonName) {
2740  if (!param)
2741  param = tlsConnectHostOrIp.c_str();
2742  certProperties.commonName = param;
2743  certProperties.setCommonName = true;
2744  } else if (ca->alg == Ssl::algSetValidAfter)
2745  certProperties.setValidAfter = true;
2746  else if (ca->alg == Ssl::algSetValidBefore)
2747  certProperties.setValidBefore = true;
2748 
2749  debugs(33, 5, HERE << "Matches certificate adaptation aglorithm: " <<
2750  alg << " param: " << (param ? param : "-"));
2751  }
2752  }
2753 
2754  certProperties.signAlgorithm = Ssl::algSignEnd;
2755  for (sslproxy_cert_sign *sg = Config.ssl_client.cert_sign; sg != NULL; sg = sg->next) {
2756  if (sg->aclList && checklist.fastCheck(sg->aclList).allowed()) {
2757  certProperties.signAlgorithm = (Ssl::CertSignAlgorithm)sg->alg;
2758  break;
2759  }
2760  }
2761  } else {// did not try to connect (e.g. client-first) or failed to connect
2762  // In case of an error while connecting to the secure server, use a
2763  // trusted certificate, with no mimicked fields and no adaptation
2764  // algorithms. There is nothing we can mimic, so we want to minimize the
2765  // number of warnings the user will have to see to get to the error page.
2766  // We will close the connection, so that the trust is not extended to
2767  // non-Squid content.
2768  certProperties.signAlgorithm = Ssl::algSignTrusted;
2769  }
2770 
2771  assert(certProperties.signAlgorithm != Ssl::algSignEnd);
2772 
2773  if (certProperties.signAlgorithm == Ssl::algSignUntrusted) {
2774  assert(port->secure.untrustedSigningCa.cert);
2775  certProperties.signWithX509.resetAndLock(port->secure.untrustedSigningCa.cert.get());
2776  certProperties.signWithPkey.resetAndLock(port->secure.untrustedSigningCa.pkey.get());
2777  } else {
2778  assert(port->secure.signingCa.cert.get());
2779  certProperties.signWithX509.resetAndLock(port->secure.signingCa.cert.get());
2780 
2781  if (port->secure.signingCa.pkey)
2782  certProperties.signWithPkey.resetAndLock(port->secure.signingCa.pkey.get());
2783  }
2784  signAlgorithm = certProperties.signAlgorithm;
2785 
2786  certProperties.signHash = Ssl::DefaultSignHash;
2787 }
2788 
2791 {
2792  debugs(33, 5, "Finding SSL certificate for " << cacheKey << " in cache");
2794  if (Security::ContextPointer *ctx = ssl_ctx_cache ? ssl_ctx_cache->get(cacheKey) : nullptr) {
2795  if (Ssl::verifySslCertificate(*ctx, certProperties)) {
2796  debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is valid");
2797  return *ctx;
2798  } else {
2799  debugs(33, 5, "Cached SSL certificate for " << certProperties.commonName << " is out of date. Delete this certificate from cache");
2800  if (ssl_ctx_cache)
2801  ssl_ctx_cache->del(cacheKey);
2802  }
2803  }
2804  return Security::ContextPointer(nullptr);
2805 }
2806 
2807 void
2809 {
2811  if (!ssl_ctx_cache || !ssl_ctx_cache->add(cacheKey, new Security::ContextPointer(ctx))) {
2812  // If it is not in storage delete after using. Else storage deleted it.
2813  fd_table[clientConnection->fd].html?amicTlsContext = ctx;
2814  }
2815 }
2816 
2817 void
2819 {
2820  // If we are called, then CONNECT has succeeded. Finalize it.
2821  if (auto xact = pipeline.front()) {
2822  if (xact->http && xact->http->request && xact->http->request->method == Http::METHOD_CONNECT)
2823  xact->finished();
2824  // cannot proceed with encryption if requests wait for plain responses
2825  Must(pipeline.empty());
2826  }
2827  /* careful: finished() above frees request, host, etc. */
2828 
2829  if (port->secure.generateHostCertificates) {
2830  Ssl::CertificateProperties certProperties;
2831  buildSslCertGenerationParams(certProperties);
2832 
2833  // Disable caching for bumpPeekAndSplice mode
2836  Ssl::InRamCertificateDbKey(certProperties, sslBumpCertKey);
2838 
2840  if (ctx) {
2841  getSslContextDone(ctx);
2842  return;
2843  }
2844  }
2845 
2846 #if USE_SSL_CRTD
2847  try {
2848  debugs(33, 5, HERE << "Generating SSL certificate for " << certProperties.commonName << " using ssl_crtd.");
2851  request_message.composeRequest(certProperties);
2852  debugs(33, 5, HERE << "SSL crtd request: " << request_message.compose().c_str());
2853  Ssl::Helper::Submit(request_message, sslCrtdHandleReplyWrapper, this);
2854  return;
2855  } catch (const std::exception &e) {
2856  debugs(33, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtd " <<
2857  "request for " << certProperties.commonName <<
2858  " certificate: " << e.what() << "; will now block to " <<
2859  "generate that certificate.");
2860  // fall through to do blocking in-process generation.
2861  }
2862 #endif // USE_SSL_CRTD
2863 
2864  debugs(33, 5, HERE << "Generating SSL certificate for " << certProperties.commonName);
2867  auto ssl = fd_table[clientConnection->fd].ssl.get();
2868  if (!Ssl::configureSSL(ssl, certProperties, *port))
2869  debugs(33, 5, "Failed to set certificates to ssl object for PeekAndSplice mode");
2870 
2873  } else {
2875  if (dynCtx && !sslBumpCertKey.isEmpty())
2877  getSslContextDone(dynCtx);
2878  }
2879  return;
2880  }
2881 
2883  getSslContextDone(nil);
2884 }
2885 
2886 void
2888 {
2889  if (port->secure.generateHostCertificates && !ctx) {
2890  debugs(33, 2, "Failed to generate TLS context for " << tlsConnectHostOrIp);
2891  }
2892 
2893  // If generated ssl context = NULL, try to use static ssl context.
2894  if (!ctx) {
2895  if (!port->secure.staticContext) {
2896  debugs(83, DBG_IMPORTANT, "Closing " << clientConnection->remote << " as lacking TLS context");
2898  return;
2899  } else {
2900  debugs(33, 5, "Using static TLS context.");
2901  ctx = port->secure.staticContext;
2902  }
2903  }
2904 
2905  if (!httpsCreate(this, ctx))
2906  return;
2907 
2908  // bumped intercepted conns should already have Config.Timeout.request set
2909  // but forwarded connections may only have Config.Timeout.lifetime. [Re]set
2910  // to make sure the connection does not get stuck on non-SSL clients.
2912  AsyncCall::Pointer timeoutCall = JobCallback(33, 5, TimeoutDialer,
2915 
2916  switchedToHttps_ = true;
2917 
2918  auto ssl = fd_table[clientConnection->fd].ssl.get();
2919  BIO *b = SSL_get_rbio(ssl);
2920  Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
2921  bio->setReadBufData(inBuf);
2922  inBuf.clear();
2924 }
2925 
2926 void
2928 {
2930  Must(http->request);
2931  auto &request = http->request;
2932 
2933  // Depending on receivedFirstByte_, we are at the start of either an
2934  // established CONNECT tunnel with the client or an intercepted TCP (and
2935  // presumably TLS) connection from the client. Expect TLS Client Hello.
2936  const auto insideConnectTunnel = receivedFirstByte_;
2937  debugs(33, 5, (insideConnectTunnel ? "post-CONNECT " : "raw TLS ") << clientConnection);
2938 
2939  tlsConnectHostOrIp = request->url.hostOrIp();
2940  tlsConnectPort = request->url.port();
2941  resetSslCommonName(request->url.host());
2942 
2943  // We are going to read new request
2944  flags.readMore = true;
2945 
2946  // keep version major.minor details the same.
2947  // but we are now performing the HTTPS handshake traffic
2949 
2950  // If sslServerBump is set, then we have decided to deny CONNECT
2951  // and now want to switch to SSL to send the error to the client
2952  // without even peeking at the origin server certificate.
2953  if (bumpServerMode == Ssl::bumpServerFirst && !sslServerBump) {
2954  request->flags.sslPeek = true;
2955  sslServerBump = new Ssl::ServerBump(http);
2956  } else if (bumpServerMode == Ssl::bumpPeek || bumpServerMode == Ssl::bumpStare) {
2957  request->flags.sslPeek = true;
2958  sslServerBump = new Ssl::ServerBump(http, nullptr, bumpServerMode);
2959  }
2960 
2961  // commSetConnTimeout() was called for this request before we switched.
2962  // Fix timeout to request_start_timeout
2964  AsyncCall::Pointer timeoutCall = JobCallback(33, 5,
2965  TimeoutDialer, this, ConnStateData::requestTimeout);
2967  // Also reset receivedFirstByte_ flag to allow this timeout work in the case we have
2968  // a bumbed "connect" request on non transparent port.
2969  receivedFirstByte_ = false;
2970  // Get more data to peek at Tls
2971  parsingTlsHandshake = true;
2972 
2973  // If the protocol has changed, then reset preservingClientData_.
2974  // Otherwise, its value initially set in start() is still valid/fresh.
2975  // shouldPreserveClientData() uses parsingTlsHandshake which is reset above.
2976  if (insideConnectTunnel)
2978 
2979  readSomeData();
2980 }
2981 
2982 void
2984 {
2986 
2987  assert(!inBuf.isEmpty());
2989  fd_note(clientConnection->fd, "Parsing TLS handshake");
2990 
2991  bool unsupportedProtocol = false;
2992  try {
2993  if (!tlsParser.parseHello(inBuf)) {
2994  // need more data to finish parsing
2995  readSomeData();
2996  return;
2997  }
2998  }
2999  catch (const std::exception &ex) {
3000  debugs(83, 2, "error on FD " << clientConnection->fd << ": " << ex.what());
3001  unsupportedProtocol = true;
3002  }
3003 
3004  parsingTlsHandshake = false;
3005 
3006  // client data may be needed for splicing and for
3007  // tunneling unsupportedProtocol after an error
3009 
3010  // Even if the parser failed, each TLS detail should either be set
3011  // correctly or still be "unknown"; copying unknown detail is a no-op.
3014  if (details && !details->serverName.isEmpty()) {
3015  resetSslCommonName(details->serverName.c_str());
3016  tlsClientSni_ = details->serverName;
3017  }
3018 
3019  // We should disable read/write handlers
3021 
3022  if (unsupportedProtocol) {
3023  Http::StreamPointer context = pipeline.front();
3024  Must(context && context->http);
3025  HttpRequest::Pointer request = context->http->request;
3026  debugs(83, 5, "Got something other than TLS Client Hello. Cannot SslBump.");
3028  context->http->al->ssl.bumpMode = Ssl::bumpSplice;
3029  if (!clientTunnelOnError(this, context, request, HttpRequestMethod(), ERR_PROTOCOL_UNKNOWN))
3031  return;
3032  }
3033 
3034  if (!sslServerBump || sslServerBump->act.step1 == Ssl::bumpClientFirst) { // Either means client-first.
3036  return;
3037  } else if (sslServerBump->act.step1 == Ssl::bumpServerFirst) {
3038  Http::StreamPointer context = pipeline.front();
3039  ClientHttpRequest *http = context ? context->http : nullptr;
3040  // will call httpsPeeked() with certificate and connection, eventually
3042  } else {
3045  }
3046 }
3047 
3049 {
3050  ConnStateData *connState = (ConnStateData *) data;
3051 
3052  // if the connection is closed or closing, just return.
3053  if (!connState->isOpen())
3054  return;
3055 
3056  debugs(33, 5, "Answer: " << answer << " kind:" << answer.kind);
3057  assert(connState->serverBump());
3058  Ssl::BumpMode bumpAction;
3059  if (answer.allowed()) {
3060  bumpAction = (Ssl::BumpMode)answer.kind;
3061  } else
3062  bumpAction = Ssl::bumpSplice;
3063 
3064  connState->serverBump()->act.step2 = bumpAction;
3065  connState->sslBumpMode = bumpAction;
3066  Http::StreamPointer context = connState->pipeline.front();
3067  if (ClientHttpRequest *http = (context ? context->http : nullptr))
3068  http->al->ssl.bumpMode = bumpAction;
3069 
3070  if (bumpAction == Ssl::bumpTerminate) {
3071  connState->clientConnection->close();
3072  } else if (bumpAction != Ssl::bumpSplice) {
3073  connState->startPeekAndSplice();
3074  } else if (!connState->splice())
3075  connState->clientConnection->close();
3076 }
3077 
3078 bool
3080 {
3081  // normally we can splice here, because we just got client hello message
3082 
3083  // fde::ssl/tls_read_method() probably reads from our own inBuf. If so, then
3084  // we should not lose any raw bytes when switching to raw I/O here.
3085  if (fd_table[clientConnection->fd].ssl.get())
3086  fd_table[clientConnection->fd].useDefaultIo();
3087 
3088  // XXX: assuming that there was an HTTP/1.1 CONNECT to begin with...
3089  // reset the current protocol to HTTP/1.1 (was "HTTPS" for the bumping process)
3091  assert(!pipeline.empty());
3092  Http::StreamPointer context = pipeline.front();
3093  Must(context);
3094  Must(context->http);
3095  ClientHttpRequest *http = context->http;
3097  context->finished();
3098  if (transparent()) {
3099  // For transparent connections, make a new fake CONNECT request, now
3100  // with SNI as target. doCallout() checks, adaptations may need that.
3101  return fakeAConnectRequest("splice", preservedClientData);
3102  } else {
3103  // For non transparent connections make a new tunneled CONNECT, which
3104  // also sets the HttpRequest::flags::forceTunnel flag to avoid
3105  // respond with "Connection Established" to the client.
3106  // This fake CONNECT request required to allow use of SNI in
3107  // doCallout() checks and adaptations.
3109  }
3110 }
3111 
3112 void
3114 {
3115  // This is the Step2 of the SSL bumping
3117  Http::StreamPointer context = pipeline.front();
3118  ClientHttpRequest *http = context ? context->http : nullptr;
3119 
3120  if (sslServerBump->step == Ssl::bumpStep1) {
3122  // Run a accessList check to check if want to splice or continue bumping
3123 
3125  acl_checklist->al = http ? http->al : nullptr;
3126  //acl_checklist->src_addr = params.conn->remote;
3127  //acl_checklist->my_addr = s->s;
3131  const char *log_uri = http ? http->log_uri : nullptr;
3132  acl_checklist->syncAle(sslServerBump->request.getRaw(), log_uri);
3133  acl_checklist->nonBlockingCheck(httpsSslBumpStep2AccessCheckDone, this);
3134  return;
3135  }
3136 
3137  // will call httpsPeeked() with certificate and connection, eventually
3138  Security::ContextPointer unConfiguredCTX(Ssl::createSSLContext(port->secure.signingCa.cert, port->secure.signingCa.pkey, port->secure));
3139  fd_table[clientConnection->fd].html?amicTlsContext = unConfiguredCTX;
3140 
3141  if (!httpsCreate(this, unConfiguredCTX))
3142  return;
3143 
3144  switchedToHttps_ = true;
3145 
3146  auto ssl = fd_table[clientConnection->fd].ssl.get();
3147  BIO *b = SSL_get_rbio(ssl);
3148  Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3149  bio->setReadBufData(inBuf);
3150  bio->hold(true);
3151 
3152  // Here squid should have all of the client hello message so the
3153  // tlsAttemptHandshake() should return 0.
3154  // This block exist only to force openSSL parse client hello and detect
3155  // ERR_SECURE_ACCEPT_FAIL error, which should be checked and splice if required.
3156  if (tlsAttemptHandshake(this, nullptr) < 0) {
3157  debugs(83, 2, "TLS handshake failed.");
3158  HttpRequest::Pointer request(http ? http->request : nullptr);
3159  if (!clientTunnelOnError(this, context, request, HttpRequestMethod(), ERR_SECURE_ACCEPT_FAIL))
3161  return;
3162  }
3163 
3164  // We need to reset inBuf here, to be used by incoming requests in the case
3165  // of SSL bump
3166  inBuf.clear();
3167 
3168  debugs(83, 5, "Peek and splice at step2 done. Start forwarding the request!!! ");
3170 }
3171 
3172 void
3174 {
3175  auto ssl = fd_table[clientConnection->fd].ssl.get();
3176  BIO *b = SSL_get_rbio(ssl);
3177  assert(b);
3178  Ssl::ClientBio *bio = static_cast<Ssl::ClientBio *>(BIO_get_data(b));
3179 
3180  debugs(33, 5, "PeekAndSplice mode, proceed with client negotiation. Currrent state:" << SSL_state_string_long(ssl));
3181  bio->hold(false);
3182 
3184  switchedToHttps_ = true;
3185 }
3186 
3187 void
3189 {
3190  Must(sslServerBump != NULL);
3191  Must(sslServerBump->request == pic.request);
3192  Must(pipeline.empty() || pipeline.front()->http == nullptr || pipeline.front()->http->request == pic.request.getRaw());
3193 
3194  if (Comm::IsConnOpen(pic.connection)) {
3196  debugs(33, 5, "bumped HTTPS server: " << tlsConnectHostOrIp);
3197  } else
3198  debugs(33, 5, "Error while bumping: " << tlsConnectHostOrIp);
3199 
3201 }
3202 
3203 #endif /* USE_OPENSSL */
3204 
3205 bool
3206 ConnStateData::initiateTunneledRequest(HttpRequest::Pointer const &cause, Http::MethodType const method, const char *reason, const SBuf &payload)
3207 {
3208  // fake a CONNECT request to force connState to tunnel
3209  SBuf connectHost;
3210  unsigned short connectPort = 0;
3211 
3212  if (pinning.serverConnection != nullptr) {
3213  static char ip[MAX_IPSTRLEN];
3214  connectHost = pinning.serverConnection->remote.toStr(ip, sizeof(ip));
3215  connectPort = pinning.serverConnection->remote.port();
3216  } else if (cause) {
3217  connectHost = cause->url.hostOrIp();
3218  connectPort = cause->url.port();
3219 #if USE_OPENSSL
3220  } else if (!tlsConnectHostOrIp.isEmpty()) {
3221  connectHost = tlsConnectHostOrIp;
3222  connectPort = tlsConnectPort;
3223 #endif
3224  } else if (transparent()) {
3225  static char ip[MAX_IPSTRLEN];
3226  connectHost = clientConnection->local.toStr(ip, sizeof(ip));
3227  connectPort = clientConnection->local.port();
3228  } else {
3229  debugs(33, 2, "Not able to compute URL, abort request tunneling for " << reason);
3230  return false;
3231  }
3232 
3233  debugs(33, 2, "Request tunneling for " << reason);
3234  ClientHttpRequest *http = buildFakeRequest(method, connectHost, connectPort, payload);
3236  request->flags.forceTunnel = true;
3237  http->calloutContext = new ClientRequestContext(http);
3238  http->doCallouts();
3239  clientProcessRequestFinished(this, request);
3240  return true;
3241 }
3242 
3243 bool
3244 ConnStateData::fakeAConnectRequest(const char *reason, const SBuf &payload)
3245 {
3246  debugs(33, 2, "fake a CONNECT request to force connState to tunnel for " << reason);
3247 
3248  SBuf connectHost;
3249  assert(transparent());
3250  const unsigned short connectPort = clientConnection->local.port();
3251 
3252 #if USE_OPENSSL
3253  if (!tlsClientSni_.isEmpty())
3254  connectHost.assign(tlsClientSni_);
3255  else
3256 #endif
3257  {
3258  static char ip[MAX_IPSTRLEN];
3259  clientConnection->local.toHostStr(ip, sizeof(ip));
3260  connectHost.assign(ip);
3261  }
3262 
3263  ClientHttpRequest *http = buildFakeRequest(Http::METHOD_CONNECT, connectHost, connectPort, payload);
3264 
3265  http->calloutContext = new ClientRequestContext(http);
3267  http->doCallouts();
3268  clientProcessRequestFinished(this, request);
3269  return true;
3270 }
3271 
3273 ConnStateData::buildFakeRequest(Http::MethodType const method, SBuf &useHost, unsigned short usePort, const SBuf &payload)
3274 {
3275  ClientHttpRequest *http = new ClientHttpRequest(this);
3276  Http::Stream *stream = new Http::Stream(clientConnection, http);
3277 
3278  StoreIOBuffer tempBuffer;
3279  tempBuffer.data = stream->reqbuf;
3280  tempBuffer.length = HTTP_REQBUF_SZ;
3281 
3282  ClientStreamData newServer = new clientReplyContext(http);
3283  ClientStreamData newClient = stream;
3286  clientSocketDetach, newClient, tempBuffer);
3287 
3288  stream->flags.parsed_ok = 1; // Do we need it?
3289  stream->mayUseConnection(true);
3290 
3291  AsyncCall::Pointer timeoutCall = commCbCall(5, 4, "clientLifetimeTimeout",
3294 
3295  stream->registerWithConn();
3296 
3299  // Setup Http::Request object. Maybe should be replaced by a call to (modified)
3300  // clientProcessRequest
3303  request->url.setScheme(proto, nullptr);
3304  request->method = method;
3305  request->url.host(useHost.c_str());
3306  request->url.port(usePort);
3307 
3308  http->uri = SBufToCstring(request->effectiveRequestUri());
3309  http->initRequest(request.getRaw());
3310 
3311  request->manager(this, http->al);
3312 
3313  if (proto == AnyP::PROTO_HTTP)
3314  request->header.putStr(Http::HOST, useHost.c_str());
3315 
3316  request->sources |= ((switchedToHttps() || port->transport.protocol == AnyP::PROTO_HTTPS) ? Http::Message::srcHttps : Http::Message::srcHttp);
3317 #if USE_AUTH
3318  if (getAuth())
3319  request->auth_user_request = getAuth();
3320 #endif
3321 
3322  inBuf = payload;
3323  flags.readMore = false;
3324 
3325  return http;
3326 }
3327 
3329 static bool
3331 {
3332  if (!Comm::IsConnOpen(c)) {
3333  Must(NHttpSockets > 0); // we tried to open some
3334  --NHttpSockets; // there will be fewer sockets than planned
3335  Must(HttpSockets[NHttpSockets] < 0); // no extra fds received
3336 
3337  if (!NHttpSockets) // we could not open any listen sockets at all
3338  fatalf("Unable to open %s",FdNote(portType));
3339 
3340  return false;
3341  }
3342  return true;
3343 }
3344 
3346 static bool
3348 {
3349  bool found = false;
3350  for (int i = 0; i < NHttpSockets && !found; ++i) {
3351  if ((found = HttpSockets[i] < 0))
3352  HttpSockets[i] = conn->fd;
3353  }
3354  return found;
3355 }
3356 
3357 static void
3359 {
3360  for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) {
3361  const SBuf &scheme = AnyP::UriScheme(s->transport.protocol).image();
3362 
3364  debugs(1, DBG_IMPORTANT, "WARNING: You have too many '" << scheme << "_port' lines.");
3365  debugs(1, DBG_IMPORTANT, " The limit is " << MAXTCPLISTENPORTS << " HTTP ports.");
3366  continue;
3367  }
3368 
3369 #if USE_OPENSSL
3370  if (s->flags.tunnelSslBumping) {
3371  if (!Config.accessList.ssl_bump) {
3372  debugs(33, DBG_IMPORTANT, "WARNING: No ssl_bump configured. Disabling ssl-bump on " << scheme << "_port " << s->s);
3373  s->flags.tunnelSslBumping = false;
3374  }
3375  if (!s->secure.staticContext && !s->secure.generateHostCertificates) {
3376  debugs(1, DBG_IMPORTANT, "Will not bump SSL at " << scheme << "_port " << s->s << " due to TLS initialization failure.");
3377  s->flags.tunnelSslBumping = false;
3378  if (s->transport.protocol == AnyP::PROTO_HTTP)
3379  s->secure.encryptTransport = false;
3380  }
3381  if (s->flags.tunnelSslBumping) {
3382  // Create ssl_ctx cache for this port.
3383  Ssl::TheGlobalContextStorage.addLocalStorage(s->s, s->secure.html?amicCertMemCacheSize);
3384  }
3385  }
3386 #endif
3387 
3388  if (s->secure.encryptTransport && !s->secure.staticContext) {
3389  debugs(1, DBG_CRITICAL, "ERROR: Ignoring " << scheme << "_port " << s->s << " due to TLS context initialization failure.");
3390  continue;
3391  }
3392 
3393  // Fill out a Comm::Connection which IPC will open as a listener for us
3394  // then pass back when active so we can start a TcpAcceptor subscription.
3395  s->listenConn = new Comm::Connection;
3396  s->listenConn->local = s->s;
3397 
3398  s->listenConn->flags = COMM_NONBLOCKING | (s->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) |
3399  (s->flags.natIntercept ? COMM_INTERCEPTION : 0);
3400 
3401  typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
3402  if (s->transport.protocol == AnyP::PROTO_HTTP) {
3403  // setup the subscriptions such that new connections accepted by listenConn are handled by HTTP
3406 
3407  AsyncCall::Pointer listenCall = asyncCall(33,2, "clientListenerConnectionOpened",
3409  Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnHttpSocket, listenCall);
3410 
3411  } else if (s->transport.protocol == AnyP::PROTO_HTTPS) {
3412  // setup the subscriptions such that new connections accepted by listenConn are handled by HTTPS
3415 
3416  AsyncCall::Pointer listenCall = asyncCall(33, 2, "clientListenerConnectionOpened",
3418  s, Ipc::fdnHttpsSocket, sub));
3419  Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnHttpsSocket, listenCall);
3420  }
3421 
3422  HttpSockets[NHttpSockets] = -1; // set in clientListenerConnectionOpened
3423  ++NHttpSockets;
3424  }
3425 }
3426 
3427 void
3429 {
3430  // Fill out a Comm::Connection which IPC will open as a listener for us
3431  port->listenConn = new Comm::Connection;
3432  port->listenConn->local = port->s;
3433  port->listenConn->flags =
3435  (port->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) |
3436  (port->flags.natIntercept ? COMM_INTERCEPTION : 0);
3437 
3438  // route new connections to subCall
3439  typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
3441  AsyncCall::Pointer listenCall =
3442  asyncCall(33, 2, "clientListenerConnectionOpened",
3444  port, fdNote, sub));
3445  Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, port->listenConn, fdNote, listenCall);
3446 
3448  HttpSockets[NHttpSockets] = -1;
3449  ++NHttpSockets;
3450 }
3451 
3453 static void
3455 {
3456  Must(s != NULL);
3457 
3458  if (!OpenedHttpSocket(s->listenConn, portTypeNote))
3459  return;
3460 
3461  Must(Comm::IsConnOpen(s->listenConn));
3462 
3463  // TCP: setup a job to handle accept() with subscribed handler
3464  AsyncJob::Start(new Comm::TcpAcceptor(s, FdNote(portTypeNote), sub));
3465 
3466  debugs(1, DBG_IMPORTANT, "Accepting " <<
3467  (s->flags.natIntercept ? "NAT intercepted " : "") <<
3468  (s->flags.tproxyIntercept ? "TPROXY intercepted " : "") <<
3469  (s->flags.tunnelSslBumping ? "SSL bumped " : "") <<
3470  (s->flags.accelSurrogate ? "reverse-proxy " : "")
3471  << FdNote(portTypeNote) << " connections at "
3472  << s->listenConn);
3473 
3474  Must(AddOpenedHttpSocket(s->listenConn)); // otherwise, we have received a fd we did not ask for
3475 }
3476 
3477 void
3479 {
3482 
3483  if (NHttpSockets < 1)
3484  fatal("No HTTP, HTTPS, or FTP ports configured");
3485 }
3486 
3487 void
3489 {
3490  for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) {
3491  if (s->listenConn != NULL) {
3492  debugs(1, DBG_IMPORTANT, "Closing HTTP(S) port " << s->listenConn->local);
3493  s->listenConn->close();
3494  s->listenConn = NULL;
3495  }
3496  }
3497 
3499 
3500  // TODO see if we can drop HttpSockets array entirely */
3501  for (int i = 0; i < NHttpSockets; ++i) {
3502  HttpSockets[i] = -1;
3503  }
3504 
3505  NHttpSockets = 0;
3506 }
3507 
3508 int
3510 {
3511  SBuf vary(request->vary_headers);
3512  int has_vary = entry->getReply()->header.has(Http::HdrType::VARY);
3513 #if X_ACCELERATOR_VARY
3514 
3515  has_vary |=
3517 #endif
3518 
3519  if (!has_vary || entry->mem_obj->vary_headers.isEmpty()) {
3520  if (!vary.isEmpty()) {
3521  /* Oops... something odd is going on here.. */
3522  debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary object on second attempt, '" <<
3523  entry->mem_obj->urlXXX() << "' '" << vary << "'");
3524  request->vary_headers.clear();
3525  return VARY_CANCEL;
3526  }
3527 
3528  if (!has_vary) {
3529  /* This is not a varying object */
3530  return VARY_NONE;
3531  }
3532 
3533  /* virtual "vary" object found. Calculate the vary key and
3534  * continue the search
3535  */
3536  vary = httpMakeVaryMark(request, entry->getReply());
3537 
3538  if (!vary.isEmpty()) {
3539  request->vary_headers = vary;
3540  return VARY_OTHER;
3541  } else {
3542  /* Ouch.. we cannot handle this kind of variance */
3543  /* XXX This cannot really happen, but just to be complete */
3544  return VARY_CANCEL;
3545  }
3546  } else {
3547  if (vary.isEmpty()) {
3548  vary = httpMakeVaryMark(request, entry->getReply());
3549 
3550  if (!vary.isEmpty())
3551  request->vary_headers = vary;
3552  }
3553 
3554  if (vary.isEmpty()) {
3555  /* Ouch.. we cannot handle this kind of variance */
3556  /* XXX This cannot really happen, but just to be complete */
3557  return VARY_CANCEL;
3558  } else if (vary.cmp(entry->mem_obj->vary_headers) == 0) {
3559  return VARY_MATCH;
3560  } else {
3561  /* Oops.. we have already been here and still haven't
3562  * found the requested variant. Bail out
3563  */
3564  debugs(33, DBG_IMPORTANT, "varyEvaluateMatch: Oops. Not a Vary match on second attempt, '" <<
3565  entry->mem_obj->urlXXX() << "' '" << vary << "'");
3566  return VARY_CANCEL;
3567  }
3568  }
3569 }
3570 
3573 {
3574  const auto checklist = new ACLFilledChecklist(acl, nullptr, nullptr);
3575  clientAclChecklistFill(*checklist, http);
3576  return checklist;
3577 }
3578 
3579 void
3581 {
3582  checklist.setRequest(http->request);
3583  checklist.al = http->al;
3584  checklist.syncAle(http->request, http->log_uri);
3585 
3586  // TODO: If http->getConn is always http->request->clientConnectionManager,
3587  // then call setIdent() inside checklist.setRequest(). Otherwise, restore
3588  // USE_IDENT lost in commit 94439e4.
3589  ConnStateData * conn = http->getConn();
3590  const char *ident = (cbdataReferenceValid(conn) &&
3591  conn && conn->clientConnection) ?
3592  conn->clientConnection->rfc931 : dash_str;
3593  checklist.setIdent(ident);
3594 }
3595 
3596 bool
3598 {
3600 }
3601 
3604 {
3605  bodyPipe = new BodyPipe(this);
3606  if (size >= 0)
3607  bodyPipe->setBodySize(size);
3608  else
3610  return bodyPipe;
3611 }
3612 
3613 int64_t
3615 {
3616  if (!bodyPipe)
3617  return 0; // request without a body or read/produced all body bytes
3618 
3619  if (!bodyPipe->bodySizeKnown())
3620  return -1; // probably need to read more, but we cannot be sure
3621 
3622  const int64_t needToProduce = bodyPipe->unproducedSize();
3623  const int64_t haveAvailable = static_cast<int64_t>(inBuf.length());
3624 
3625  if (needToProduce <= haveAvailable)
3626  return 0; // we have read what we need (but are waiting for pipe space)
3627 
3628  return needToProduce - haveAvailable;
3629 }
3630 
3631 void
3633 {
3634  debugs(33, 4, HERE << "receiving error (" << clientConnection << "): " << error <<
3635  "; old sending error: " <<
3636  (stoppedSending() ? stoppedSending_ : "none"));
3637 
3638  if (const char *oldError = stoppedReceiving()) {
3639  debugs(33, 3, HERE << "already stopped receiving: " << oldError);
3640  return; // nothing has changed as far as this connection is concerned
3641  }
3642 
3644 
3645  if (const char *sendError = stoppedSending()) {
3646  debugs(33, 3, HERE << "closing because also stopped sending: " << sendError);
3648  }
3649 }
3650 
3651 void
3653 {
3654  if (bodyPipe != NULL) {
3655  debugs(33, 4, HERE << "no consumer for virgin body " << bodyPipe->status());
3657  }
3658 }
3659 
3661 void
3663 {
3664  Must(bodyPipe != NULL);
3665  debugs(33, 5, HERE << "start dechunking" << bodyPipe->status());
3666  assert(!bodyParser);
3668 }
3669 
3671 void
3673 {
3674  debugs(33, 5, HERE << "finish dechunking: " << withSuccess);
3675 
3676  if (bodyPipe != NULL) {
3677  debugs(33, 7, HERE << "dechunked tail: " << bodyPipe->status());
3678  BodyPipe::Pointer myPipe = bodyPipe;
3679  stopProducingFor(bodyPipe, withSuccess); // sets bodyPipe->bodySize()
3680  Must(!bodyPipe); // we rely on it being nil after we are done with body
3681  if (withSuccess) {
3682  Must(myPipe->bodySizeKnown());
3683  Http::StreamPointer context = pipeline.front();
3684  if (context != NULL && context->http && context->http->request)
3685  context->http->request->setContentLength(myPipe->bodySize());
3686  }
3687  }
3688 
3689  delete bodyParser;
3690  bodyParser = NULL;
3691 }
3692 
3693 // XXX: this is an HTTP/1-only operation
3694 void
3696 {
3697  if (!isOpen()) {
3698  debugs(33, 3, HERE << "ignoring 1xx due to earlier closure");
3699  return;
3700  }
3701 
3702  // HTTP/1 1xx status messages are only valid when there is a transaction to trigger them
3703  if (!pipeline.empty()) {
3704  HttpReply::Pointer rep(msg.reply);
3705  Must(rep);
3706  // remember the callback
3708 
3711 
3712  if (!writeControlMsgAndCall(rep.getRaw(), call)) {
3713  // but still inform the caller (so it may resume its operation)
3715  }
3716  return;
3717  }
3718 
3719  debugs(33, 3, HERE << " closing due to missing context for 1xx");
3721 }
3722 
3723 void
3725 {
3727 
3728  if (Http::StreamPointer deferredRequest = pipeline.front()) {
3729  debugs(33, 3, clientConnection << ": calling PushDeferredIfNeeded after control msg wrote");
3730  ClientSocketContextPushDeferredIfNeeded(deferredRequest, this);
3731  }
3732 }
3733 
3735 void
3737 {
3738  // FwdState might repin a failed connection sooner than this close
3739  // callback is called for the failed connection.
3740  assert(pinning.serverConnection == io.conn);
3741  pinning.closeHandler = NULL; // Comm unregisters handlers before calling
3742  const bool sawZeroReply = pinning.zeroReply; // reset when unpinning
3743  pinning.serverConnection->noteClosure();
3744  unpinConnection(false);
3745 
3746  if (sawZeroReply && clientConnection != NULL) {
3747  debugs(33, 3, "Closing client connection on pinned zero reply.");
3749  }
3750 
3751 }
3752 
3753 void
3755 {
3756  pinConnection(pinServer, *request);
3757 }
3758 
3759 void
3761 {
3762  Must(pic.connection);
3763  Must(pic.request);
3764  pinConnection(pic.connection, *pic.request);
3765 
3766  // monitor pinned server connection for remote-end closures.
3768 
3769  if (pipeline.empty())
3770  kick(); // in case clientParseRequests() was blocked by a busy pic.connection
3771 }
3772 
3774 void
3776 {
3777  if (Comm::IsConnOpen(pinning.serverConnection) &&
3778  pinning.serverConnection->fd == pinServer->fd) {
3779  debugs(33, 3, "already pinned" << pinServer);
3780  return;
3781  }
3782 
3783  unpinConnection(true); // closes pinned connection, if any, and resets fields
3784 
3785  pinning.serverConnection = pinServer;
3786 
3787  debugs(33, 3, HERE << pinning.serverConnection);
3788 
3789  Must(pinning.serverConnection != NULL);
3790 
3791  const char *pinnedHost = "[unknown]";
3792  pinning.host = xstrdup(request.url.host());
3793  pinning.port = request.url.port();
3794  pinnedHost = pinning.host;
3795  pinning.pinned = true;
3796  if (CachePeer *aPeer = pinServer->getPeer())
3797  pinning.peer = cbdataReference(aPeer);
3798  pinning.auth = request.flags.connectionAuth;
3799  char stmp[MAX_IPSTRLEN];
3800  char desc[FD_DESC_SZ];
3801  snprintf(desc, FD_DESC_SZ, "%s pinned connection for %s (%d)",
3802  (pinning.auth || !pinning.peer) ? pinnedHost : pinning.peer->name,
3804  clientConnection->fd);
3805  fd_note(pinning.serverConnection->fd, desc);
3806 
3808  pinning.closeHandler = JobCallback(33, 5,
3810  // remember the pinned connection so that cb does not unpin a fresher one
3811  typedef CommCloseCbParams Params;
3812  Params &params = GetCommParams<Params>(pinning.closeHandler);
3813  params.conn = pinning.serverConnection;
3814  comm_add_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3815 }
3816 
3819 void
3821 {
3822  if (pinning.readHandler != NULL)
3823  return; // already monitoring
3824 
3826  pinning.readHandler = JobCallback(33, 3,
3828  Comm::Read(pinning.serverConnection, pinning.readHandler);
3829 }
3830 
3831 void
3833 {
3834  if (pinning.readHandler != NULL) {
3835  Comm::ReadCancel(pinning.serverConnection->fd, pinning.readHandler);
3836  pinning.readHandler = NULL;
3837  }
3838 }
3839 
3840 #if USE_OPENSSL
3841 bool
3843 {
3844  // A ready-for-reading connection means that the TLS server either closed
3845  // the connection, sent us some unexpected HTTP data, or started TLS
3846  // renegotiations. We should close the connection except for the last case.
3847 
3848  Must(pinning.serverConnection != nullptr);
3849  auto ssl = fd_table[pinning.serverConnection->fd].ssl.get();
3850  if (!ssl)
3851  return false;
3852 
3853  char buf[1];
3854  const int readResult = SSL_read(ssl, buf, sizeof(buf));
3855 
3856  if (readResult > 0 || SSL_pending(ssl) > 0) {
3857  debugs(83, 2, pinning.serverConnection << " TLS application data read");
3858  return false;
3859  }
3860 
3861  switch(const int error = SSL_get_error(ssl, readResult)) {
3862  case SSL_ERROR_WANT_WRITE:
3863  debugs(83, DBG_IMPORTANT, pinning.serverConnection << " TLS SSL_ERROR_WANT_WRITE request for idle pinned connection");
3864  // fall through to restart monitoring, for now
3865  case SSL_ERROR_NONE:
3866  case SSL_ERROR_WANT_READ:
3868  return true;
3869 
3870  default:
3871  debugs(83, 2, pinning.serverConnection << " TLS error: " << error);
3872  return false;
3873  }
3874 
3875  // not reached
3876  return true;
3877 }
3878 #endif
3879 
3882 void
3884 {
3885  pinning.readHandler = NULL; // Comm unregisters handlers before calling
3886 
3887  if (io.flag == Comm::ERR_CLOSING)
3888  return; // close handler will clean up
3889 
3890  Must(pinning.serverConnection == io.conn);
3891 
3892 #if USE_OPENSSL
3894  return;
3895 #endif
3896 
3897  const bool clientIsIdle = pipeline.empty();
3898 
3899  debugs(33, 3, "idle pinned " << pinning.serverConnection << " read " <<
3900  io.size << (clientIsIdle ? " with idle client" : ""));
3901 
3902  pinning.serverConnection->close();
3903 
3904  // If we are still sending data to the client, do not close now. When we are done sending,
3905  // ConnStateData::kick() checks pinning.serverConnection and will close.
3906  // However, if we are idle, then we must close to inform the idle client and minimize races.
3907  if (clientIsIdle && clientConnection != NULL)
3909 }
3910 
3913 {
3914  debugs(33, 7, HERE << pinning.serverConnection);
3915 
3916  bool valid = true;
3917  if (!Comm::IsConnOpen(pinning.serverConnection))
3918  valid = false;
3919  else if (pinning.auth && pinning.host && request && strcasecmp(pinning.host, request->url.host()) != 0)
3920  valid = false;
3921  else if (request && pinning.port != request->url.port())
3922  valid = false;
3923  else if (pinning.peer && !cbdataReferenceValid(pinning.peer))
3924  valid = false;
3925 
3926  if (!valid) {
3927  /* The pinning info is not safe, remove any pinning info */
3928  unpinConnection(true);
3929  }
3930 
3931  return pinning.serverConnection;
3932 }
3933 
3936 {
3937  debugs(33, 7, pinning.serverConnection);
3938  if (validatePinnedConnection(request) != nullptr)
3940 
3941  return pinning.serverConnection; // closed if validation failed
3942 }
3943 
3944 void
3946 {
3947  debugs(33, 3, HERE << pinning.serverConnection);
3948 
3949  if (pinning.peer)
3951 
3952  if (Comm::IsConnOpen(pinning.serverConnection)) {
3953  if (pinning.closeHandler != NULL) {
3954  comm_remove_close_handler(pinning.serverConnection->fd, pinning.closeHandler);
3955  pinning.closeHandler = NULL;
3956  }
3957 
3959 
3960  // close the server side socket if requested
3961  if (andClose)
3962  pinning.serverConnection->close();
3963  pinning.serverConnection = NULL;
3964  }
3965 
3966  safe_free(pinning.host);
3967 
3968  pinning.zeroReply = false;
3969 
3970  /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host
3971  * connection has gone away */
3972 }
3973 
3974 void
3976 {
3977  // if we are parsing request body, its request is responsible for logging
3978  if (bodyPipe)
3979  return;
3980 
3981  // a request currently using this connection is responsible for logging
3982  if (!pipeline.empty() && pipeline.back()->mayUseConnection())
3983  return;
3984 
3985  /* Either we are waiting for the very first transaction, or
3986  * we are done with the Nth transaction and are waiting for N+1st.
3987  * XXX: We assume that if anything was added to inBuf, then it could
3988  * only be consumed by actions already covered by the above checks.
3989  */
3990 
3991  // do not log connections that closed after a transaction (it is normal)
3992  // TODO: access_log needs ACLs to match received-no-bytes connections
3993  if (pipeline.nrequests && inBuf.isEmpty())
3994  return;
3995 
3996  /* Create a temporary ClientHttpRequest object. Its destructor will log. */
3997  ClientHttpRequest http(this);
3998  http.req_sz = inBuf.length();
3999  // XXX: Or we died while waiting for the pinned connection to become idle.
4000  http.setErrorUri("error:transaction-end-before-headers");
4001 }
4002 
4003 bool
4005 {
4006  // PROXY protocol bytes are meant for us and, hence, cannot be tunneled
4008  return false;
4009 
4010  // If our decision here is negative, configuration changes are irrelevant.
4011  // Otherwise, clientTunnelOnError() rechecks configuration before tunneling.
4013  return false;
4014 
4015  // TODO: Figure out whether/how we can support FTP tunneling.
4016  if (port->transport.protocol == AnyP::PROTO_FTP)
4017  return false;
4018 
4019 #if USE_OPENSSL
4020  if (parsingTlsHandshake)
4021  return true;
4022 
4023  // the 1st HTTP request on a bumped connection
4025  return true;
4026 #endif
4027 
4028  // the 1st HTTP request on a connection to a plain intercepting port
4029  if (!pipeline.nrequests && !port->secure.encryptTransport && transparent())
4030  return true;
4031 
4032  return false;
4033 }
4034 
4037 {
4038  if (!theNotes)
4039  theNotes = new NotePairs;
4040  return theNotes;
4041 }
4042 
4043 std::ostream &
4044 operator <<(std::ostream &os, const ConnStateData::PinnedIdleContext &pic)
4045 {
4046  return os << pic.connection << ", request=" << pic.request;
4047 }
4048 
sslproxy_cert_adapt * cert_adapt
Definition: SquidConfig.h:518
static void clientHttpConnectionsOpen(void)
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:404
void nonBlockingCheck(ACLCB *callback, void *callback_data)
Definition: Checklist.cc:238
int internalHostnameIs(const char *arg)
Definition: internal.cc:156
const Security::CertErrors * sslErrors() const
SSL [certificate validation] errors.
Definition: ServerBump.cc:69
bool setCommonName
Replace the CN field of the mimicing subject with the given.
Definition: gadgets.h:221
HierarchyLogEntry hier
Definition: HttpRequest.h:157
bool splice()
Splice a bumped client connection on peek-and-splice mode.
virtual ~ConnStateData()
Definition: client_side.cc:613
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
char * skipLeadingSpace(char *aString)
void clientProcessRequestFinished(ConnStateData *conn, const HttpRequest::Pointer &request)
#define debug_log
change-avoidance macro; new code should call DebugStream() instead
Definition: Debug.h:110
#define HTTP_REQBUF_SZ
Definition: defines.h:216
static bool clientPingHasFinished(ping_data const *aPing)
Definition: client_side.cc:256
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
const char * status() const
Definition: BodyPipe.cc:445
IdentConfig TheConfig
Definition: Ident.cc:73
#define SQUIDSTRINGPH
Definition: SquidString.h:21
const char * FdNote(int fdNodeId)
converts FdNoteId into a string
Definition: FdNotes.cc:16
err_type errType
Definition: HttpRequest.h:161
void count(double val)
Definition: StatHist.cc:57
static CTCB clientLifetimeTimeout
Definition: client_side.cc:174
void setAuth(const Auth::UserRequest::Pointer &aur, const char *cause)
Definition: client_side.cc:502
#define fd_table
Definition: fde.h:174
Ssl::ServerBump * serverBump()
Definition: client_side.h:253
bool setValidAfter
Do not mimic "Not Valid After" field.
Definition: gadgets.h:219
int timeout
Definition: PingData.h:24
NotePairs::Pointer theNotes
Definition: client_side.h:440
#define COMM_SELECT_READ
Definition: defines.h:36
void StartListening(int sock_type, int proto, const Comm::ConnectionPointer &listenConn, FdNoteId fdNote, AsyncCall::Pointer &callback)
void(* Handler)(AnyP::PortCfgPointer &portCfg, const Ipc::FdNoteId note, const Subscription::Pointer &sub)
Definition: client_side.cc:150
unsigned short port() const
Definition: Address.cc:778
StatCounters statCounter
Definition: StatCounters.cc:12
#define assert(EX)
Definition: assert.h:17
SBuf & authority(bool requirePort=false) const
Definition: Uri.cc:503
bool isOpen(const int fd)
Definition: comm.cc:86
void const char HLPCB * callback
Definition: stub_helper.cc:16
#define SQUIDSTRINGPRINT(s)
Definition: SquidString.h:22
void clientStreamInit(dlink_list *list, CSR *func, CSD *rdetach, CSS *readstatus, ClientStreamData readdata, CSCB *callback, CSD *cdetach, ClientStreamData callbackdata, StoreIOBuffer tailBuffer)
time_t request
Definition: SquidConfig.h:112
bool setValidBefore
Do not mimic "Not Valid Before" field.
Definition: gadgets.h:220
static int tlsAttemptHandshake(ConnStateData *conn, PF *callback)
SBuf & assign(const SBuf &S)
Definition: SBuf.cc:83
bool timedout
_TIMEDOUT: terminated due to a lifetime or I/O timeout
Definition: LogTags.h:73
bool verifySslCertificate(Security::ContextPointer &, CertificateProperties const &)
Definition: support.cc:872
const char * stoppedReceiving() const
true if we stopped receiving the request
Definition: client_side.h:150
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
char reqbuf[HTTP_REQBUF_SZ]
Definition: Stream.h:134
Ssl::BumpMode step1
The SSL bump mode at step1.
Definition: ServerBump.h:50
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:216
SBuf image() const
Definition: UriScheme.h:50
https_port or bumped http_port tunnel; HTTPS server
Definition: Message.h:33
Security::NegotiationHistory * tlsNegotiations()
Definition: Connection.cc:130
ConnStateData * NewServer(MasterXactionPointer &xact)
create a new HTTP connection handler; never returns NULL
Definition: Http1Server.cc:360
bool internalCheck(const SBuf &urlPath)
Definition: internal.cc:69
std::vector< HttpHdrRangeSpec * >::iterator iterator
int errDetail
errType-specific detail about the transaction error
Definition: HttpRequest.h:162
void parseTlsHandshake()
void fd_note(int fd, const char *s)
Definition: fd.cc:246
Definition: Server.h:28
err_type handleChunkedRequestBody()
parses available chunked encoded body bytes, checks size, returns errors
void setReplyToError(err_type, Http::StatusCode, const HttpRequestMethod &, char const *, Ip::Address &, HttpRequest *, const char *, Auth::UserRequest::Pointer)
builds error using clientBuildError() and calls setReplyToError() below
bool persistent() const
Definition: Message.cc:261
bool conflictingContentLength() const
Definition: HttpHeader.h:108
void CTCB(const CommTimeoutCbParams &params)
Definition: CommCalls.h:39
SBuf vary_headers
The variant second-stage cache key. Generated from Vary header pattern for this request.
Definition: HttpRequest.h:171
ConnStateData(const MasterXactionPointer &xact)
void accessLogLog(AccessLogEntry::Pointer &, ACLChecklist *checklist)
Definition: access_log.cc:144
static void clientUpdateStatHistCounters(const LogTags &logType, int svc_time)
Definition: client_side.cc:214
static char * prepareAcceleratedURL(ConnStateData *conn, const Http1::RequestParserPointer &hp)
HttpRequest::Pointer request
to-server request that initiated serverConnection
Definition: client_side.h:174
bool needProxyProtocolHeader_
whether PROXY protocol header is still expected
Definition: client_side.h:403
int NHttpSockets
Definition: PortCfg.cc:24
bool multipartRangeRequest() const
Definition: client_side.cc:687
virtual void afterClientRead()
processing to be done after a Comm::Read()
void resetSslCommonName(const char *name)
Definition: client_side.h:261
void path(const char *p)
Definition: Uri.h:93
Definition: SBuf.h:86
bool fakeAConnectRequest(const char *reason, const SBuf &payload)
void configureUnconfiguredSslContext(Security::ContextPointer &, Ssl::CertSignAlgorithm signAlgorithm, AnyP::PortCfg &)
Definition: support.cc:822
void startPinnedConnectionMonitoring()
virtual void swanSong()
Definition: Server.cc:47
SQUIDCEXTERN CSD clientReplyDetach
Definition: client_side.h:466
ClientHttpRequest * buildFakeRequest(Http::MethodType const method, SBuf &useHost, unsigned short usePort, const SBuf &payload)
build a fake http request
Subscription::Pointer sub
The handler to be subscribed for this connetion listener.
Definition: client_side.cc:168
virtual bool canDial(AsyncCall &) const
Definition: client_side.cc:159
#define xcalloc
Definition: membanger.c:57
static void httpsAccept(const CommAcceptCbParams &params)
ByteCounter hit_kbytes_out
Definition: StatCounters.h:47
Ip::Address src_addr
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:828
HttpRequestMethod method
Definition: HttpRequest.h:114
virtual bool connFinishedWithConn(int size)
??
Ssl::ServerBump * sslServerBump
HTTPS server cert. fetching state for bump-ssl-server-first.
Definition: client_side.h:429
void setErrorUri(const char *errorUri)
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
void error(char *format,...)
int i
Definition: membanger.c:49
SBuf sslBumpCertKey
Key to use to store/retrieve generated certificate.
Definition: client_side.h:426
#define xstrdup
SBuf & append(const SBuf &S)
Definition: SBuf.cc:195
SBuf preservedClientData
Definition: client_side.h:317
virtual void dial(AsyncCall &)
Definition: client_side.cc:160
#define SQUID_X509_V_ERR_DOMAIN_MISMATCH
Definition: support.h:44
ConnStateData * getConn() const
T * get() const
Returns raw and possibly nullptr pointer.
ListeningStartedDialer(Handler aHandler, AnyP::PortCfgPointer &aPortCfg, const Ipc::FdNoteId note, const Subscription::Pointer &aSub)
Definition: client_side.cc:151
int HttpSockets[MAXTCPLISTENPORTS]
Definition: PortCfg.cc:25
int clientdbEstablished(const Ip::Address &addr, int delta)
Definition: client_db.cc:183
struct timeval start_time
The time the master transaction started.
#define safe_free(x)
Definition: xalloc.h:73
void stopPinnedConnectionMonitoring()
The caller assumes responsibility for connection closure detection.
const char * bumpMode(int bm)
Definition: support.h:149
bool CreateServerSession(const Security::ContextPointer &, const Comm::ConnectionPointer &, Security::PeerOptions &, const char *squidCtx)
Definition: Session.cc:194
Definition: Flag.h:16
Adaptation::History::Pointer adaptLogHistory() const
Returns possibly nil history, creating it if adapt. logging is enabled.
Definition: HttpRequest.cc:426
void banAction(const Acl::Answer &action)
add action to the list of banned actions
Definition: Checklist.cc:402
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
void StopListening()
reject new connections to any configured ftp_port
Definition: FtpServer.cc:285
virtual void swanSong()
Definition: client_side.cc:583
http_port or HTTP server
Definition: Message.h:39
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:160
bool bodySizeKnown() const
Definition: BodyPipe.h:109
void clientdbUpdate(const Ip::Address &addr, const LogTags &ltype, AnyP::ProtocolType p, size_t size)
Definition: client_db.cc:139
void registerWithConn()
register this stream with the Server
Definition: Stream.cc:52
Ssl::BumpMode step2
The SSL bump mode at step2.
Definition: ServerBump.h:51
std::string commonName
A CN to use for the generated certificate.
Definition: gadgets.h:222
void clear()
Definition: SBuf.cc:178
void clientStreamAbort(clientStreamNode *thisObject, ClientHttpRequest *http)
CachePeer * getPeer() const
Definition: Connection.cc:98
void doPeekAndSpliceStep()
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
bool isEmpty() const
Definition: SBuf.h:420
HttpReply const * getReply() const
Definition: store.cc:1676
char * prepareTlsSwitchingURL(const Http1::RequestParserPointer &hp)
#define COMM_INTERCEPTION
Definition: Connection.h:49
Comm::ConnectionPointer tcpClient
TCP/IP level details about the client connection.
bool SessionIsResumed(const Security::SessionPointer &)
whether the session is a resumed one
Definition: Session.cc:213
const char * CertAdaptAlgorithmStr[]
Definition: gadgets.cc:202
AnyP::ProtocolVersion http_ver
Definition: Message.h:73
#define xisspace(x)
Definition: xis.h:17
int64_t mayNeedToReadMoreBody() const
bool isTcpHit() const
determine if the log tag code indicates a cache HIT
Definition: LogTags.cc:91
#define DBG_CRITICAL
Definition: Debug.h:45
struct timeval stop
Definition: PingData.h:20
int conn
the current server connection FD
Definition: Transport.cc:26
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
AnyP::PortCfgPointer HttpPortList
list of Squid http(s)_port configured
Definition: PortCfg.cc:21
void SBufToCstring(char *d, const SBuf &s)
Definition: SBuf.h:741
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
void applyClientMask(const Address &mask)
Definition: Address.cc:105
uint64_t unproducedSize() const
Definition: BodyPipe.cc:179
void notePinnedConnectionBecameIdle(PinnedIdleContext pic)
Called when a pinned connection becomes available for forwarding the next request.
void connStateClosed(const CommCloseCbParams &io)
Definition: client_side.cc:495
static const CharacterSet ALPHA
Definition: CharacterSet.h:73
static void clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub)
process clientHttpConnectionsOpen result
bool isOpen() const
Definition: client_side.cc:606
Ssl::CertSignAlgorithm signAlgorithm
The signing algorithm to use.
Definition: client_side.h:430
Ip::Address log_addr
Definition: client_side.h:128
struct timeval current_time
Definition: stub_time.cc:15
void prepareLogWithRequestDetails(HttpRequest *, AccessLogEntry::Pointer &)
Definition: client_side.cc:324
void clientAclChecklistFill(ACLFilledChecklist &checklist, ClientHttpRequest *http)
void setReplyToStoreEntry(StoreEntry *e, const char *reason)
replaces current response store entry with the given one
void requestTimeout(const CommTimeoutCbParams &params)
void append(char const *buf, int len)
Definition: String.cc:161
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:609
Security::CertPointer serverCert
Definition: ServerBump.h:48
const char * findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *end)
std::string const & getBody() const
Current body. If parsing is not finished the method returns incompleted body.
struct SquidConfig::@123 ssl_client
Acl::Answer const & fastCheck()
Definition: Checklist.cc:336
void clientSetKeepaliveFlag(ClientHttpRequest *http)
decide whether to expect multiple requests on the corresponding connection
Definition: client_side.cc:640
Security::ContextPointer GenerateSslContextUsingPkeyAndCertFromMemory(const char *data, Security::ServerOptions &, bool trusted)
Definition: support.cc:769
void finishDechunkingRequest(bool withSuccess)
put parsed content into input buffer and clean up
bool readMore
needs comm_read (for this request or new requests)
Definition: client_side.h:131
StatHist hitSvcTime
Definition: StatCounters.h:51
ftp_port or FTP server
Definition: Message.h:40
void startPeekAndSplice()
Initializes and starts a peek-and-splice negotiation with the SSL client.
void stopSending(const char *error)
note response sending error and close as soon as we read the request
Definition: client_side.cc:972
bool connectionAuth
Definition: RequestFlags.h:76
struct StatCounters::@135 cd
void * BIO_get_data(BIO *table)
Definition: openssl.h:60
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
char * toUrl(char *buf, unsigned int len) const
Definition: Address.cc:884
FdNoteId
We cannot send char* FD notes to other processes. Pass int IDs and convert.
Definition: FdNotes.h:20
Helper::ResultCode result
The helper response &#39;result&#39; field.
Definition: Reply.h:59
void manager(const CbcPointer< ConnStateData > &aMgr, const AccessLogEntryPointer &al)
associates the request with a from-client connection manager
Definition: HttpRequest.cc:737
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
static const CharacterSet HEXDIG
Definition: CharacterSet.h:85
void fatalf(const char *fmt,...)
Definition: fatal.cc:68
void Read(const Comm::ConnectionPointer &conn, AsyncCall::Pointer &callback)
Definition: Read.cc:40
sslproxy_cert_sign * cert_sign
Definition: SquidConfig.h:517
Security::ContextPointer getTlsContextFromCache(const SBuf &cacheKey, const Ssl::CertificateProperties &certProperties)
static const std::string code_new_certificate
String code for "new_certificate" messages.
Definition: crtd_message.h:76
Security::ContextPointer GenerateSslContext(CertificateProperties const &, Security::ServerOptions &, bool trusted)
Definition: support.cc:783
size_t appendDomainLen
Definition: SquidConfig.h:216
Security::PrivateKeyPointer signWithPkey
The key of the signing certificate.
Definition: gadgets.h:218
static Pointer Start(AsyncJob *job)
starts a freshly created job (i.e., makes the job asynchronous)
Definition: AsyncJob.cc:23
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:168
void IOACB(const CommAcceptCbParams &params)
Definition: CommCalls.h:33
std::vector< ClientDelayPool::Pointer > pools
const char * stoppedSending() const
true if we stopped sending the response
Definition: client_side.h:152
ACLFilledChecklist * clientAclChecklistCreate(const acl_access *acl, ClientHttpRequest *http)
time_t lifetime
Definition: SquidConfig.h:108
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:707
int tvSubMsec(struct timeval, struct timeval)
Definition: stub_time.cc:20
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
const char * xstrerr(int error)
Definition: xstrerror.cc:83
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:24
Http::StreamPointer front() const
get the first request context in the pipeline
Definition: Pipeline.cc:28
static void clientUpdateHierCounters(HierarchyLogEntry *)
Definition: client_side.cc:265
common API for all StartListening() callbacks
void storeTlsContextToCache(const SBuf &cacheKey, Security::ContextPointer &ctx)
unsigned int major
major version number
const char * stoppedReceiving_
the reason why we no longer read the request or nil
Definition: client_side.h:436
int pipeline_max_prefetch
Definition: SquidConfig.h:350
void const char HLPCB void * data
Definition: stub_helper.cc:16
bool configureSSLUsingPkeyAndCertFromMemory(SSL *ssl, const char *data, AnyP::PortCfg &port)
Definition: support.cc:852
CbDataList< Security::CertError > CertErrors
Holds a list of X.509 certificate errors.
Definition: forward.h:57
LogTags logType
the processing tags associated with this request transaction.
Definition: parse.c:104
static int clientIsContentLengthValid(HttpRequest *r)
checks body length of non-chunked requests
Definition: client_side.cc:653
int urlCheckRequest(const HttpRequest *r)
Definition: Uri.cc:790
struct Ssl::ServerBump::@128 act
bumping actions at various bumping steps
StatusCode
Definition: StatusCode.h:20
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:802
int64_t content_length
Definition: Message.h:84
virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io)
Our close handler called by Comm when the pinned connection is closed.
int getMyPort(void)
Definition: tools.cc:1003
void port(unsigned short p)
Definition: Uri.h:88
virtual void sendControlMsg(HttpControlMsg)
called to send the 1xx message and notify the Source
bool parseProxyProtocolHeader()
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
thrown by modern "incremental" parsers when they need more data
Definition: forward.h:18
#define cbdataReference(var)
Definition: cbdata.h:341
Http1::TeChunkedParser * bodyParser
parses HTTP/1.1 chunked request body
Definition: client_side.h:100
virtual int pipelinePrefetchMax() const
returning N allows a pipeline of 1+N requests (see pipeline_prefetch)
void setIdent(const char *userIdentity)
configure rfc931 user identity for the first time
void sslCrtdHandleReply(const Helper::Reply &reply)
Proccess response from ssl_crtd.
#define DBG_IMPORTANT
Definition: Debug.h:46
void readSomeData()
maybe grow the inBuf and schedule Comm::Read()
Definition: Server.cc:85
void clientProcessRequest(ConnStateData *conn, const Http1::RequestParserPointer &hp, Http::Stream *context)
Comm::ConnectionPointer conn
opened listening socket
void SetSelect(int, unsigned int, PF *, void *, time_t)
Mark an FD to be watched for its IO status.
Definition: ModDevPoll.cc:225
ConnStateData * conn() const
The client connection manager.
Security::ContextPointer GetFrom(Security::SessionPointer &s)
Helper function to retrieve a (non-locked) ContextPointer from a SessionPointer.
Definition: Session.h:90
ConnStateData * NewServer(MasterXactionPointer &xact)
create a new HTTPS connection handler; never returns NULL
Definition: Http1Server.cc:366
GlobalContextStorage TheGlobalContextStorage
Global cache for store all SSL server certificates.
virtual bool writeControlMsgAndCall(HttpReply *rep, AsyncCall::Pointer &call)=0
handle a control message received by context from a peer and call back
Security::CertPointer mimicCert
Certificate to mimic.
Definition: gadgets.h:216
void httpHeaderAddContRange(HttpHeader *, HttpHdrRangeSpec, int64_t)
bool allowed() const
Definition: Acl.h:143
static const CharacterSet DIGIT
Definition: CharacterSet.h:81
struct StatCounters::@136 netdb
bool connectedOk() const
whether there was a successful connection to (and peeking at) the origin server
Definition: ServerBump.h:41
#define FD_DESC_SZ
Definition: defines.h:46
AnyP::ProtocolVersion transferProtocol
Definition: Server.h:104
SBuf tlsConnectHostOrIp
The TLS server host name appears in CONNECT request or the server ip address for the intercepted requ...
Definition: client_side.h:420
NotePairs::Pointer notes()
bool add(const Key &key, EntryValue *t)
Add an entry to the map.
Definition: LruMap.h:140
void unpinConnection(const bool andClose)
Undo pinConnection() and, optionally, close the pinned connection.
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
bool parseHello(const SBuf &data)
Definition: Handshake.cc:517
void reset()
Definition: MemBuf.cc:132
void setScheme(const AnyP::ProtocolType &p, const char *str)
convert the URL scheme to that given
Definition: Uri.h:70
void expectNoForwarding()
cleans up virgin request [body] forwarding state
#define USER_IDENT_SZ
Definition: defines.h:60
dlink_node node
Definition: clientStream.h:88
enum Http::_method_t MethodType
CertSignAlgorithm
Definition: gadgets.h:150
bool chunked() const
whether message uses chunked Transfer-Encoding
Definition: HttpHeader.h:192
mb_size_t size
Definition: MemBuf.h:135
AnyP::PortCfgPointer portCfg
from HttpPortList
Definition: client_side.cc:166
virtual void endingShutdown()
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength)
Definition: client_side.cc:677
ClientHttpRequest * http
Definition: Stream.h:132
int varyEvaluateMatch(StoreEntry *entry, HttpRequest *request)
bool receivedFirstByte_
true if at least one byte received on this connection
Definition: Server.h:112
SQUIDCEXTERN CSR clientGetMoreData
Definition: client_side.h:464
void httpsSslBumpStep2AccessCheckDone(Acl::Answer answer, void *data)
#define COMM_SELECT_WRITE
Definition: defines.h:37
void quitAfterError(HttpRequest *request)
Definition: LruMap.h:17
bool initiateTunneledRequest(HttpRequest::Pointer const &cause, Http::MethodType const method, const char *reason, const SBuf &payload)
generates and sends to tunnel.cc a fake request with a given payload
BodyPipe::Pointer expectRequestBody(int64_t size)
static ClientDelayPools * Instance()
void addLocalStorage(Ip::Address const &address, size_t size_of_store)
Create new SSL context storage for the local listening address/port.
std::ostream & startPrint(std::ostream &os) const
starts printing arguments, return os
void getSslContextDone(Security::ContextPointer &)
finish configuring the newly created SSL context"
bundles HTTP 1xx reply and the "successfully forwarded" callback
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
char * xstrncpy(char *dst, const char *src, size_t n)
Definition: xstring.cc:37
virtual void afterClientWrite(size_t)
processing to sync state after a Comm::Write()
Definition: client_side.cc:996
void stopReceiving(const char *error)
note request receiving error and close as soon as we write the response
std::shared_ptr< SSL_CTX > ContextPointer
Definition: Context.h:29
void fatal(const char *message)
Definition: fatal.cc:28
void abortChunkedRequestBody(const err_type error)
quit on errors related to chunked request body handling
void clientStartListeningOn(AnyP::PortCfgPointer &port, const RefCount< CommCbFunPtrCallT< CommAcceptCbPtrFun > > &subCall, const Ipc::FdNoteId fdNote)
accept requests to a given port and inform subCall about them
bool handleRequestBodyData()
Security::ContextPointer createSSLContext(Security::CertPointer &x509, Security::PrivateKeyPointer &pkey, Security::ServerOptions &)
Create SSL context and apply ssl certificate and private key to it.
Definition: support.cc:752
void retrieveParsedInfo(Security::TlsDetails::Pointer const &details)
Extract information from parser stored in TlsDetails object.
static bool AddOpenedHttpSocket(const Comm::ConnectionPointer &conn)
find any unused HttpSockets[] slot and store fd there or return false
struct SquidConfig::@113 accessList
HttpRequest * request
int incoming_sockets_accepted
ClientInfo * clientdbGetInfo(const Ip::Address &addr)
Definition: client_db.cc:120
std::string compose() const
static bool OpenedHttpSocket(const Comm::ConnectionPointer &c, const Ipc::FdNoteId portType)
check FD after clientHttp[s]ConnectionOpened, adjust HttpSockets as needed
void PF(int, void *)
Definition: forward.h:18
const char * internalHostname(void)
Definition: internal.cc:140
bool concurrentRequestQueueFilled() const
StatHist nearMissSvcTime
Definition: StatCounters.h:49
Ip::Address client_netmask
Definition: SquidConfig.h:242
const char * c_str()
Definition: SBuf.cc:526
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Tokenizer.h:44
optimized set of C chars, with quick membership test and merge support
Definition: CharacterSet.h:17
void setReadBufData(SBuf &data)
Definition: bio.h:91
ClientStreamData data
Definition: clientStream.h:94
AccessLogEntry::Pointer al
access.log entry
ClientDelayConfig ClientDelay
Definition: SquidConfig.h:445
struct Http::Stream::@80 flags
#define FQDN_LOOKUP_IF_MISS
Definition: defines.h:48
Definition: parse.c:160
void host(const char *src)
Definition: Uri.cc:47
Http::MethodType id() const
Definition: RequestMethod.h:73
bool commIsHalfClosed(int fd)
Definition: comm.h:101
const MemBuf & other() const
Definition: Reply.h:42
#define APP_FULLNAME
Definition: version.h:22
void wroteControlMsg(const CommIoCbParams &)
callback to handle Comm::Write completion
#define MAXTCPLISTENPORTS
Definition: PortCfg.h:78
ClientRequestContext * calloutContext
static int port
Definition: ldap_backend.cc:69
MemObject * mem_obj
Definition: Store.h:199
virtual void doneWithControlMsg()
time_t request_start_timeout
Definition: SquidConfig.h:119
void setLogUriToRequestUri()
sets log_uri when we know the current request
SQUIDCEXTERN int tvSubUsec(struct timeval, struct timeval)
Definition: util.c:39
clientStreamNode * getClientReplyContext() const
Definition: Stream.cc:547
void setCode(std::string const &aCode)
Set new request/reply code to compose.
bool switchedToHttps() const
Definition: client_side.h:252
Security::HandshakeParser tlsParser
Definition: client_side.h:276
char const * termedBuf() const
Definition: SquidString.h:91
char * internalLocalUri(const char *dir, const SBuf &name)
Definition: internal.cc:130
int log_mime_hdrs
Definition: SquidConfig.h:286
char * content()
start of the added data
Definition: MemBuf.h:41
struct ClientHttpRequest::Out out
static void clientNegotiateSSL(int fd, void *data)
unsigned short tlsConnectPort
The TLS server port number as passed in the CONNECT request.
Definition: client_side.h:421
bool proxyProtocolError(const char *reason)
static void clientUpdateStatCounters(const LogTags &logType)
Definition: client_side.cc:200
Parsed Parse(const SBuf &)
Definition: Parser.cc:241
static void httpsEstablish(ConnStateData *connState, const Security::ContextPointer &ctx)
void const char * buf
Definition: stub_helper.cc:16
int kind
which custom access list verb matched
Definition: Acl.h:155
virtual void receivedFirstByte()
Update flags and timeout after the first byte received.
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:153
class AccessLogEntry::CacheDetails cache
struct SquidConfig::@112 onoff
void commSetTcpKeepalive(int fd, int idle, int interval, int timeout)
Definition: comm.cc:1176
Http::StreamPointer back() const
get the last request context in the pipeline
Definition: Pipeline.cc:40
acl_access * cert_error
Definition: SquidConfig.h:516
bool skip(const SBuf &tokenToSkip)
Definition: Tokenizer.cc:179
const char * fqdncache_gethostbyaddr(const Ip::Address &addr, int flags)
Definition: fqdncache.cc:480
virtual void syncAle(HttpRequest *adaptedRequest, const char *logUri) const
assigns uninitialized adapted_request and url ALE components
CertSignAlgorithm signAlgorithm
The signing algorithm to use.
Definition: gadgets.h:223
struct SquidConfig::@107 Addrs
struct StatCounters::@131 icp
Ip::Address local
Definition: Connection.h:135
void clientSocketDetach(clientStreamNode *node, ClientHttpRequest *http)
Definition: client_side.cc:830
HttpHeader header
Definition: Message.h:75
bool handleIdleClientPinnedTlsRead()
bool accelerated
Definition: RequestFlags.h:58
bool mayUseConnection() const
Definition: Stream.h:140
virtual void doneWithControlMsg()
void StartListening()
accept connections on all configured ftp_ports
Definition: FtpServer.cc:266
void clean()
Definition: MemBuf.cc:113
struct timeval start
Definition: PingData.h:18
SBuf vary_headers
Definition: MemObject.h:174
bool serveDelayedError(Http::Stream *)
void postHttpsAccept()
the second part of old httpsAccept, waiting for future HttpsServer home
void tvSub(struct timeval &res, struct timeval const &t1, struct timeval const &t2)
Definition: time.cc:41
Security::CertPointer signWithX509
Certificate to sign the generated request.
Definition: gadgets.h:217
StatHist allSvcTime
Definition: StatCounters.h:52
void composeRequest(Ssl::CertificateProperties const &)
bool clientParseRequests()
Traffic parsing.
void comm_reset_close(const Comm::ConnectionPointer &conn)
Definition: comm.cc:792
Http::Stream * abortRequestParsing(const char *const errUri)
stop parsing the request and create context for relaying error info
char rfc931[USER_IDENT_SZ]
Definition: Connection.h:165
Ipc::FdNoteId portTypeNote
Type of IPC socket being opened.
Definition: client_side.cc:167
bool configureSSL(SSL *ssl, CertificateProperties const &properties, AnyP::PortCfg &port)
Definition: support.cc:829
int half_closed_clients
Definition: SquidConfig.h:291
char * buf
Definition: MemBuf.h:134
bool checkX509ServerValidity(X509 *cert, const char *server)
Definition: support.cc:251
RequestFlags flags
Definition: HttpRequest.h:141
unsigned short initial
void clientOpenListenSockets(void)
#define MYNAME
Definition: Debug.h:166
char * url
Definition: tcp-banger2.c:114
TlsDetails::Pointer details
TLS handshake meta info or nil.
Definition: Handshake.h:70
bool empty() const
whether there are none or any requests currently pipelined
Definition: Pipeline.h:56
int has(Http::HdrType id) const
Definition: HttpHeader.cc:977
SBuf sslCommonName_
CN name for SSL certificate generation.
Definition: client_side.h:422
bool proxyProtocolValidateClient()
Ip::Address remote
Definition: Connection.h:138
StatHist missSvcTime
Definition: StatCounters.h:48
#define COMM_TRANSPARENT
Definition: Connection.h:48
LogTags_ot oldType
a set of client protocol, cache use, and other transaction outcome tags
Definition: LogTags.h:84
Definition: Xaction.cc:47
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1038
void InRamCertificateDbKey(const Ssl::CertificateProperties &certProperties, SBuf &key)
Definition: support.cc:1199
void kick()
try to make progress on a transaction or read more I/O
Definition: client_side.cc:891
void readNextRequest()
Definition: client_side.cc:851
void clean()
Definition: HttpHeader.cc:187
const char * stoppedSending_
the reason why we no longer write the response or nil
Definition: client_side.h:434
MasterXaction::Pointer xaction
Transaction which this call is part of.
Definition: CommCalls.h:105
String rangeBoundaryStr() const
Definition: client_side.cc:771
const Acl::Tree * changeAcl(const Acl::Tree *t)
Definition: Checklist.h:176
int dnsWait
sum of DNS lookup delays in milliseconds, for dt
Definition: HttpRequest.h:159
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
void getSslContextStart()
Start to create dynamic Security::ContextPointer for host or uses static port SSL context...
#define PROF_start(probename)
Definition: Profiler.h:62
bool proxyKeepalive
Definition: RequestFlags.h:38
bool shouldPreserveClientData() const
void httpsPeeked(PinnedIdleContext pic)
called by FwdState when it is done bumping the server
virtual bool handleReadData()
LocalContextStorage * getLocalStorage(Ip::Address const &address)
Return the local storage for the given listening address/port.
int cmp(const SBuf &S, const size_type n) const
shorthand version for compare()
Definition: SBuf.h:264
bool transparent() const
SQUIDCEXTERN CSS clientReplyStatus
Definition: client_side.h:465
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:149
AnyP::PortCfgPointer port
void IDCB(const char *ident, void *data)
Definition: Ident.h:17
void detailError(err_type aType, int aDetail)
sets error detail if no earlier detail was available
Definition: HttpRequest.cc:476
static char * prepareTransparentURL(ConnStateData *conn, const Http1::RequestParserPointer &hp)
void hold(bool h)
Prevents or allow writting on socket.
Definition: bio.h:86
void clientConnectionsClose()
struct ClientHttpRequest::Flags flags
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
struct StatCounters::@129 client_http
acl_access * on_unsupported_protocol
Definition: SquidConfig.h:399
const Comm::ConnectionPointer validatePinnedConnection(HttpRequest *request)
ProxyProtocol::HeaderPointer proxyProtocolHeader_
the parsed PROXY protocol header
Definition: client_side.h:406
String extacl_user
Definition: HttpRequest.h:179
static IOACB httpAccept
Definition: client_side.cc:173
C * getRaw() const
Definition: RefCount.h:74
void startDechunkingRequest()
initialize dechunking state
Comm::ConnectionPointer borrowPinnedConnection(HttpRequest *request)
Returns validated pinnned server connection (and stops its monitoring).
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
struct ConnStateData::@39 flags
int64_t maxRequestBodySize
Definition: SquidConfig.h:129
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod & method
Definition: stub_store.cc:112
static bool httpsCreate(const ConnStateData *connState, const Security::ContextPointer &ctx)
Create TLS