client_side_request.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 85 Client-side Request Routines */
10 
11 /*
12  * General logic of request processing:
13  *
14  * We run a series of tests to determine if access will be permitted, and to do
15  * any redirection. Then we call into the result clientStream to retrieve data.
16  * From that point on it's up to reply management.
17  */
18 
19 #include "squid.h"
20 #include "acl/FilledChecklist.h"
21 #include "acl/Gadgets.h"
22 #include "anyp/PortCfg.h"
23 #include "base/AsyncJobCalls.h"
24 #include "client_side.h"
25 #include "client_side_reply.h"
26 #include "client_side_request.h"
27 #include "ClientRequestContext.h"
28 #include "clientStream.h"
29 #include "comm/Connection.h"
30 #include "comm/Write.h"
31 #include "error/Detail.h"
32 #include "errorpage.h"
33 #include "fd.h"
34 #include "fde.h"
35 #include "format/Token.h"
36 #include "FwdState.h"
37 #include "gopher.h"
38 #include "helper.h"
39 #include "helper/Reply.h"
40 #include "http.h"
41 #include "http/Stream.h"
42 #include "HttpHdrCc.h"
43 #include "HttpReply.h"
44 #include "HttpRequest.h"
45 #include "ip/NfMarkConfig.h"
46 #include "ip/QosConfig.h"
47 #include "ipcache.h"
48 #include "log/access_log.h"
49 #include "MemObject.h"
50 #include "Parsing.h"
51 #include "proxyp/Header.h"
52 #include "redirect.h"
53 #include "rfc1738.h"
54 #include "sbuf/StringConvert.h"
55 #include "SquidConfig.h"
56 #include "Store.h"
57 #include "StrList.h"
58 #include "tools.h"
59 #include "wordlist.h"
60 #if USE_AUTH
61 #include "auth/UserRequest.h"
62 #endif
63 #if USE_ADAPTATION
64 #include "adaptation/AccessCheck.h"
65 #include "adaptation/Answer.h"
66 #include "adaptation/Iterator.h"
67 #include "adaptation/Service.h"
68 #if ICAP_CLIENT
70 #endif
71 #endif
72 #if USE_OPENSSL
73 #include "ssl/ServerBump.h"
74 #include "ssl/support.h"
75 #endif
76 
77 #if FOLLOW_X_FORWARDED_FOR
78 static void clientFollowXForwardedForCheck(Acl::Answer answer, void *data);
79 #endif /* FOLLOW_X_FORWARDED_FOR */
80 
82 
84 
85 /* Local functions */
86 /* other */
87 static void clientAccessCheckDoneWrapper(Acl::Answer, void *);
88 #if USE_OPENSSL
89 static void sslBumpAccessCheckDoneWrapper(Acl::Answer, void *);
90 #endif
91 static int clientHierarchical(ClientHttpRequest * http);
95 static void checkNoCacheDoneWrapper(Acl::Answer, void *);
100 
102 {
103  /*
104  * Release our "lock" on our parent, ClientHttpRequest, if we
105  * still have one
106  */
107 
108  if (http)
110 
111  delete error;
112  debugs(85,3, "ClientRequestContext destructed, this=" << this);
113 }
114 
116  http(cbdataReference(anHttp))
117 {
118  debugs(85, 3, "ClientRequestContext constructed, this=" << this);
119 }
120 
122 
124 #if USE_ADAPTATION
125  AsyncJob("ClientHttpRequest"),
126 #endif
127  al(new AccessLogEntry()),
128  conn_(cbdataReference(aConn))
129 {
132  if (aConn) {
133  al->tcpClient = aConn->clientConnection;
134  al->cache.port = aConn->port;
135  al->cache.caddr = aConn->log_addr;
137  al->updateError(aConn->bareError);
138 
139 #if USE_OPENSSL
140  if (aConn->clientConnection != nullptr && aConn->clientConnection->isOpen()) {
141  if (auto ssl = fd_table[aConn->clientConnection->fd].ssl.get())
142  al->cache.sslClientCert.resetWithoutLocking(SSL_get_peer_certificate(ssl));
143  }
144 #endif
145  }
147 }
148 
149 /*
150  * returns true if client specified that the object must come from the cache
151  * without contacting origin server
152  */
153 bool
155 {
156  assert(request);
157  return request->cache_control &&
159 }
160 
173 static void
175 {
176  // Can be set at compile time with -D compiler flag
177 #ifndef FAILURE_MODE_TIME
178 #define FAILURE_MODE_TIME 300
179 #endif
180 
181  if (hcode == HIER_NONE)
182  return;
183 
184  // don't bother when ICP is disabled.
185  if (Config.Port.icp <= 0)
186  return;
187 
188  static double magic_factor = 100.0;
189  double n_good;
190  double n_bad;
191 
192  n_good = magic_factor / (1.0 + request_failure_ratio);
193 
194  n_bad = magic_factor - n_good;
195 
196  switch (etype) {
197 
198  case ERR_DNS_FAIL:
199 
200  case ERR_CONNECT_FAIL:
202 
203  case ERR_READ_ERROR:
204  ++n_bad;
205  break;
206 
207  default:
208  ++n_good;
209  }
210 
211  request_failure_ratio = n_bad / n_good;
212 
214  return;
215 
216  if (request_failure_ratio < 1.0)
217  return;
218 
219  debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
220  std::setprecision(3) << request_failure_ratio);
221 
222  debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
223  FAILURE_MODE_TIME / 60 << " minutes...");
224 
226 
227  request_failure_ratio = 0.8; /* reset to something less than 1.0 */
228 }
229 
231 {
232  debugs(33, 3, "httpRequestFree: " << uri);
233 
234  // Even though freeResources() below may destroy the request,
235  // we no longer set request->body_pipe to NULL here
236  // because we did not initiate that pipe (ConnStateData did)
237 
238  /* the ICP check here was erroneous
239  * - StoreEntry::releaseRequest was always called if entry was valid
240  */
241 
242  logRequest();
243 
244  loggingEntry(nullptr);
245 
246  if (request)
248 
249  freeResources();
250 
251 #if USE_ADAPTATION
253 
254  if (adaptedBodySource != nullptr)
256 #endif
257 
258  delete calloutContext;
259 
260  if (conn_)
262 
263  /* moving to the next connection is handled by the context free */
265 }
266 
276 int
277 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
278  CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
279  char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
280 {
281  size_t url_sz;
282  ClientHttpRequest *http = new ClientHttpRequest(nullptr);
284  StoreIOBuffer tempBuffer;
285  if (http->al != nullptr)
286  http->al->cache.start_time = current_time;
287  /* this is only used to adjust the connection offset in client_side.c */
288  http->req_sz = 0;
289  tempBuffer.length = taillen;
290  tempBuffer.data = tailbuf;
291  /* client stream setup */
293  clientReplyStatus, new clientReplyContext(http), streamcallback,
294  streamdetach, streamdata, tempBuffer);
295  /* make it visible in the 'current acctive requests list' */
296  /* Set flags */
297  /* internal requests only makes sense in an
298  * accelerator today. TODO: accept flags ? */
299  http->flags.accel = true;
300  /* allow size for url rewriting */
301  url_sz = strlen(url) + Config.appendDomainLen + 5;
302  http->uri = (char *)xcalloc(url_sz, 1);
303  strcpy(http->uri, url); // XXX: polluting http->uri before parser validation
304 
305  request = HttpRequest::FromUrlXXX(http->uri, mx, method);
306  if (!request) {
307  debugs(85, 5, "Invalid URL: " << http->uri);
308  return -1;
309  }
310 
311  /*
312  * now update the headers in request with our supplied headers.
313  * HttpRequest::FromUrl() should return a blank header set, but
314  * we use Update to be sure of correctness.
315  */
316  if (header)
317  request->header.update(header);
318 
319  /* http struct now ready */
320 
321  /*
322  * build new header list *? TODO
323  */
324  request->flags.accelerated = http->flags.accel;
325 
326  /* this is an internally created
327  * request, not subject to acceleration
328  * target overrides */
329  // TODO: detect and handle internal requests of internal objects?
330 
331  /* Internally created requests cannot have bodies today */
332  request->content_length = 0;
333 
334  request->client_addr.setNoAddr();
335 
336 #if FOLLOW_X_FORWARDED_FOR
337  request->indirect_client_addr.setNoAddr();
338 #endif /* FOLLOW_X_FORWARDED_FOR */
339 
340  request->my_addr.setNoAddr(); /* undefined for internal requests */
341 
342  request->my_addr.port(0);
343 
344  request->http_ver = Http::ProtocolVersion();
345 
346  http->initRequest(request);
347 
348  /* optional - skip the access check ? */
349  http->calloutContext = new ClientRequestContext(http);
350 
351  http->calloutContext->http_access_done = false;
352 
353  http->calloutContext->redirect_done = true;
354 
355  http->calloutContext->no_cache_done = true;
356 
357  http->doCallouts();
358 
359  return 0;
360 }
361 
362 bool
364 {
365  ClientHttpRequest *http_ = http;
366 
367  if (cbdataReferenceValid(http_))
368  return true;
369 
370  http = nullptr;
371 
372  cbdataReferenceDone(http_);
373 
374  return false;
375 }
376 
377 #if FOLLOW_X_FORWARDED_FOR
392 static void
394 {
395  ClientRequestContext *calloutContext = (ClientRequestContext *) data;
396 
397  if (!calloutContext->httpStateIsValid())
398  return;
399 
400  ClientHttpRequest *http = calloutContext->http;
401  HttpRequest *request = http->request;
402 
403  if (answer.allowed() && request->x_forwarded_for_iterator.size() != 0) {
404 
405  /*
406  * Remove the last comma-delimited element from the
407  * x_forwarded_for_iterator and use it to repeat the cycle.
408  */
409  const char *p;
410  const char *asciiaddr;
411  int l;
412  Ip::Address addr;
413  p = request->x_forwarded_for_iterator.termedBuf();
414  l = request->x_forwarded_for_iterator.size();
415 
416  /*
417  * XXX x_forwarded_for_iterator should really be a list of
418  * IP addresses, but it's a String instead. We have to
419  * walk backwards through the String, biting off the last
420  * comma-delimited part each time. As long as the data is in
421  * a String, we should probably implement and use a variant of
422  * strListGetItem() that walks backwards instead of forwards
423  * through a comma-separated list. But we don't even do that;
424  * we just do the work in-line here.
425  */
426  /* skip trailing space and commas */
427  while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
428  --l;
429  request->x_forwarded_for_iterator.cut(l);
430  /* look for start of last item in list */
431  while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
432  --l;
433  asciiaddr = p+l;
434  if ((addr = asciiaddr)) {
435  request->indirect_client_addr = addr;
436  request->x_forwarded_for_iterator.cut(l);
439  /* override the default src_addr tested if we have to go deeper than one level into XFF */
440  Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
441  }
443  return;
444  }
445  }
446 
447  /* clean up, and pass control to clientAccessCheck */
449  /*
450  * Ensure that the access log shows the indirect client
451  * instead of the direct client.
452  */
453  http->al->cache.caddr = request->indirect_client_addr;
454  if (ConnStateData *conn = http->getConn())
455  conn->log_addr = request->indirect_client_addr;
456  }
457  request->x_forwarded_for_iterator.clean();
458  request->flags.done_follow_x_forwarded_for = true;
459 
460  if (answer.conflicted()) {
461  debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
462  }
463 
464  /* process actual access ACL as normal. */
465  calloutContext->clientAccessCheck();
466 }
467 #endif /* FOLLOW_X_FORWARDED_FOR */
468 
469 static void
471 {
472  ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
473  c->hostHeaderIpVerify(ia, dns);
474 }
475 
476 void
478 {
480 
481  // note the DNS details for the transaction stats.
482  http->request->recordLookup(dns);
483 
484  // Is the NAT destination IP in DNS?
485  if (ia && ia->have(clientConn->local)) {
486  debugs(85, 3, "validate IP " << clientConn->local << " possible from Host:");
487  http->request->flags.hostVerified = true;
488  http->doCallouts();
489  return;
490  }
491  debugs(85, 3, "FAIL: validate IP " << clientConn->local << " possible from Host:");
492  hostHeaderVerifyFailed("local IP", "any domain IP");
493 }
494 
495 void
497 {
498  // IP address validation for Host: failed. Admin wants to ignore them.
499  // NP: we do not yet handle CONNECT tunnels well, so ignore for them
501  debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
502  " (" << A << " does not match " << B << ") on URL: " << http->request->effectiveRequestUri());
503 
504  // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
505  // The problems here are about WRITE for new cache content, which means flags.cachable
506  http->request->flags.cachable = false; // MUST NOT cache (for now)
507  // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
508  http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
509  // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
510  http->doCallouts();
511  return;
512  }
513 
514  debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
515  http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
516  if (const char *ua = http->request->header.getStr(Http::HdrType::USER_AGENT))
517  debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << ua);
518  debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << http->request->effectiveRequestUri());
519 
520  // IP address validation for Host: failed. reject the connection.
522  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
523  assert (repContext);
524  repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
525  nullptr,
526  http->getConn(),
527  http->request,
528  nullptr,
529 #if USE_AUTH
530  http->getConn() != nullptr && http->getConn()->getAuth() != nullptr ?
532 #else
533  NULL);
534 #endif
536  clientStreamRead(node, http, node->readBuffer);
537 }
538 
539 void
541 {
542  // Require a Host: header.
543  const char *host = http->request->header.getStr(Http::HdrType::HOST);
544 
545  if (!host) {
546  // TODO: dump out the HTTP/1.1 error about missing host header.
547  // otherwise this is fine, can't forge a header value when its not even set.
548  debugs(85, 3, "validate skipped with no Host: header present.");
549  http->doCallouts();
550  return;
551  }
552 
553  if (http->request->flags.internal) {
554  // TODO: kill this when URL handling allows partial URLs out of accel mode
555  // and we no longer screw with the URL just to add our internal host there
556  debugs(85, 6, "validate skipped due to internal composite URL.");
557  http->doCallouts();
558  return;
559  }
560 
561  // Locate if there is a port attached, strip ready for IP lookup
562  char *portStr = nullptr;
563  char *hostB = xstrdup(host);
564  host = hostB;
565  if (host[0] == '[') {
566  // IPv6 literal.
567  portStr = strchr(hostB, ']');
568  if (portStr && *(++portStr) != ':') {
569  portStr = nullptr;
570  }
571  } else {
572  // Domain or IPv4 literal with port
573  portStr = strrchr(hostB, ':');
574  }
575 
576  uint16_t port = 0;
577  if (portStr) {
578  *portStr = '\0'; // strip the ':'
579  if (*(++portStr) != '\0') {
580  char *end = nullptr;
581  int64_t ret = strtoll(portStr, &end, 10);
582  if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
583  // invalid port details. Replace the ':'
584  *(--portStr) = ':';
585  portStr = nullptr;
586  } else
587  port = (ret & 0xFFFF);
588  }
589  }
590 
591  debugs(85, 3, "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
593  // verify the Host: port (if any) matches the apparent destination
594  if (portStr && port != http->getConn()->clientConnection->local.port()) {
595  debugs(85, 3, "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
596  " matches Host: port " << port << " (" << portStr << ")");
597  hostHeaderVerifyFailed("intercepted port", portStr);
598  } else {
599  // XXX: match the scheme default port against the apparent destination
600 
601  // verify the destination DNS is one of the Host: headers IPs
603  }
604  } else if (!Config.onoff.hostStrictVerify) {
605  debugs(85, 3, "validate skipped.");
606  http->doCallouts();
607  } else if (strlen(host) != strlen(http->request->url.host())) {
608  // Verify forward-proxy requested URL domain matches the Host: header
609  debugs(85, 3, "FAIL on validate URL domain length " << http->request->url.host() << " matches Host: " << host);
611  } else if (matchDomainName(host, http->request->url.host()) != 0) {
612  // Verify forward-proxy requested URL domain matches the Host: header
613  debugs(85, 3, "FAIL on validate URL domain " << http->request->url.host() << " matches Host: " << host);
615  } else if (portStr && port != http->request->url.port()) {
616  // Verify forward-proxy requested URL domain matches the Host: header
617  debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: port " << portStr);
618  hostHeaderVerifyFailed("URL port", portStr);
619  } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->url.port() != http->request->url.getScheme().defaultPort()) {
620  // Verify forward-proxy requested URL domain matches the Host: header
621  // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
622  debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: default port " << http->request->url.getScheme().defaultPort());
623  hostHeaderVerifyFailed("URL port", "default port");
624  } else {
625  // Okay no problem.
626  debugs(85, 3, "validate passed.");
627  http->request->flags.hostVerified = true;
628  http->doCallouts();
629  }
630  safe_free(hostB);
631 }
632 
633 /* This is the entry point for external users of the client_side routines */
634 void
636 {
637 #if FOLLOW_X_FORWARDED_FOR
638  if (!http->request->flags.doneFollowXff() &&
641 
642  /* we always trust the direct client address for actual use */
645 
646  /* setup the XFF iterator for processing */
648 
649  /* begin by checking to see if we trust direct client enough to walk XFF */
652  return;
653  }
654 #endif
655 
656  if (Config.accessList.http) {
659  } else {
660  debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
662  }
663 }
664 
670 void
672 {
676  } else {
677  debugs(85, 2, "No adapted_http_access configuration. default: ALLOW");
679  }
680 }
681 
682 void
684 {
685  ClientRequestContext *calloutContext = (ClientRequestContext *) data;
686 
687  if (!calloutContext->httpStateIsValid())
688  return;
689 
690  calloutContext->clientAccessCheckDone(answer);
691 }
692 
693 void
695 {
696  acl_checklist = nullptr;
697  err_type page_id;
698  Http::StatusCode status;
699  debugs(85, 2, "The request " << http->request->method << ' ' <<
700  http->uri << " is " << answer <<
701  "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
702 
703 #if USE_AUTH
704  char const *proxy_auth_msg = "<null>";
705  if (http->getConn() != nullptr && http->getConn()->getAuth() != nullptr)
706  proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
707  else if (http->request->auth_user_request != nullptr)
708  proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
709 #endif
710 
711  if (!answer.allowed()) {
712  // auth has a grace period where credentials can be expired but okay not to challenge.
713 
714  /* Send an auth challenge or error */
715  // XXX: do we still need aclIsProxyAuth() ?
716  bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
717  debugs(85, 5, "Access Denied: " << http->uri);
718  debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
719 #if USE_AUTH
720  if (auth_challenge)
721  debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
722 #endif
723 
724  /*
725  * NOTE: get page_id here, based on AclMatchedName because if
726  * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
727  * the clientCreateStoreEntry() call just below. Pedro Ribeiro
728  * <pribeiro@isel.pt>
729  */
731 
733 
734  if (auth_challenge) {
735 #if USE_AUTH
736  if (http->request->flags.sslBumped) {
737  /*SSL Bumped request, authentication is not possible*/
738  status = Http::scForbidden;
739  } else if (!http->flags.accel) {
740  /* Proxy authorisation needed */
742  } else {
743  /* WWW authorisation needed */
744  status = Http::scUnauthorized;
745  }
746 #else
747  // need auth, but not possible to do.
748  status = Http::scForbidden;
749 #endif
750  if (page_id == ERR_NONE)
751  page_id = ERR_CACHE_ACCESS_DENIED;
752  } else {
753  status = Http::scForbidden;
754 
755  if (page_id == ERR_NONE)
756  page_id = ERR_ACCESS_DENIED;
757  }
758 
759  error = clientBuildError(page_id, status, nullptr, http->getConn(), http->request, http->al);
760 
761 #if USE_AUTH
763  http->getConn() != nullptr && http->getConn()->getAuth() != nullptr ?
765 #endif
766 
767  readNextRequest = true;
768  }
769 
770  /* ACCESS_ALLOWED continues here ... */
771  xfree(http->uri);
773  http->doCallouts();
774 }
775 
776 #if USE_ADAPTATION
777 void
779 {
780  debugs(93,3, this << " adaptationAclCheckDone called");
781 
782 #if ICAP_CLIENT
784  if (ih != nullptr) {
785  if (getConn() != nullptr && getConn()->clientConnection != nullptr) {
786  ih->rfc931 = getConn()->clientConnection->rfc931;
787 #if USE_OPENSSL
788  if (getConn()->clientConnection->isOpen()) {
789  ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl.get());
790  }
791 #endif
792  }
793  ih->log_uri = log_uri;
794  ih->req_sz = req_sz;
795  }
796 #endif
797 
798  if (!g) {
799  debugs(85,3, "no adaptation needed");
800  doCallouts();
801  return;
802  }
803 
804  startAdaptation(g);
805 }
806 
807 #endif
808 
809 static void
811 {
812  ClientRequestContext *context = (ClientRequestContext *)data;
813  ClientHttpRequest *http = context->http;
814  context->acl_checklist = nullptr;
815 
816  if (answer.allowed())
818  else {
819  Helper::Reply const nilReply(Helper::Error);
820  context->clientRedirectDone(nilReply);
821  }
822 }
823 
824 void
826 {
827  debugs(33, 5, "'" << http->uri << "'");
832  } else
834 }
835 
840 static void
842 {
843  ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
844  ClientHttpRequest *http = context->http;
845  context->acl_checklist = nullptr;
846 
847  if (answer.allowed())
848  storeIdStart(http, clientStoreIdDoneWrapper, context);
849  else {
850  debugs(85, 3, "access denied expected ERR reply handling: " << answer);
851  Helper::Reply const nilReply(Helper::Error);
852  context->clientStoreIdDone(nilReply);
853  }
854 }
855 
861 void
863 {
864  debugs(33, 5,"'" << http->uri << "'");
865 
866  if (Config.accessList.store_id) {
869  } else
871 }
872 
873 static int
875 {
876  HttpRequest *request = http->request;
877  HttpRequestMethod method = request->method;
878 
879  // intercepted requests MUST NOT (yet) be sent to peers unless verified
880  if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
881  return 0;
882 
883  /*
884  * IMS needs a private key, so we can use the hierarchy for IMS only if our
885  * neighbors support private keys
886  */
887 
888  if (request->flags.ims && !neighbors_do_private_keys)
889  return 0;
890 
891  /*
892  * This is incorrect: authenticating requests can be sent via a hierarchy
893  * (they can even be cached if the correct headers are set on the reply)
894  */
895  if (request->flags.auth)
896  return 0;
897 
898  if (method == Http::METHOD_TRACE)
899  return 1;
900 
901  if (method != Http::METHOD_GET)
902  return 0;
903 
904  if (request->flags.loopDetected)
905  return 0;
906 
907  if (request->url.getScheme() == AnyP::PROTO_HTTP)
908  return method.respMaybeCacheable();
909 
910  if (request->url.getScheme() == AnyP::PROTO_GOPHER)
911  return gopherCachable(request);
912 
913  if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
914  return 0;
915 
916  return 1;
917 }
918 
919 static void
921 {
922  HttpRequest *request = http->request;
923  HttpHeader *req_hdr = &request->header;
924  ConnStateData *http_conn = http->getConn();
925 
926  /* Internal requests such as those from ESI includes may be without
927  * a client connection
928  */
929  if (!http_conn)
930  return;
931 
932  request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
933  if (!request->flags.connectionAuthDisabled) {
934  if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
935  if (http_conn->pinning.auth) {
936  request->flags.connectionAuth = true;
937  request->flags.auth = true;
938  } else {
939  request->flags.connectionProxyAuth = true;
940  }
941  // These should already be linked correctly.
942  assert(request->clientConnectionManager == http_conn);
943  }
944  }
945 
946  /* check if connection auth is used, and flag as candidate for pinning
947  * in such case.
948  * Note: we may need to set flags.connectionAuth even if the connection
949  * is already pinned if it was pinned earlier due to proxy auth
950  */
951  if (!request->flags.connectionAuth) {
954  HttpHeaderEntry *e;
955  int may_pin = 0;
956  while ((e = req_hdr->getEntry(&pos))) {
958  const char *value = e->value.rawBuf();
959  if (strncasecmp(value, "NTLM ", 5) == 0
960  ||
961  strncasecmp(value, "Negotiate ", 10) == 0
962  ||
963  strncasecmp(value, "Kerberos ", 9) == 0) {
964  if (e->id == Http::HdrType::AUTHORIZATION) {
965  request->flags.connectionAuth = true;
966  may_pin = 1;
967  } else {
968  request->flags.connectionProxyAuth = true;
969  may_pin = 1;
970  }
971  }
972  }
973  }
974  if (may_pin && !request->pinnedConnection()) {
975  // These should already be linked correctly. Just need the ServerConnection to pinn.
976  assert(request->clientConnectionManager == http_conn);
977  }
978  }
979  }
980 }
981 
982 static void
984 {
985  HttpRequest *request = http->request;
986  HttpHeader *req_hdr = &request->header;
987  bool no_cache = false;
988 
989  request->imslen = -1;
991 
992  if (request->ims > 0)
993  request->flags.ims = true;
994 
995  if (!request->flags.ignoreCc) {
996  if (request->cache_control) {
997  if (request->cache_control->hasNoCache())
998  no_cache=true;
999 
1000  // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1001  } else if (req_hdr->has(Http::HdrType::PRAGMA))
1002  no_cache = req_hdr->hasListMember(Http::HdrType::PRAGMA,"no-cache",',');
1003  }
1004 
1005  if (request->method == Http::METHOD_OTHER) {
1006  no_cache=true;
1007  }
1008 
1009  if (no_cache) {
1010 #if USE_HTTP_VIOLATIONS
1011 
1013  request->flags.nocacheHack = true;
1014  else if (refresh_nocache_hack)
1015  request->flags.nocacheHack = true;
1016  else
1017 #endif
1018 
1019  request->flags.noCache = true;
1020  }
1021 
1022  /* ignore range header in non-GETs or non-HEADs */
1024  // XXX: initialize if we got here without HttpRequest::parseHeader()
1025  if (!request->range)
1026  request->range = req_hdr->getRange();
1027 
1028  if (request->range) {
1029  request->flags.isRanged = true;
1031  /* XXX: This is suboptimal. We should give the stream the range set,
1032  * and thereby let the top of the stream set the offset when the
1033  * size becomes known. As it is, we will end up requesting from 0
1034  * for evey -X range specification.
1035  * RBC - this may be somewhat wrong. We should probably set the range
1036  * iter up at this point.
1037  */
1038  node->readBuffer.offset = request->range->lowestOffset(0);
1039  }
1040  }
1041 
1042  /* Only HEAD and GET requests permit a Range or Request-Range header.
1043  * If these headers appear on any other type of request, delete them now.
1044  */
1045  else {
1046  req_hdr->delById(Http::HdrType::RANGE);
1048  request->ignoreRange("neither HEAD nor GET");
1049  }
1050 
1051  if (req_hdr->has(Http::HdrType::AUTHORIZATION))
1052  request->flags.auth = true;
1053 
1054  clientCheckPinning(http);
1055 
1056  if (!request->url.userInfo().isEmpty())
1057  request->flags.auth = true;
1058 
1059  if (req_hdr->has(Http::HdrType::VIA)) {
1060  String s = req_hdr->getList(Http::HdrType::VIA);
1061  /*
1062  * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1063  * Note ThisCache2 has a space prepended to the hostname so we don't
1064  * accidentally match super-domains.
1065  */
1066 
1067  if (strListIsSubstr(&s, ThisCache2, ',')) {
1068  request->flags.loopDetected = true;
1069  }
1070 
1071 #if USE_FORW_VIA_DB
1073 
1074 #endif
1075 
1076  s.clean();
1077  }
1078 
1079  // headers only relevant to reverse-proxy
1080  if (request->flags.accelerated) {
1081  // check for a cdn-info member with a cdn-id matching surrogate_id
1082  // XXX: HttpHeader::hasListMember() does not handle OWS around ";" yet
1084  request->flags.loopDetected = true;
1085  }
1086 
1087  if (request->flags.loopDetected) {
1088  debugObj(33, DBG_IMPORTANT, "WARNING: Forwarding loop detected for:\n",
1090  }
1091 
1092 #if USE_FORW_VIA_DB
1093 
1094  if (req_hdr->has(Http::HdrType::X_FORWARDED_FOR)) {
1097  s.clean();
1098  }
1099 
1100 #endif
1101 
1102  request->flags.cachable = http->request->maybeCacheable();
1103 
1104  if (clientHierarchical(http))
1105  request->flags.hierarchical = true;
1106 
1107  debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1108  (request->flags.noCache ? "SET" : "NOT SET"));
1109  debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1110  (request->flags.cachable ? "SET" : "NOT SET"));
1111  debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1112  (request->flags.hierarchical ? "SET" : "NOT SET"));
1113 
1114 }
1115 
1116 void
1117 clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1118 {
1119  ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1120 
1121  if (!calloutContext->httpStateIsValid())
1122  return;
1123 
1124  calloutContext->clientRedirectDone(result);
1125 }
1126 
1127 void
1128 clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1129 {
1130  ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1131 
1132  if (!calloutContext->httpStateIsValid())
1133  return;
1134 
1135  calloutContext->clientStoreIdDone(result);
1136 }
1137 
1138 void
1140 {
1141  HttpRequest *old_request = http->request;
1142  debugs(85, 5, "'" << http->uri << "' result=" << reply);
1145 
1146  // Put helper response Notes into the transaction state record (ALE) eventually
1147  // do it early to ensure that no matter what the outcome the notes are present.
1148  if (http->al)
1149  http->al->syncNotes(old_request);
1150 
1151  UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1152 
1153  switch (reply.result) {
1154  case Helper::TimedOut:
1156  static const auto d = MakeNamedErrorDetail("REDIRECTOR_TIMEDOUT");
1158  debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: Timedout");
1159  }
1160  break;
1161 
1162  case Helper::Unknown:
1163  case Helper::TT:
1164  // Handler in redirect.cc should have already mapped Unknown
1165  // IF it contained valid entry for the old URL-rewrite helper protocol
1166  debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1167  break;
1168 
1169  case Helper::BrokenHelper:
1170  debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply);
1171  break;
1172 
1173  case Helper::Error:
1174  // no change to be done.
1175  break;
1176 
1177  case Helper::Okay: {
1178  // #1: redirect with a specific status code OK status=NNN url="..."
1179  // #2: redirect with a default status code OK url="..."
1180  // #3: re-write the URL OK rewrite-url="..."
1181 
1182  const char *statusNote = reply.notes.findFirst("status");
1183  const char *urlNote = reply.notes.findFirst("url");
1184 
1185  if (urlNote != nullptr) {
1186  // HTTP protocol redirect to be done.
1187 
1188  // TODO: change default redirect status for appropriate requests
1189  // Squid defaults to 302 status for now for better compatibility with old clients.
1190  // HTTP/1.0 client should get 302 (Http::scFound)
1191  // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1192  // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1194  if (statusNote != nullptr) {
1195  const char * result = statusNote;
1196  status = static_cast<Http::StatusCode>(atoi(result));
1197  }
1198 
1199  if (status == Http::scMovedPermanently
1200  || status == Http::scFound
1201  || status == Http::scSeeOther
1202  || status == Http::scPermanentRedirect
1203  || status == Http::scTemporaryRedirect) {
1204  http->redirect.status = status;
1205  http->redirect.location = xstrdup(urlNote);
1206  // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1207  } else {
1208  debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1209  }
1210  } else {
1211  // URL-rewrite wanted. Ew.
1212  urlNote = reply.notes.findFirst("rewrite-url");
1213 
1214  // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1215  if (urlNote != nullptr && strcmp(urlNote, http->uri)) {
1216  AnyP::Uri tmpUrl;
1217  if (tmpUrl.parse(old_request->method, SBuf(urlNote))) {
1218  HttpRequest *new_request = old_request->clone();
1219  new_request->url = tmpUrl;
1220  debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri());
1221 
1222  // update the new request to flag the re-writing was done on it
1223  new_request->flags.redirected = true;
1224 
1225  // unlink bodypipe from the old request. Not needed there any longer.
1226  if (old_request->body_pipe != nullptr) {
1227  old_request->body_pipe = nullptr;
1228  debugs(61,2, "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1229  " from request " << old_request << " to " << new_request);
1230  }
1231 
1232  http->resetRequest(new_request);
1233  old_request = nullptr;
1234  } else {
1235  debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1236  old_request->method << " " << urlNote << " " << old_request->http_ver);
1237  }
1238  }
1239  }
1240  }
1241  break;
1242  }
1243 
1244  /* XXX PIPELINE: This is inaccurate during pipelining */
1245 
1246  if (http->getConn() != nullptr && Comm::IsConnOpen(http->getConn()->clientConnection))
1248 
1249  assert(http->uri);
1250 
1251  http->doCallouts();
1252 }
1253 
1257 void
1259 {
1260  HttpRequest *old_request = http->request;
1261  debugs(85, 5, "'" << http->uri << "' result=" << reply);
1264 
1265  // Put helper response Notes into the transaction state record (ALE) eventually
1266  // do it early to ensure that no matter what the outcome the notes are present.
1267  if (http->al)
1268  http->al->syncNotes(old_request);
1269 
1270  UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1271 
1272  switch (reply.result) {
1273  case Helper::Unknown:
1274  case Helper::TT:
1275  // Handler in redirect.cc should have already mapped Unknown
1276  // IF it contained valid entry for the old helper protocol
1277  debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1278  break;
1279 
1280  case Helper::TimedOut:
1281  // Timeouts for storeID are not implemented
1282  case Helper::BrokenHelper:
1283  debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply);
1284  break;
1285 
1286  case Helper::Error:
1287  // no change to be done.
1288  break;
1289 
1290  case Helper::Okay: {
1291  const char *urlNote = reply.notes.findFirst("store-id");
1292 
1293  // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1294  if (urlNote != nullptr && strcmp(urlNote, http->uri) ) {
1295  // Debug section required for some very specific cases.
1296  debugs(85, 9, "Setting storeID with: " << urlNote );
1297  http->request->store_id = urlNote;
1298  http->store_id = urlNote;
1299  }
1300  }
1301  break;
1302  }
1303 
1304  http->doCallouts();
1305 }
1306 
1310 void
1312 {
1313  if (Config.accessList.noCache) {
1316  } else {
1317  /* unless otherwise specified, we try to cache. */
1319  }
1320 }
1321 
1322 static void
1324 {
1325  ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1326 
1327  if (!calloutContext->httpStateIsValid())
1328  return;
1329 
1330  calloutContext->checkNoCacheDone(answer);
1331 }
1332 
1333 void
1335 {
1336  acl_checklist = nullptr;
1337  if (answer.denied()) {
1338  http->request->flags.noCache = true; // do not read reply from cache
1339  http->request->flags.cachable = false; // do not store reply into cache
1340  }
1341  http->doCallouts();
1342 }
1343 
1344 #if USE_OPENSSL
1345 bool
1347 {
1348  if (!http->getConn()) {
1349  http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1350  return false;
1351  }
1352 
1354  if (http->request->flags.forceTunnel) {
1355  debugs(85, 5, "not needed; already decided to tunnel " << http->getConn());
1356  if (bumpMode != Ssl::bumpEnd)
1357  http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1358  return false;
1359  }
1360 
1361  // If SSL connection tunneling or bumping decision has been made, obey it.
1362  if (bumpMode != Ssl::bumpEnd) {
1363  debugs(85, 5, "SslBump already decided (" << bumpMode <<
1364  "), " << "ignoring ssl_bump for " << http->getConn());
1365 
1366  // We need the following "if" for transparently bumped TLS connection,
1367  // because in this case we are running ssl_bump access list before
1368  // the doCallouts runs. It can be removed after the bug #4340 fixed.
1369  // We do not want to proceed to bumping steps:
1370  // - if the TLS connection with the client is already established
1371  // because we are accepting normal HTTP requests on TLS port,
1372  // or because of the client-first bumping mode
1373  // - When the bumping is already started
1374  if (!http->getConn()->switchedToHttps() &&
1375  !http->getConn()->serverBump())
1376  http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1377  http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1378  return false;
1379  }
1380 
1381  // If we have not decided yet, decide whether to bump now.
1382 
1383  // Bumping here can only start with a CONNECT request on a bumping port
1384  // (bumping of intercepted SSL conns is decided before we get 1st request).
1385  // We also do not bump redirected CONNECT requests.
1388  !http->getConn()->port->flags.tunnelSslBumping) {
1389  http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1390  debugs(85, 5, "cannot SslBump this request");
1391  return false;
1392  }
1393 
1394  // Do not bump during authentication: clients would not proxy-authenticate
1395  // if we delay a 407 response and respond with 200 OK to CONNECT.
1397  http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1398  debugs(85, 5, "no SslBump during proxy authentication");
1399  return false;
1400  }
1401 
1402  if (error) {
1403  debugs(85, 5, "SslBump applies. Force bump action on error " << errorTypeName(error->type));
1406  return false;
1407  }
1408 
1409  debugs(85, 5, "SslBump possible, checking ACL");
1410 
1412  aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1413  return true;
1414 }
1415 
1420 static void
1422 {
1423  ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1424 
1425  if (!calloutContext->httpStateIsValid())
1426  return;
1427  calloutContext->sslBumpAccessCheckDone(answer);
1428 }
1429 
1430 void
1432 {
1433  if (!httpStateIsValid())
1434  return;
1435 
1436  const Ssl::BumpMode bumpMode = answer.allowed() ?
1437  static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpSplice;
1438  http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1439  http->al->ssl.bumpMode = bumpMode; // for logging
1440 
1441  if (bumpMode == Ssl::bumpTerminate) {
1442  const Comm::ConnectionPointer clientConn = http->getConn() ? http->getConn()->clientConnection : nullptr;
1443  if (Comm::IsConnOpen(clientConn)) {
1444  debugs(85, 3, "closing after Ssl::bumpTerminate ");
1445  clientConn->close();
1446  }
1447  return;
1448  }
1449 
1450  http->doCallouts();
1451 }
1452 #endif
1453 
1454 /*
1455  * Identify requests that do not go through the store and client side stream
1456  * and forward them to the appropriate location. All other requests, request
1457  * them.
1458  */
1459 void
1461 {
1462  debugs(85, 4, request->method << ' ' << uri);
1463 
1464  const bool untouchedConnect = request->method == Http::METHOD_CONNECT && !redirect.status;
1465 
1466 #if USE_OPENSSL
1467  if (untouchedConnect && sslBumpNeeded()) {
1469  sslBumpStart();
1470  return;
1471  }
1472 #endif
1473 
1474  if (untouchedConnect || request->flags.forceTunnel) {
1475  getConn()->stopReading(); // tunnels read for themselves
1476  tunnelStart(this);
1477  return;
1478  }
1479 
1480  httpStart();
1481 }
1482 
1483 void
1485 {
1486  // XXX: Re-initializes rather than updates. Should not be needed at all.
1488  debugs(85, 4, loggingTags().c_str() << " for '" << uri << "'");
1489 
1490  /* no one should have touched this */
1491  assert(out.offset == 0);
1492  /* Use the Stream Luke */
1494  clientStreamRead(node, this, node->readBuffer);
1495 }
1496 
1497 #if USE_OPENSSL
1498 
1499 void
1501 {
1502  debugs(83, 3, "sslBump required: "<< Ssl::bumpMode(mode));
1503  sslBumpNeed_ = mode;
1504 }
1505 
1506 // called when comm_write has completed
1507 static void
1508 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1509 {
1510  ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1511  debugs(85, 5, "responded to CONNECT: " << r << " ? " << errflag);
1512 
1513  assert(r && cbdataReferenceValid(r));
1514  r->sslBumpEstablish(errflag);
1515 }
1516 
1517 void
1519 {
1520  // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1521  if (errflag == Comm::ERR_CLOSING)
1522  return;
1523 
1524  if (errflag) {
1525  debugs(85, 3, "CONNECT response failure in SslBump: " << errflag);
1527  return;
1528  }
1529 
1530 #if USE_AUTH
1531  // Preserve authentication info for the ssl-bumped request
1532  if (request->auth_user_request != nullptr)
1533  getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1534 #endif
1535 
1536  assert(sslBumpNeeded());
1538 }
1539 
1540 void
1542 {
1543  debugs(85, 5, "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1544  "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1546 
1547  AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1549 
1551  CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1552  params.flag = Comm::OK;
1553  params.conn = getConn()->clientConnection;
1554  ScheduleCallHere(bumpCall);
1555  return;
1556  }
1557 
1559 
1560  const auto mb = al->reply->pack();
1561  // send an HTTP 200 response to kick client SSL negotiation
1562  // TODO: Unify with tunnel.cc and add a Server(?) header
1563  Comm::Write(getConn()->clientConnection, mb, bumpCall);
1564  delete mb;
1565 }
1566 
1567 #endif
1568 
1569 void
1571 {
1572  if (request)
1574  else
1575  al->updateError(error);
1576 }
1577 
1578 bool
1580 {
1581  // TODO: See also (and unify with) clientReplyContext::storeNotOKTransferDone()
1582  int64_t contentLength =
1584  assert(contentLength >= 0);
1585 
1586  if (out.offset < contentLength)
1587  return false;
1588 
1589  return true;
1590 }
1591 
1592 void
1594 {
1595  entry_ = newEntry;
1596 }
1597 
1598 void
1600 {
1601  if (loggingEntry_)
1602  loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1603 
1604  loggingEntry_ = newEntry;
1605 
1606  if (loggingEntry_)
1607  loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1608 }
1609 
1610 void
1612 {
1613  assignRequest(aRequest);
1614  if (const auto csd = getConn()) {
1615  if (!csd->notes()->empty())
1616  request->notes()->appendNewOnly(csd->notes().getRaw());
1617  }
1618  // al is created in the constructor
1619  assert(al);
1620  if (!al->request) {
1621  al->request = request;
1623  al->syncNotes(request);
1624  }
1625 }
1626 
1627 void
1629 {
1630  assert(request != newRequest);
1631  clearRequest();
1632  assignRequest(newRequest);
1633  xfree(uri);
1635 }
1636 
1637 void
1639 {
1640  assert(newRequest);
1641  assert(!request);
1642  const_cast<HttpRequest *&>(request) = newRequest;
1645 }
1646 
1647 void
1649 {
1650  HttpRequest *oldRequest = request;
1651  HTTPMSGUNLOCK(oldRequest);
1652  const_cast<HttpRequest *&>(request) = nullptr;
1653  absorbLogUri(nullptr);
1654 }
1655 
1656 /*
1657  * doCallouts() - This function controls the order of "callout"
1658  * executions, including non-blocking access control checks, the
1659  * redirector, and ICAP. Previously, these callouts were chained
1660  * together such that "clientAccessCheckDone()" would call
1661  * "clientRedirectStart()" and so on.
1662  *
1663  * The ClientRequestContext (aka calloutContext) class holds certain
1664  * state data for the callout/callback operations. Previously
1665  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1666  * for a short time. ClientRequestContext would then delete itself
1667  * and pass control back to ClientHttpRequest when all callouts
1668  * were finished.
1669  *
1670  * This caused some problems for ICAP because we want to make the
1671  * ICAP callout after checking ACLs, but before checking the no_cache
1672  * list. We can't stuff the ICAP state into the ClientRequestContext
1673  * class because we still need the ICAP state after ClientRequestContext
1674  * goes away.
1675  *
1676  * Note that ClientRequestContext is created before the first call
1677  * to doCallouts().
1678  *
1679  * If one of the callouts notices that ClientHttpRequest is no
1680  * longer valid, it should call cbdataReferenceDone() so that
1681  * ClientHttpRequest's reference count goes to zero and it will get
1682  * deleted. ClientHttpRequest will then delete ClientRequestContext.
1683  *
1684  * Note that we set the _done flags here before actually starting
1685  * the callout. This is strictly for convenience.
1686  */
1687 
1688 void
1690 {
1692 
1693  if (!calloutContext->error) {
1694  // CVE-2009-0801: verify the Host: header is consistent with other known details.
1696  debugs(83, 3, "Doing calloutContext->hostHeaderVerify()");
1699  return;
1700  }
1701 
1703  debugs(83, 3, "Doing calloutContext->clientAccessCheck()");
1706  return;
1707  }
1708 
1709 #if USE_ADAPTATION
1714  request, nullptr, calloutContext->http->al, this))
1715  return; // will call callback
1716  }
1717 #endif
1718 
1719  if (!calloutContext->redirect_done) {
1720  calloutContext->redirect_done = true;
1721 
1722  if (Config.Program.redirect) {
1723  debugs(83, 3, "Doing calloutContext->clientRedirectStart()");
1726  return;
1727  }
1728  }
1729 
1731  debugs(83, 3, "Doing calloutContext->clientAccessCheck2()");
1734  return;
1735  }
1736 
1737  if (!calloutContext->store_id_done) {
1738  calloutContext->store_id_done = true;
1739 
1740  if (Config.Program.store_id) {
1741  debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1744  return;
1745  }
1746  }
1747 
1749  debugs(83, 3, "Doing clientInterpretRequestHeaders()");
1752  }
1753 
1754  if (!calloutContext->no_cache_done) {
1755  calloutContext->no_cache_done = true;
1756 
1758  debugs(83, 3, "Doing calloutContext->checkNoCache()");
1760  return;
1761  }
1762  }
1763  } // if !calloutContext->error
1764 
1765  // Set appropriate MARKs and CONNMARKs if needed.
1766  if (getConn() && Comm::IsConnOpen(getConn()->clientConnection)) {
1767  ACLFilledChecklist ch(nullptr, request, nullptr);
1768  ch.al = calloutContext->http->al;
1770  ch.my_addr = request->my_addr;
1771  ch.syncAle(request, log_uri);
1772 
1775  tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1776  if (tos)
1777  Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1778 
1779  const auto packetMark = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1780  if (!packetMark.isEmpty())
1781  Ip::Qos::setSockNfmark(getConn()->clientConnection, packetMark.mark);
1782 
1783  const auto connmark = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfConnmarkToClient, &ch);
1784  if (!connmark.isEmpty())
1785  Ip::Qos::setNfConnmark(getConn()->clientConnection, Ip::Qos::dirAccepted, connmark);
1786  }
1787  }
1788 
1789 #if USE_OPENSSL
1790  // Even with calloutContext->error, we call sslBumpAccessCheck() to decide
1791  // whether SslBump applies to this transaction. If it applies, we will
1792  // attempt to bump the client to serve the error.
1796  return;
1797  /* else no ssl bump required*/
1798  }
1799 #endif
1800 
1801  if (calloutContext->error) {
1802  // XXX: prformance regression. c_str() reallocates
1803  SBuf storeUriBuf(request->storeId());
1804  const char *storeUri = storeUriBuf.c_str();
1805  StoreEntry *e = storeCreateEntry(storeUri, storeUri, request->flags, request->method);
1806 #if USE_OPENSSL
1807  if (sslBumpNeeded()) {
1808  // We have to serve an error, so bump the client first.
1810  // set final error but delay sending until we bump
1811  Ssl::ServerBump *srvBump = new Ssl::ServerBump(this, e, Ssl::bumpClientFirst);
1813  calloutContext->error = nullptr;
1814  getConn()->setServerBump(srvBump);
1815  e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1816  } else
1817 #endif
1818  {
1819  // send the error to the client now
1821  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1822  assert (repContext);
1823  repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1825  calloutContext->error = nullptr;
1827  getConn()->flags.readMore = true; // resume any pipeline reads.
1829  clientStreamRead(node, this, node->readBuffer);
1830  e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1831  return;
1832  }
1833  }
1834 
1836  delete calloutContext;
1837  calloutContext = nullptr;
1838 #if HEADERS_LOG
1839 
1840  headersLog(0, 1, request->method, request);
1841 #endif
1842 
1843  debugs(83, 3, "calling processRequest()");
1844  processRequest();
1845 
1846 #if ICAP_CLIENT
1848  if (ih != nullptr)
1849  ih->logType = loggingTags();
1850 #endif
1851 }
1852 
1853 void
1855 {
1856  assert(request);
1857  const auto canonicalUri = request->canonicalCleanUrl();
1858  absorbLogUri(xstrndup(canonicalUri, MAX_URL));
1859 }
1860 
1861 void
1863 {
1864  assert(rawUri);
1865  // Should(!request);
1866 
1867  // TODO: SBuf() performance regression, fix by converting rawUri to SBuf
1868  char *canonicalUri = urlCanonicalCleanWithoutRequest(SBuf(rawUri), method, AnyP::UriScheme());
1869 
1870  absorbLogUri(AnyP::Uri::cleanup(canonicalUri));
1871 
1872  char *cleanedRawUri = AnyP::Uri::cleanup(rawUri);
1873  al->setVirginUrlForMissingRequest(SBuf(cleanedRawUri));
1874  xfree(cleanedRawUri);
1875 }
1876 
1877 void
1879 {
1880  xfree(log_uri);
1881  const_cast<char *&>(log_uri) = aUri;
1882 }
1883 
1884 void
1886 {
1887  assert(!uri);
1888  assert(aUri);
1889  // Should(!request);
1890 
1891  uri = xstrdup(aUri);
1892  // TODO: SBuf() performance regression, fix by converting setErrorUri() parameter to SBuf
1893  const SBuf errorUri(aUri);
1894  const auto canonicalUri = urlCanonicalCleanWithoutRequest(errorUri, HttpRequestMethod(), AnyP::UriScheme());
1895  absorbLogUri(xstrndup(canonicalUri, MAX_URL));
1896 
1897  al->setVirginUrlForMissingRequest(errorUri);
1898 }
1899 
1900 // XXX: This should not be a _request_ method. Move range_iter elsewhere.
1901 int64_t
1903 {
1904  assert(request);
1905  assert(request->range);
1906 
1908  range_iter.end = request->range->end();
1909  range_iter.debt_size = 0;
1910  const auto multipart = request->range->specs.size() > 1;
1911  if (multipart)
1913  range_iter.valid = true; // TODO: Remove.
1914  range_iter.updateSpec(); // TODO: Refactor to initialize rather than update.
1915 
1917  const auto &firstRange = *range_iter.pos;
1918  assert(firstRange);
1919  out.offset = firstRange->offset;
1920 
1921  return multipart ? mRangeCLen() : firstRange->length;
1922 }
1923 
1924 #if USE_ADAPTATION
1926 void
1928 {
1929  debugs(85, 3, "adaptation needed for " << this);
1933  new Adaptation::Iterator(request, nullptr, al, g));
1934 
1935  // we could try to guess whether we can bypass this adaptation
1936  // initiation failure, but it should not really happen
1938 }
1939 
1940 void
1942 {
1943  assert(cbdataReferenceValid(this)); // indicates bug
1946 
1947  switch (answer.kind) {
1949  handleAdaptedHeader(const_cast<Http::Message*>(answer.message.getRaw()));
1950  break;
1951 
1953  handleAdaptationBlock(answer);
1954  break;
1955 
1957  static const auto d = MakeNamedErrorDetail("CLT_REQMOD_ABORT");
1958  handleAdaptationFailure(d, !answer.final);
1959  break;
1960  }
1961  }
1962 }
1963 
1964 void
1966 {
1967  assert(msg);
1968 
1969  if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1970  // update the new message to flag whether URL re-writing was done on it
1971  if (request->effectiveRequestUri() != new_req->effectiveRequestUri())
1972  new_req->flags.redirected = true;
1973  resetRequest(new_req);
1974  assert(request->method.id());
1975  } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1976  debugs(85,3, "REQMOD reply is HTTP reply");
1977 
1978  // subscribe to receive reply body
1979  if (new_rep->body_pipe != nullptr) {
1980  adaptedBodySource = new_rep->body_pipe;
1981  int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1982  assert(consumer_ok);
1983  }
1984 
1986  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1987  assert(repContext);
1988  repContext->createStoreEntry(request->method, request->flags);
1989 
1992  storeEntry()->replaceHttpReply(new_rep);
1994 
1995  al->reply = new_rep;
1996 
1997  if (!adaptedBodySource) // no body
1998  storeEntry()->complete();
1999  clientGetMoreData(node, this);
2000  }
2001 
2002  // we are done with getting headers (but may be receiving body)
2004 
2006  doCallouts();
2007 }
2008 
2009 void
2011 {
2012  static const auto d = MakeNamedErrorDetail("REQMOD_BLOCK");
2014  AclMatchedName = answer.ruleId.termedBuf();
2017  AclMatchedName = nullptr;
2018 }
2019 
2020 void
2022 {
2023  if (!adaptedBodySource)
2024  return;
2025 
2027 }
2028 
2029 void
2031 {
2033  assert(adaptedBodySource != nullptr);
2034 
2035  if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
2036  const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
2037 
2038  if (spaceAvailable < contentSize ) {
2039  // No or partial body data consuming
2040  typedef NullaryMemFunT<ClientHttpRequest> Dialer;
2041  AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
2042  Dialer(this, &ClientHttpRequest::resumeBodyStorage));
2043  storeEntry()->deferProducer(call);
2044  }
2045 
2046  if (!spaceAvailable)
2047  return;
2048 
2049  if (spaceAvailable < contentSize )
2050  contentSize = spaceAvailable;
2051 
2053  const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
2054  storeEntry()->write(ioBuf);
2055  // assume StoreEntry::write() writes the entire ioBuf
2057  bpc.buf.consume(contentSize);
2058  bpc.checkIn();
2059  }
2060 
2063  // else wait for more body data
2064 }
2065 
2066 void
2068 {
2070 
2071  // distinguish this code path from future noteBodyProducerAborted() that
2072  // would continue storing/delivering (truncated) reply if necessary (TODO)
2074 
2075  // should we end request satisfaction now?
2076  if (adaptedBodySource != nullptr && adaptedBodySource->exhausted())
2078 }
2079 
2080 void
2082 {
2083  debugs(85,4, this << " ends request satisfaction");
2086 
2087  // TODO: anything else needed to end store entry formation correctly?
2089  // We received the entire reply per receivedWholeAdaptedReply.
2090  // We are called when we consumed everything received (per our callers).
2091  // We consume only what we store per noteMoreBodyDataAvailable().
2092  storeEntry()->completeSuccessfully("received, consumed, and, hence, stored the entire REQMOD reply");
2093  } else {
2094  storeEntry()->completeTruncated("REQMOD request satisfaction default");
2095  }
2096 }
2097 
2098 void
2100 {
2103 
2104  debugs(85,3, "REQMOD body production failed");
2105  if (request_satisfaction_mode) { // too late to recover or serve an error
2106  static const auto d = MakeNamedErrorDetail("CLT_REQMOD_RESP_BODY");
2109  Must(Comm::IsConnOpen(c));
2110  c->close(); // drastic, but we may be writing a response already
2111  } else {
2112  static const auto d = MakeNamedErrorDetail("CLT_REQMOD_REQ_BODY");
2114  }
2115 }
2116 
2117 void
2119 {
2120  debugs(85,3, "handleAdaptationFailure(" << bypassable << ")");
2121 
2122  const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2123  const bool usedPipe = request->body_pipe != nullptr &&
2124  request->body_pipe->consumedSize() > 0;
2125 
2126  if (bypassable && !usedStore && !usedPipe) {
2127  debugs(85,3, "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2128  if (calloutContext)
2129  doCallouts();
2130  return;
2131  }
2132 
2133  debugs(85,3, "ICAP REQMOD callout failed, responding with error");
2134 
2136  clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2137  assert(repContext);
2138 
2139  calloutsError(ERR_ICAP_FAILURE, errDetail);
2140 
2141  if (calloutContext)
2142  doCallouts();
2143 }
2144 
2145 void
2146 ClientHttpRequest::callException(const std::exception &ex)
2147 {
2148  if (const auto clientConn = getConn() ? getConn()->clientConnection : nullptr) {
2149  if (Comm::IsConnOpen(clientConn)) {
2150  debugs(85, 3, "closing after exception: " << ex.what());
2151  clientConn->close(); // initiate orderly top-to-bottom cleanup
2152  return;
2153  }
2154  }
2155  debugs(85, DBG_IMPORTANT, "ClientHttpRequest exception without connection. Ignoring " << ex.what());
2156  // XXX: Normally, we mustStop() but we cannot do that here because it is
2157  // likely to leave Http::Stream and ConnStateData with a dangling http
2158  // pointer. See r13480 or XXX in Http::Stream class description.
2159 }
2160 #endif
2161 
2162 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2163 void
2165 {
2166  // The original author of the code also wanted to pass an errno to
2167  // setReplyToError, but it seems unlikely that the errno reflects the
2168  // true cause of the error at this point, so I did not pass it.
2169  if (calloutContext) {
2170  ConnStateData * c = getConn();
2172  nullptr, c, request, al);
2173 #if USE_AUTH
2175  c != nullptr && c->getAuth() != nullptr ? c->getAuth() : request->auth_user_request;
2176 #endif
2177  calloutContext->error->detailError(errDetail);
2179  if (c != nullptr)
2180  c->expectNoForwarding();
2181  }
2182  //else if(calloutContext == NULL) is it possible?
2183 }
2184 
#define ScheduleCallHere(call)
Definition: AsyncCall.h:164
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:154
CommCbFunPtrCallT< Dialer > * commCbCall(int debugSection, int debugLevel, const char *callName, const Dialer &dialer)
Definition: CommCalls.h:342
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
const char * errorTypeName(err_type err)
Definition: Error.h:53
ACLFilledChecklist * Filled(ACLChecklist *checklist)
convenience and safety wrapper for dynamic_cast<ACLFilledChecklist*>
Ip::NfMarkConfig aclFindNfMarkConfig(acl_nfmark *head, ACLChecklist *ch)
Checks for a netfilter mark value to apply depending on the ACL.
Definition: FwdState.cc:1513
tos_t aclMapTOS(acl_tos *head, ACLChecklist *ch)
Checks for a TOS value to apply depending on the ACL.
Definition: FwdState.cc:1501
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
void UpdateRequestNotes(ConnStateData *csd, HttpRequest &request, NotePairs const &helperNotes)
Definition: HttpRequest.cc:769
void httpRequestPack(void *obj, Packable *p)
Definition: HttpRequest.cc:362
@ LOG_TCP_DENIED
Definition: LogTags.h:52
@ LOG_TAG_NONE
Definition: LogTags.h:38
time_t squid_curtime
Definition: stub_libtime.cc:20
void SBufToCstring(char *d, const SBuf &s)
Definition: SBuf.h:752
class SquidConfig Config
Definition: SquidConfig.cc:12
int strListIsSubstr(const String *list, const char *s, char del)
Definition: StrList.cc:63
SBuf StringToSBuf(const String &s)
create a new SBuf from a String by copying contents
Definition: StringConvert.h:17
#define Must(condition)
Definition: TextException.h:71
int conn
the current server connection FD
Definition: Transport.cc:26
int matchDomainName(const char *h, const char *d, MatchDomainNameFlags flags)
Definition: Uri.cc:736
char * urlCanonicalCleanWithoutRequest(const SBuf &url, const HttpRequestMethod &method, const AnyP::UriScheme &scheme)
Definition: Uri.cc:620
void fvdbCountVia(const SBuf &)
void fvdbCountForwarded(const SBuf &)
count occurrences of the given X-Forwarded-For header value
void error(char *format,...)
#define assert(EX)
Definition: assert.h:19
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:398
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
#define cbdataReference(var)
Definition: cbdata.h:341
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
void nonBlockingCheck(ACLCB *callback, void *callback_data)
Definition: Checklist.cc:237
virtual void syncAle(HttpRequest *adaptedRequest, const char *logUri) const
assigns uninitialized adapted_request and url ALE components
Ip::Address src_addr
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
AnyP::PortCfgPointer port
Security::CertPointer sslClientCert
cert received from the client
struct timeval start_time
The time the master transaction started.
int bumpMode
whether and how the request was SslBumped
HttpReplyPointer reply
class AccessLogEntry::CacheDetails cache
HierarchyLogEntry hier
void syncNotes(HttpRequest *request)
Comm::ConnectionPointer tcpClient
TCP/IP level details about the client connection.
HttpRequest * request
void setVirginUrlForMissingRequest(const SBuf &vu)
Remember Client URI (or equivalent) when there is no HttpRequest.
ProxyProtocol::HeaderPointer proxyProtocolHeader
see ConnStateData::proxyProtocolHeader_
class AccessLogEntry::SslDetails ssl
void updateError(const Error &)
sets (or updates the already stored) transaction error as needed
int kind
the matched custom access list verb (or zero)
Definition: Acl.h:164
bool denied() const
Definition: Acl.h:156
bool conflicted() const
whether Squid is uncertain about the allowed() or denied() answer
Definition: Acl.h:159
bool allowed() const
Definition: Acl.h:150
static bool Start(Method method, VectPoint vp, HttpRequest *req, HttpReply *, const AccessLogEntryPointer &, Adaptation::Initiator *)
Definition: AccessCheck.cc:30
summarizes adaptation service answer for the noteAdaptationAnswer() API
Definition: Answer.h:23
String ruleId
ACL (or similar rule) name that blocked forwarding.
Definition: Answer.h:40
Kind kind
the type of the answer
Definition: Answer.h:42
Http::MessagePointer message
HTTP request or response to forward.
Definition: Answer.h:39
bool final
whether the error, if any, cannot be bypassed
Definition: Answer.h:41
@ akForward
forward the supplied adapted HTTP message
Definition: Answer.h:27
@ akBlock
block or deny the master xaction; see authority
Definition: Answer.h:28
@ akError
no adapted message will come; see bypassable
Definition: Answer.h:29
CbcPointer< Initiate > initiateAdaptation(Initiate *x)
< starts freshly created initiate and returns a safe pointer to it
Definition: Initiator.cc:23
void clearAdaptation(CbcPointer< Initiate > &x)
clears the pointer (does not call announceInitiatorAbort)
Definition: Initiator.cc:32
void announceInitiatorAbort(CbcPointer< Initiate > &x)
inform the transaction about abnormal termination and clear the pointer
Definition: Initiator.cc:38
bool initiated(const CbcPointer< AsyncJob > &job) const
Must(initiated(initiate)) instead of Must(initiate.set()), for clarity.
Definition: Initiator.h:52
iterates services in ServiceGroup, starting adaptation launchers
Definition: Iterator.h:32
unsigned short defaultPort() const
Definition: UriScheme.cc:71
Definition: Uri.h:31
void port(unsigned short p)
Definition: Uri.h:94
AnyP::UriScheme const & getScheme() const
Definition: Uri.h:67
static char * cleanup(const char *uri)
Definition: Uri.cc:910
void host(const char *src)
Definition: Uri.cc:99
bool parse(const HttpRequestMethod &, const SBuf &url)
Definition: Uri.cc:254
char const * denyMessage(char const *const default_message=nullptr) const
Definition: UserRequest.cc:127
void stopConsumingFrom(RefCount< BodyPipe > &)
Definition: BodyPipe.cc:118
MemBuf & buf
Definition: BodyPipe.h:74
bool exhausted() const
Definition: BodyPipe.cc:174
uint64_t consumedSize() const
Definition: BodyPipe.h:111
bool setConsumerIfNotLate(const Consumer::Pointer &aConsumer)
Definition: BodyPipe.cc:228
const MemBuf & buf() const
Definition: BodyPipe.h:137
int64_t prepPartialResponseGeneration()
ClientHttpRequest(ConnStateData *)
virtual void callException(const std::exception &)
called when the job throws during an async call
struct ClientHttpRequest::Out out
void clearRequest()
resets the current request and log_uri to nil
HttpRequest *const request
virtual void noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer)
void resumeBodyStorage()
called by StoreEntry when it has more buffer space available
virtual void noteBodyProductionEnded(BodyPipe::Pointer)
bool receivedWholeAdaptedReply
noteBodyProductionEnded() was called
StoreEntry * loggingEntry_
int64_t mRangeCLen() const
Definition: client_side.cc:753
void calloutsError(const err_type, const ErrorDetail::Pointer &)
Build an error reply. For use with the callouts.
void absorbLogUri(char *)
assigns log_uri with aUri without copying the entire C-string
String rangeBoundaryStr() const
Definition: client_side.cc:793
void initRequest(HttpRequest *)
void setLogUriToRequestUri()
sets log_uri when we know the current request
void updateError(const Error &)
if necessary, stores new error information (if any)
virtual void noteMoreBodyDataAvailable(BodyPipe::Pointer)
CbcPointer< Adaptation::Initiate > virginHeadSource
void updateLoggingTags(const LogTags_ot code)
update the code in the transaction processing tags
void setLogUriToRawUri(const char *, const HttpRequestMethod &)
size_t req_sz
raw request size on input, not current request size
ConnStateData * conn_
virtual void noteAdaptationAnswer(const Adaptation::Answer &)
void setErrorUri(const char *)
BodyPipe::Pointer adaptedBodySource
Ssl::BumpMode sslBumpNeed_
whether (and how) the request needs to be bumped
HttpHdrRangeIter range_iter
void handleAdaptedHeader(Http::Message *)
struct ClientHttpRequest::Flags flags
void resetRequest(HttpRequest *)
StoreEntry * loggingEntry() const
void assignRequest(HttpRequest *)
void handleAdaptationBlock(const Adaptation::Answer &)
virtual void noteBodyProducerAborted(BodyPipe::Pointer)
void sslBumpEstablish(Comm::Flag)
StoreEntry * storeEntry() const
bool sslBumpNeeded() const
returns true if and only if the request needs to be bumped
ClientRequestContext * calloutContext
Ssl::BumpMode sslBumpNeed() const
returns raw sslBump mode value
void handleAdaptationFailure(const ErrorDetail::Pointer &, bool bypassable=false)
const AccessLogEntry::Pointer al
access.log entry
MemObject * memObject() const
void startAdaptation(const Adaptation::ServiceGroupPointer &)
Initiate an asynchronous adaptation transaction which will call us back.
const LogTags & loggingTags() const
the processing tags associated with this request transaction.
ConnStateData * getConn() const
struct ClientHttpRequest::Redirect redirect
void clientAccessCheckDone(const Acl::Answer &)
void clientStoreIdDone(const Helper::Reply &)
void clientRedirectDone(const Helper::Reply &)
void sslBumpAccessCheckDone(const Acl::Answer &answer)
The callback function for ssl-bump access check list.
ClientRequestContext(ClientHttpRequest *)
bool readNextRequest
whether Squid should read after error handling
ClientHttpRequest * http
ACLChecklist * acl_checklist
need ptr back so we can unregister if needed
ErrorState * error
saved error page for centralized/delayed processing
void hostHeaderVerifyFailed(const char *A, const char *B)
void hostHeaderIpVerify(const ipcache_addrs *, const Dns::LookupDetails &)
void checkNoCacheDone(const Acl::Answer &)
static void Reset()
forgets the current context, setting it to nil/unknown
Definition: CodeContext.cc:75
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
bool isOpen() const
Definition: Connection.h:99
char rfc931[USER_IDENT_SZ]
Definition: Connection.h:174
Ip::Address local
Definition: Connection.h:144
bool auth
pinned for www authentication
Definition: client_side.h:148
bool switchedToHttps() const
Definition: client_side.h:285
int port
port of pinned connection
Definition: client_side.h:146
Comm::ConnectionPointer serverConnection
Definition: client_side.h:144
void switchToHttps(ClientHttpRequest *, Ssl::BumpMode bumpServerMode)
struct ConnStateData::@38 flags
void setAuth(const Auth::UserRequest::Pointer &aur, const char *cause)
Definition: client_side.cc:507
const ProxyProtocol::HeaderPointer & proxyProtocolHeader() const
Definition: client_side.h:361
Error bareError
a problem that occurred without a request (e.g., while parsing headers)
Definition: client_side.h:382
void expectNoForwarding()
cleans up virgin request [body] forwarding state
Ssl::ServerBump * serverBump()
Definition: client_side.h:286
struct ConnStateData::@39 pinning
bool isOpen() const
Definition: client_side.cc:653
Ip::Address log_addr
Definition: client_side.h:137
Ssl::BumpMode sslBumpMode
ssl_bump decision (Ssl::bumpEnd if n/a).
Definition: client_side.h:305
bool readMore
needs comm_read (for this request or new requests)
Definition: client_side.h:140
const Auth::UserRequest::Pointer & getAuth() const
Definition: client_side.h:124
void setServerBump(Ssl::ServerBump *srvBump)
Definition: client_side.h:287
bool have(const Ip::Address &ip, size_t *position=nullptr) const
Definition: ipcache.cc:973
encapsulates DNS lookup results
Definition: LookupDetails.h:21
err_type type
Definition: errorpage.h:168
void detailError(const ErrorDetail::Pointer &dCode)
set error type-specific detail code
Definition: errorpage.h:109
Auth::UserRequest::Pointer auth_user_request
Definition: errorpage.h:173
Http::StatusCode httpStatus
Definition: errorpage.h:171
a transaction problem
Definition: Error.h:18
err_type category
primary error classification (or ERR_NONE)
Definition: Error.h:35
void update(const Error &)
if necessary, stores the given error information (if any)
Definition: Error.cc:16
NotePairs notes
Definition: Reply.h:62
Helper::ResultCode result
The helper response 'result' field.
Definition: Reply.h:59
bool hasOnlyIfCached() const
Definition: HttpHdrCc.h:145
HttpHdrRange::iterator pos
HttpHdrRange::iterator end
iterator begin()
iterator end()
std::vector< HttpHdrRangeSpec * > specs
Http::HdrType id
Definition: HttpHeader.h:63
int delById(Http::HdrType id)
Definition: HttpHeader.cc:671
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:818
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:588
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1160
time_t getTime(Http::HdrType id) const
Definition: HttpHeader.cc:1143
HttpHdrRange * getRange() const
Definition: HttpHeader.cc:1217
int has(Http::HdrType id) const
Definition: HttpHeader.cc:967
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1659
MemBuf * pack() const
Definition: HttpReply.cc:110
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:374
static HttpReplyPointer MakeConnectionEstablished()
construct and return an HTTP/200 (Connection Established) response
Definition: HttpReply.cc:119
bool respMaybeCacheable() const
Http::MethodType id() const
Definition: RequestMethod.h:70
void recordLookup(const Dns::LookupDetails &detail)
Definition: HttpRequest.cc:589
HttpHdrRange * range
Definition: HttpRequest.h:143
HttpRequestMethod method
Definition: HttpRequest.h:114
Ip::Address indirect_client_addr
Definition: HttpRequest.h:152
static HttpRequest * FromUrlXXX(const char *url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:529
String store_id
Definition: HttpRequest.h:139
RequestFlags flags
Definition: HttpRequest.h:141
String x_forwarded_for_iterator
Definition: HttpRequest.h:187
void detailError(const err_type c, const ErrorDetail::Pointer &d)
sets error detail if no earlier detail was available
Definition: HttpRequest.h:101
NotePairs::Pointer notes()
Definition: HttpRequest.cc:761
virtual HttpRequest * clone() const
Definition: HttpRequest.cc:176
const SBuf storeId()
Definition: HttpRequest.cc:742
bool maybeCacheable()
Definition: HttpRequest.cc:539
char * canonicalCleanUrl() const
Definition: HttpRequest.cc:823
Ip::Address my_addr
Definition: HttpRequest.h:155
Auth::UserRequest::Pointer auth_user_request
Definition: HttpRequest.h:127
Error error
the first transaction problem encountered (or falsy)
Definition: HttpRequest.h:161
Adaptation::Icap::History::Pointer icapHistory() const
Returns possibly nil history, creating it if icap logging is enabled.
Definition: HttpRequest.cc:390
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
Ip::Address client_addr
Definition: HttpRequest.h:149
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:753
common parts of HttpRequest and HttpReply
Definition: Message.h:26
HttpHeader header
Definition: Message.h:75
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:98
HttpHdrCc * cache_control
Definition: Message.h:77
AnyP::ProtocolVersion http_ver
Definition: Message.h:73
unsigned short port() const
Definition: Address.cc:778
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
void consume(mb_size_t sz)
removes sz bytes and "packs" by moving content left
Definition: MemBuf.cc:168
const HttpReply & baseReply() const
Definition: MemObject.h:59
void appendNewOnly(const NotePairs *src)
Definition: Notes.cc:381
const char * findFirst(const char *noteKey) const
Definition: Notes.cc:297
Definition: Range.h:19
C * getRaw() const
Definition: RefCount.h:80
bool doneFollowXff() const
Definition: RequestFlags.h:119
bool interceptTproxy
Set for requests handled by a "tproxy" port.
Definition: RequestFlags.h:66
bool forceTunnel
whether to forward via TunnelStateData (instead of FwdState)
Definition: RequestFlags.h:111
bool hostVerified
Definition: RequestFlags.h:64
bool intercepted
Definition: RequestFlags.h:62
bool hierarchical
Definition: RequestFlags.h:34
Definition: SBuf.h:94
const char * c_str()
Definition: SBuf.cc:516
void resetWithoutLocking(T *t)
Reset raw pointer - unlock any previous one and save new one without locking.
Comm::ConnectionPointer clientConnection
Definition: Server.h:98
void stopReading()
cancels Comm::Read() if it is scheduled
Definition: Server.cc:60
size_t appendDomainLen
Definition: SquidConfig.h:221
acl_access * adapted_http
Definition: SquidConfig.h:366
int log_uses_indirect_client
Definition: SquidConfig.h:334
struct SquidConfig::@97 Port
int acl_uses_indirect_client
Definition: SquidConfig.h:332
wordlist * store_id
Definition: SquidConfig.h:200
acl_access * followXFF
Definition: SquidConfig.h:397
acl_access * redirector
Definition: SquidConfig.h:383
int hostStrictVerify
Definition: SquidConfig.h:343
int reload_into_ims
Definition: SquidConfig.h:299
acl_access * http
Definition: SquidConfig.h:365
struct SquidConfig::@111 accessList
wordlist * redirect
Definition: SquidConfig.h:199
char * surrogate_id
Definition: SquidConfig.h:218
AclDenyInfoList * denyInfoList
Definition: SquidConfig.h:414
struct SquidConfig::@110 onoff
unsigned short icp
Definition: SquidConfig.h:139
struct SquidConfig::UrlHelperTimeout onUrlRewriteTimeout
acl_access * ssl_bump
Definition: SquidConfig.h:394
struct SquidConfig::@102 Program
struct SquidConfig::@103 Accel
acl_access * noCache
Definition: SquidConfig.h:372
void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply)
Definition: store.cc:992
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:210
void completeTruncated(const char *whyWeConsiderTheReplyTruncated)
Definition: store.cc:999
int unlock(const char *context)
Definition: store.cc:444
void complete()
Definition: store.cc:1006
void write(StoreIOBuffer)
Definition: store.cc:755
void lock(const char *context)
Definition: store.cc:420
bool timestampsSet()
Definition: store.cc:1370
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1682
bool isEmpty() const
Definition: Store.h:66
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:339
char const * rawBuf() const
Definition: SquidString.h:86
void clean()
Definition: String.cc:118
char const * termedBuf() const
Definition: SquidString.h:92
void CSD(clientStreamNode *, ClientHttpRequest *)
client stream detach
void CSR(clientStreamNode *, ClientHttpRequest *)
client stream read
void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer)
client stream read callback
clientStream_status_t CSS(clientStreamNode *, ClientHttpRequest *)
ACLFilledChecklist * clientAclChecklistCreate(const acl_access *acl, ClientHttpRequest *http)
static void clientInterpretRequestHeaders(ClientHttpRequest *http)
static void clientRedirectAccessCheckDone(Acl::Answer answer, void *data)
ErrorState * clientBuildError(err_type, Http::StatusCode, char const *url, const ConnStateData *, HttpRequest *, const AccessLogEntry::Pointer &)
static void clientFollowXForwardedForCheck(Acl::Answer answer, void *data)
#define FAILURE_MODE_TIME
static void checkNoCacheDoneWrapper(Acl::Answer, void *)
static void sslBumpAccessCheckDoneWrapper(Acl::Answer, void *)
static void hostHeaderIpVerifyWrapper(const ipcache_addrs *ia, const Dns::LookupDetails &dns, void *data)
static void clientCheckPinning(ClientHttpRequest *http)
static int clientHierarchical(ClientHttpRequest *http)
static HLPCB clientStoreIdDoneWrapper
static void clientStoreIdAccessCheckDone(Acl::Answer answer, void *data)
int clientBeginRequest(const HttpRequestMethod &method, char const *url, CSCB *streamcallback, CSD *streamdetach, ClientStreamData streamdata, HttpHeader const *header, char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
static void checkFailureRatio(err_type, hier_code)
static HLPCB clientRedirectDoneWrapper
SQUIDCEXTERN CSR clientGetMoreData
static void clientAccessCheckDoneWrapper(Acl::Answer, void *)
SQUIDCEXTERN CSS clientReplyStatus
SQUIDCEXTERN CSD clientReplyDetach
static void SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
void tunnelStart(ClientHttpRequest *)
Definition: tunnel.cc:1118
#define DBG_IMPORTANT
Definition: Stream.h:41
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
#define DBG_CRITICAL
Definition: Stream.h:40
#define REDIRECT_DONE
Definition: defines.h:57
#define REDIRECT_PENDING
Definition: defines.h:56
#define MAX_URL
Definition: defines.h:78
static int port
Definition: ldap_backend.cc:70
err_type
Definition: forward.h:14
@ ERR_ACCESS_DENIED
Definition: forward.h:18
@ ERR_CONNECT_FAIL
Definition: forward.h:30
@ ERR_SECURE_CONNECT_FAIL
Definition: forward.h:31
@ ERR_DNS_FAIL
Definition: forward.h:35
@ ERR_GATEWAY_FAILURE
Definition: forward.h:67
@ ERR_NONE
Definition: forward.h:15
@ ERR_ICAP_FAILURE
Definition: forward.h:64
@ ERR_CONFLICT_HOST
Definition: forward.h:48
@ ERR_READ_ERROR
Definition: forward.h:28
@ ERR_CACHE_ACCESS_DENIED
Definition: forward.h:19
void fd_note(int fd, const char *s)
Definition: fd.cc:216
#define fd_table
Definition: fde.h:189
int refresh_nocache_hack
char ThisCache2[RFC2181_MAXHOSTNAMELEN<< 1]
int neighbors_do_private_keys
double request_failure_ratio
time_t hit_only_mode_until
err_type aclGetDenyInfoPage(AclDenyInfoList **head, const char *name, int redirect_allowed)
Definition: Gadgets.cc:42
int aclIsProxyAuth(const char *name)
Definition: Gadgets.cc:73
const char * AclMatchedName
Definition: Acl.cc:29
@ ACCESS_AUTH_REQUIRED
Definition: Acl.h:114
@ ACCESS_DENIED
Definition: Acl.h:109
@ ACCESS_ALLOWED
Definition: Acl.h:110
void clientStreamRead(clientStreamNode *thisObject, ClientHttpRequest *http, StoreIOBuffer readBuffer)
void clientStreamInit(dlink_list *list, CSR *func, CSD *rdetach, CSS *readstatus, ClientStreamData readdata, CSCB *callback, CSD *cdetach, ClientStreamData callbackdata, StoreIOBuffer tailBuffer)
void errorAppendEntry(StoreEntry *entry, ErrorState *err)
Definition: errorpage.cc:719
void ipcache_nbgethostbyname(const char *name, IPH *handler, void *handlerData)
Definition: ipcache.cc:599
int gopherCachable(const HttpRequest *req)
Definition: gopher.cc:346
BumpMode
Definition: support.h:126
const char * sslGetUserEmail(SSL *ssl)
Definition: support.cc:885
const char * bumpMode(int bm)
Definition: support.h:138
@ bumpTerminate
Definition: support.h:126
@ bumpEnd
Definition: support.h:126
@ bumpClientFirst
Definition: support.h:126
@ bumpSplice
Definition: support.h:126
@ bumpBump
Definition: support.h:126
void HLPCB(void *, const Helper::Reply &)
Definition: forward.h:27
hier_code
Definition: hier_code.h:12
@ HIER_NONE
Definition: hier_code.h:13
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:150
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:161
unsigned char tos_t
Definition: forward.h:27
static uint32 A
Definition: md4.c:43
static uint32 B
Definition: md4.c:43
@ methodReqmod
Definition: Elements.h:17
@ pointPreCache
Definition: Elements.h:18
@ PROTO_GOPHER
Definition: ProtocolType.h:30
@ PROTO_HTTP
Definition: ProtocolType.h:25
@ PROTO_CACHE_OBJECT
Definition: ProtocolType.h:32
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:27
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:33
Flag
Definition: Flag.h:15
@ OK
Definition: Flag.h:16
@ ERR_CLOSING
Definition: Flag.h:25
@ Unknown
Definition: ResultCode.h:17
@ BrokenHelper
Definition: ResultCode.h:20
@ Error
Definition: ResultCode.h:19
@ TimedOut
Definition: ResultCode.h:21
@ Okay
Definition: ResultCode.h:18
StatusCode
Definition: StatusCode.h:20
@ scForbidden
Definition: StatusCode.h:47
@ scUnauthorized
Definition: StatusCode.h:45
@ scFound
Definition: StatusCode.h:38
@ scInternalServerError
Definition: StatusCode.h:71
@ scConflict
Definition: StatusCode.h:53
@ scPermanentRedirect
Definition: StatusCode.h:43
@ scSeeOther
Definition: StatusCode.h:39
@ scProxyAuthenticationRequired
Definition: StatusCode.h:51
@ scTemporaryRedirect
Definition: StatusCode.h:42
@ scMovedPermanently
Definition: StatusCode.h:37
@ METHOD_TRACE
Definition: MethodType.h:30
@ METHOD_OTHER
Definition: MethodType.h:93
@ METHOD_CONNECT
Definition: MethodType.h:29
@ METHOD_GET
Definition: MethodType.h:25
@ METHOD_HEAD
Definition: MethodType.h:28
@ PROXY_AUTHORIZATION
@ IF_MODIFIED_SINCE
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
bool setNfConnmark(Comm::ConnectionPointer &conn, const ConnectionDirection connDir, const NfMarkConfig &cm)
@ dirAccepted
accepted (from a client by Squid)
Definition: QosConfig.h:67
Config TheConfig
Globally available instance of Qos::Config.
Definition: QosConfig.cc:283
int setSockTos(const Comm::ConnectionPointer &conn, tos_t tos)
Definition: QosConfig.cc:570
int setSockNfmark(const Comm::ConnectionPointer &conn, nfmark_t mark)
Definition: QosConfig.cc:602
#define xfree
#define xstrdup
void storeIdStart(ClientHttpRequest *http, HLPCB *handler, void *data)
Definition: redirect.cc:311
void redirectStart(ClientHttpRequest *http, HLPCB *handler, void *data)
Definition: redirect.cc:285
@ toutActBypass
Definition: redirect.h:16
#define SQUIDCEXTERN
Definition: squid.h:21
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:734
int64_t strtoll(const char *nptr, char **endptr, int base)
Definition: strtoll.c:61
char * url
Definition: tcp-banger2.c:114
char method[16]
Definition: tcp-banger2.c:115
int content_length
Definition: tcp-banger2.c:121
Definition: parse.c:104
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
struct timeval current_time
the current UNIX time in timeval {seconds, microseconds} format
Definition: gadgets.cc:17
void debugObj(int section, int level, const char *label, void *obj, ObjPackMethod pm)
Definition: tools.cc:935
void(* ObjPackMethod)(void *obj, Packable *p)
Definition: tools.h:33
#define NULL
Definition: types.h:166
void * xcalloc(size_t n, size_t sz)
Definition: xalloc.cc:71
#define safe_free(x)
Definition: xalloc.h:73
#define xisspace(x)
Definition: xis.h:17
char * xstrndup(const char *s, size_t n)
Definition: xstring.cc:56

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors