http.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 11 Hypertext Transfer Protocol (HTTP) */
10 
11 /*
12  * Anonymizing patch by lutz@as-node.jena.thur.de
13  * have a look into http-anon.c to get more information.
14  */
15 
16 #include "squid.h"
17 #include "acl/FilledChecklist.h"
18 #include "base/AsyncJobCalls.h"
19 #include "base/TextException.h"
20 #include "base64.h"
21 #include "CachePeer.h"
22 #include "client_side.h"
23 #include "comm/Connection.h"
24 #include "comm/Read.h"
25 #include "comm/Write.h"
26 #include "CommRead.h"
27 #include "err_detail_type.h"
28 #include "errorpage.h"
29 #include "fd.h"
30 #include "fde.h"
31 #include "globals.h"
32 #include "http.h"
35 #include "http/Stream.h"
36 #include "HttpControlMsg.h"
37 #include "HttpHdrCc.h"
38 #include "HttpHdrContRange.h"
39 #include "HttpHdrSc.h"
40 #include "HttpHdrScTarget.h"
41 #include "HttpHeaderTools.h"
42 #include "HttpReply.h"
43 #include "HttpRequest.h"
45 #include "log/access_log.h"
46 #include "MemBuf.h"
47 #include "MemObject.h"
48 #include "neighbors.h"
49 #include "pconn.h"
51 #include "profiler/Profiler.h"
52 #include "refresh.h"
53 #include "RefreshPattern.h"
54 #include "rfc1738.h"
55 #include "SquidConfig.h"
56 #include "SquidTime.h"
57 #include "StatCounters.h"
58 #include "Store.h"
59 #include "StrList.h"
60 #include "tools.h"
61 #include "util.h"
62 
63 #if USE_AUTH
64 #include "auth/UserRequest.h"
65 #endif
66 #if USE_DELAY_POOLS
67 #include "DelayPools.h"
68 #endif
69 
70 #define SQUID_ENTER_THROWING_CODE() try {
71 #define SQUID_EXIT_THROWING_CODE(status) \
72  status = true; \
73  } \
74  catch (const std::exception &e) { \
75  debugs (11, 1, "Exception error:" << e.what()); \
76  status = false; \
77  }
78 
80 
81 static const char *const crlf = "\r\n";
82 
84 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request,
85  HttpHeader * hdr_out, const int we_do_ranges, const Http::StateFlags &);
86 
88  AsyncJob("HttpStateData"),
89  Client(theFwdState),
90  lastChunk(0),
91  httpChunkDecoder(NULL),
92  payloadSeen(0),
93  payloadTruncated(0),
94  sawDateGoBack(false)
95 {
96  debugs(11,5,HERE << "HttpStateData " << this << " created");
97  ignoreCacheControl = false;
98  surrogateNoStore = false;
100 
101  if (fwd->serverConnection() != NULL)
102  _peer = cbdataReference(fwd->serverConnection()->getPeer()); /* might be NULL */
103 
104  flags.peering = _peer;
107 
108  if (_peer) {
109  /*
110  * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
111  * We might end up getting the object from somewhere else if,
112  * for example, the request to this neighbor fails.
113  */
115  entry->releaseRequest(true);
116 
117 #if USE_DELAY_POOLS
119 #endif
120  }
121 
122  /*
123  * register the handler to free HTTP state data when the FD closes
124  */
128 }
129 
131 {
132  /*
133  * don't forget that ~Client() gets called automatically
134  */
135 
136  if (httpChunkDecoder)
137  delete httpChunkDecoder;
138 
140 
141  delete upgradeHeaderOut;
142 
143  debugs(11,5, HERE << "HttpStateData " << this << " destroyed; " << serverConnection);
144 }
145 
148 {
149  return serverConnection;
150 }
151 
152 void
154 {
155  debugs(11, 5, "httpStateFree: FD " << params.fd << ", httpState=" << params.data);
156  doneWithFwd = "httpStateConnClosed()"; // assume FwdState is monitoring too
157  mustStop("HttpStateData::httpStateConnClosed");
158 }
159 
160 void
162 {
163  debugs(11, 4, serverConnection << ": '" << entry->url() << "'");
164 
165  if (entry->store_status == STORE_PENDING) {
167  }
168 
169  closeServer();
170  mustStop("HttpStateData::httpTimeout");
171 }
172 
173 static StoreEntry *
175  assert(newEntry->mem_obj);
176  return newEntry->mem_obj->request ?
178  storeGetPublic(newEntry->mem_obj->storeId(), newEntry->mem_obj->method);
179 }
180 
183 static void
185 {
186  int remove = 0;
187  int forbidden = 0;
188 
189  // If the incoming response already goes into a public entry, then there is
190  // nothing to remove. This protects ready-for-collapsing entries as well.
191  if (!EBIT_TEST(e->flags, KEY_PRIVATE))
192  return;
193 
194  // If the new/incoming response cannot be stored, then it does not
195  // compete with the old stored response for the public key, and the
196  // old stored response should be left as is.
197  if (e->mem_obj->request && !e->mem_obj->request->flags.cachable)
198  return;
199 
200  switch (status) {
201 
202  case Http::scOkay:
203 
205 
207 
209 
210  case Http::scFound:
211 
212  case Http::scSeeOther:
213 
214  case Http::scGone:
215 
216  case Http::scNotFound:
217  remove = 1;
218 
219  break;
220 
221  case Http::scForbidden:
222 
224  forbidden = 1;
225 
226  break;
227 
228 #if WORK_IN_PROGRESS
229 
231  forbidden = 1;
232 
233  break;
234 
235 #endif
236 
237  default:
238 #if QUESTIONABLE
239  /*
240  * Any 2xx response should eject previously cached entities...
241  */
242 
243  if (status >= 200 && status < 300)
244  remove = 1;
245 
246 #endif
247 
248  break;
249  }
250 
251  if (!remove && !forbidden)
252  return;
253 
255 
256  if (pe != NULL) {
257  assert(e != pe);
258 #if USE_HTCP
260 #endif
261  pe->release(true);
262  }
263 
268  if (e->mem_obj->request)
270  else
272 
273  if (pe != NULL) {
274  assert(e != pe);
275 #if USE_HTCP
277 #endif
278  pe->release(true);
279  }
280 }
281 
282 void
284 {
285  if (request->flags.accelerated && reply->surrogate_control) {
287 
288  if (sctusable) {
289  if (sctusable->hasNoStore() ||
291  && sctusable->noStoreRemote())) {
292  surrogateNoStore = true;
293  // Be conservative for now and make it non-shareable because
294  // there is no enough information here to make the decision.
295  entry->makePrivate(false);
296  }
297 
298  /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
299  * accelerated request or not...
300  * Still, this is an abstraction breach. - RC
301  */
302  if (sctusable->hasMaxAge()) {
303  if (sctusable->maxAge() < sctusable->maxStale())
304  reply->expires = reply->date + sctusable->maxAge();
305  else
306  reply->expires = reply->date + sctusable->maxStale();
307 
308  /* And update the timestamps */
309  entry->timestampsSet();
310  }
311 
312  /* We ignore cache-control directives as per the Surrogate specification */
313  ignoreCacheControl = true;
314 
315  delete sctusable;
316  }
317  }
318 }
319 
322 {
323  HttpReply const *rep = finalReply();
324  HttpHeader const *hdr = &rep->header;
325  const char *v;
326 #if USE_HTTP_VIOLATIONS
327 
328  const RefreshPattern *R = NULL;
329 
330  /* This strange looking define first looks up the refresh pattern
331  * and then checks if the specified flag is set. The main purpose
332  * of this is to simplify the refresh pattern lookup and USE_HTTP_VIOLATIONS
333  * condition
334  */
335 #define REFRESH_OVERRIDE(flag) \
336  ((R = (R ? R : refreshLimits(entry->mem_obj->storeId()))) , \
337  (R && R->flags.flag))
338 #else
339 #define REFRESH_OVERRIDE(flag) 0
340 #endif
341 
343  return decision.make(ReuseDecision::doNotCacheButShare, "the entry has been released");
344 
345  // RFC 7234 section 4: a cache MUST use the most recent response
346  // (as determined by the Date header field)
347  // TODO: whether such responses could be shareable?
348  if (sawDateGoBack)
349  return decision.make(ReuseDecision::reuseNot, "the response has an older date header");
350 
351  // Check for Surrogate/1.0 protocol conditions
352  // NP: reverse-proxy traffic our parent server has instructed us never to cache
353  if (surrogateNoStore)
354  return decision.make(ReuseDecision::reuseNot, "Surrogate-Control:no-store");
355 
356  // RFC 2616: HTTP/1.1 Cache-Control conditions
357  if (!ignoreCacheControl) {
358  // XXX: check to see if the request headers alone were enough to prevent caching earlier
359  // (ie no-store request header) no need to check those all again here if so.
360  // for now we are not reliably doing that so we waste CPU re-checking request CC
361 
362  // RFC 2616 section 14.9.2 - MUST NOT cache any response with request CC:no-store
364  !REFRESH_OVERRIDE(ignore_no_store))
365  return decision.make(ReuseDecision::reuseNot,
366  "client request Cache-Control:no-store");
367 
368  // NP: request CC:no-cache only means cache READ is forbidden. STORE is permitted.
370  /* TODO: we are allowed to cache when no-cache= has parameters.
371  * Provided we strip away any of the listed headers unless they are revalidated
372  * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
373  * That is a bit tricky for squid right now so we avoid caching entirely.
374  */
375  return decision.make(ReuseDecision::reuseNot,
376  "server reply Cache-Control:no-cache has parameters");
377  }
378 
379  // NP: request CC:private is undefined. We ignore.
380  // NP: other request CC flags are limiters on HIT/MISS. We don't care about here.
381 
382  // RFC 2616 section 14.9.2 - MUST NOT cache any response with CC:no-store
383  if (rep->cache_control && rep->cache_control->hasNoStore() &&
384  !REFRESH_OVERRIDE(ignore_no_store))
385  return decision.make(ReuseDecision::reuseNot,
386  "server reply Cache-Control:no-store");
387 
388  // RFC 2616 section 14.9.1 - MUST NOT cache any response with CC:private in a shared cache like Squid.
389  // CC:private overrides CC:public when both are present in a response.
390  // TODO: add a shared/private cache configuration possibility.
391  if (rep->cache_control &&
392  rep->cache_control->hasPrivate() &&
393  !REFRESH_OVERRIDE(ignore_private)) {
394  /* TODO: we are allowed to cache when private= has parameters.
395  * Provided we strip away any of the listed headers unless they are revalidated
396  * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
397  * That is a bit tricky for squid right now so we avoid caching entirely.
398  */
399  return decision.make(ReuseDecision::reuseNot,
400  "server reply Cache-Control:private");
401  }
402  }
403 
404  // RFC 2068, sec 14.9.4 - MUST NOT cache any response with Authentication UNLESS certain CC controls are present
405  // allow HTTP violations to IGNORE those controls (ie re-block caching Auth)
406  if (request && (request->flags.auth || request->flags.authSent)) {
407  if (!rep->cache_control)
408  return decision.make(ReuseDecision::reuseNot,
409  "authenticated and server reply missing Cache-Control");
410 
411  if (ignoreCacheControl)
412  return decision.make(ReuseDecision::reuseNot,
413  "authenticated and ignoring Cache-Control");
414 
415  bool mayStore = false;
416  // HTTPbis pt6 section 3.2: a response CC:public is present
417  if (rep->cache_control->hasPublic()) {
418  debugs(22, 3, HERE << "Authenticated but server reply Cache-Control:public");
419  mayStore = true;
420 
421  // HTTPbis pt6 section 3.2: a response CC:must-revalidate is present
422  } else if (rep->cache_control->hasMustRevalidate()) {
423  debugs(22, 3, HERE << "Authenticated but server reply Cache-Control:must-revalidate");
424  mayStore = true;
425 
426 #if USE_HTTP_VIOLATIONS
427  // NP: given the must-revalidate exception we should also be able to exempt no-cache.
428  // HTTPbis WG verdict on this is that it is omitted from the spec due to being 'unexpected' by
429  // some. The caching+revalidate is not exactly unsafe though with Squids interpretation of no-cache
430  // (without parameters) as equivalent to must-revalidate in the reply.
431  } else if (rep->cache_control->hasNoCacheWithoutParameters()) {
432  debugs(22, 3, HERE << "Authenticated but server reply Cache-Control:no-cache (equivalent to must-revalidate)");
433  mayStore = true;
434 #endif
435 
436  // HTTPbis pt6 section 3.2: a response CC:s-maxage is present
437  } else if (rep->cache_control->hasSMaxAge()) {
438  debugs(22, 3, HERE << "Authenticated but server reply Cache-Control:s-maxage");
439  mayStore = true;
440  }
441 
442  if (!mayStore)
443  return decision.make(ReuseDecision::reuseNot, "authenticated transaction");
444 
445  // NP: response CC:no-cache is equivalent to CC:must-revalidate,max-age=0. We MAY cache, and do so.
446  // NP: other request CC flags are limiters on HIT/MISS/REFRESH. We don't care about here.
447  }
448 
449  /* HACK: The "multipart/x-mixed-replace" content type is used for
450  * continuous push replies. These are generally dynamic and
451  * probably should not be cachable
452  */
453  if ((v = hdr->getStr(Http::HdrType::CONTENT_TYPE)))
454  if (!strncasecmp(v, "multipart/x-mixed-replace", 25))
455  return decision.make(ReuseDecision::reuseNot, "Content-Type:multipart/x-mixed-replace");
456 
457  // TODO: if possible, provide more specific message for each status code
458  static const char *shareableError = "shareable error status code";
459  static const char *nonShareableError = "non-shareable error status code";
461  const char *statusReason = nonShareableError;
462 
463  switch (rep->sline.status()) {
464 
465  /* There are several situations when a non-cacheable response may be
466  * still shareable (e.g., among collapsed clients). We assume that these
467  * are 3xx and 5xx responses, indicating server problems and some of
468  * 4xx responses, common for all clients with a given cache key (e.g.,
469  * 404 Not Found or 414 URI Too Long). On the other hand, we should not
470  * share non-cacheable client-specific errors, such as 400 Bad Request
471  * or 406 Not Acceptable.
472  */
473 
474  /* Responses that are cacheable */
475 
476  case Http::scOkay:
477 
479 
481 
484 
485  case Http::scGone:
486  /*
487  * Don't cache objects that need to be refreshed on next request,
488  * unless we know how to refresh it.
489  */
490 
491  if (refreshIsCachable(entry) || REFRESH_OVERRIDE(store_stale))
492  decision.make(ReuseDecision::cachePositively, "refresh check returned cacheable");
493  else
494  decision.make(ReuseDecision::doNotCacheButShare, "refresh check returned non-cacheable");
495  break;
496 
497  /* Responses that only are cacheable if the server says so */
498 
499  case Http::scFound:
501  if (rep->date <= 0)
502  decision.make(ReuseDecision::doNotCacheButShare, "Date is missing/invalid");
503  else if (rep->expires > rep->date)
504  decision.make(ReuseDecision::cachePositively, "Expires > Date");
505  else
506  decision.make(ReuseDecision::doNotCacheButShare, "Expires <= Date");
507  break;
508 
509  /* These responses can be negatively cached. Most can also be shared. */
510  case Http::scNoContent:
511  case Http::scUseProxy:
512  case Http::scForbidden:
513  case Http::scNotFound:
515  case Http::scUriTooLong:
518  case Http::scBadGateway:
522  statusAnswer = ReuseDecision::doNotCacheButShare;
523  statusReason = shareableError;
524  // fall through to the actual decision making below
525 
526  case Http::scBadRequest: // no sharing; perhaps the server did not like something specific to this request
527 #if USE_HTTP_VIOLATIONS
528  if (Config.negativeTtl > 0)
529  decision.make(ReuseDecision::cacheNegatively, "Config.negativeTtl > 0");
530  else
531 #endif
532  decision.make(statusAnswer, statusReason);
533  break;
534 
535  /* these responses can never be cached, some
536  of them can be shared though */
537  case Http::scSeeOther:
538  case Http::scNotModified:
543  // TODO: use more specific reason for non-error status codes
544  decision.make(ReuseDecision::doNotCacheButShare, shareableError);
545  break;
546 
547  case Http::scPartialContent: /* Not yet supported. TODO: make shareable for suitable ranges */
549  case Http::scRequestTimeout: // TODO: is this shareable?
550  case Http::scConflict: // TODO: is this shareable?
556  case Http::scLocked: // TODO: is this shareable?
560  case Http::scInvalidHeader: /* Squid header parsing error */
562  decision.make(ReuseDecision::reuseNot, nonShareableError);
563  break;
564 
565  default:
566  /* RFC 2616 section 6.1.1: an unrecognized response MUST NOT be cached. */
567  decision.make(ReuseDecision::reuseNot, "unknown status code");
568  break;
569  }
570 
571  return decision.answer;
572 }
573 
575 static void
577 {
578  static const SBuf asterisk("*");
579  const char *pos = nullptr;
580  const char *item = nullptr;
581  int ilen = 0;
582 
583  while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
584  SBuf name(item, ilen);
585  if (name == asterisk) {
586  vstr = asterisk;
587  break;
588  }
589  name.toLower();
590  if (!vstr.isEmpty())
591  vstr.append(", ", 2);
592  vstr.append(name);
593  String hdr(request.header.getByName(name));
594  const char *value = hdr.termedBuf();
595  if (value) {
596  value = rfc1738_escape_part(value);
597  vstr.append("=\"", 2);
598  vstr.append(value);
599  vstr.append("\"", 1);
600  }
601 
602  hdr.clean();
603  }
604 }
605 
606 /*
607  * For Vary, store the relevant request headers as
608  * virtual headers in the reply
609  * Returns an empty SBuf if the variance cannot be stored
610  */
611 SBuf
613 {
614  SBuf vstr;
615  String vary;
616 
617  vary = reply->header.getList(Http::HdrType::VARY);
618  assembleVaryKey(vary, vstr, *request);
619 
620 #if X_ACCELERATOR_VARY
621  vary.clean();
623  assembleVaryKey(vary, vstr, *request);
624 #endif
625 
626  debugs(11, 3, vstr);
627  return vstr;
628 }
629 
630 void
632 {
633  if (flags.keepalive)
634  if (flags.peering && !flags.tunneling)
636 
637  if (reply->keep_alive) {
638  if (flags.peering && !flags.tunneling)
640 
642  && reply->bodySize(request->method) == -1 && !flags.chunked) {
643  debugs(11, DBG_IMPORTANT, "keepaliveAccounting: Impossible keep-alive header from '" << entry->url() << "'" );
644  // debugs(11, 2, "GOT HTTP REPLY HDR:\n---------\n" << readBuf->content() << "\n----------" );
645  flags.keepalive_broken = true;
646  }
647  }
648 }
649 
650 void
652 {
653  if (reply->date > -1 && flags.toOrigin) {
654  int skew = abs((int)(reply->date - squid_curtime));
655 
656  if (skew > 86400)
657  debugs(11, 3, "" << request->url.host() << "'s clock is skewed by " << skew << " seconds!");
658  }
659 }
660 
667 void
669 {
672  /* NP: all exit points to this function MUST call ctx_exit(ctx) */
673  Ctx ctx = ctx_enter(entry->mem_obj->urlXXX());
674 
675  debugs(11, 3, "processReplyHeader: key '" << entry->getMD5Text() << "'");
676 
678 
679  if (!inBuf.length()) {
680  ctx_exit(ctx);
681  return;
682  }
683 
684  /* Attempt to parse the first line; this will define where the protocol, status, reason-phrase and header begin */
685  {
686  if (hp == NULL)
688 
689  bool parsedOk = hp->parse(inBuf);
690 
691  // sync the buffers after parsing.
692  inBuf = hp->remaining();
693 
694  if (hp->needsMoreData()) {
695  if (eof) { // no more data coming
696  /* Bug 2879: Replies may terminate with \r\n then EOF instead of \r\n\r\n.
697  * We also may receive truncated responses.
698  * Ensure here that we have at minimum two \r\n when EOF is seen.
699  */
700  inBuf.append("\r\n\r\n", 4);
701  // retry the parse
702  parsedOk = hp->parse(inBuf);
703  // sync the buffers after parsing.
704  inBuf = hp->remaining();
705  } else {
706  debugs(33, 5, "Incomplete response, waiting for end of response headers");
707  ctx_exit(ctx);
708  return;
709  }
710  }
711 
712  if (!parsedOk) {
713  // unrecoverable parsing error
714  // TODO: Use Raw! XXX: inBuf no longer has the [beginning of the] malformed header.
715  debugs(11, 3, "Non-HTTP-compliant header:\n---------\n" << inBuf << "\n----------");
716  flags.headers_parsed = true;
717  HttpReply *newrep = new HttpReply;
718  newrep->sline.set(Http::ProtocolVersion(), hp->parseStatusCode);
719  setVirginReply(newrep);
720  ctx_exit(ctx);
721  return;
722  }
723  }
724 
725  /* We know the whole response is in parser now */
726  debugs(11, 2, "HTTP Server " << serverConnection);
727  debugs(11, 2, "HTTP Server RESPONSE:\n---------\n" <<
728  hp->messageProtocol() << " " << hp->messageStatus() << " " << hp->reasonPhrase() << "\n" <<
729  hp->mimeHeader() <<
730  "----------");
731 
732  // reset payload tracking to begin after message headers
734 
735  HttpReply *newrep = new HttpReply;
736  // XXX: RFC 7230 indicates we MAY ignore the reason phrase,
737  // and use an empty string on unknown status.
738  // We do that now to avoid performance regression from using SBuf::c_str()
739  newrep->sline.set(Http::ProtocolVersion(1,1), hp->messageStatus() /* , hp->reasonPhrase() */);
740  newrep->sline.protocol = newrep->sline.version.protocol = hp->messageProtocol().protocol;
741  newrep->sline.version.major = hp->messageProtocol().major;
742  newrep->sline.version.minor = hp->messageProtocol().minor;
743 
744  // parse headers
745  if (!newrep->parseHeader(*hp)) {
746  // XXX: when Http::ProtocolVersion is a function, remove this hack. just set with messageProtocol()
748  newrep->sline.version.protocol = hp->messageProtocol().protocol;
749  newrep->sline.version.major = hp->messageProtocol().major;
750  newrep->sline.version.minor = hp->messageProtocol().minor;
751  debugs(11, 2, "error parsing response headers mime block");
752  }
753 
754  // done with Parser, now process using the HttpReply
755  hp = NULL;
756 
758 
759  newrep->removeStaleWarnings();
760 
761  if (newrep->sline.protocol == AnyP::PROTO_HTTP && Http::Is1xx(newrep->sline.status())) {
762  handle1xx(newrep);
763  ctx_exit(ctx);
764  return;
765  }
766 
767  flags.chunked = false;
768  if (newrep->sline.protocol == AnyP::PROTO_HTTP && newrep->header.chunked()) {
769  flags.chunked = true;
771  }
772 
775 
776  HttpReply *vrep = setVirginReply(newrep);
777  flags.headers_parsed = true;
778 
779  keepaliveAccounting(vrep);
780 
781  checkDateSkew(vrep);
782 
784 
786 
787  ctx_exit(ctx);
788 }
789 
791 void
793 {
794  if (fwd->al)
795  fwd->al->reply = reply;
796 
797  HttpReply::Pointer msg(reply); // will destroy reply if unused
798 
799  // one 1xx at a time: we must not be called while waiting for previous 1xx
801  flags.handling1xx = true;
802 
803  const auto statusCode = reply->sline.status();
804 
805  // drop1xx() needs to handle HTTP 101 (Switching Protocols) responses
806  // specially because they indicate that the server has stopped speaking HTTP
809 
810  if (statusCode == Http::scContinue && request->forcedBodyContinuation)
811  return drop1xx("we have sent it already");
812 
813  if (!request->canHandle1xx())
814  return drop1xx("the client does not support it");
815 
816 #if USE_HTTP_VIOLATIONS
817  // check whether the 1xx response forwarding is allowed by squid.conf
818  if (Config.accessList.reply) {
820  ch.al = fwd->al;
821  ch.reply = reply;
822  ch.syncAle(originalRequest().getRaw(), nullptr);
823  HTTPMSGLOCK(ch.reply);
824  if (!ch.fastCheck().allowed()) // TODO: support slow lookups?
825  return drop1xx("http_reply_access blocked it");
826  }
827 #endif // USE_HTTP_VIOLATIONS
828 
830  if (const auto reason = blockSwitchingProtocols(*reply))
831  return drop1xx(reason);
832  }
833 
834  debugs(11, 2, HERE << "forwarding 1xx to client");
835 
836  // the Sink will use this to call us back after writing 1xx to the client
837  typedef NullaryMemFunT<HttpStateData> CbDialer;
838  const AsyncCall::Pointer cb = JobCallback(11, 3, CbDialer, this,
842  // If the call is not fired, then the Sink is gone, and HttpStateData
843  // will terminate due to an aborted store entry or another similar error.
844  // If we get stuck, it is not handle1xx fault if we could get stuck
845  // for similar reasons without a 1xx response.
846 }
847 
850 void
851 HttpStateData::drop1xx(const char *reason)
852 {
854  debugs(11, 2, "bad 101 because " << reason);
855  const auto err = new ErrorState(ERR_INVALID_RESP, Http::scBadGateway, request.getRaw(), fwd->al);
856  fwd->fail(err);
857  closeServer();
858  mustStop("prohibited HTTP/101 response");
859  return;
860  }
861 
862  debugs(11, 2, "ignoring 1xx because " << reason);
863  proceedAfter1xx();
864 }
865 
868 const char *
870 {
871  if (!upgradeHeaderOut)
872  return "Squid offered no Upgrade at all, but server switched to a tunnel";
873 
874  // See RFC 7230 section 6.7 for the corresponding MUSTs
875 
876  if (!reply.header.has(Http::HdrType::UPGRADE))
877  return "server did not send an Upgrade header field";
878 
879  if (!reply.header.hasListMember(Http::HdrType::CONNECTION, "upgrade", ','))
880  return "server did not send 'Connection: upgrade'";
881 
882  const auto acceptedProtos = reply.header.getList(Http::HdrType::UPGRADE);
883  const char *pos = nullptr;
884  const char *accepted = nullptr;
885  int acceptedLen = 0;
886  while (strListGetItem(&acceptedProtos, ',', &accepted, &acceptedLen, &pos)) {
887  debugs(11, 5, "server accepted at least" << Raw(nullptr, accepted, acceptedLen));
888  return nullptr; // OK: let the client validate server's selection
889  }
890 
891  return "server sent an essentially empty Upgrade header field";
892 }
893 
895 void
897 {
899 
901  // pass server connection ownership to request->clientConnectionManager
904  AsyncCall::Pointer call = asyncCall(11, 3, "ConnStateData::noteTakeServerConnectionControl",
907  ScheduleCallHere(call);
910  closeHandler = nullptr;
911  serverConnection = nullptr;
912  doneWithFwd = "switched protocols";
914  return;
915  }
916 
917  debugs(11, 2, "continuing with " << payloadSeen << " bytes in buffer after 1xx");
919 }
920 
924 bool
926 {
927  if (!_peer)
928  return true;
929 
930  // we are talking "through" rather than "to" our _peer
931  if (flags.tunneling)
932  return true;
933 
934  /*If this peer does not support connection pinning (authenticated
935  connections) return false
936  */
937  if (!_peer->connection_auth)
938  return false;
939 
940  const auto &rep = entry->mem().freshestReply();
941 
942  /*The peer supports connection pinning and the http reply status
943  is not unauthorized, so the related connection can be pinned
944  */
945  if (rep.sline.status() != Http::scUnauthorized)
946  return true;
947 
948  /*The server respond with Http::scUnauthorized and the peer configured
949  with "connection-auth=on" we know that the peer supports pinned
950  connections
951  */
952  if (_peer->connection_auth == 1)
953  return true;
954 
955  /*At this point peer has configured with "connection-auth=auto"
956  parameter so we need some extra checks to decide if we are going
957  to allow pinned connections or not
958  */
959 
960  /*if the peer configured with originserver just allow connection
961  pinning (squid 2.6 behaviour)
962  */
964  return true;
965 
966  /*if the connections it is already pinned it is OK*/
967  if (request->flags.pinned)
968  return true;
969 
970  /*Allow pinned connections only if the Proxy-support header exists in
971  reply and has in its list the "Session-Based-Authentication"
972  which means that the peer supports connection pinning.
973  */
974  if (rep.header.hasListMember(Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication", ','))
975  return true;
976 
977  return false;
978 }
979 
980 // Called when we parsed (and possibly adapted) the headers but
981 // had not starting storing (a.k.a., sending) the body yet.
982 void
984 {
986 
987  Ctx ctx = ctx_enter(entry->mem_obj->urlXXX());
988  HttpReply *rep = finalReply();
989  const Http::StatusCode statusCode = rep->sline.status();
990 
991  entry->timestampsSet();
992 
993  /* Check if object is cacheable or not based on reply code */
994  debugs(11, 3, "HTTP CODE: " << statusCode);
995 
996  if (StoreEntry *oldEntry = findPreviouslyCachedEntry(entry)) {
997  oldEntry->lock("HttpStateData::haveParsedReplyHeaders");
998  sawDateGoBack = rep->olderThan(oldEntry->hasFreshestReply());
999  oldEntry->unlock("HttpStateData::haveParsedReplyHeaders");
1000  }
1001 
1004 
1005  bool varyFailure = false;
1006  if (rep->header.has(Http::HdrType::VARY)
1007 #if X_ACCELERATOR_VARY
1009 #endif
1010  ) {
1011  const SBuf vary(httpMakeVaryMark(request.getRaw(), rep));
1012 
1013  if (vary.isEmpty()) {
1014  // TODO: check whether such responses are shareable.
1015  // Do not share for now.
1016  entry->makePrivate(false);
1017  if (fwd->reforwardableStatus(rep->sline.status()))
1019  varyFailure = true;
1020  } else {
1021  entry->mem_obj->vary_headers = vary;
1022 
1023  // RFC 7231 section 7.1.4
1024  // Vary:* can be cached, but has mandatory revalidation
1025  static const SBuf asterisk("*");
1026  if (vary == asterisk)
1028  }
1029  }
1030 
1031  if (!varyFailure) {
1032  /*
1033  * If its not a reply that we will re-forward, then
1034  * allow the client to get it.
1035  */
1036  if (fwd->reforwardableStatus(rep->sline.status()))
1038 
1039  ReuseDecision decision(entry, statusCode);
1040 
1041  switch (reusableReply(decision)) {
1042 
1044  entry->makePrivate(false);
1045  break;
1046 
1048  if (!entry->makePublic()) {
1049  decision.make(ReuseDecision::doNotCacheButShare, "public key creation error");
1050  entry->makePrivate(true);
1051  }
1052  break;
1053 
1055  if (!entry->cacheNegatively()) {
1056  decision.make(ReuseDecision::doNotCacheButShare, "public key creation error");
1057  entry->makePrivate(true);
1058  }
1059  break;
1060 
1062  entry->makePrivate(true);
1063  break;
1064 
1065  default:
1066  assert(0);
1067  break;
1068  }
1069  debugs(11, 3, "decided: " << decision);
1070  }
1071 
1072  if (!ignoreCacheControl) {
1073  if (rep->cache_control) {
1074  // We are required to revalidate on many conditions.
1075  // For security reasons we do so even if storage was caused by refresh_pattern ignore-* option
1076 
1077  // CC:must-revalidate or CC:proxy-revalidate
1078  const bool ccMustRevalidate = (rep->cache_control->hasProxyRevalidate() || rep->cache_control->hasMustRevalidate());
1079 
1080  // CC:no-cache (only if there are no parameters)
1081  const bool ccNoCacheNoParams = rep->cache_control->hasNoCacheWithoutParameters();
1082 
1083  // CC:s-maxage=N
1084  const bool ccSMaxAge = rep->cache_control->hasSMaxAge();
1085 
1086  // CC:private (yes, these can sometimes be stored)
1087  const bool ccPrivate = rep->cache_control->hasPrivate();
1088 
1089  if (ccNoCacheNoParams || ccPrivate)
1091  else if (ccMustRevalidate || ccSMaxAge)
1093  }
1094 #if USE_HTTP_VIOLATIONS // response header Pragma::no-cache is undefined in HTTP
1095  else {
1096  // Expensive calculation. So only do it IF the CC: header is not present.
1097 
1098  /* HACK: Pragma: no-cache in _replies_ is not documented in HTTP,
1099  * but servers like "Active Imaging Webcast/2.0" sure do use it */
1100  if (rep->header.has(Http::HdrType::PRAGMA) &&
1101  rep->header.hasListMember(Http::HdrType::PRAGMA,"no-cache",','))
1103  }
1104 #endif
1105  }
1106 
1107 #if HEADERS_LOG
1108  headersLog(1, 0, request->method, rep);
1109 
1110 #endif
1111 
1112  ctx_exit(ctx);
1113 }
1114 
1117 {
1118  const HttpReply *rep = virginReply();
1122  static SBuf close("close", 5);
1123  if (httpHeaderHasConnDir(&rep->header, close))
1125 
1130  if (!flags.keepalive)
1132 
1137  if (!flags.request_sent) {
1138  debugs(11, 2, "Request not yet fully sent " << request->method << ' ' << entry->url());
1140  }
1141 
1154  if (!rep->keep_alive)
1156 
1157  return COMPLETE_PERSISTENT_MSG;
1158 }
1159 
1162 {
1163  debugs(11, 3, HERE << serverConnection << " eof=" << eof);
1164  if (eof) // already reached EOF
1166 
1167  /* If server fd is closing (but we have not been notified yet), stop Comm
1168  I/O to avoid assertions. TODO: Change Comm API to handle callers that
1169  want more I/O after async closing (usually initiated by others). */
1170  // XXX: add canReceive or s/canSend/canTalkToServer/
1173 
1179  if (lastChunk && flags.chunked)
1180  return statusIfComplete();
1181 
1182  const HttpReply *vrep = virginReply();
1183  debugs(11, 5, "persistentConnStatus: content_length=" << vrep->content_length);
1184 
1185  const int64_t clen = vrep->bodySize(request->method);
1186 
1187  debugs(11, 5, "persistentConnStatus: clen=" << clen);
1188 
1189  /* If the body size is unknown we must wait for EOF */
1190  if (clen < 0)
1191  return INCOMPLETE_MSG;
1192 
1195  if (clen > 0) {
1196  debugs(11,5, "payloadSeen=" << payloadSeen << " content_length=" << vrep->content_length);
1197 
1198  if (payloadSeen < vrep->content_length)
1199  return INCOMPLETE_MSG;
1200 
1201  if (payloadTruncated > 0) // already read more than needed
1202  return COMPLETE_NONPERSISTENT_MSG; // disable pconns
1203  }
1204 
1207  return statusIfComplete();
1208 }
1209 
1210 static void
1211 readDelayed(void *context, CommRead const &)
1212 {
1213  HttpStateData *state = static_cast<HttpStateData*>(context);
1214  state->flags.do_next_read = true;
1215  state->maybeReadVirginBody();
1216 }
1217 
1218 void
1220 {
1221  Must(!flags.do_next_read); // XXX: should have been set false by mayReadVirginBody()
1222  flags.do_next_read = false;
1223 
1224  debugs(11, 5, io.conn);
1225 
1226  // Bail out early on Comm::ERR_CLOSING - close handlers will tidy up for us
1227  if (io.flag == Comm::ERR_CLOSING) {
1228  debugs(11, 3, "http socket closing");
1229  return;
1230  }
1231 
1233  abortTransaction("store entry aborted while reading reply");
1234  return;
1235  }
1236 
1238  Must(io.conn->fd == serverConnection->fd);
1239 
1240  /*
1241  * Don't reset the timeout value here. The value should be
1242  * counting Config.Timeout.request and applies to the request
1243  * as a whole, not individual read() calls.
1244  * Plus, it breaks our lame *HalfClosed() detection
1245  */
1246 
1248  CommIoCbParams rd(this); // will be expanded with ReadNow results
1249  rd.conn = io.conn;
1251 
1252  if (rd.size <= 0) {
1253  assert(entry->mem_obj);
1254  AsyncCall::Pointer nilCall;
1255  entry->mem_obj->delayRead(DeferredRead(readDelayed, this, CommRead(io.conn, NULL, 0, nilCall)));
1256  return;
1257  }
1258 
1259  switch (Comm::ReadNow(rd, inBuf)) {
1260  case Comm::INPROGRESS:
1261  if (inBuf.isEmpty())
1262  debugs(33, 2, io.conn << ": no data to process, " << xstrerr(rd.xerrno));
1263  flags.do_next_read = true;
1265  return;
1266 
1267  case Comm::OK:
1268  {
1269  payloadSeen += rd.size;
1270 #if USE_DELAY_POOLS
1271  DelayId delayId = entry->mem_obj->mostBytesAllowed();
1272  delayId.bytesIn(rd.size);
1273 #endif
1274 
1275  statCounter.server.all.kbytes_in += rd.size;
1276  statCounter.server.http.kbytes_in += rd.size;
1277  ++ IOStats.Http.reads;
1278 
1279  int bin = 0;
1280  for (int clen = rd.size - 1; clen; ++bin)
1281  clen >>= 1;
1282 
1283  ++ IOStats.Http.read_hist[bin];
1284 
1286  }
1287 
1288  /* Continue to process previously read data */
1289  break;
1290 
1291  case Comm::ENDFILE: // close detected by 0-byte read
1292  eof = 1;
1293  flags.do_next_read = false;
1294 
1295  /* Continue to process previously read data */
1296  break;
1297 
1298  // case Comm::COMM_ERROR:
1299  default: // no other flags should ever occur
1300  debugs(11, 2, io.conn << ": read failure: " << xstrerr(rd.xerrno));
1301  const auto err = new ErrorState(ERR_READ_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1302  err->xerrno = rd.xerrno;
1303  fwd->fail(err);
1304  flags.do_next_read = false;
1305  closeServer();
1306  mustStop("HttpStateData::readReply");
1307  return;
1308  }
1309 
1310  /* Process next response from buffer */
1311  processReply();
1312 }
1313 
1316 void
1318 {
1319 
1320  if (flags.handling1xx) { // we came back after handling a 1xx response
1321  debugs(11, 5, HERE << "done with 1xx handling");
1322  flags.handling1xx = false;
1324  }
1325 
1326  if (!flags.headers_parsed) { // have not parsed headers yet?
1327  PROF_start(HttpStateData_processReplyHeader);
1329  PROF_stop(HttpStateData_processReplyHeader);
1330 
1331  if (!continueAfterParsingHeader()) // parsing error or need more data
1332  return; // TODO: send errors to ICAP
1333 
1334  adaptOrFinalizeReply(); // may write to, abort, or "close" the entry
1335  }
1336 
1337  // kick more reads if needed and/or process the response body, if any
1338  PROF_start(HttpStateData_processReplyBody);
1339  processReplyBody(); // may call serverComplete()
1340  PROF_stop(HttpStateData_processReplyBody);
1341 }
1342 
1346 bool
1348 {
1349  if (flags.handling1xx) {
1350  debugs(11, 5, HERE << "wait for 1xx handling");
1352  return false;
1353  }
1354 
1355  if (!flags.headers_parsed && !eof) {
1356  debugs(11, 9, "needs more at " << inBuf.length());
1357  flags.do_next_read = true;
1361  maybeReadVirginBody(); // schedules all kinds of reads; TODO: rename
1362  return false;
1363  }
1364 
1368 
1369  if (flags.headers_parsed) { // parsed headers, possibly with errors
1370  // check for header parsing errors
1371  if (HttpReply *vrep = virginReply()) {
1372  const Http::StatusCode s = vrep->sline.status();
1373  const AnyP::ProtocolVersion &v = vrep->sline.version;
1374  if (s == Http::scInvalidHeader && v != Http::ProtocolVersion(0,9)) {
1375  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Bad header encountered from " << entry->url() << " AKA " << request->url);
1377  } else if (s == Http::scHeaderTooLarge) {
1378  fwd->dontRetry(true);
1379  error = ERR_TOO_BIG;
1380  } else if (vrep->header.conflictingContentLength()) {
1381  fwd->dontRetry(true);
1383  } else if (vrep->header.unsupportedTe()) {
1384  fwd->dontRetry(true);
1386  } else {
1387  return true; // done parsing, got reply, and no error
1388  }
1389  } else {
1390  // parsed headers but got no reply
1391  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: No reply at all for " << entry->url() << " AKA " << request->url);
1393  }
1394  } else {
1395  assert(eof);
1396  if (inBuf.length()) {
1398  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry->url() << " AKA " << request->url);
1399  } else {
1401  debugs(11, (request->flags.accelerated?DBG_IMPORTANT:2), "WARNING: HTTP: Invalid Response: No object data received for " << entry->url() << " AKA " << request->url);
1402  }
1403  }
1404 
1405  assert(error != ERR_NONE);
1406  entry->reset();
1408  flags.do_next_read = false;
1409  closeServer();
1410  mustStop("HttpStateData::continueAfterParsingHeader");
1411  return false; // quit on error
1412 }
1413 
1416 void
1418 {
1420 
1421  HttpReply *vrep = virginReply();
1422  int64_t clen = -1;
1423  if (!vrep->expectingBody(request->method, clen) || clen < 0)
1424  return; // no body or a body of unknown size, including chunked
1425 
1426  if (payloadSeen - payloadTruncated <= clen)
1427  return; // we did not read too much or already took care of the extras
1428 
1429  if (const int64_t extras = payloadSeen - payloadTruncated - clen) {
1430  // server sent more that the advertised content length
1431  debugs(11, 5, "payloadSeen=" << payloadSeen <<
1432  " clen=" << clen << '/' << vrep->content_length <<
1433  " truncated=" << payloadTruncated << '+' << extras);
1434 
1435  inBuf.chop(0, inBuf.length() - extras);
1436  payloadTruncated += extras;
1437  }
1438 }
1439 
1444 void
1446 {
1447  truncateVirginBody(); // if needed
1448  const char *data = inBuf.rawContent();
1449  int len = inBuf.length();
1451  inBuf.consume(len);
1452 }
1453 
1454 bool
1456 {
1457  const char *data = NULL;
1458  int len;
1459  bool wasThereAnException = false;
1460  assert(flags.chunked);
1463  MemBuf decodedData;
1464  decodedData.init();
1465  httpChunkDecoder->setPayloadBuffer(&decodedData);
1466  const bool doneParsing = httpChunkDecoder->parse(inBuf);
1467  inBuf = httpChunkDecoder->remaining(); // sync buffers after parse
1468  len = decodedData.contentSize();
1469  data=decodedData.content();
1471  if (doneParsing) {
1472  lastChunk = 1;
1473  flags.do_next_read = false;
1474  }
1475  SQUID_EXIT_THROWING_CODE(wasThereAnException);
1476  return wasThereAnException;
1477 }
1478 
1486 void
1488 {
1489  if (!flags.headers_parsed) {
1490  flags.do_next_read = true;
1492  return;
1493  }
1494 
1495 #if USE_ADAPTATION
1496  debugs(11,5, HERE << "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
1498  return;
1499 
1500 #endif
1501 
1502  /*
1503  * At this point the reply headers have been parsed and consumed.
1504  * That means header content has been removed from readBuf and
1505  * it contains only body data.
1506  */
1507  if (entry->isAccepting()) {
1508  if (flags.chunked) {
1509  if (!decodeAndWriteReplyBody()) {
1510  flags.do_next_read = false;
1511  serverComplete();
1512  return;
1513  }
1514  } else
1515  writeReplyBody();
1516  }
1517 
1518  // storing/sending methods like earlier adaptOrFinalizeReply() or
1519  // above writeReplyBody() may release/abort the store entry.
1521  // TODO: In some cases (e.g., 304), we should keep persistent conn open.
1522  // Detect end-of-reply (and, hence, pool our idle pconn) earlier (ASAP).
1523  abortTransaction("store entry aborted while storing reply");
1524  return;
1525  } else
1526  switch (persistentConnStatus()) {
1527  case INCOMPLETE_MSG: {
1528  debugs(11, 5, "processReplyBody: INCOMPLETE_MSG from " << serverConnection);
1529  /* Wait for more data or EOF condition */
1531  if (flags.keepalive_broken) {
1533  } else {
1535  }
1536 
1537  flags.do_next_read = true;
1538  }
1539  break;
1540 
1541  case COMPLETE_PERSISTENT_MSG: {
1542  debugs(11, 5, "processReplyBody: COMPLETE_PERSISTENT_MSG from " << serverConnection);
1543 
1544  // TODO: Remove serverConnectionSaved but preserve exception safety.
1545 
1547  flags.do_next_read = false;
1548 
1550  closeHandler = NULL;
1551 
1552  Ip::Address client_addr; // XXX: Remove as unused. Why was it added?
1554  client_addr = request->client_addr;
1555 
1556  auto serverConnectionSaved = serverConnection;
1558  serverConnection = nullptr;
1559 
1560  bool ispinned = false; // TODO: Rename to isOrShouldBePinned
1561  if (request->flags.pinned) {
1562  ispinned = true;
1563  } else if (request->flags.connectionAuth && request->flags.authSent) {
1564  ispinned = true;
1565  }
1566 
1567  if (ispinned) {
1568  if (request->clientConnectionManager.valid()) {
1570  ConnStateData,
1571  notePinnedConnectionBecameIdle,
1572  ConnStateData::PinnedIdleContext(serverConnectionSaved, request));
1573  } else {
1574  // must not pool/share ispinned connections, even orphaned ones
1575  serverConnectionSaved->close();
1576  }
1577  } else {
1578  fwdPconnPool->push(serverConnectionSaved, request->url.host());
1579  }
1580 
1581  serverComplete();
1582  return;
1583  }
1584 
1586  debugs(11, 5, "processReplyBody: COMPLETE_NONPERSISTENT_MSG from " << serverConnection);
1587  if (flags.chunked && !lastChunk)
1588  entry->lengthWentBad("missing last-chunk");
1589 
1590  serverComplete();
1591  return;
1592  }
1593 
1595 }
1596 
1597 bool
1599 {
1600  // TODO: Be more precise here. For example, if/when reading trailer, we may
1601  // not be doneWithServer() yet, but we should return false. Similarly, we
1602  // could still be writing the request body after receiving the whole reply.
1603  return !doneWithServer();
1604 }
1605 
1606 void
1608 {
1609  // too late to read
1611  return;
1612 
1613  if (!maybeMakeSpaceAvailable(false))
1614  return;
1615 
1616  // XXX: get rid of the do_next_read flag
1617  // check for the proper reasons preventing read(2)
1618  if (!flags.do_next_read)
1619  return;
1620 
1621  flags.do_next_read = false;
1622 
1623  // must not already be waiting for read(2) ...
1625 
1626  // wait for read(2) to be possible.
1628  AsyncCall::Pointer call = JobCallback(11, 5, Dialer, this, HttpStateData::readReply);
1630 }
1631 
1632 bool
1634 {
1635  // how much we are allowed to buffer
1636  const int limitBuffer = (flags.headers_parsed ? Config.readAheadGap : Config.maxReplyHeaderSize);
1637 
1638  if (limitBuffer < 0 || inBuf.length() >= (SBuf::size_type)limitBuffer) {
1639  // when buffer is at or over limit already
1640  debugs(11, 7, "will not read up to " << limitBuffer << ". buffer has (" << inBuf.length() << "/" << inBuf.spaceSize() << ") from " << serverConnection);
1641  debugs(11, DBG_DATA, "buffer has {" << inBuf << "}");
1642  // Process next response from buffer
1643  processReply();
1644  return false;
1645  }
1646 
1647  // how much we want to read
1648  const size_t read_size = calcBufferSpaceToReserve(inBuf.spaceSize(), (limitBuffer - inBuf.length()));
1649 
1650  if (!read_size) {
1651  debugs(11, 7, "will not read up to " << read_size << " into buffer (" << inBuf.length() << "/" << inBuf.spaceSize() << ") from " << serverConnection);
1652  return false;
1653  }
1654 
1655  // just report whether we could grow or not, do not actually do it
1656  if (doGrow)
1657  return (read_size >= 2);
1658 
1659  // we may need to grow the buffer
1660  inBuf.reserveSpace(read_size);
1661  debugs(11, 8, (!flags.do_next_read ? "will not" : "may") <<
1662  " read up to " << read_size << " bytes info buf(" << inBuf.length() << "/" << inBuf.spaceSize() <<
1663  ") from " << serverConnection);
1664 
1665  return (inBuf.spaceSize() >= 2); // only read if there is 1+ bytes of space available
1666 }
1667 
1669 void
1671 {
1672  debugs(11, 5, HERE << serverConnection << ": size " << io.size << ": errflag " << io.flag << ".");
1673 #if URL_CHECKSUM_DEBUG
1674 
1675  entry->mem_obj->checkUrlChecksum();
1676 #endif
1677 
1678  // XXX: Keep in sync with Client::sentRequestBody().
1679  // TODO: Extract common parts.
1680 
1681  if (io.size > 0) {
1682  fd_bytes(io.fd, io.size, FD_WRITE);
1683  statCounter.server.all.kbytes_out += io.size;
1684  statCounter.server.http.kbytes_out += io.size;
1685  }
1686 
1687  if (io.flag == Comm::ERR_CLOSING)
1688  return;
1689 
1690  // both successful and failed writes affect response times
1692 
1693  if (io.flag) {
1694  const auto err = new ErrorState(ERR_WRITE_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1695  err->xerrno = io.xerrno;
1696  fwd->fail(err);
1697  closeServer();
1698  mustStop("HttpStateData::wroteLast");
1699  return;
1700  }
1701 
1702  sendComplete();
1703 }
1704 
1706 void
1708 {
1709  /*
1710  * Set the read timeout here because it hasn't been set yet.
1711  * We only set the read timeout after the request has been
1712  * fully written to the peer. If we start the timeout
1713  * after connection establishment, then we are likely to hit
1714  * the timeout for POST/PUT requests that have very large
1715  * request bodies.
1716  */
1718  AsyncCall::Pointer timeoutCall = JobCallback(11, 5,
1719  TimeoutDialer, this, HttpStateData::httpTimeout);
1720 
1722  flags.request_sent = true;
1723 }
1724 
1725 void
1727 {
1728  debugs(11,5, HERE << "closing HTTP server " << serverConnection << " this " << this);
1729 
1733  closeHandler = NULL;
1735  }
1736 }
1737 
1738 bool
1740 {
1742 }
1743 
1744 /*
1745  * Fixup authentication request headers for special cases
1746  */
1747 static void
1749 {
1750  /* Nothing to do unless we are forwarding to a peer */
1751  if (!flags.peering)
1752  return;
1753 
1754  // This request is going "through" rather than "to" our _peer.
1755  if (flags.tunneling)
1756  return;
1757 
1758  /* Needs to be explicitly enabled */
1759  if (!request->peer_login)
1760  return;
1761 
1763  /* Maybe already dealt with? */
1764  if (hdr_out->has(header))
1765  return;
1766 
1767  /* Nothing to do here for PASSTHRU */
1768  if (strcmp(request->peer_login, "PASSTHRU") == 0)
1769  return;
1770 
1771  // Dangerous and undocumented PROXYPASS is a single-signon to servers with
1772  // the proxy password. Only Basic Authentication can work this way. This
1773  // statement forwards a "basic" Proxy-Authorization value from our client
1774  // to an originserver peer. Other PROXYPASS cases are handled lower.
1775  if (flags.toOrigin &&
1776  strcmp(request->peer_login, "PROXYPASS") == 0 &&
1778 
1779  const char *auth = hdr_in->getStr(Http::HdrType::PROXY_AUTHORIZATION);
1780 
1781  if (auth && strncasecmp(auth, "basic ", 6) == 0) {
1782  hdr_out->putStr(header, auth);
1783  return;
1784  }
1785  }
1786 
1787  char loginbuf[base64_encode_len(MAX_LOGIN_SZ)];
1788  size_t blen;
1789  struct base64_encode_ctx ctx;
1790  base64_encode_init(&ctx);
1791 
1792  /* Special mode to pass the username to the upstream cache */
1793  if (*request->peer_login == '*') {
1794  const char *username = "-";
1795 
1796  if (request->extacl_user.size())
1797  username = request->extacl_user.termedBuf();
1798 #if USE_AUTH
1799  else if (request->auth_user_request != NULL)
1800  username = request->auth_user_request->username();
1801 #endif
1802 
1803  blen = base64_encode_update(&ctx, loginbuf, strlen(username), reinterpret_cast<const uint8_t*>(username));
1804  blen += base64_encode_update(&ctx, loginbuf+blen, strlen(request->peer_login +1), reinterpret_cast<const uint8_t*>(request->peer_login +1));
1805  blen += base64_encode_final(&ctx, loginbuf+blen);
1806  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1807  return;
1808  }
1809 
1810  /* external_acl provided credentials */
1811  if (request->extacl_user.size() && request->extacl_passwd.size() &&
1812  (strcmp(request->peer_login, "PASS") == 0 ||
1813  strcmp(request->peer_login, "PROXYPASS") == 0)) {
1814 
1815  blen = base64_encode_update(&ctx, loginbuf, request->extacl_user.size(), reinterpret_cast<const uint8_t*>(request->extacl_user.rawBuf()));
1816  blen += base64_encode_update(&ctx, loginbuf+blen, 1, reinterpret_cast<const uint8_t*>(":"));
1817  blen += base64_encode_update(&ctx, loginbuf+blen, request->extacl_passwd.size(), reinterpret_cast<const uint8_t*>(request->extacl_passwd.rawBuf()));
1818  blen += base64_encode_final(&ctx, loginbuf+blen);
1819  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1820  return;
1821  }
1822  // if no external user credentials are available to fake authentication with PASS acts like PASSTHRU
1823  if (strcmp(request->peer_login, "PASS") == 0)
1824  return;
1825 
1826  /* Kerberos login to peer */
1827 #if HAVE_AUTH_MODULE_NEGOTIATE && HAVE_KRB5 && HAVE_GSSAPI
1828  if (strncmp(request->peer_login, "NEGOTIATE",strlen("NEGOTIATE")) == 0) {
1829  char *Token=NULL;
1830  char *PrincipalName=NULL,*p;
1831  int negotiate_flags = 0;
1832 
1833  if ((p=strchr(request->peer_login,':')) != NULL ) {
1834  PrincipalName=++p;
1835  }
1836  if (request->flags.auth_no_keytab) {
1837  negotiate_flags |= PEER_PROXY_NEGOTIATE_NOKEYTAB;
1838  }
1839  Token = peer_proxy_negotiate_auth(PrincipalName, request->peer_host, negotiate_flags);
1840  if (Token) {
1841  httpHeaderPutStrf(hdr_out, header, "Negotiate %s",Token);
1842  }
1843  return;
1844  }
1845 #endif /* HAVE_KRB5 && HAVE_GSSAPI */
1846 
1847  blen = base64_encode_update(&ctx, loginbuf, strlen(request->peer_login), reinterpret_cast<const uint8_t*>(request->peer_login));
1848  blen += base64_encode_final(&ctx, loginbuf+blen);
1849  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1850  return;
1851 }
1852 
1853 /*
1854  * build request headers and append them to a given MemBuf
1855  * used by buildRequestPrefix()
1856  * note: initialised the HttpHeader, the caller is responsible for Clean()-ing
1857  */
1858 void
1860  StoreEntry * entry,
1861  const AccessLogEntryPointer &al,
1862  HttpHeader * hdr_out,
1863  const Http::StateFlags &flags)
1864 {
1865  /* building buffer for complex strings */
1866 #define BBUF_SZ (MAX_URL+32)
1867  LOCAL_ARRAY(char, bbuf, BBUF_SZ);
1868  LOCAL_ARRAY(char, ntoabuf, MAX_IPSTRLEN);
1869  const HttpHeader *hdr_in = &request->header;
1870  const HttpHeaderEntry *e = NULL;
1872  assert (hdr_out->owner == hoRequest);
1873 
1874  /* use our IMS header if the cached entry has Last-Modified time */
1875  if (request->lastmod > -1)
1877 
1878  // Add our own If-None-Match field if the cached entry has a strong ETag.
1879  // copyOneHeaderFromClientsideRequestToUpstreamRequest() adds client ones.
1880  if (request->etag.size() > 0) {
1882  request->etag.termedBuf()));
1883  }
1884 
1885  bool we_do_ranges = decideIfWeDoRanges (request);
1886 
1887  String strConnection (hdr_in->getList(Http::HdrType::CONNECTION));
1888 
1889  while ((e = hdr_in->getEntry(&pos)))
1890  copyOneHeaderFromClientsideRequestToUpstreamRequest(e, strConnection, request, hdr_out, we_do_ranges, flags);
1891 
1892  /* Abstraction break: We should interpret multipart/byterange responses
1893  * into offset-length data, and this works around our inability to do so.
1894  */
1895  if (!we_do_ranges && request->multipartRangeRequest()) {
1896  /* don't cache the result */
1897  request->flags.cachable = false;
1898  /* pretend it's not a range request */
1899  request->ignoreRange("want to request the whole object");
1900  request->flags.isRanged = false;
1901  }
1902 
1903  hdr_out->addVia(request->http_ver, hdr_in);
1904 
1905  if (request->flags.accelerated) {
1906  /* Append Surrogate-Capabilities */
1907  String strSurrogate(hdr_in->getList(Http::HdrType::SURROGATE_CAPABILITY));
1908 #if USE_SQUID_ESI
1909  snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0 ESI/1.0\"", Config.Accel.surrogate_id);
1910 #else
1911  snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0\"", Config.Accel.surrogate_id);
1912 #endif
1913  strListAdd(&strSurrogate, bbuf, ',');
1914  hdr_out->putStr(Http::HdrType::SURROGATE_CAPABILITY, strSurrogate.termedBuf());
1915  }
1916 
1918  if (strcmp(opt_forwarded_for, "delete") != 0) {
1919 
1920  String strFwd = hdr_in->getList(Http::HdrType::X_FORWARDED_FOR);
1921 
1922  // if we cannot double strFwd size, then it grew past 50% of the limit
1923  if (!strFwd.canGrowBy(strFwd.size())) {
1924  // There is probably a forwarding loop with Via detection disabled.
1925  // If we do nothing, String will assert on overflow soon.
1926  // TODO: Terminate all transactions with huge XFF?
1927  strFwd = "error";
1928 
1929  static int warnedCount = 0;
1930  if (warnedCount++ < 100) {
1931  const SBuf url(entry ? SBuf(entry->url()) : request->effectiveRequestUri());
1932  debugs(11, DBG_IMPORTANT, "Warning: likely forwarding loop with " << url);
1933  }
1934  }
1935 
1936  if (strcmp(opt_forwarded_for, "on") == 0) {
1938  if ( request->client_addr.isNoAddr() )
1939  strListAdd(&strFwd, "unknown", ',');
1940  else
1941  strListAdd(&strFwd, request->client_addr.toStr(ntoabuf, MAX_IPSTRLEN), ',');
1942  } else if (strcmp(opt_forwarded_for, "off") == 0) {
1944  strListAdd(&strFwd, "unknown", ',');
1945  } else if (strcmp(opt_forwarded_for, "transparent") == 0) {
1947  } else if (strcmp(opt_forwarded_for, "truncate") == 0) {
1949  if ( request->client_addr.isNoAddr() )
1950  strFwd = "unknown";
1951  else
1952  strFwd = request->client_addr.toStr(ntoabuf, MAX_IPSTRLEN);
1953  }
1954  if (strFwd.size() > 0)
1955  hdr_out->putStr(Http::HdrType::X_FORWARDED_FOR, strFwd.termedBuf());
1956  }
1959  /* append Host if not there already */
1960  if (!hdr_out->has(Http::HdrType::HOST)) {
1961  if (request->peer_domain) {
1963  } else {
1964  SBuf authority = request->url.authority();
1965  hdr_out->putStr(Http::HdrType::HOST, authority.c_str());
1966  }
1967  }
1968 
1969  /* append Authorization if known in URL, not in header and going direct */
1970  if (!hdr_out->has(Http::HdrType::AUTHORIZATION)) {
1971  if (flags.toOrigin && !request->url.userInfo().isEmpty()) {
1972  static char result[base64_encode_len(MAX_URL*2)]; // should be big enough for a single URI segment
1973  struct base64_encode_ctx ctx;
1974  base64_encode_init(&ctx);
1975  size_t blen = base64_encode_update(&ctx, result, request->url.userInfo().length(), reinterpret_cast<const uint8_t*>(request->url.userInfo().rawContent()));
1976  blen += base64_encode_final(&ctx, result+blen);
1977  result[blen] = '\0';
1978  if (blen)
1979  httpHeaderPutStrf(hdr_out, Http::HdrType::AUTHORIZATION, "Basic %.*s", (int)blen, result);
1980  }
1981  }
1982 
1983  /* Fixup (Proxy-)Authorization special cases. Plain relaying dealt with above */
1984  httpFixupAuthentication(request, hdr_in, hdr_out, flags);
1985 
1986  /* append Cache-Control, add max-age if not there already */
1987  {
1988  HttpHdrCc *cc = hdr_in->getCc();
1989 
1990  if (!cc)
1991  cc = new HttpHdrCc();
1992 
1993 #if 0 /* see bug 2330 */
1994  /* Set no-cache if determined needed but not found */
1995  if (request->flags.nocache)
1997 #endif
1998 
1999  /* Add max-age only without no-cache */
2000  if (!cc->hasMaxAge() && !cc->hasNoCache()) {
2001  // XXX: performance regression. c_str() reallocates
2003  cc->maxAge(getMaxAge(entry ? entry->url() : tmp.c_str()));
2004  }
2005 
2006  /* Enforce sibling relations */
2007  if (flags.only_if_cached)
2008  cc->onlyIfCached(true);
2009 
2010  hdr_out->putCc(cc);
2011 
2012  delete cc;
2013  }
2014 
2015  // Always send Connection because HTTP/1.0 servers need explicit "keep-alive"
2016  // while HTTP/1.1 servers need explicit "close", and we do not always know
2017  // the server expectations.
2018  hdr_out->putStr(Http::HdrType::CONNECTION, flags.keepalive ? "keep-alive" : "close");
2019 
2020  /* append Front-End-Https */
2021  if (flags.front_end_https) {
2023  hdr_out->putStr(Http::HdrType::FRONT_END_HTTPS, "On");
2024  }
2025 
2026  if (flags.chunked_request) {
2027  // Do not just copy the original value so that if the client-side
2028  // starts decode other encodings, this code may remain valid.
2029  hdr_out->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
2030  }
2031 
2032  /* Now mangle the headers. */
2033  httpHdrMangleList(hdr_out, request, al, ROR_REQUEST);
2034 
2035  strConnection.clean();
2036 }
2037 
2040 void
2042 {
2044  return; // forward nothing by default
2045 
2046  /* RFC 7230 section 6.7 paragraph 10:
2047  * A server MUST ignore an Upgrade header field that is received in
2048  * an HTTP/1.0 request.
2049  */
2050  if (request->http_ver == Http::ProtocolVersion(1,0))
2051  return;
2052 
2053  const auto &hdrIn = request->header;
2054  if (!hdrIn.has(Http::HdrType::UPGRADE))
2055  return;
2056  const auto upgradeIn = hdrIn.getList(Http::HdrType::UPGRADE);
2057 
2058  String upgradeOut;
2059 
2060  ACLFilledChecklist ch(nullptr, request.getRaw());
2061  ch.al = fwd->al;
2062  const char *pos = nullptr;
2063  const char *offeredStr = nullptr;
2064  int offeredStrLen = 0;
2065  while (strListGetItem(&upgradeIn, ',', &offeredStr, &offeredStrLen, &pos)) {
2066  const ProtocolView offeredProto(offeredStr, offeredStrLen);
2067  debugs(11, 5, "checks all rules applicable to " << offeredProto);
2068  Config.http_upgrade_request_protocols->forApplicable(offeredProto, [&ch, offeredStr, offeredStrLen, &upgradeOut] (const SBuf &cfgProto, const acl_access *guard) {
2069  debugs(11, 5, "checks " << cfgProto << " rule(s)");
2070  ch.changeAcl(guard);
2071  const auto answer = ch.fastCheck();
2072  if (answer.implicit)
2073  return false; // keep looking for an explicit rule match
2074  if (answer.allowed())
2075  strListAdd(upgradeOut, offeredStr, offeredStrLen);
2076  // else drop the offer (explicitly denied cases and ACL errors)
2077  return true; // stop after an explicit rule match or an error
2078  });
2079  }
2080 
2081  if (upgradeOut.size()) {
2082  hdrOut.putStr(Http::HdrType::UPGRADE, upgradeOut.termedBuf());
2083 
2084  /* RFC 7230 section 6.7 paragraph 10:
2085  * When Upgrade is sent, the sender MUST also send a Connection header
2086  * field that contains an "upgrade" connection option, in
2087  * order to prevent Upgrade from being accidentally forwarded by
2088  * intermediaries that might not implement the listed protocols.
2089  *
2090  * NP: Squid does not truly implement the protocol(s) in this Upgrade.
2091  * For now we are treating an explicit blind tunnel as "implemented"
2092  * regardless of the security implications.
2093  */
2094  hdrOut.putStr(Http::HdrType::CONNECTION, "upgrade");
2095  }
2096 }
2097 
2102 void
2103 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request, HttpHeader * hdr_out, const int we_do_ranges, const Http::StateFlags &flags)
2104 {
2105  debugs(11, 5, "httpBuildRequestHeader: " << e->name << ": " << e->value );
2106 
2107  switch (e->id) {
2108 
2116  if (!flags.toOrigin && request->peer_login &&
2117  (strcmp(request->peer_login, "PASS") == 0 ||
2118  strcmp(request->peer_login, "PROXYPASS") == 0 ||
2119  strcmp(request->peer_login, "PASSTHRU") == 0)) {
2120  hdr_out->addEntry(e->clone());
2121  }
2122  break;
2123 
2127  case Http::HdrType::TE:
2130  case Http::HdrType::TRAILER:
2132  break;
2133 
2136  break;
2137 
2144  if (!flags.toOriginPeer()) {
2145  hdr_out->addEntry(e->clone());
2146  } else {
2151  if (request->peer_login &&
2152  (strcmp(request->peer_login, "PASS") == 0 ||
2153  strcmp(request->peer_login, "PASSTHRU") == 0 ||
2154  strcmp(request->peer_login, "PROXYPASS") == 0)) {
2155  hdr_out->addEntry(e->clone());
2156  }
2157  }
2158 
2159  break;
2160 
2161  case Http::HdrType::HOST:
2168  if (request->peer_domain)
2169  hdr_out->putStr(Http::HdrType::HOST, request->peer_domain);
2170  else if (request->flags.redirected && !Config.onoff.redir_rewrites_host)
2171  hdr_out->addEntry(e->clone());
2172  else {
2173  SBuf authority = request->url.authority();
2174  hdr_out->putStr(Http::HdrType::HOST, authority.c_str());
2175  }
2176 
2177  break;
2178 
2187  // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2188  if (hdr_out->has(Http::HdrType::IF_MODIFIED_SINCE))
2189  break;
2190  else if (Config.onoff.cache_miss_revalidate || !request->flags.cachable || request->flags.auth)
2191  hdr_out->addEntry(e->clone());
2192  break;
2193 
2202  // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2203  if (hdr_out->hasListMember(Http::HdrType::IF_MATCH, "*", ',') || Config.onoff.cache_miss_revalidate || !request->flags.cachable || request->flags.auth)
2204  hdr_out->addEntry(e->clone());
2205  break;
2206 
2211  const int64_t hops = e->getInt64();
2212 
2213  if (hops > 0)
2214  hdr_out->putInt64(Http::HdrType::MAX_FORWARDS, hops - 1);
2215  }
2216 
2217  break;
2218 
2219  case Http::HdrType::VIA:
2224  if (!Config.onoff.via)
2225  hdr_out->addEntry(e->clone());
2226 
2227  break;
2228 
2229  case Http::HdrType::RANGE:
2230 
2232 
2236  if (!we_do_ranges)
2237  hdr_out->addEntry(e->clone());
2238 
2239  break;
2240 
2241  case Http::HdrType::PROXY_CONNECTION: // SHOULD ignore. But doing so breaks things.
2242  break;
2243 
2245  // pass through unless we chunk; also, keeping this away from default
2246  // prevents request smuggling via Connection: Content-Length tricks
2247  if (!flags.chunked_request)
2248  hdr_out->addEntry(e->clone());
2249  break;
2250 
2252 
2257  break;
2258 
2262  if (!flags.front_end_https)
2263  hdr_out->addEntry(e->clone());
2264 
2265  break;
2266 
2267  default:
2271  if (strConnection.size()>0 && strListIsMember(&strConnection, e->name, ',')) {
2272  debugs(11, 2, "'" << e->name << "' header cropped by Connection: definition");
2273  return;
2274  }
2275 
2276  hdr_out->addEntry(e->clone());
2277  }
2278 }
2279 
2280 bool
2282 {
2283  bool result = true;
2284  /* decide if we want to do Ranges ourselves
2285  * and fetch the whole object now)
2286  * We want to handle Ranges ourselves iff
2287  * - we can actually parse client Range specs
2288  * - the specs are expected to be simple enough (e.g. no out-of-order ranges)
2289  * - reply will be cachable
2290  * (If the reply will be uncachable we have to throw it away after
2291  * serving this request, so it is better to forward ranges to
2292  * the server and fetch only the requested content)
2293  */
2294 
2295  int64_t roffLimit = request->getRangeOffsetLimit();
2296 
2297  if (NULL == request->range || !request->flags.cachable
2299  result = false;
2300 
2301  debugs(11, 8, "decideIfWeDoRanges: range specs: " <<
2302  request->range << ", cachable: " <<
2303  request->flags.cachable << "; we_do_ranges: " << result);
2304 
2305  return result;
2306 }
2307 
2308 /* build request prefix and append it to a given MemBuf;
2309  * return the length of the prefix */
2310 mb_size_t
2312 {
2313  const int offset = mb->size;
2314  /* Uses a local httpver variable to print the HTTP label
2315  * since the HttpRequest may have an older version label.
2316  * XXX: This could create protocol bugs as the headers sent and
2317  * flow control should all be based on the HttpRequest version
2318  * not the one we are sending. Needs checking.
2319  */
2320  const AnyP::ProtocolVersion httpver = Http::ProtocolVersion();
2322  mb->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " %s/%d.%d\r\n",
2324  SQUIDSBUFPRINT(url),
2326  httpver.major,httpver.minor);
2327  /* build and pack headers */
2328  {
2329  HttpHeader hdr(hoRequest);
2330  forwardUpgrade(hdr);
2332 
2334  request->flags.authSent = true;
2335  else if (hdr.has(Http::HdrType::AUTHORIZATION))
2336  request->flags.authSent = true;
2337 
2338  // The late placement of this check supports reply_header_add mangling,
2339  // but also complicates optimizing upgradeHeaderOut-like lookups.
2340  if (hdr.has(Http::HdrType::UPGRADE)) {
2343  }
2344 
2345  hdr.packInto(mb);
2346  hdr.clean();
2347  }
2348  /* append header terminator */
2349  mb->append(crlf, 2);
2350  return mb->size - offset;
2351 }
2352 
2353 /* This will be called when connect completes. Write request. */
2354 bool
2356 {
2357  MemBuf mb;
2358 
2359  debugs(11, 5, HERE << serverConnection << ", request " << request << ", this " << this << ".");
2360 
2362  debugs(11,3, HERE << "cannot send request to closing " << serverConnection);
2363  assert(closeHandler != NULL);
2364  return false;
2365  }
2366 
2368  AsyncCall::Pointer timeoutCall = JobCallback(11, 5,
2369  TimeoutDialer, this, HttpStateData::httpTimeout);
2371  flags.do_next_read = true;
2373 
2374  if (request->body_pipe != NULL) {
2375  if (!startRequestBodyFlow()) // register to receive body data
2376  return false;
2378  requestSender = JobCallback(11,5,
2379  Dialer, this, HttpStateData::sentRequestBody);
2380 
2382  // use chunked encoding if we do not know the length
2383  if (request->content_length < 0)
2384  flags.chunked_request = true;
2385  } else {
2388  requestSender = JobCallback(11,5,
2389  Dialer, this, HttpStateData::wroteLast);
2390  }
2391 
2392  /*
2393  * Is keep-alive okay for all request methods?
2394  */
2396  flags.keepalive = true;
2397  else if (request->flags.pinned)
2399  else if (!Config.onoff.server_pconns)
2400  flags.keepalive = false;
2401  else if (flags.tunneling)
2402  // tunneled non pinned bumped requests must not keepalive
2404  else if (_peer == NULL)
2405  flags.keepalive = true;
2406  else if (_peer->stats.n_keepalives_sent < 10)
2407  flags.keepalive = true;
2408  else if ((double) _peer->stats.n_keepalives_recv /
2409  (double) _peer->stats.n_keepalives_sent > 0.50)
2410  flags.keepalive = true;
2411 
2412  if (_peer && !flags.tunneling) {
2413  /*The old code here was
2414  if (neighborType(_peer, request->url) == PEER_SIBLING && ...
2415  which is equivalent to:
2416  if (neighborType(_peer, URL()) == PEER_SIBLING && ...
2417  or better:
2418  if (((_peer->type == PEER_MULTICAST && p->options.mcast_siblings) ||
2419  _peer->type == PEER_SIBLINGS ) && _peer->options.allow_miss)
2420  flags.only_if_cached = 1;
2421 
2422  But I suppose it was a bug
2423  */
2425  flags.only_if_cached = true;
2426 
2428  }
2429 
2430  mb.init();
2432  buildRequestPrefix(&mb);
2433 
2434  debugs(11, 2, "HTTP Server " << serverConnection);
2435  debugs(11, 2, "HTTP Server REQUEST:\n---------\n" << mb.buf << "\n----------");
2436 
2438  return true;
2439 }
2440 
2441 bool
2443 {
2444  // parent's implementation can handle the no-encoding case
2445  if (!flags.chunked_request)
2447 
2448  MemBuf raw;
2449 
2451  if (!requestBodySource->getMoreData(raw))
2452  return false; // no request body bytes to chunk yet
2453 
2454  // optimization: pre-allocate buffer size that should be enough
2455  const mb_size_t rawDataSize = raw.contentSize();
2456  // we may need to send: hex-chunk-size CRLF raw-data CRLF last-chunk
2457  buf.init(16 + 2 + rawDataSize + 2 + 5, raw.max_capacity);
2458 
2459  buf.appendf("%x\r\n", static_cast<unsigned int>(rawDataSize));
2460  buf.append(raw.content(), rawDataSize);
2461  buf.append("\r\n", 2);
2462 
2463  Must(rawDataSize > 0); // we did not accidentally created last-chunk above
2464 
2465  // Do not send last-chunk unless we successfully received everything
2468  flags.sentLastChunk = true;
2469  buf.append("0\r\n\r\n", 5);
2470  }
2471 
2472  return true;
2473 }
2474 
2475 void
2477 {
2478  debugs(11, 3, fwd->request->method << ' ' << fwd->entry->url());
2479  AsyncJob::Start(new HttpStateData(fwd));
2480 }
2481 
2482 void
2484 {
2485  if (!sendRequest()) {
2486  debugs(11, 3, "httpStart: aborted");
2487  mustStop("HttpStateData::start failed");
2488  return;
2489  }
2490 
2491  ++ statCounter.server.all.requests;
2492  ++ statCounter.server.http.requests;
2493 
2494  /*
2495  * We used to set the read timeout here, but not any more.
2496  * Now its set in httpSendComplete() after the full request,
2497  * including request body, has been written to the server.
2498  */
2499 }
2500 
2502 bool
2504 {
2505 #if USE_HTTP_VIOLATIONS
2506  if (!Config.accessList.brokenPosts) {
2507  debugs(11, 5, HERE << "No brokenPosts list");
2508  return false;
2509  }
2510 
2512  ch.al = fwd->al;
2513  ch.syncAle(originalRequest().getRaw(), nullptr);
2514  if (!ch.fastCheck().allowed()) {
2515  debugs(11, 5, HERE << "didn't match brokenPosts");
2516  return false;
2517  }
2518 
2520  debugs(11, 3, HERE << "ignoring broken POST for closed " << serverConnection);
2521  assert(closeHandler != NULL);
2522  return true; // prevent caller from proceeding as if nothing happened
2523  }
2524 
2525  debugs(11, 3, "finishingBrokenPost: fixing broken POST");
2527  requestSender = JobCallback(11,5,
2528  Dialer, this, HttpStateData::wroteLast);
2530  return true;
2531 #else
2532  return false;
2533 #endif /* USE_HTTP_VIOLATIONS */
2534 }
2535 
2537 bool
2539 {
2540  if (flags.sentLastChunk) {
2541  debugs(11, 5, HERE << "already sent last-chunk");
2542  return false;
2543  }
2544 
2545  Must(receivedWholeRequestBody); // or we should not be sending last-chunk
2546  flags.sentLastChunk = true;
2547 
2549  requestSender = JobCallback(11,5, Dialer, this, HttpStateData::wroteLast);
2550  Comm::Write(serverConnection, "0\r\n\r\n", 5, requestSender, NULL);
2551  return true;
2552 }
2553 
2554 void
2556 {
2558  debugs(11,5, HERE << serverConnection);
2559 
2560  // do we need to write something after the last body byte?
2562  return;
2564  return;
2565 
2566  sendComplete();
2567 }
2568 
2569 // more origin request body data is available
2570 void
2572 {
2574  // XXX: we should check this condition in other callbacks then!
2575  // TODO: Check whether this can actually happen: We should unsubscribe
2576  // as a body consumer when the above condition(s) are detected.
2577  debugs(11, DBG_IMPORTANT, HERE << "Transaction aborted while reading HTTP body");
2578  return;
2579  }
2580 
2582 
2583  if (requestBodySource->buf().hasContent()) {
2584  // XXX: why does not this trigger a debug message on every request?
2585 
2587  flags.abuse_detected = true;
2588  debugs(11, DBG_IMPORTANT, "http handleMoreRequestBodyAvailable: Likely proxy abuse detected '" << request->client_addr << "' -> '" << entry->url() << "'" );
2589 
2590  if (virginReply()->sline.status() == Http::scInvalidHeader) {
2591  closeServer();
2592  mustStop("HttpStateData::handleMoreRequestBodyAvailable");
2593  return;
2594  }
2595  }
2596  }
2597 
2599 }
2600 
2601 // premature end of the request body
2602 void
2604 {
2606  if (entry->isEmpty()) {
2607  debugs(11, 3, "request body aborted: " << serverConnection);
2608  // We usually get here when ICAP REQMOD aborts during body processing.
2609  // We might also get here if client-side aborts, but then our response
2610  // should not matter because either client-side will provide its own or
2611  // there will be no response at all (e.g., if the the client has left).
2613  err->detailError(ERR_DETAIL_SRV_REQMOD_REQ_BODY);
2614  fwd->fail(err);
2615  }
2616 
2617  abortTransaction("request body producer aborted");
2618 }
2619 
2620 // called when we wrote request headers(!) or a part of the body
2621 void
2623 {
2624  if (io.size > 0)
2625  statCounter.server.http.kbytes_out += io.size;
2626 
2628 }
2629 
2630 void
2631 HttpStateData::abortAll(const char *reason)
2632 {
2633  debugs(11,5, HERE << "aborting transaction for " << reason <<
2634  "; " << serverConnection << ", this " << this);
2635  mustStop(reason);
2636 }
2637 
2639  : answer(HttpStateData::ReuseDecision::reuseNot), reason(nullptr), entry(e), statusCode(code) {}
2640 
2643 {
2644  answer = ans;
2645  reason = why;
2646  return answer;
2647 }
2648 
2649 std::ostream &operator <<(std::ostream &os, const HttpStateData::ReuseDecision &d)
2650 {
2651  static const char *ReuseMessages[] = {
2652  "do not cache and do not share", // reuseNot
2653  "cache positively and share", // cachePositively
2654  "cache negatively and share", // cacheNegatively
2655  "do not cache but share" // doNotCacheButShare
2656  };
2657 
2660  return os << ReuseMessages[d.answer] << " because " << d.reason <<
2661  "; HTTP status " << d.statusCode << " " << *(d.entry);
2662 }
2663 
const MemBuf & buf() const
Definition: BodyPipe.h:137
void drop1xx(const char *reason)
Definition: http.cc:851
void Read(const Comm::ConnectionPointer &conn, AsyncCall::Pointer &callback)
Definition: Read.cc:40
bool proxy_only
Definition: CachePeer.h:82
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: http.cc:983
const char * xstrerr(int error)
Definition: xstrerror.cc:83
void makePrivate(const bool shareable)
Definition: store.cc:172
bool handling1xx
Whether we are processing an HTTP 1xx control message.
Definition: StateFlags.h:23
char method[16]
Definition: tcp-banger2.c:115
bool hasProxyRevalidate() const
Definition: HttpHdrCc.h:117
bool maybeMakeSpaceAvailable(bool grow)
Definition: http.cc:1633
char * buf
Definition: MemBuf.h:134
struct StatCounters::@130::@140 all
AsyncCall::Pointer comm_add_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:983
int connection_auth
0 - off, 1 - on, 2 - auto
Definition: CachePeer.h:195
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:418
void truncateVirginBody()
Definition: http.cc:1417
char * peer_host
Definition: HttpRequest.h:166
bool keepalive_broken
Definition: StateFlags.h:51
unsigned int major
major version number
StoreEntry * entry
Definition: Client.h:164
@ ERR_ICAP_FAILURE
Definition: err_type.h:62
bool keepalive
whether to keep the connection persistent
Definition: StateFlags.h:19
@ scMethodNotAllowed
Definition: StatusCode.h:49
bool hasPrivate(const String **val=nullptr) const
Definition: HttpHdrCc.h:74
@ scNonAuthoritativeInformation
Definition: StatusCode.h:29
AnyP::ProtocolVersion http_ver
Definition: Message.h:73
void userInfo(const SBuf &s)
Definition: Uri.h:79
#define base64_encode_len(length)
Definition: base64.h:169
@ HDR_X_ACCELERATOR_VARY
void ignoreRange(const char *reason)
forgets about the cached Range header (for a reason)
Definition: HttpRequest.cc:646
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
@ scUnauthorized
Definition: StatusCode.h:45
@ scPayloadTooLarge
Definition: StatusCode.h:57
@ METHOD_HEAD
Definition: MethodType.h:28
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
http_hdr_owner_type owner
Definition: HttpHeader.h:173
bool ignoreCacheControl
Definition: http.h:70
Definition: Client.h:29
struct CachePeer::@31 options
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:98
bool spoofClientIp
Definition: RequestFlags.h:70
const char * storeId() const
Definition: MemObject.cc:54
void releaseRequest(const bool shareable=false)
Definition: store.cc:472
@ ENTRY_REVALIDATE_STALE
Definition: enums.h:100
int Ctx
Definition: Debug.h:37
@ FD_WRITE
Definition: enums.h:24
@ scBadRequest
Definition: StatusCode.h:44
bool chunked
reading a chunked response; TODO: rename
Definition: StateFlags.h:55
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1208
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
mb_size_t size
Definition: MemBuf.h:135
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: http.cc:2442
int reads
Definition: IoStats.h:19
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:165
HttpHeader header
Definition: Message.h:75
bool offsetLimitExceeded(const int64_t limit) const
#define EBIT_SET(flag, bit)
Definition: defines.h:105
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1709
Http::StateFlags flags
Definition: http.h:67
@ scMisdirectedRequest
Definition: StatusCode.h:62
Http::StatusCode peer_reply_status
last HTTP status code received
bool MonitorsRead(int fd)
whether the FD socket is being monitored for read
Definition: Read.cc:29
Answers make(const Answers ans, const char *why)
stores the corresponding decision
Definition: http.cc:2642
bool isEmpty() const
Definition: SBuf.h:420
MemObject * mem_obj
Definition: Store.h:213
AsyncCall::Pointer requestSender
Definition: Client.h:170
RequestFlags flags
Definition: HttpRequest.h:141
String * upgradeHeaderOut
Upgrade header value sent to the origin server or cache peer.
Definition: http.h:74
void checkDateSkew(HttpReply *)
Definition: http.cc:651
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
int64_t payloadTruncated
positive when we read more than we wanted
Definition: http.h:155
virtual void abortAll(const char *reason)
abnormal transaction termination; reason is for debugging only
Definition: http.cc:2631
void reserveSpace(size_type minSpace)
Definition: SBuf.h:429
@ scLengthRequired
Definition: StatusCode.h:55
const Http::StatusCode statusCode
HTTP status for debugging.
Definition: http.h:41
virtual bool mayReadVirginReplyBody() const
whether we may receive more virgin response body bytes
Definition: http.cc:1598
bool surrogateNoStore
Definition: http.h:71
const char * url() const
Definition: store.cc:1622
a reference to a protocol name[/version] string; no 0-termination is assumed
int lastChunk
Definition: http.h:66
Comm::Flag ReadNow(CommIoCbParams &params, SBuf &buf)
Definition: Read.cc:81
size_t base64_encode_final(struct base64_encode_ctx *ctx, char *dst)
Definition: base64.c:308
static void httpFixupAuthentication(HttpRequest *request, const HttpHeader *hdr_in, HttpHeader *hdr_out, const Http::StateFlags &flags)
Definition: http.cc:1748
int64_t getRangeOffsetLimit()
Definition: HttpRequest.cc:619
MemObject & mem()
Definition: Store.h:52
@ ERR_NONE
Definition: err_type.h:13
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:331
void error(char *format,...)
struct SquidConfig::@113 accessList
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:174
unsigned int minor
minor version number
@ PROXY_CONNECTION
#define PEER_PROXY_NEGOTIATE_NOKEYTAB
bool finishingBrokenPost()
if broken posts are enabled for the request, try to fix and return true
Definition: http.cc:2503
CachePeer * _peer
Definition: http.h:64
bool connectionAuth
Definition: RequestFlags.h:76
@ scUnsupportedMediaType
Definition: StatusCode.h:59
void httpHdrMangleList(HttpHeader *l, HttpRequest *request, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep)
bool hasContent() const
Definition: MemBuf.h:54
HttpReplyPointer reply
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:375
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
const char * ProtocolType_str[]
@ KEY_PRIVATE
Definition: enums.h:102
@ INPROGRESS
Definition: Flag.h:22
Definition: SBuf.h:86
virtual void sendControlMsg(HttpControlMsg)
called to send the 1xx message and notify the Source
mb_size_t buildRequestPrefix(MemBuf *mb)
Definition: http.cc:2311
size_t maxReplyHeaderSize
Definition: SquidConfig.h:134
ConnectionStatus statusIfComplete() const
Definition: http.cc:1116
AsyncCall::Pointer closeHandler
Definition: http.h:91
char * opt_forwarded_for
ConnectionStatus
Definition: http.h:92
@ scUseProxy
Definition: StatusCode.h:41
int commSetConnTimeout(const Comm::ConnectionPointer &conn, int timeout, AsyncCall::Pointer &callback)
Definition: comm.cc:567
HttpStateData(FwdState *)
Definition: http.cc:87
@ scMovedPermanently
Definition: StatusCode.h:37
void noStoreRemote(bool v)
static void assembleVaryKey(String &vary, SBuf &vstr, const HttpRequest &request)
assemble a variant key (vary-mark) from the given Vary header and HTTP request
Definition: http.cc:576
#define SQUID_ENTER_THROWING_CODE()
Definition: http.cc:70
DelayId mostBytesAllowed() const
Definition: MemObject.cc:478
bool peering
Whether the next TCP hop is a cache_peer, including originserver.
Definition: StateFlags.h:33
BodyPipe::Pointer requestBodySource
Definition: Client.h:169
a representation of a refresh pattern.
const char * blockSwitchingProtocols(const HttpReply &) const
Definition: http.cc:869
PconnPool * fwdPconnPool
a collection of previously used persistent Squid-to-peer HTTP(S) connections
Definition: FwdState.cc:79
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:858
bool sendRequest()
Definition: http.cc:2355
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
bool chunked_request
writing a chunked request
Definition: StateFlags.h:56
int detect_broken_server_pconns
Definition: SquidConfig.h:320
C * getRaw() const
Definition: RefCount.h:80
size_type spaceSize() const
Definition: SBuf.h:382
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:216
uint16_t flags
Definition: Store.h:224
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
const SBuf & image() const
void processReplyBody()
Definition: http.cc:1487
void toLower()
converts all characters to lower case;
Definition: SBuf.cc:821
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:26
bool allow_miss
Definition: CachePeer.h:103
IoStats IOStats
@ OK
Definition: Flag.h:16
bool isAccepting() const
Definition: store.cc:2050
#define SQUID_EXIT_THROWING_CODE(status)
Definition: http.cc:71
Http::StatusLine sline
Definition: HttpReply.h:60
time_t getMaxAge(const char *url)
Definition: refresh.cc:642
@ ENDFILE
Definition: Flag.h:27
Comm::ConnectionPointer serverConnection
Definition: http.h:90
int surrogate_is_remote
Definition: SquidConfig.h:318
StatusCode
Definition: StatusCode.h:20
static const char *const crlf
Definition: http.cc:81
void adaptOrFinalizeReply()
Definition: Client.cc:956
void unregister(Comm::ConnectionPointer &conn)
Definition: FwdState.cc:503
bool multipartRangeRequest() const
Definition: HttpRequest.cc:437
void fail(ErrorState *err)
Definition: FwdState.cc:471
void keepaliveAccounting(HttpReply *)
Definition: http.cc:631
bool startRequestBodyFlow()
Definition: Client.cc:219
@ scRequestedRangeNotSatisfied
Definition: StatusCode.h:60
@ ERR_CLOSING
Definition: Flag.h:25
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:517
bool Is1xx(const int sc)
whether this is an informational 1xx response status code
Definition: StatusCode.h:92
#define cbdataReference(var)
Definition: cbdata.h:341
int front_end_https
0 - off, 1 - on, 2 - auto
Definition: CachePeer.h:194
#define PROF_stop(probename)
Definition: Profiler.h:63
bool connectionAuthDisabled
Definition: RequestFlags.h:78
void reset()
Definition: store.cc:1677
@ IF_MODIFIED_SINCE
#define DBG_IMPORTANT
Definition: Debug.h:46
bool decodeAndWriteReplyBody()
Definition: http.cc:1455
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
StoreEntry * storeGetPublicByRequest(HttpRequest *req, const KeyScope keyScope)
Definition: store.cc:554
int32_t mask
Definition: HttpHdrCc.h:173
ProtocolType protocol
which protocol this version is for
void readReply(const CommIoCbParams &io)
Definition: http.cc:1219
static bool decideIfWeDoRanges(HttpRequest *orig_request)
Definition: http.cc:2281
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:294
@ scSeeOther
Definition: StatusCode.h:39
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:612
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *req, const HttpRequestMethod &method, const KeyScope keyScope)
Definition: store.cc:548
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:802
AnyP::ProtocolType protocol
Definition: StatusLine.h:68
void serverComplete()
Definition: Client.cc:160
@ METHOD_OPTIONS
Definition: MethodType.h:31
@ CONTENT_LENGTH
SBuf & chop(size_type pos, size_type n=npos)
Definition: SBuf.cc:540
bool chunked() const
Definition: HttpHeader.h:165
mb_size_t max_capacity
Definition: MemBuf.h:142
peer_t neighborType(const CachePeer *p, const AnyP::Uri &url)
Definition: neighbors.cc:115
#define MAX_IPSTRLEN
Length of buffer that needs to be allocated to old a null-terminated IP-string.
Definition: forward.h:23
@ scNotAcceptable
Definition: StatusCode.h:50
@ COMPLETE_NONPERSISTENT_MSG
Definition: http.h:95
Definition: Range.h:18
@ scGatewayTimeout
Definition: StatusCode.h:75
Definition: Debug.h:188
@ ENTRY_FWD_HDR_WAIT
Definition: enums.h:111
#define MAX_URL
Definition: defines.h:118
bool persistent() const
Definition: Message.cc:261
void sendComplete()
successfully wrote the entire request (including body, last-chunk, etc.)
Definition: http.cc:1707
time_t lastmod
Definition: HttpRequest.h:168
void strListAdd(String &str, const char *item, const size_t itemSize, const char delimiter)
Appends the given item of a given size to a delimiter-separated list in str.
Definition: StrList.cc:18
bool canGrowBy(const size_type growthLen) const
whether appending growthLen characters is safe (i.e., unlikely to assert)
Definition: SquidString.h:123
@ HTCP_CLR_INVALIDATION
Definition: enums.h:243
void bytesIn(int qty)
Definition: DelayId.cc:154
HttpHdrScTarget * getMergedTarget(const char *ourtarget)
Definition: HttpHdrSc.cc:296
String etag
A strong etag of the cached entry. Used for refreshing that entry.
Definition: HttpRequest.h:192
void processReplyHeader()
Definition: http.cc:668
ReuseDecision(const StoreEntry *e, const Http::StatusCode code)
Definition: http.cc:2638
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const
Definition: HttpReply.cc:488
bool httpHeaderHasConnDir(const HttpHeader *hdr, const SBuf &directive)
static void httpMaybeRemovePublic(StoreEntry *, Http::StatusCode)
Definition: http.cc:184
time_t expires
Definition: HttpReply.h:48
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
int64_t getInt64() const
Definition: HttpHeader.cc:1582
@ scTemporaryRedirect
Definition: StatusCode.h:42
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:170
unsigned int front_end_https
send "Front-End-Https: On" header (off/on/auto=2)
Definition: StateFlags.h:18
#define NULL
Definition: types.h:166
@ COMPLETE_PERSISTENT_MSG
Definition: http.h:94
const char * rawContent() const
Definition: SBuf.cc:519
SBuf consume(size_type n=npos)
Definition: SBuf.cc:491
int n_keepalives_sent
Definition: CachePeer.h:51
@ scFailedDependency
Definition: StatusCode.h:65
void maxAge(int32_t v)
Definition: HttpHdrCc.h:123
@ scSwitchingProtocols
Definition: StatusCode.h:23
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
virtual void doneSendingRequestBody()=0
Definition: Client.cc:320
time_t read
Definition: SquidConfig.h:109
@ scForbidden
Definition: StatusCode.h:47
@ ROR_REQUEST
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
bool dontRetry()
Definition: FwdState.h:102
struct StatCounters::@130::@140 http
void mustStop(const char *aReason)
Definition: AsyncJob.cc:69
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
const HttpReply * virginReply() const
Definition: Client.cc:103
SBuf inBuf
I/O buffer for receiving server responses.
Definition: http.h:69
bundles HTTP 1xx reply and the "successfully forwarded" callback
@ scInsufficientStorage
Definition: StatusCode.h:78
SBuf vary_headers
Definition: MemObject.h:204
@ scPreconditionFailed
Definition: StatusCode.h:56
@ scNotImplemented
Definition: StatusCode.h:72
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
MemBlob::size_type size_type
Definition: SBuf.h:89
#define MAX_LOGIN_SZ
Definition: defines.h:119
bool hasNoCacheWithParameters() const
Definition: HttpHdrCc.h:87
const Comm::ConnectionPointer & serverConnection() const
Definition: FwdState.h:110
struct IoStats::@81 Http
void forApplicable(const ProtocolView &, const Visitor &) const
bool hasNoStore() const
virtual void handleMoreRequestBodyAvailable()
Definition: http.cc:2571
void handle1xx(HttpReply *msg)
ignore or start forwarding the 1xx response (a.k.a., control message)
Definition: http.cc:792
@ ENTRY_REVALIDATE_ALWAYS
Definition: enums.h:85
const Acl::Answer & fastCheck()
Definition: Checklist.cc:336
@ srcHttp
http_port or HTTP server
Definition: Message.h:39
static Pointer Start(AsyncJob *job)
starts a freshly created job (i.e., makes the job asynchronous)
Definition: AsyncJob.cc:23
@ scBadGateway
Definition: StatusCode.h:73
const StoreEntry * entry
entry for debugging
Definition: http.h:40
Definition: MemBuf.h:23
void doneSendingRequestBody()
Definition: http.cc:2555
bool toOriginPeer() const
Whether the next TCP/HTTP hop is an originserver cache_peer.
Definition: StateFlags.h:49
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
CachePeer * getPeer() const
Definition: Connection.cc:105
void push(const Comm::ConnectionPointer &serverConn, const char *domain)
Definition: pconn.cc:412
unsigned char code
Definition: html_quote.c:20
bool canHandle1xx() const
whether the client is likely to be able to handle a 1xx reply
Definition: HttpRequest.cc:660
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:157
virtual void handleRequestBodyProducerAborted()
Definition: http.cc:2603
@ scPartialContent
Definition: StatusCode.h:32
@ scGone
Definition: StatusCode.h:54
Http::HdrType id
Definition: HttpHeader.h:63
void proceedAfter1xx()
restores state and resumes processing after 1xx is ignored or forwarded
Definition: http.cc:896
ConnectionStatus persistentConnStatus() const
Definition: http.cc:1161
err_type
Definition: err_type.h:12
bool hasSMaxAge(int32_t *val=nullptr) const
Definition: HttpHdrCc.h:127
void const char HLPCB void * data
Definition: stub_helper.cc:16
@ INCOMPLETE_MSG
Definition: http.h:93
#define CallJobHere(debugSection, debugLevel, job, Class, method)
Definition: AsyncJobCalls.h:58
char * peer_domain
Definition: HttpRequest.h:173
@ scUnprocessableEntity
Definition: StatusCode.h:63
#define BBUF_SZ
void httpTimeout(const CommTimeoutCbParams &params)
Definition: http.cc:161
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
Answers answer
the decision id
Definition: http.h:38
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:781
bool hasNoCache(const String **val=nullptr) const
Definition: HttpHdrCc.h:89
HttpRequestMethod method
Definition: MemObject.h:137
virtual bool doneWithServer() const
Definition: http.cc:1739
store_status_t store_status
Definition: Store.h:236
@ scPermanentRedirect
Definition: StatusCode.h:43
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1059
int neighbors_do_private_keys
HttpUpgradeProtocolAccess * http_upgrade_request_protocols
http_upgrade_request_protocols
Definition: SquidConfig.h:483
#define assert(EX)
Definition: assert.h:19
@ scUriTooLong
Definition: StatusCode.h:58
void setPayloadBuffer(MemBuf *parsedContent)
set the buffer to be used to store decoded chunk data
@ scContinue
Definition: StatusCode.h:22
HttpHdrCc * cache_control
Definition: Message.h:77
FwdState::Pointer fwd
Definition: Client.h:165
HierarchyLogEntry hier
Definition: HttpRequest.h:157
acl_access * brokenPosts
Definition: SquidConfig.h:379
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:160
virtual void syncAle(HttpRequest *adaptedRequest, const char *logUri) const
assigns uninitialized adapted_request and url ALE components
void neighborsHtcpClear(StoreEntry *e, HttpRequest *req, const HttpRequestMethod &method, htcp_clr_reason reason)
Definition: neighbors.cc:1807
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:547
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Parser.h:98
void abortTransaction(const char *reason)
Definition: http.h:113
@ TRANSFER_ENCODING
bool no_delay
Definition: CachePeer.h:101
@ scServiceUnavailable
Definition: StatusCode.h:74
@ ERR_READ_ERROR
Definition: err_type.h:26
const AnyP::UriScheme & getScheme() const
Definition: Uri.h:67
@ METHOD_TRACE
Definition: MethodType.h:30
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
bool adaptationAccessCheckPending
Definition: Client.h:177
void onlyIfCached(bool v)
Definition: HttpHdrCc.h:146
void writeReplyBody()
Definition: http.cc:1445
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
@ scInternalServerError
Definition: StatusCode.h:71
HttpHdrRange * range
Definition: HttpRequest.h:143
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:117
void httpStateConnClosed(const CommCloseCbParams &params)
Definition: http.cc:153
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:584
bool sentLastChunk
do not try to write last-chunk again
Definition: StateFlags.h:57
bool cacheNegatively()
Definition: store.cc:186
const char * c_str()
Definition: SBuf.cc:526
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:35
int commUnsetConnTimeout(const Comm::ConnectionPointer &conn)
Definition: comm.cc:593
virtual const Comm::ConnectionPointer & dataConnection() const
Definition: http.cc:147
bool olderThan(const HttpReply *them) const
Definition: HttpReply.cc:664
SBuf & authority(bool requirePort=false) const
Definition: Uri.cc:574
void base64_encode_init(struct base64_encode_ctx *ctx)
Definition: base64.c:232
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:180
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:404
time_t squid_curtime
Definition: stub_time.cc:17
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:249
SBuf & append(const SBuf &S)
Definition: SBuf.cc:195
bool isNoAddr() const
Definition: Address.cc:284
int64_t content_length
Definition: Message.h:84
time_t date
Definition: HttpReply.h:44
HttpReply * finalReply()
Definition: Client.cc:130
struct SquidConfig::@112 onoff
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1084
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:256
bool parseHeader(Http1::Parser &hp)
parses reply header using Parser
Definition: HttpReply.cc:467
@ ERR_INVALID_RESP
Definition: err_type.h:39
@ scNotModified
Definition: StatusCode.h:40
ReuseDecision::Answers reusableReply(ReuseDecision &decision)
Definition: http.cc:321
@ ERR_ZERO_SIZE_OBJECT
Definition: err_type.h:44
uint32_t sources
The message sources.
Definition: Message.h:100
const Acl::Tree * changeAcl(const Acl::Tree *t)
Definition: Checklist.h:176
bool mustKeepalive
Definition: RequestFlags.h:74
struct StatCounters::@130 server
Http1::TeChunkedParser * httpChunkDecoder
Definition: http.h:150
Ctx ctx_enter(const char *descr)
Definition: debug.cc:721
void wroteLast(const CommIoCbParams &io)
called after writing the very last request byte (body, last-chunk, etc)
Definition: http.cc:1670
#define fd_table
Definition: fde.h:189
Http1::ResponseParserPointer hp
Parser being used at present to parse the HTTP/ICY server response.
Definition: http.h:149
assists in making and relaying entry caching/sharing decision
Definition: http.h:29
@ scFound
Definition: StatusCode.h:38
virtual bool parse(const SBuf &)
@ PROTO_HTTPS
Definition: ProtocolType.h:27
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1014
@ scRequestTimeout
Definition: StatusCode.h:52
bool accelerated
Definition: RequestFlags.h:58
void maxAge(int v)
HttpRequestMethod method
Definition: HttpRequest.h:114
void path(const char *p)
Definition: Uri.h:99
const char * reason
the decision reason
Definition: http.h:39
@ scNotFound
Definition: StatusCode.h:48
acl_access * reply
Definition: SquidConfig.h:383
noteTakeServerConnectionControl() callback parameter
Definition: client_side.h:201
@ ENTRY_ABORTED
Definition: enums.h:115
@ PROTO_HTTP
Definition: ProtocolType.h:25
char * host
Definition: CachePeer.h:40
int eof
Definition: http.h:65
const char * termedBuf() const
Definition: SquidString.h:91
bool allowed() const
Definition: Acl.h:143
int cache_miss_revalidate
Definition: SquidConfig.h:325
bool sawDateGoBack
Definition: http.h:159
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:989
int server_pconns
Definition: SquidConfig.h:310
AnyP::ProtocolVersion version
breakdown of protocol version label: (HTTP/ICY) and (0.9/1.0/1.1)
Definition: StatusLine.h:70
const char * doneWithFwd
Definition: Client.h:184
void delayRead(DeferredRead const &)
Definition: MemObject.cc:457
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:344
bool peerSupportsConnectionPinning() const
Definition: http.cc:925
bool timestampsSet()
Definition: store.cc:1449
@ scNoContent
Definition: StatusCode.h:30
@ PROXY_AUTHENTICATE
bool forcedBodyContinuation
whether we have responded with HTTP 100 or FTP 150 already
Definition: HttpRequest.h:195
virtual void maybeReadVirginBody()
read response data from the network
Definition: http.cc:1607
int has(Http::HdrType id) const
Definition: HttpHeader.cc:1007
void processReply()
Definition: http.cc:1317
char * content()
start of the added data
Definition: MemBuf.h:41
#define acl_access
Definition: forward.h:44
bool hasMaxAge(int32_t *val=nullptr) const
Definition: HttpHdrCc.h:122
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1068
void set(const AnyP::ProtocolVersion &newVersion, Http::StatusCode newStatus, const char *newReason=NULL)
Definition: StatusLine.cc:30
bool finishingChunkedRequest()
if needed, write last-chunk to end the request body and return true
Definition: http.cc:2538
size_type size() const
Definition: SquidString.h:72
char * url
Definition: tcp-banger2.c:114
@ ERR_TOO_BIG
Definition: err_type.h:38
bool isEmpty() const
Definition: Store.h:67
bool refreshIsCachable(const StoreEntry *entry)
Definition: refresh.cc:528
bool reforwardableStatus(const Http::StatusCode s) const
Definition: FwdState.cc:1323
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
int redir_rewrites_host
Definition: SquidConfig.h:302
@ SURROGATE_CAPABILITY
@ scProxyAuthenticationRequired
Definition: StatusCode.h:51
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
void Controller::create() STUB void Controller Controller nil
size_t base64_encode_update(struct base64_encode_ctx *ctx, char *dst, size_t length, const uint8_t *src)
Definition: base64.c:265
char * surrogate_id
Definition: SquidConfig.h:216
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
StoreEntry * entry
Definition: FwdState.h:172
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:542
@ ERR_READ_TIMEOUT
Definition: err_type.h:24
void release(const bool shareable=false)
Definition: store.cc:1196
virtual void start()
called by AsyncStart; do not call directly
Definition: http.cc:2483
const HttpReply & freshestReply() const
Definition: MemObject.h:67
ssize_t mb_size_t
Definition: MemBuf.h:17
HttpHdrSc * surrogate_control
Definition: HttpReply.h:52
@ scInvalidHeader
Definition: StatusCode.h:86
@ scPaymentRequired
Definition: StatusCode.h:46
@ scLocked
Definition: StatusCode.h:64
@ ERR_WRITE_ERROR
Definition: err_type.h:27
static StoreEntry * findPreviouslyCachedEntry(StoreEntry *newEntry)
Definition: http.cc:174
@ scConflict
Definition: StatusCode.h:53
int n_keepalives_recv
Definition: CachePeer.h:52
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:98
#define REFRESH_OVERRIDE(flag)
static void httpBuildRequestHeader(HttpRequest *request, StoreEntry *entry, const AccessLogEntryPointer &al, HttpHeader *hdr_out, const Http::StateFlags &flags)
Definition: http.cc:1859
@ srcHttps
https_port or bumped http_port tunnel; HTTPS server
Definition: Message.h:33
virtual void sentRequestBody(const CommIoCbParams &io)
Definition: http.cc:2622
int64_t readAheadGap
Definition: SquidConfig.h:95
void setNoDelay(bool const)
Definition: store.cc:273
bool hasNoCacheWithoutParameters() const
Definition: HttpHdrCc.h:88
bool hasMustRevalidate() const
Definition: HttpHdrCc.h:112
void processSurrogateControl(HttpReply *)
Definition: http.cc:283
int64_t payloadSeen
amount of message payload/body received so far.
Definition: http.h:153
struct SquidConfig::@98 Timeout
bool continueAfterParsingHeader()
Definition: http.cc:1347
static void readDelayed(void *context, CommRead const &)
Definition: http.cc:1211
@ scMultipleChoices
Definition: StatusCode.h:36
const char * urlXXX() const
Definition: MemObject.h:128
bool serverSwitchedProtocols
Definition: StateFlags.h:28
@ scOkay
Definition: StatusCode.h:26
std::ostream & operator<<(std::ostream &os, const HttpStateData::ReuseDecision &d)
Definition: http.cc:2649
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:628
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1070
@ scHeaderTooLarge
Definition: StatusCode.h:87
#define rfc1738_escape_part(x)
Definition: rfc1738.h:55
@ CC_NO_CACHE
Definition: HttpHdrCc.h:23
time_t lifetime
Definition: SquidConfig.h:111
virtual void closeServer()
Definition: http.cc:1726
#define false
Definition: GnuRegex.c:233
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:705
static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest *request, HttpHeader *hdr_out, const int we_do_ranges, const Http::StateFlags &)
Definition: http.cc:2103
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:1015
HttpRequest * request
Definition: FwdState.h:173
Ip::Address client_addr
Definition: HttpRequest.h:149
bool hasNoStore() const
Definition: HttpHdrCc.h:102
void host(const char *src)
Definition: Uri.cc:98
void forwardUpgrade(HttpHeader &)
Definition: http.cc:2041
@ PEER_SIBLING
Definition: enums.h:29
@ scExpectationFailed
Definition: StatusCode.h:61
HttpRequestPointer request
Definition: Client.h:166
int read_hist[histSize]
Definition: IoStats.h:21
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
virtual void noteTakeServerConnectionControl(ServerConnectionContext)
Definition: client_side.h:217
short int keep_alive
Definition: HttpReply.h:57
@ hoRequest
Definition: HttpHeader.h:36
HttpRequestPointer request
Definition: MemObject.h:188
void clean()
Definition: HttpHeader.cc:190
struct CachePeer::@29 stats
void comm_remove_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:1010
const char * getMD5Text() const
Definition: store.cc:205
void const char * buf
Definition: stub_helper.cc:16
~HttpStateData()
Definition: http.cc:130
bool originserver
Definition: CachePeer.h:117
time_t negativeTtl
Definition: SquidConfig.h:99
void ctx_exit(Ctx ctx)
Definition: debug.cc:737
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1239
#define SQUIDSBUFPH
Definition: SBuf.h:31
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1050
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1554
CbcPointer< ConnStateData > clientConnectionManager
Definition: HttpRequest.h:233
#define PROF_start(probename)
Definition: Profiler.h:62
bool hasMaxAge() const
bool hasPublic() const
Definition: HttpHdrCc.h:69
class SquidConfig Config
Definition: SquidConfig.cc:12
struct SquidConfig::@105 Accel
void httpStart(FwdState *fwd)
Definition: http.cc:2476
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
StatCounters statCounter
Definition: StatCounters.cc:12
void removeStaleWarnings()
Remove Warnings with warn-date different from Date value.
Definition: HttpReply.cc:599
void clean()
Definition: String.cc:125
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
@ PROXY_AUTHORIZATION
void maxStale(int v)
#define DBG_DATA
Definition: Debug.h:48
#define CallJobHere1(debugSection, debugLevel, job, Class, method, arg1)
Definition: AsyncJobCalls.h:63
@ STORE_PENDING
Definition: enums.h:51
@ ERR_DETAIL_SRV_REQMOD_REQ_BODY

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors