http.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 11 Hypertext Transfer Protocol (HTTP) */
10 
11 /*
12  * Anonymizing patch by lutz@as-node.jena.thur.de
13  * have a look into http-anon.c to get more information.
14  */
15 
16 #include "squid.h"
17 #include "acl/FilledChecklist.h"
18 #include "base/AsyncJobCalls.h"
19 #include "base/DelayedAsyncCalls.h"
20 #include "base/Raw.h"
21 #include "base/TextException.h"
22 #include "base64.h"
23 #include "CachePeer.h"
24 #include "client_side.h"
25 #include "comm/Connection.h"
26 #include "comm/Read.h"
27 #include "comm/Write.h"
28 #include "error/Detail.h"
29 #include "errorpage.h"
30 #include "fd.h"
31 #include "fde.h"
32 #include "globals.h"
33 #include "http.h"
36 #include "http/Stream.h"
37 #include "HttpControlMsg.h"
38 #include "HttpHdrCc.h"
39 #include "HttpHdrContRange.h"
40 #include "HttpHdrSc.h"
41 #include "HttpHdrScTarget.h"
42 #include "HttpHeaderTools.h"
43 #include "HttpReply.h"
44 #include "HttpRequest.h"
46 #include "log/access_log.h"
47 #include "MemBuf.h"
48 #include "MemObject.h"
49 #include "neighbors.h"
50 #include "pconn.h"
52 #include "refresh.h"
53 #include "RefreshPattern.h"
54 #include "rfc1738.h"
55 #include "SquidConfig.h"
56 #include "StatCounters.h"
57 #include "Store.h"
58 #include "StrList.h"
59 #include "tools.h"
60 #include "util.h"
61 
62 #if USE_AUTH
63 #include "auth/UserRequest.h"
64 #endif
65 #if USE_DELAY_POOLS
66 #include "DelayPools.h"
67 #endif
68 
70 
71 static const char *const crlf = "\r\n";
72 
74 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request,
75  HttpHeader * hdr_out, const int we_do_ranges, const Http::StateFlags &);
76 
78  AsyncJob("HttpStateData"),
79  Client(theFwdState)
80 {
81  debugs(11,5, "HttpStateData " << this << " created");
83 
84  if (fwd->serverConnection() != NULL)
85  _peer = cbdataReference(fwd->serverConnection()->getPeer()); /* might be NULL */
86 
90 
91  if (_peer) {
92  /*
93  * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
94  * We might end up getting the object from somewhere else if,
95  * for example, the request to this neighbor fails.
96  */
98  entry->releaseRequest(true);
99 
100 #if USE_DELAY_POOLS
102 #endif
103  }
104 
105  /*
106  * register the handler to free HTTP state data when the FD closes
107  */
111 }
112 
114 {
115  /*
116  * don't forget that ~Client() gets called automatically
117  */
118 
119  if (httpChunkDecoder)
120  delete httpChunkDecoder;
121 
123 
124  delete upgradeHeaderOut;
125 
126  debugs(11,5, "HttpStateData " << this << " destroyed; " << serverConnection);
127 }
128 
131 {
132  return serverConnection;
133 }
134 
135 void
137 {
138  debugs(11, 5, "httpStateFree: FD " << params.fd << ", httpState=" << params.data);
139  doneWithFwd = "httpStateConnClosed()"; // assume FwdState is monitoring too
140  mustStop("HttpStateData::httpStateConnClosed");
141 }
142 
143 void
145 {
146  debugs(11, 4, serverConnection << ": '" << entry->url() << "'");
147 
148  if (entry->store_status == STORE_PENDING) {
150  }
151 
152  closeServer();
153  mustStop("HttpStateData::httpTimeout");
154 }
155 
156 static StoreEntry *
158  assert(newEntry->mem_obj);
159  return newEntry->mem_obj->request ?
161  storeGetPublic(newEntry->mem_obj->storeId(), newEntry->mem_obj->method);
162 }
163 
166 static void
168 {
169  int remove = 0;
170  int forbidden = 0;
171 
172  // If the incoming response already goes into a public entry, then there is
173  // nothing to remove. This protects ready-for-collapsing entries as well.
174  if (!EBIT_TEST(e->flags, KEY_PRIVATE))
175  return;
176 
177  // If the new/incoming response cannot be stored, then it does not
178  // compete with the old stored response for the public key, and the
179  // old stored response should be left as is.
180  if (e->mem_obj->request && !e->mem_obj->request->flags.cachable)
181  return;
182 
183  switch (status) {
184 
185  case Http::scOkay:
186 
188 
190 
192 
193  case Http::scFound:
194 
195  case Http::scSeeOther:
196 
197  case Http::scGone:
198 
199  case Http::scNotFound:
200  remove = 1;
201 
202  break;
203 
204  case Http::scForbidden:
205 
207  forbidden = 1;
208 
209  break;
210 
211 #if WORK_IN_PROGRESS
212 
214  forbidden = 1;
215 
216  break;
217 
218 #endif
219 
220  default:
221  break;
222  }
223 
224  if (!remove && !forbidden)
225  return;
226 
228 
229  if (pe != NULL) {
230  assert(e != pe);
231 #if USE_HTCP
233 #endif
234  pe->release(true);
235  }
236 
241  if (e->mem_obj->request)
243  else
245 
246  if (pe != NULL) {
247  assert(e != pe);
248 #if USE_HTCP
250 #endif
251  pe->release(true);
252  }
253 }
254 
255 void
257 {
258  if (request->flags.accelerated && reply->surrogate_control) {
260 
261  if (sctusable) {
262  if (sctusable->hasNoStore() ||
264  && sctusable->noStoreRemote())) {
265  surrogateNoStore = true;
266  // Be conservative for now and make it non-shareable because
267  // there is no enough information here to make the decision.
268  entry->makePrivate(false);
269  }
270 
271  /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
272  * accelerated request or not...
273  * Still, this is an abstraction breach. - RC
274  */
275  if (sctusable->hasMaxAge()) {
276  if (sctusable->maxAge() < sctusable->maxStale())
277  reply->expires = reply->date + sctusable->maxAge();
278  else
279  reply->expires = reply->date + sctusable->maxStale();
280 
281  /* And update the timestamps */
282  entry->timestampsSet();
283  }
284 
285  /* We ignore cache-control directives as per the Surrogate specification */
286  ignoreCacheControl = true;
287 
288  delete sctusable;
289  }
290  }
291 }
292 
295 {
296  HttpReply const *rep = finalReply();
297  HttpHeader const *hdr = &rep->header;
298  const char *v;
299 #if USE_HTTP_VIOLATIONS
300 
301  const RefreshPattern *R = NULL;
302 
303  /* This strange looking define first looks up the refresh pattern
304  * and then checks if the specified flag is set. The main purpose
305  * of this is to simplify the refresh pattern lookup and USE_HTTP_VIOLATIONS
306  * condition
307  */
308 #define REFRESH_OVERRIDE(flag) \
309  ((R = (R ? R : refreshLimits(entry->mem_obj->storeId()))) , \
310  (R && R->flags.flag))
311 #else
312 #define REFRESH_OVERRIDE(flag) 0
313 #endif
314 
316  return decision.make(ReuseDecision::doNotCacheButShare, "the entry has been released");
317 
318  // RFC 7234 section 4: a cache MUST use the most recent response
319  // (as determined by the Date header field)
320  // TODO: whether such responses could be shareable?
321  if (sawDateGoBack)
322  return decision.make(ReuseDecision::reuseNot, "the response has an older date header");
323 
324  // Check for Surrogate/1.0 protocol conditions
325  // NP: reverse-proxy traffic our parent server has instructed us never to cache
326  if (surrogateNoStore)
327  return decision.make(ReuseDecision::reuseNot, "Surrogate-Control:no-store");
328 
329  // RFC 2616: HTTP/1.1 Cache-Control conditions
330  if (!ignoreCacheControl) {
331  // XXX: check to see if the request headers alone were enough to prevent caching earlier
332  // (ie no-store request header) no need to check those all again here if so.
333  // for now we are not reliably doing that so we waste CPU re-checking request CC
334 
335  // RFC 2616 section 14.9.2 - MUST NOT cache any response with request CC:no-store
337  !REFRESH_OVERRIDE(ignore_no_store))
338  return decision.make(ReuseDecision::reuseNot,
339  "client request Cache-Control:no-store");
340 
341  // NP: request CC:no-cache only means cache READ is forbidden. STORE is permitted.
343  /* TODO: we are allowed to cache when no-cache= has parameters.
344  * Provided we strip away any of the listed headers unless they are revalidated
345  * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
346  * That is a bit tricky for squid right now so we avoid caching entirely.
347  */
348  return decision.make(ReuseDecision::reuseNot,
349  "server reply Cache-Control:no-cache has parameters");
350  }
351 
352  // NP: request CC:private is undefined. We ignore.
353  // NP: other request CC flags are limiters on HIT/MISS. We don't care about here.
354 
355  // RFC 2616 section 14.9.2 - MUST NOT cache any response with CC:no-store
356  if (rep->cache_control && rep->cache_control->hasNoStore() &&
357  !REFRESH_OVERRIDE(ignore_no_store))
358  return decision.make(ReuseDecision::reuseNot,
359  "server reply Cache-Control:no-store");
360 
361  // RFC 2616 section 14.9.1 - MUST NOT cache any response with CC:private in a shared cache like Squid.
362  // CC:private overrides CC:public when both are present in a response.
363  // TODO: add a shared/private cache configuration possibility.
364  if (rep->cache_control &&
365  rep->cache_control->hasPrivate() &&
366  !REFRESH_OVERRIDE(ignore_private)) {
367  /* TODO: we are allowed to cache when private= has parameters.
368  * Provided we strip away any of the listed headers unless they are revalidated
369  * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
370  * That is a bit tricky for squid right now so we avoid caching entirely.
371  */
372  return decision.make(ReuseDecision::reuseNot,
373  "server reply Cache-Control:private");
374  }
375  }
376 
377  // RFC 2068, sec 14.9.4 - MUST NOT cache any response with Authentication UNLESS certain CC controls are present
378  // allow HTTP violations to IGNORE those controls (ie re-block caching Auth)
379  if (request && (request->flags.auth || request->flags.authSent)) {
380  if (!rep->cache_control)
381  return decision.make(ReuseDecision::reuseNot,
382  "authenticated and server reply missing Cache-Control");
383 
384  if (ignoreCacheControl)
385  return decision.make(ReuseDecision::reuseNot,
386  "authenticated and ignoring Cache-Control");
387 
388  bool mayStore = false;
389  // HTTPbis pt6 section 3.2: a response CC:public is present
390  if (rep->cache_control->hasPublic()) {
391  debugs(22, 3, "Authenticated but server reply Cache-Control:public");
392  mayStore = true;
393 
394  // HTTPbis pt6 section 3.2: a response CC:must-revalidate is present
395  } else if (rep->cache_control->hasMustRevalidate()) {
396  debugs(22, 3, "Authenticated but server reply Cache-Control:must-revalidate");
397  mayStore = true;
398 
399 #if USE_HTTP_VIOLATIONS
400  // NP: given the must-revalidate exception we should also be able to exempt no-cache.
401  // HTTPbis WG verdict on this is that it is omitted from the spec due to being 'unexpected' by
402  // some. The caching+revalidate is not exactly unsafe though with Squids interpretation of no-cache
403  // (without parameters) as equivalent to must-revalidate in the reply.
404  } else if (rep->cache_control->hasNoCacheWithoutParameters()) {
405  debugs(22, 3, "Authenticated but server reply Cache-Control:no-cache (equivalent to must-revalidate)");
406  mayStore = true;
407 #endif
408 
409  // HTTPbis pt6 section 3.2: a response CC:s-maxage is present
410  } else if (rep->cache_control->hasSMaxAge()) {
411  debugs(22, 3, "Authenticated but server reply Cache-Control:s-maxage");
412  mayStore = true;
413  }
414 
415  if (!mayStore)
416  return decision.make(ReuseDecision::reuseNot, "authenticated transaction");
417 
418  // NP: response CC:no-cache is equivalent to CC:must-revalidate,max-age=0. We MAY cache, and do so.
419  // NP: other request CC flags are limiters on HIT/MISS/REFRESH. We don't care about here.
420  }
421 
422  /* HACK: The "multipart/x-mixed-replace" content type is used for
423  * continuous push replies. These are generally dynamic and
424  * probably should not be cachable
425  */
426  if ((v = hdr->getStr(Http::HdrType::CONTENT_TYPE)))
427  if (!strncasecmp(v, "multipart/x-mixed-replace", 25))
428  return decision.make(ReuseDecision::reuseNot, "Content-Type:multipart/x-mixed-replace");
429 
430  // TODO: if possible, provide more specific message for each status code
431  static const char *shareableError = "shareable error status code";
432  static const char *nonShareableError = "non-shareable error status code";
434  const char *statusReason = nonShareableError;
435 
436  switch (rep->sline.status()) {
437 
438  /* There are several situations when a non-cacheable response may be
439  * still shareable (e.g., among collapsed clients). We assume that these
440  * are 3xx and 5xx responses, indicating server problems and some of
441  * 4xx responses, common for all clients with a given cache key (e.g.,
442  * 404 Not Found or 414 URI Too Long). On the other hand, we should not
443  * share non-cacheable client-specific errors, such as 400 Bad Request
444  * or 406 Not Acceptable.
445  */
446 
447  /* Responses that are cacheable */
448 
449  case Http::scOkay:
450 
452 
454 
457 
458  case Http::scGone:
459  /*
460  * Don't cache objects that need to be refreshed on next request,
461  * unless we know how to refresh it.
462  */
463 
464  if (refreshIsCachable(entry) || REFRESH_OVERRIDE(store_stale))
465  decision.make(ReuseDecision::cachePositively, "refresh check returned cacheable");
466  else
467  decision.make(ReuseDecision::doNotCacheButShare, "refresh check returned non-cacheable");
468  break;
469 
470  /* Responses that only are cacheable if the server says so */
471 
472  case Http::scFound:
474  if (rep->date <= 0)
475  decision.make(ReuseDecision::doNotCacheButShare, "Date is missing/invalid");
476  else if (rep->expires > rep->date)
477  decision.make(ReuseDecision::cachePositively, "Expires > Date");
478  else
479  decision.make(ReuseDecision::doNotCacheButShare, "Expires <= Date");
480  break;
481 
482  /* These responses can be negatively cached. Most can also be shared. */
483  case Http::scNoContent:
484  case Http::scUseProxy:
485  case Http::scForbidden:
486  case Http::scNotFound:
488  case Http::scUriTooLong:
491  case Http::scBadGateway:
495  statusAnswer = ReuseDecision::doNotCacheButShare;
496  statusReason = shareableError;
497  /* [[fallthrough]] to the actual decision making below */
498 
499  case Http::scBadRequest: // no sharing; perhaps the server did not like something specific to this request
500 #if USE_HTTP_VIOLATIONS
501  if (Config.negativeTtl > 0)
502  decision.make(ReuseDecision::cacheNegatively, "Config.negativeTtl > 0");
503  else
504 #endif
505  decision.make(statusAnswer, statusReason);
506  break;
507 
508  /* these responses can never be cached, some
509  of them can be shared though */
510  case Http::scSeeOther:
511  case Http::scNotModified:
516  // TODO: use more specific reason for non-error status codes
517  decision.make(ReuseDecision::doNotCacheButShare, shareableError);
518  break;
519 
520  case Http::scPartialContent: /* Not yet supported. TODO: make shareable for suitable ranges */
522  case Http::scRequestTimeout: // TODO: is this shareable?
523  case Http::scConflict: // TODO: is this shareable?
529  case Http::scLocked: // TODO: is this shareable?
533  case Http::scInvalidHeader: /* Squid header parsing error */
535  decision.make(ReuseDecision::reuseNot, nonShareableError);
536  break;
537 
538  default:
539  /* RFC 2616 section 6.1.1: an unrecognized response MUST NOT be cached. */
540  decision.make(ReuseDecision::reuseNot, "unknown status code");
541  break;
542  }
543 
544  return decision.answer;
545 }
546 
548 static void
550 {
551  static const SBuf asterisk("*");
552  const char *pos = nullptr;
553  const char *item = nullptr;
554  int ilen = 0;
555 
556  while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
557  SBuf name(item, ilen);
558  if (name == asterisk) {
559  vstr = asterisk;
560  break;
561  }
562  name.toLower();
563  if (!vstr.isEmpty())
564  vstr.append(", ", 2);
565  vstr.append(name);
566  String hdr(request.header.getByName(name));
567  const char *value = hdr.termedBuf();
568  if (value) {
569  value = rfc1738_escape_part(value);
570  vstr.append("=\"", 2);
571  vstr.append(value);
572  vstr.append("\"", 1);
573  }
574 
575  hdr.clean();
576  }
577 }
578 
579 /*
580  * For Vary, store the relevant request headers as
581  * virtual headers in the reply
582  * Returns an empty SBuf if the variance cannot be stored
583  */
584 SBuf
586 {
587  SBuf vstr;
588  String vary;
589 
590  vary = reply->header.getList(Http::HdrType::VARY);
591  assembleVaryKey(vary, vstr, *request);
592 
593 #if X_ACCELERATOR_VARY
594  vary.clean();
596  assembleVaryKey(vary, vstr, *request);
597 #endif
598 
599  debugs(11, 3, vstr);
600  return vstr;
601 }
602 
603 void
605 {
606  if (flags.keepalive)
607  if (flags.peering && !flags.tunneling)
609 
610  if (reply->keep_alive) {
611  if (flags.peering && !flags.tunneling)
613 
615  && reply->bodySize(request->method) == -1 && !flags.chunked) {
616  debugs(11, DBG_IMPORTANT, "keepaliveAccounting: Impossible keep-alive header from '" << entry->url() << "'" );
617  // debugs(11, 2, "GOT HTTP REPLY HDR:\n---------\n" << readBuf->content() << "\n----------" );
618  flags.keepalive_broken = true;
619  }
620  }
621 }
622 
623 void
625 {
626  if (reply->date > -1 && flags.toOrigin) {
627  int skew = abs((int)(reply->date - squid_curtime));
628 
629  if (skew > 86400)
630  debugs(11, 3, "" << request->url.host() << "'s clock is skewed by " << skew << " seconds!");
631  }
632 }
633 
640 void
642 {
645  debugs(11, 3, "processReplyHeader: key '" << entry->getMD5Text() << "'");
646 
648 
649  if (!inBuf.length())
650  return;
651 
652  /* Attempt to parse the first line; this will define where the protocol, status, reason-phrase and header begin */
653  {
654  if (hp == NULL)
656 
657  bool parsedOk = hp->parse(inBuf);
658  // remember the actual received status-code before returning on errors,
659  // overwriting any previously stored value from earlier forwarding attempts
660  request->hier.peer_reply_status = hp->messageStatus(); // may still be scNone
661 
662  // sync the buffers after parsing.
663  inBuf = hp->remaining();
664 
665  if (hp->needsMoreData()) {
666  if (eof) { // no more data coming
667  assert(!parsedOk);
668  // fall through to handle this premature EOF as an error
669  } else {
670  debugs(33, 5, "Incomplete response, waiting for end of response headers");
671  return;
672  }
673  }
674 
675  if (!parsedOk) {
676  // unrecoverable parsing error
677  // TODO: Use Raw! XXX: inBuf no longer has the [beginning of the] malformed header.
678  debugs(11, 3, "Non-HTTP-compliant header:\n---------\n" << inBuf << "\n----------");
679  flags.headers_parsed = true;
680  HttpReply *newrep = new HttpReply;
681  // hp->needsMoreData() means hp->parseStatusCode is unusable, but, here,
682  // it also means that the reply header got truncated by a premature EOF
683  assert(!hp->needsMoreData() || eof);
684  const auto scode = hp->needsMoreData() ? Http::scInvalidHeader : hp->parseStatusCode;
685  newrep->sline.set(Http::ProtocolVersion(), scode);
686  setVirginReply(newrep);
687  return;
688  }
689  }
690 
691  /* We know the whole response is in parser now */
692  debugs(11, 2, "HTTP Server " << serverConnection);
693  debugs(11, 2, "HTTP Server RESPONSE:\n---------\n" <<
694  hp->messageProtocol() << " " << hp->messageStatus() << " " << hp->reasonPhrase() << "\n" <<
695  hp->mimeHeader() <<
696  "----------");
697 
698  // reset payload tracking to begin after message headers
700 
701  HttpReply *newrep = new HttpReply;
702  // XXX: RFC 7230 indicates we MAY ignore the reason phrase,
703  // and use an empty string on unknown status.
704  // We do that now to avoid performance regression from using SBuf::c_str()
705  newrep->sline.set(hp->messageProtocol(), hp->messageStatus() /* , hp->reasonPhrase() */);
706 
707  // parse headers
708  if (!newrep->parseHeader(*hp)) {
709  newrep->sline.set(hp->messageProtocol(), Http::scInvalidHeader);
710  debugs(11, 2, "error parsing response headers mime block");
711  }
712 
713  // done with Parser, now process using the HttpReply
714  hp = NULL;
715 
717 
718  newrep->removeStaleWarnings();
719 
720  if (newrep->sline.version.protocol == AnyP::PROTO_HTTP && Http::Is1xx(newrep->sline.status())) {
721  handle1xx(newrep);
722  return;
723  }
724 
725  flags.chunked = false;
726  if (newrep->sline.version.protocol == AnyP::PROTO_HTTP && newrep->header.chunked()) {
727  flags.chunked = true;
729  }
730 
733 
734  HttpReply *vrep = setVirginReply(newrep);
735  flags.headers_parsed = true;
736 
737  keepaliveAccounting(vrep);
738 
739  checkDateSkew(vrep);
740 
742 }
743 
745 void
747 {
748  if (fwd->al)
749  fwd->al->reply = reply;
750 
751  HttpReply::Pointer msg(reply); // will destroy reply if unused
752 
753  // one 1xx at a time: we must not be called while waiting for previous 1xx
755  flags.handling1xx = true;
756 
757  const auto statusCode = reply->sline.status();
758 
759  // drop1xx() needs to handle HTTP 101 (Switching Protocols) responses
760  // specially because they indicate that the server has stopped speaking HTTP
763 
764  if (statusCode == Http::scContinue && request->forcedBodyContinuation)
765  return drop1xx("we have sent it already");
766 
767  if (!request->canHandle1xx())
768  return drop1xx("the client does not support it");
769 
770 #if USE_HTTP_VIOLATIONS
771  // check whether the 1xx response forwarding is allowed by squid.conf
772  if (Config.accessList.reply) {
774  ch.al = fwd->al;
775  ch.reply = reply;
776  ch.syncAle(originalRequest().getRaw(), nullptr);
777  HTTPMSGLOCK(ch.reply);
778  if (!ch.fastCheck().allowed()) // TODO: support slow lookups?
779  return drop1xx("http_reply_access blocked it");
780  }
781 #endif // USE_HTTP_VIOLATIONS
782 
784  if (const auto reason = blockSwitchingProtocols(*reply))
785  return drop1xx(reason);
786  }
787 
788  debugs(11, 2, "forwarding 1xx to client");
789 
790  // the Sink will use this to call us back after writing 1xx to the client
791  typedef NullaryMemFunT<HttpStateData> CbDialer;
792  const AsyncCall::Pointer cb = JobCallback(11, 3, CbDialer, this,
796  // If the call is not fired, then the Sink is gone, and HttpStateData
797  // will terminate due to an aborted store entry or another similar error.
798  // If we get stuck, it is not handle1xx fault if we could get stuck
799  // for similar reasons without a 1xx response.
800 }
801 
804 void
805 HttpStateData::drop1xx(const char *reason)
806 {
808  debugs(11, 2, "bad 101 because " << reason);
809  const auto err = new ErrorState(ERR_INVALID_RESP, Http::scBadGateway, request.getRaw(), fwd->al);
810  fwd->fail(err);
811  closeServer();
812  mustStop("prohibited HTTP/101 response");
813  return;
814  }
815 
816  debugs(11, 2, "ignoring 1xx because " << reason);
817  proceedAfter1xx();
818 }
819 
822 const char *
824 {
825  if (!upgradeHeaderOut)
826  return "Squid offered no Upgrade at all, but server switched to a tunnel";
827 
828  // See RFC 7230 section 6.7 for the corresponding MUSTs
829 
830  if (!reply.header.has(Http::HdrType::UPGRADE))
831  return "server did not send an Upgrade header field";
832 
833  if (!reply.header.hasListMember(Http::HdrType::CONNECTION, "upgrade", ','))
834  return "server did not send 'Connection: upgrade'";
835 
836  const auto acceptedProtos = reply.header.getList(Http::HdrType::UPGRADE);
837  const char *pos = nullptr;
838  const char *accepted = nullptr;
839  int acceptedLen = 0;
840  while (strListGetItem(&acceptedProtos, ',', &accepted, &acceptedLen, &pos)) {
841  debugs(11, 5, "server accepted at least" << Raw(nullptr, accepted, acceptedLen));
842  return nullptr; // OK: let the client validate server's selection
843  }
844 
845  return "server sent an essentially empty Upgrade header field";
846 }
847 
849 void
851 {
853 
855  // pass server connection ownership to request->clientConnectionManager
858  AsyncCall::Pointer call = asyncCall(11, 3, "ConnStateData::noteTakeServerConnectionControl",
861  ScheduleCallHere(call);
864  closeHandler = nullptr;
865  serverConnection = nullptr;
866  doneWithFwd = "switched protocols";
868  return;
869  }
870 
871  debugs(11, 2, "continuing with " << payloadSeen << " bytes in buffer after 1xx");
873 }
874 
878 bool
880 {
881  if (!_peer)
882  return true;
883 
884  // we are talking "through" rather than "to" our _peer
885  if (flags.tunneling)
886  return true;
887 
888  /*If this peer does not support connection pinning (authenticated
889  connections) return false
890  */
891  if (!_peer->connection_auth)
892  return false;
893 
894  const auto &rep = entry->mem().freshestReply();
895 
896  /*The peer supports connection pinning and the http reply status
897  is not unauthorized, so the related connection can be pinned
898  */
899  if (rep.sline.status() != Http::scUnauthorized)
900  return true;
901 
902  /*The server respond with Http::scUnauthorized and the peer configured
903  with "connection-auth=on" we know that the peer supports pinned
904  connections
905  */
906  if (_peer->connection_auth == 1)
907  return true;
908 
909  /*At this point peer has configured with "connection-auth=auto"
910  parameter so we need some extra checks to decide if we are going
911  to allow pinned connections or not
912  */
913 
914  /*if the peer configured with originserver just allow connection
915  pinning (squid 2.6 behaviour)
916  */
918  return true;
919 
920  /*if the connections it is already pinned it is OK*/
921  if (request->flags.pinned)
922  return true;
923 
924  /*Allow pinned connections only if the Proxy-support header exists in
925  reply and has in its list the "Session-Based-Authentication"
926  which means that the peer supports connection pinning.
927  */
928  if (rep.header.hasListMember(Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication", ','))
929  return true;
930 
931  return false;
932 }
933 
934 // Called when we parsed (and possibly adapted) the headers but
935 // had not starting storing (a.k.a., sending) the body yet.
936 void
938 {
940 
941  HttpReply *rep = finalReply();
942  const Http::StatusCode statusCode = rep->sline.status();
943 
944  entry->timestampsSet();
945 
946  /* Check if object is cacheable or not based on reply code */
947  debugs(11, 3, "HTTP CODE: " << statusCode);
948 
949  if (StoreEntry *oldEntry = findPreviouslyCachedEntry(entry)) {
950  oldEntry->lock("HttpStateData::haveParsedReplyHeaders");
951  sawDateGoBack = rep->olderThan(oldEntry->hasFreshestReply());
952  oldEntry->unlock("HttpStateData::haveParsedReplyHeaders");
953  }
954 
957 
958  bool varyFailure = false;
959  if (rep->header.has(Http::HdrType::VARY)
960 #if X_ACCELERATOR_VARY
962 #endif
963  ) {
964  const SBuf vary(httpMakeVaryMark(request.getRaw(), rep));
965 
966  if (vary.isEmpty()) {
967  // TODO: check whether such responses are shareable.
968  // Do not share for now.
969  entry->makePrivate(false);
970  if (fwd->reforwardableStatus(rep->sline.status()))
972  varyFailure = true;
973  } else {
974  entry->mem_obj->vary_headers = vary;
975 
976  // RFC 7231 section 7.1.4
977  // Vary:* can be cached, but has mandatory revalidation
978  static const SBuf asterisk("*");
979  if (vary == asterisk)
981  }
982  }
983 
984  if (!varyFailure) {
985  /*
986  * If its not a reply that we will re-forward, then
987  * allow the client to get it.
988  */
989  if (fwd->reforwardableStatus(rep->sline.status()))
991 
992  ReuseDecision decision(entry, statusCode);
993 
994  switch (reusableReply(decision)) {
995 
997  entry->makePrivate(false);
998  break;
999 
1001  if (!entry->makePublic()) {
1002  decision.make(ReuseDecision::doNotCacheButShare, "public key creation error");
1003  entry->makePrivate(true);
1004  }
1005  break;
1006 
1008  if (!entry->cacheNegatively()) {
1009  decision.make(ReuseDecision::doNotCacheButShare, "public key creation error");
1010  entry->makePrivate(true);
1011  }
1012  break;
1013 
1015  entry->makePrivate(true);
1016  break;
1017 
1018  default:
1019  assert(0);
1020  break;
1021  }
1022  debugs(11, 3, "decided: " << decision);
1023  }
1024 
1025  if (!ignoreCacheControl) {
1026  if (rep->cache_control) {
1027  // We are required to revalidate on many conditions.
1028  // For security reasons we do so even if storage was caused by refresh_pattern ignore-* option
1029 
1030  // CC:must-revalidate or CC:proxy-revalidate
1031  const bool ccMustRevalidate = (rep->cache_control->hasProxyRevalidate() || rep->cache_control->hasMustRevalidate());
1032 
1033  // CC:no-cache (only if there are no parameters)
1034  const bool ccNoCacheNoParams = rep->cache_control->hasNoCacheWithoutParameters();
1035 
1036  // CC:s-maxage=N
1037  const bool ccSMaxAge = rep->cache_control->hasSMaxAge();
1038 
1039  // CC:private (yes, these can sometimes be stored)
1040  const bool ccPrivate = rep->cache_control->hasPrivate();
1041 
1042  if (ccNoCacheNoParams || ccPrivate)
1044  else if (ccMustRevalidate || ccSMaxAge)
1046  }
1047 #if USE_HTTP_VIOLATIONS // response header Pragma::no-cache is undefined in HTTP
1048  else {
1049  // Expensive calculation. So only do it IF the CC: header is not present.
1050 
1051  /* HACK: Pragma: no-cache in _replies_ is not documented in HTTP,
1052  * but servers like "Active Imaging Webcast/2.0" sure do use it */
1053  if (rep->header.has(Http::HdrType::PRAGMA) &&
1054  rep->header.hasListMember(Http::HdrType::PRAGMA,"no-cache",','))
1056  }
1057 #endif
1058  }
1059 
1060 #if HEADERS_LOG
1061  headersLog(1, 0, request->method, rep);
1062 
1063 #endif
1064 }
1065 
1068 {
1069  const HttpReply *rep = virginReply();
1073  static SBuf close("close", 5);
1074  if (httpHeaderHasConnDir(&rep->header, close))
1076 
1081  if (!flags.keepalive)
1083 
1087  if (flags.forceClose)
1089 
1094  if (!flags.request_sent) {
1095  debugs(11, 2, "Request not yet fully sent " << request->method << ' ' << entry->url());
1097  }
1098 
1111  if (!rep->keep_alive)
1113 
1114  return COMPLETE_PERSISTENT_MSG;
1115 }
1116 
1119 {
1120  debugs(11, 3, serverConnection << " eof=" << eof);
1121  if (eof) // already reached EOF
1123 
1124  /* If server fd is closing (but we have not been notified yet), stop Comm
1125  I/O to avoid assertions. TODO: Change Comm API to handle callers that
1126  want more I/O after async closing (usually initiated by others). */
1127  // XXX: add canReceive or s/canSend/canTalkToServer/
1130 
1136  if (lastChunk && flags.chunked)
1137  return statusIfComplete();
1138 
1139  const HttpReply *vrep = virginReply();
1140  debugs(11, 5, "persistentConnStatus: content_length=" << vrep->content_length);
1141 
1142  const int64_t clen = vrep->bodySize(request->method);
1143 
1144  debugs(11, 5, "persistentConnStatus: clen=" << clen);
1145 
1146  /* If the body size is unknown we must wait for EOF */
1147  if (clen < 0)
1148  return INCOMPLETE_MSG;
1149 
1152  if (clen > 0) {
1153  debugs(11,5, "payloadSeen=" << payloadSeen << " content_length=" << vrep->content_length);
1154 
1155  if (payloadSeen < vrep->content_length)
1156  return INCOMPLETE_MSG;
1157 
1158  if (payloadTruncated > 0) // already read more than needed
1159  return COMPLETE_NONPERSISTENT_MSG; // disable pconns
1160  }
1161 
1164  return statusIfComplete();
1165 }
1166 
1167 void
1169 {
1170  flags.do_next_read = true;
1172 }
1173 
1174 void
1176 {
1177  Must(!flags.do_next_read); // XXX: should have been set false by mayReadVirginBody()
1178  flags.do_next_read = false;
1179 
1180  debugs(11, 5, io.conn);
1181 
1182  // Bail out early on Comm::ERR_CLOSING - close handlers will tidy up for us
1183  if (io.flag == Comm::ERR_CLOSING) {
1184  debugs(11, 3, "http socket closing");
1185  return;
1186  }
1187 
1189  abortTransaction("store entry aborted while reading reply");
1190  return;
1191  }
1192 
1194  Must(io.conn->fd == serverConnection->fd);
1195 
1196  /*
1197  * Don't reset the timeout value here. The value should be
1198  * counting Config.Timeout.request and applies to the request
1199  * as a whole, not individual read() calls.
1200  * Plus, it breaks our lame *HalfClosed() detection
1201  */
1202 
1204  CommIoCbParams rd(this); // will be expanded with ReadNow results
1205  rd.conn = io.conn;
1207 
1208  if (rd.size <= 0) {
1209  delayRead();
1210  return;
1211  }
1212 
1213  switch (Comm::ReadNow(rd, inBuf)) {
1214  case Comm::INPROGRESS:
1215  if (inBuf.isEmpty())
1216  debugs(33, 2, io.conn << ": no data to process, " << xstrerr(rd.xerrno));
1217  flags.do_next_read = true;
1219  return;
1220 
1221  case Comm::OK:
1222  {
1223  payloadSeen += rd.size;
1224 #if USE_DELAY_POOLS
1225  DelayId delayId = entry->mem_obj->mostBytesAllowed();
1226  delayId.bytesIn(rd.size);
1227 #endif
1228 
1229  statCounter.server.all.kbytes_in += rd.size;
1230  statCounter.server.http.kbytes_in += rd.size;
1231  ++ IOStats.Http.reads;
1232 
1233  int bin = 0;
1234  for (int clen = rd.size - 1; clen; ++bin)
1235  clen >>= 1;
1236 
1237  ++ IOStats.Http.read_hist[bin];
1238 
1240  }
1241 
1242  /* Continue to process previously read data */
1243  break;
1244 
1245  case Comm::ENDFILE: // close detected by 0-byte read
1246  eof = 1;
1247  flags.do_next_read = false;
1248 
1249  /* Continue to process previously read data */
1250  break;
1251 
1252  // case Comm::COMM_ERROR:
1253  default: // no other flags should ever occur
1254  debugs(11, 2, io.conn << ": read failure: " << xstrerr(rd.xerrno));
1255  const auto err = new ErrorState(ERR_READ_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1256  err->xerrno = rd.xerrno;
1257  fwd->fail(err);
1258  flags.do_next_read = false;
1259  closeServer();
1260  mustStop("HttpStateData::readReply");
1261  return;
1262  }
1263 
1264  /* Process next response from buffer */
1265  processReply();
1266 }
1267 
1270 void
1272 {
1273 
1274  if (flags.handling1xx) { // we came back after handling a 1xx response
1275  debugs(11, 5, "done with 1xx handling");
1276  flags.handling1xx = false;
1278  }
1279 
1281  abortTransaction("store entry aborted while we were waiting for processReply()");
1282  return;
1283  }
1284 
1285  if (!flags.headers_parsed) { // have not parsed headers yet?
1287 
1288  if (!continueAfterParsingHeader()) // parsing error or need more data
1289  return; // TODO: send errors to ICAP
1290 
1291  adaptOrFinalizeReply(); // may write to, abort, or "close" the entry
1292  }
1293 
1294  // kick more reads if needed and/or process the response body, if any
1295  processReplyBody(); // may call serverComplete()
1296 }
1297 
1301 bool
1303 {
1304  if (flags.handling1xx) {
1305  debugs(11, 5, "wait for 1xx handling");
1307  return false;
1308  }
1309 
1310  if (!flags.headers_parsed && !eof) {
1311  debugs(11, 9, "needs more at " << inBuf.length());
1312  flags.do_next_read = true;
1316  maybeReadVirginBody(); // schedules all kinds of reads; TODO: rename
1317  return false;
1318  }
1319 
1323 
1324  if (flags.headers_parsed) { // parsed headers, possibly with errors
1325  // check for header parsing errors
1326  if (HttpReply *vrep = virginReply()) {
1327  const Http::StatusCode s = vrep->sline.status();
1328  const AnyP::ProtocolVersion &v = vrep->sline.version;
1329  if (s == Http::scInvalidHeader && v != Http::ProtocolVersion(0,9)) {
1330  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Bad header encountered from " << entry->url() << " AKA " << request->url);
1332  } else if (s == Http::scHeaderTooLarge) {
1333  fwd->dontRetry(true);
1334  error = ERR_TOO_BIG;
1335  } else if (vrep->header.conflictingContentLength()) {
1336  fwd->dontRetry(true);
1338  } else if (vrep->header.unsupportedTe()) {
1339  fwd->dontRetry(true);
1341  } else {
1342  return true; // done parsing, got reply, and no error
1343  }
1344  } else {
1345  // parsed headers but got no reply
1346  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: No reply at all for " << entry->url() << " AKA " << request->url);
1348  }
1349  } else {
1350  assert(eof);
1351  if (inBuf.length()) {
1353  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry->url() << " AKA " << request->url);
1354  } else {
1356  debugs(11, (request->flags.accelerated?DBG_IMPORTANT:2), "WARNING: HTTP: Invalid Response: No object data received for " << entry->url() << " AKA " << request->url);
1357  }
1358  }
1359 
1360  assert(error != ERR_NONE);
1361  entry->reset();
1363  flags.do_next_read = false;
1364  closeServer();
1365  mustStop("HttpStateData::continueAfterParsingHeader");
1366  return false; // quit on error
1367 }
1368 
1371 void
1373 {
1375 
1376  HttpReply *vrep = virginReply();
1377  int64_t clen = -1;
1378  if (!vrep->expectingBody(request->method, clen) || clen < 0)
1379  return; // no body or a body of unknown size, including chunked
1380 
1381  if (payloadSeen - payloadTruncated <= clen)
1382  return; // we did not read too much or already took care of the extras
1383 
1384  if (const int64_t extras = payloadSeen - payloadTruncated - clen) {
1385  // server sent more that the advertised content length
1386  debugs(11, 5, "payloadSeen=" << payloadSeen <<
1387  " clen=" << clen << '/' << vrep->content_length <<
1388  " truncated=" << payloadTruncated << '+' << extras);
1389 
1390  inBuf.chop(0, inBuf.length() - extras);
1391  payloadTruncated += extras;
1392  }
1393 }
1394 
1399 void
1401 {
1402  truncateVirginBody(); // if needed
1403  const char *data = inBuf.rawContent();
1404  int len = inBuf.length();
1405  addVirginReplyBody(data, len);
1406  inBuf.consume(len);
1407 
1408  // after addVirginReplyBody() wrote (when not adapting) everything we have
1409  // received to Store, check whether we have received/parsed the entire reply
1410  int64_t clen = -1;
1411  const char *parsedWhole = nullptr;
1412  if (!virginReply()->expectingBody(request->method, clen))
1413  parsedWhole = "http parsed header-only reply";
1414  else if (clen >= 0 && clen == payloadSeen - payloadTruncated)
1415  parsedWhole = "http parsed Content-Length body bytes";
1416  else if (clen < 0 && eof)
1417  parsedWhole = "http parsed body ending with expected/required EOF";
1418  if (parsedWhole)
1419  markParsedVirginReplyAsWhole(parsedWhole);
1420 }
1421 
1422 bool
1424 {
1425  assert(flags.chunked);
1427  try {
1428  MemBuf decodedData;
1429  decodedData.init();
1430  httpChunkDecoder->setPayloadBuffer(&decodedData);
1431  const bool doneParsing = httpChunkDecoder->parse(inBuf);
1432  inBuf = httpChunkDecoder->remaining(); // sync buffers after parse
1433  addVirginReplyBody(decodedData.content(), decodedData.contentSize());
1434  if (doneParsing) {
1435  lastChunk = 1;
1436  flags.do_next_read = false;
1437  markParsedVirginReplyAsWhole("http parsed last-chunk");
1438  }
1439  return true;
1440  }
1441  catch (...) {
1442  debugs (11, 2, "de-chunking failure: " << CurrentException);
1443  }
1444  return false;
1445 }
1446 
1454 void
1456 {
1457  if (!flags.headers_parsed) {
1458  flags.do_next_read = true;
1460  return;
1461  }
1462 
1463 #if USE_ADAPTATION
1464  debugs(11,5, "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
1466  return;
1467 
1468 #endif
1469 
1470  /*
1471  * At this point the reply headers have been parsed and consumed.
1472  * That means header content has been removed from readBuf and
1473  * it contains only body data.
1474  */
1475  if (entry->isAccepting()) {
1476  if (flags.chunked) {
1477  if (!decodeAndWriteReplyBody()) {
1478  flags.do_next_read = false;
1479  serverComplete();
1480  return;
1481  }
1482  } else
1483  writeReplyBody();
1484  }
1485 
1486  // storing/sending methods like earlier adaptOrFinalizeReply() or
1487  // above writeReplyBody() may release/abort the store entry.
1489  // TODO: In some cases (e.g., 304), we should keep persistent conn open.
1490  // Detect end-of-reply (and, hence, pool our idle pconn) earlier (ASAP).
1491  abortTransaction("store entry aborted while storing reply");
1492  return;
1493  } else
1494  switch (persistentConnStatus()) {
1495  case INCOMPLETE_MSG: {
1496  debugs(11, 5, "processReplyBody: INCOMPLETE_MSG from " << serverConnection);
1497  /* Wait for more data or EOF condition */
1499  if (flags.keepalive_broken) {
1501  } else {
1503  }
1504 
1505  flags.do_next_read = true;
1506  }
1507  break;
1508 
1509  case COMPLETE_PERSISTENT_MSG: {
1510  debugs(11, 5, "processReplyBody: COMPLETE_PERSISTENT_MSG from " << serverConnection);
1511 
1512  // TODO: Remove serverConnectionSaved but preserve exception safety.
1513 
1515  flags.do_next_read = false;
1516 
1518  closeHandler = NULL;
1519 
1520  Ip::Address client_addr; // XXX: Remove as unused. Why was it added?
1522  client_addr = request->client_addr;
1523 
1524  auto serverConnectionSaved = serverConnection;
1526  serverConnection = nullptr;
1527 
1528  bool ispinned = false; // TODO: Rename to isOrShouldBePinned
1529  if (request->flags.pinned) {
1530  ispinned = true;
1531  } else if (request->flags.connectionAuth && request->flags.authSent) {
1532  ispinned = true;
1533  }
1534 
1535  if (ispinned) {
1538  ConnStateData,
1539  notePinnedConnectionBecameIdle,
1540  ConnStateData::PinnedIdleContext(serverConnectionSaved, request));
1541  } else {
1542  // must not pool/share ispinned connections, even orphaned ones
1543  serverConnectionSaved->close();
1544  }
1545  } else {
1546  fwdPconnPool->push(serverConnectionSaved, request->url.host());
1547  }
1548 
1549  serverComplete();
1550  return;
1551  }
1552 
1554  debugs(11, 5, "processReplyBody: COMPLETE_NONPERSISTENT_MSG from " << serverConnection);
1555 
1556  serverComplete();
1557  return;
1558  }
1559 
1561 }
1562 
1563 bool
1565 {
1566  // TODO: Be more precise here. For example, if/when reading trailer, we may
1567  // not be doneWithServer() yet, but we should return false. Similarly, we
1568  // could still be writing the request body after receiving the whole reply.
1569  return !doneWithServer();
1570 }
1571 
1572 void
1574 {
1575  // too late to read
1577  return;
1578 
1579  if (!maybeMakeSpaceAvailable(false))
1580  return;
1581 
1582  // XXX: get rid of the do_next_read flag
1583  // check for the proper reasons preventing read(2)
1584  if (!flags.do_next_read)
1585  return;
1586 
1587  flags.do_next_read = false;
1588 
1589  // must not already be waiting for read(2) ...
1591 
1592  // wait for read(2) to be possible.
1594  AsyncCall::Pointer call = JobCallback(11, 5, Dialer, this, HttpStateData::readReply);
1596 }
1597 
1598 bool
1600 {
1601  // how much we are allowed to buffer
1602  const int limitBuffer = (flags.headers_parsed ? Config.readAheadGap : Config.maxReplyHeaderSize);
1603 
1604  if (limitBuffer < 0 || inBuf.length() >= (SBuf::size_type)limitBuffer) {
1605  // when buffer is at or over limit already
1606  debugs(11, 7, "will not read up to " << limitBuffer << ". buffer has (" << inBuf.length() << "/" << inBuf.spaceSize() << ") from " << serverConnection);
1607  debugs(11, DBG_DATA, "buffer has {" << inBuf << "}");
1608  // Process next response from buffer
1609  processReply();
1610  return false;
1611  }
1612 
1613  // how much we want to read
1614  const size_t read_size = calcBufferSpaceToReserve(inBuf.spaceSize(), (limitBuffer - inBuf.length()));
1615 
1616  if (!read_size) {
1617  debugs(11, 7, "will not read up to " << read_size << " into buffer (" << inBuf.length() << "/" << inBuf.spaceSize() << ") from " << serverConnection);
1618  return false;
1619  }
1620 
1621  // just report whether we could grow or not, do not actually do it
1622  if (doGrow)
1623  return (read_size >= 2);
1624 
1625  // we may need to grow the buffer
1626  inBuf.reserveSpace(read_size);
1627  debugs(11, 8, (!flags.do_next_read ? "will not" : "may") <<
1628  " read up to " << read_size << " bytes info buf(" << inBuf.length() << "/" << inBuf.spaceSize() <<
1629  ") from " << serverConnection);
1630 
1631  return (inBuf.spaceSize() >= 2); // only read if there is 1+ bytes of space available
1632 }
1633 
1635 void
1637 {
1638  debugs(11, 5, serverConnection << ": size " << io.size << ": errflag " << io.flag << ".");
1639 #if URL_CHECKSUM_DEBUG
1640 
1641  entry->mem_obj->checkUrlChecksum();
1642 #endif
1643 
1644  // XXX: Keep in sync with Client::sentRequestBody().
1645  // TODO: Extract common parts.
1646 
1647  if (io.size > 0) {
1648  fd_bytes(io.fd, io.size, FD_WRITE);
1649  statCounter.server.all.kbytes_out += io.size;
1650  statCounter.server.http.kbytes_out += io.size;
1651  }
1652 
1653  if (io.flag == Comm::ERR_CLOSING)
1654  return;
1655 
1656  // both successful and failed writes affect response times
1658 
1659  if (io.flag) {
1660  const auto err = new ErrorState(ERR_WRITE_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1661  err->xerrno = io.xerrno;
1662  fwd->fail(err);
1663  closeServer();
1664  mustStop("HttpStateData::wroteLast");
1665  return;
1666  }
1667 
1668  sendComplete();
1669 }
1670 
1672 void
1674 {
1675  /*
1676  * Set the read timeout here because it hasn't been set yet.
1677  * We only set the read timeout after the request has been
1678  * fully written to the peer. If we start the timeout
1679  * after connection establishment, then we are likely to hit
1680  * the timeout for POST/PUT requests that have very large
1681  * request bodies.
1682  */
1684  AsyncCall::Pointer timeoutCall = JobCallback(11, 5,
1685  TimeoutDialer, this, HttpStateData::httpTimeout);
1686 
1688  flags.request_sent = true;
1689 }
1690 
1691 void
1693 {
1694  debugs(11,5, "closing HTTP server " << serverConnection << " this " << this);
1695 
1699  closeHandler = NULL;
1701  }
1702 }
1703 
1704 bool
1706 {
1708 }
1709 
1710 /*
1711  * Fixup authentication request headers for special cases
1712  */
1713 static void
1715 {
1716  /* Nothing to do unless we are forwarding to a peer */
1717  if (!flags.peering)
1718  return;
1719 
1720  // This request is going "through" rather than "to" our _peer.
1721  if (flags.tunneling)
1722  return;
1723 
1724  /* Needs to be explicitly enabled */
1725  if (!request->peer_login)
1726  return;
1727 
1729  /* Maybe already dealt with? */
1730  if (hdr_out->has(header))
1731  return;
1732 
1733  /* Nothing to do here for PASSTHRU */
1734  if (strcmp(request->peer_login, "PASSTHRU") == 0)
1735  return;
1736 
1737  // Dangerous and undocumented PROXYPASS is a single-signon to servers with
1738  // the proxy password. Only Basic Authentication can work this way. This
1739  // statement forwards a "basic" Proxy-Authorization value from our client
1740  // to an originserver peer. Other PROXYPASS cases are handled lower.
1741  if (flags.toOrigin &&
1742  strcmp(request->peer_login, "PROXYPASS") == 0 &&
1744 
1745  const char *auth = hdr_in->getStr(Http::HdrType::PROXY_AUTHORIZATION);
1746 
1747  if (auth && strncasecmp(auth, "basic ", 6) == 0) {
1748  hdr_out->putStr(header, auth);
1749  return;
1750  }
1751  }
1752 
1753  char loginbuf[base64_encode_len(MAX_LOGIN_SZ)];
1754  size_t blen;
1755  struct base64_encode_ctx ctx;
1756  base64_encode_init(&ctx);
1757 
1758  /* Special mode to pass the username to the upstream cache */
1759  if (*request->peer_login == '*') {
1760  const char *username = "-";
1761 
1762  if (request->extacl_user.size())
1763  username = request->extacl_user.termedBuf();
1764 #if USE_AUTH
1765  else if (request->auth_user_request != NULL)
1766  username = request->auth_user_request->username();
1767 #endif
1768 
1769  blen = base64_encode_update(&ctx, loginbuf, strlen(username), reinterpret_cast<const uint8_t*>(username));
1770  blen += base64_encode_update(&ctx, loginbuf+blen, strlen(request->peer_login +1), reinterpret_cast<const uint8_t*>(request->peer_login +1));
1771  blen += base64_encode_final(&ctx, loginbuf+blen);
1772  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1773  return;
1774  }
1775 
1776  /* external_acl provided credentials */
1777  if (request->extacl_user.size() && request->extacl_passwd.size() &&
1778  (strcmp(request->peer_login, "PASS") == 0 ||
1779  strcmp(request->peer_login, "PROXYPASS") == 0)) {
1780 
1781  blen = base64_encode_update(&ctx, loginbuf, request->extacl_user.size(), reinterpret_cast<const uint8_t*>(request->extacl_user.rawBuf()));
1782  blen += base64_encode_update(&ctx, loginbuf+blen, 1, reinterpret_cast<const uint8_t*>(":"));
1783  blen += base64_encode_update(&ctx, loginbuf+blen, request->extacl_passwd.size(), reinterpret_cast<const uint8_t*>(request->extacl_passwd.rawBuf()));
1784  blen += base64_encode_final(&ctx, loginbuf+blen);
1785  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1786  return;
1787  }
1788  // if no external user credentials are available to fake authentication with PASS acts like PASSTHRU
1789  if (strcmp(request->peer_login, "PASS") == 0)
1790  return;
1791 
1792  /* Kerberos login to peer */
1793 #if HAVE_AUTH_MODULE_NEGOTIATE && HAVE_KRB5 && HAVE_GSSAPI
1794  if (strncmp(request->peer_login, "NEGOTIATE",strlen("NEGOTIATE")) == 0) {
1795  char *Token=NULL;
1796  char *PrincipalName=NULL,*p;
1797  int negotiate_flags = 0;
1798 
1799  if ((p=strchr(request->peer_login,':')) != NULL ) {
1800  PrincipalName=++p;
1801  }
1802  if (request->flags.auth_no_keytab) {
1803  negotiate_flags |= PEER_PROXY_NEGOTIATE_NOKEYTAB;
1804  }
1805  Token = peer_proxy_negotiate_auth(PrincipalName, request->peer_host, negotiate_flags);
1806  if (Token) {
1807  httpHeaderPutStrf(hdr_out, header, "Negotiate %s",Token);
1808  }
1809  return;
1810  }
1811 #endif /* HAVE_KRB5 && HAVE_GSSAPI */
1812 
1813  blen = base64_encode_update(&ctx, loginbuf, strlen(request->peer_login), reinterpret_cast<const uint8_t*>(request->peer_login));
1814  blen += base64_encode_final(&ctx, loginbuf+blen);
1815  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1816  return;
1817 }
1818 
1819 /*
1820  * build request headers and append them to a given MemBuf
1821  * used by buildRequestPrefix()
1822  * note: initialised the HttpHeader, the caller is responsible for Clean()-ing
1823  */
1824 void
1826  StoreEntry * entry,
1827  const AccessLogEntryPointer &al,
1828  HttpHeader * hdr_out,
1829  const Http::StateFlags &flags)
1830 {
1831  /* building buffer for complex strings */
1832 #define BBUF_SZ (MAX_URL+32)
1833  LOCAL_ARRAY(char, bbuf, BBUF_SZ);
1834  LOCAL_ARRAY(char, ntoabuf, MAX_IPSTRLEN);
1835  const HttpHeader *hdr_in = &request->header;
1836  const HttpHeaderEntry *e = NULL;
1838  assert (hdr_out->owner == hoRequest);
1839 
1840  /* use our IMS header if the cached entry has Last-Modified time */
1841  if (request->lastmod > -1)
1843 
1844  // Add our own If-None-Match field if the cached entry has a strong ETag.
1845  // copyOneHeaderFromClientsideRequestToUpstreamRequest() adds client ones.
1846  if (request->etag.size() > 0) {
1848  request->etag.termedBuf()));
1849  }
1850 
1851  bool we_do_ranges = decideIfWeDoRanges (request);
1852 
1853  String strConnection (hdr_in->getList(Http::HdrType::CONNECTION));
1854 
1855  while ((e = hdr_in->getEntry(&pos)))
1856  copyOneHeaderFromClientsideRequestToUpstreamRequest(e, strConnection, request, hdr_out, we_do_ranges, flags);
1857 
1858  /* Abstraction break: We should interpret multipart/byterange responses
1859  * into offset-length data, and this works around our inability to do so.
1860  */
1861  if (!we_do_ranges && request->multipartRangeRequest()) {
1862  /* don't cache the result */
1863  request->flags.cachable = false;
1864  /* pretend it's not a range request */
1865  request->ignoreRange("want to request the whole object");
1866  request->flags.isRanged = false;
1867  }
1868 
1869  hdr_out->addVia(request->http_ver, hdr_in);
1870 
1871  if (request->flags.accelerated) {
1872  /* Append Surrogate-Capabilities */
1873  String strSurrogate(hdr_in->getList(Http::HdrType::SURROGATE_CAPABILITY));
1874 #if USE_SQUID_ESI
1875  snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0 ESI/1.0\"", Config.Accel.surrogate_id);
1876 #else
1877  snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0\"", Config.Accel.surrogate_id);
1878 #endif
1879  strListAdd(&strSurrogate, bbuf, ',');
1880  hdr_out->putStr(Http::HdrType::SURROGATE_CAPABILITY, strSurrogate.termedBuf());
1881  }
1882 
1884  if (strcmp(opt_forwarded_for, "delete") != 0) {
1885 
1886  String strFwd = hdr_in->getList(Http::HdrType::X_FORWARDED_FOR);
1887 
1888  // if we cannot double strFwd size, then it grew past 50% of the limit
1889  if (!strFwd.canGrowBy(strFwd.size())) {
1890  // There is probably a forwarding loop with Via detection disabled.
1891  // If we do nothing, String will assert on overflow soon.
1892  // TODO: Terminate all transactions with huge XFF?
1893  strFwd = "error";
1894 
1895  static int warnedCount = 0;
1896  if (warnedCount++ < 100) {
1897  const SBuf url(entry ? SBuf(entry->url()) : request->effectiveRequestUri());
1898  debugs(11, DBG_IMPORTANT, "WARNING: likely forwarding loop with " << url);
1899  }
1900  }
1901 
1902  if (strcmp(opt_forwarded_for, "on") == 0) {
1904  if ( request->client_addr.isNoAddr() )
1905  strListAdd(&strFwd, "unknown", ',');
1906  else
1907  strListAdd(&strFwd, request->client_addr.toStr(ntoabuf, MAX_IPSTRLEN), ',');
1908  } else if (strcmp(opt_forwarded_for, "off") == 0) {
1910  strListAdd(&strFwd, "unknown", ',');
1911  } else if (strcmp(opt_forwarded_for, "transparent") == 0) {
1913  } else if (strcmp(opt_forwarded_for, "truncate") == 0) {
1915  if ( request->client_addr.isNoAddr() )
1916  strFwd = "unknown";
1917  else
1918  strFwd = request->client_addr.toStr(ntoabuf, MAX_IPSTRLEN);
1919  }
1920  if (strFwd.size() > 0)
1921  hdr_out->putStr(Http::HdrType::X_FORWARDED_FOR, strFwd.termedBuf());
1922  }
1925  /* append Host if not there already */
1926  if (!hdr_out->has(Http::HdrType::HOST)) {
1927  if (request->peer_domain) {
1929  } else {
1930  SBuf authority = request->url.authority();
1931  hdr_out->putStr(Http::HdrType::HOST, authority.c_str());
1932  }
1933  }
1934 
1935  /* append Authorization if known in URL, not in header and going direct */
1936  if (!hdr_out->has(Http::HdrType::AUTHORIZATION)) {
1937  if (flags.toOrigin && !request->url.userInfo().isEmpty()) {
1938  static char result[base64_encode_len(MAX_URL*2)]; // should be big enough for a single URI segment
1939  struct base64_encode_ctx ctx;
1940  base64_encode_init(&ctx);
1941  size_t blen = base64_encode_update(&ctx, result, request->url.userInfo().length(), reinterpret_cast<const uint8_t*>(request->url.userInfo().rawContent()));
1942  blen += base64_encode_final(&ctx, result+blen);
1943  result[blen] = '\0';
1944  if (blen)
1945  httpHeaderPutStrf(hdr_out, Http::HdrType::AUTHORIZATION, "Basic %.*s", (int)blen, result);
1946  }
1947  }
1948 
1949  /* Fixup (Proxy-)Authorization special cases. Plain relaying dealt with above */
1950  httpFixupAuthentication(request, hdr_in, hdr_out, flags);
1951 
1952  /* append Cache-Control, add max-age if not there already */
1953  {
1954  HttpHdrCc *cc = hdr_in->getCc();
1955 
1956  if (!cc)
1957  cc = new HttpHdrCc();
1958 
1959  /* Add max-age only without no-cache */
1960  if (!cc->hasMaxAge() && !cc->hasNoCache()) {
1961  // XXX: performance regression. c_str() reallocates
1963  cc->maxAge(getMaxAge(entry ? entry->url() : tmp.c_str()));
1964  }
1965 
1966  /* Enforce sibling relations */
1967  if (flags.only_if_cached)
1968  cc->onlyIfCached(true);
1969 
1970  hdr_out->putCc(cc);
1971 
1972  delete cc;
1973  }
1974 
1975  // Always send Connection because HTTP/1.0 servers need explicit
1976  // "keep-alive", HTTP/1.1 servers need explicit "close", Upgrade recipients
1977  // need bare "upgrade", and we do not always know the server expectations.
1978  if (!hdr_out->has(Http::HdrType::CONNECTION)) // forwardUpgrade() may add it
1979  hdr_out->putStr(Http::HdrType::CONNECTION, flags.keepalive ? "keep-alive" : "close");
1980 
1981  /* append Front-End-Https */
1982  if (flags.front_end_https) {
1984  hdr_out->putStr(Http::HdrType::FRONT_END_HTTPS, "On");
1985  }
1986 
1987  if (flags.chunked_request) {
1988  // Do not just copy the original value so that if the client-side
1989  // starts decode other encodings, this code may remain valid.
1990  hdr_out->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
1991  }
1992 
1993  /* Now mangle the headers. */
1994  httpHdrMangleList(hdr_out, request, al, ROR_REQUEST);
1995 
1996  strConnection.clean();
1997 }
1998 
2001 void
2003 {
2005  return; // forward nothing by default
2006 
2007  /* RFC 7230 section 6.7 paragraph 10:
2008  * A server MUST ignore an Upgrade header field that is received in
2009  * an HTTP/1.0 request.
2010  */
2011  if (request->http_ver == Http::ProtocolVersion(1,0))
2012  return;
2013 
2014  const auto &hdrIn = request->header;
2015  if (!hdrIn.has(Http::HdrType::UPGRADE))
2016  return;
2017  const auto upgradeIn = hdrIn.getList(Http::HdrType::UPGRADE);
2018 
2019  String upgradeOut;
2020 
2021  ACLFilledChecklist ch(nullptr, request.getRaw());
2022  ch.al = fwd->al;
2023  const char *pos = nullptr;
2024  const char *offeredStr = nullptr;
2025  int offeredStrLen = 0;
2026  while (strListGetItem(&upgradeIn, ',', &offeredStr, &offeredStrLen, &pos)) {
2027  const ProtocolView offeredProto(offeredStr, offeredStrLen);
2028  debugs(11, 5, "checks all rules applicable to " << offeredProto);
2029  Config.http_upgrade_request_protocols->forApplicable(offeredProto, [&ch, offeredStr, offeredStrLen, &upgradeOut] (const SBuf &cfgProto, const acl_access *guard) {
2030  debugs(11, 5, "checks " << cfgProto << " rule(s)");
2031  ch.changeAcl(guard);
2032  const auto answer = ch.fastCheck();
2033  if (answer.implicit)
2034  return false; // keep looking for an explicit rule match
2035  if (answer.allowed())
2036  strListAdd(upgradeOut, offeredStr, offeredStrLen);
2037  // else drop the offer (explicitly denied cases and ACL errors)
2038  return true; // stop after an explicit rule match or an error
2039  });
2040  }
2041 
2042  if (upgradeOut.size()) {
2043  hdrOut.putStr(Http::HdrType::UPGRADE, upgradeOut.termedBuf());
2044 
2045  /* RFC 7230 section 6.7 paragraph 10:
2046  * When Upgrade is sent, the sender MUST also send a Connection header
2047  * field that contains an "upgrade" connection option, in
2048  * order to prevent Upgrade from being accidentally forwarded by
2049  * intermediaries that might not implement the listed protocols.
2050  *
2051  * NP: Squid does not truly implement the protocol(s) in this Upgrade.
2052  * For now we are treating an explicit blind tunnel as "implemented"
2053  * regardless of the security implications.
2054  */
2055  hdrOut.putStr(Http::HdrType::CONNECTION, "upgrade");
2056 
2057  // Connection:close and Connection:keepalive confuse some Upgrade
2058  // recipients, so we do not send those headers. Our Upgrade request
2059  // implicitly offers connection persistency per HTTP/1.1 defaults.
2060  // Update the keepalive flag to reflect that offer.
2061  // * If the server upgrades, then we would not be talking HTTP past the
2062  // HTTP 101 control message, and HTTP persistence would be irrelevant.
2063  // * Otherwise, our request will contradict onoff.server_pconns=off or
2064  // other no-keepalive conditions (if any). We compensate by copying
2065  // the original no-keepalive decision now and honoring it later.
2067  flags.keepalive = true; // should already be true in most cases
2068  }
2069 }
2070 
2075 void
2076 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request, HttpHeader * hdr_out, const int we_do_ranges, const Http::StateFlags &flags)
2077 {
2078  debugs(11, 5, "httpBuildRequestHeader: " << e->name << ": " << e->value );
2079 
2080  switch (e->id) {
2081 
2089  if (!flags.toOrigin && request->peer_login &&
2090  (strcmp(request->peer_login, "PASS") == 0 ||
2091  strcmp(request->peer_login, "PROXYPASS") == 0 ||
2092  strcmp(request->peer_login, "PASSTHRU") == 0)) {
2093  hdr_out->addEntry(e->clone());
2094  }
2095  break;
2096 
2100  case Http::HdrType::TE:
2103  case Http::HdrType::TRAILER:
2105  break;
2106 
2109  break;
2110 
2117  if (!flags.toOriginPeer()) {
2118  hdr_out->addEntry(e->clone());
2119  } else {
2124  if (request->peer_login &&
2125  (strcmp(request->peer_login, "PASS") == 0 ||
2126  strcmp(request->peer_login, "PASSTHRU") == 0 ||
2127  strcmp(request->peer_login, "PROXYPASS") == 0)) {
2128  hdr_out->addEntry(e->clone());
2129  }
2130  }
2131 
2132  break;
2133 
2134  case Http::HdrType::HOST:
2141  if (request->peer_domain)
2142  hdr_out->putStr(Http::HdrType::HOST, request->peer_domain);
2143  else if (request->flags.redirected && !Config.onoff.redir_rewrites_host)
2144  hdr_out->addEntry(e->clone());
2145  else {
2146  SBuf authority = request->url.authority();
2147  hdr_out->putStr(Http::HdrType::HOST, authority.c_str());
2148  }
2149 
2150  break;
2151 
2160  // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2161  if (hdr_out->has(Http::HdrType::IF_MODIFIED_SINCE))
2162  break;
2163  else if (Config.onoff.cache_miss_revalidate || !request->flags.cachable || request->flags.auth)
2164  hdr_out->addEntry(e->clone());
2165  break;
2166 
2175  // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2176  if (hdr_out->hasListMember(Http::HdrType::IF_MATCH, "*", ',') || Config.onoff.cache_miss_revalidate || !request->flags.cachable || request->flags.auth)
2177  hdr_out->addEntry(e->clone());
2178  break;
2179 
2184  const int64_t hops = e->getInt64();
2185 
2186  if (hops > 0)
2187  hdr_out->putInt64(Http::HdrType::MAX_FORWARDS, hops - 1);
2188  }
2189 
2190  break;
2191 
2192  case Http::HdrType::VIA:
2197  if (!Config.onoff.via)
2198  hdr_out->addEntry(e->clone());
2199 
2200  break;
2201 
2202  case Http::HdrType::RANGE:
2203 
2205 
2209  if (!we_do_ranges)
2210  hdr_out->addEntry(e->clone());
2211 
2212  break;
2213 
2214  case Http::HdrType::PROXY_CONNECTION: // SHOULD ignore. But doing so breaks things.
2215  break;
2216 
2218  // pass through unless we chunk; also, keeping this away from default
2219  // prevents request smuggling via Connection: Content-Length tricks
2220  if (!flags.chunked_request)
2221  hdr_out->addEntry(e->clone());
2222  break;
2223 
2225 
2230  break;
2231 
2235  if (!flags.front_end_https)
2236  hdr_out->addEntry(e->clone());
2237 
2238  break;
2239 
2240  default:
2244  if (strConnection.size()>0 && strListIsMember(&strConnection, e->name, ',')) {
2245  debugs(11, 2, "'" << e->name << "' header cropped by Connection: definition");
2246  return;
2247  }
2248 
2249  hdr_out->addEntry(e->clone());
2250  }
2251 }
2252 
2253 bool
2255 {
2256  bool result = true;
2257  /* decide if we want to do Ranges ourselves
2258  * and fetch the whole object now)
2259  * We want to handle Ranges ourselves iff
2260  * - we can actually parse client Range specs
2261  * - the specs are expected to be simple enough (e.g. no out-of-order ranges)
2262  * - reply will be cachable
2263  * (If the reply will be uncachable we have to throw it away after
2264  * serving this request, so it is better to forward ranges to
2265  * the server and fetch only the requested content)
2266  */
2267 
2268  int64_t roffLimit = request->getRangeOffsetLimit();
2269 
2270  if (NULL == request->range || !request->flags.cachable
2272  result = false;
2273 
2274  debugs(11, 8, "decideIfWeDoRanges: range specs: " <<
2275  request->range << ", cachable: " <<
2276  request->flags.cachable << "; we_do_ranges: " << result);
2277 
2278  return result;
2279 }
2280 
2281 /* build request prefix and append it to a given MemBuf;
2282  * return the length of the prefix */
2283 mb_size_t
2285 {
2286  const int offset = mb->size;
2287  /* Uses a local httpver variable to print the HTTP label
2288  * since the HttpRequest may have an older version label.
2289  * XXX: This could create protocol bugs as the headers sent and
2290  * flow control should all be based on the HttpRequest version
2291  * not the one we are sending. Needs checking.
2292  */
2293  const AnyP::ProtocolVersion httpver = Http::ProtocolVersion();
2295  mb->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " %s/%d.%d\r\n",
2297  SQUIDSBUFPRINT(url),
2299  httpver.major,httpver.minor);
2300  /* build and pack headers */
2301  {
2302  HttpHeader hdr(hoRequest);
2303  forwardUpgrade(hdr); // before httpBuildRequestHeader() for CONNECTION
2305 
2307  request->flags.authSent = true;
2308  else if (hdr.has(Http::HdrType::AUTHORIZATION))
2309  request->flags.authSent = true;
2310 
2311  // The late placement of this check supports reply_header_add mangling,
2312  // but also complicates optimizing upgradeHeaderOut-like lookups.
2313  if (hdr.has(Http::HdrType::UPGRADE)) {
2316  }
2317 
2318  hdr.packInto(mb);
2319  hdr.clean();
2320  }
2321  /* append header terminator */
2322  mb->append(crlf, 2);
2323  return mb->size - offset;
2324 }
2325 
2326 /* This will be called when connect completes. Write request. */
2327 bool
2329 {
2330  MemBuf mb;
2331 
2332  debugs(11, 5, serverConnection << ", request " << request << ", this " << this << ".");
2333 
2335  debugs(11,3, "cannot send request to closing " << serverConnection);
2336  assert(closeHandler != NULL);
2337  return false;
2338  }
2339 
2341  AsyncCall::Pointer timeoutCall = JobCallback(11, 5,
2342  TimeoutDialer, this, HttpStateData::httpTimeout);
2344  flags.do_next_read = true;
2346 
2347  if (request->body_pipe != NULL) {
2348  if (!startRequestBodyFlow()) // register to receive body data
2349  return false;
2351  requestSender = JobCallback(11,5,
2352  Dialer, this, HttpStateData::sentRequestBody);
2353 
2355  // use chunked encoding if we do not know the length
2356  if (request->content_length < 0)
2357  flags.chunked_request = true;
2358  } else {
2361  requestSender = JobCallback(11,5,
2362  Dialer, this, HttpStateData::wroteLast);
2363  }
2364 
2365  /*
2366  * Is keep-alive okay for all request methods?
2367  */
2369  flags.keepalive = true;
2370  else if (request->flags.pinned)
2372  else if (!Config.onoff.server_pconns)
2373  flags.keepalive = false;
2374  else if (flags.tunneling)
2375  // tunneled non pinned bumped requests must not keepalive
2377  else if (_peer == NULL)
2378  flags.keepalive = true;
2379  else if (_peer->stats.n_keepalives_sent < 10)
2380  flags.keepalive = true;
2381  else if ((double) _peer->stats.n_keepalives_recv /
2382  (double) _peer->stats.n_keepalives_sent > 0.50)
2383  flags.keepalive = true;
2384 
2385  if (_peer && !flags.tunneling) {
2386  /*The old code here was
2387  if (neighborType(_peer, request->url) == PEER_SIBLING && ...
2388  which is equivalent to:
2389  if (neighborType(_peer, URL()) == PEER_SIBLING && ...
2390  or better:
2391  if (((_peer->type == PEER_MULTICAST && p->options.mcast_siblings) ||
2392  _peer->type == PEER_SIBLINGS ) && _peer->options.allow_miss)
2393  flags.only_if_cached = 1;
2394 
2395  But I suppose it was a bug
2396  */
2398  flags.only_if_cached = true;
2399 
2401  }
2402 
2403  mb.init();
2405  buildRequestPrefix(&mb);
2406 
2407  debugs(11, 2, "HTTP Server " << serverConnection);
2408  debugs(11, 2, "HTTP Server REQUEST:\n---------\n" << mb.buf << "\n----------");
2409 
2411  return true;
2412 }
2413 
2414 bool
2416 {
2417  // parent's implementation can handle the no-encoding case
2418  if (!flags.chunked_request)
2419  return Client::getMoreRequestBody(buf);
2420 
2421  MemBuf raw;
2422 
2424  if (!requestBodySource->getMoreData(raw))
2425  return false; // no request body bytes to chunk yet
2426 
2427  // optimization: pre-allocate buffer size that should be enough
2428  const mb_size_t rawDataSize = raw.contentSize();
2429  // we may need to send: hex-chunk-size CRLF raw-data CRLF last-chunk
2430  buf.init(16 + 2 + rawDataSize + 2 + 5, raw.max_capacity);
2431 
2432  buf.appendf("%x\r\n", static_cast<unsigned int>(rawDataSize));
2433  buf.append(raw.content(), rawDataSize);
2434  buf.append("\r\n", 2);
2435 
2436  Must(rawDataSize > 0); // we did not accidentally created last-chunk above
2437 
2438  // Do not send last-chunk unless we successfully received everything
2441  flags.sentLastChunk = true;
2442  buf.append("0\r\n\r\n", 5);
2443  }
2444 
2445  return true;
2446 }
2447 
2448 void
2450 {
2451  debugs(11, 3, fwd->request->method << ' ' << fwd->entry->url());
2452  AsyncJob::Start(new HttpStateData(fwd));
2453 }
2454 
2455 void
2457 {
2458  if (!sendRequest()) {
2459  debugs(11, 3, "httpStart: aborted");
2460  mustStop("HttpStateData::start failed");
2461  return;
2462  }
2463 
2464  ++ statCounter.server.all.requests;
2465  ++ statCounter.server.http.requests;
2466 
2467  /*
2468  * We used to set the read timeout here, but not any more.
2469  * Now its set in httpSendComplete() after the full request,
2470  * including request body, has been written to the server.
2471  */
2472 }
2473 
2475 bool
2477 {
2478 #if USE_HTTP_VIOLATIONS
2479  if (!Config.accessList.brokenPosts) {
2480  debugs(11, 5, "No brokenPosts list");
2481  return false;
2482  }
2483 
2485  ch.al = fwd->al;
2486  ch.syncAle(originalRequest().getRaw(), nullptr);
2487  if (!ch.fastCheck().allowed()) {
2488  debugs(11, 5, "didn't match brokenPosts");
2489  return false;
2490  }
2491 
2493  debugs(11, 3, "ignoring broken POST for closed " << serverConnection);
2494  assert(closeHandler != NULL);
2495  return true; // prevent caller from proceeding as if nothing happened
2496  }
2497 
2498  debugs(11, 3, "finishingBrokenPost: fixing broken POST");
2500  requestSender = JobCallback(11,5,
2501  Dialer, this, HttpStateData::wroteLast);
2503  return true;
2504 #else
2505  return false;
2506 #endif /* USE_HTTP_VIOLATIONS */
2507 }
2508 
2510 bool
2512 {
2513  if (flags.sentLastChunk) {
2514  debugs(11, 5, "already sent last-chunk");
2515  return false;
2516  }
2517 
2518  Must(receivedWholeRequestBody); // or we should not be sending last-chunk
2519  flags.sentLastChunk = true;
2520 
2522  requestSender = JobCallback(11,5, Dialer, this, HttpStateData::wroteLast);
2523  Comm::Write(serverConnection, "0\r\n\r\n", 5, requestSender, NULL);
2524  return true;
2525 }
2526 
2527 void
2529 {
2531  debugs(11,5, serverConnection);
2532 
2533  // do we need to write something after the last body byte?
2535  return;
2537  return;
2538 
2539  sendComplete();
2540 }
2541 
2542 // more origin request body data is available
2543 void
2545 {
2547  // XXX: we should check this condition in other callbacks then!
2548  // TODO: Check whether this can actually happen: We should unsubscribe
2549  // as a body consumer when the above condition(s) are detected.
2550  debugs(11, DBG_IMPORTANT, "Transaction aborted while reading HTTP body");
2551  return;
2552  }
2553 
2555 
2556  if (requestBodySource->buf().hasContent()) {
2557  // XXX: why does not this trigger a debug message on every request?
2558 
2560  flags.abuse_detected = true;
2561  debugs(11, DBG_IMPORTANT, "http handleMoreRequestBodyAvailable: Likely proxy abuse detected '" << request->client_addr << "' -> '" << entry->url() << "'" );
2562 
2563  if (virginReply()->sline.status() == Http::scInvalidHeader) {
2564  closeServer();
2565  mustStop("HttpStateData::handleMoreRequestBodyAvailable");
2566  return;
2567  }
2568  }
2569  }
2570 
2572 }
2573 
2574 // premature end of the request body
2575 void
2577 {
2579  if (entry->isEmpty()) {
2580  debugs(11, 3, "request body aborted: " << serverConnection);
2581  // We usually get here when ICAP REQMOD aborts during body processing.
2582  // We might also get here if client-side aborts, but then our response
2583  // should not matter because either client-side will provide its own or
2584  // there will be no response at all (e.g., if the the client has left).
2586  static const auto d = MakeNamedErrorDetail("SRV_REQMOD_REQ_BODY");
2587  err->detailError(d);
2588  fwd->fail(err);
2589  }
2590 
2591  abortTransaction("request body producer aborted");
2592 }
2593 
2594 // called when we wrote request headers(!) or a part of the body
2595 void
2597 {
2598  if (io.size > 0)
2599  statCounter.server.http.kbytes_out += io.size;
2600 
2602 }
2603 
2604 void
2605 HttpStateData::abortAll(const char *reason)
2606 {
2607  debugs(11,5, "aborting transaction for " << reason <<
2608  "; " << serverConnection << ", this " << this);
2609  mustStop(reason);
2610 }
2611 
2613  : answer(HttpStateData::ReuseDecision::reuseNot), reason(nullptr), entry(e), statusCode(code) {}
2614 
2617 {
2618  answer = ans;
2619  reason = why;
2620  return answer;
2621 }
2622 
2623 std::ostream &operator <<(std::ostream &os, const HttpStateData::ReuseDecision &d)
2624 {
2625  static const char *ReuseMessages[] = {
2626  "do not cache and do not share", // reuseNot
2627  "cache positively and share", // cachePositively
2628  "cache negatively and share", // cacheNegatively
2629  "do not cache but share" // doNotCacheButShare
2630  };
2631 
2634  return os << ReuseMessages[d.answer] << " because " << d.reason <<
2635  "; HTTP status " << d.statusCode << " " << *(d.entry);
2636 }
2637 
const MemBuf & buf() const
Definition: BodyPipe.h:137
void drop1xx(const char *reason)
Definition: http.cc:805
void Read(const Comm::ConnectionPointer &conn, AsyncCall::Pointer &callback)
Definition: Read.cc:40
bool proxy_only
Definition: CachePeer.h:82
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: http.cc:937
struct SquidConfig::@96 Timeout
const char * xstrerr(int error)
Definition: xstrerror.cc:83
void makePrivate(const bool shareable)
Definition: store.cc:171
bool handling1xx
Whether we are processing an HTTP 1xx control message.
Definition: StateFlags.h:30
char method[16]
Definition: tcp-banger2.c:115
bool hasProxyRevalidate() const
Definition: HttpHdrCc.h:117
bool maybeMakeSpaceAvailable(bool grow)
Definition: http.cc:1599
char * buf
Definition: MemBuf.h:134
AsyncCall::Pointer comm_add_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:921
int connection_auth
0 - off, 1 - on, 2 - auto
Definition: CachePeer.h:195
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:438
void truncateVirginBody()
Definition: http.cc:1372
char * peer_host
Definition: HttpRequest.h:165
bool keepalive_broken
Definition: StateFlags.h:58
unsigned int major
major version number
StoreEntry * entry
Definition: Client.h:176
bool keepalive
whether the Squid-sent request offers to keep the connection persistent
Definition: StateFlags.h:21
@ scMethodNotAllowed
Definition: StatusCode.h:49
bool hasPrivate(const String **val=nullptr) const
Definition: HttpHdrCc.h:74
@ scNonAuthoritativeInformation
Definition: StatusCode.h:29
AnyP::ProtocolVersion http_ver
Definition: Message.h:73
void userInfo(const SBuf &s)
Definition: Uri.h:79
#define base64_encode_len(length)
Definition: base64.h:169
@ HDR_X_ACCELERATOR_VARY
void ignoreRange(const char *reason)
forgets about the cached Range header (for a reason)
Definition: HttpRequest.cc:629
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
@ scUnauthorized
Definition: StatusCode.h:45
@ METHOD_HEAD
Definition: MethodType.h:28
void delayRead()
Definition: Client.cc:1029
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
http_hdr_owner_type owner
Definition: HttpHeader.h:173
bool ignoreCacheControl
Definition: http.h:69
Definition: Client.h:35
@ ERR_READ_ERROR
Definition: forward.h:28
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:98
bool spoofClientIp
Definition: RequestFlags.h:70
const char * storeId() const
Definition: MemObject.cc:53
void releaseRequest(const bool shareable=false)
Definition: store.cc:433
struct IoStats::@78 Http
@ scBadRequest
Definition: StatusCode.h:44
bool chunked
reading a chunked response; TODO: rename
Definition: StateFlags.h:62
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1192
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
mb_size_t size
Definition: MemBuf.h:135
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: http.cc:2415
int reads
Definition: IoStats.h:19
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:164
HttpHeader header
Definition: Message.h:75
bool offsetLimitExceeded(const int64_t limit) const
#define EBIT_SET(flag, bit)
Definition: defines.h:67
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1691
Http::StateFlags flags
Definition: http.h:67
@ scMisdirectedRequest
Definition: StatusCode.h:62
Http::StatusCode peer_reply_status
last HTTP status code received
bool MonitorsRead(int fd)
whether the FD socket is being monitored for read
Definition: Read.cc:29
Answers make(const Answers ans, const char *why)
stores the corresponding decision
Definition: http.cc:2616
bool isEmpty() const
Definition: SBuf.h:431
struct CachePeer::@30 options
MemObject * mem_obj
Definition: Store.h:219
AsyncCall::Pointer requestSender
Definition: Client.h:182
RequestFlags flags
Definition: HttpRequest.h:141
String * upgradeHeaderOut
Upgrade header value sent to the origin server or cache peer.
Definition: http.h:73
void checkDateSkew(HttpReply *)
Definition: http.cc:624
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
int64_t payloadTruncated
positive when we read more than we wanted
Definition: http.h:157
virtual void abortAll(const char *reason)
abnormal transaction termination; reason is for debugging only
Definition: http.cc:2605
void reserveSpace(size_type minSpace)
Definition: SBuf.h:440
@ scLengthRequired
Definition: StatusCode.h:55
const Http::StatusCode statusCode
HTTP status for debugging.
Definition: http.h:41
virtual bool mayReadVirginReplyBody() const
whether we may receive more virgin response body bytes
Definition: http.cc:1564
bool surrogateNoStore
Definition: http.h:70
const char * url() const
Definition: store.cc:1544
a reference to a protocol name[/version] string; no 0-termination is assumed
int lastChunk
Definition: http.h:66
Comm::Flag ReadNow(CommIoCbParams &params, SBuf &buf)
Definition: Read.cc:81
@ KEY_PRIVATE
Definition: enums.h:102
size_t base64_encode_final(struct base64_encode_ctx *ctx, char *dst)
Definition: base64.c:308
static void httpFixupAuthentication(HttpRequest *request, const HttpHeader *hdr_in, HttpHeader *hdr_out, const Http::StateFlags &flags)
Definition: http.cc:1714
@ ENTRY_FWD_HDR_WAIT
Definition: enums.h:111
int64_t getRangeOffsetLimit()
Definition: HttpRequest.cc:602
MemObject & mem()
Definition: Store.h:51
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
#define ScheduleCallHere(call)
Definition: AsyncCall.h:164
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:351
void error(char *format,...)
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:171
unsigned int minor
minor version number
@ PROXY_CONNECTION
#define PEER_PROXY_NEGOTIATE_NOKEYTAB
bool finishingBrokenPost()
if broken posts are enabled for the request, try to fix and return true
Definition: http.cc:2476
CachePeer * _peer
Definition: http.h:64
bool connectionAuth
Definition: RequestFlags.h:76
@ scUnsupportedMediaType
Definition: StatusCode.h:59
void httpHdrMangleList(HttpHeader *l, HttpRequest *request, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep)
bool hasContent() const
Definition: MemBuf.h:54
HttpReplyPointer reply
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:374
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
const char * ProtocolType_str[]
@ INPROGRESS
Definition: Flag.h:22
Definition: SBuf.h:94
virtual void sendControlMsg(HttpControlMsg)
called to send the 1xx message and notify the Source
mb_size_t buildRequestPrefix(MemBuf *mb)
Definition: http.cc:2284
size_t maxReplyHeaderSize
Definition: SquidConfig.h:135
ConnectionStatus statusIfComplete() const
Definition: http.cc:1067
AsyncCall::Pointer closeHandler
Definition: http.h:93
char * opt_forwarded_for
ConnectionStatus
Definition: http.h:94
@ scUseProxy
Definition: StatusCode.h:41
int commSetConnTimeout(const Comm::ConnectionPointer &conn, int timeout, AsyncCall::Pointer &callback)
Definition: comm.cc:563
HttpStateData(FwdState *)
Definition: http.cc:77
@ scMovedPermanently
Definition: StatusCode.h:37
void noStoreRemote(bool v)
static void assembleVaryKey(String &vary, SBuf &vstr, const HttpRequest &request)
assemble a variant key (vary-mark) from the given Vary header and HTTP request
Definition: http.cc:549
DelayId mostBytesAllowed() const
Definition: MemObject.cc:462
@ ERR_INVALID_RESP
Definition: forward.h:41
bool peering
Whether the next TCP hop is a cache_peer, including originserver.
Definition: StateFlags.h:40
BodyPipe::Pointer requestBodySource
Definition: Client.h:181
a representation of a refresh pattern.
const char * blockSwitchingProtocols(const HttpReply &) const
Definition: http.cc:823
PconnPool * fwdPconnPool
a collection of previously used persistent Squid-to-peer HTTP(S) connections
Definition: FwdState.cc:77
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:842
bool sendRequest()
Definition: http.cc:2328
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
bool chunked_request
writing a chunked request
Definition: StateFlags.h:63
int detect_broken_server_pconns
Definition: SquidConfig.h:320
C * getRaw() const
Definition: RefCount.h:80
size_type spaceSize() const
Definition: SBuf.h:393
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:209
uint16_t flags
Definition: Store.h:230
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
const SBuf & image() const
Acl::Answer const & fastCheck()
Definition: Checklist.cc:332
void processReplyBody()
Definition: http.cc:1455
void toLower()
converts all characters to lower case;
Definition: SBuf.cc:811
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:27
bool allow_miss
Definition: CachePeer.h:103
IoStats IOStats
@ OK
Definition: Flag.h:16
bool isAccepting() const
Definition: store.cc:1951
Http::StatusLine sline
Definition: HttpReply.h:56
time_t getMaxAge(const char *url)
Definition: refresh.cc:633
@ ENDFILE
Definition: Flag.h:27
Comm::ConnectionPointer serverConnection
Definition: http.h:92
@ ERR_NONE
Definition: forward.h:15
int surrogate_is_remote
Definition: SquidConfig.h:319
StatusCode
Definition: StatusCode.h:20
static const char *const crlf
Definition: http.cc:71
err_type
Definition: forward.h:14
struct SquidConfig::@103 Accel
void adaptOrFinalizeReply()
Definition: Client.cc:996
void unregister(Comm::ConnectionPointer &conn)
Definition: FwdState.cc:530
bool multipartRangeRequest() const
Definition: HttpRequest.cc:435
void fail(ErrorState *err)
Definition: FwdState.cc:489
void keepaliveAccounting(HttpReply *)
Definition: http.cc:604
bool startRequestBodyFlow()
Definition: Client.cc:239
@ scRequestedRangeNotSatisfied
Definition: StatusCode.h:60
@ ERR_CLOSING
Definition: Flag.h:25
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:537
bool Is1xx(const int sc)
whether this is an informational 1xx response status code
Definition: StatusCode.h:92
#define cbdataReference(var)
Definition: cbdata.h:341
int front_end_https
0 - off, 1 - on, 2 - auto
Definition: CachePeer.h:194
bool connectionAuthDisabled
Definition: RequestFlags.h:78
void reset()
Definition: store.cc:1599
@ IF_MODIFIED_SINCE
bool decodeAndWriteReplyBody()
Definition: http.cc:1423
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
StoreEntry * storeGetPublicByRequest(HttpRequest *req, const KeyScope keyScope)
Definition: store.cc:491
ProtocolType protocol
which protocol this version is for
void readReply(const CommIoCbParams &io)
Definition: http.cc:1175
static bool decideIfWeDoRanges(HttpRequest *orig_request)
Definition: http.cc:2254
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:294
@ scSeeOther
Definition: StatusCode.h:39
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:585
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *req, const HttpRequestMethod &method, const KeyScope keyScope)
Definition: store.cc:485
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:792
AnyP::UriScheme const & getScheme() const
Definition: Uri.h:67
void serverComplete()
Definition: Client.cc:180
@ METHOD_OPTIONS
Definition: MethodType.h:31
struct StatCounters::@128 server
@ CONTENT_LENGTH
#define DBG_DATA
Definition: Stream.h:43
SBuf & chop(size_type pos, size_type n=npos)
Definition: SBuf.cc:530
bool chunked() const
Definition: HttpHeader.h:165
mb_size_t max_capacity
Definition: MemBuf.h:142
peer_t neighborType(const CachePeer *p, const AnyP::Uri &url)
Definition: neighbors.cc:115
#define MAX_IPSTRLEN
Length of buffer that needs to be allocated to old a null-terminated IP-string.
Definition: forward.h:25
@ scNotAcceptable
Definition: StatusCode.h:50
@ ENTRY_ABORTED
Definition: enums.h:115
@ COMPLETE_NONPERSISTENT_MSG
Definition: http.h:97
Definition: Range.h:19
@ scGatewayTimeout
Definition: StatusCode.h:75
Definition: Raw.h:21
#define MAX_URL
Definition: defines.h:78
bool persistent() const
Definition: Message.cc:247
void sendComplete()
successfully wrote the entire request (including body, last-chunk, etc.)
Definition: http.cc:1673
time_t lastmod
Definition: HttpRequest.h:167
void strListAdd(String &str, const char *item, const size_t itemSize, const char delimiter)
Appends the given item of a given size to a delimiter-separated list in str.
Definition: StrList.cc:18
bool canGrowBy(const size_type growthLen) const
whether appending growthLen characters is safe (i.e., unlikely to assert)
Definition: SquidString.h:124
@ ERR_ICAP_FAILURE
Definition: forward.h:64
@ HTCP_CLR_INVALIDATION
Definition: enums.h:243
void bytesIn(int qty)
Definition: DelayId.cc:152
HttpHdrScTarget * getMergedTarget(const char *ourtarget)
Definition: HttpHdrSc.cc:296
String etag
A strong etag of the cached entry. Used for refreshing that entry.
Definition: HttpRequest.h:191
void processReplyHeader()
Definition: http.cc:641
ReuseDecision(const StoreEntry *e, const Http::StatusCode code)
Definition: http.cc:2612
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const
Definition: HttpReply.cc:487
bool httpHeaderHasConnDir(const HttpHeader *hdr, const SBuf &directive)
static void httpMaybeRemovePublic(StoreEntry *, Http::StatusCode)
Definition: http.cc:167
time_t expires
Definition: HttpReply.h:44
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
int64_t getInt64() const
Definition: HttpHeader.cc:1563
@ scTemporaryRedirect
Definition: StatusCode.h:42
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:184
unsigned int front_end_https
send "Front-End-Https: On" header (off/on/auto=2)
Definition: StateFlags.h:18
#define NULL
Definition: types.h:166
@ COMPLETE_PERSISTENT_MSG
Definition: http.h:96
const char * rawContent() const
Definition: SBuf.cc:509
SBuf consume(size_type n=npos)
Definition: SBuf.cc:481
int n_keepalives_sent
Definition: CachePeer.h:51
@ scFailedDependency
Definition: StatusCode.h:65
void maxAge(int32_t v)
Definition: HttpHdrCc.h:123
@ scSwitchingProtocols
Definition: StatusCode.h:23
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
virtual void doneSendingRequestBody()=0
Definition: Client.cc:340
time_t read
Definition: SquidConfig.h:110
@ scForbidden
Definition: StatusCode.h:47
@ ROR_REQUEST
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
bool dontRetry()
Definition: FwdState.h:98
void mustStop(const char *aReason)
Definition: AsyncJob.cc:70
const HttpReply * virginReply() const
Definition: Client.cc:109
SBuf inBuf
I/O buffer for receiving server responses.
Definition: http.h:68
bundles HTTP 1xx reply and the "successfully forwarded" callback
@ scInsufficientStorage
Definition: StatusCode.h:78
SBuf vary_headers
Definition: MemObject.h:204
@ scPreconditionFailed
Definition: StatusCode.h:56
@ scNotImplemented
Definition: StatusCode.h:72
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
MemBlob::size_type size_type
Definition: SBuf.h:96
#define MAX_LOGIN_SZ
Definition: defines.h:79
bool hasNoCacheWithParameters() const
Definition: HttpHdrCc.h:87
void forApplicable(const ProtocolView &, const Visitor &) const
bool hasNoStore() const
virtual void handleMoreRequestBodyAvailable()
Definition: http.cc:2544
void handle1xx(HttpReply *msg)
ignore or start forwarding the 1xx response (a.k.a., control message)
Definition: http.cc:746
@ srcHttp
http_port or HTTP server
Definition: Message.h:39
@ scBadGateway
Definition: StatusCode.h:73
const StoreEntry * entry
entry for debugging
Definition: http.h:40
Definition: MemBuf.h:24
void doneSendingRequestBody()
Definition: http.cc:2528
bool toOriginPeer() const
Whether the next TCP/HTTP hop is an originserver cache_peer.
Definition: StateFlags.h:56
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:98
CachePeer * getPeer() const
Definition: Connection.cc:124
struct CachePeer::@28 stats
void push(const Comm::ConnectionPointer &serverConn, const char *domain)
Definition: pconn.cc:412
unsigned char code
Definition: html_quote.c:20
bool canHandle1xx() const
whether the client is likely to be able to handle a 1xx reply
Definition: HttpRequest.cc:643
virtual void handleRequestBodyProducerAborted()
Definition: http.cc:2576
@ scPartialContent
Definition: StatusCode.h:32
@ scGone
Definition: StatusCode.h:54
Http::HdrType id
Definition: HttpHeader.h:63
void proceedAfter1xx()
restores state and resumes processing after 1xx is ignored or forwarded
Definition: http.cc:850
ConnectionStatus persistentConnStatus() const
Definition: http.cc:1118
bool hasSMaxAge(int32_t *val=nullptr) const
Definition: HttpHdrCc.h:127
@ INCOMPLETE_MSG
Definition: http.h:95
#define CallJobHere(debugSection, debugLevel, job, Class, method)
Definition: AsyncJobCalls.h:58
char * peer_domain
Definition: HttpRequest.h:172
@ scUnprocessableEntity
Definition: StatusCode.h:63
#define BBUF_SZ
void httpTimeout(const CommTimeoutCbParams &params)
Definition: http.cc:144
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
Answers answer
the decision id
Definition: http.h:38
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:765
bool hasNoCache(const String **val=nullptr) const
Definition: HttpHdrCc.h:89
HttpRequestMethod method
Definition: MemObject.h:137
virtual bool doneWithServer() const
Definition: http.cc:1705
store_status_t store_status
Definition: Store.h:242
@ scPermanentRedirect
Definition: StatusCode.h:43
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1043
int neighbors_do_private_keys
@ ERR_ZERO_SIZE_OBJECT
Definition: forward.h:46
HttpUpgradeProtocolAccess * http_upgrade_request_protocols
http_upgrade_request_protocols
Definition: SquidConfig.h:485
#define assert(EX)
Definition: assert.h:19
@ scUriTooLong
Definition: StatusCode.h:58
char const * termedBuf() const
Definition: SquidString.h:92
@ scContentTooLarge
Definition: StatusCode.h:57
void setPayloadBuffer(MemBuf *parsedContent)
set the buffer to be used to store decoded chunk data
@ scContinue
Definition: StatusCode.h:22
HttpHdrCc * cache_control
Definition: Message.h:77
FwdState::Pointer fwd
Definition: Client.h:177
HierarchyLogEntry hier
Definition: HttpRequest.h:157
acl_access * brokenPosts
Definition: SquidConfig.h:381
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:161
virtual void syncAle(HttpRequest *adaptedRequest, const char *logUri) const
assigns uninitialized adapted_request and url ALE components
void neighborsHtcpClear(StoreEntry *e, HttpRequest *req, const HttpRequestMethod &method, htcp_clr_reason reason)
Definition: neighbors.cc:1815
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:570
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Parser.h:98
void abortTransaction(const char *reason)
Definition: http.h:115
@ TRANSFER_ENCODING
bool no_delay
Definition: CachePeer.h:101
@ scServiceUnavailable
Definition: StatusCode.h:74
@ METHOD_TRACE
Definition: MethodType.h:30
#define cbdataReferenceDone(var)
Definition: cbdata.h:350
bool adaptationAccessCheckPending
Definition: Client.h:189
void onlyIfCached(bool v)
Definition: HttpHdrCc.h:146
void writeReplyBody()
Definition: http.cc:1400
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
@ scInternalServerError
Definition: StatusCode.h:71
HttpHdrRange * range
Definition: HttpRequest.h:143
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:116
void httpStateConnClosed(const CommCloseCbParams &params)
Definition: http.cc:136
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:568
bool sentLastChunk
do not try to write last-chunk again
Definition: StateFlags.h:64
bool cacheNegatively()
Definition: store.cc:185
const char * c_str()
Definition: SBuf.cc:516
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:33
int commUnsetConnTimeout(const Comm::ConnectionPointer &conn)
Definition: comm.cc:589
virtual const Comm::ConnectionPointer & dataConnection() const
Definition: http.cc:130
bool olderThan(const HttpReply *them) const
Definition: HttpReply.cc:663
SBuf & authority(bool requirePort=false) const
Definition: Uri.cc:566
void base64_encode_init(struct base64_encode_ctx *ctx)
Definition: base64.c:232
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:195
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:415
time_t squid_curtime
Definition: stub_libtime.cc:20
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:210
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
bool isNoAddr() const
Definition: Address.cc:284
int64_t content_length
Definition: Message.h:84
time_t date
Definition: HttpReply.h:40
HttpReply * finalReply()
Definition: Client.cc:129
void putCc(const HttpHdrCc *cc)
Definition: HttpHeader.cc:1068
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:226
bool parseHeader(Http1::Parser &hp)
parses reply header using Parser
Definition: HttpReply.cc:466
@ scNotModified
Definition: StatusCode.h:40
ReuseDecision::Answers reusableReply(ReuseDecision &decision)
Definition: http.cc:294
void markParsedVirginReplyAsWhole(const char *reasonWeAreSure)
Definition: Client.cc:158
uint32_t sources
The message sources.
Definition: Message.h:100
const Acl::Tree * changeAcl(const Acl::Tree *t)
Definition: Checklist.h:176
bool mustKeepalive
Definition: RequestFlags.h:74
Http1::TeChunkedParser * httpChunkDecoder
Definition: http.h:152
void wroteLast(const CommIoCbParams &io)
called after writing the very last request byte (body, last-chunk, etc)
Definition: http.cc:1636
#define fd_table
Definition: fde.h:189
Http1::ResponseParserPointer hp
Parser being used at present to parse the HTTP/ICY server response.
Definition: http.h:151
@ ERR_READ_TIMEOUT
Definition: forward.h:26
assists in making and relaying entry caching/sharing decision
Definition: http.h:30
virtual void noteDelayAwareReadChance()
Definition: http.cc:1168
@ scFound
Definition: StatusCode.h:38
virtual bool parse(const SBuf &)
@ PROTO_HTTPS
Definition: ProtocolType.h:27
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1063
@ scRequestTimeout
Definition: StatusCode.h:52
bool accelerated
Definition: RequestFlags.h:58
void maxAge(int v)
HttpRequestMethod method
Definition: HttpRequest.h:114
void path(const char *p)
Definition: Uri.h:99
const char * reason
the decision reason
Definition: http.h:39
struct SquidConfig::@111 accessList
@ scNotFound
Definition: StatusCode.h:48
acl_access * reply
Definition: SquidConfig.h:385
noteTakeServerConnectionControl() callback parameter
Definition: client_side.h:214
@ PROTO_HTTP
Definition: ProtocolType.h:25
char * host
Definition: CachePeer.h:40
int eof
Definition: http.h:65
bool allowed() const
Definition: Acl.h:149
int cache_miss_revalidate
Definition: SquidConfig.h:325
bool sawDateGoBack
Definition: http.h:161
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:1038
int server_pconns
Definition: SquidConfig.h:311
struct StatCounters::@128::@138 http
AnyP::ProtocolVersion version
breakdown of protocol version label: (HTTP/ICY) and (0.9/1.0/1.1)
Definition: StatusLine.h:62
const char * doneWithFwd
Definition: Client.h:199
struct SquidConfig::@110 onoff
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:364
bool peerSupportsConnectionPinning() const
Definition: http.cc:879
bool timestampsSet()
Definition: store.cc:1371
@ scNoContent
Definition: StatusCode.h:30
@ PROXY_AUTHENTICATE
@ ERR_TOO_BIG
Definition: forward.h:40
bool forcedBodyContinuation
whether we have responded with HTTP 100 or FTP 150 already
Definition: HttpRequest.h:194
virtual void maybeReadVirginBody()
read response data from the network
Definition: http.cc:1573
int has(Http::HdrType id) const
Definition: HttpHeader.cc:991
void processReply()
Definition: http.cc:1271
char * content()
start of the added data
Definition: MemBuf.h:41
#define acl_access
Definition: forward.h:45
bool hasMaxAge(int32_t *val=nullptr) const
Definition: HttpHdrCc.h:122
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1052
@ ERR_WRITE_ERROR
Definition: forward.h:29
void set(const AnyP::ProtocolVersion &newVersion, Http::StatusCode newStatus, const char *newReason=NULL)
Definition: StatusLine.cc:35
bool finishingChunkedRequest()
if needed, write last-chunk to end the request body and return true
Definition: http.cc:2511
size_type size() const
Definition: SquidString.h:73
char * url
Definition: tcp-banger2.c:114
bool isEmpty() const
Definition: Store.h:66
bool refreshIsCachable(const StoreEntry *entry)
Definition: refresh.cc:519
struct StatCounters::@128::@138 all
bool reforwardableStatus(const Http::StatusCode s) const
Definition: FwdState.cc:1407
#define Must(condition)
Definition: TextException.h:71
int redir_rewrites_host
Definition: SquidConfig.h:303
@ SURROGATE_CAPABILITY
#define DBG_IMPORTANT
Definition: Stream.h:41
@ scProxyAuthenticationRequired
Definition: StatusCode.h:51
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:154
void Controller::create() STUB void Controller Controller nil
size_t base64_encode_update(struct base64_encode_ctx *ctx, char *dst, size_t length, const uint8_t *src)
Definition: base64.c:265
char * surrogate_id
Definition: SquidConfig.h:218
StoreEntry * entry
Definition: FwdState.h:169
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:479
void release(const bool shareable=false)
Definition: store.cc:1122
virtual void start()
called by AsyncStart; do not call directly
Definition: http.cc:2456
const HttpReply & freshestReply() const
Definition: MemObject.h:67
ssize_t mb_size_t
Definition: MemBuf.h:17
HttpHdrSc * surrogate_control
Definition: HttpReply.h:48
@ scInvalidHeader
Definition: StatusCode.h:86
@ scPaymentRequired
Definition: StatusCode.h:46
@ scLocked
Definition: StatusCode.h:64
static StoreEntry * findPreviouslyCachedEntry(StoreEntry *newEntry)
Definition: http.cc:157
@ scConflict
Definition: StatusCode.h:53
int n_keepalives_recv
Definition: CachePeer.h:52
#define REFRESH_OVERRIDE(flag)
@ FD_WRITE
Definition: enums.h:24
static void httpBuildRequestHeader(HttpRequest *request, StoreEntry *entry, const AccessLogEntryPointer &al, HttpHeader *hdr_out, const Http::StateFlags &flags)
Definition: http.cc:1825
@ srcHttps
https_port or bumped http_port tunnel; HTTPS server
Definition: Message.h:33
virtual void sentRequestBody(const CommIoCbParams &io)
Definition: http.cc:2596
int64_t readAheadGap
Definition: SquidConfig.h:96
void setNoDelay(bool const)
Definition: store.cc:234
bool hasNoCacheWithoutParameters() const
Definition: HttpHdrCc.h:88
bool hasMustRevalidate() const
Definition: HttpHdrCc.h:112
void processSurrogateControl(HttpReply *)
Definition: http.cc:256
int64_t payloadSeen
amount of message payload/body received so far.
Definition: http.h:155
@ ENTRY_REVALIDATE_STALE
Definition: enums.h:100
bool continueAfterParsingHeader()
Definition: http.cc:1302
@ scMultipleChoices
Definition: StatusCode.h:36
@ ENTRY_REVALIDATE_ALWAYS
Definition: enums.h:85
bool serverSwitchedProtocols
Definition: StateFlags.h:35
@ scOkay
Definition: StatusCode.h:26
std::ostream & operator<<(std::ostream &os, const HttpStateData::ReuseDecision &d)
Definition: http.cc:2623
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:612
@ scHeaderTooLarge
Definition: StatusCode.h:87
#define rfc1738_escape_part(x)
Definition: rfc1738.h:55
time_t lifetime
Definition: SquidConfig.h:112
virtual void closeServer()
Definition: http.cc:1692
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:752
static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest *request, HttpHeader *hdr_out, const int we_do_ranges, const Http::StateFlags &)
Definition: http.cc:2076
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=0)
Definition: HttpHeader.cc:999
HttpRequest * request
Definition: FwdState.h:170
Ip::Address client_addr
Definition: HttpRequest.h:149
bool hasNoStore() const
Definition: HttpHdrCc.h:102
void host(const char *src)
Definition: Uri.cc:99
void forwardUpgrade(HttpHeader &)
Definition: http.cc:2002
@ PEER_SIBLING
Definition: enums.h:29
@ scExpectationFailed
Definition: StatusCode.h:61
HttpRequestPointer request
Definition: Client.h:178
int read_hist[histSize]
Definition: IoStats.h:21
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
virtual void noteTakeServerConnectionControl(ServerConnectionContext)
Definition: client_side.h:230
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
short int keep_alive
Definition: HttpReply.h:53
@ hoRequest
Definition: HttpHeader.h:36
HttpRequestPointer request
Definition: MemObject.h:188
void clean()
Definition: HttpHeader.cc:190
void comm_remove_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:950
Comm::ConnectionPointer const & serverConnection() const
Definition: FwdState.h:106
const char * getMD5Text() const
Definition: store.cc:204
~HttpStateData()
Definition: http.cc:113
bool originserver
Definition: CachePeer.h:117
time_t negativeTtl
Definition: SquidConfig.h:100
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1223
#define SQUIDSBUFPH
Definition: SBuf.h:31
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1034
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1535
CbcPointer< ConnStateData > clientConnectionManager
Definition: HttpRequest.h:232
bool hasMaxAge() const
bool hasPublic() const
Definition: HttpHdrCc.h:69
class SquidConfig Config
Definition: SquidConfig.cc:12
void httpStart(FwdState *fwd)
Definition: http.cc:2449
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
StatCounters statCounter
Definition: StatCounters.cc:12
void removeStaleWarnings()
Remove Warnings with warn-date different from Date value.
Definition: HttpReply.cc:598
void clean()
Definition: String.cc:118
static void Start(const Pointer &job)
Definition: AsyncJob.cc:24
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
@ PROXY_AUTHORIZATION
void maxStale(int v)
#define CallJobHere1(debugSection, debugLevel, job, Class, method, arg1)
Definition: AsyncJobCalls.h:63
@ STORE_PENDING
Definition: enums.h:51

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors