Client.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #include "squid.h"
10 #include "acl/FilledChecklist.h"
11 #include "acl/Gadgets.h"
12 #include "base/TextException.h"
13 #include "clients/Client.h"
14 #include "comm/Connection.h"
15 #include "comm/forward.h"
16 #include "comm/Write.h"
17 #include "error/Detail.h"
18 #include "errorpage.h"
19 #include "fd.h"
20 #include "HttpHdrContRange.h"
21 #include "HttpReply.h"
22 #include "HttpRequest.h"
23 #include "SquidConfig.h"
24 #include "SquidTime.h"
25 #include "StatCounters.h"
26 #include "Store.h"
27 #include "tools.h"
28 
29 #if USE_ADAPTATION
30 #include "adaptation/AccessCheck.h"
31 #include "adaptation/Answer.h"
32 #include "adaptation/Iterator.h"
33 #include "base/AsyncCall.h"
34 #endif
35 
36 // implemented in client_side_reply.cc until sides have a common parent
37 void purgeEntriesByUrl(HttpRequest * req, const char *url);
38 
39 Client::Client(FwdState *theFwdState) :
40  AsyncJob("Client"),
41  fwd(theFwdState),
42  request(fwd->request)
43 {
44  entry = fwd->entry;
45  entry->lock("Client");
46 }
47 
49 {
50  // paranoid: check that swanSong has been called
52 #if USE_ADAPTATION
55 #endif
56 
57  entry->unlock("Client");
58 
61 
62  if (responseBodyBuffer != NULL) {
63  delete responseBodyBuffer;
65  }
66 }
67 
68 void
70 {
71  // get rid of our piping obligations
72  if (requestBodySource != NULL)
74 
75 #if USE_ADAPTATION
77 #endif
78 
79  if (!doneWithServer())
80  closeServer();
81 
82  if (!doneWithFwd) {
83  doneWithFwd = "swanSong()";
85  }
86 
88 #if USE_ADAPTATION
89  Initiator::swanSong();
91 #endif
92 
93  // paranoid: check that swanSong has been called
94  // extra paranoid: yeah, I really mean it. they MUST pass here.
96 #if USE_ADAPTATION
99 #endif
100 }
101 
102 HttpReply *
104 {
106  return theVirginReply;
107 }
108 
109 const HttpReply *
111 {
113  return theVirginReply;
114 }
115 
116 HttpReply *
118 {
119  debugs(11,5, HERE << this << " setting virgin reply to " << rep);
121  assert(rep);
122  theVirginReply = rep;
124  if (fwd->al)
126  return theVirginReply;
127 }
128 
129 HttpReply *
131 {
133  return theFinalReply;
134 }
135 
136 HttpReply *
138 {
139  debugs(11,5, HERE << this << " setting final reply to " << rep);
140 
142  assert(rep);
143  theFinalReply = rep;
145  if (fwd->al)
147 
148  // give entry the reply because haveParsedReplyHeaders() expects it there
149  entry->replaceHttpReply(theFinalReply, false); // but do not write yet
150  haveParsedReplyHeaders(); // update the entry/reply (e.g., set timestamps)
152  entry->release();
153  entry->startWriting(); // write the updated entry to store
154 
155  return theFinalReply;
156 }
157 
158 // called when no more server communication is expected; may quit
159 void
161 {
162  debugs(11,5,HERE << "serverComplete " << this);
163 
164  if (!doneWithServer()) {
165  closeServer();
167  }
168 
169  completed = true;
171 
172  if (requestBodySource != NULL)
174 
175  if (responseBodyBuffer != NULL)
176  return;
177 
178  serverComplete2();
179 }
180 
181 void
183 {
184  debugs(11,5,HERE << "serverComplete2 " << this);
185 
186 #if USE_ADAPTATION
189 
190  if (!doneWithAdaptation())
191  return;
192 #endif
193 
195 }
196 
197 bool Client::doneAll() const
198 {
199  return doneWithServer() &&
200 #if USE_ADAPTATION
201  doneWithAdaptation() &&
204 #endif
206 }
207 
208 // FTP side overloads this to work around multiple calls to fwd->complete
209 void
211 {
212  debugs(11,5, HERE << "completing forwarding for " << fwd);
213  assert(fwd != NULL);
214  doneWithFwd = "completeForwarding()";
215  fwd->complete();
216 }
217 
218 // Register to receive request body
220 {
222  assert(r->body_pipe != NULL);
225  debugs(11,3, HERE << "expecting request body from " <<
227  return true;
228  }
229 
230  debugs(11,3, HERE << "aborting on partially consumed request body: " <<
233  return false;
234 }
235 
236 // Entry-dependent callbacks use this check to quit if the entry went bad
237 bool
238 Client::abortOnBadEntry(const char *abortReason)
239 {
240  if (entry->isAccepting())
241  return false;
242 
243  debugs(11,5, HERE << "entry is not Accepting!");
244  abortOnData(abortReason);
245  return true;
246 }
247 
248 // more request or adapted response body is available
249 void
251 {
252 #if USE_ADAPTATION
253  if (adaptedBodySource == bp) {
255  return;
256  }
257 #endif
258  if (requestBodySource == bp)
260 }
261 
262 // the entire request or adapted response body was provided, successfully
263 void
265 {
266 #if USE_ADAPTATION
267  if (adaptedBodySource == bp) {
269  return;
270  }
271 #endif
272  if (requestBodySource == bp)
274 }
275 
276 // premature end of the request or adapted response body production
277 void
279 {
280 #if USE_ADAPTATION
281  if (adaptedBodySource == bp) {
283  return;
284  }
285 #endif
286  if (requestBodySource == bp)
288 }
289 
290 bool
291 Client::abortOnData(const char *reason)
292 {
293  abortAll(reason);
294  return true;
295 }
296 
297 // more origin request body data is available
298 void
300 {
301  if (!requestSender)
303  else
304  debugs(9,3, HERE << "waiting for request body write to complete");
305 }
306 
307 // there will be no more handleMoreRequestBodyAvailable calls
308 void
310 {
312  if (!requestSender)
314  else
315  debugs(9,3, HERE << "waiting for request body write to complete");
316 }
317 
318 // called when we are done sending request body; kids extend this
319 void
321 {
322  debugs(9,3, HERE << "done sending request body");
325 
326  // kids extend this
327 }
328 
329 // called when body producers aborts; kids extend this
330 void
332 {
333  if (requestSender != NULL)
334  debugs(9,3, HERE << "fyi: request body aborted while we were sending");
335 
336  fwd->dontRetry(true); // the problem is not with the server
337  stopConsumingFrom(requestBodySource); // requestSender, if any, will notice
338 
339  // kids extend this
340 }
341 
342 // called when we wrote request headers(!) or a part of the body
343 void
345 {
346  debugs(11, 5, "sentRequestBody: FD " << io.fd << ": size " << io.size << ": errflag " << io.flag << ".");
347  debugs(32,3,HERE << "sentRequestBody called");
348 
350 
351  if (io.size > 0) {
352  fd_bytes(io.fd, io.size, FD_WRITE);
353  statCounter.server.all.kbytes_out += io.size;
354  // kids should increment their counters
355  }
356 
357  if (io.flag == Comm::ERR_CLOSING)
358  return;
359 
360  if (!requestBodySource) {
361  debugs(9,3, HERE << "detected while-we-were-sending abort");
362  return; // do nothing;
363  }
364 
365  // both successful and failed writes affect response times
367 
368  if (io.flag) {
369  debugs(11, DBG_IMPORTANT, "sentRequestBody error: FD " << io.fd << ": " << xstrerr(io.xerrno));
370  ErrorState *err;
372  err->xerrno = io.xerrno;
373  fwd->fail(err);
374  abortOnData("I/O error while sending request body");
375  return;
376  }
377 
379  abortOnData("store entry aborted while sending request body");
380  return;
381  }
382 
385  else if (receivedWholeRequestBody)
387  else
388  debugs(9,3, HERE << "waiting for body production end or abort");
389 }
390 
391 void
393 {
396 
398 
399  if (!Comm::IsConnOpen(conn)) {
400  debugs(9,3, HERE << "cannot send request body to closing " << conn);
401  return; // wait for the kid's close handler; TODO: assert(closer);
402  }
403 
404  MemBuf buf;
405  if (getMoreRequestBody(buf) && buf.contentSize() > 0) {
406  debugs(9,3, HERE << "will write " << buf.contentSize() << " request body bytes");
408  requestSender = JobCallback(93,3, Dialer, this, Client::sentRequestBody);
410  } else {
411  debugs(9,3, HERE << "will wait for more request body bytes or eof");
413  }
414 }
415 
417 bool
419 {
420  // default implementation does not encode request body content
423 }
424 
425 // Compares hosts in urls, returns false if different, no sheme, or no host.
426 static bool
427 sameUrlHosts(const char *url1, const char *url2)
428 {
429  // XXX: Want AnyP::Uri::parse() here, but it uses static storage and copying
430  const char *host1 = strchr(url1, ':');
431  const char *host2 = strchr(url2, ':');
432 
433  if (host1 && host2) {
434  // skip scheme slashes
435  do {
436  ++host1;
437  ++host2;
438  } while (*host1 == '/' && *host2 == '/');
439 
440  if (!*host1)
441  return false; // no host
442 
443  // increment while the same until we reach the end of the URL/host
444  while (*host1 && *host1 != '/' && *host1 == *host2) {
445  ++host1;
446  ++host2;
447  }
448  return *host1 == *host2;
449  }
450 
451  return false; // no URL scheme
452 }
453 
454 // purges entries that match the value of a given HTTP [response] header
455 static void
456 purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
457 {
458  const auto hdrUrl = rep->header.getStr(hdr);
459  if (!hdrUrl)
460  return;
461 
462  /*
463  * If the URL is relative, make it absolute so we can find it.
464  * If it's absolute, make sure the host parts match to avoid DOS attacks
465  * as per RFC 2616 13.10.
466  */
467  SBuf absUrlMaker;
468  const char *absUrl = nullptr;
469  if (urlIsRelative(hdrUrl)) {
470  if (req->method.id() == Http::METHOD_CONNECT)
471  absUrl = hdrUrl; // TODO: merge authority-uri and hdrUrl
472  else if (req->url.getScheme() == AnyP::PROTO_URN)
473  absUrl = req->url.absolute().c_str();
474  else {
475  AnyP::Uri tmpUrl = req->url;
476  if (*hdrUrl == '/') {
477  // RFC 3986 section 4.2: absolute-path reference
478  // for this logic replace the entire request-target URI path
479  tmpUrl.path(hdrUrl);
480  } else {
481  tmpUrl.addRelativePath(reqUrl);
482  }
483  absUrlMaker = tmpUrl.absolute();
484  absUrl = absUrlMaker.c_str();
485  }
486  } else if (!sameUrlHosts(reqUrl, hdrUrl)) {
487  return;
488  } else
489  absUrl = hdrUrl;
490 
491  purgeEntriesByUrl(req, absUrl);
492 }
493 
494 // some HTTP methods should purge matching cache entries
495 void
497 {
498  // only some HTTP methods should purge matching cache entries
499  if (!request->method.purgesOthers())
500  return;
501 
502  // and probably only if the response was successful
503  if (theFinalReply->sline.status() >= 400)
504  return;
505 
506  // XXX: should we use originalRequest() here?
508  const char *reqUrl = tmp.c_str();
509  debugs(88, 5, "maybe purging due to " << request->method << ' ' << tmp);
510  purgeEntriesByUrl(request.getRaw(), reqUrl);
513 }
514 
516 void
518 {
521 
522  // adaptation may overwrite old offset computed using the virgin response
523  currentOffset = 0;
524  if (const auto cr = theFinalReply->contentRange()) {
525  if (cr->spec.offset != HttpHdrRangeSpec::UnknownPosition)
526  currentOffset = cr->spec.offset;
527  }
528 }
529 
531 bool
533 {
534  if (const Acl::Tree *acl = Config.accessList.storeMiss) {
535  // This relatively expensive check is not in StoreEntry::checkCachable:
536  // That method lacks HttpRequest and may be called too many times.
537  ACLFilledChecklist ch(acl, originalRequest().getRaw());
538  ch.reply = const_cast<HttpReply*>(&entry->mem().freshestReply()); // ACLFilledChecklist API bug
539  HTTPMSGLOCK(ch.reply);
540  ch.al = fwd->al;
541  if (!ch.fastCheck().allowed()) { // when in doubt, block
542  debugs(20, 3, "store_miss prohibits caching");
543  return true;
544  }
545  }
546  return false;
547 }
548 
551 {
552  return request;
553 }
554 
555 #if USE_ADAPTATION
556 void
559 {
560  debugs(11, 5, "Client::startAdaptation() called");
561  // check whether we should be sending a body as well
562  // start body pipe to feed ICAP transaction if needed
564  HttpReply *vrep = virginReply();
565  assert(!vrep->body_pipe);
566  int64_t size = 0;
567  if (vrep->expectingBody(cause->method, size) && size) {
568  virginBodyDestination = new BodyPipe(this);
570  debugs(93, 6, HERE << "will send virgin reply body to " <<
571  virginBodyDestination << "; size: " << size);
572  if (size > 0)
574  }
575 
577  new Adaptation::Iterator(vrep, cause, fwd->al, group));
580 }
581 
582 // properly cleans up ICAP-related state
583 // may be called multiple times
585 {
586  debugs(11,5, HERE << "cleaning ICAP; ACL: " << adaptationAccessCheckPending);
587 
590 
592 
593  if (adaptedBodySource != NULL)
595 
596  if (!adaptationAccessCheckPending) // we cannot cancel a pending callback
597  assert(doneWithAdaptation()); // make sure the two methods are in sync
598 }
599 
600 bool
602 {
605 }
606 
607 // sends virgin reply body to ICAP, buffering excesses if needed
608 void
609 Client::adaptVirginReplyBody(const char *data, ssize_t len)
610 {
612 
613  if (!virginBodyDestination) {
614  debugs(11,3, HERE << "ICAP does not want more virgin body");
615  return;
616  }
617 
618  // grow overflow area if already overflowed
619  if (responseBodyBuffer) {
623  }
624 
625  const ssize_t putSize = virginBodyDestination->putMoreData(data, len);
626  data += putSize;
627  len -= putSize;
628 
629  // if we had overflow area, shrink it as necessary
630  if (responseBodyBuffer) {
631  if (putSize == responseBodyBuffer->contentSize()) {
632  delete responseBodyBuffer;
634  } else {
635  responseBodyBuffer->consume(putSize);
636  }
637  return;
638  }
639 
640  // if we did not have an overflow area, create it as needed
641  if (len > 0) {
644  responseBodyBuffer->init(4096, SQUID_TCP_SO_RCVBUF * 10);
646  }
647 }
648 
649 // can supply more virgin response body data
650 void
652 {
653  if (responseBodyBuffer) {
654  addVirginReplyBody(NULL, 0); // kick the buffered fragment alive again
655  if (completed && !responseBodyBuffer) {
656  serverComplete2();
657  return;
658  }
659  }
661 }
662 
663 // the consumer of our virgin response body aborted
664 void
666 {
668 
669  // do not force closeServer here in case we need to bypass AdaptationQueryAbort
670 
671  if (doneWithAdaptation()) // we may still be receiving adapted response
673 }
674 
675 // received adapted response headers (body may follow)
676 void
678 {
679  clearAdaptation(adaptedHeadSource); // we do not expect more messages
680 
681  switch (answer.kind) {
683  handleAdaptedHeader(const_cast<Http::Message*>(answer.message.getRaw()));
684  break;
685 
687  handleAdaptationBlocked(answer);
688  break;
689 
692  break;
693  }
694 }
695 
696 void
698 {
699  if (abortOnBadEntry("entry went bad while waiting for adapted headers")) {
700  // If the adapted response has a body, the ICAP side needs to know
701  // that nobody will consume that body. We will be destroyed upon
702  // return. Tell the ICAP side that it is on its own.
703  HttpReply *rep = dynamic_cast<HttpReply*>(msg);
704  assert(rep);
705  if (rep->body_pipe != NULL)
707 
708  return;
709  }
710 
711  HttpReply *rep = dynamic_cast<HttpReply*>(msg);
712  assert(rep);
713  debugs(11,5, HERE << this << " setting adapted reply to " << rep);
714  setFinalReply(rep);
715 
717  if (rep->body_pipe != NULL) {
718  // subscribe to receive adapted body
720  // assume that ICAP does not auto-consume on failures
721  const bool result = adaptedBodySource->setConsumerIfNotLate(this);
722  assert(result);
723  } else {
724  // no body
725  if (doneWithAdaptation()) // we may still be sending virgin response
727  }
728 }
729 
730 void
732 {
733  if (abortOnBadEntry("store entry aborted while kick producer callback"))
734  return;
735 
736  if (!adaptedBodySource)
737  return;
738 
740 
743 }
744 
745 // more adapted response body is available
746 void
748 {
749  if (abortOnBadEntry("entry refuses adapted body"))
750  return;
751 
752  assert(entry);
753 
754  size_t contentSize = adaptedBodySource->buf().contentSize();
755 
756  if (!contentSize)
757  return; // XXX: bytesWanted asserts on zero-size ranges
758 
759  const size_t spaceAvailable = entry->bytesWanted(Range<size_t>(0, contentSize), true);
760 
761  if (spaceAvailable < contentSize ) {
762  // No or partial body data consuming
763  typedef NullaryMemFunT<Client> Dialer;
764  AsyncCall::Pointer call = asyncCall(93, 5, "Client::resumeBodyStorage",
765  Dialer(this, &Client::resumeBodyStorage));
766  entry->deferProducer(call);
767  }
768 
769  if (!spaceAvailable) {
770  debugs(11, 5, HERE << "NOT storing " << contentSize << " bytes of adapted " <<
771  "response body at offset " << adaptedBodySource->consumedSize());
772  return;
773  }
774 
775  if (spaceAvailable < contentSize ) {
776  debugs(11, 5, HERE << "postponing storage of " <<
777  (contentSize - spaceAvailable) << " body bytes");
778  contentSize = spaceAvailable;
779  }
780 
781  debugs(11,5, HERE << "storing " << contentSize << " bytes of adapted " <<
782  "response body at offset " << adaptedBodySource->consumedSize());
783 
785  const StoreIOBuffer ioBuf(&bpc.buf, currentOffset, contentSize);
786  currentOffset += ioBuf.length;
787  entry->write(ioBuf);
788  bpc.buf.consume(contentSize);
789  bpc.checkIn();
790 }
791 
792 // the entire adapted response body was produced, successfully
793 void
795 {
796  if (abortOnBadEntry("entry went bad while waiting for adapted body eof"))
797  return;
798 
799  // end consumption if we consumed everything
802  // else resumeBodyStorage() will eventually consume the rest
803 }
804 
805 void
807 {
810 }
811 
812 // premature end of the adapted response body
814 {
815  if (abortOnBadEntry("entry went bad while waiting for the now-aborted adapted body"))
816  return;
817 
818  Must(adaptedBodySource != nullptr);
819  if (!adaptedBodySource->exhausted()) {
820  debugs(11,5, "waiting to consume the remainder of the aborted adapted body");
821  return; // resumeBodyStorage() should eventually consume the rest
822  }
823 
825 
827  return;
828 
829  entry->lengthWentBad("body adaptation aborted");
830  handleAdaptationCompleted(); // the user should get a truncated response
831 }
832 
833 // common part of noteAdaptationAnswer and handleAdaptedBodyProductionEnded
834 void
836 {
837  debugs(11,5, HERE << "handleAdaptationCompleted");
838  cleanAdaptation();
839 
840  // We stop reading origin response because we have no place to put it(*) and
841  // cannot use it. If some origin servers do not like that or if we want to
842  // reuse more pconns, we can add code to discard unneeded origin responses.
843  // (*) TODO: Is it possible that the adaptation xaction is still running?
844  if (mayReadVirginReplyBody()) {
845  debugs(11,3, HERE << "closing origin conn due to ICAP completion");
846  closeServer();
847  }
848 
850 }
851 
852 // common part of noteAdaptation*Aborted and noteBodyConsumerAborted methods
853 void
855 {
856  debugs(11,5, HERE << "handleAdaptationAborted; bypassable: " << bypassable <<
857  ", entry empty: " << entry->isEmpty());
858 
859  if (abortOnBadEntry("entry went bad while ICAP aborted"))
860  return;
861 
862  // TODO: bypass if possible
864  abortAll("adaptation failure with a filled entry");
865 }
866 
869 bool
871 {
872  if (entry->isEmpty()) {
873  debugs(11,8, "adaptation failure with an empty entry: " << *entry);
875  static const auto d = MakeNamedErrorDetail("ICAP_RESPMOD_EARLY");
876  err->detailError(d);
877  fwd->fail(err);
878  fwd->dontRetry(true);
879  abortAll("adaptation failure with an empty entry");
880  return true; // handled
881  }
882 
883  if (request) { // update logged info directly
884  static const auto d = MakeNamedErrorDetail("ICAP_RESPMOD_LATE");
886  }
887 
888  return false; // the caller must handle
889 }
890 
891 // adaptation service wants us to deny HTTP client access to this response
892 void
894 {
895  debugs(11,5, HERE << answer.ruleId);
896 
897  if (abortOnBadEntry("entry went bad while ICAP aborted"))
898  return;
899 
900  if (!entry->isEmpty()) { // too late to block (should not really happen)
901  if (request) {
902  static const auto d = MakeNamedErrorDetail("RESPMOD_BLOCK_LATE");
904  }
905  abortAll("late adaptation block");
906  return;
907  }
908 
909  debugs(11,7, HERE << "creating adaptation block response");
910 
911  err_type page_id =
913  if (page_id == ERR_NONE)
914  page_id = ERR_ACCESS_DENIED;
915 
916  const auto err = new ErrorState(page_id, Http::scForbidden, request.getRaw(), fwd->al);
917  static const auto d = MakeNamedErrorDetail("RESPMOD_BLOCK_EARLY");
918  err->detailError(d);
919  fwd->fail(err);
920  fwd->dontRetry(true);
921 
922  abortOnData("timely adaptation block");
923 }
924 
925 void
927 {
929 
930  if (abortOnBadEntry("entry went bad while waiting for ICAP ACL check"))
931  return;
932 
933  // TODO: Should non-ICAP and ICAP REPMOD pre-cache paths check this?
934  // That check now only happens on REQMOD pre-cache and REPMOD post-cache, in processReplyAccess().
935  if (virginReply()->expectedBodyTooLarge(*request)) {
937  return;
938  }
939  // TODO: Should we check receivedBodyTooLarge as well?
940 
941  if (!group) {
942  debugs(11,3, HERE << "no adapation needed");
945  return;
946  }
947 
948  startAdaptation(group, originalRequest().getRaw());
950 }
951 #endif
952 
953 void
955 {
956  const auto err = new ErrorState(ERR_TOO_BIG, Http::scForbidden, request.getRaw(), fwd->al);
957  fwd->fail(err);
958  fwd->dontRetry(true);
959  abortOnData("Virgin body too large.");
960 }
961 
962 // TODO: when HttpStateData sends all errors to ICAP,
963 // we should be able to move this at the end of setVirginReply().
964 void
966 {
967 #if USE_ADAPTATION
968  // TODO: merge with client side and return void to hide the on/off logic?
969  // The callback can be called with a NULL service if adaptation is off.
972  originalRequest().getRaw(), virginReply(), fwd->al, this);
973  debugs(11,5, HERE << "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
975  return;
976 #endif
977 
979 }
980 
982 void
983 Client::adjustBodyBytesRead(const int64_t delta)
984 {
985  int64_t &bodyBytesRead = originalRequest()->hier.bodyBytesRead;
986 
987  // if we got here, do not log a dash even if we got nothing from the server
988  if (bodyBytesRead < 0)
989  bodyBytesRead = 0;
990 
991  bodyBytesRead += delta; // supports negative and zero deltas
992 
993  // check for overflows ("infinite" response?) and underflows (a bug)
994  Must(bodyBytesRead >= 0);
995 }
996 
997 void
998 Client::addVirginReplyBody(const char *data, ssize_t len)
999 {
1001 
1002 #if USE_ADAPTATION
1003  assert(!adaptationAccessCheckPending); // or would need to buffer while waiting
1004  if (startedAdaptation) {
1006  return;
1007  }
1008 #endif
1010 }
1011 
1012 // writes virgin or adapted reply body to store
1013 void
1014 Client::storeReplyBody(const char *data, ssize_t len)
1015 {
1016  // write even if len is zero to push headers towards the client side
1018 
1019  currentOffset += len;
1020 }
1021 
1022 size_t
1023 Client::calcBufferSpaceToReserve(size_t space, const size_t wantSpace) const
1024 {
1025  if (space < wantSpace) {
1026  const size_t maxSpace = SBuf::maxSize; // absolute best
1027  space = min(wantSpace, maxSpace); // do not promise more than asked
1028  }
1029 
1030 #if USE_ADAPTATION
1031  if (responseBodyBuffer) {
1032  return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1033  }
1034 
1035  if (virginBodyDestination != NULL) {
1036  /*
1037  * BodyPipe buffer has a finite size limit. We
1038  * should not read more data from the network than will fit
1039  * into the pipe buffer or we _lose_ what did not fit if
1040  * the response ends sooner that BodyPipe frees up space:
1041  * There is no code to keep pumping data into the pipe once
1042  * response ends and serverComplete() is called.
1043  */
1044  const size_t adaptor_space = virginBodyDestination->buf().potentialSpaceSize();
1045 
1046  debugs(11,9, "Client may read up to min(" <<
1047  adaptor_space << ", " << space << ") bytes");
1048 
1049  if (adaptor_space < space)
1050  space = adaptor_space;
1051  }
1052 #endif
1053 
1054  return space;
1055 }
1056 
1057 size_t
1058 Client::replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
1059 {
1060  size_t space = readBuf.spaceSize(); // available space w/o heroic measures
1061  if (space < minSpace) {
1062  const size_t maxSpace = readBuf.potentialSpaceSize(); // absolute best
1063  space = min(minSpace, maxSpace); // do not promise more than asked
1064  }
1065 
1066 #if USE_ADAPTATION
1067  if (responseBodyBuffer) {
1068  return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1069  }
1070 
1071  if (virginBodyDestination != NULL) {
1072  /*
1073  * BodyPipe buffer has a finite size limit. We
1074  * should not read more data from the network than will fit
1075  * into the pipe buffer or we _lose_ what did not fit if
1076  * the response ends sooner that BodyPipe frees up space:
1077  * There is no code to keep pumping data into the pipe once
1078  * response ends and serverComplete() is called.
1079  *
1080  * If the pipe is totally full, don't register the read handler.
1081  * The BodyPipe will call our noteMoreBodySpaceAvailable() method
1082  * when it has free space again.
1083  */
1084  size_t adaptation_space =
1086 
1087  debugs(11,9, "Client may read up to min(" <<
1088  adaptation_space << ", " << space << ") bytes");
1089 
1090  if (adaptation_space < space)
1091  space = adaptation_space;
1092  }
1093 #endif
1094 
1095  return space;
1096 }
1097 
virtual void noteMoreBodyDataAvailable(BodyPipe::Pointer)
Definition: Client.cc:250
const MemBuf & buf() const
Definition: BodyPipe.h:137
const char * xstrerr(int error)
Definition: xstrerror.cc:83
void handleMoreRequestBodyAvailable()
Definition: Client.cc:299
static bool sameUrlHosts(const char *url1, const char *url2)
Definition: Client.cc:427
void storeReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:1014
HttpReply * theVirginReply
Definition: Client.h:190
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:418
StoreEntry * entry
Definition: Client.h:164
bool initiated(const CbcPointer< AsyncJob > &job) const
Must(initiated(initiate)) instead of Must(initiate.set()), for clarity.
Definition: Initiator.h:52
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
common parts of HttpRequest and HttpReply
Definition: Message.h:26
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:98
void endAdaptedBodyConsumption()
called when the entire adapted response body is consumed
Definition: Client.cc:806
Definition: Uri.h:31
void adaptVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:609
@ FD_WRITE
Definition: enums.h:24
struct StatCounters::@131 server
void stopPeerClock(const bool force)
Definition: access_log.cc:312
virtual void swanSong()
Definition: Client.cc:69
bool completed
Definition: Client.h:86
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1207
HttpHeader header
Definition: Message.h:75
AsyncCall::Pointer requestSender
Definition: Client.h:170
virtual void swanSong()
Definition: AsyncJob.h:54
bool final
whether the error, if any, cannot be bypassed
Definition: Answer.h:41
Client(FwdState *)
Definition: Client.cc:39
struct StatCounters::@131::@141 all
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
MemObject & mem()
Definition: Store.h:52
void handleAdaptationCompleted()
Definition: Client.cc:835
void lock(const char *context)
Definition: store.cc:459
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:331
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:166
HttpReplyPointer reply
virtual void abortAll(const char *reason)=0
abnormal transaction termination; reason is for debugging only
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
Definition: SBuf.h:87
void maybePurgeOthers()
Definition: Client.cc:496
void serverComplete2()
Definition: Client.cc:182
bool purgesOthers() const
BodyPipe::Pointer requestBodySource
Definition: Client.h:169
static void purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
Definition: Client.cc:456
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
C * getRaw() const
Definition: RefCount.h:80
virtual void noteBodyConsumerAborted(BodyPipe::Pointer)
Definition: Client.cc:665
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:216
uint16_t flags
Definition: Store.h:224
virtual const Comm::ConnectionPointer & dataConnection() const =0
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
void cleanAdaptation()
Definition: Client.cc:584
Acl::Answer const & fastCheck()
Definition: Checklist.cc:336
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:26
void adjustBodyBytesRead(const int64_t delta)
initializes bodyBytesRead stats if needed and applies delta
Definition: Client.cc:983
bool isAccepting() const
Definition: store.cc:1987
Http::StatusLine sline
Definition: HttpReply.h:56
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:150
@ ERR_NONE
Definition: forward.h:15
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1719
virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer)
Definition: Client.cc:651
err_type
Definition: forward.h:14
void adaptOrFinalizeReply()
Definition: Client.cc:965
MemBuf * responseBodyBuffer
Definition: Client.h:161
void fail(ErrorState *err)
Definition: FwdState.cc:470
virtual void noteBodyProductionEnded(BodyPipe::Pointer)
Definition: Client.cc:264
BodyPipe::Pointer virginBodyDestination
Definition: Client.h:173
bool startRequestBodyFlow()
Definition: Client.cc:219
@ akForward
forward the supplied adapted HTTP message
Definition: Answer.h:27
void startWriting()
Definition: store.cc:1735
void detailError(const err_type c, const ErrorDetail::Pointer &d)
sets error detail if no earlier detail was available
Definition: HttpRequest.h:101
@ ERR_CLOSING
Definition: Flag.h:25
CbcPointer< Initiate > initiateAdaptation(Initiate *x)
< starts freshly created initiate and returns a safe pointer to it
Definition: Initiator.cc:23
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:517
int64_t currentOffset
Definition: Client.h:160
bool setConsumerIfNotLate(const Consumer::Pointer &aConsumer)
Definition: BodyPipe.cc:228
#define DBG_IMPORTANT
Definition: Debug.h:46
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb.
Definition: SBuf.h:95
AclDenyInfoList * denyInfoList
Definition: SquidConfig.h:413
virtual bool mayReadVirginReplyBody() const =0
whether we may receive more virgin response body bytes
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:294
virtual void noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer group)
Definition: Client.cc:926
bool exhausted() const
Definition: BodyPipe.cc:174
AnyP::UriScheme const & getScheme() const
Definition: Uri.h:67
void serverComplete()
Definition: Client.cc:160
virtual void completeForwarding()
Definition: Client.cc:210
Definition: Range.h:19
@ ERR_ACCESS_DENIED
Definition: forward.h:18
@ PROTO_URN
Definition: ProtocolType.h:37
void handleMoreAdaptedBodyAvailable()
Definition: Client.cc:747
MemBuf & buf
Definition: BodyPipe.h:74
void handleAdaptedHeader(Http::Message *msg)
Definition: Client.cc:697
@ ERR_ICAP_FAILURE
Definition: forward.h:64
Http::MethodType id() const
Definition: RequestMethod.h:70
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const
Definition: HttpReply.cc:488
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
int size
Definition: ModDevPoll.cc:77
#define NULL
Definition: types.h:166
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:96
virtual void doneSendingRequestBody()=0
Definition: Client.cc:320
@ scForbidden
Definition: StatusCode.h:47
err_type aclGetDenyInfoPage(AclDenyInfoList **head, const char *name, int redirect_allowed)
Definition: Gadgets.cc:41
bool dontRetry()
Definition: FwdState.h:94
void resumeBodyStorage()
called by StoreEntry when it has more buffer space available
Definition: Client.cc:731
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
const HttpReply * virginReply() const
Definition: Client.cc:110
void stopConsumingFrom(RefCount< BodyPipe > &)
Definition: BodyPipe.cc:118
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
iterates services in ServiceGroup, starting adaptation launchers
Definition: Iterator.h:32
HttpReply * theFinalReply
Definition: Client.h:191
bool blockCaching()
whether to prevent caching of an otherwise cachable response
Definition: Client.cc:532
virtual void noteBodyProducerAborted(BodyPipe::Pointer)
Definition: Client.cc:278
virtual void noteAdaptationAnswer(const Adaptation::Answer &answer)
Definition: Client.cc:677
virtual void processReplyBody()=0
void clearAdaptation(CbcPointer< Initiate > &x)
clears the pointer (does not call announceInitiatorAbort)
Definition: Initiator.cc:32
@ scBadGateway
Definition: StatusCode.h:73
Definition: MemBuf.h:24
@ CONTENT_LOCATION
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:157
Kind kind
the type of the answer
Definition: Answer.h:42
@ METHOD_CONNECT
Definition: MethodType.h:29
void const char HLPCB void * data
Definition: stub_helper.cc:16
void handleAdaptedBodyProducerAborted()
Definition: Client.cc:813
int unlock(const char *context)
Definition: store.cc:483
virtual ~Client()
Definition: Client.cc:48
Definition: Tree.h:21
int conn
the current server connection FD
Definition: Transport.cc:26
#define assert(EX)
Definition: assert.h:19
void sendMoreRequestBody()
Definition: Client.cc:392
char const * termedBuf() const
Definition: SquidString.h:91
static bool Start(Method method, VectPoint vp, HttpRequest *req, HttpReply *rep, AccessLogEntry::Pointer &al, Adaptation::Initiator *initiator)
Definition: AccessCheck.cc:30
FwdState::Pointer fwd
Definition: Client.h:165
HierarchyLogEntry hier
Definition: HttpRequest.h:157
void purgeEntriesByUrl(HttpRequest *req, const char *url)
bool handledEarlyAdaptationAbort()
Definition: Client.cc:870
HttpReply * setFinalReply(HttpReply *r)
Definition: Client.cc:137
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:161
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:550
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
bool adaptationAccessCheckPending
Definition: Client.h:177
void write(StoreIOBuffer)
Definition: store.cc:797
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
@ scInternalServerError
Definition: StatusCode.h:71
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:117
const char * c_str()
Definition: SBuf.cc:516
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:35
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:180
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:249
static int64_t const UnknownPosition
void addRelativePath(const char *relUrl)
Definition: Uri.cc:711
HttpReply * finalReply()
Definition: Client.cc:130
int xerrno
Definition: errorpage.h:177
bool abortOnBadEntry(const char *abortReason)
Entry-dependent callbacks use this check to quit if the entry went bad.
Definition: Client.cc:238
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:256
summarizes adaptation service answer for the noteAdaptationAnswer() API
Definition: Answer.h:23
BodyPipe::Pointer adaptedBodySource
Definition: Client.h:175
virtual bool doneAll() const
whether positive goal has been reached
Definition: Client.cc:197
virtual bool doneWithAdaptation() const
Definition: Client.cc:601
void sendBodyIsTooLargeError()
Definition: Client.cc:954
int64_t bodyBytesRead
number of body bytes received from the next hop or -1
SBuf & absolute() const
Definition: Uri.cc:583
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1023
void startAdaptation(const Adaptation::ServiceGroupPointer &group, HttpRequest *cause)
Initiate an asynchronous adaptation transaction which will call us back.
Definition: Client.cc:558
@ pointPreCache
Definition: Elements.h:18
void handleAdaptationAborted(bool bypassable=false)
Definition: Client.cc:854
HttpRequestMethod method
Definition: HttpRequest.h:114
void path(const char *p)
Definition: Uri.h:99
Http::MessagePointer message
HTTP request or response to forward.
Definition: Answer.h:39
@ ENTRY_ABORTED
Definition: enums.h:115
@ methodRespmod
Definition: Elements.h:17
bool allowed() const
Definition: Acl.h:143
size_t putMoreData(const char *buf, size_t size)
Definition: BodyPipe.cc:213
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:998
const char * status() const
Definition: BodyPipe.cc:445
struct SquidConfig::@114 accessList
const char * doneWithFwd
Definition: Client.h:184
void complete()
Definition: FwdState.cc:529
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:344
mb_size_t potentialSpaceSize() const
Definition: MemBuf.cc:164
@ akError
no adapted message will come; see bypassable
Definition: Answer.h:29
@ ERR_TOO_BIG
Definition: forward.h:40
const HttpHdrContRange * contentRange() const
Definition: HttpReply.cc:343
char * content()
start of the added data
Definition: MemBuf.h:41
mb_size_t spaceSize() const
Definition: MemBuf.cc:158
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
@ ERR_WRITE_ERROR
Definition: forward.h:29
void announceInitiatorAbort(CbcPointer< Initiate > &x)
inform the transaction about abnormal termination and clear the pointer
Definition: Initiator.cc:38
bool isEmpty() const
Definition: Store.h:67
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:73
acl_access * storeMiss
Definition: SquidConfig.h:373
void handleRequestBodyProductionEnded()
Definition: Client.cc:309
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
virtual void maybeReadVirginBody()=0
read response data from the network
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
StoreEntry * entry
Definition: FwdState.h:164
bool urlIsRelative(const char *url)
Definition: Uri.cc:675
void release(const bool shareable=false)
Definition: store.cc:1154
virtual void closeServer()=0
virtual bool abortOnData(const char *reason)
Definition: Client.cc:291
const HttpReply & freshestReply() const
Definition: MemObject.h:67
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:98
void handleAdaptedBodyProductionEnded()
Definition: Client.cc:794
bool startedAdaptation
Definition: Client.h:178
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:378
@ akBlock
block or deny the master xaction; see authority
Definition: Answer.h:28
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1028
void consume(mb_size_t sz)
removes sz bytes and "packs" by moving content left
Definition: MemBuf.cc:171
CbcPointer< Adaptation::Initiate > adaptedHeadSource
Definition: Client.h:174
size_t replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
Definition: Client.cc:1058
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:688
void handleAdaptationBlocked(const Adaptation::Answer &answer)
Definition: Client.cc:893
A const & min(A const &lhs, A const &rhs)
HttpRequest * request
Definition: FwdState.h:165
String ruleId
ACL (or similar rule) name that blocked forwarding.
Definition: Answer.h:40
HttpRequestPointer request
Definition: Client.h:166
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
void const char * buf
Definition: stub_helper.cc:16
uint64_t consumedSize() const
Definition: BodyPipe.h:111
void handleUnregisteredServerEnd()
Definition: FwdState.cc:766
class SquidConfig Config
Definition: SquidConfig.cc:12
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
StatCounters statCounter
Definition: StatCounters.cc:12
virtual bool doneWithServer() const =0

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors