Client.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #include "squid.h"
10 #include "acl/FilledChecklist.h"
11 #include "acl/Gadgets.h"
12 #include "base/TextException.h"
13 #include "clients/Client.h"
14 #include "comm/Connection.h"
15 #include "comm/forward.h"
16 #include "comm/Write.h"
17 #include "err_detail_type.h"
18 #include "errorpage.h"
19 #include "fd.h"
20 #include "HttpHdrContRange.h"
21 #include "HttpReply.h"
22 #include "HttpRequest.h"
23 #include "SquidConfig.h"
24 #include "SquidTime.h"
25 #include "StatCounters.h"
26 #include "Store.h"
27 #include "tools.h"
28 #include "URL.h"
29 
30 #if USE_ADAPTATION
31 #include "adaptation/AccessCheck.h"
32 #include "adaptation/Answer.h"
33 #include "adaptation/Iterator.h"
34 #include "base/AsyncCall.h"
35 #endif
36 
37 // implemented in client_side_reply.cc until sides have a common parent
38 void purgeEntriesByUrl(HttpRequest * req, const char *url);
39 
40 Client::Client(FwdState *theFwdState) :
41  AsyncJob("Client"),
42  fwd(theFwdState),
43  request(fwd->request)
44 {
45  entry = fwd->entry;
46  entry->lock("Client");
47 }
48 
50 {
51  // paranoid: check that swanSong has been called
53 #if USE_ADAPTATION
56 #endif
57 
58  entry->unlock("Client");
59 
62 
63  if (responseBodyBuffer != NULL) {
64  delete responseBodyBuffer;
66  }
67 }
68 
69 void
71 {
72  // get rid of our piping obligations
73  if (requestBodySource != NULL)
75 
76 #if USE_ADAPTATION
78 #endif
79 
80  if (!doneWithServer())
81  closeServer();
82 
83  if (!doneWithFwd) {
84  doneWithFwd = "swanSong()";
86  }
87 
89 #if USE_ADAPTATION
90  Initiator::swanSong();
92 #endif
93 
94  // paranoid: check that swanSong has been called
95  // extra paranoid: yeah, I really mean it. they MUST pass here.
97 #if USE_ADAPTATION
100 #endif
101 }
102 
103 HttpReply *
105 {
107  return theVirginReply;
108 }
109 
110 const HttpReply *
112 {
114  return theVirginReply;
115 }
116 
117 HttpReply *
119 {
120  debugs(11,5, HERE << this << " setting virgin reply to " << rep);
122  assert(rep);
123  theVirginReply = rep;
125  return theVirginReply;
126 }
127 
128 HttpReply *
130 {
132  return theFinalReply;
133 }
134 
135 HttpReply *
137 {
138  debugs(11,5, HERE << this << " setting final reply to " << rep);
139 
141  assert(rep);
142  theFinalReply = rep;
144 
145  // give entry the reply because haveParsedReplyHeaders() expects it there
146  entry->replaceHttpReply(theFinalReply, false); // but do not write yet
147  haveParsedReplyHeaders(); // update the entry/reply (e.g., set timestamps)
149  entry->release();
150  entry->startWriting(); // write the updated entry to store
151 
152  return theFinalReply;
153 }
154 
155 // called when no more server communication is expected; may quit
156 void
158 {
159  debugs(11,5,HERE << "serverComplete " << this);
160 
161  if (!doneWithServer()) {
162  closeServer();
164  }
165 
166  completed = true;
168 
169  if (requestBodySource != NULL)
171 
172  if (responseBodyBuffer != NULL)
173  return;
174 
175  serverComplete2();
176 }
177 
178 void
180 {
181  debugs(11,5,HERE << "serverComplete2 " << this);
182 
183 #if USE_ADAPTATION
186 
187  if (!doneWithAdaptation())
188  return;
189 #endif
190 
192 }
193 
194 bool Client::doneAll() const
195 {
196  return doneWithServer() &&
197 #if USE_ADAPTATION
198  doneWithAdaptation() &&
201 #endif
203 }
204 
205 // FTP side overloads this to work around multiple calls to fwd->complete
206 void
208 {
209  debugs(11,5, HERE << "completing forwarding for " << fwd);
210  assert(fwd != NULL);
211  doneWithFwd = "completeForwarding()";
212  fwd->complete();
213 }
214 
215 // Register to receive request body
217 {
219  assert(r->body_pipe != NULL);
222  debugs(11,3, HERE << "expecting request body from " <<
224  return true;
225  }
226 
227  debugs(11,3, HERE << "aborting on partially consumed request body: " <<
230  return false;
231 }
232 
233 // Entry-dependent callbacks use this check to quit if the entry went bad
234 bool
235 Client::abortOnBadEntry(const char *abortReason)
236 {
237  if (entry->isAccepting())
238  return false;
239 
240  debugs(11,5, HERE << "entry is not Accepting!");
241  abortOnData(abortReason);
242  return true;
243 }
244 
245 // more request or adapted response body is available
246 void
248 {
249 #if USE_ADAPTATION
250  if (adaptedBodySource == bp) {
252  return;
253  }
254 #endif
255  if (requestBodySource == bp)
257 }
258 
259 // the entire request or adapted response body was provided, successfully
260 void
262 {
263 #if USE_ADAPTATION
264  if (adaptedBodySource == bp) {
266  return;
267  }
268 #endif
269  if (requestBodySource == bp)
271 }
272 
273 // premature end of the request or adapted response body production
274 void
276 {
277 #if USE_ADAPTATION
278  if (adaptedBodySource == bp) {
280  return;
281  }
282 #endif
283  if (requestBodySource == bp)
285 }
286 
287 bool
288 Client::abortOnData(const char *reason)
289 {
290  abortAll(reason);
291  return true;
292 }
293 
294 // more origin request body data is available
295 void
297 {
298  if (!requestSender)
300  else
301  debugs(9,3, HERE << "waiting for request body write to complete");
302 }
303 
304 // there will be no more handleMoreRequestBodyAvailable calls
305 void
307 {
309  if (!requestSender)
311  else
312  debugs(9,3, HERE << "waiting for request body write to complete");
313 }
314 
315 // called when we are done sending request body; kids extend this
316 void
318 {
319  debugs(9,3, HERE << "done sending request body");
322 
323  // kids extend this
324 }
325 
326 // called when body producers aborts; kids extend this
327 void
329 {
330  if (requestSender != NULL)
331  debugs(9,3, HERE << "fyi: request body aborted while we were sending");
332 
333  fwd->dontRetry(true); // the problem is not with the server
334  stopConsumingFrom(requestBodySource); // requestSender, if any, will notice
335 
336  // kids extend this
337 }
338 
339 // called when we wrote request headers(!) or a part of the body
340 void
342 {
343  debugs(11, 5, "sentRequestBody: FD " << io.fd << ": size " << io.size << ": errflag " << io.flag << ".");
344  debugs(32,3,HERE << "sentRequestBody called");
345 
347 
348  if (io.size > 0) {
349  fd_bytes(io.fd, io.size, FD_WRITE);
350  statCounter.server.all.kbytes_out += io.size;
351  // kids should increment their counters
352  }
353 
354  if (io.flag == Comm::ERR_CLOSING)
355  return;
356 
357  if (!requestBodySource) {
358  debugs(9,3, HERE << "detected while-we-were-sending abort");
359  return; // do nothing;
360  }
361 
362  // both successful and failed writes affect response times
364 
365  if (io.flag) {
366  debugs(11, DBG_IMPORTANT, "sentRequestBody error: FD " << io.fd << ": " << xstrerr(io.xerrno));
367  ErrorState *err;
369  err->xerrno = io.xerrno;
370  fwd->fail(err);
371  abortOnData("I/O error while sending request body");
372  return;
373  }
374 
376  abortOnData("store entry aborted while sending request body");
377  return;
378  }
379 
382  else if (receivedWholeRequestBody)
384  else
385  debugs(9,3, HERE << "waiting for body production end or abort");
386 }
387 
388 void
390 {
393 
395 
396  if (!Comm::IsConnOpen(conn)) {
397  debugs(9,3, HERE << "cannot send request body to closing " << conn);
398  return; // wait for the kid's close handler; TODO: assert(closer);
399  }
400 
401  MemBuf buf;
402  if (getMoreRequestBody(buf) && buf.contentSize() > 0) {
403  debugs(9,3, HERE << "will write " << buf.contentSize() << " request body bytes");
405  requestSender = JobCallback(93,3, Dialer, this, Client::sentRequestBody);
406  Comm::Write(conn, &buf, requestSender);
407  } else {
408  debugs(9,3, HERE << "will wait for more request body bytes or eof");
410  }
411 }
412 
414 bool
416 {
417  // default implementation does not encode request body content
419  return requestBodySource->getMoreData(buf);
420 }
421 
422 // Compares hosts in urls, returns false if different, no sheme, or no host.
423 static bool
424 sameUrlHosts(const char *url1, const char *url2)
425 {
426  // XXX: Want urlHostname() here, but it uses static storage and copying
427  const char *host1 = strchr(url1, ':');
428  const char *host2 = strchr(url2, ':');
429 
430  if (host1 && host2) {
431  // skip scheme slashes
432  do {
433  ++host1;
434  ++host2;
435  } while (*host1 == '/' && *host2 == '/');
436 
437  if (!*host1)
438  return false; // no host
439 
440  // increment while the same until we reach the end of the URL/host
441  while (*host1 && *host1 != '/' && *host1 == *host2) {
442  ++host1;
443  ++host2;
444  }
445  return *host1 == *host2;
446  }
447 
448  return false; // no URL scheme
449 }
450 
451 // purges entries that match the value of a given HTTP [response] header
452 static void
453 purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
454 {
455  const char *hdrUrl, *absUrl;
456 
457  absUrl = NULL;
458  hdrUrl = rep->header.getStr(hdr);
459  if (hdrUrl == NULL) {
460  return;
461  }
462 
463  /*
464  * If the URL is relative, make it absolute so we can find it.
465  * If it's absolute, make sure the host parts match to avoid DOS attacks
466  * as per RFC 2616 13.10.
467  */
468  if (urlIsRelative(hdrUrl)) {
469  absUrl = urlMakeAbsolute(req, hdrUrl);
470  if (absUrl != NULL) {
471  hdrUrl = absUrl;
472  }
473  } else if (!sameUrlHosts(reqUrl, hdrUrl)) {
474  return;
475  }
476 
477  purgeEntriesByUrl(req, hdrUrl);
478 
479  if (absUrl != NULL) {
480  safe_free(absUrl);
481  }
482 }
483 
484 // some HTTP methods should purge matching cache entries
485 void
487 {
488  // only some HTTP methods should purge matching cache entries
489  if (!request->method.purgesOthers())
490  return;
491 
492  // and probably only if the response was successful
493  if (theFinalReply->sline.status() >= 400)
494  return;
495 
496  // XXX: should we use originalRequest() here?
498  const char *reqUrl = tmp.c_str();
499  debugs(88, 5, "maybe purging due to " << request->method << ' ' << tmp);
500  purgeEntriesByUrl(request.getRaw(), reqUrl);
503 }
504 
506 void
508 {
511 
512  // adaptation may overwrite old offset computed using the virgin response
513  const bool partial = theFinalReply->contentRange();
514  currentOffset = partial ? theFinalReply->contentRange()->spec.offset : 0;
515 }
516 
518 bool
520 {
521  if (const Acl::Tree *acl = Config.accessList.storeMiss) {
522  // This relatively expensive check is not in StoreEntry::checkCachable:
523  // That method lacks HttpRequest and may be called too many times.
524  ACLFilledChecklist ch(acl, originalRequest().getRaw());
525  ch.reply = const_cast<HttpReply*>(entry->getReply()); // ACLFilledChecklist API bug
526  HTTPMSGLOCK(ch.reply);
527  if (!ch.fastCheck().allowed()) { // when in doubt, block
528  debugs(20, 3, "store_miss prohibits caching");
529  return true;
530  }
531  }
532  return false;
533 }
534 
537 {
538  return request;
539 }
540 
541 #if USE_ADAPTATION
542 void
545 {
546  debugs(11, 5, "Client::startAdaptation() called");
547  // check whether we should be sending a body as well
548  // start body pipe to feed ICAP transaction if needed
550  HttpReply *vrep = virginReply();
551  assert(!vrep->body_pipe);
552  int64_t size = 0;
553  if (vrep->expectingBody(cause->method, size) && size) {
554  virginBodyDestination = new BodyPipe(this);
556  debugs(93, 6, HERE << "will send virgin reply body to " <<
557  virginBodyDestination << "; size: " << size);
558  if (size > 0)
560  }
561 
563  new Adaptation::Iterator(vrep, cause, fwd->al, group));
566 }
567 
568 // properly cleans up ICAP-related state
569 // may be called multiple times
571 {
572  debugs(11,5, HERE << "cleaning ICAP; ACL: " << adaptationAccessCheckPending);
573 
576 
578 
579  if (adaptedBodySource != NULL)
581 
582  if (!adaptationAccessCheckPending) // we cannot cancel a pending callback
583  assert(doneWithAdaptation()); // make sure the two methods are in sync
584 }
585 
586 bool
588 {
591 }
592 
593 // sends virgin reply body to ICAP, buffering excesses if needed
594 void
595 Client::adaptVirginReplyBody(const char *data, ssize_t len)
596 {
598 
599  if (!virginBodyDestination) {
600  debugs(11,3, HERE << "ICAP does not want more virgin body");
601  return;
602  }
603 
604  // grow overflow area if already overflowed
605  if (responseBodyBuffer) {
606  responseBodyBuffer->append(data, len);
607  data = responseBodyBuffer->content();
609  }
610 
611  const ssize_t putSize = virginBodyDestination->putMoreData(data, len);
612  data += putSize;
613  len -= putSize;
614 
615  // if we had overflow area, shrink it as necessary
616  if (responseBodyBuffer) {
617  if (putSize == responseBodyBuffer->contentSize()) {
618  delete responseBodyBuffer;
620  } else {
621  responseBodyBuffer->consume(putSize);
622  }
623  return;
624  }
625 
626  // if we did not have an overflow area, create it as needed
627  if (len > 0) {
630  responseBodyBuffer->init(4096, SQUID_TCP_SO_RCVBUF * 10);
631  responseBodyBuffer->append(data, len);
632  }
633 }
634 
635 // can supply more virgin response body data
636 void
638 {
639  if (responseBodyBuffer) {
640  addVirginReplyBody(NULL, 0); // kick the buffered fragment alive again
641  if (completed && !responseBodyBuffer) {
642  serverComplete2();
643  return;
644  }
645  }
647 }
648 
649 // the consumer of our virgin response body aborted
650 void
652 {
654 
655  // do not force closeServer here in case we need to bypass AdaptationQueryAbort
656 
657  if (doneWithAdaptation()) // we may still be receiving adapted response
659 }
660 
661 // received adapted response headers (body may follow)
662 void
664 {
665  clearAdaptation(adaptedHeadSource); // we do not expect more messages
666 
667  switch (answer.kind) {
669  handleAdaptedHeader(const_cast<Http::Message*>(answer.message.getRaw()));
670  break;
671 
673  handleAdaptationBlocked(answer);
674  break;
675 
678  break;
679  }
680 }
681 
682 void
684 {
685  if (abortOnBadEntry("entry went bad while waiting for adapted headers")) {
686  // If the adapted response has a body, the ICAP side needs to know
687  // that nobody will consume that body. We will be destroyed upon
688  // return. Tell the ICAP side that it is on its own.
689  HttpReply *rep = dynamic_cast<HttpReply*>(msg);
690  assert(rep);
691  if (rep->body_pipe != NULL)
693 
694  return;
695  }
696 
697  HttpReply *rep = dynamic_cast<HttpReply*>(msg);
698  assert(rep);
699  debugs(11,5, HERE << this << " setting adapted reply to " << rep);
700  setFinalReply(rep);
701 
703  if (rep->body_pipe != NULL) {
704  // subscribe to receive adapted body
706  // assume that ICAP does not auto-consume on failures
707  const bool result = adaptedBodySource->setConsumerIfNotLate(this);
708  assert(result);
709  } else {
710  // no body
711  if (doneWithAdaptation()) // we may still be sending virgin response
713  }
714 }
715 
716 void
718 {
719  if (abortOnBadEntry("store entry aborted while kick producer callback"))
720  return;
721 
722  if (!adaptedBodySource)
723  return;
724 
726 
729 }
730 
731 // more adapted response body is available
732 void
734 {
735  if (abortOnBadEntry("entry refuses adapted body"))
736  return;
737 
738  assert(entry);
739 
740  size_t contentSize = adaptedBodySource->buf().contentSize();
741 
742  if (!contentSize)
743  return; // XXX: bytesWanted asserts on zero-size ranges
744 
745  const size_t spaceAvailable = entry->bytesWanted(Range<size_t>(0, contentSize), true);
746 
747  if (spaceAvailable < contentSize ) {
748  // No or partial body data consuming
749  typedef NullaryMemFunT<Client> Dialer;
750  AsyncCall::Pointer call = asyncCall(93, 5, "Client::resumeBodyStorage",
751  Dialer(this, &Client::resumeBodyStorage));
752  entry->deferProducer(call);
753  }
754 
755  if (!spaceAvailable) {
756  debugs(11, 5, HERE << "NOT storing " << contentSize << " bytes of adapted " <<
757  "response body at offset " << adaptedBodySource->consumedSize());
758  return;
759  }
760 
761  if (spaceAvailable < contentSize ) {
762  debugs(11, 5, HERE << "postponing storage of " <<
763  (contentSize - spaceAvailable) << " body bytes");
764  contentSize = spaceAvailable;
765  }
766 
767  debugs(11,5, HERE << "storing " << contentSize << " bytes of adapted " <<
768  "response body at offset " << adaptedBodySource->consumedSize());
769 
771  const StoreIOBuffer ioBuf(&bpc.buf, currentOffset, contentSize);
772  currentOffset += ioBuf.length;
773  entry->write(ioBuf);
774  bpc.buf.consume(contentSize);
775  bpc.checkIn();
776 }
777 
778 // the entire adapted response body was produced, successfully
779 void
781 {
782  if (abortOnBadEntry("entry went bad while waiting for adapted body eof"))
783  return;
784 
785  // end consumption if we consumed everything
788  // else resumeBodyStorage() will eventually consume the rest
789 }
790 
791 void
793 {
796 }
797 
798 // premature end of the adapted response body
800 {
801  if (abortOnBadEntry("entry went bad while waiting for the now-aborted adapted body"))
802  return;
803 
804  Must(adaptedBodySource != nullptr);
805  if (!adaptedBodySource->exhausted()) {
806  debugs(11,5, "waiting to consume the remainder of the aborted adapted body");
807  return; // resumeBodyStorage() should eventually consume the rest
808  }
809 
811 
813  return;
814 
815  entry->lengthWentBad("body adaptation aborted");
816  handleAdaptationCompleted(); // the user should get a truncated response
817 }
818 
819 // common part of noteAdaptationAnswer and handleAdaptedBodyProductionEnded
820 void
822 {
823  debugs(11,5, HERE << "handleAdaptationCompleted");
824  cleanAdaptation();
825 
826  // We stop reading origin response because we have no place to put it(*) and
827  // cannot use it. If some origin servers do not like that or if we want to
828  // reuse more pconns, we can add code to discard unneeded origin responses.
829  // (*) TODO: Is it possible that the adaptation xaction is still running?
830  if (mayReadVirginReplyBody()) {
831  debugs(11,3, HERE << "closing origin conn due to ICAP completion");
832  closeServer();
833  }
834 
836 }
837 
838 // common part of noteAdaptation*Aborted and noteBodyConsumerAborted methods
839 void
841 {
842  debugs(11,5, HERE << "handleAdaptationAborted; bypassable: " << bypassable <<
843  ", entry empty: " << entry->isEmpty());
844 
845  if (abortOnBadEntry("entry went bad while ICAP aborted"))
846  return;
847 
848  // TODO: bypass if possible
850  abortAll("adaptation failure with a filled entry");
851 }
852 
855 bool
857 {
858  if (entry->isEmpty()) {
859  debugs(11,8, "adaptation failure with an empty entry: " << *entry);
862  fwd->fail(err);
863  fwd->dontRetry(true);
864  abortAll("adaptation failure with an empty entry");
865  return true; // handled
866  }
867 
868  if (request) // update logged info directly
870 
871  return false; // the caller must handle
872 }
873 
874 // adaptation service wants us to deny HTTP client access to this response
875 void
877 {
878  debugs(11,5, HERE << answer.ruleId);
879 
880  if (abortOnBadEntry("entry went bad while ICAP aborted"))
881  return;
882 
883  if (!entry->isEmpty()) { // too late to block (should not really happen)
884  if (request)
886  abortAll("late adaptation block");
887  return;
888  }
889 
890  debugs(11,7, HERE << "creating adaptation block response");
891 
892  err_type page_id =
894  if (page_id == ERR_NONE)
895  page_id = ERR_ACCESS_DENIED;
896 
897  ErrorState *err = new ErrorState(page_id, Http::scForbidden, request.getRaw());
899  fwd->fail(err);
900  fwd->dontRetry(true);
901 
902  abortOnData("timely adaptation block");
903 }
904 
905 void
907 {
909 
910  if (abortOnBadEntry("entry went bad while waiting for ICAP ACL check"))
911  return;
912 
913  // TODO: Should non-ICAP and ICAP REPMOD pre-cache paths check this?
914  // That check now only happens on REQMOD pre-cache and REPMOD post-cache, in processReplyAccess().
915  if (virginReply()->expectedBodyTooLarge(*request)) {
917  return;
918  }
919  // TODO: Should we check receivedBodyTooLarge as well?
920 
921  if (!group) {
922  debugs(11,3, HERE << "no adapation needed");
925  return;
926  }
927 
928  startAdaptation(group, originalRequest().getRaw());
930 }
931 #endif
932 
933 void
935 {
937  fwd->fail(err);
938  fwd->dontRetry(true);
939  abortOnData("Virgin body too large.");
940 }
941 
942 // TODO: when HttpStateData sends all errors to ICAP,
943 // we should be able to move this at the end of setVirginReply().
944 void
946 {
947 #if USE_ADAPTATION
948  // TODO: merge with client side and return void to hide the on/off logic?
949  // The callback can be called with a NULL service if adaptation is off.
952  originalRequest().getRaw(), virginReply(), fwd->al, this);
953  debugs(11,5, HERE << "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
955  return;
956 #endif
957 
959 }
960 
962 void
963 Client::adjustBodyBytesRead(const int64_t delta)
964 {
965  int64_t &bodyBytesRead = originalRequest()->hier.bodyBytesRead;
966 
967  // if we got here, do not log a dash even if we got nothing from the server
968  if (bodyBytesRead < 0)
969  bodyBytesRead = 0;
970 
971  bodyBytesRead += delta; // supports negative and zero deltas
972 
973  // check for overflows ("infinite" response?) and undeflows (a bug)
974  Must(bodyBytesRead >= 0);
975 }
976 
977 void
978 Client::addVirginReplyBody(const char *data, ssize_t len)
979 {
980  adjustBodyBytesRead(len);
981 
982 #if USE_ADAPTATION
983  assert(!adaptationAccessCheckPending); // or would need to buffer while waiting
984  if (startedAdaptation) {
985  adaptVirginReplyBody(data, len);
986  return;
987  }
988 #endif
989  storeReplyBody(data, len);
990 }
991 
992 // writes virgin or adapted reply body to store
993 void
994 Client::storeReplyBody(const char *data, ssize_t len)
995 {
996  // write even if len is zero to push headers towards the client side
997  entry->write (StoreIOBuffer(len, currentOffset, (char*)data));
998 
999  currentOffset += len;
1000 }
1001 
1002 size_t
1003 Client::calcBufferSpaceToReserve(size_t space, const size_t wantSpace) const
1004 {
1005  if (space < wantSpace) {
1006  const size_t maxSpace = SBuf::maxSize; // absolute best
1007  space = min(wantSpace, maxSpace); // do not promise more than asked
1008  }
1009 
1010 #if USE_ADAPTATION
1011  if (responseBodyBuffer) {
1012  return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1013  }
1014 
1015  if (virginBodyDestination != NULL) {
1016  /*
1017  * BodyPipe buffer has a finite size limit. We
1018  * should not read more data from the network than will fit
1019  * into the pipe buffer or we _lose_ what did not fit if
1020  * the response ends sooner that BodyPipe frees up space:
1021  * There is no code to keep pumping data into the pipe once
1022  * response ends and serverComplete() is called.
1023  */
1024  const size_t adaptor_space = virginBodyDestination->buf().potentialSpaceSize();
1025 
1026  debugs(11,9, "Client may read up to min(" <<
1027  adaptor_space << ", " << space << ") bytes");
1028 
1029  if (adaptor_space < space)
1030  space = adaptor_space;
1031  }
1032 #endif
1033 
1034  return space;
1035 }
1036 
1037 size_t
1038 Client::replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
1039 {
1040  size_t space = readBuf.spaceSize(); // available space w/o heroic measures
1041  if (space < minSpace) {
1042  const size_t maxSpace = readBuf.potentialSpaceSize(); // absolute best
1043  space = min(minSpace, maxSpace); // do not promise more than asked
1044  }
1045 
1046 #if USE_ADAPTATION
1047  if (responseBodyBuffer) {
1048  return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1049  }
1050 
1051  if (virginBodyDestination != NULL) {
1052  /*
1053  * BodyPipe buffer has a finite size limit. We
1054  * should not read more data from the network than will fit
1055  * into the pipe buffer or we _lose_ what did not fit if
1056  * the response ends sooner that BodyPipe frees up space:
1057  * There is no code to keep pumping data into the pipe once
1058  * response ends and serverComplete() is called.
1059  *
1060  * If the pipe is totally full, don't register the read handler.
1061  * The BodyPipe will call our noteMoreBodySpaceAvailable() method
1062  * when it has free space again.
1063  */
1064  size_t adaptation_space =
1066 
1067  debugs(11,9, "Client may read up to min(" <<
1068  adaptation_space << ", " << space << ") bytes");
1069 
1070  if (adaptation_space < space)
1071  space = adaptation_space;
1072  }
1073 #endif
1074 
1075  return space;
1076 }
1077 
iterates services in ServiceGroup, starting adaptation launchers
Definition: Iterator.h:31
HierarchyLogEntry hier
Definition: HttpRequest.h:145
summarizes adaptation service answer for the noteAdaptationAnswer() API
Definition: Answer.h:22
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:96
struct StatCounters::@129 server
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:68
virtual bool isAccepting() const
Definition: store.cc:2068
CbcPointer< Initiate > initiateAdaptation(Initiate *x)
< starts freshly created initiate and returns a safe pointer to it
Definition: Initiator.cc:23
StatCounters statCounter
Definition: StatCounters.cc:12
#define assert(EX)
Definition: assert.h:17
mb_size_t spaceSize() const
Definition: MemBuf.cc:167
bool startedAdaptation
Definition: Client.h:178
Definition: Tree.h:20
void purgeEntriesByUrl(HttpRequest *req, const char *url)
MemBuf * responseBodyBuffer
Definition: Client.h:161
String ruleId
ACL (or similar rule) name that blocked forwarding.
Definition: Answer.h:40
void handleRequestBodyProductionEnded()
Definition: Client.cc:306
virtual const Comm::ConnectionPointer & dataConnection() const =0
void handleAdaptationBlocked(const Adaptation::Answer &answer)
Definition: Client.cc:876
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:225
virtual void release(const bool shareable=false)
Definition: store.cc:1250
int unlock(const char *context)
Definition: store.cc:485
const char * status() const
Definition: BodyPipe.cc:440
bool startRequestBodyFlow()
Definition: Client.cc:216
virtual bool abortOnData(const char *reason)
Definition: Client.cc:288
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1003
void adaptOrFinalizeReply()
Definition: Client.cc:945
mb_size_t potentialSpaceSize() const
Definition: MemBuf.cc:173
Definition: SBuf.h:87
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:415
virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer)
Definition: Client.cc:637
void consume(mb_size_t sz)
removes sz bytes and "packs" by moving content left
Definition: MemBuf.cc:180
void handleMoreRequestBodyAvailable()
Definition: Client.cc:296
const HttpReply * virginReply() const
Definition: Client.cc:111
HttpRequestMethod method
Definition: HttpRequest.h:102
void serverComplete()
Definition: Client.cc:157
const MemBuf & buf() const
Definition: BodyPipe.h:137
void handleMoreAdaptedBodyAvailable()
Definition: Client.cc:733
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
Http::MessagePointer message
HTTP request or response to forward.
Definition: Answer.h:39
bool initiated(const CbcPointer< AsyncJob > &job) const
Must(initiated(initiate)) instead of Must(initiate.set()), for clarity.
Definition: Initiator.h:52
StoreEntry * entry
Definition: Client.h:164
void storeReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:994
void announceInitiatorAbort(CbcPointer< Initiate > &x)
inform the transaction about abnormal termination and clear the pointer
Definition: Initiator.cc:38
#define safe_free(x)
Definition: xalloc.h:73
bool exhausted() const
Definition: BodyPipe.cc:174
virtual void noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer group)
Definition: Client.cc:906
Definition: Range.h:18
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:154
void handleAdaptedHeader(Http::Message *msg)
Definition: Client.cc:683
HttpRequest * request
Definition: FwdState.h:138
virtual bool doneWithServer() const =0
Definition: enums.h:24
virtual void noteAdaptationAnswer(const Adaptation::Answer &answer)
Definition: Client.cc:663
int conn
the current server connection FD
Definition: Transport.cc:26
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
virtual void processReplyBody()=0
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
virtual ~Client()
Definition: Client.cc:49
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb. ...
Definition: SBuf.h:96
HttpReply * theVirginReply
Definition: Client.h:190
size_t putMoreData(const char *buf, size_t size)
Definition: BodyPipe.cc:213
void replaceHttpReply(HttpReply *, bool andStartWriting=true)
Definition: store.cc:1854
no adapted message will come; see bypassable
Definition: Answer.h:29
uint64_t consumedSize() const
Definition: BodyPipe.h:111
void resumeBodyStorage()
called by StoreEntry when it has more buffer space available
Definition: Client.cc:717
void maybePurgeOthers()
Definition: Client.cc:486
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
int64_t bodyBytesRead
number of body bytes received from the next hop or -1
virtual void noteBodyProducerAborted(BodyPipe::Pointer)
Definition: Client.cc:275
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:105
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:261
const HttpHdrContRange * contentRange() const
Definition: HttpReply.cc:329
void sendMoreRequestBody()
Definition: Client.cc:389
const char * xstrerr(int error)
Definition: xstrerror.cc:83
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:24
AsyncCall::Pointer requestSender
Definition: Client.h:170
void sendBodyIsTooLargeError()
Definition: Client.cc:934
void const char HLPCB void * data
Definition: stub_helper.cc:16
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
Client(FwdState *)
Definition: Client.cc:40
void handleAdaptedBodyProducerAborted()
Definition: Client.cc:799
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const
Definition: HttpReply.cc:461
virtual void noteBodyProductionEnded(BodyPipe::Pointer)
Definition: Client.cc:261
virtual void abortAll(const char *reason)=0
abnormal transaction termination; reason is for debugging only
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
BodyPipe::Pointer virginBodyDestination
Definition: Client.h:173
#define DBG_IMPORTANT
Definition: Debug.h:45
virtual void noteBodyConsumerAborted(BodyPipe::Pointer)
Definition: Client.cc:651
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:507
bool urlIsRelative(const char *url)
Definition: url.cc:568
FwdState::Pointer fwd
Definition: Client.h:165
virtual void swanSong()
Definition: AsyncJob.h:55
void complete()
Definition: FwdState.cc:513
acl_access * storeMiss
Definition: SquidConfig.h:364
void serverComplete2()
Definition: Client.cc:179
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:328
bool handledEarlyAdaptationAbort()
Definition: Client.cc:856
void startAdaptation(const Adaptation::ServiceGroupPointer &group, HttpRequest *cause)
Initiate an asynchronous adaptation transaction which will call us back.
Definition: Client.cc:544
virtual HttpReply const * getReply() const
Definition: store.cc:1742
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:341
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
BodyPipe::Pointer requestBodySource
Definition: Client.h:169
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:536
char const * termedBuf() const
Definition: SquidString.h:90
virtual bool doneAll() const
whether positive goal has been reached
Definition: Client.cc:194
void handleUnregisteredServerEnd()
Definition: FwdState.cc:715
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:673
common parts of HttpRequest and HttpReply
Definition: Message.h:24
struct SquidConfig::@113 accessList
bool allowed() const
Definition: Acl.h:141
void adjustBodyBytesRead(const int64_t delta)
initializes bodyBytesRead stats if needed and applies delta
Definition: Client.cc:963
bool setConsumerIfNotLate(const Consumer::Pointer &aConsumer)
Definition: BodyPipe.cc:228
const char * c_str()
Definition: SBuf.cc:546
void cleanAdaptation()
Definition: Client.cc:570
uint16_t flags
Definition: Store.h:173
Http::StatusLine sline
Definition: HttpReply.h:60
virtual bool mayReadVirginReplyBody() const =0
whether we may receive more virgin response body bytes
virtual void completeForwarding()
Definition: Client.cc:207
bool blockCaching()
whether to prevent caching of an otherwise cachable response
Definition: Client.cc:519
virtual void closeServer()=0
virtual void maybeReadVirginBody()=0
read response data from the network
int64_t currentOffset
Definition: Client.h:160
void handleAdaptedBodyProductionEnded()
Definition: Client.cc:780
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:978
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
char * content()
start of the added data
Definition: MemBuf.h:41
void const char * buf
Definition: stub_helper.cc:16
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:147
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1080
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:295
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
char * urlMakeAbsolute(const HttpRequest *req, const char *relUrl)
Definition: url.cc:601
#define Must(cond)
Definition: TextException.h:89
HttpHeader header
Definition: Message.h:74
virtual void doneSendingRequestBody()=0
Definition: Client.cc:317
void stopPeerClock(const bool force)
Definition: access_log.cc:303
bool final
whether the error, if any, cannot be bypassed
Definition: Answer.h:41
AclDenyInfoList * denyInfoList
Definition: SquidConfig.h:403
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:139
struct StatCounters::@129::@138 all
allow_t const & fastCheck()
Definition: Checklist.cc:336
HttpRequestPointer request
Definition: Client.h:166
virtual void write(StoreIOBuffer)
Definition: store.cc:822
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
static bool sameUrlHosts(const char *url1, const char *url2)
Definition: Client.cc:424
HttpReply * theFinalReply
Definition: Client.h:191
HttpHdrRangeSpec spec
bool dontRetry()
Definition: FwdState.h:102
Kind kind
the type of the answer
Definition: Answer.h:42
void detailError(err_type aType, int aDetail)
sets error detail if no earlier detail was available
Definition: HttpRequest.cc:456
forward the supplied adapted HTTP message
Definition: Answer.h:27
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:118
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
HttpReply * setFinalReply(HttpReply *r)
Definition: Client.cc:136
block or deny the master xaction; see authority
Definition: Answer.h:28
static bool Start(Method method, VectPoint vp, HttpRequest *req, HttpReply *rep, AccessLogEntry::Pointer &al, Adaptation::Initiator *initiator)
Definition: AccessCheck.cc:30
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:180
Definition: MemBuf.h:23
void stopConsumingFrom(RefCount< BodyPipe > &)
Definition: BodyPipe.cc:118
void adaptVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:595
BodyPipe::Pointer adaptedBodySource
Definition: Client.h:175
CbcPointer< Adaptation::Initiate > adaptedHeadSource
Definition: Client.h:174
void fail(ErrorState *err)
Definition: FwdState.cc:458
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:97
bool adaptationAccessCheckPending
Definition: Client.h:177
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1203
void detailError(int dCode)
set error type-specific detail code
Definition: errorpage.h:96
virtual size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:226
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:350
int xerrno
Definition: errorpage.h:148
virtual void swanSong()
Definition: Client.cc:70
void startWriting()
Definition: store.cc:1870
void handleAdaptationCompleted()
Definition: Client.cc:821
size_t replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
Definition: Client.cc:1038
virtual bool isEmpty() const
Definition: Store.h:60
virtual void noteMoreBodyDataAvailable(BodyPipe::Pointer)
Definition: Client.cc:247
bool abortOnBadEntry(const char *abortReason)
Entry-dependent callbacks use this check to quit if the entry went bad.
Definition: Client.cc:235
HttpReply * finalReply()
Definition: Client.cc:129
static void purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
Definition: Client.cc:453
virtual bool doneWithAdaptation() const
Definition: Client.cc:587
StoreEntry * entry
Definition: FwdState.h:137
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
C * getRaw() const
Definition: RefCount.h:74
void endAdaptedBodyConsumption()
called when the entire adapted response body is consumed
Definition: Client.cc:792
const char * doneWithFwd
Definition: Client.h:184
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
bool purgesOthers() const
void handleAdaptationAborted(bool bypassable=false)
Definition: Client.cc:840
void HTTPMSGUNLOCK(Http::Message *a)
Definition: Message.h:144
int size
Definition: ModDevPoll.cc:77
bool completed
Definition: Client.h:86
A const & min(A const &lhs, A const &rhs)
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:35
err_type
Definition: err_type.h:12
err_type aclGetDenyInfoPage(AclDenyInfoList **head, const char *name, int redirect_allowed)
Definition: Gadgets.cc:40
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
void clearAdaptation(CbcPointer< Initiate > &x)
clears the pointer (does not call announceInitiatorAbort)
Definition: Initiator.cc:32
void lock(const char *context)
Definition: store.cc:448

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors