Client.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #include "squid.h"
10 #include "acl/FilledChecklist.h"
11 #include "acl/Gadgets.h"
12 #include "base/TextException.h"
13 #include "clients/Client.h"
14 #include "comm/Connection.h"
15 #include "comm/forward.h"
16 #include "comm/Write.h"
17 #include "err_detail_type.h"
18 #include "errorpage.h"
19 #include "fd.h"
20 #include "HttpHdrContRange.h"
21 #include "HttpReply.h"
22 #include "HttpRequest.h"
23 #include "SquidConfig.h"
24 #include "SquidTime.h"
25 #include "StatCounters.h"
26 #include "Store.h"
27 #include "tools.h"
28 
29 #if USE_ADAPTATION
30 #include "adaptation/AccessCheck.h"
31 #include "adaptation/Answer.h"
32 #include "adaptation/Iterator.h"
33 #include "base/AsyncCall.h"
34 #endif
35 
36 // implemented in client_side_reply.cc until sides have a common parent
37 void purgeEntriesByUrl(HttpRequest * req, const char *url);
38 
39 Client::Client(FwdState *theFwdState) :
40  AsyncJob("Client"),
41  fwd(theFwdState),
42  request(fwd->request)
43 {
44  entry = fwd->entry;
45  entry->lock("Client");
46 }
47 
49 {
50  // paranoid: check that swanSong has been called
52 #if USE_ADAPTATION
55 #endif
56 
57  entry->unlock("Client");
58 
61 
62  if (responseBodyBuffer != NULL) {
63  delete responseBodyBuffer;
65  }
66 }
67 
68 void
70 {
71  // get rid of our piping obligations
72  if (requestBodySource != NULL)
74 
75 #if USE_ADAPTATION
77 #endif
78 
79  if (!doneWithServer())
80  closeServer();
81 
82  if (!doneWithFwd) {
83  doneWithFwd = "swanSong()";
85  }
86 
88 #if USE_ADAPTATION
89  Initiator::swanSong();
91 #endif
92 
93  // paranoid: check that swanSong has been called
94  // extra paranoid: yeah, I really mean it. they MUST pass here.
96 #if USE_ADAPTATION
99 #endif
100 }
101 
102 HttpReply *
104 {
106  return theVirginReply;
107 }
108 
109 const HttpReply *
111 {
113  return theVirginReply;
114 }
115 
116 HttpReply *
118 {
119  debugs(11,5, HERE << this << " setting virgin reply to " << rep);
121  assert(rep);
122  theVirginReply = rep;
124  return theVirginReply;
125 }
126 
127 HttpReply *
129 {
131  return theFinalReply;
132 }
133 
134 HttpReply *
136 {
137  debugs(11,5, HERE << this << " setting final reply to " << rep);
138 
140  assert(rep);
141  theFinalReply = rep;
143 
144  // give entry the reply because haveParsedReplyHeaders() expects it there
145  entry->replaceHttpReply(theFinalReply, false); // but do not write yet
146  haveParsedReplyHeaders(); // update the entry/reply (e.g., set timestamps)
148  entry->release();
149  entry->startWriting(); // write the updated entry to store
150 
151  return theFinalReply;
152 }
153 
154 // called when no more server communication is expected; may quit
155 void
157 {
158  debugs(11,5,HERE << "serverComplete " << this);
159 
160  if (!doneWithServer()) {
161  closeServer();
163  }
164 
165  completed = true;
167 
168  if (requestBodySource != NULL)
170 
171  if (responseBodyBuffer != NULL)
172  return;
173 
174  serverComplete2();
175 }
176 
177 void
179 {
180  debugs(11,5,HERE << "serverComplete2 " << this);
181 
182 #if USE_ADAPTATION
185 
186  if (!doneWithAdaptation())
187  return;
188 #endif
189 
191 }
192 
193 bool Client::doneAll() const
194 {
195  return doneWithServer() &&
196 #if USE_ADAPTATION
197  doneWithAdaptation() &&
200 #endif
202 }
203 
204 // FTP side overloads this to work around multiple calls to fwd->complete
205 void
207 {
208  debugs(11,5, HERE << "completing forwarding for " << fwd);
209  assert(fwd != NULL);
210  doneWithFwd = "completeForwarding()";
211  fwd->complete();
212 }
213 
214 // Register to receive request body
216 {
218  assert(r->body_pipe != NULL);
221  debugs(11,3, HERE << "expecting request body from " <<
223  return true;
224  }
225 
226  debugs(11,3, HERE << "aborting on partially consumed request body: " <<
229  return false;
230 }
231 
232 // Entry-dependent callbacks use this check to quit if the entry went bad
233 bool
234 Client::abortOnBadEntry(const char *abortReason)
235 {
236  if (entry->isAccepting())
237  return false;
238 
239  debugs(11,5, HERE << "entry is not Accepting!");
240  abortOnData(abortReason);
241  return true;
242 }
243 
244 // more request or adapted response body is available
245 void
247 {
248 #if USE_ADAPTATION
249  if (adaptedBodySource == bp) {
251  return;
252  }
253 #endif
254  if (requestBodySource == bp)
256 }
257 
258 // the entire request or adapted response body was provided, successfully
259 void
261 {
262 #if USE_ADAPTATION
263  if (adaptedBodySource == bp) {
265  return;
266  }
267 #endif
268  if (requestBodySource == bp)
270 }
271 
272 // premature end of the request or adapted response body production
273 void
275 {
276 #if USE_ADAPTATION
277  if (adaptedBodySource == bp) {
279  return;
280  }
281 #endif
282  if (requestBodySource == bp)
284 }
285 
286 bool
287 Client::abortOnData(const char *reason)
288 {
289  abortAll(reason);
290  return true;
291 }
292 
293 // more origin request body data is available
294 void
296 {
297  if (!requestSender)
299  else
300  debugs(9,3, HERE << "waiting for request body write to complete");
301 }
302 
303 // there will be no more handleMoreRequestBodyAvailable calls
304 void
306 {
308  if (!requestSender)
310  else
311  debugs(9,3, HERE << "waiting for request body write to complete");
312 }
313 
314 // called when we are done sending request body; kids extend this
315 void
317 {
318  debugs(9,3, HERE << "done sending request body");
321 
322  // kids extend this
323 }
324 
325 // called when body producers aborts; kids extend this
326 void
328 {
329  if (requestSender != NULL)
330  debugs(9,3, HERE << "fyi: request body aborted while we were sending");
331 
332  fwd->dontRetry(true); // the problem is not with the server
333  stopConsumingFrom(requestBodySource); // requestSender, if any, will notice
334 
335  // kids extend this
336 }
337 
338 // called when we wrote request headers(!) or a part of the body
339 void
341 {
342  debugs(11, 5, "sentRequestBody: FD " << io.fd << ": size " << io.size << ": errflag " << io.flag << ".");
343  debugs(32,3,HERE << "sentRequestBody called");
344 
346 
347  if (io.size > 0) {
348  fd_bytes(io.fd, io.size, FD_WRITE);
349  statCounter.server.all.kbytes_out += io.size;
350  // kids should increment their counters
351  }
352 
353  if (io.flag == Comm::ERR_CLOSING)
354  return;
355 
356  if (!requestBodySource) {
357  debugs(9,3, HERE << "detected while-we-were-sending abort");
358  return; // do nothing;
359  }
360 
361  // both successful and failed writes affect response times
363 
364  if (io.flag) {
365  debugs(11, DBG_IMPORTANT, "sentRequestBody error: FD " << io.fd << ": " << xstrerr(io.xerrno));
366  ErrorState *err;
368  err->xerrno = io.xerrno;
369  fwd->fail(err);
370  abortOnData("I/O error while sending request body");
371  return;
372  }
373 
375  abortOnData("store entry aborted while sending request body");
376  return;
377  }
378 
381  else if (receivedWholeRequestBody)
383  else
384  debugs(9,3, HERE << "waiting for body production end or abort");
385 }
386 
387 void
389 {
392 
394 
395  if (!Comm::IsConnOpen(conn)) {
396  debugs(9,3, HERE << "cannot send request body to closing " << conn);
397  return; // wait for the kid's close handler; TODO: assert(closer);
398  }
399 
400  MemBuf buf;
401  if (getMoreRequestBody(buf) && buf.contentSize() > 0) {
402  debugs(9,3, HERE << "will write " << buf.contentSize() << " request body bytes");
404  requestSender = JobCallback(93,3, Dialer, this, Client::sentRequestBody);
405  Comm::Write(conn, &buf, requestSender);
406  } else {
407  debugs(9,3, HERE << "will wait for more request body bytes or eof");
409  }
410 }
411 
413 bool
415 {
416  // default implementation does not encode request body content
418  return requestBodySource->getMoreData(buf);
419 }
420 
421 // Compares hosts in urls, returns false if different, no sheme, or no host.
422 static bool
423 sameUrlHosts(const char *url1, const char *url2)
424 {
425  // XXX: Want urlHostname() here, but it uses static storage and copying
426  const char *host1 = strchr(url1, ':');
427  const char *host2 = strchr(url2, ':');
428 
429  if (host1 && host2) {
430  // skip scheme slashes
431  do {
432  ++host1;
433  ++host2;
434  } while (*host1 == '/' && *host2 == '/');
435 
436  if (!*host1)
437  return false; // no host
438 
439  // increment while the same until we reach the end of the URL/host
440  while (*host1 && *host1 != '/' && *host1 == *host2) {
441  ++host1;
442  ++host2;
443  }
444  return *host1 == *host2;
445  }
446 
447  return false; // no URL scheme
448 }
449 
450 // purges entries that match the value of a given HTTP [response] header
451 static void
452 purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
453 {
454  const char *hdrUrl, *absUrl;
455 
456  absUrl = NULL;
457  hdrUrl = rep->header.getStr(hdr);
458  if (hdrUrl == NULL) {
459  return;
460  }
461 
462  /*
463  * If the URL is relative, make it absolute so we can find it.
464  * If it's absolute, make sure the host parts match to avoid DOS attacks
465  * as per RFC 2616 13.10.
466  */
467  if (urlIsRelative(hdrUrl)) {
468  absUrl = urlMakeAbsolute(req, hdrUrl);
469  if (absUrl != NULL) {
470  hdrUrl = absUrl;
471  }
472  } else if (!sameUrlHosts(reqUrl, hdrUrl)) {
473  return;
474  }
475 
476  purgeEntriesByUrl(req, hdrUrl);
477 
478  if (absUrl != NULL) {
479  safe_free(absUrl);
480  }
481 }
482 
483 // some HTTP methods should purge matching cache entries
484 void
486 {
487  // only some HTTP methods should purge matching cache entries
488  if (!request->method.purgesOthers())
489  return;
490 
491  // and probably only if the response was successful
492  if (theFinalReply->sline.status() >= 400)
493  return;
494 
495  // XXX: should we use originalRequest() here?
497  const char *reqUrl = tmp.c_str();
498  debugs(88, 5, "maybe purging due to " << request->method << ' ' << tmp);
499  purgeEntriesByUrl(request.getRaw(), reqUrl);
502 }
503 
505 void
507 {
510 
511  // adaptation may overwrite old offset computed using the virgin response
512  const bool partial = theFinalReply->contentRange();
513  currentOffset = partial ? theFinalReply->contentRange()->spec.offset : 0;
514 }
515 
517 bool
519 {
520  if (const Acl::Tree *acl = Config.accessList.storeMiss) {
521  // This relatively expensive check is not in StoreEntry::checkCachable:
522  // That method lacks HttpRequest and may be called too many times.
523  ACLFilledChecklist ch(acl, originalRequest().getRaw());
524  ch.reply = const_cast<HttpReply*>(entry->getReply()); // ACLFilledChecklist API bug
525  HTTPMSGLOCK(ch.reply);
526  if (!ch.fastCheck().allowed()) { // when in doubt, block
527  debugs(20, 3, "store_miss prohibits caching");
528  return true;
529  }
530  }
531  return false;
532 }
533 
536 {
537  return request;
538 }
539 
540 #if USE_ADAPTATION
541 void
544 {
545  debugs(11, 5, "Client::startAdaptation() called");
546  // check whether we should be sending a body as well
547  // start body pipe to feed ICAP transaction if needed
549  HttpReply *vrep = virginReply();
550  assert(!vrep->body_pipe);
551  int64_t size = 0;
552  if (vrep->expectingBody(cause->method, size) && size) {
553  virginBodyDestination = new BodyPipe(this);
555  debugs(93, 6, HERE << "will send virgin reply body to " <<
556  virginBodyDestination << "; size: " << size);
557  if (size > 0)
559  }
560 
562  new Adaptation::Iterator(vrep, cause, fwd->al, group));
565 }
566 
567 // properly cleans up ICAP-related state
568 // may be called multiple times
570 {
571  debugs(11,5, HERE << "cleaning ICAP; ACL: " << adaptationAccessCheckPending);
572 
575 
577 
578  if (adaptedBodySource != NULL)
580 
581  if (!adaptationAccessCheckPending) // we cannot cancel a pending callback
582  assert(doneWithAdaptation()); // make sure the two methods are in sync
583 }
584 
585 bool
587 {
590 }
591 
592 // sends virgin reply body to ICAP, buffering excesses if needed
593 void
594 Client::adaptVirginReplyBody(const char *data, ssize_t len)
595 {
597 
598  if (!virginBodyDestination) {
599  debugs(11,3, HERE << "ICAP does not want more virgin body");
600  return;
601  }
602 
603  // grow overflow area if already overflowed
604  if (responseBodyBuffer) {
605  responseBodyBuffer->append(data, len);
606  data = responseBodyBuffer->content();
608  }
609 
610  const ssize_t putSize = virginBodyDestination->putMoreData(data, len);
611  data += putSize;
612  len -= putSize;
613 
614  // if we had overflow area, shrink it as necessary
615  if (responseBodyBuffer) {
616  if (putSize == responseBodyBuffer->contentSize()) {
617  delete responseBodyBuffer;
619  } else {
620  responseBodyBuffer->consume(putSize);
621  }
622  return;
623  }
624 
625  // if we did not have an overflow area, create it as needed
626  if (len > 0) {
629  responseBodyBuffer->init(4096, SQUID_TCP_SO_RCVBUF * 10);
630  responseBodyBuffer->append(data, len);
631  }
632 }
633 
634 // can supply more virgin response body data
635 void
637 {
638  if (responseBodyBuffer) {
639  addVirginReplyBody(NULL, 0); // kick the buffered fragment alive again
640  if (completed && !responseBodyBuffer) {
641  serverComplete2();
642  return;
643  }
644  }
646 }
647 
648 // the consumer of our virgin response body aborted
649 void
651 {
653 
654  // do not force closeServer here in case we need to bypass AdaptationQueryAbort
655 
656  if (doneWithAdaptation()) // we may still be receiving adapted response
658 }
659 
660 // received adapted response headers (body may follow)
661 void
663 {
664  clearAdaptation(adaptedHeadSource); // we do not expect more messages
665 
666  switch (answer.kind) {
668  handleAdaptedHeader(const_cast<Http::Message*>(answer.message.getRaw()));
669  break;
670 
672  handleAdaptationBlocked(answer);
673  break;
674 
677  break;
678  }
679 }
680 
681 void
683 {
684  if (abortOnBadEntry("entry went bad while waiting for adapted headers")) {
685  // If the adapted response has a body, the ICAP side needs to know
686  // that nobody will consume that body. We will be destroyed upon
687  // return. Tell the ICAP side that it is on its own.
688  HttpReply *rep = dynamic_cast<HttpReply*>(msg);
689  assert(rep);
690  if (rep->body_pipe != NULL)
692 
693  return;
694  }
695 
696  HttpReply *rep = dynamic_cast<HttpReply*>(msg);
697  assert(rep);
698  debugs(11,5, HERE << this << " setting adapted reply to " << rep);
699  setFinalReply(rep);
700 
702  if (rep->body_pipe != NULL) {
703  // subscribe to receive adapted body
705  // assume that ICAP does not auto-consume on failures
706  const bool result = adaptedBodySource->setConsumerIfNotLate(this);
707  assert(result);
708  } else {
709  // no body
710  if (doneWithAdaptation()) // we may still be sending virgin response
712  }
713 }
714 
715 void
717 {
718  if (abortOnBadEntry("store entry aborted while kick producer callback"))
719  return;
720 
721  if (!adaptedBodySource)
722  return;
723 
725 
728 }
729 
730 // more adapted response body is available
731 void
733 {
734  if (abortOnBadEntry("entry refuses adapted body"))
735  return;
736 
737  assert(entry);
738 
739  size_t contentSize = adaptedBodySource->buf().contentSize();
740 
741  if (!contentSize)
742  return; // XXX: bytesWanted asserts on zero-size ranges
743 
744  const size_t spaceAvailable = entry->bytesWanted(Range<size_t>(0, contentSize), true);
745 
746  if (spaceAvailable < contentSize ) {
747  // No or partial body data consuming
748  typedef NullaryMemFunT<Client> Dialer;
749  AsyncCall::Pointer call = asyncCall(93, 5, "Client::resumeBodyStorage",
750  Dialer(this, &Client::resumeBodyStorage));
751  entry->deferProducer(call);
752  }
753 
754  if (!spaceAvailable) {
755  debugs(11, 5, HERE << "NOT storing " << contentSize << " bytes of adapted " <<
756  "response body at offset " << adaptedBodySource->consumedSize());
757  return;
758  }
759 
760  if (spaceAvailable < contentSize ) {
761  debugs(11, 5, HERE << "postponing storage of " <<
762  (contentSize - spaceAvailable) << " body bytes");
763  contentSize = spaceAvailable;
764  }
765 
766  debugs(11,5, HERE << "storing " << contentSize << " bytes of adapted " <<
767  "response body at offset " << adaptedBodySource->consumedSize());
768 
770  const StoreIOBuffer ioBuf(&bpc.buf, currentOffset, contentSize);
771  currentOffset += ioBuf.length;
772  entry->write(ioBuf);
773  bpc.buf.consume(contentSize);
774  bpc.checkIn();
775 }
776 
777 // the entire adapted response body was produced, successfully
778 void
780 {
781  if (abortOnBadEntry("entry went bad while waiting for adapted body eof"))
782  return;
783 
784  // end consumption if we consumed everything
787  // else resumeBodyStorage() will eventually consume the rest
788 }
789 
790 void
792 {
795 }
796 
797 // premature end of the adapted response body
799 {
800  if (abortOnBadEntry("entry went bad while waiting for the now-aborted adapted body"))
801  return;
802 
803  Must(adaptedBodySource != nullptr);
804  if (!adaptedBodySource->exhausted()) {
805  debugs(11,5, "waiting to consume the remainder of the aborted adapted body");
806  return; // resumeBodyStorage() should eventually consume the rest
807  }
808 
810 
812  return;
813 
814  entry->lengthWentBad("body adaptation aborted");
815  handleAdaptationCompleted(); // the user should get a truncated response
816 }
817 
818 // common part of noteAdaptationAnswer and handleAdaptedBodyProductionEnded
819 void
821 {
822  debugs(11,5, HERE << "handleAdaptationCompleted");
823  cleanAdaptation();
824 
825  // We stop reading origin response because we have no place to put it(*) and
826  // cannot use it. If some origin servers do not like that or if we want to
827  // reuse more pconns, we can add code to discard unneeded origin responses.
828  // (*) TODO: Is it possible that the adaptation xaction is still running?
829  if (mayReadVirginReplyBody()) {
830  debugs(11,3, HERE << "closing origin conn due to ICAP completion");
831  closeServer();
832  }
833 
835 }
836 
837 // common part of noteAdaptation*Aborted and noteBodyConsumerAborted methods
838 void
840 {
841  debugs(11,5, HERE << "handleAdaptationAborted; bypassable: " << bypassable <<
842  ", entry empty: " << entry->isEmpty());
843 
844  if (abortOnBadEntry("entry went bad while ICAP aborted"))
845  return;
846 
847  // TODO: bypass if possible
849  abortAll("adaptation failure with a filled entry");
850 }
851 
854 bool
856 {
857  if (entry->isEmpty()) {
858  debugs(11,8, "adaptation failure with an empty entry: " << *entry);
861  fwd->fail(err);
862  fwd->dontRetry(true);
863  abortAll("adaptation failure with an empty entry");
864  return true; // handled
865  }
866 
867  if (request) // update logged info directly
869 
870  return false; // the caller must handle
871 }
872 
873 // adaptation service wants us to deny HTTP client access to this response
874 void
876 {
877  debugs(11,5, HERE << answer.ruleId);
878 
879  if (abortOnBadEntry("entry went bad while ICAP aborted"))
880  return;
881 
882  if (!entry->isEmpty()) { // too late to block (should not really happen)
883  if (request)
885  abortAll("late adaptation block");
886  return;
887  }
888 
889  debugs(11,7, HERE << "creating adaptation block response");
890 
891  err_type page_id =
893  if (page_id == ERR_NONE)
894  page_id = ERR_ACCESS_DENIED;
895 
896  ErrorState *err = new ErrorState(page_id, Http::scForbidden, request.getRaw());
898  fwd->fail(err);
899  fwd->dontRetry(true);
900 
901  abortOnData("timely adaptation block");
902 }
903 
904 void
906 {
908 
909  if (abortOnBadEntry("entry went bad while waiting for ICAP ACL check"))
910  return;
911 
912  // TODO: Should non-ICAP and ICAP REPMOD pre-cache paths check this?
913  // That check now only happens on REQMOD pre-cache and REPMOD post-cache, in processReplyAccess().
914  if (virginReply()->expectedBodyTooLarge(*request)) {
916  return;
917  }
918  // TODO: Should we check receivedBodyTooLarge as well?
919 
920  if (!group) {
921  debugs(11,3, HERE << "no adapation needed");
924  return;
925  }
926 
927  startAdaptation(group, originalRequest().getRaw());
929 }
930 #endif
931 
932 void
934 {
936  fwd->fail(err);
937  fwd->dontRetry(true);
938  abortOnData("Virgin body too large.");
939 }
940 
941 // TODO: when HttpStateData sends all errors to ICAP,
942 // we should be able to move this at the end of setVirginReply().
943 void
945 {
946 #if USE_ADAPTATION
947  // TODO: merge with client side and return void to hide the on/off logic?
948  // The callback can be called with a NULL service if adaptation is off.
951  originalRequest().getRaw(), virginReply(), fwd->al, this);
952  debugs(11,5, HERE << "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
954  return;
955 #endif
956 
958 }
959 
961 void
962 Client::adjustBodyBytesRead(const int64_t delta)
963 {
964  int64_t &bodyBytesRead = originalRequest()->hier.bodyBytesRead;
965 
966  // if we got here, do not log a dash even if we got nothing from the server
967  if (bodyBytesRead < 0)
968  bodyBytesRead = 0;
969 
970  bodyBytesRead += delta; // supports negative and zero deltas
971 
972  // check for overflows ("infinite" response?) and undeflows (a bug)
973  Must(bodyBytesRead >= 0);
974 }
975 
976 void
977 Client::addVirginReplyBody(const char *data, ssize_t len)
978 {
979  adjustBodyBytesRead(len);
980 
981 #if USE_ADAPTATION
982  assert(!adaptationAccessCheckPending); // or would need to buffer while waiting
983  if (startedAdaptation) {
984  adaptVirginReplyBody(data, len);
985  return;
986  }
987 #endif
988  storeReplyBody(data, len);
989 }
990 
991 // writes virgin or adapted reply body to store
992 void
993 Client::storeReplyBody(const char *data, ssize_t len)
994 {
995  // write even if len is zero to push headers towards the client side
996  entry->write (StoreIOBuffer(len, currentOffset, (char*)data));
997 
998  currentOffset += len;
999 }
1000 
1001 size_t
1002 Client::calcBufferSpaceToReserve(size_t space, const size_t wantSpace) const
1003 {
1004  if (space < wantSpace) {
1005  const size_t maxSpace = SBuf::maxSize; // absolute best
1006  space = min(wantSpace, maxSpace); // do not promise more than asked
1007  }
1008 
1009 #if USE_ADAPTATION
1010  if (responseBodyBuffer) {
1011  return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1012  }
1013 
1014  if (virginBodyDestination != NULL) {
1015  /*
1016  * BodyPipe buffer has a finite size limit. We
1017  * should not read more data from the network than will fit
1018  * into the pipe buffer or we _lose_ what did not fit if
1019  * the response ends sooner that BodyPipe frees up space:
1020  * There is no code to keep pumping data into the pipe once
1021  * response ends and serverComplete() is called.
1022  */
1023  const size_t adaptor_space = virginBodyDestination->buf().potentialSpaceSize();
1024 
1025  debugs(11,9, "Client may read up to min(" <<
1026  adaptor_space << ", " << space << ") bytes");
1027 
1028  if (adaptor_space < space)
1029  space = adaptor_space;
1030  }
1031 #endif
1032 
1033  return space;
1034 }
1035 
1036 size_t
1037 Client::replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
1038 {
1039  size_t space = readBuf.spaceSize(); // available space w/o heroic measures
1040  if (space < minSpace) {
1041  const size_t maxSpace = readBuf.potentialSpaceSize(); // absolute best
1042  space = min(minSpace, maxSpace); // do not promise more than asked
1043  }
1044 
1045 #if USE_ADAPTATION
1046  if (responseBodyBuffer) {
1047  return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1048  }
1049 
1050  if (virginBodyDestination != NULL) {
1051  /*
1052  * BodyPipe buffer has a finite size limit. We
1053  * should not read more data from the network than will fit
1054  * into the pipe buffer or we _lose_ what did not fit if
1055  * the response ends sooner that BodyPipe frees up space:
1056  * There is no code to keep pumping data into the pipe once
1057  * response ends and serverComplete() is called.
1058  *
1059  * If the pipe is totally full, don't register the read handler.
1060  * The BodyPipe will call our noteMoreBodySpaceAvailable() method
1061  * when it has free space again.
1062  */
1063  size_t adaptation_space =
1065 
1066  debugs(11,9, "Client may read up to min(" <<
1067  adaptation_space << ", " << space << ") bytes");
1068 
1069  if (adaptation_space < space)
1070  space = adaptation_space;
1071  }
1072 #endif
1073 
1074  return space;
1075 }
1076 
iterates services in ServiceGroup, starting adaptation launchers
Definition: Iterator.h:31
HierarchyLogEntry hier
Definition: HttpRequest.h:149
summarizes adaptation service answer for the noteAdaptationAnswer() API
Definition: Answer.h:22
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:96
struct StatCounters::@129 server
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:68
bool isAccepting() const
Definition: store.cc:2047
CbcPointer< Initiate > initiateAdaptation(Initiate *x)
&lt; starts freshly created initiate and returns a safe pointer to it
Definition: Initiator.cc:23
StatCounters statCounter
Definition: StatCounters.cc:12
#define assert(EX)
Definition: assert.h:17
mb_size_t spaceSize() const
Definition: MemBuf.cc:158
bool startedAdaptation
Definition: Client.h:178
Definition: Tree.h:20
MemBuf * responseBodyBuffer
Definition: Client.h:161
String ruleId
ACL (or similar rule) name that blocked forwarding.
Definition: Answer.h:40
void handleRequestBodyProductionEnded()
Definition: Client.cc:305
virtual const Comm::ConnectionPointer & dataConnection() const =0
void handleAdaptationBlocked(const Adaptation::Answer &answer)
Definition: Client.cc:875
char * urlMakeAbsolute(const HttpRequest *req, const char *relUrl)
Definition: Uri.cc:601
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:216
void release(const bool shareable=false)
Definition: store.cc:1189
int unlock(const char *context)
Definition: store.cc:457
const char * status() const
Definition: BodyPipe.cc:440
bool startRequestBodyFlow()
Definition: Client.cc:215
virtual bool abortOnData(const char *reason)
Definition: Client.cc:287
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1002
void adaptOrFinalizeReply()
Definition: Client.cc:944
bool urlIsRelative(const char *url)
Definition: Uri.cc:568
mb_size_t potentialSpaceSize() const
Definition: MemBuf.cc:164
Definition: SBuf.h:86
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:414
virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer)
Definition: Client.cc:636
void consume(mb_size_t sz)
removes sz bytes and &quot;packs&quot; by moving content left
Definition: MemBuf.cc:171
void handleMoreRequestBodyAvailable()
Definition: Client.cc:295
const HttpReply * virginReply() const
Definition: Client.cc:110
HttpRequestMethod method
Definition: HttpRequest.h:106
void serverComplete()
Definition: Client.cc:156
const MemBuf & buf() const
Definition: BodyPipe.h:137
void handleMoreAdaptedBodyAvailable()
Definition: Client.cc:732
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
Http::MessagePointer message
HTTP request or response to forward.
Definition: Answer.h:39
bool initiated(const CbcPointer< AsyncJob > &job) const
Must(initiated(initiate)) instead of Must(initiate.set()), for clarity.
Definition: Initiator.h:52
StoreEntry * entry
Definition: Client.h:164
void storeReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:993
void announceInitiatorAbort(CbcPointer< Initiate > &x)
inform the transaction about abnormal termination and clear the pointer
Definition: Initiator.cc:38
#define safe_free(x)
Definition: xalloc.h:73
bool exhausted() const
Definition: BodyPipe.cc:174
virtual void noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer group)
Definition: Client.cc:905
Definition: Range.h:18
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:160
void handleAdaptedHeader(Http::Message *msg)
Definition: Client.cc:682
HttpRequest * request
Definition: FwdState.h:138
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
virtual bool doneWithServer() const =0
Definition: enums.h:24
virtual void noteAdaptationAnswer(const Adaptation::Answer &answer)
Definition: Client.cc:662
int conn
the current server connection FD
Definition: Transport.cc:26
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
virtual void processReplyBody()=0
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
virtual ~Client()
Definition: Client.cc:48
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be &lt; MAX(size_type)/2. Currently 256Mb. ...
Definition: SBuf.h:95
HttpReply * theVirginReply
Definition: Client.h:190
size_t putMoreData(const char *buf, size_t size)
Definition: BodyPipe.cc:213
void replaceHttpReply(HttpReply *, bool andStartWriting=true)
Definition: store.cc:1782
no adapted message will come; see bypassable
Definition: Answer.h:29
uint64_t consumedSize() const
Definition: BodyPipe.h:111
void resumeBodyStorage()
called by StoreEntry when it has more buffer space available
Definition: Client.cc:716
void maybePurgeOthers()
Definition: Client.cc:485
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
int64_t bodyBytesRead
number of body bytes received from the next hop or -1
virtual void noteBodyProducerAborted(BodyPipe::Pointer)
Definition: Client.cc:274
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:260
const HttpHdrContRange * contentRange() const
Definition: HttpReply.cc:339
void sendMoreRequestBody()
Definition: Client.cc:388
const char * xstrerr(int error)
Definition: xstrerror.cc:83
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:24
AsyncCall::Pointer requestSender
Definition: Client.h:170
void sendBodyIsTooLargeError()
Definition: Client.cc:933
void const char HLPCB void * data
Definition: stub_helper.cc:16
bool isEmpty() const
Definition: Store.h:60
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
Client(FwdState *)
Definition: Client.cc:39
void handleAdaptedBodyProducerAborted()
Definition: Client.cc:798
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const
Definition: HttpReply.cc:484
virtual void noteBodyProductionEnded(BodyPipe::Pointer)
Definition: Client.cc:260
virtual void abortAll(const char *reason)=0
abnormal transaction termination; reason is for debugging only
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
BodyPipe::Pointer virginBodyDestination
Definition: Client.h:173
#define DBG_IMPORTANT
Definition: Debug.h:46
virtual void noteBodyConsumerAborted(BodyPipe::Pointer)
Definition: Client.cc:650
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:506
FwdState::Pointer fwd
Definition: Client.h:165
virtual void swanSong()
Definition: AsyncJob.h:54
void complete()
Definition: FwdState.cc:514
acl_access * storeMiss
Definition: SquidConfig.h:367
void serverComplete2()
Definition: Client.cc:178
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:327
bool handledEarlyAdaptationAbort()
Definition: Client.cc:855
void startAdaptation(const Adaptation::ServiceGroupPointer &group, HttpRequest *cause)
Initiate an asynchronous adaptation transaction which will call us back.
Definition: Client.cc:543
HttpReply const * getReply() const
Definition: store.cc:1670
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:340
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
BodyPipe::Pointer requestBodySource
Definition: Client.h:169
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:535
char const * termedBuf() const
Definition: SquidString.h:91
virtual bool doneAll() const
whether positive goal has been reached
Definition: Client.cc:193
void handleUnregisteredServerEnd()
Definition: FwdState.cc:715
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:687
common parts of HttpRequest and HttpReply
Definition: Message.h:25
struct SquidConfig::@113 accessList
bool allowed() const
Definition: Acl.h:141
void adjustBodyBytesRead(const int64_t delta)
initializes bodyBytesRead stats if needed and applies delta
Definition: Client.cc:962
bool setConsumerIfNotLate(const Consumer::Pointer &aConsumer)
Definition: BodyPipe.cc:228
const char * c_str()
Definition: SBuf.cc:526
void cleanAdaptation()
Definition: Client.cc:569
uint16_t flags
Definition: Store.h:208
Http::StatusLine sline
Definition: HttpReply.h:60
virtual bool mayReadVirginReplyBody() const =0
whether we may receive more virgin response body bytes
virtual void completeForwarding()
Definition: Client.cc:206
bool blockCaching()
whether to prevent caching of an otherwise cachable response
Definition: Client.cc:518
virtual void closeServer()=0
virtual void maybeReadVirginBody()=0
read response data from the network
int64_t currentOffset
Definition: Client.h:160
void handleAdaptedBodyProductionEnded()
Definition: Client.cc:779
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:977
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
char * content()
start of the added data
Definition: MemBuf.h:41
void const char * buf
Definition: stub_helper.cc:16
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:153
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1050
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:295
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
HttpHeader header
Definition: Message.h:75
virtual void doneSendingRequestBody()=0
Definition: Client.cc:316
void stopPeerClock(const bool force)
Definition: access_log.cc:303
bool final
whether the error, if any, cannot be bypassed
Definition: Answer.h:41
AclDenyInfoList * denyInfoList
Definition: SquidConfig.h:407
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:139
prohibits making the key public
Definition: enums.h:90
struct StatCounters::@129::@138 all
allow_t const & fastCheck()
Definition: Checklist.cc:336
HttpRequestPointer request
Definition: Client.h:166
void purgeEntriesByUrl(HttpRequest *req, const char *url)
void write(StoreIOBuffer)
Definition: store.cc:797
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
static bool sameUrlHosts(const char *url1, const char *url2)
Definition: Client.cc:423
HttpReply * theFinalReply
Definition: Client.h:191
HttpHdrRangeSpec spec
bool dontRetry()
Definition: FwdState.h:102
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:149
Kind kind
the type of the answer
Definition: Answer.h:42
void detailError(err_type aType, int aDetail)
sets error detail if no earlier detail was available
Definition: HttpRequest.cc:456
forward the supplied adapted HTTP message
Definition: Answer.h:27
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:117
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
HttpReply * setFinalReply(HttpReply *r)
Definition: Client.cc:135
block or deny the master xaction; see authority
Definition: Answer.h:28
static bool Start(Method method, VectPoint vp, HttpRequest *req, HttpReply *rep, AccessLogEntry::Pointer &al, Adaptation::Initiator *initiator)
Definition: AccessCheck.cc:30
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:180
Definition: MemBuf.h:23
void stopConsumingFrom(RefCount< BodyPipe > &)
Definition: BodyPipe.cc:118
void adaptVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:594
BodyPipe::Pointer adaptedBodySource
Definition: Client.h:175
CbcPointer< Adaptation::Initiate > adaptedHeadSource
Definition: Client.h:174
void fail(ErrorState *err)
Definition: FwdState.cc:459
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:98
bool adaptationAccessCheckPending
Definition: Client.h:177
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1212
void detailError(int dCode)
set error type-specific detail code
Definition: errorpage.h:97
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:228
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:352
int xerrno
Definition: errorpage.h:149
virtual void swanSong()
Definition: Client.cc:69
void startWriting()
Definition: store.cc:1798
void handleAdaptationCompleted()
Definition: Client.cc:820
size_t replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
Definition: Client.cc:1037
virtual void noteMoreBodyDataAvailable(BodyPipe::Pointer)
Definition: Client.cc:246
bool abortOnBadEntry(const char *abortReason)
Entry-dependent callbacks use this check to quit if the entry went bad.
Definition: Client.cc:234
HttpReply * finalReply()
Definition: Client.cc:128
static void purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
Definition: Client.cc:452
virtual bool doneWithAdaptation() const
Definition: Client.cc:586
StoreEntry * entry
Definition: FwdState.h:137
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
C * getRaw() const
Definition: RefCount.h:74
void endAdaptedBodyConsumption()
called when the entire adapted response body is consumed
Definition: Client.cc:791
const char * doneWithFwd
Definition: Client.h:184
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
bool purgesOthers() const
void handleAdaptationAborted(bool bypassable=false)
Definition: Client.cc:839
int size
Definition: ModDevPoll.cc:77
bool completed
Definition: Client.h:86
A const & min(A const &lhs, A const &rhs)
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:35
err_type
Definition: err_type.h:12
err_type aclGetDenyInfoPage(AclDenyInfoList **head, const char *name, int redirect_allowed)
Definition: Gadgets.cc:40
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
void clearAdaptation(CbcPointer< Initiate > &x)
clears the pointer (does not call announceInitiatorAbort)
Definition: Initiator.cc:32
void lock(const char *context)
Definition: store.cc:433

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors