Client.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9#include "squid.h"
10#include "acl/FilledChecklist.h"
11#include "acl/Gadgets.h"
12#include "base/TextException.h"
13#include "clients/Client.h"
14#include "comm/Connection.h"
15#include "comm/forward.h"
16#include "comm/Write.h"
17#include "error/Detail.h"
18#include "errorpage.h"
19#include "fd.h"
20#include "HttpHdrContRange.h"
21#include "HttpReply.h"
22#include "HttpRequest.h"
23#include "SquidConfig.h"
24#include "StatCounters.h"
25#include "Store.h"
26#include "tools.h"
27
28#if USE_ADAPTATION
30#include "adaptation/Answer.h"
31#include "adaptation/Iterator.h"
32#include "base/AsyncCall.h"
33#endif
34
35// implemented in client_side_reply.cc until sides have a common parent
36void purgeEntriesByUrl(HttpRequest * req, const char *url);
37
38Client::Client(FwdState *theFwdState) :
39 AsyncJob("Client"),
40 fwd(theFwdState),
41 request(fwd->request)
42{
43 entry = fwd->entry;
44 entry->lock("Client");
45}
46
48{
49 // paranoid: check that swanSong has been called
51#if USE_ADAPTATION
54#endif
55
56 entry->unlock("Client");
57
60
61 if (responseBodyBuffer != nullptr) {
62 delete responseBodyBuffer;
63 responseBodyBuffer = nullptr;
64 }
65}
66
67void
69{
70 // get rid of our piping obligations
71 if (requestBodySource != nullptr)
73
74#if USE_ADAPTATION
76#endif
77
78 if (!doneWithServer())
80
81 if (!doneWithFwd) {
82 doneWithFwd = "swanSong()";
84 }
85
87#if USE_ADAPTATION
88 Initiator::swanSong();
90#endif
91
92 // paranoid: check that swanSong has been called
93 // extra paranoid: yeah, I really mean it. they MUST pass here.
95#if USE_ADAPTATION
98#endif
99}
100
101HttpReply *
103{
105 return theVirginReply;
106}
107
108const HttpReply *
110{
112 return theVirginReply;
113}
114
115HttpReply *
117{
118 debugs(11,5, this << " setting virgin reply to " << rep);
120 assert(rep);
121 theVirginReply = rep;
123 if (fwd->al)
125 return theVirginReply;
126}
127
128HttpReply *
130{
132 return theFinalReply;
133}
134
135HttpReply *
137{
138 debugs(11,5, this << " setting final reply to " << rep);
139
141 assert(rep);
142 theFinalReply = rep;
144 if (fwd->al)
146
147 // give entry the reply because haveParsedReplyHeaders() expects it there
148 entry->replaceHttpReply(theFinalReply, false); // but do not write yet
149 haveParsedReplyHeaders(); // update the entry/reply (e.g., set timestamps)
151 entry->release();
152 entry->startWriting(); // write the updated entry to store
153
154 return theFinalReply;
155}
156
157void
158Client::markParsedVirginReplyAsWhole(const char *reasonWeAreSure)
159{
160 assert(reasonWeAreSure);
161 debugs(11, 3, reasonWeAreSure);
162
163 // The code storing adapted reply takes care of markStoredReplyAsWhole().
164 // We need to take care of the remaining regular network-to-store case.
165#if USE_ADAPTATION
166 if (startedAdaptation) {
167 debugs(11, 5, "adaptation handles markStoredReplyAsWhole()");
168 return;
169 }
170#endif
171
172 // Convert the "parsed whole virgin reply" event into the "stored..." event
173 // because, without adaptation, we store everything we parse: There is no
174 // buffer for parsed content; addVirginReplyBody() stores every parsed byte.
175 fwd->markStoredReplyAsWhole(reasonWeAreSure);
176}
177
178// called when no more server communication is expected; may quit
179void
181{
182 debugs(11,5, "serverComplete " << this);
183
184 if (!doneWithServer()) {
185 closeServer();
187 }
188
189 completed = true;
191
192 if (requestBodySource != nullptr)
194
195 if (responseBodyBuffer != nullptr)
196 return;
197
199}
200
201void
203{
204 debugs(11,5, "serverComplete2 " << this);
205
206#if USE_ADAPTATION
207 if (virginBodyDestination != nullptr)
209
210 if (!doneWithAdaptation())
211 return;
212#endif
213
215}
216
217bool Client::doneAll() const
218{
219 return doneWithServer() &&
220#if USE_ADAPTATION
224#endif
226}
227
228// FTP side overloads this to work around multiple calls to fwd->complete
229void
231{
232 debugs(11,5, "completing forwarding for " << fwd);
233 assert(fwd != nullptr);
234 doneWithFwd = "completeForwarding()";
235 fwd->complete();
236}
237
238// Register to receive request body
240{
242 assert(r->body_pipe != nullptr);
245 debugs(11,3, "expecting request body from " <<
247 return true;
248 }
249
250 debugs(11,3, "aborting on partially consumed request body: " <<
252 requestBodySource = nullptr;
253 return false;
254}
255
256// Entry-dependent callbacks use this check to quit if the entry went bad
257bool
258Client::abortOnBadEntry(const char *abortReason)
259{
260 if (entry->isAccepting())
261 return false;
262
263 debugs(11,5, "entry is not Accepting!");
264 abortOnData(abortReason);
265 return true;
266}
267
268// more request or adapted response body is available
269void
271{
272#if USE_ADAPTATION
273 if (adaptedBodySource == bp) {
275 return;
276 }
277#endif
278 if (requestBodySource == bp)
280}
281
282// the entire request or adapted response body was provided, successfully
283void
285{
286#if USE_ADAPTATION
287 if (adaptedBodySource == bp) {
289 return;
290 }
291#endif
292 if (requestBodySource == bp)
294}
295
296// premature end of the request or adapted response body production
297void
299{
300#if USE_ADAPTATION
301 if (adaptedBodySource == bp) {
303 return;
304 }
305#endif
306 if (requestBodySource == bp)
308}
309
310bool
311Client::abortOnData(const char *reason)
312{
313 abortAll(reason);
314 return true;
315}
316
317// more origin request body data is available
318void
320{
321 if (!requestSender)
323 else
324 debugs(9,3, "waiting for request body write to complete");
325}
326
327// there will be no more handleMoreRequestBodyAvailable calls
328void
330{
332 if (!requestSender)
334 else
335 debugs(9,3, "waiting for request body write to complete");
336}
337
338// called when we are done sending request body; kids extend this
339void
341{
342 debugs(9,3, "done sending request body");
343 assert(requestBodySource != nullptr);
345
346 // kids extend this
347}
348
349// called when body producers aborts; kids extend this
350void
352{
353 if (requestSender != nullptr)
354 debugs(9,3, "fyi: request body aborted while we were sending");
355
356 fwd->dontRetry(true); // the problem is not with the server
357 stopConsumingFrom(requestBodySource); // requestSender, if any, will notice
358
359 // kids extend this
360}
361
362// called when we wrote request headers(!) or a part of the body
363void
365{
366 debugs(11, 5, "sentRequestBody: FD " << io.fd << ": size " << io.size << ": errflag " << io.flag << ".");
367 debugs(32,3, "sentRequestBody called");
368
369 requestSender = nullptr;
370
371 if (io.size > 0) {
372 fd_bytes(io.fd, io.size, FD_WRITE);
374 // kids should increment their counters
375 }
376
377 if (io.flag == Comm::ERR_CLOSING)
378 return;
379
380 if (!requestBodySource) {
381 debugs(9,3, "detected while-we-were-sending abort");
382 return; // do nothing;
383 }
384
385 // both successful and failed writes affect response times
387
388 if (io.flag) {
389 debugs(11, DBG_IMPORTANT, "ERROR: sentRequestBody failure: FD " << io.fd << ": " << xstrerr(io.xerrno));
390 ErrorState *err;
392 err->xerrno = io.xerrno;
393 fwd->fail(err);
394 abortOnData("I/O error while sending request body");
395 return;
396 }
397
399 abortOnData("store entry aborted while sending request body");
400 return;
401 }
402
407 else
408 debugs(9,3, "waiting for body production end or abort");
409}
410
411void
413{
414 assert(requestBodySource != nullptr);
416
418
419 if (!Comm::IsConnOpen(conn)) {
420 debugs(9,3, "cannot send request body to closing " << conn);
421 return; // wait for the kid's close handler; TODO: assert(closer);
422 }
423
424 MemBuf buf;
425 if (getMoreRequestBody(buf) && buf.contentSize() > 0) {
426 debugs(9,3, "will write " << buf.contentSize() << " request body bytes");
430 } else {
431 debugs(9,3, "will wait for more request body bytes or eof");
432 requestSender = nullptr;
433 }
434}
435
437bool
439{
440 // default implementation does not encode request body content
441 Must(requestBodySource != nullptr);
442 return requestBodySource->getMoreData(buf);
443}
444
445// Compares hosts in urls, returns false if different, no sheme, or no host.
446static bool
447sameUrlHosts(const char *url1, const char *url2)
448{
449 // XXX: Want AnyP::Uri::parse() here, but it uses static storage and copying
450 const char *host1 = strchr(url1, ':');
451 const char *host2 = strchr(url2, ':');
452
453 if (host1 && host2) {
454 // skip scheme slashes
455 do {
456 ++host1;
457 ++host2;
458 } while (*host1 == '/' && *host2 == '/');
459
460 if (!*host1)
461 return false; // no host
462
463 // increment while the same until we reach the end of the URL/host
464 while (*host1 && *host1 != '/' && *host1 == *host2) {
465 ++host1;
466 ++host2;
467 }
468 return *host1 == *host2;
469 }
470
471 return false; // no URL scheme
472}
473
474// purges entries that match the value of a given HTTP [response] header
475static void
477{
478 const auto hdrUrl = rep->header.getStr(hdr);
479 if (!hdrUrl)
480 return;
481
482 /*
483 * If the URL is relative, make it absolute so we can find it.
484 * If it's absolute, make sure the host parts match to avoid DOS attacks
485 * as per RFC 2616 13.10.
486 */
487 SBuf absUrlMaker;
488 const char *absUrl = nullptr;
489 if (urlIsRelative(hdrUrl)) {
490 if (req->method.id() == Http::METHOD_CONNECT)
491 absUrl = hdrUrl; // TODO: merge authority-uri and hdrUrl
492 else if (req->url.getScheme() == AnyP::PROTO_URN)
493 absUrl = req->url.absolute().c_str();
494 else {
495 AnyP::Uri tmpUrl = req->url;
496 if (*hdrUrl == '/') {
497 // RFC 3986 section 4.2: absolute-path reference
498 // for this logic replace the entire request-target URI path
499 tmpUrl.path(hdrUrl);
500 } else {
501 tmpUrl.addRelativePath(reqUrl);
502 }
503 absUrlMaker = tmpUrl.absolute();
504 absUrl = absUrlMaker.c_str();
505 }
506 } else if (!sameUrlHosts(reqUrl, hdrUrl)) {
507 return;
508 } else
509 absUrl = hdrUrl;
510
511 purgeEntriesByUrl(req, absUrl);
512}
513
514// some HTTP methods should purge matching cache entries
515void
517{
518 // only some HTTP methods should purge matching cache entries
520 return;
521
522 // and probably only if the response was successful
523 if (theFinalReply->sline.status() >= 400)
524 return;
525
526 // XXX: should we use originalRequest() here?
528 const char *reqUrl = tmp.c_str();
529 debugs(88, 5, "maybe purging due to " << request->method << ' ' << tmp);
533}
534
536void
538{
541
542 // adaptation may overwrite old offset computed using the virgin response
543 currentOffset = 0;
544 if (const auto cr = theFinalReply->contentRange()) {
545 if (cr->spec.offset != HttpHdrRangeSpec::UnknownPosition)
546 currentOffset = cr->spec.offset;
547 }
548}
549
551bool
553{
554 if (const Acl::Tree *acl = Config.accessList.storeMiss) {
555 // This relatively expensive check is not in StoreEntry::checkCachable:
556 // That method lacks HttpRequest and may be called too many times.
557 ACLFilledChecklist ch(acl, originalRequest().getRaw());
558 ch.reply = const_cast<HttpReply*>(&entry->mem().freshestReply()); // ACLFilledChecklist API bug
559 HTTPMSGLOCK(ch.reply);
560 ch.al = fwd->al;
561 if (!ch.fastCheck().allowed()) { // when in doubt, block
562 debugs(20, 3, "store_miss prohibits caching");
563 return true;
564 }
565 }
566 return false;
567}
568
571{
572 return request;
573}
574
575#if USE_ADAPTATION
577void
579{
580 debugs(11, 5, "Client::startAdaptation() called");
581 // check whether we should be sending a body as well
582 // start body pipe to feed ICAP transaction if needed
584 HttpReply *vrep = virginReply();
585 assert(!vrep->body_pipe);
586 int64_t size = 0;
587 if (vrep->expectingBody(cause->method, size) && size) {
590 debugs(93, 6, "will send virgin reply body to " <<
591 virginBodyDestination << "; size: " << size);
592 if (size > 0)
594 }
595
597 new Adaptation::Iterator(vrep, cause, fwd->al, group));
600}
601
602// properly cleans up ICAP-related state
603// may be called multiple times
605{
606 debugs(11,5, "cleaning ICAP; ACL: " << adaptationAccessCheckPending);
607
608 if (virginBodyDestination != nullptr)
610
612
613 if (adaptedBodySource != nullptr)
615
616 if (!adaptationAccessCheckPending) // we cannot cancel a pending callback
617 assert(doneWithAdaptation()); // make sure the two methods are in sync
618}
619
620bool
622{
625}
626
627// sends virgin reply body to ICAP, buffering excesses if needed
628void
629Client::adaptVirginReplyBody(const char *data, ssize_t len)
630{
632
634 debugs(11,3, "ICAP does not want more virgin body");
635 return;
636 }
637
638 // grow overflow area if already overflowed
639 if (responseBodyBuffer) {
640 responseBodyBuffer->append(data, len);
641 data = responseBodyBuffer->content();
643 }
644
645 const ssize_t putSize = virginBodyDestination->putMoreData(data, len);
646 data += putSize;
647 len -= putSize;
648
649 // if we had overflow area, shrink it as necessary
650 if (responseBodyBuffer) {
651 if (putSize == responseBodyBuffer->contentSize()) {
652 delete responseBodyBuffer;
653 responseBodyBuffer = nullptr;
654 } else {
655 responseBodyBuffer->consume(putSize);
656 }
657 return;
658 }
659
660 // if we did not have an overflow area, create it as needed
661 if (len > 0) {
664 responseBodyBuffer->init(4096, SQUID_TCP_SO_RCVBUF * 10);
665 responseBodyBuffer->append(data, len);
666 }
667}
668
669// can supply more virgin response body data
670void
672{
673 if (responseBodyBuffer) {
674 addVirginReplyBody(nullptr, 0); // kick the buffered fragment alive again
677 return;
678 }
679 }
681}
682
683// the consumer of our virgin response body aborted
684void
686{
688
689 // do not force closeServer here in case we need to bypass AdaptationQueryAbort
690
691 if (doneWithAdaptation()) // we may still be receiving adapted response
693}
694
695// received adapted response headers (body may follow)
696void
698{
699 clearAdaptation(adaptedHeadSource); // we do not expect more messages
700
701 switch (answer.kind) {
703 handleAdaptedHeader(const_cast<Http::Message*>(answer.message.getRaw()));
704 break;
705
708 break;
709
712 break;
713 }
714}
715
716void
718{
719 if (abortOnBadEntry("entry went bad while waiting for adapted headers")) {
720 // If the adapted response has a body, the ICAP side needs to know
721 // that nobody will consume that body. We will be destroyed upon
722 // return. Tell the ICAP side that it is on its own.
723 HttpReply *rep = dynamic_cast<HttpReply*>(msg);
724 assert(rep);
725 if (rep->body_pipe != nullptr)
727
728 return;
729 }
730
731 HttpReply *rep = dynamic_cast<HttpReply*>(msg);
732 assert(rep);
733 debugs(11,5, this << " setting adapted reply to " << rep);
734 setFinalReply(rep);
735
737 if (rep->body_pipe != nullptr) {
738 // subscribe to receive adapted body
740 // assume that ICAP does not auto-consume on failures
741 const bool result = adaptedBodySource->setConsumerIfNotLate(this);
742 assert(result);
743 } else {
744 // no body
745 fwd->markStoredReplyAsWhole("setFinalReply() stored header-only adapted reply");
746 if (doneWithAdaptation()) // we may still be sending virgin response
748 }
749}
750
751void
753{
754 if (abortOnBadEntry("store entry aborted while kick producer callback"))
755 return;
756
758 return;
759
761
762 if (adaptedBodySource != nullptr && adaptedBodySource->exhausted())
764}
765
766// more adapted response body is available
767void
769{
770 if (abortOnBadEntry("entry refuses adapted body"))
771 return;
772
773 assert(entry);
774
775 size_t contentSize = adaptedBodySource->buf().contentSize();
776
777 if (!contentSize)
778 return; // XXX: bytesWanted asserts on zero-size ranges
779
780 const size_t spaceAvailable = entry->bytesWanted(Range<size_t>(0, contentSize), true);
781
782 if (spaceAvailable < contentSize ) {
783 // No or partial body data consuming
784 typedef NullaryMemFunT<Client> Dialer;
785 AsyncCall::Pointer call = asyncCall(93, 5, "Client::resumeBodyStorage",
786 Dialer(this, &Client::resumeBodyStorage));
787 entry->deferProducer(call);
788 }
789
790 if (!spaceAvailable) {
791 debugs(11, 5, "NOT storing " << contentSize << " bytes of adapted " <<
792 "response body at offset " << adaptedBodySource->consumedSize());
793 return;
794 }
795
796 if (spaceAvailable < contentSize ) {
797 debugs(11, 5, "postponing storage of " <<
798 (contentSize - spaceAvailable) << " body bytes");
799 contentSize = spaceAvailable;
800 }
801
802 debugs(11,5, "storing " << contentSize << " bytes of adapted " <<
803 "response body at offset " << adaptedBodySource->consumedSize());
804
806 const StoreIOBuffer ioBuf(&bpc.buf, currentOffset, contentSize);
807 currentOffset += ioBuf.length;
808 entry->write(ioBuf);
809 bpc.buf.consume(contentSize);
810 bpc.checkIn();
811}
812
813// the entire adapted response body was produced, successfully
814void
816{
817 if (abortOnBadEntry("entry went bad while waiting for adapted body eof"))
818 return;
819
820 // distinguish this code path from handleAdaptedBodyProducerAborted()
822
823 // end consumption if we consumed everything
824 if (adaptedBodySource != nullptr && adaptedBodySource->exhausted())
826 // else resumeBodyStorage() will eventually consume the rest
827}
828
829void
831{
833
835 // We received the entire adapted reply per receivedWholeAdaptedReply.
836 // We are called when we consumed everything received (per our callers).
837 // We consume only what we store per handleMoreAdaptedBodyAvailable().
838 fwd->markStoredReplyAsWhole("received,consumed=>stored the entire RESPMOD reply");
839 }
840
842}
843
844// premature end of the adapted response body
846{
847 if (abortOnBadEntry("entry went bad while waiting for the now-aborted adapted body"))
848 return;
849
850 Must(adaptedBodySource != nullptr);
852 debugs(11,5, "waiting to consume the remainder of the aborted adapted body");
853 return; // resumeBodyStorage() should eventually consume the rest
854 }
855
857
859 return;
860
861 handleAdaptationCompleted(); // the user should get a truncated response
862}
863
864// common part of noteAdaptationAnswer and handleAdaptedBodyProductionEnded
865void
867{
868 debugs(11,5, "handleAdaptationCompleted");
870
871 // We stop reading origin response because we have no place to put it(*) and
872 // cannot use it. If some origin servers do not like that or if we want to
873 // reuse more pconns, we can add code to discard unneeded origin responses.
874 // (*) TODO: Is it possible that the adaptation xaction is still running?
876 debugs(11,3, "closing origin conn due to ICAP completion");
877 closeServer();
878 }
879
881}
882
883// common part of noteAdaptation*Aborted and noteBodyConsumerAborted methods
884void
886{
887 debugs(11,5, "handleAdaptationAborted; bypassable: " << bypassable <<
888 ", entry empty: " << entry->isEmpty());
889
890 if (abortOnBadEntry("entry went bad while ICAP aborted"))
891 return;
892
893 // TODO: bypass if possible
895 abortAll("adaptation failure with a filled entry");
896}
897
900bool
902{
903 if (entry->isEmpty()) {
904 debugs(11,8, "adaptation failure with an empty entry: " << *entry);
906 static const auto d = MakeNamedErrorDetail("ICAP_RESPMOD_EARLY");
907 err->detailError(d);
908 fwd->fail(err);
909 fwd->dontRetry(true);
910 abortAll("adaptation failure with an empty entry");
911 return true; // handled
912 }
913
914 if (request) { // update logged info directly
915 static const auto d = MakeNamedErrorDetail("ICAP_RESPMOD_LATE");
917 }
918
919 return false; // the caller must handle
920}
921
922// adaptation service wants us to deny HTTP client access to this response
923void
925{
926 debugs(11,5, answer.ruleId);
927
928 if (abortOnBadEntry("entry went bad while ICAP aborted"))
929 return;
930
931 if (!entry->isEmpty()) { // too late to block (should not really happen)
932 if (request) {
933 static const auto d = MakeNamedErrorDetail("RESPMOD_BLOCK_LATE");
935 }
936 abortAll("late adaptation block");
937 return;
938 }
939
940 debugs(11,7, "creating adaptation block response");
941
942 err_type page_id =
944 if (page_id == ERR_NONE)
945 page_id = ERR_ACCESS_DENIED;
946
947 const auto err = new ErrorState(page_id, Http::scForbidden, request.getRaw(), fwd->al);
948 static const auto d = MakeNamedErrorDetail("RESPMOD_BLOCK_EARLY");
949 err->detailError(d);
950 fwd->fail(err);
951 fwd->dontRetry(true);
952
953 abortOnData("timely adaptation block");
954}
955
956void
958{
960
961 if (abortOnBadEntry("entry went bad while waiting for ICAP ACL check"))
962 return;
963
964 // TODO: Should non-ICAP and ICAP REPMOD pre-cache paths check this?
965 // That check now only happens on REQMOD pre-cache and REPMOD post-cache, in processReplyAccess().
966 if (virginReply()->expectedBodyTooLarge(*request)) {
968 return;
969 }
970 // TODO: Should we check receivedBodyTooLarge as well?
971
972 if (!group) {
973 debugs(11,3, "no adapation needed");
976 return;
977 }
978
979 startAdaptation(group, originalRequest().getRaw());
981}
982#endif
983
984void
986{
987 const auto err = new ErrorState(ERR_TOO_BIG, Http::scForbidden, request.getRaw(), fwd->al);
988 fwd->fail(err);
989 fwd->dontRetry(true);
990 abortOnData("Virgin body too large.");
991}
992
993// TODO: when HttpStateData sends all errors to ICAP,
994// we should be able to move this at the end of setVirginReply().
995void
997{
998#if USE_ADAPTATION
999 // TODO: merge with client side and return void to hide the on/off logic?
1000 // The callback can be called with a NULL service if adaptation is off.
1003 originalRequest().getRaw(), virginReply(), fwd->al, this);
1004 debugs(11,5, "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
1006 return;
1007#endif
1008
1010}
1011
1013void
1015{
1016 int64_t &bodyBytesRead = originalRequest()->hier.bodyBytesRead;
1017
1018 // if we got here, do not log a dash even if we got nothing from the server
1019 if (bodyBytesRead < 0)
1020 bodyBytesRead = 0;
1021
1022 bodyBytesRead += delta; // supports negative and zero deltas
1023
1024 // check for overflows ("infinite" response?) and underflows (a bug)
1025 Must(bodyBytesRead >= 0);
1026}
1027
1028void
1030{
1031 using DeferredReadDialer = NullaryMemFunT<Client>;
1032 AsyncCall::Pointer call = asyncCall(11, 5, "Client::noteDelayAwareReadChance",
1033 DeferredReadDialer(this, &Client::noteDelayAwareReadChance));
1034 entry->mem().delayRead(call);
1035}
1036
1037void
1038Client::addVirginReplyBody(const char *data, ssize_t len)
1039{
1041
1042#if USE_ADAPTATION
1043 assert(!adaptationAccessCheckPending); // or would need to buffer while waiting
1044 if (startedAdaptation) {
1045 adaptVirginReplyBody(data, len);
1046 return;
1047 }
1048#endif
1049 storeReplyBody(data, len);
1050}
1051
1052// writes virgin or adapted reply body to store
1053void
1054Client::storeReplyBody(const char *data, ssize_t len)
1055{
1056 // write even if len is zero to push headers towards the client side
1057 entry->write (StoreIOBuffer(len, currentOffset, (char*)data));
1058
1059 currentOffset += len;
1060}
1061
1062size_t
1063Client::calcBufferSpaceToReserve(size_t space, const size_t wantSpace) const
1064{
1065 if (space < wantSpace) {
1066 const size_t maxSpace = SBuf::maxSize; // absolute best
1067 space = min(wantSpace, maxSpace); // do not promise more than asked
1068 }
1069
1070#if USE_ADAPTATION
1071 if (responseBodyBuffer) {
1072 return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1073 }
1074
1075 if (virginBodyDestination != nullptr) {
1076 /*
1077 * BodyPipe buffer has a finite size limit. We
1078 * should not read more data from the network than will fit
1079 * into the pipe buffer or we _lose_ what did not fit if
1080 * the response ends sooner that BodyPipe frees up space:
1081 * There is no code to keep pumping data into the pipe once
1082 * response ends and serverComplete() is called.
1083 */
1084 const size_t adaptor_space = virginBodyDestination->buf().potentialSpaceSize();
1085
1086 debugs(11,9, "Client may read up to min(" <<
1087 adaptor_space << ", " << space << ") bytes");
1088
1089 if (adaptor_space < space)
1090 space = adaptor_space;
1091 }
1092#endif
1093
1094 return space;
1095}
1096
1097size_t
1098Client::replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
1099{
1100 size_t space = readBuf.spaceSize(); // available space w/o heroic measures
1101 if (space < minSpace) {
1102 const size_t maxSpace = readBuf.potentialSpaceSize(); // absolute best
1103 space = min(minSpace, maxSpace); // do not promise more than asked
1104 }
1105
1106#if USE_ADAPTATION
1107 if (responseBodyBuffer) {
1108 return 0; // Stop reading if already overflowed waiting for ICAP to catch up
1109 }
1110
1111 if (virginBodyDestination != nullptr) {
1112 /*
1113 * BodyPipe buffer has a finite size limit. We
1114 * should not read more data from the network than will fit
1115 * into the pipe buffer or we _lose_ what did not fit if
1116 * the response ends sooner that BodyPipe frees up space:
1117 * There is no code to keep pumping data into the pipe once
1118 * response ends and serverComplete() is called.
1119 *
1120 * If the pipe is totally full, don't register the read handler.
1121 * The BodyPipe will call our noteMoreBodySpaceAvailable() method
1122 * when it has free space again.
1123 */
1124 size_t adaptation_space =
1126
1127 debugs(11,9, "Client may read up to min(" <<
1128 adaptation_space << ", " << space << ") bytes");
1129
1130 if (adaptation_space < space)
1131 space = adaptation_space;
1132 }
1133#endif
1134
1135 return space;
1136}
1137
AsyncCall * asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:154
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:69
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
int size
Definition: ModDevPoll.cc:75
class SquidConfig Config
Definition: SquidConfig.cc:12
StatCounters statCounter
Definition: StatCounters.cc:12
#define Must(condition)
Definition: TextException.h:71
int conn
the current server connection FD
Definition: Transport.cc:26
bool urlIsRelative(const char *url)
Definition: Uri.cc:676
#define assert(EX)
Definition: assert.h:19
Acl::Answer const & fastCheck()
Definition: Checklist.cc:332
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
HttpReplyPointer reply
bool allowed() const
Definition: Acl.h:150
Definition: Tree.h:21
static bool Start(Method method, VectPoint vp, HttpRequest *req, HttpReply *, const AccessLogEntryPointer &, Adaptation::Initiator *)
Definition: AccessCheck.cc:30
summarizes adaptation service answer for the noteAdaptationAnswer() API
Definition: Answer.h:23
String ruleId
ACL (or similar rule) name that blocked forwarding.
Definition: Answer.h:40
Kind kind
the type of the answer
Definition: Answer.h:42
Http::MessagePointer message
HTTP request or response to forward.
Definition: Answer.h:39
bool final
whether the error, if any, cannot be bypassed
Definition: Answer.h:41
@ akForward
forward the supplied adapted HTTP message
Definition: Answer.h:27
@ akBlock
block or deny the master xaction; see authority
Definition: Answer.h:28
@ akError
no adapted message will come; see bypassable
Definition: Answer.h:29
CbcPointer< Initiate > initiateAdaptation(Initiate *x)
< starts freshly created initiate and returns a safe pointer to it
Definition: Initiator.cc:23
void clearAdaptation(CbcPointer< Initiate > &x)
clears the pointer (does not call announceInitiatorAbort)
Definition: Initiator.cc:32
void announceInitiatorAbort(CbcPointer< Initiate > &x)
inform the transaction about abnormal termination and clear the pointer
Definition: Initiator.cc:38
bool initiated(const CbcPointer< AsyncJob > &job) const
Must(initiated(initiate)) instead of Must(initiate.set()), for clarity.
Definition: Initiator.h:52
iterates services in ServiceGroup, starting adaptation launchers
Definition: Iterator.h:32
Definition: Uri.h:31
AnyP::UriScheme const & getScheme() const
Definition: Uri.h:67
void path(const char *p)
Definition: Uri.h:99
void addRelativePath(const char *relUrl)
Definition: Uri.cc:712
SBuf & absolute() const
Definition: Uri.cc:584
virtual bool doneAll() const
whether positive goal has been reached
Definition: AsyncJob.cc:97
virtual void swanSong()
Definition: AsyncJob.h:59
void stopConsumingFrom(RefCount< BodyPipe > &)
Definition: BodyPipe.cc:118
MemBuf & buf
Definition: BodyPipe.h:74
void expectNoConsumption()
there will be no more setConsumer() calls
Definition: BodyPipe.cc:267
size_t putMoreData(const char *buf, size_t size)
Definition: BodyPipe.cc:213
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:294
const MemBuf & buf() const
Definition: BodyPipe.h:137
bool exhausted() const
Definition: BodyPipe.cc:174
void setBodySize(uint64_t aSize)
Definition: BodyPipe.cc:147
uint64_t consumedSize() const
Definition: BodyPipe.h:111
bool setConsumerIfNotLate(const Consumer::Pointer &aConsumer)
Definition: BodyPipe.cc:228
const char * status() const
Definition: BodyPipe.cc:446
void stopProducingFor(RefCount< BodyPipe > &, bool atEof)
Definition: BodyPipe.cc:107
const char * doneWithFwd
Definition: Client.h:199
bool adaptationAccessCheckPending
Definition: Client.h:189
void endAdaptedBodyConsumption()
called when the entire adapted response body is consumed
Definition: Client.cc:830
void adaptVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:629
virtual void maybeReadVirginBody()=0
read response data from the network
Client(FwdState *)
Definition: Client.cc:38
size_t replyBodySpace(const MemBuf &readBuf, const size_t minSpace) const
Definition: Client.cc:1098
void sendMoreRequestBody()
Definition: Client.cc:412
void maybePurgeOthers()
Definition: Client.cc:516
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:195
virtual void noteBodyProductionEnded(BodyPipe::Pointer)
Definition: Client.cc:284
void handleAdaptationCompleted()
Definition: Client.cc:866
HttpReply * theFinalReply
Definition: Client.h:206
void serverComplete2()
Definition: Client.cc:202
void sendBodyIsTooLargeError()
Definition: Client.cc:985
BodyPipe::Pointer adaptedBodySource
Definition: Client.h:187
BodyPipe::Pointer requestBodySource
Definition: Client.h:181
virtual void noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer group)
Definition: Client.cc:957
virtual void noteMoreBodyDataAvailable(BodyPipe::Pointer)
Definition: Client.cc:270
AsyncCall::Pointer requestSender
Definition: Client.h:182
bool completed
Definition: Client.h:90
void resumeBodyStorage()
called by StoreEntry when it has more buffer space available
Definition: Client.cc:752
virtual void completeForwarding()
Definition: Client.cc:230
void serverComplete()
Definition: Client.cc:180
void delayRead()
Definition: Client.cc:1029
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:570
virtual bool abortOnData(const char *reason)
Definition: Client.cc:311
void handleMoreAdaptedBodyAvailable()
Definition: Client.cc:768
const HttpReply * virginReply() const
Definition: Client.cc:109
int64_t currentOffset
Definition: Client.h:172
bool handledEarlyAdaptationAbort()
Definition: Client.cc:901
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:438
void startAdaptation(const Adaptation::ServiceGroupPointer &group, HttpRequest *cause)
Initiate an asynchronous adaptation transaction which will call us back.
Definition: Client.cc:578
virtual void abortAll(const char *reason)=0
abnormal transaction termination; reason is for debugging only
bool abortOnBadEntry(const char *abortReason)
Entry-dependent callbacks use this check to quit if the entry went bad.
Definition: Client.cc:258
void storeReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:1054
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:351
bool receivedWholeAdaptedReply
handleAdaptedBodyProductionEnded() was called
Definition: Client.h:193
virtual void noteAdaptationAnswer(const Adaptation::Answer &answer)
Definition: Client.cc:697
bool startRequestBodyFlow()
Definition: Client.cc:239
void handleAdaptedBodyProducerAborted()
Definition: Client.cc:845
virtual ~Client()
Definition: Client.cc:47
void handleRequestBodyProductionEnded()
Definition: Client.cc:329
bool blockCaching()
whether to prevent caching of an otherwise cachable response
Definition: Client.cc:552
void handleAdaptedBodyProductionEnded()
Definition: Client.cc:815
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:116
void handleAdaptationBlocked(const Adaptation::Answer &answer)
Definition: Client.cc:924
BodyPipe::Pointer virginBodyDestination
Definition: Client.h:185
HttpReply * finalReply()
Definition: Client.cc:129
void adaptOrFinalizeReply()
Definition: Client.cc:996
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1063
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:1038
void cleanAdaptation()
Definition: Client.cc:604
CbcPointer< Adaptation::Initiate > adaptedHeadSource
Definition: Client.h:186
HttpRequestPointer request
Definition: Client.h:178
void handleAdaptedHeader(Http::Message *msg)
Definition: Client.cc:717
virtual bool doneWithServer() const =0
void adjustBodyBytesRead(const int64_t delta)
initializes bodyBytesRead stats if needed and applies delta
Definition: Client.cc:1014
void handleAdaptationAborted(bool bypassable=false)
Definition: Client.cc:885
void markParsedVirginReplyAsWhole(const char *reasonWeAreSure)
Definition: Client.cc:158
virtual bool doneAll() const
whether positive goal has been reached
Definition: Client.cc:217
bool startedAdaptation
Definition: Client.h:190
virtual void noteDelayAwareReadChance()=0
StoreEntry * entry
Definition: Client.h:176
virtual void noteBodyConsumerAborted(BodyPipe::Pointer)
Definition: Client.cc:685
virtual bool mayReadVirginReplyBody() const =0
whether we may receive more virgin response body bytes
virtual void closeServer()=0
HttpReply * setFinalReply(HttpReply *r)
Definition: Client.cc:136
void handleMoreRequestBodyAvailable()
Definition: Client.cc:319
HttpReply * theVirginReply
Definition: Client.h:205
virtual void processReplyBody()=0
MemBuf * responseBodyBuffer
Definition: Client.h:173
virtual bool doneWithAdaptation() const
Definition: Client.cc:621
virtual void noteBodyProducerAborted(BodyPipe::Pointer)
Definition: Client.cc:298
virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer)
Definition: Client.cc:671
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:364
virtual const Comm::ConnectionPointer & dataConnection() const =0
virtual void doneSendingRequestBody()=0
Definition: Client.cc:340
virtual void swanSong()
Definition: Client.cc:68
FwdState::Pointer fwd
Definition: Client.h:177
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:537
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:88
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:90
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:87
int xerrno
Definition: errorpage.h:177
void handleUnregisteredServerEnd()
Definition: FwdState.cc:830
HttpRequest * request
Definition: FwdState.h:170
void complete()
Definition: FwdState.cc:556
void fail(ErrorState *err)
Definition: FwdState.cc:488
bool dontRetry()
Definition: FwdState.h:98
StoreEntry * entry
Definition: FwdState.h:169
void markStoredReplyAsWhole(const char *whyWeAreSure)
Definition: FwdState.cc:605
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:171
int64_t bodyBytesRead
number of body bytes received from the next hop or -1
void stopPeerClock(const bool force)
Definition: access_log.cc:303
static int64_t const UnknownPosition
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1160
Http::StatusLine sline
Definition: HttpReply.h:56
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const
Definition: HttpReply.cc:487
const HttpHdrContRange * contentRange() const
Definition: HttpReply.cc:342
bool purgesOthers() const
Http::MethodType id() const
Definition: RequestMethod.h:70
HttpRequestMethod method
Definition: HttpRequest.h:114
HierarchyLogEntry hier
Definition: HttpRequest.h:157
void detailError(const err_type c, const ErrorDetail::Pointer &d)
sets error detail if no earlier detail was available
Definition: HttpRequest.h:101
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:747
common parts of HttpRequest and HttpReply
Definition: Message.h:26
HttpHeader header
Definition: Message.h:74
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:97
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
Definition: MemBuf.h:24
mb_size_t spaceSize() const
Definition: MemBuf.cc:155
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:209
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
char * content()
start of the added data
Definition: MemBuf.h:41
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
mb_size_t potentialSpaceSize() const
Definition: MemBuf.cc:161
void consume(mb_size_t sz)
removes sz bytes and "packs" by moving content left
Definition: MemBuf.cc:168
const HttpReply & freshestReply() const
Definition: MemObject.h:67
void delayRead(const AsyncCallPointer &)
Definition: MemObject.cc:439
Definition: Range.h:19
C * getRaw() const
Definition: RefCount.h:80
Definition: SBuf.h:94
const char * c_str()
Definition: SBuf.cc:516
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb.
Definition: SBuf.h:102
struct SquidConfig::@112 accessList
AclDenyInfoList * denyInfoList
Definition: SquidConfig.h:406
acl_access * storeMiss
Definition: SquidConfig.h:366
struct StatCounters::@129::@139 all
ByteCounter kbytes_out
Definition: StatCounters.h:46
struct StatCounters::@129 server
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:210
bool isAccepting() const
Definition: store.cc:1956
uint16_t flags
Definition: Store.h:232
MemObject & mem()
Definition: Store.h:51
int unlock(const char *context)
Definition: store.cc:442
void startWriting()
Definition: store.cc:1688
void release(const bool shareable=false)
Definition: store.cc:1119
void write(StoreIOBuffer)
Definition: store.cc:753
void lock(const char *context)
Definition: store.cc:418
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1672
bool isEmpty() const
Definition: Store.h:66
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:339
char const * termedBuf() const
Definition: SquidString.h:92
static bool sameUrlHosts(const char *url1, const char *url2)
Definition: Client.cc:447
static void purgeEntriesByHeader(HttpRequest *req, const char *reqUrl, Http::Message *rep, Http::HdrType hdr)
Definition: Client.cc:476
void purgeEntriesByUrl(HttpRequest *req, const char *url)
A const & min(A const &lhs, A const &rhs)
#define DBG_IMPORTANT
Definition: Stream.h:41
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
@ FD_WRITE
Definition: enums.h:24
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:98
@ ENTRY_ABORTED
Definition: enums.h:115
err_type
Definition: forward.h:14
@ ERR_ACCESS_DENIED
Definition: forward.h:18
@ ERR_NONE
Definition: forward.h:15
@ ERR_ICAP_FAILURE
Definition: forward.h:64
@ ERR_WRITE_ERROR
Definition: forward.h:29
@ ERR_TOO_BIG
Definition: forward.h:40
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:227
err_type aclGetDenyInfoPage(AclDenyInfoList **head, const char *name, int redirect_allowed)
Definition: Gadgets.cc:42
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:149
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:160
@ methodRespmod
Definition: Elements.h:17
@ pointPreCache
Definition: Elements.h:18
@ PROTO_URN
Definition: ProtocolType.h:36
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:27
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:33
@ ERR_CLOSING
Definition: Flag.h:25
@ scForbidden
Definition: StatusCode.h:47
@ scInternalServerError
Definition: StatusCode.h:71
@ scBadGateway
Definition: StatusCode.h:73
@ METHOD_CONNECT
Definition: MethodType.h:29
@ CONTENT_LOCATION
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
const char * xstrerr(int error)
Definition: xstrerror.cc:83

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors