store_client.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 90 Storage Manager Client-Side Interface */
10
11#include "squid.h"
12#include "acl/FilledChecklist.h"
14#include "base/CodeContext.h"
15#include "event.h"
16#include "globals.h"
17#include "HttpReply.h"
18#include "HttpRequest.h"
19#include "MemBuf.h"
20#include "MemObject.h"
21#include "mime_header.h"
22#include "sbuf/Stream.h"
23#include "SquidConfig.h"
24#include "SquidMath.h"
25#include "StatCounters.h"
26#include "Store.h"
27#include "store/SwapMetaIn.h"
28#include "store_swapin.h"
29#include "StoreClient.h"
30#if USE_DELAY_POOLS
31#include "DelayPools.h"
32#endif
33
34/*
35 * NOTE: 'Header' refers to the swapfile metadata header.
36 * 'OBJHeader' refers to the object header, with canonical
37 * processed object headers (which may derive from FTP/HTTP etc
38 * upstream protocols
39 * 'Body' refers to the swapfile body, which is the full
40 * HTTP reply (including HTTP headers and body).
41 */
44static void storeClientCopy2(StoreEntry * e, store_client * sc);
45static bool CheckQuickAbortIsReasonable(StoreEntry * entry);
46
48
49/* StoreClient */
50
51bool
53{
55 return false;
56
58 return true;
59
61 fillChecklist(checklist);
62 return checklist.fastCheck().allowed();
63}
64
65bool
66StoreClient::startCollapsingOn(const StoreEntry &e, const bool doingRevalidation) const
67{
69 return false; // collapsing is impossible due to the entry state
70
71 if (!onCollapsingPath())
72 return false; // collapsing is impossible due to Squid configuration
73
74 /* collapsing is possible; the caller must collapse */
75
76 if (const auto tags = loggingTags()) {
77 if (doingRevalidation)
78 tags->collapsingHistory.revalidationCollapses++;
79 else
80 tags->collapsingHistory.otherCollapses++;
81 }
82
83 debugs(85, 5, e << " doingRevalidation=" << doingRevalidation);
84 return true;
85}
86
87/* store_client */
88
89int
91{
92 return type;
93}
94
95#if STORE_CLIENT_LIST_DEBUG
96static store_client *
97storeClientListSearch(const MemObject * mem, void *data)
98{
100 store_client *sc = nullptr;
101
102 for (node = mem->clients.head; node; node = node->next) {
103 sc = node->data;
104
105 if (sc->owner == data)
106 return sc;
107 }
108
109 return nullptr;
110}
111
112int
113storeClientIsThisAClient(store_client * sc, void *someClient)
114{
115 return sc->owner == someClient;
116}
117
118#endif
119#include "HttpRequest.h"
120
121/* add client with fd to client list */
124{
125 MemObject *mem = e->mem_obj;
127 assert(mem);
128#if STORE_CLIENT_LIST_DEBUG
129
130 if (storeClientListSearch(mem, data) != NULL)
131 /* XXX die! */
132 assert(1 == 0);
133#else
134 (void)data;
135#endif
136
137 sc = new store_client (e);
138
139 mem->addClient(sc);
140
141 return sc;
142}
143
145void
147{
148 sc->finishCallback();
149}
150
152void
154{
157
158 // XXX: Some legacy code relies on zero-length buffers having nil data
159 // pointers. Some other legacy code expects "correct" result.offset even
160 // when there is no body to return. Accommodate all those expectations.
161 auto result = StoreIOBuffer(0, copyInto.offset, nullptr);
162 if (object_ok && parsingBuffer && parsingBuffer->contentSize())
163 result = parsingBuffer->packBack();
164 result.flags.error = object_ok ? 0 : 1;
165
166 // no HTTP headers and no body bytes (but not because there was no space)
167 atEof_ = !sendingHttpHeaders() && !result.length && copyInto.length;
168
169 parsingBuffer.reset();
170 ++answers;
171
172 STCB *temphandler = _callback.callback_handler;
173 const auto cbdata = _callback.cbData.validDone();
175 copyInto.data = nullptr;
176
177 if (cbdata)
178 temphandler(cbdata, result);
179}
180
182#if STORE_CLIENT_LIST_DEBUG
183 owner(cbdataReference(data)),
184#endif
185 entry(e),
186 type(e->storeClientType()),
187 object_ok(true),
188 atEof_(false),
189 answers(0)
190{
191 Assure(entry);
192 entry->lock("store_client");
193
194 flags.disk_io_pending = false;
195 flags.store_copying = false;
196 ++ entry->refcount;
197
198 if (getType() == STORE_DISK_CLIENT) {
199 /* assert we'll be able to get the data we want */
200 /* maybe we should open swapin_sio here */
202 }
203}
204
206{
207 assert(entry);
208 entry->unlock("store_client");
209}
210
211/* copy bytes requested by the client */
212void
214 StoreEntry * e,
215 StoreIOBuffer copyInto,
216 STCB * callback,
217 void *data)
218{
219 assert (sc != nullptr);
220 sc->copy(e, copyInto,callback,data);
221}
222
223void
225 StoreIOBuffer copyRequest,
226 STCB * callback_fn,
227 void *data)
228{
229 assert (anEntry == entry);
230 assert (callback_fn);
231 assert (data);
233 debugs(90, 3, "store_client::copy: " << entry->getMD5Text() << ", from " <<
234 copyRequest.offset << ", for length " <<
235 (int) copyRequest.length << ", cb " << callback_fn << ", cbdata " <<
236 data);
237
238#if STORE_CLIENT_LIST_DEBUG
239
240 assert(this == storeClientListSearch(entry->mem_obj, data));
241#endif
242
244 _callback = Callback(callback_fn, data);
245 copyInto.data = copyRequest.data;
246 copyInto.length = copyRequest.length;
247 copyInto.offset = copyRequest.offset;
248 Assure(copyInto.offset >= 0);
249
250 if (!copyInto.length) {
251 // During the first storeClientCopy() call, a zero-size buffer means
252 // that we will have to drop any HTTP response body bytes we read (with
253 // the HTTP headers from disk). After that, it means we cannot return
254 // anything to the caller at all.
255 debugs(90, 2, "WARNING: zero-size storeClientCopy() buffer: " << copyInto);
256 // keep going; moreToRead() should prevent any from-Store reading
257 }
258
259 // Our nextHttpReadOffset() expects the first copy() call to have zero
260 // offset. More complex code could handle a positive first offset, but it
261 // would only be useful when reading responses from memory: We would not
262 // _delay_ the response (to read the requested HTTP body bytes from disk)
263 // when we already can respond with HTTP headers.
265
266 parsingBuffer.emplace(copyInto);
267
268 static bool copying (false);
269 assert (!copying);
270 copying = true;
271 /* we might be blocking comm reads due to readahead limits
272 * now we have a new offset, trigger those reads...
273 */
275 copying = false;
276
277 anEntry->lock("store_client::copy"); // see deletion note below
278
279 storeClientCopy2(entry, this);
280
281 // Bug 3480: This store_client object may be deleted now if, for example,
282 // the client rejects the hit response copied above. Use on-stack pointers!
283
284#if USE_ADAPTATION
285 anEntry->kickProducer();
286#endif
287 anEntry->unlock("store_client::copy");
288
289 // Add no code here. This object may no longer exist.
290}
291
293bool
295{
296 if (!copyInto.length)
297 return false; // the client supplied a zero-size buffer
298
300 return true; // there may be more coming
301
302 /* STORE_OK, including aborted entries: no more data is coming */
303
304 if (canReadFromMemory())
305 return true; // memory has the first byte wanted by the client
306
307 if (!entry->hasDisk())
308 return false; // cannot read anything from disk either
309
310 if (entry->objectLen() >= 0 && copyInto.offset >= entry->contentLen())
311 return false; // the disk cannot have byte(s) wanted by the client
312
313 // we cannot be sure until we swap in metadata and learn contentLen(),
314 // but the disk may have the byte(s) wanted by the client
315 return true;
316}
317
318static void
320{
321 /* reentrancy not allowed - note this could lead to
322 * dropped notifications about response data availability
323 */
324
325 if (sc->flags.store_copying) {
326 debugs(90, 3, "prevented recursive copying for " << *e);
327 return;
328 }
329
330 debugs(90, 3, "storeClientCopy2: " << e->getMD5Text());
331 assert(sc->_callback.pending());
332 /*
333 * We used to check for ENTRY_ABORTED here. But there were some
334 * problems. For example, we might have a slow client (or two) and
335 * the peer server is reading far ahead and swapping to disk. Even
336 * if the peer aborts, we want to give the client(s)
337 * everything we got before the abort condition occurred.
338 */
339 sc->doCopy(e);
340}
341
344bool
346{
347 return !answeredOnce() && entry->mem().baseReply().hdr_sz > 0;
348}
349
350void
352{
354 Assure(!flags.disk_io_pending);
355 Assure(!flags.store_copying);
356
357 assert (anEntry == entry);
358 flags.store_copying = true;
359 MemObject *mem = entry->mem_obj;
360
361 debugs(33, 5, this << " into " << copyInto <<
362 " hi: " << mem->endOffset() <<
363 " objectLen: " << entry->objectLen() <<
364 " past_answers: " << answers);
365
366 const auto sendHttpHeaders = sendingHttpHeaders();
367
368 if (!sendHttpHeaders && !moreToRead()) {
369 /* There is no more to send! */
370 debugs(33, 3, "There is no more to send!");
371 noteNews();
372 flags.store_copying = false;
373 return;
374 }
375
376 if (!sendHttpHeaders && anEntry->store_status == STORE_PENDING && nextHttpReadOffset() >= mem->endOffset()) {
377 debugs(90, 3, "store_client::doCopy: Waiting for more");
378 flags.store_copying = false;
379 return;
380 }
381
382 /*
383 * Slight weirdness here. We open a swapin file for any
384 * STORE_DISK_CLIENT, even if we can copy the requested chunk
385 * from memory in the next block. We must try to open the
386 * swapin file before sending any data to the client side. If
387 * we postpone the open, and then can not open the file later
388 * on, the client loses big time. Its transfer just gets cut
389 * off. Better to open it early (while the client side handler
390 * is clientCacheHit) so that we can fall back to a cache miss
391 * if needed.
392 */
393
394 if (STORE_DISK_CLIENT == getType() && swapin_sio == nullptr) {
395 if (!startSwapin())
396 return; // failure
397 }
398
399 // send any immediately available body bytes even if we also sendHttpHeaders
400 if (canReadFromMemory()) {
402 noteNews(); // will sendHttpHeaders (if needed) as well
403 flags.store_copying = false;
404 return;
405 }
406
407 if (sendHttpHeaders) {
408 debugs(33, 5, "just send HTTP headers: " << mem->baseReply().hdr_sz);
409 noteNews();
410 flags.store_copying = false;
411 return;
412 }
413
414 // no information that the client needs is available immediately
416}
417
419bool
421{
422 debugs(90, 3, "store_client::doCopy: Need to open swap in file");
423 /* gotta open the swapin file */
424
426 /* yuck -- this causes a TCP_SWAPFAIL_MISS on the client side */
427 fail();
428 flags.store_copying = false;
429 return false;
430 } else if (!flags.disk_io_pending) {
431 /* Don't set store_io_pending here */
432 storeSwapInStart(this);
433
434 if (swapin_sio == nullptr) {
435 fail();
436 flags.store_copying = false;
437 return false;
438 }
439
440 return true;
441 } else {
442 debugs(90, DBG_IMPORTANT, "WARNING: Averted multiple fd operation (1)");
443 flags.store_copying = false;
444 return false;
445 }
446}
447
448void
450{
452 if (error)
453 fail();
454 else
455 noteNews();
456}
457
458void
460{
461 /* What the client wants is not in memory. Schedule a disk read */
462 if (getType() == STORE_DISK_CLIENT) {
463 // we should have called startSwapin() already
464 assert(swapin_sio != nullptr);
465 } else if (!swapin_sio && !startSwapin()) {
466 debugs(90, 3, "bailing after swapin start failure for " << *entry);
467 assert(!flags.store_copying);
468 return;
469 }
470
471 assert(!flags.disk_io_pending);
472
473 debugs(90, 3, "reading " << *entry << " from disk");
474
475 fileRead();
476
477 flags.store_copying = false;
478}
479
481bool
483{
484 const auto &mem = entry->mem();
485 const auto memReadOffset = nextHttpReadOffset();
486 return mem.inmem_lo <= memReadOffset && memReadOffset < mem.endOffset() &&
487 parsingBuffer->spaceSize();
488}
489
491int64_t
493{
495 const auto &mem = entry->mem();
496 const auto hdr_sz = mem.baseReply().hdr_sz;
497 // Certain SMP cache manager transactions do not store HTTP headers in
498 // mem_hdr; they store just a kid-specific piece of the future report body.
499 // In such cases, hdr_sz ought to be zero. In all other (known) cases,
500 // mem_hdr contains HTTP response headers (positive hdr_sz if parsed)
501 // followed by HTTP response body. This code math accommodates all cases.
502 return NaturalSum<int64_t>(hdr_sz, copyInto.offset, parsingBuffer->contentSize()).value();
503}
504
508void
510{
512 const auto readInto = parsingBuffer->space().positionAt(nextHttpReadOffset());
513
514 debugs(90, 3, "copying HTTP body bytes from memory into " << readInto);
515 const auto sz = entry->mem_obj->data_hdr.copy(readInto);
516 Assure(sz > 0); // our canReadFromMemory() precondition guarantees that
517 parsingBuffer->appended(readInto.data, sz);
518}
519
520void
522{
523 MemObject *mem = entry->mem_obj;
524
526 assert(!flags.disk_io_pending);
527 flags.disk_io_pending = true;
528
529 // mem->swap_hdr_sz is zero here during initial read(s)
530 const auto nextStoreReadOffset = NaturalSum<int64_t>(mem->swap_hdr_sz, nextHttpReadOffset()).value();
531
532 // XXX: If fileRead() is called when we do not yet know mem->swap_hdr_sz,
533 // then we must start reading from disk offset zero to learn it: we cannot
534 // compute correct HTTP response start offset on disk without it. However,
535 // late startSwapin() calls imply that the assertion below might fail.
536 Assure(mem->swap_hdr_sz > 0 || !nextStoreReadOffset);
537
538 // TODO: Remove this assertion. Introduced in 1998 commit 3157c72, it
539 // assumes that swapped out memory is freed unconditionally, but we no
540 // longer do that because trimMemory() path checks lowestMemReaderOffset().
541 // It is also misplaced: We are not swapping out anything here and should
542 // not care about any swapout invariants.
543 if (mem->swap_hdr_sz != 0)
544 if (entry->swappingOut())
545 assert(mem->swapout.sio->offset() > nextStoreReadOffset);
546
547 // XXX: We should let individual cache_dirs limit the read size instead, but
548 // we cannot do that without more fixes and research because:
549 // * larger reads corrupt responses when cache_dir uses SharedMemory::get();
550 // * we do not know how to find all I/O code that assumes this limit;
551 // * performance effects of larger disk reads may be negative somewhere.
552 const decltype(StoreIOBuffer::length) maxReadSize = SM_PAGE_SIZE;
553
555 // also, do not read more than we can return (via a copyInto.length buffer)
556 const auto readSize = std::min(copyInto.length, maxReadSize);
557 lastDiskRead = parsingBuffer->makeSpace(readSize).positionAt(nextStoreReadOffset);
558 debugs(90, 5, "into " << lastDiskRead);
559
566 this);
567}
568
569void
570store_client::readBody(const char * const buf, const ssize_t lastIoResult)
571{
572 Assure(flags.disk_io_pending);
573 flags.disk_io_pending = false;
576 debugs(90, 3, "got " << lastIoResult << " using " << *parsingBuffer);
577
578 if (lastIoResult < 0)
579 return fail();
580
581 if (!lastIoResult) {
582 if (answeredOnce())
583 return noteNews();
584
585 debugs(90, DBG_CRITICAL, "ERROR: Truncated HTTP headers in on-disk object");
586 return fail();
587 }
588
589 assert(lastDiskRead.data == buf);
590 lastDiskRead.length = lastIoResult;
591
592 parsingBuffer->appended(buf, lastIoResult);
593
594 // we know swap_hdr_sz by now and were reading beyond swap metadata because
595 // readHead() would have been called otherwise (to read swap metadata)
596 const auto swap_hdr_sz = entry->mem().swap_hdr_sz;
597 Assure(swap_hdr_sz > 0);
598 Assure(!Less(lastDiskRead.offset, swap_hdr_sz));
599
600 // Map lastDiskRead (i.e. the disk area we just read) to an HTTP reply part.
601 // The bytes are the same, but disk and HTTP offsets differ by swap_hdr_sz.
602 const auto httpOffset = lastDiskRead.offset - swap_hdr_sz;
603 const auto httpPart = StoreIOBuffer(lastDiskRead).positionAt(httpOffset);
604
607}
608
610void
612{
613 // We cannot de-serialize on-disk HTTP response without MemObject because
614 // without MemObject::swap_hdr_sz we cannot know where that response starts.
617
618 if (!answeredOnce()) {
619 // All on-disk responses have HTTP headers. First disk body read(s)
620 // include HTTP headers that we must parse (if needed) and skip.
621 const auto haveHttpHeaders = entry->mem_obj->baseReply().pstate == Http::Message::psParsed;
622 if (!haveHttpHeaders && !parseHttpHeadersFromDisk())
623 return;
625 }
626
627 noteNews();
628}
629
633void
635{
636 // XXX: Reject [memory-]uncachable/unshareable responses instead of assuming
637 // that an HTTP response should be written to MemObject's data_hdr (and that
638 // it may purge already cached entries) just because it "fits" and was
639 // loaded from disk. For example, this response may already be marked for
640 // release. The (complex) cachability decision(s) should be made outside
641 // (and obeyed by) this low-level code.
642 if (httpResponsePart.length && entry->mem_obj->inmem_lo == 0 && entry->objectLen() <= (int64_t)Config.Store.maxInMemObjSize && Config.onoff.memory_cache_disk) {
643 storeGetMemSpace(httpResponsePart.length);
644 // XXX: This "recheck" is not needed because storeGetMemSpace() cannot
645 // purge mem_hdr bytes of a locked entry, and we do lock ours. And
646 // inmem_lo offset itself should not be relevant to appending new bytes.
647 //
648 // recheck for the above call may purge entry's data from the memory cache
649 if (entry->mem_obj->inmem_lo == 0) {
650 // XXX: This code assumes a non-shared memory cache.
651 if (httpResponsePart.offset == entry->mem_obj->endOffset())
652 entry->mem_obj->write(httpResponsePart);
653 }
654 }
655}
656
657void
659{
660 debugs(90, 3, (object_ok ? "once" : "again"));
661 if (!object_ok)
662 return; // we failed earlier; nothing to do now
663
664 object_ok = false;
665
666 noteNews();
667}
668
670void
672{
673 /* synchronous open failures callback from the store,
674 * before startSwapin detects the failure.
675 * TODO: fix this inconsistent behaviour - probably by
676 * having storeSwapInStart become a callback functions,
677 * not synchronous
678 */
679
681 debugs(90, 5, "client lost interest");
682 return;
683 }
684
685 if (_callback.notifier) {
686 debugs(90, 5, "earlier news is being delivered by " << _callback.notifier);
687 return;
688 }
689
690 _callback.notifier = asyncCall(90, 4, "store_client::FinishCallback", cbdataDialer(store_client::FinishCallback, this));
692
694}
695
696static void
697storeClientReadHeader(void *data, const char *buf, ssize_t len, StoreIOState::Pointer)
698{
699 store_client *sc = (store_client *)data;
700 sc->readHeader(buf, len);
701}
702
703static void
704storeClientReadBody(void *data, const char *buf, ssize_t len, StoreIOState::Pointer)
705{
706 store_client *sc = (store_client *)data;
707 sc->readBody(buf, len);
708}
709
710void
711store_client::readHeader(char const *buf, ssize_t len)
712{
713 MemObject *const mem = entry->mem_obj;
714
715 assert(flags.disk_io_pending);
716 flags.disk_io_pending = false;
718
719 // abort if we fail()'d earlier
720 if (!object_ok)
721 return;
722
724 debugs(90, 3, "got " << len << " using " << *parsingBuffer);
725
726 if (len < 0)
727 return fail();
728
729 try {
730 Assure(!parsingBuffer->contentSize());
731 parsingBuffer->appended(buf, len);
733 parsingBuffer->consume(mem->swap_hdr_sz);
734 } catch (...) {
735 debugs(90, DBG_IMPORTANT, "ERROR: Failed to unpack Store entry metadata: " << CurrentException);
736 fail();
737 return;
738 }
739
742}
743
744/*
745 * This routine hasn't been optimised to take advantage of the
746 * passed sc. Yet.
747 */
748int
750{
751 MemObject *mem = e->mem_obj;
752#if STORE_CLIENT_LIST_DEBUG
753 assert(sc == storeClientListSearch(e->mem_obj, data));
754#else
755 (void)data;
756#endif
757
758 if (mem == nullptr)
759 return 0;
760
761 debugs(90, 3, "storeUnregister: called for '" << e->getMD5Text() << "'");
762
763 if (sc == nullptr) {
764 debugs(90, 3, "storeUnregister: No matching client for '" << e->getMD5Text() << "'");
765 return 0;
766 }
767
768 if (mem->clientCount() == 0) {
769 debugs(90, 3, "storeUnregister: Consistency failure - store client being unregistered is not in the mem object's list for '" << e->getMD5Text() << "'");
770 return 0;
771 }
772
773 dlinkDelete(&sc->node, &mem->clients);
774 -- mem->nclients;
775
776 const auto swapoutFinished = e->swappedOut() || e->swapoutFailed();
777 if (e->store_status == STORE_OK && !swapoutFinished)
778 e->swapOut();
779
780 if (sc->swapin_sio != nullptr) {
782 sc->swapin_sio = nullptr;
784 }
785
786 if (sc->_callback.callback_handler || sc->_callback.notifier) {
787 debugs(90, 3, "forgetting store_client callback for " << *e);
788 // Do not notify: Callers want to stop copying and forget about this
789 // pending copy request. Some would mishandle a notification from here.
790 if (sc->_callback.notifier)
791 sc->_callback.notifier->cancel("storeUnregister");
792 }
793
794#if STORE_CLIENT_LIST_DEBUG
795 cbdataReferenceDone(sc->owner);
796
797#endif
798
799 // We must lock to safely dereference e below, after deleting sc and after
800 // calling CheckQuickAbortIsReasonable().
801 e->lock("storeUnregister");
802
803 // XXX: We might be inside sc store_client method somewhere up the call
804 // stack. TODO: Convert store_client to AsyncJob to make destruction async.
805 delete sc;
806
808 e->abort();
809 else
810 mem->kickReads();
811
812#if USE_ADAPTATION
813 e->kickProducer();
814#endif
815
816 e->unlock("storeUnregister");
817 return 1;
818}
819
820/* Call handlers waiting for data to be appended to E. */
821void
823{
825 debugs(90, 3, "DELAY_SENDING is on, exiting " << *this);
826 return;
827 }
829 debugs(90, 3, "ENTRY_FWD_HDR_WAIT is on, exiting " << *this);
830 return;
831 }
832
833 /* Commit what we can to disk, if appropriate */
834 swapOut();
835 int i = 0;
837 dlink_node *nx = nullptr;
839
840 debugs(90, 3, mem_obj->nclients << " clients; " << *this << ' ' << getMD5Text());
841 /* walk the entire list looking for valid callbacks */
842
843 const auto savedContext = CodeContext::Current();
844 for (node = mem_obj->clients.head; node; node = nx) {
845 sc = (store_client *)node->data;
846 nx = node->next;
847 ++i;
848
849 if (!sc->_callback.pending())
850 continue;
851
852 if (sc->flags.disk_io_pending)
853 continue;
854
855 if (sc->flags.store_copying)
856 continue;
857
858 // XXX: If invokeHandlers() is (indirectly) called from a store_client
859 // method, then the above three conditions may not be sufficient to
860 // prevent us from reentering the same store_client object! This
861 // probably does not happen in the current code, but no observed
862 // invariant prevents this from (accidentally) happening in the future.
863
864 // TODO: Convert store_client into AsyncJob; make this call asynchronous
865 CodeContext::Reset(sc->_callback.codeContext);
866 debugs(90, 3, "checking client #" << i);
867 storeClientCopy2(this, sc);
868 }
869 CodeContext::Reset(savedContext);
870}
871
872// Does not account for remote readers/clients.
873int
875{
876 MemObject *mem = e->mem_obj;
877 int npend = nullptr == mem ? 0 : mem->nclients;
878 debugs(90, 3, "storePendingNClients: returning " << npend);
879 return npend;
880}
881
882/* return true if the request should be aborted */
883static bool
885{
886 assert(entry);
887 debugs(90, 3, "entry=" << *entry);
888
889 if (storePendingNClients(entry) > 0) {
890 debugs(90, 3, "quick-abort? NO storePendingNClients() > 0");
891 return false;
892 }
893
894 if (Store::Root().transientReaders(*entry)) {
895 debugs(90, 3, "quick-abort? NO still have one or more transient readers");
896 return false;
897 }
898
899 if (entry->store_status != STORE_PENDING) {
900 debugs(90, 3, "quick-abort? NO store_status != STORE_PENDING");
901 return false;
902 }
903
904 if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) {
905 debugs(90, 3, "quick-abort? NO ENTRY_SPECIAL");
906 return false;
907 }
908
909 if (shutting_down) {
910 debugs(90, 3, "quick-abort? YES avoid heavy optional work during shutdown");
911 return true;
912 }
913
914 MemObject * const mem = entry->mem_obj;
915 assert(mem);
916 debugs(90, 3, "mem=" << mem);
917
918 if (mem->request && !mem->request->flags.cachable) {
919 debugs(90, 3, "quick-abort? YES !mem->request->flags.cachable");
920 return true;
921 }
922
923 if (EBIT_TEST(entry->flags, KEY_PRIVATE)) {
924 debugs(90, 3, "quick-abort? YES KEY_PRIVATE");
925 return true;
926 }
927
928 const auto &reply = mem->baseReply();
929
930 if (reply.hdr_sz <= 0) {
931 // TODO: Check whether this condition works for HTTP/0 responses.
932 debugs(90, 3, "quick-abort? YES no object data received yet");
933 return true;
934 }
935
936 if (Config.quickAbort.min < 0) {
937 debugs(90, 3, "quick-abort? NO disabled");
938 return false;
939 }
940
941 if (mem->request && mem->request->range && mem->request->getRangeOffsetLimit() < 0) {
942 // the admin has configured "range_offset_limit none"
943 debugs(90, 3, "quick-abort? NO admin configured range replies to full-download");
944 return false;
945 }
946
947 if (reply.content_length < 0) {
948 // XXX: cf.data.pre does not document what should happen in this case
949 // We know that quick_abort is enabled, but no limit can be applied.
950 debugs(90, 3, "quick-abort? YES unknown content length");
951 return true;
952 }
953 const auto expectlen = reply.hdr_sz + reply.content_length;
954
955 int64_t curlen = mem->endOffset();
956
957 if (curlen > expectlen) {
958 debugs(90, 3, "quick-abort? YES bad content length (" << curlen << " of " << expectlen << " bytes received)");
959 return true;
960 }
961
962 if ((expectlen - curlen) < (Config.quickAbort.min << 10)) {
963 debugs(90, 3, "quick-abort? NO only a little more object left to receive");
964 return false;
965 }
966
967 if ((expectlen - curlen) > (Config.quickAbort.max << 10)) {
968 debugs(90, 3, "quick-abort? YES too much left to go");
969 return true;
970 }
971
972 // XXX: This is absurd! TODO: For positives, "a/(b/c) > d" is "a*c > b*d".
973 if (expectlen < 100) {
974 debugs(90, 3, "quick-abort? NO avoid FPE");
975 return false;
976 }
977
978 if ((curlen / (expectlen / 100)) > (Config.quickAbort.pct)) {
979 debugs(90, 3, "quick-abort? NO past point of no return");
980 return false;
981 }
982
983 debugs(90, 3, "quick-abort? YES default");
984 return true;
985}
986
990bool
992{
993 try {
994 return tryParsingHttpHeaders();
995 } catch (...) {
996 // XXX: Our parser enforces Config.maxReplyHeaderSize limit, but our
997 // packer does not. Since packing might increase header size, we may
998 // cache a header that we cannot parse and get here. Same for MemStore.
999 debugs(90, DBG_CRITICAL, "ERROR: Cannot parse on-disk HTTP headers" <<
1000 Debug::Extra << "exception: " << CurrentException <<
1001 Debug::Extra << "raw input size: " << parsingBuffer->contentSize() << " bytes" <<
1002 Debug::Extra << "current buffer capacity: " << parsingBuffer->capacity() << " bytes");
1003 fail();
1004 return false;
1005 }
1006}
1007
1010bool
1012{
1014 Assure(!copyInto.offset); // otherwise, parsingBuffer cannot have HTTP response headers
1015 auto &adjustableReply = entry->mem().adjustableBaseReply();
1016 if (adjustableReply.parseTerminatedPrefix(parsingBuffer->c_str(), parsingBuffer->contentSize()))
1017 return true;
1018
1019 // TODO: Optimize by checking memory as well. For simplicity sake, we
1020 // continue on the disk-reading path, but readFromMemory() can give us the
1021 // missing header bytes immediately if a concurrent request put those bytes
1022 // into memory while we were waiting for our disk response.
1024 return false;
1025}
1026
1028void
1030{
1031 const auto hdr_sz = entry->mem_obj->baseReply().hdr_sz;
1032 Assure(hdr_sz > 0); // all on-disk responses have HTTP headers
1033 if (Less(parsingBuffer->contentSize(), hdr_sz)) {
1034 debugs(90, 5, "discovered " << hdr_sz << "-byte HTTP headers in memory after reading some of them from disk: " << *parsingBuffer);
1035 parsingBuffer->consume(parsingBuffer->contentSize()); // skip loaded HTTP header prefix
1036 } else {
1037 parsingBuffer->consume(hdr_sz); // skip loaded HTTP headers
1038 const auto httpBodyBytesAfterHeader = parsingBuffer->contentSize(); // may be zero
1039 Assure(httpBodyBytesAfterHeader <= copyInto.length);
1040 debugs(90, 5, "read HTTP body prefix: " << httpBodyBytesAfterHeader);
1041 }
1042}
1043
1044void
1045store_client::dumpStats(MemBuf * output, int clientNumber) const
1046{
1047 if (_callback.pending())
1048 return;
1049
1050 output->appendf("\tClient #%d, %p\n", clientNumber, this);
1051 output->appendf("\t\tcopy_offset: %" PRId64 "\n", copyInto.offset);
1052 output->appendf("\t\tcopy_size: %zu\n", copyInto.length);
1053 output->append("\t\tflags:", 8);
1054
1055 if (flags.disk_io_pending)
1056 output->append(" disk_io_pending", 16);
1057
1058 if (flags.store_copying)
1059 output->append(" store_copying", 14);
1060
1061 if (_callback.notifier)
1062 output->append(" notifying", 10);
1063
1064 output->append("\n",1);
1065}
1066
1067bool
1069{
1070 return callback_handler && !notifier;
1071}
1072
1074 callback_handler(function),
1075 cbData(data),
1076 codeContext(CodeContext::Current())
1077{
1078}
1079
1080#if USE_DELAY_POOLS
1081int
1083{
1084 // TODO: To avoid using stale copyInto, return zero if !_callback.pending()?
1086}
1087
1088void
1090{
1091 delayId = delay_id;
1092}
1093#endif
1094
#define Assure(condition)
Definition: Assure.h:35
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
RefCount< AsyncCallT< Dialer > > asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
UnaryCbdataDialer< Argument1 > cbdataDialer(typename UnaryCbdataDialer< Argument1 >::Handler *handler, Argument1 *arg1)
class SquidConfig Config
Definition: SquidConfig.cc:12
constexpr bool Less(const A a, const B b)
whether integer a is less than integer b, with correct overflow handling
Definition: SquidMath.h:48
StatCounters statCounter
Definition: StatCounters.cc:12
void(void *, StoreIOBuffer) STCB
Definition: StoreClient.h:32
int storeClientIsThisAClient(store_client *sc, void *someClient)
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
void error(char *format,...)
#define assert(EX)
Definition: assert.h:17
#define cbdataReferenceDone(var)
Definition: cbdata.h:352
#define cbdataReference(var)
Definition: cbdata.h:343
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:320
Acl::Answer const & fastCheck()
Definition: Checklist.cc:332
bool allowed() const
Definition: Acl.h:156
virtual void fillChecklist(ACLFilledChecklist &) const =0
configure the given checklist (to reflect the current transaction state)
void * validDone()
Definition: cbdata.h:391
static const Pointer & Current()
Definition: CodeContext.cc:33
static void Reset()
forgets the current context, setting it to nil/unknown
Definition: CodeContext.cc:77
static std::ostream & Extra(std::ostream &)
Definition: debug.cc:1313
int bytesWanted(int min, int max) const
Definition: DelayId.cc:129
HttpHdrRange * range
Definition: HttpRequest.h:143
int64_t getRangeOffsetLimit()
Definition: HttpRequest.cc:594
RequestFlags flags
Definition: HttpRequest.h:141
int hdr_sz
Definition: Message.h:81
ParseState pstate
the current parsing state
Definition: Message.h:94
Definition: MemBuf.h:24
void append(const char *c, int sz) override
Definition: MemBuf.cc:209
StoreIOState::Pointer sio
Definition: MemObject.h:162
size_t clientCount() const
Definition: MemObject.h:152
int nclients
Definition: MemObject.h:156
SwapOut swapout
Definition: MemObject.h:169
HttpRequestPointer request
Definition: MemObject.h:212
void addClient(store_client *)
Definition: MemObject.cc:303
dlink_list clients
Definition: MemObject.h:150
mem_hdr data_hdr
Definition: MemObject.h:148
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:136
int64_t inmem_lo
Definition: MemObject.h:149
size_t swap_hdr_sz
Definition: MemObject.h:223
int64_t endOffset() const
Definition: MemObject.cc:214
const HttpReply & baseReply() const
Definition: MemObject.h:60
HttpReply & adjustableBaseReply()
Definition: MemObject.cc:121
void kickReads()
Definition: MemObject.cc:459
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
SupportOrVeto cachable
whether the response may be stored in the cache
Definition: RequestFlags.h:35
struct SquidConfig::@107 accessList
struct SquidConfig::@106 onoff
struct SquidConfig::@92 quickAbort
acl_access * collapsedForwardingAccess
Definition: SquidConfig.h:406
int memory_cache_disk
Definition: SquidConfig.h:336
size_t maxInMemObjSize
Definition: SquidConfig.h:268
struct SquidConfig::@104 Store
int collapsed_forwarding
Definition: SquidConfig.h:323
int64_t min
Definition: SquidConfig.h:94
int64_t max
Definition: SquidConfig.h:96
struct StatCounters::@131 swap
bool onCollapsingPath() const
whether Squid configuration allows collapsing for this transaction
Definition: store_client.cc:52
virtual LogTags * loggingTags() const =0
bool startCollapsingOn(const StoreEntry &, const bool doingRevalidation) const
Definition: store_client.cc:66
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:136
uint16_t flags
Definition: Store.h:232
void invokeHandlers()
MemObject & mem()
Definition: Store.h:51
int unlock(const char *context)
Definition: store.cc:455
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1915
void lock(const char *context)
Definition: store.cc:431
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:134
const char * getMD5Text() const
Definition: store.cc:206
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:362
MemObject * mem_obj
Definition: Store.h:221
void abort()
Definition: store.cc:1063
int64_t objectLen() const
Definition: Store.h:257
store_status_t store_status
Definition: Store.h:244
int64_t contentLen() const
Definition: Store.h:258
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:138
uint16_t refcount
Definition: Store.h:231
void swapOut()
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:216
int64_t offset
Definition: StoreIOBuffer.h:58
StoreIOBuffer & positionAt(const int64_t newOffset)
convenience method for changing the offset of a being-configured buffer
Definition: StoreIOBuffer.h:47
@ readerDone
success or failure: either way, stop swapping in
Definition: StoreIOState.h:60
off_t offset() const
Definition: StoreIOState.h:48
void STRCB(void *their_data, const char *buf, ssize_t len, StoreIOState::Pointer self)
Definition: StoreIOState.h:29
Definition: cbdata.cc:38
ssize_t copy(StoreIOBuffer const &) const
Definition: stmem.cc:189
int getType() const
Definition: store_client.cc:90
bool sendingHttpHeaders() const
void readHeader(const char *buf, ssize_t len)
void skipHttpHeadersFromDisk()
skips HTTP header bytes previously loaded from disk
void handleBodyFromDisk()
de-serializes HTTP response (partially) read from disk storage
void setDelayId(DelayId delay_id)
struct store_client::@140 flags
std::optional< Store::ParsingBuffer > parsingBuffer
Definition: StoreClient.h:187
void scheduleDiskRead()
bool startSwapin()
opens the swapin "file" if possible; otherwise, fail()s and returns false
bool answeredOnce() const
Definition: StoreClient.h:147
void noteNews()
if necessary and possible, informs the Store reader about copy() result
bool parseHttpHeadersFromDisk()
void readFromMemory()
bool moreToRead() const
Whether Store has (or possibly will have) more entry data for us.
void noteSwapInDone(bool error)
StoreIOBuffer lastDiskRead
buffer used for the last storeRead() call
Definition: StoreClient.h:189
void readBody(const char *buf, ssize_t len)
DelayId delayId
Definition: StoreClient.h:131
int64_t nextHttpReadOffset() const
The offset of the next stored HTTP response byte wanted by the client.
StoreEntry * entry
Definition: StoreClient.h:116
void copy(StoreEntry *, StoreIOBuffer, STCB *, void *)
int bytesWanted() const
bool tryParsingHttpHeaders()
void doCopy(StoreEntry *e)
void maybeWriteFromDiskToMemory(const StoreIOBuffer &)
bool canReadFromMemory() const
whether at least one byte wanted by the client is in memory
struct store_client::Callback _callback
StoreIOBuffer copyInto
Definition: StoreClient.h:178
void finishCallback()
finishes a copy()-STCB sequence by synchronously calling STCB
store_client(StoreEntry *)
static void FinishCallback(store_client *)
finishCallback() wrapper; TODO: Add NullaryMemFunT for non-jobs.
uint64_t answers
the total number of finishCallback() calls
Definition: StoreClient.h:181
StoreIOState::Pointer swapin_sio
Definition: StoreClient.h:117
void dumpStats(MemBuf *output, int clientNumber) const
A const & min(A const &lhs, A const &rhs)
#define DBG_IMPORTANT
Definition: Stream.h:38
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:194
#define DBG_CRITICAL
Definition: Stream.h:37
#define SM_PAGE_SIZE
Definition: defines.h:65
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
@ ENTRY_SPECIAL
Definition: enums.h:84
@ KEY_PRIVATE
Definition: enums.h:102
@ ENTRY_FWD_HDR_WAIT
Definition: enums.h:111
@ DELAY_SENDING
Definition: enums.h:97
@ ENTRY_ABORTED
Definition: enums.h:115
@ STORE_DISK_CLIENT
Definition: enums.h:74
@ STORE_PENDING
Definition: enums.h:51
@ STORE_OK
Definition: enums.h:50
int shutting_down
Controller & Root()
safely access controller singleton
Definition: Controller.cc:938
void UnpackHitSwapMeta(char const *, ssize_t, StoreEntry &)
deserializes entry metadata from the given buffer into the cache hit entry
Definition: SwapMetaIn.cc:310
static int sc[16]
Definition: smbdes.c:121
void storeGetMemSpace(int size)
Definition: store.cc:1107
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:875
int storeUnregister(store_client *sc, StoreEntry *e, void *data)
void storeClientCopy(store_client *sc, StoreEntry *e, StoreIOBuffer copyInto, STCB *callback, void *data)
int storePendingNClients(const StoreEntry *e)
static void storeClientCopy2(StoreEntry *e, store_client *sc)
store_client * storeClientListAdd(StoreEntry *e, void *data)
static bool CheckQuickAbortIsReasonable(StoreEntry *entry)
static StoreIOState::STRCB storeClientReadBody
Definition: store_client.cc:42
static StoreIOState::STRCB storeClientReadHeader
Definition: store_client.cc:43
void storeClose(StoreIOState::Pointer sio, int how)
Definition: store_io.cc:65
void storeRead(StoreIOState::Pointer sio, char *buf, size_t size, off_t offset, StoreIOState::STRCB *callback, void *callback_data)
Definition: store_io.cc:79
void storeSwapInStart(store_client *sc)
Definition: store_swapin.cc:21
Definition: parse.c:104
struct node * next
Definition: parse.c:105
CallbackData cbData
the first STCB callback parameter
Definition: StoreClient.h:206
STCB * callback_handler
where to deliver the answer
Definition: StoreClient.h:205
AsyncCall::Pointer notifier
a scheduled asynchronous finishCallback() call (or nil)
Definition: StoreClient.h:210
#define NULL
Definition: types.h:145
#define PRId64
Definition: types.h:104

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors