store.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 20 Storage Manager */
10 
11 #include "squid.h"
12 #include "base/AsyncCbdataCalls.h"
13 #include "base/PackableStream.h"
14 #include "base/TextException.h"
15 #include "CacheDigest.h"
16 #include "CacheManager.h"
17 #include "CollapsedForwarding.h"
18 #include "comm/Connection.h"
19 #include "comm/Read.h"
20 #include "DebugMessages.h"
21 #if HAVE_DISKIO_MODULE_IPCIO
22 #include "DiskIO/IpcIo/IpcIoFile.h"
23 #endif
24 #include "ETag.h"
25 #include "event.h"
26 #include "fde.h"
27 #include "globals.h"
28 #include "http.h"
29 #include "HttpReply.h"
30 #include "HttpRequest.h"
31 #include "mem_node.h"
32 #include "MemObject.h"
33 #include "MemStore.h"
34 #include "mgr/Registration.h"
35 #include "mgr/StoreIoAction.h"
36 #include "profiler/Profiler.h"
37 #include "repl_modules.h"
38 #include "RequestFlags.h"
39 #include "SquidConfig.h"
40 #include "SquidTime.h"
41 #include "StatCounters.h"
42 #include "stmem.h"
43 #include "Store.h"
44 #include "store/Controller.h"
45 #include "store/Disk.h"
46 #include "store/Disks.h"
47 #include "store_digest.h"
48 #include "store_key_md5.h"
49 #include "store_log.h"
50 #include "store_rebuild.h"
51 #include "StoreClient.h"
52 #include "StoreIOState.h"
53 #include "StoreMeta.h"
54 #include "StrList.h"
55 #include "swap_log_op.h"
56 #include "tools.h"
57 #if USE_DELAY_POOLS
58 #include "DelayPools.h"
59 #endif
60 
64 #include "mem/Pool.h"
65 
66 #include <climits>
67 #include <stack>
68 
69 #define REBUILD_TIMESTAMP_DELTA_MAX 2
70 
71 #define STORE_IN_MEM_BUCKETS (229)
72 
73 // TODO: Convert these string constants to enum string-arrays generated
74 
75 const char *memStatusStr[] = {
76  "NOT_IN_MEMORY",
77  "IN_MEMORY"
78 };
79 
80 const char *pingStatusStr[] = {
81  "PING_NONE",
82  "PING_WAITING",
83  "PING_DONE"
84 };
85 
86 const char *storeStatusStr[] = {
87  "STORE_OK",
88  "STORE_PENDING"
89 };
90 
91 const char *swapStatusStr[] = {
92  "SWAPOUT_NONE",
93  "SWAPOUT_WRITING",
94  "SWAPOUT_DONE",
95  "SWAPOUT_FAILED"
96 };
97 
98 /*
99  * This defines an repl type
100  */
101 
102 typedef struct _storerepl_entry storerepl_entry_t;
103 
105  const char *typestr;
107 };
108 
110 
111 /*
112  * local function prototypes
113  */
114 static int getKeyCounter(void);
117 
118 /*
119  * local variables
120  */
121 static std::stack<StoreEntry*> LateReleaseStack;
123 
124 void
126 {
127  assert(output);
128  Root().stat(*output);
129 }
130 
132 static void
134 {
135  assert(e);
136  PackableStream stream(*e);
138 #if HAVE_DISKIO_MODULE_IPCIO
139  stream << "\n";
140  IpcIoFile::StatQueue(stream);
141 #endif
142  stream.flush();
143 }
144 
145 // XXX: new/delete operators need to be replaced with MEMPROXY_CLASS
146 // definitions but doing so exposes bug 4370, and maybe 4354 and 4355
147 void *
148 StoreEntry::operator new (size_t bytecount)
149 {
150  assert(bytecount == sizeof (StoreEntry));
151 
152  if (!pool) {
153  pool = memPoolCreate ("StoreEntry", bytecount);
154  }
155 
156  return pool->alloc();
157 }
158 
159 void
160 StoreEntry::operator delete (void *address)
161 {
162  pool->freeOne(address);
163 }
164 
165 bool
167 {
168  /* This object can be cached for a long time */
169  return !EBIT_TEST(flags, RELEASE_REQUEST) && setPublicKey(scope);
170 }
171 
172 void
173 StoreEntry::makePrivate(const bool shareable)
174 {
175  releaseRequest(shareable); /* delete object when not used */
176 }
177 
178 void
180 {
183  shareableWhenPrivate = false;
184 }
185 
186 bool
188 {
189  /* This object may be negatively cached */
190  if (makePublic()) {
191  negativeCache();
192  return true;
193  }
194  return false;
195 }
196 
197 size_t
199 {
200  if (!pool)
201  return 0;
202  return pool->getInUseCount();
203 }
204 
205 const char *
207 {
208  return storeKeyText((const cache_key *)key);
209 }
210 
211 #include "comm.h"
212 
213 void
214 StoreEntry::DeferReader(void *theContext, CommRead const &aRead)
215 {
216  StoreEntry *anEntry = (StoreEntry *)theContext;
217  anEntry->delayAwareRead(aRead.conn,
218  aRead.buf,
219  aRead.len,
220  aRead.callback);
221 }
222 
223 void
225 {
226  size_t amountToRead = bytesWanted(Range<size_t>(0, len));
227  /* sketch: readdeferer* = getdeferer.
228  * ->deferRead (fd, buf, len, callback, DelayAwareRead, this)
229  */
230 
231  if (amountToRead <= 0) {
232  assert (mem_obj);
233  mem_obj->delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
234  return;
235  }
236 
237  if (fd_table[conn->fd].closing()) {
238  // Readers must have closing callbacks if they want to be notified. No
239  // readers appeared to care around 2009/12/14 as they skipped reading
240  // for other reasons. Closing may already be true at the delyaAwareRead
241  // call time or may happen while we wait after delayRead() above.
242  debugs(20, 3, "will not read from closing " << conn << " for " << callback);
243  return; // the read callback will never be called
244  }
245 
246  comm_read(conn, buf, amountToRead, callback);
247 }
248 
249 size_t
250 StoreEntry::bytesWanted (Range<size_t> const aRange, bool ignoreDelayPools) const
251 {
252  if (mem_obj == NULL)
253  return aRange.end;
254 
255 #if URL_CHECKSUM_DEBUG
256 
257  mem_obj->checkUrlChecksum();
258 
259 #endif
260 
262  return 0;
263 
264  return mem_obj->mostBytesWanted(aRange.end, ignoreDelayPools);
265 }
266 
267 bool
269 {
270  return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0);
271 }
272 
273 void
274 StoreEntry::setNoDelay(bool const newValue)
275 {
276  if (mem_obj)
277  mem_obj->setNoDelay(newValue);
278 }
279 
280 // XXX: Type names mislead. STORE_DISK_CLIENT actually means that we should
281 // open swapin file, aggressively trim memory, and ignore read-ahead gap.
282 // It does not mean we will read from disk exclusively (or at all!).
283 // STORE_MEM_CLIENT covers all other cases, including in-memory entries,
284 // newly created entries, and entries not backed by disk or memory cache.
285 // XXX: May create STORE_DISK_CLIENT with no disk caching configured.
286 // XXX: Collapsed clients cannot predict their type.
289 {
290  /* The needed offset isn't in memory
291  * XXX TODO: this is wrong for range requests
292  * as the needed offset may *not* be 0, AND
293  * offset 0 in the memory object is the HTTP headers.
294  */
295 
296  assert(mem_obj);
297 
298  if (mem_obj->inmem_lo)
299  return STORE_DISK_CLIENT;
300 
301  if (EBIT_TEST(flags, ENTRY_ABORTED)) {
302  /* I don't think we should be adding clients to aborted entries */
303  debugs(20, DBG_IMPORTANT, "storeClientType: adding to ENTRY_ABORTED entry");
304  return STORE_MEM_CLIENT;
305  }
306 
307  if (swapoutFailed())
308  return STORE_MEM_CLIENT;
309 
310  if (store_status == STORE_OK) {
311  /* the object has completed. */
312 
313  if (mem_obj->inmem_lo == 0 && !isEmpty()) {
314  if (swappedOut()) {
315  debugs(20,7, HERE << mem_obj << " lo: " << mem_obj->inmem_lo << " hi: " << mem_obj->endOffset() << " size: " << mem_obj->object_sz);
316  if (mem_obj->endOffset() == mem_obj->object_sz) {
317  /* hot object fully swapped in (XXX: or swapped out?) */
318  return STORE_MEM_CLIENT;
319  }
320  } else {
321  /* Memory-only, or currently being swapped out */
322  return STORE_MEM_CLIENT;
323  }
324  }
325  return STORE_DISK_CLIENT;
326  }
327 
328  /* here and past, entry is STORE_PENDING */
329  /*
330  * If this is the first client, let it be the mem client
331  */
332  if (mem_obj->nclients == 1)
333  return STORE_MEM_CLIENT;
334 
335  /*
336  * If there is no disk file to open yet, we must make this a
337  * mem client. If we can't open the swapin file before writing
338  * to the client, there is no guarantee that we will be able
339  * to open it later when we really need it.
340  */
341  if (swap_status == SWAPOUT_NONE)
343 
344  /*
345  * otherwise, make subsequent clients read from disk so they
346  * can not delay the first, and vice-versa.
347  */
348  return STORE_DISK_CLIENT;
349 }
350 
352  mem_obj(NULL),
353  timestamp(-1),
354  lastref(-1),
355  expires(-1),
356  lastModified_(-1),
357  swap_file_sz(0),
358  refcount(0),
359  flags(0),
360  swap_filen(-1),
361  swap_dirn(-1),
362  mem_status(NOT_IN_MEMORY),
363  ping_status(PING_NONE),
364  store_status(STORE_PENDING),
365  swap_status(SWAPOUT_NONE),
366  lock_count(0),
367  shareableWhenPrivate(false)
368 {
369  debugs(20, 5, "StoreEntry constructed, this=" << this);
370 }
371 
373 {
374  debugs(20, 5, "StoreEntry destructed, this=" << this);
375 }
376 
377 #if USE_ADAPTATION
378 void
380 {
381  if (!deferredProducer)
382  deferredProducer = producer;
383  else
384  debugs(20, 5, "Deferred producer call is already set to: " <<
385  *deferredProducer << ", requested call: " << *producer);
386 }
387 
388 void
390 {
391  if (deferredProducer != NULL) {
394  }
395 }
396 #endif
397 
398 void
400 {
401  debugs(20, 3, mem_obj << " in " << *this);
402 
403  // Store::Root() is FATALly missing during shutdown
404  if (hasTransients() && !shutting_down)
406  if (hasMemStore() && !shutting_down)
407  Store::Root().memoryDisconnect(*this);
408 
409  if (auto memObj = mem_obj) {
411  mem_obj = NULL;
412  delete memObj;
413  }
414 }
415 
416 void
417 destroyStoreEntry(void *data)
418 {
419  debugs(20, 3, HERE << "destroyStoreEntry: destroying " << data);
420  StoreEntry *e = static_cast<StoreEntry *>(static_cast<hash_link *>(data));
421  assert(e != NULL);
422 
423  // Store::Root() is FATALly missing during shutdown
424  if (e->hasDisk() && !shutting_down)
425  e->disk().disconnect(*e);
426 
427  e->destroyMemObject();
428 
429  e->hashDelete();
430 
431  assert(e->key == NULL);
432 
433  delete e;
434 }
435 
436 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
437 
438 void
440 {
441  debugs(20, 3, "StoreEntry::hashInsert: Inserting Entry " << *this << " key '" << storeKeyText(someKey) << "'");
442  assert(!key);
443  key = storeKeyDup(someKey);
444  hash_join(store_table, this);
445 }
446 
447 void
449 {
450  if (key) { // some test cases do not create keys and do not hashInsert()
452  storeKeyFree((const cache_key *)key);
453  key = NULL;
454  }
455 }
456 
457 /* -------------------------------------------------------------------------- */
458 
459 void
460 StoreEntry::lock(const char *context)
461 {
462  ++lock_count;
463  debugs(20, 3, context << " locked key " << getMD5Text() << ' ' << *this);
464 }
465 
466 void
468 {
470 }
471 
472 void
473 StoreEntry::releaseRequest(const bool shareable)
474 {
475  debugs(20, 3, shareable << ' ' << *this);
476  if (!shareable)
477  shareableWhenPrivate = false; // may already be false
479  return;
480  setPrivateKey(shareable, true);
481 }
482 
483 int
484 StoreEntry::unlock(const char *context)
485 {
486  debugs(20, 3, (context ? context : "somebody") <<
487  " unlocking key " << getMD5Text() << ' ' << *this);
488  assert(lock_count > 0);
489  --lock_count;
490 
491  if (lock_count)
492  return (int) lock_count;
493 
494  abandon(context);
495  return 0;
496 }
497 
500 void
501 StoreEntry::doAbandon(const char *context)
502 {
503  debugs(20, 5, *this << " via " << (context ? context : "somebody"));
504  assert(!locked());
505  assert(storePendingNClients(this) == 0);
506 
507  // Both aborted local writers and aborted local readers (of remote writers)
508  // are STORE_PENDING, but aborted readers should never release().
510  (store_status == STORE_PENDING && !Store::Root().transientsReader(*this))) {
511  this->release();
512  return;
513  }
514 
515  Store::Root().handleIdleEntry(*this); // may delete us
516 }
517 
518 StoreEntry *
519 storeGetPublic(const char *uri, const HttpRequestMethod& method)
520 {
521  return Store::Root().find(storeKeyPublic(uri, method));
522 }
523 
524 StoreEntry *
526 {
527  return Store::Root().find(storeKeyPublicByRequestMethod(req, method, keyScope));
528 }
529 
530 StoreEntry *
532 {
533  StoreEntry *e = storeGetPublicByRequestMethod(req, req->method, keyScope);
534 
535  if (e == NULL && req->method == Http::METHOD_HEAD)
536  /* We can generate a HEAD reply from a cached GET object */
538 
539  return e;
540 }
541 
542 static int
544 {
545  static int key_counter = 0;
546 
547  if (++key_counter < 0)
548  key_counter = 1;
549 
550  return key_counter;
551 }
552 
553 /* RBC 20050104 AFAICT this should become simpler:
554  * rather than reinserting with a special key it should be marked
555  * as 'released' and then cleaned up when refcounting indicates.
556  * the StoreHashIndex could well implement its 'released' in the
557  * current manner.
558  * Also, clean log writing should skip over ia,t
559  * Otherwise, we need a 'remove from the index but not the store
560  * concept'.
561  */
562 void
563 StoreEntry::setPrivateKey(const bool shareable, const bool permanent)
564 {
565  debugs(20, 3, shareable << permanent << ' ' << *this);
566  if (permanent)
567  EBIT_SET(flags, RELEASE_REQUEST); // may already be set
568  if (!shareable)
569  shareableWhenPrivate = false; // may already be false
570 
572  return;
573 
574  if (key) {
575  Store::Root().evictCached(*this); // all caches/workers will know
576  hashDelete();
577  }
578 
579  if (mem_obj && mem_obj->hasUris())
580  mem_obj->id = getKeyCounter();
581  const cache_key *newkey = storeKeyPrivate();
582 
583  assert(hash_lookup(store_table, newkey) == NULL);
585  shareableWhenPrivate = shareable;
586  hashInsert(newkey);
587 }
588 
589 bool
591 {
592  debugs(20, 3, *this);
593  if (key && !EBIT_TEST(flags, KEY_PRIVATE))
594  return true; // already public
595 
596  assert(mem_obj);
597 
598  /*
599  * We can't make RELEASE_REQUEST objects public. Depending on
600  * when RELEASE_REQUEST gets set, we might not be swapping out
601  * the object. If we're not swapping out, then subsequent
602  * store clients won't be able to access object data which has
603  * been freed from memory.
604  *
605  * If RELEASE_REQUEST is set, setPublicKey() should not be called.
606  */
607 
609 
610  try {
611  EntryGuard newVaryMarker(adjustVary(), "setPublicKey+failure");
612  const cache_key *pubKey = calcPublicKey(scope);
613  Store::Root().addWriting(this, pubKey);
614  forcePublicKey(pubKey);
615  newVaryMarker.unlockAndReset("setPublicKey+success");
616  return true;
617  } catch (const std::exception &ex) {
618  debugs(20, 2, "for " << *this << " failed: " << ex.what());
619  }
620  return false;
621 }
622 
623 void
625 {
626  if (!key || EBIT_TEST(flags, KEY_PRIVATE))
627  return; // probably the old public key was deleted or made private
628 
629  // TODO: adjustVary() when collapsed revalidation supports that
630 
631  const cache_key *newKey = calcPublicKey(ksDefault);
632  if (!storeKeyHashCmp(key, newKey))
633  return; // probably another collapsed revalidation beat us to this change
634 
635  forcePublicKey(newKey);
636 }
637 
640 void
642 {
643  debugs(20, 3, storeKeyText(newkey) << " for " << *this);
644  assert(mem_obj);
645 
646  if (StoreEntry *e2 = (StoreEntry *)hash_lookup(store_table, newkey)) {
647  assert(e2 != this);
648  debugs(20, 3, "releasing clashing " << *e2);
649  e2->release(true);
650  }
651 
652  if (key)
653  hashDelete();
654 
655  clearPrivate();
656 
657  assert(mem_obj->hasUris());
658  hashInsert(newkey);
659 
660  if (hasDisk())
662 }
663 
666 const cache_key *
668 {
669  assert(mem_obj);
670  return mem_obj->request ? storeKeyPublicByRequest(mem_obj->request.getRaw(), keyScope) :
671  storeKeyPublic(mem_obj->storeId(), mem_obj->method, keyScope);
672 }
673 
679 StoreEntry *
681 {
682  assert(mem_obj);
683 
684  if (!mem_obj->request)
685  return nullptr;
686 
688  const auto &reply = mem_obj->freshestReply();
689 
690  if (mem_obj->vary_headers.isEmpty()) {
691  /* First handle the case where the object no longer varies */
692  request->vary_headers.clear();
693  } else {
694  if (!request->vary_headers.isEmpty() && request->vary_headers.cmp(mem_obj->vary_headers) != 0) {
695  /* Oops.. the variance has changed. Kill the base object
696  * to record the new variance key
697  */
698  request->vary_headers.clear(); /* free old "bad" variance key */
700  pe->release(true);
701  }
702 
703  /* Make sure the request knows the variance status */
704  if (request->vary_headers.isEmpty())
705  request->vary_headers = httpMakeVaryMark(request.getRaw(), &reply);
706  }
707 
708  // TODO: storeGetPublic() calls below may create unlocked entries.
709  // We should add/use storeHas() API or lock/unlock those entries.
711  /* Create "vary" base object */
713  // XXX: storeCreateEntry() already tries to make `pe` public under
714  // certain conditions. If those conditions do not apply to Vary markers,
715  // then refactor to call storeCreatePureEntry() above. Otherwise,
716  // refactor to simply check whether `pe` is already public below.
717  if (!pe->makePublic()) {
718  pe->unlock("StoreEntry::adjustVary+failed_makePublic");
719  throw TexcHere("failed to make Vary marker public");
720  }
721  /* We are allowed to do this typecast */
722  const HttpReplyPointer rep(new HttpReply);
723  rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
724  auto vary = reply.header.getList(Http::HdrType::VARY);
725 
726  if (vary.size()) {
727  /* Again, we own this structure layout */
728  rep->header.putStr(Http::HdrType::VARY, vary.termedBuf());
729  vary.clean();
730  }
731 
732 #if X_ACCELERATOR_VARY
733  vary = reply.header.getList(Http::HdrType::HDR_X_ACCELERATOR_VARY);
734 
735  if (vary.size() > 0) {
736  /* Again, we own this structure layout */
737  rep->header.putStr(Http::HdrType::HDR_X_ACCELERATOR_VARY, vary.termedBuf());
738  vary.clean();
739  }
740 
741 #endif
742  pe->replaceHttpReply(rep, false); // no write until timestampsSet()
743 
744  pe->timestampsSet();
745 
746  pe->startWriting(); // after timestampsSet()
747 
748  pe->completeSuccessfully("wrote the entire Vary marker object");
749 
750  return pe;
751  }
752  return nullptr;
753 }
754 
755 StoreEntry *
756 storeCreatePureEntry(const char *url, const char *log_url, const HttpRequestMethod& method)
757 {
758  StoreEntry *e = NULL;
759  debugs(20, 3, "storeCreateEntry: '" << url << "'");
760 
761  e = new StoreEntry();
762  e->createMemObject(url, log_url, method);
763 
765  e->refcount = 0;
766  e->lastref = squid_curtime;
767  e->timestamp = -1; /* set in StoreEntry::timestampsSet() */
768  e->ping_status = PING_NONE;
770  return e;
771 }
772 
773 StoreEntry *
774 storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod& method)
775 {
776  StoreEntry *e = storeCreatePureEntry(url, logUrl, method);
777  e->lock("storeCreateEntry");
778 
779  if (!neighbors_do_private_keys && flags.hierarchical && flags.cachable && e->setPublicKey())
780  return e;
781 
782  e->setPrivateKey(false, !flags.cachable);
783  return e;
784 }
785 
786 /* Mark object as expired */
787 void
789 {
790  debugs(20, 3, "StoreEntry::expireNow: '" << getMD5Text() << "'");
792 }
793 
794 void
796 {
797  assert(mem_obj != NULL);
798  /* This assert will change when we teach the store to update */
799  PROF_start(StoreEntry_write);
801 
802  // XXX: caller uses content offset, but we also store headers
803  writeBuffer.offset += mem_obj->baseReply().hdr_sz;
804 
805  debugs(20, 5, "storeWrite: writing " << writeBuffer.length << " bytes for '" << getMD5Text() << "'");
806  PROF_stop(StoreEntry_write);
807  storeGetMemSpace(writeBuffer.length);
808  mem_obj->write(writeBuffer);
809 
811  debugs(20, 3, "allow Store clients to get entry content after buffering too much for " << *this);
813  }
814 
815  invokeHandlers();
816 }
817 
818 /* Append incoming data from a primary server to an entry. */
819 void
820 StoreEntry::append(char const *buf, int len)
821 {
822  assert(mem_obj != NULL);
823  assert(len >= 0);
825 
826  StoreIOBuffer tempBuffer;
827  tempBuffer.data = (char *)buf;
828  tempBuffer.length = len;
829  /*
830  * XXX sigh, offset might be < 0 here, but it gets "corrected"
831  * later. This offset crap is such a mess.
832  */
833  tempBuffer.offset = mem_obj->endOffset() - mem_obj->baseReply().hdr_sz;
834  write(tempBuffer);
835 }
836 
837 void
838 StoreEntry::vappendf(const char *fmt, va_list vargs)
839 {
840  LOCAL_ARRAY(char, buf, 4096);
841  *buf = 0;
842  int x;
843 
844  va_list ap;
845  /* Fix of bug 753r. The value of vargs is undefined
846  * after vsnprintf() returns. Make a copy of vargs
847  * in case we loop around and call vsnprintf() again.
848  */
849  va_copy(ap,vargs);
850  errno = 0;
851  if ((x = vsnprintf(buf, sizeof(buf), fmt, ap)) < 0) {
852  fatal(xstrerr(errno));
853  return;
854  }
855  va_end(ap);
856 
857  if (x < static_cast<int>(sizeof(buf))) {
858  append(buf, x);
859  return;
860  }
861 
862  // okay, do it the slow way.
863  char *buf2 = new char[x+1];
864  int y = vsnprintf(buf2, x+1, fmt, vargs);
865  assert(y >= 0 && y == x);
866  append(buf2, y);
867  delete[] buf2;
868 }
869 
870 // deprecated. use StoreEntry::appendf() instead.
871 void
872 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
873 {
874  va_list args;
875  va_start(args, fmt);
876  e->vappendf(fmt, args);
877  va_end(args);
878 }
879 
880 // deprecated. use StoreEntry::appendf() instead.
881 void
882 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
883 {
884  e->vappendf(fmt, vargs);
885 }
886 
888 
889  struct {
892  int too_big;
898  } no;
899 
900  struct {
901  int Default;
902  } yes;
904 
905 int
907 {
908  if (Config.max_open_disk_fds == 0)
909  return 0;
910 
912  return 1;
913 
914  return 0;
915 }
916 
917 int
919 {
921  return 0;
922 
923  if (STORE_OK == store_status)
924  if (mem_obj->object_sz >= 0 &&
926  return 1;
927 
928  const auto clen = mem().baseReply().content_length;
929  if (clen >= 0 && clen < Config.Store.minObjectSize)
930  return 1;
931  return 0;
932 }
933 
934 bool
936 {
938  return true;
939 
940  const auto clen = mem_obj->baseReply().content_length;
941  return (clen >= 0 && clen > store_maxobjsize);
942 }
943 
944 // TODO: move "too many open..." checks outside -- we are called too early/late
945 bool
947 {
948  // XXX: This method is used for both memory and disk caches, but some
949  // checks are specific to disk caches. Move them to mayStartSwapOut().
950 
951  // XXX: This method may be called several times, sometimes with different
952  // outcomes, making store_check_cachable_hist counters misleading.
953 
954  // check this first to optimize handling of repeated calls for uncachables
956  debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
958  return 0; // avoid rerequesting release below
959  }
960 
962  debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
964  } else if (!mem_obj) {
965  // XXX: In bug 4131, we forgetHit() without mem_obj, so we need
966  // this segfault protection, but how can we get such a HIT?
967  debugs(20, 2, "StoreEntry::checkCachable: NO: missing parts: " << *this);
969  } else if (checkTooBig()) {
970  debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
972  } else if (checkTooSmall()) {
973  debugs(20, 2, "StoreEntry::checkCachable: NO: too small");
975  } else if (EBIT_TEST(flags, KEY_PRIVATE)) {
976  debugs(20, 3, "StoreEntry::checkCachable: NO: private key");
978  } else if (hasDisk()) {
979  /*
980  * the remaining cases are only relevant if we haven't
981  * started swapping out the object yet.
982  */
983  return 1;
984  } else if (storeTooManyDiskFilesOpen()) {
985  debugs(20, 2, "StoreEntry::checkCachable: NO: too many disk files open");
987  } else if (fdNFree() < RESERVED_FD) {
988  debugs(20, 2, "StoreEntry::checkCachable: NO: too many FD's open");
990  } else {
992  return 1;
993  }
994 
995  releaseRequest();
996  return 0;
997 }
998 
999 void
1001 {
1002  storeAppendPrintf(sentry, "Category\t Count\n");
1003  storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
1005  storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
1007  storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
1008  0); // TODO: Remove this backward compatibility hack.
1009  storeAppendPrintf(sentry, "no.missing_parts\t%d\n",
1011  storeAppendPrintf(sentry, "no.too_big\t%d\n",
1013  storeAppendPrintf(sentry, "no.too_small\t%d\n",
1015  storeAppendPrintf(sentry, "no.private_key\t%d\n",
1017  storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
1019  storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
1021  storeAppendPrintf(sentry, "yes.default\t%d\n",
1023 }
1024 
1025 void
1026 StoreEntry::lengthWentBad(const char *reason)
1027 {
1028  debugs(20, 3, "because " << reason << ": " << *this);
1030  releaseRequest();
1031 }
1032 
1033 void
1034 StoreEntry::completeSuccessfully(const char * const whyWeAreSure)
1035 {
1036  debugs(20, 3, whyWeAreSure << "; " << *this);
1037  complete();
1038 }
1039 
1040 void
1041 StoreEntry::completeTruncated(const char * const truncationReason)
1042 {
1043  lengthWentBad(truncationReason);
1044  complete();
1045 }
1046 
1047 void
1049 {
1050  debugs(20, 3, "storeComplete: '" << getMD5Text() << "'");
1051 
1052  // To preserve forwarding retries, call FwdState::complete() instead.
1054 
1055  if (store_status != STORE_PENDING) {
1056  /*
1057  * if we're not STORE_PENDING, then probably we got aborted
1058  * and there should be NO clients on this entry
1059  */
1061  assert(mem_obj->nclients == 0);
1062  return;
1063  }
1064 
1066 
1068 
1070 
1072  lengthWentBad("!validLength() in complete()");
1073 
1074 #if USE_CACHE_DIGESTS
1075  if (mem_obj->request)
1077 
1078 #endif
1079  /*
1080  * We used to call invokeHandlers, then storeSwapOut. However,
1081  * Madhukar Reddy <myreddy@persistence.com> reported that
1082  * responses without content length would sometimes get released
1083  * in client_side, thinking that the response is incomplete.
1084  */
1085  invokeHandlers();
1086 }
1087 
1088 /*
1089  * Someone wants to abort this transfer. Set the reason in the
1090  * request structure, call the callback and mark the
1091  * entry for releasing
1092  */
1093 void
1095 {
1098  assert(mem_obj != NULL);
1099  debugs(20, 6, "storeAbort: " << getMD5Text());
1100 
1101  lock("StoreEntry::abort"); /* lock while aborting */
1102  negativeCache();
1103 
1104  releaseRequest();
1105 
1107 
1108  // allow the Store clients to be told about the problem
1110 
1112 
1114 
1115  /* Notify the server side */
1116 
1117  if (mem_obj->abortCallback) {
1119  mem_obj->abortCallback = nullptr;
1120  }
1121 
1122  /* XXX Should we reverse these two, so that there is no
1123  * unneeded disk swapping triggered?
1124  */
1125  /* Notify the client side */
1126  invokeHandlers();
1127 
1128  // abort swap out, invalidating what was created so far (release follows)
1130 
1131  unlock("StoreEntry::abort"); /* unlock */
1132 }
1133 
1137 void
1139 {
1141  if (!shutting_down) // Store::Root() is FATALly missing during shutdown
1144 }
1145 
1146 /* thunk through to Store::Root().maintain(). Note that this would be better still
1147  * if registered against the root store itself, but that requires more complex
1148  * update logic - bigger fish to fry first. Long term each store when
1149  * it becomes active will self register
1150  */
1151 void
1153 {
1154  Store::Root().maintain();
1155 
1156  /* Reregister a maintain event .. */
1157  eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1);
1158 
1159 }
1160 
1161 /* The maximum objects to scan for maintain storage space */
1162 #define MAINTAIN_MAX_SCAN 1024
1163 #define MAINTAIN_MAX_REMOVE 64
1164 
1165 void
1166 StoreEntry::release(const bool shareable)
1167 {
1168  PROF_start(storeRelease);
1169  debugs(20, 3, shareable << ' ' << *this << ' ' << getMD5Text());
1170  /* If, for any reason we can't discard this object because of an
1171  * outstanding request, mark it for pending release */
1172 
1173  if (locked()) {
1174  releaseRequest(shareable);
1175  PROF_stop(storeRelease);
1176  return;
1177  }
1178 
1180  /* TODO: Teach disk stores to handle releases during rebuild instead. */
1181 
1182  // lock the entry until rebuilding is done
1183  lock("storeLateRelease");
1184  releaseRequest(shareable);
1185  LateReleaseStack.push(this);
1186  PROF_stop(storeRelease);
1187  return;
1188  }
1189 
1190  storeLog(STORE_LOG_RELEASE, this);
1191  Store::Root().evictCached(*this);
1192  destroyStoreEntry(static_cast<hash_link *>(this));
1193  PROF_stop(storeRelease);
1194 }
1195 
1196 static void
1198 {
1199  StoreEntry *e;
1200  static int n = 0;
1201 
1203  eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1204  return;
1205  }
1206 
1207  // TODO: this works but looks unelegant.
1208  for (int i = 0; i < 10; ++i) {
1209  if (LateReleaseStack.empty()) {
1210  debugs(20, Important(30), "storeLateRelease: released " << n << " objects");
1211  return;
1212  } else {
1213  e = LateReleaseStack.top();
1214  LateReleaseStack.pop();
1215  }
1216 
1217  e->unlock("storeLateRelease");
1218  ++n;
1219  }
1220 
1221  eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
1222 }
1223 
1227 bool
1229 {
1230  int64_t diff;
1231  assert(mem_obj != NULL);
1232  const auto reply = &mem_obj->baseReply();
1233  debugs(20, 3, "storeEntryValidLength: Checking '" << getMD5Text() << "'");
1234  debugs(20, 5, "storeEntryValidLength: object_len = " <<
1235  objectLen());
1236  debugs(20, 5, "storeEntryValidLength: hdr_sz = " << reply->hdr_sz);
1237  debugs(20, 5, "storeEntryValidLength: content_length = " << reply->content_length);
1238 
1239  if (reply->content_length < 0) {
1240  debugs(20, 5, "storeEntryValidLength: Unspecified content length: " << getMD5Text());
1241  return 1;
1242  }
1243 
1244  if (reply->hdr_sz == 0) {
1245  debugs(20, 5, "storeEntryValidLength: Zero header size: " << getMD5Text());
1246  return 1;
1247  }
1248 
1249  if (mem_obj->method == Http::METHOD_HEAD) {
1250  debugs(20, 5, "storeEntryValidLength: HEAD request: " << getMD5Text());
1251  return 1;
1252  }
1253 
1254  if (reply->sline.status() == Http::scNotModified)
1255  return 1;
1256 
1257  if (reply->sline.status() == Http::scNoContent)
1258  return 1;
1259 
1260  diff = reply->hdr_sz + reply->content_length - objectLen();
1261 
1262  if (diff == 0)
1263  return 1;
1264 
1265  debugs(20, 3, "storeEntryValidLength: " << (diff < 0 ? -diff : diff) << " bytes too " << (diff < 0 ? "big" : "small") <<"; '" << getMD5Text() << "'" );
1266 
1267  return 0;
1268 }
1269 
1270 static void
1272 {
1273  Mgr::RegisterAction("storedir", "Store Directory Stats", Store::Stats, 0, 1);
1274  Mgr::RegisterAction("store_io", "Store IO Interface Stats", &Mgr::StoreIoAction::Create, 0, 1);
1275  Mgr::RegisterAction("store_check_cachable_stats", "storeCheckCachable() Stats",
1276  storeCheckCachableStats, 0, 1);
1277  Mgr::RegisterAction("store_queues", "SMP Transients and Caching Queues", StatQueues, 0, 1);
1278 }
1279 
1280 void
1282 {
1283  storeKeyInit();
1285  storeDigestInit();
1286  storeLogOpen();
1287  eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1288  Store::Root().init();
1290 
1292 }
1293 
1294 void
1296 {
1297  Store::Root().configure();
1298 }
1299 
1300 bool
1302 {
1303  if (!checkCachable())
1304  return 0;
1305 
1306  if (mem_obj == NULL)
1307  return 0;
1308 
1309  if (mem_obj->data_hdr.size() == 0)
1310  return 0;
1311 
1312  if (mem_obj->inmem_lo != 0)
1313  return 0;
1314 
1316  return 0;
1317 
1318  return 1;
1319 }
1320 
1321 int
1323 {
1325  return 0;
1326 
1327  if (expires <= squid_curtime)
1328  return 0;
1329 
1330  if (store_status != STORE_OK)
1331  return 0;
1332 
1333  return 1;
1334 }
1335 
1342 void
1344 {
1345  // XXX: should make the default for expires 0 instead of -1
1346  // so we can distinguish "Expires: -1" from nothing.
1347  if (expires <= 0)
1348 #if USE_HTTP_VIOLATIONS
1350 #else
1352 #endif
1353  if (expires > squid_curtime) {
1355  debugs(20, 6, "expires = " << expires << " +" << (expires-squid_curtime) << ' ' << *this);
1356  }
1357 }
1358 
1359 void
1361 {
1363 #if USE_CACHE_DIGESTS
1364  delete store_digest;
1365 #endif
1366  store_digest = NULL;
1367 }
1368 
1369 int
1370 expiresMoreThan(time_t expires, time_t when)
1371 {
1372  if (expires < 0) /* No Expires given */
1373  return 1;
1374 
1375  return (expires > (squid_curtime + when));
1376 }
1377 
1378 int
1380 {
1382  return 0;
1383 
1385  if (expires <= squid_curtime)
1386  return 0;
1387 
1389  return 0;
1390 
1391  // now check that the entry has a cache backing or is collapsed
1392  if (hasDisk()) // backed by a disk cache
1393  return 1;
1394 
1395  if (swappingOut()) // will be backed by a disk cache
1396  return 1;
1397 
1398  if (!mem_obj) // not backed by a memory cache and not collapsed
1399  return 0;
1400 
1401  // StoreEntry::storeClientType() assumes DISK_CLIENT here, but there is no
1402  // disk cache backing that store_client constructor will assert. XXX: This
1403  // is wrong for range requests (that could feed off nibbled memory) and for
1404  // entries backed by the shared memory cache (that could, in theory, get
1405  // nibbled bytes from that cache, but there is no such "memoryIn" code).
1406  if (mem_obj->inmem_lo) // in memory cache, but got nibbled at
1407  return 0;
1408 
1409  // The following check is correct but useless at this position. TODO: Move
1410  // it up when the shared memory cache can either replenish locally nibbled
1411  // bytes or, better, does not use local RAM copy at all.
1412  // if (mem_obj->memCache.index >= 0) // backed by a shared memory cache
1413  // return 1;
1414 
1415  return 1;
1416 }
1417 
1418 bool
1420 {
1421  debugs(20, 7, *this << " had " << describeTimestamps());
1422 
1423  // TODO: Remove change-reducing "&" before the official commit.
1424  const auto reply = &mem().freshestReply();
1425 
1426  time_t served_date = reply->date;
1427  int age = reply->header.getInt(Http::HdrType::AGE);
1428  /* Compute the timestamp, mimicking RFC2616 section 13.2.3. */
1429  /* make sure that 0 <= served_date <= squid_curtime */
1430 
1431  if (served_date < 0 || served_date > squid_curtime)
1432  served_date = squid_curtime;
1433 
1434  /* Bug 1791:
1435  * If the returned Date: is more than 24 hours older than
1436  * the squid_curtime, then one of us needs to use NTP to set our
1437  * clock. We'll pretend that our clock is right.
1438  */
1439  else if (served_date < (squid_curtime - 24 * 60 * 60) )
1440  served_date = squid_curtime;
1441 
1442  /*
1443  * Compensate with Age header if origin server clock is ahead
1444  * of us and there is a cache in between us and the origin
1445  * server. But DONT compensate if the age value is larger than
1446  * squid_curtime because it results in a negative served_date.
1447  */
1448  if (age > squid_curtime - served_date)
1449  if (squid_curtime > age)
1450  served_date = squid_curtime - age;
1451 
1452  // compensate for Squid-to-server and server-to-Squid delays
1453  if (mem_obj && mem_obj->request) {
1454  struct timeval responseTime;
1455  if (mem_obj->request->hier.peerResponseTime(responseTime))
1456  served_date -= responseTime.tv_sec;
1457  }
1458 
1459  time_t exp = 0;
1460  if (reply->expires > 0 && reply->date > -1)
1461  exp = served_date + (reply->expires - reply->date);
1462  else
1463  exp = reply->expires;
1464 
1465  if (timestamp == served_date && expires == exp) {
1466  // if the reply lacks LMT, then we now know that our effective
1467  // LMT (i.e., timestamp) will stay the same, otherwise, old and
1468  // new modification times must match
1469  if (reply->last_modified < 0 || reply->last_modified == lastModified())
1470  return false; // nothing has changed
1471  }
1472 
1473  expires = exp;
1474 
1475  lastModified_ = reply->last_modified;
1476 
1477  timestamp = served_date;
1478 
1479  debugs(20, 5, *this << " has " << describeTimestamps());
1480  return true;
1481 }
1482 
1483 bool
1485 {
1486  assert(mem_obj);
1487  assert(e304.mem_obj);
1488 
1489  // update reply before calling timestampsSet() below
1490  const auto &oldReply = mem_obj->freshestReply();
1491  const auto updatedReply = oldReply.recreateOnNotModified(e304.mem_obj->baseReply());
1492  if (updatedReply) // HTTP 304 brought in new information
1493  mem_obj->updateReply(*updatedReply);
1494  // else continue to use the previous update, if any
1495 
1496  if (!timestampsSet() && !updatedReply)
1497  return false;
1498 
1499  // Keep the old mem_obj->vary_headers; see HttpHeader::skipUpdateHeader().
1500 
1501  debugs(20, 5, "updated basics in " << *this << " with " << e304);
1502  mem_obj->appliedUpdates = true; // helps in triage; may already be true
1503  return true;
1504 }
1505 
1506 void
1508 {
1509  assert(mem_obj);
1512 }
1513 
1514 void
1516 {
1517  assert(mem_obj);
1518  if (mem_obj->abortCallback) {
1519  mem_obj->abortCallback->cancel(reason);
1520  mem_obj->abortCallback = nullptr;
1521  }
1522 }
1523 
1524 void
1525 StoreEntry::dump(int l) const
1526 {
1527  debugs(20, l, "StoreEntry->key: " << getMD5Text());
1528  debugs(20, l, "StoreEntry->next: " << next);
1529  debugs(20, l, "StoreEntry->mem_obj: " << mem_obj);
1530  debugs(20, l, "StoreEntry->timestamp: " << timestamp);
1531  debugs(20, l, "StoreEntry->lastref: " << lastref);
1532  debugs(20, l, "StoreEntry->expires: " << expires);
1533  debugs(20, l, "StoreEntry->lastModified_: " << lastModified_);
1534  debugs(20, l, "StoreEntry->swap_file_sz: " << swap_file_sz);
1535  debugs(20, l, "StoreEntry->refcount: " << refcount);
1536  debugs(20, l, "StoreEntry->flags: " << storeEntryFlags(this));
1537  debugs(20, l, "StoreEntry->swap_dirn: " << swap_dirn);
1538  debugs(20, l, "StoreEntry->swap_filen: " << swap_filen);
1539  debugs(20, l, "StoreEntry->lock_count: " << lock_count);
1540  debugs(20, l, "StoreEntry->mem_status: " << mem_status);
1541  debugs(20, l, "StoreEntry->ping_status: " << ping_status);
1542  debugs(20, l, "StoreEntry->store_status: " << store_status);
1543  debugs(20, l, "StoreEntry->swap_status: " << swap_status);
1544 }
1545 
1546 /*
1547  * NOTE, this function assumes only two mem states
1548  */
1549 void
1551 {
1552  if (new_status == mem_status)
1553  return;
1554 
1555  // are we using a shared memory cache?
1556  if (MemStore::Enabled()) {
1557  // This method was designed to update replacement policy, not to
1558  // actually purge something from the memory cache (TODO: rename?).
1559  // Shared memory cache does not have a policy that needs updates.
1560  mem_status = new_status;
1561  return;
1562  }
1563 
1564  assert(mem_obj != NULL);
1565 
1566  if (new_status == IN_MEMORY) {
1567  assert(mem_obj->inmem_lo == 0);
1568 
1569  if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1570  debugs(20, 4, "not inserting special " << *this << " into policy");
1571  } else {
1572  mem_policy->Add(mem_policy, this, &mem_obj->repl);
1573  debugs(20, 4, "inserted " << *this << " key: " << getMD5Text());
1574  }
1575 
1576  ++hot_obj_count; // TODO: maintain for the shared hot cache as well
1577  } else {
1578  if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1579  debugs(20, 4, "not removing special " << *this << " from policy");
1580  } else {
1582  debugs(20, 4, "removed " << *this);
1583  }
1584 
1585  --hot_obj_count;
1586  }
1587 
1588  mem_status = new_status;
1589 }
1590 
1591 const char *
1593 {
1594  if (mem_obj == NULL)
1595  return "[null_mem_obj]";
1596  else
1597  return mem_obj->storeId();
1598 }
1599 
1600 void
1602 {
1603  assert(!mem_obj);
1604  mem_obj = new MemObject();
1605 }
1606 
1607 void
1608 StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl, const HttpRequestMethod &aMethod)
1609 {
1610  assert(!mem_obj);
1611  ensureMemObject(aUrl, aLogUrl, aMethod);
1612 }
1613 
1614 void
1615 StoreEntry::ensureMemObject(const char *aUrl, const char *aLogUrl, const HttpRequestMethod &aMethod)
1616 {
1617  if (!mem_obj)
1618  mem_obj = new MemObject();
1619  mem_obj->setUris(aUrl, aLogUrl, aMethod);
1620 }
1621 
1626 void
1628 {
1630 }
1631 
1637 void
1639 {
1640  if (EBIT_TEST(flags, DELAY_SENDING)) {
1642  invokeHandlers();
1643  }
1644 }
1645 
1646 void
1648 {
1649  debugs(20, 3, url());
1650  mem().reset();
1651  expires = lastModified_ = timestamp = -1;
1652 }
1653 
1654 /*
1655  * storeFsInit
1656  *
1657  * This routine calls the SETUP routine for each fs type.
1658  * I don't know where the best place for this is, and I'm not going to shuffle
1659  * around large chunks of code right now (that can be done once its working.)
1660  */
1661 void
1663 {
1664  storeReplSetup();
1665 }
1666 
1667 /*
1668  * called to add another store removal policy module
1669  */
1670 void
1671 storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
1672 {
1673  int i;
1674 
1675  /* find the number of currently known repl types */
1676  for (i = 0; storerepl_list && storerepl_list[i].typestr; ++i) {
1677  if (strcmp(storerepl_list[i].typestr, type) == 0) {
1678  debugs(20, DBG_IMPORTANT, "WARNING: Trying to load store replacement policy " << type << " twice.");
1679  return;
1680  }
1681  }
1682 
1683  /* add the new type */
1684  storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
1685 
1686  memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
1687 
1689 
1690  storerepl_list[i].create = create;
1691 }
1692 
1693 /*
1694  * Create a removal policy instance
1695  */
1696 RemovalPolicy *
1698 {
1699  storerepl_entry_t *r;
1700 
1701  for (r = storerepl_list; r && r->typestr; ++r) {
1702  if (strcmp(r->typestr, settings->type) == 0)
1703  return r->create(settings->args);
1704  }
1705 
1706  debugs(20, DBG_IMPORTANT, "ERROR: Unknown policy " << settings->type);
1707  debugs(20, DBG_IMPORTANT, "ERROR: Be sure to have set cache_replacement_policy");
1708  debugs(20, DBG_IMPORTANT, "ERROR: and memory_replacement_policy in squid.conf!");
1709  fatalf("ERROR: Unknown policy %s\n", settings->type);
1710  return NULL; /* NOTREACHED */
1711 }
1712 
1713 void
1715 {
1716  lock("StoreEntry::storeErrorResponse");
1717  buffer();
1719  flush();
1720  completeSuccessfully("replaceHttpReply() stored the entire error");
1721  negativeCache();
1722  releaseRequest(false); // if it is safe to negatively cache, sharing is OK
1723  unlock("StoreEntry::storeErrorResponse");
1724 }
1725 
1726 /*
1727  * Replace a store entry with
1728  * a new reply. This eats the reply.
1729  */
1730 void
1731 StoreEntry::replaceHttpReply(const HttpReplyPointer &rep, const bool andStartWriting)
1732 {
1733  debugs(20, 3, "StoreEntry::replaceHttpReply: " << url());
1734 
1735  if (!mem_obj) {
1736  debugs(20, DBG_CRITICAL, "Attempt to replace object with no in-memory representation");
1737  return;
1738  }
1739 
1740  mem_obj->replaceBaseReply(rep);
1741 
1742  if (andStartWriting)
1743  startWriting();
1744 }
1745 
1746 void
1748 {
1749  /* TODO: when we store headers separately remove the header portion */
1750  /* TODO: mark the length of the headers ? */
1751  /* We ONLY want the headers */
1752  assert (isEmpty());
1753  assert(mem_obj);
1754 
1755  // Per MemObject replies definitions, we can only write our base reply.
1756  // Currently, all callers replaceHttpReply() first, so there is no updated
1757  // reply here anyway. Eventually, we may need to support the
1758  // updateOnNotModified(),startWriting() sequence as well.
1760  const auto rep = &mem_obj->baseReply();
1761 
1762  buffer();
1763  rep->packHeadersUsingSlowPacker(*this);
1765 
1766  rep->body.packInto(this);
1767  flush();
1768 
1769  // The entry headers are written, new clients
1770  // should not collapse anymore.
1771  if (hittingRequiresCollapsing()) {
1772  setCollapsingRequirement(false);
1774  }
1775 }
1776 
1777 char const *
1779 {
1780  StoreMeta *tlv_list = storeSwapMetaBuild(this);
1781  int swap_hdr_sz;
1782  char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
1783  storeSwapTLVFree(tlv_list);
1784  assert (swap_hdr_sz >= 0);
1785  length = static_cast<size_t>(swap_hdr_sz);
1786  return result;
1787 }
1788 
1795 void
1797 {
1798  if (mem_obj && !Store::Root().transientsReader(*this) && // this worker is responsible
1799  hasTransients() && // other workers may be interested
1800  !hasMemStore() && // rejected by the shared memory cache
1802  debugs(20, 7, "cannot be shared: " << *this);
1803  if (!shutting_down) // Store::Root() is FATALly missing during shutdown
1804  Store::Root().stopSharing(*this);
1805  }
1806 }
1807 
1808 void
1810 {
1812 }
1813 
1814 void
1816 {
1817  // Abandon our transient entry if neither shared memory nor disk wants it.
1818  assert(mem_obj);
1819  mem_obj->swapout.decision = decision;
1821 }
1822 
1823 void
1824 StoreEntry::trimMemory(const bool preserveSwappable)
1825 {
1826  /*
1827  * DPW 2007-05-09
1828  * Bug #1943. We must not let go any data for IN_MEMORY
1829  * objects. We have to wait until the mem_status changes.
1830  */
1831  if (mem_status == IN_MEMORY)
1832  return;
1833 
1835  return; // cannot trim because we do not load them again
1836 
1837  if (preserveSwappable)
1839  else
1841 
1842  debugs(88, 7, *this << " inmem_lo=" << mem_obj->inmem_lo);
1843 }
1844 
1845 bool
1846 StoreEntry::modifiedSince(const time_t ims, const int imslen) const
1847 {
1848  const time_t mod_time = lastModified();
1849 
1850  debugs(88, 3, "modifiedSince: '" << url() << "'");
1851 
1852  debugs(88, 3, "modifiedSince: mod_time = " << mod_time);
1853 
1854  if (mod_time < 0)
1855  return true;
1856 
1857  assert(imslen < 0); // TODO: Either remove imslen or support it properly.
1858 
1859  if (mod_time > ims) {
1860  debugs(88, 3, "--> YES: entry newer than client");
1861  return true;
1862  } else if (mod_time < ims) {
1863  debugs(88, 3, "--> NO: entry older than client");
1864  return false;
1865  } else {
1866  debugs(88, 3, "--> NO: same LMT");
1867  return false;
1868  }
1869 }
1870 
1871 bool
1873 {
1874  if (const auto reply = hasFreshestReply()) {
1875  etag = reply->header.getETag(Http::HdrType::ETAG);
1876  if (etag.str)
1877  return true;
1878  }
1879  return false;
1880 }
1881 
1882 bool
1884 {
1885  const String reqETags = request.header.getList(Http::HdrType::IF_MATCH);
1886  return hasOneOfEtags(reqETags, false);
1887 }
1888 
1889 bool
1891 {
1892  const String reqETags = request.header.getList(Http::HdrType::IF_NONE_MATCH);
1893  // weak comparison is allowed only for HEAD or full-body GET requests
1894  const bool allowWeakMatch = !request.flags.isRanged &&
1896  return hasOneOfEtags(reqETags, allowWeakMatch);
1897 }
1898 
1900 bool
1901 StoreEntry::hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
1902 {
1903  const auto repETag = mem().freshestReply().header.getETag(Http::HdrType::ETAG);
1904  if (!repETag.str) {
1905  static SBuf asterisk("*", 1);
1906  return strListIsMember(&reqETags, asterisk, ',');
1907  }
1908 
1909  bool matched = false;
1910  const char *pos = NULL;
1911  const char *item;
1912  int ilen;
1913  while (!matched && strListGetItem(&reqETags, ',', &item, &ilen, &pos)) {
1914  if (!strncmp(item, "*", ilen))
1915  matched = true;
1916  else {
1917  String str;
1918  str.append(item, ilen);
1919  ETag reqETag;
1920  if (etagParseInit(&reqETag, str.termedBuf())) {
1921  matched = allowWeakMatch ? etagIsWeakEqual(repETag, reqETag) :
1922  etagIsStrongEqual(repETag, reqETag);
1923  }
1924  }
1925  }
1926  return matched;
1927 }
1928 
1929 Store::Disk &
1931 {
1932  assert(hasDisk());
1934  assert(sd);
1935  return *sd;
1936 }
1937 
1938 bool
1939 StoreEntry::hasDisk(const sdirno dirn, const sfileno filen) const
1940 {
1941  checkDisk();
1942  if (dirn < 0 && filen < 0)
1943  return swap_dirn >= 0;
1944  Must(dirn >= 0);
1945  const bool matchingDisk = (swap_dirn == dirn);
1946  return filen < 0 ? matchingDisk : (matchingDisk && swap_filen == filen);
1947 }
1948 
1949 void
1950 StoreEntry::attachToDisk(const sdirno dirn, const sfileno fno, const swap_status_t status)
1951 {
1952  debugs(88, 3, "attaching entry with key " << getMD5Text() << " : " <<
1953  swapStatusStr[status] << " " << dirn << " " <<
1954  std::hex << std::setw(8) << std::setfill('0') <<
1955  std::uppercase << fno);
1956  checkDisk();
1957  swap_dirn = dirn;
1958  swap_filen = fno;
1959  swap_status = status;
1960  checkDisk();
1961 }
1962 
1963 void
1965 {
1966  swap_dirn = -1;
1967  swap_filen = -1;
1969 }
1970 
1971 void
1973 {
1974  try {
1975  if (swap_dirn < 0) {
1976  Must(swap_filen < 0);
1978  } else {
1979  Must(swap_filen >= 0);
1981  if (swapoutFailed()) {
1983  } else {
1984  Must(swappingOut() || swappedOut());
1985  }
1986  }
1987  } catch (...) {
1988  debugs(88, DBG_IMPORTANT, "ERROR: inconsistent disk entry state " <<
1989  *this << "; problem: " << CurrentException);
1990  throw;
1991  }
1992 }
1993 
1994 /*
1995  * return true if the entry is in a state where
1996  * it can accept more data (ie with write() method)
1997  */
1998 bool
2000 {
2001  if (STORE_PENDING != store_status)
2002  return false;
2003 
2005  return false;
2006 
2007  return true;
2008 }
2009 
2010 const char *
2012 {
2013  LOCAL_ARRAY(char, buf, 256);
2014  snprintf(buf, 256, "LV:%-9d LU:%-9d LM:%-9d EX:%-9d",
2015  static_cast<int>(timestamp),
2016  static_cast<int>(lastref),
2017  static_cast<int>(lastModified_),
2018  static_cast<int>(expires));
2019  return buf;
2020 }
2021 
2022 void
2024 {
2025  if (required)
2027  else
2029 }
2030 
2031 static std::ostream &
2032 operator <<(std::ostream &os, const Store::IoStatus &io)
2033 {
2034  switch (io) {
2035  case Store::ioUndecided:
2036  os << 'u';
2037  break;
2038  case Store::ioReading:
2039  os << 'r';
2040  break;
2041  case Store::ioWriting:
2042  os << 'w';
2043  break;
2044  case Store::ioDone:
2045  os << 'o';
2046  break;
2047  }
2048  return os;
2049 }
2050 
2051 std::ostream &operator <<(std::ostream &os, const StoreEntry &e)
2052 {
2053  os << "e:";
2054 
2055  if (e.hasTransients()) {
2056  const auto &xitTable = e.mem_obj->xitTable;
2057  os << 't' << xitTable.io << xitTable.index;
2058  }
2059 
2060  if (e.hasMemStore()) {
2061  const auto &memCache = e.mem_obj->memCache;
2062  os << 'm' << memCache.io << memCache.index << '@' << memCache.offset;
2063  }
2064 
2065  // Do not use e.hasDisk() here because its checkDisk() call may calls us.
2066  if (e.swap_filen > -1 || e.swap_dirn > -1)
2067  os << 'd' << e.swap_filen << '@' << e.swap_dirn;
2068 
2069  os << '=';
2070 
2071  // print only non-default status values, using unique letters
2072  if (e.mem_status != NOT_IN_MEMORY ||
2073  e.store_status != STORE_PENDING ||
2074  e.swap_status != SWAPOUT_NONE ||
2075  e.ping_status != PING_NONE) {
2076  if (e.mem_status != NOT_IN_MEMORY) os << 'm';
2077  if (e.store_status != STORE_PENDING) os << 's';
2078  if (e.swap_status != SWAPOUT_NONE) os << 'w' << e.swap_status;
2079  if (e.ping_status != PING_NONE) os << 'p' << e.ping_status;
2080  }
2081 
2082  // print only set flags, using unique letters
2083  if (e.flags) {
2084  if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) os << 'S';
2085  if (EBIT_TEST(e.flags, ENTRY_REVALIDATE_ALWAYS)) os << 'R';
2086  if (EBIT_TEST(e.flags, DELAY_SENDING)) os << 'P';
2087  if (EBIT_TEST(e.flags, RELEASE_REQUEST)) os << 'X';
2088  if (EBIT_TEST(e.flags, REFRESH_REQUEST)) os << 'F';
2089  if (EBIT_TEST(e.flags, ENTRY_REVALIDATE_STALE)) os << 'E';
2090  if (EBIT_TEST(e.flags, KEY_PRIVATE)) {
2091  os << 'I';
2092  if (e.shareableWhenPrivate)
2093  os << 'H';
2094  }
2095  if (EBIT_TEST(e.flags, ENTRY_FWD_HDR_WAIT)) os << 'W';
2096  if (EBIT_TEST(e.flags, ENTRY_NEGCACHED)) os << 'N';
2097  if (EBIT_TEST(e.flags, ENTRY_VALIDATED)) os << 'V';
2098  if (EBIT_TEST(e.flags, ENTRY_BAD_LENGTH)) os << 'L';
2099  if (EBIT_TEST(e.flags, ENTRY_ABORTED)) os << 'A';
2100  if (EBIT_TEST(e.flags, ENTRY_REQUIRES_COLLAPSING)) os << 'C';
2101  }
2102 
2103  return os << '/' << &e << '*' << e.locks();
2104 }
2105 
2106 void
2108 {
2110  entry_->releaseRequest(false);
2112  });
2113 }
2114 
#define EBIT_CLR(flag, bit)
Definition: defines.h:68
bool cancel(const char *reason)
Definition: AsyncCall.cc:56
int hdr_sz
Definition: Message.h:82
void fatal(const char *message)
Definition: fatal.cc:28
const cache_key * storeKeyPrivate()
const char * xstrerr(int error)
Definition: xstrerror.cc:83
void makePrivate(const bool shareable)
Definition: store.cc:173
REMOVALPOLICYCREATE * create
Definition: store.cc:106
char method[16]
Definition: tcp-banger2.c:115
void memoryDisconnect(StoreEntry &)
disassociates the entry from the memory cache, preserving cached data
Definition: Controller.cc:633
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:293
Definition: store.cc:104
HASHCMP storeKeyHashCmp
void Maintain(void *unused)
Definition: store.cc:1152
virtual void buffer()
Definition: store.cc:1627
void abandon(const char *context)
Definition: Store.h:285
#define INDEXSD(i)
Definition: SquidConfig.h:72
struct _store_check_cachable_hist::@144 no
int checkTooSmall()
Definition: store.cc:918
time_t timestamp
Definition: Store.h:225
@ HDR_X_ACCELERATOR_VARY
const cache_key * storeKeyPublicByRequest(HttpRequest *request, const KeyScope keyScope)
@ METHOD_HEAD
Definition: MethodType.h:28
IoStatus
cache "I/O" direction and status
Definition: forward.h:40
const char * typestr
Definition: store.cc:105
int64_t minObjectSize
Definition: SquidConfig.h:272
Definition: ETag.h:18
virtual ~StoreEntry()
Definition: store.cc:372
store_client_t storeClientType() const
Definition: store.cc:288
bool validLength() const
Definition: store.cc:1228
void storeReplAdd(const char *type, REMOVALPOLICYCREATE *create)
Definition: store.cc:1671
bool updateOnNotModified(const StoreEntry &e304)
Definition: store.cc:1484
AsyncCall::Pointer callback
Definition: CommRead.h:28
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1615
const char * storeId() const
Definition: MemObject.cc:54
void unregisterAbortCallback(const char *reason)
Definition: store.cc:1515
void releaseRequest(const bool shareable=false)
Definition: store.cc:473
StoreEntry * adjustVary()
Definition: store.cc:680
static OBJH storeCheckCachableStats
Definition: store.cc:115
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:1972
manages a single cache_dir
Definition: Disk.h:22
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
unsigned char cache_key
Store key.
Definition: forward.h:29
static bool Enabled()
whether Squid is correctly configured to use a shared memory cache
Definition: MemStore.h:68
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:166
HttpHeader header
Definition: Message.h:75
#define EBIT_SET(flag, bit)
Definition: defines.h:67
StoreEntry()
Definition: store.cc:351
int memory_cache_first
Definition: SquidConfig.h:342
bool isEmpty() const
Definition: SBuf.h:424
virtual void flush()
Definition: store.cc:1638
MemObject * mem_obj
Definition: Store.h:222
const cache_key * storeKeyPublic(const char *url, const HttpRequestMethod &method, const KeyScope keyScope)
mem_hdr data_hdr
Definition: MemObject.h:138
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:414
void clearPublicKeyScope()
Definition: store.cc:624
C end
Definition: Range.h:25
void destroyStoreEntry(void *data)
Definition: store.cc:417
const char * url() const
Definition: store.cc:1592
static void handler(int signo)
Definition: purge.cc:854
void trimUnSwappable()
Definition: MemObject.cc:395
void storeInit(void)
Definition: store.cc:1281
void createMemObject()
Definition: store.cc:1601
@ KEY_PRIVATE
Definition: enums.h:102
@ ENTRY_FWD_HDR_WAIT
Definition: enums.h:111
MemObject & mem()
Definition: Store.h:52
bool shareableWhenPrivate
Definition: Store.h:332
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:872
void lock(const char *context)
Definition: store.cc:460
void FreeMemory()
undo Init()
Definition: Controller.cc:958
void transientsAbandonmentCheck()
Definition: store.cc:1796
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
int64_t offset
Definition: StoreIOBuffer.h:55
SQUIDCEXTERN void hash_remove_link(hash_table *, hash_link *)
Definition: hash.cc:224
Definition: SBuf.h:87
virtual void init() override
Definition: Controller.cc:59
void detachFromDisk()
Definition: store.cc:1964
void onException() noexcept
Definition: store.cc:2107
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:467
void setNoDelay(bool const newValue)
Definition: MemObject.cc:430
Io io
current I/O state
Definition: MemObject.h:173
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:563
const char * str
quoted-string
Definition: ETag.h:20
void configure()
update configuration, including limits (re)calculation
Definition: Controller.cc:198
static std::ostream & operator<<(std::ostream &os, const Store::IoStatus &io)
Definition: store.cc:2032
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:590
Entry * entry_
the guarded Entry or nil
Definition: Store.h:391
bool hierarchical
Definition: RequestFlags.h:34
bool etagIsStrongEqual(const ETag &tag1, const ETag &tag2)
whether etags are strong-equal
Definition: ETag.cc:49
const char * memStatusStr[]
Definition: store.cc:75
int aborted_requests
Definition: StatCounters.h:152
int type
Definition: errorpage.cc:153
C * getRaw() const
Definition: RefCount.h:80
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:229
uint16_t flags
Definition: Store.h:233
struct _store_check_cachable_hist store_check_cachable_hist
void storeGetMemSpace(int size)
Definition: store.cc:1138
bool etagIsWeakEqual(const ETag &tag1, const ETag &tag2)
whether etags are weak-equal
Definition: ETag.cc:55
int hot_obj_count
size_t size() const
Definition: stmem.cc:408
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:270
static storerepl_entry_t * storerepl_list
Definition: store.cc:109
bool isAccepting() const
Definition: store.cc:1999
time_t expires
Definition: Store.h:227
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:430
Io io
current I/O state
Definition: MemObject.h:184
char * storeSwapMetaPack(tlv *tlv_list, int *length)
int64_t endOffset() const
Definition: MemObject.cc:219
static void storeRegisterWithCacheManager(void)
Definition: store.cc:1271
int64_t objectLen() const
Definition: Store.h:258
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1731
#define DBG_CRITICAL
Definition: Debug.h:40
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:55
void startWriting()
Definition: store.cc:1747
SQUIDCEXTERN void hash_join(hash_table *, hash_link *)
Definition: hash.cc:132
@ SWAP_LOG_ADD
Definition: swap_log_op.h:14
RemovalPolicy * mem_policy
Definition: MemObject.cc:45
#define PROF_stop(probename)
Definition: Profiler.h:63
void storeFsInit(void)
Definition: store.cc:1662
int store_open_disk_fd
void reset()
Definition: store.cc:1647
#define DBG_IMPORTANT
Definition: Debug.h:41
virtual void evictCached(StoreEntry &) override
Definition: Controller.cc:506
StoreEntry * storeGetPublicByRequest(HttpRequest *req, const KeyScope keyScope)
Definition: store.cc:531
void destroyMemObject()
Definition: store.cc:399
int64_t inmem_lo
Definition: MemObject.h:139
@ STORE_LOG_RELEASE
Definition: enums.h:159
int len
Definition: CommRead.h:27
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:593
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *req, const HttpRequestMethod &method, const KeyScope keyScope)
Definition: store.cc:525
#define TexcHere(msg)
legacy convenience macro; it is not difficult to type Here() now
Definition: TextException.h:58
void doAbandon(const char *context)
Definition: store.cc:501
void invokeHandlers()
void replaceBaseReply(const HttpReplyPointer &r)
Definition: MemObject.cc:131
void storeReplSetup(void)
static DeferredRead::DeferrableRead DeferReader
Definition: Store.h:45
void addWriting(StoreEntry *, const cache_key *)
Definition: Controller.cc:790
@ ENTRY_ABORTED
Definition: enums.h:115
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:65
Definition: Range.h:19
enum _mem_status_t mem_status_t
int validToSend() const
Definition: store.cc:1379
@ ENTRY_BAD_LENGTH
Definition: enums.h:114
@ STORE_MEM_CLIENT
Definition: enums.h:73
void comm_read(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer &callback)
Definition: Read.h:59
static std::stack< StoreEntry * > LateReleaseStack
Definition: store.cc:121
void storeFreeMemory(void)
Definition: store.cc:1360
void transientsDisconnect(StoreEntry &)
disassociates the entry from the intransit table
Definition: Controller.cc:669
Decision decision
current decision state
Definition: MemObject.h:156
StoreEntry * find(const cache_key *)
Definition: Controller.cc:360
virtual void disconnect(StoreEntry &)
called when the entry is about to forget its association with cache_dir
Definition: Disk.h:71
int size
Definition: ModDevPoll.cc:77
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1809
struct timeval store_complete_stop
void OBJH(StoreEntry *)
Definition: forward.h:44
#define NULL
Definition: types.h:166
RemovalPolicySettings * memPolicy
Definition: SquidConfig.h:98
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1890
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
SBuf vary_headers
Definition: MemObject.h:204
@ IN_MEMORY
Definition: enums.h:36
const HttpReply & baseReply() const
Definition: MemObject.h:59
@ ENTRY_VALIDATED
Definition: enums.h:113
void abort()
Definition: store.cc:1094
static void StatQueues(StoreEntry *e)
reports the current state of Store-related queues
Definition: store.cc:133
store_client_t
Definition: enums.h:71
void append(char const *buf, int len)
Definition: String.cc:161
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1901
void stopSharing(StoreEntry &)
stop any current (and prevent any future) SMP sharing of the given entry
Definition: Controller.cc:641
int max_open_disk_fds
Definition: SquidConfig.h:463
tlv * storeSwapMetaBuild(const StoreEntry *e)
uint16_t refcount
Definition: Store.h:232
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1939
sdirno swap_dirn
Definition: Store.h:239
@ STORE_DISK_CLIENT
Definition: enums.h:74
#define INT_MAX
Definition: types.h:76
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:98
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2023
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:152
@ ENTRY_REQUIRES_COLLAPSING
Definition: enums.h:118
cache_key * storeKeyDup(const cache_key *key)
ping_status_t ping_status
Definition: Store.h:243
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:139
const char * getSerialisedMetaData(size_t &length) const
Definition: store.cc:1778
@ ioDone
Definition: forward.h:40
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:214
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
bool peerResponseTime(struct timeval &responseTime)
Definition: access_log.cc:317
static void StatQueue(std::ostream &)
prints IPC message queue state; suitable for cache manager reports
int unlock(const char *context)
Definition: store.cc:484
void expireNow()
Definition: store.cc:788
const char * storeEntryFlags(const StoreEntry *entry)
Definition: stat.cc:273
static size_t inUseCount()
Definition: store.cc:198
HttpRequestMethod method
Definition: MemObject.h:137
store_status_t store_status
Definition: Store.h:245
int neighbors_do_private_keys
int conn
the current server connection FD
Definition: Transport.cc:26
#define assert(EX)
Definition: assert.h:19
const char * swapStatusStr[]
Definition: store.cc:91
char const * termedBuf() const
Definition: SquidString.h:92
@ DELAY_SENDING
Definition: enums.h:97
void Stats(StoreEntry *output)
Definition: store.cc:125
@ NOT_IN_MEMORY
Definition: enums.h:35
void fatalf(const char *fmt,...)
Definition: fatal.cc:68
HierarchyLogEntry hier
Definition: HttpRequest.h:157
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1815
const char * pingStatusStr[]
Definition: store.cc:80
RemovalPolicy * createRemovalPolicy(RemovalPolicySettings *settings)
Definition: store.cc:1697
int checkNegativeHit() const
Definition: store.cc:1322
virtual void stat(StoreEntry &) const override
Definition: Controller.cc:141
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:137
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
@ PING_NONE
Has not considered whether to send ICP queries to peers yet.
Definition: enums.h:41
@ ENTRY_SPECIAL
Definition: enums.h:84
void write(StoreIOBuffer)
Definition: store.cc:795
bool checkCachable()
Definition: store.cc:946
void(* Remove)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node)
Definition: RemovalPolicy.h:47
hash_table * store_table
struct _store_check_cachable_hist::@145 yes
bool cacheNegatively()
Definition: store.cc:187
mem_status_t mem_status
Definition: Store.h:241
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:336
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:381
void dump(int debug_lvl) const
Definition: store.cc:1525
@ STORE_OK
Definition: enums.h:50
RefCount< HttpReply > HttpReplyPointer
Definition: forward.h:47
static int store_dirs_rebuilding
the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
Definition: Controller.h:139
time_t squid_curtime
Definition: stub_time.cc:17
void transientsClearCollapsingRequirement(StoreEntry &e)
removes collapsing requirement (for future hits)
Definition: Controller.cc:676
static MemAllocator * pool
Definition: Store.h:323
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:250
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:774
signed_int32_t sfileno
Definition: forward.h:22
int64_t content_length
Definition: Message.h:84
int64_t store_maxobjsize
time_t date
Definition: HttpReply.h:40
void storeDirSwapLog(const StoreEntry *e, int op)
Definition: Disks.cc:837
void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply)
Definition: store.cc:1034
void storeConfigure(void)
Definition: store.cc:1295
@ scNotModified
Definition: StatusCode.h:40
void EVH(void *)
Definition: event.h:18
char * buf
Definition: CommRead.h:26
void setMemStatus(mem_status_t)
Definition: store.cc:1550
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1364
swap_status_t swap_status
Definition: Store.h:247
@ ksDefault
Definition: store_key_md5.h:19
void negativeCache()
Definition: store.cc:1343
#define fd_table
Definition: fde.h:189
void complete()
Definition: store.cc:1048
bool appliedUpdates
Definition: MemObject.h:89
#define Important(id)
Definition: DebugMessages.h:91
signed int sdirno
Definition: forward.h:23
void markEndOfReplyHeaders()
sets baseReply().hdr_sz (i.e. written reply headers size) to endOffset()
Definition: MemObject.cc:225
HttpRequestMethod method
Definition: HttpRequest.h:114
void storeKeyFree(const cache_key *key)
KeyScope
Definition: store_key_md5.h:18
struct timeval current_time
Definition: stub_time.cc:15
void(* Add)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node)
Definition: RemovalPolicy.h:46
#define memPoolCreate
Definition: Pool.h:325
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:155
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:389
bool checkDeferRead(int fd) const
Definition: store.cc:268
struct SquidConfig::@111 onoff
void swapOutFileClose(int how)
void delayRead(DeferredRead const &)
Definition: MemObject.cc:444
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:667
bool timestampsSet()
Definition: store.cc:1419
void storeSwapTLVFree(tlv *n)
@ scNoContent
Definition: StatusCode.h:30
void storeAppendVPrintf(StoreEntry *e, const char *fmt, va_list vargs)
Definition: store.cc:882
struct SquidConfig::@109 Store
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:906
const char * storeStatusStr[]
Definition: store.cc:86
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: Controller.cc:93
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:71
@ ioUndecided
Definition: forward.h:40
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1067
void hashDelete()
Definition: store.cc:448
int64_t object_sz
Definition: MemObject.h:198
bool isEmpty() const
Definition: Store.h:67
void trimSwappable()
Definition: MemObject.cc:370
void handleIdleEntry(StoreEntry &)
called when the entry is no longer needed by any transaction
Definition: Controller.cc:683
int fdNFree(void)
Definition: fd.cc:296
int RESERVED_FD
int expiresMoreThan(time_t expires, time_t when)
Definition: store.cc:1370
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:73
void storeRebuildStart(void)
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
Definition: store.cc:224
void storeLog(int tag, const StoreEntry *e)
Definition: store_log.cc:39
uint64_t swap_file_sz
Definition: Store.h:231
const char * storeKeyText(const cache_key *key)
@ REFRESH_REQUEST
Definition: enums.h:99
CacheDigest * store_digest
virtual void vappendf(const char *, va_list)
Definition: store.cc:838
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:212
Comm::ConnectionPointer conn
Definition: CommRead.h:25
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:519
void release(const bool shareable=false)
Definition: store.cc:1166
static void StatQueue(std::ostream &)
prints IPC message queue state; suitable for cache manager reports
Definition: IpcIoFile.cc:529
virtual int getInUseCount()=0
const cache_key * storeKeyPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope)
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1930
const HttpReply & freshestReply() const
Definition: MemObject.h:67
static Pointer Create(const CommandPointer &cmd)
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:641
int shutting_down
void hashInsert(const cache_key *)
Definition: store.cc:439
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1301
StoreEntry * storeCreatePureEntry(const char *url, const char *log_url, const HttpRequestMethod &method)
Definition: store.cc:756
bool checkTooBig() const
Definition: store.cc:935
void freeMemorySpace(const int spaceRequired)
Definition: Controller.cc:546
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:1950
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:820
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1824
void setNoDelay(bool const)
Definition: store.cc:274
@ writerGone
failure: caller left before swapping out everything
Definition: StoreIOState.h:71
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:379
void setHeaders(Http::StatusCode status, const char *reason, const char *ctype, int64_t clen, time_t lmt, time_t expires)
Definition: HttpReply.cc:168
@ ENTRY_REVALIDATE_STALE
Definition: enums.h:100
@ ENTRY_NEGCACHED
Definition: enums.h:112
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:217
int locked() const
Definition: Store.h:145
static EVH storeLateRelease
Definition: store.cc:116
#define SWALLOW_EXCEPTIONS(code)
Definition: TextException.h:77
RemovalPolicy * REMOVALPOLICYCREATE(wordlist *args)
Definition: RemovalPolicy.h:80
@ ENTRY_REVALIDATE_ALWAYS
Definition: enums.h:85
@ scOkay
Definition: StatusCode.h:26
@ ioWriting
Definition: forward.h:40
RemovalPolicyNode repl
Definition: MemObject.h:196
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1026
void registerAbortCallback(const AsyncCall::Pointer &)
notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
Definition: store.cc:1507
void storeLogOpen(void)
Definition: store_log.cc:124
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:133
@ METHOD_GET
Definition: MethodType.h:25
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:135
AsyncCall::Pointer abortCallback
used for notifying StoreEntry writers about 3rd-party initiated aborts
Definition: MemObject.h:195
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1846
SQUIDCEXTERN hash_link * hash_lookup(hash_table *, const void *)
Definition: hash.cc:147
void updateReply(const HttpReply &r)
(re)sets updated reply;
Definition: MemObject.h:84
void reset()
Definition: MemObject.cc:269
#define false
Definition: GnuRegex.c:233
unsigned short lock_count
Definition: Store.h:325
void * xrealloc(void *s, size_t sz)
Definition: xalloc.cc:137
static int getKeyCounter(void)
Definition: store.cc:543
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1883
@ ioReading
Definition: forward.h:40
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1872
const char * context_
default unlock() message
Definition: Store.h:392
const HttpReply * hasFreshestReply() const
Definition: Store.h:58
@ SWAPOUT_NONE
Definition: enums.h:58
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
void storeKeyInit(void)
const char * describeTimestamps() const
Definition: store.cc:2011
const HttpReplyPointer & updatedReply() const
Definition: MemObject.h:63
void clearPrivate()
Definition: store.cc:179
HttpRequestPointer request
Definition: MemObject.h:188
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:77
void completeTruncated(const char *whyWeConsiderTheReplyTruncated)
Definition: store.cc:1041
time_t lastref
Definition: Store.h:226
int nclients
Definition: MemObject.h:146
const char * getMD5Text() const
Definition: store.cc:206
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:109
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1714
time_t negativeTtl
Definition: SquidConfig.h:100
void storeDigestInit(void)
#define PROF_start(probename)
Definition: Profiler.h:62
class SquidConfig Config
Definition: SquidConfig.cc:12
int storePendingNClients(const StoreEntry *e)
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
StatCounters statCounter
Definition: StatCounters.cc:12
time_t lastModified() const
Definition: Store.h:179
@ STORE_PENDING
Definition: enums.h:51
SwapOut swapout
Definition: MemObject.h:159
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:237
Pointer recreateOnNotModified(const HttpReply &reply304) const
Definition: HttpReply.cc:263
Controller & Root()
safely access controller singleton
Definition: Controller.cc:945

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors