store.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 20 Storage Manager */
10 
11 #include "squid.h"
12 #include "base/AsyncCbdataCalls.h"
13 #include "base/PackableStream.h"
14 #include "base/TextException.h"
15 #include "CacheDigest.h"
16 #include "CacheManager.h"
17 #include "CollapsedForwarding.h"
18 #include "comm/Connection.h"
19 #include "comm/Read.h"
20 #if HAVE_DISKIO_MODULE_IPCIO
21 #include "DiskIO/IpcIo/IpcIoFile.h"
22 #endif
23 #include "ETag.h"
24 #include "event.h"
25 #include "fde.h"
26 #include "globals.h"
27 #include "http.h"
28 #include "HttpReply.h"
29 #include "HttpRequest.h"
30 #include "mem_node.h"
31 #include "MemObject.h"
32 #include "MemStore.h"
33 #include "mgr/Registration.h"
34 #include "mgr/StoreIoAction.h"
35 #include "profiler/Profiler.h"
36 #include "repl_modules.h"
37 #include "RequestFlags.h"
38 #include "SquidConfig.h"
39 #include "SquidTime.h"
40 #include "StatCounters.h"
41 #include "stmem.h"
42 #include "Store.h"
43 #include "store/Controller.h"
44 #include "store/Disk.h"
45 #include "store/Disks.h"
46 #include "store_digest.h"
47 #include "store_key_md5.h"
48 #include "store_log.h"
49 #include "store_rebuild.h"
50 #include "StoreClient.h"
51 #include "StoreIOState.h"
52 #include "StoreMeta.h"
53 #include "StrList.h"
54 #include "swap_log_op.h"
55 #include "tools.h"
56 #if USE_DELAY_POOLS
57 #include "DelayPools.h"
58 #endif
59 
63 #include "mem/Pool.h"
64 
65 #include <climits>
66 #include <stack>
67 
68 #define REBUILD_TIMESTAMP_DELTA_MAX 2
69 
70 #define STORE_IN_MEM_BUCKETS (229)
71 
72 // TODO: Convert these string constants to enum string-arrays generated
73 
74 const char *memStatusStr[] = {
75  "NOT_IN_MEMORY",
76  "IN_MEMORY"
77 };
78 
79 const char *pingStatusStr[] = {
80  "PING_NONE",
81  "PING_WAITING",
82  "PING_DONE"
83 };
84 
85 const char *storeStatusStr[] = {
86  "STORE_OK",
87  "STORE_PENDING"
88 };
89 
90 const char *swapStatusStr[] = {
91  "SWAPOUT_NONE",
92  "SWAPOUT_WRITING",
93  "SWAPOUT_DONE",
94  "SWAPOUT_FAILED"
95 };
96 
97 /*
98  * This defines an repl type
99  */
100 
102 
104  const char *typestr;
106 };
107 
109 
110 /*
111  * local function prototypes
112  */
113 static int getKeyCounter(void);
116 
117 /*
118  * local variables
119  */
120 static std::stack<StoreEntry*> LateReleaseStack;
122 
123 void
125 {
126  assert(output);
127  Root().stat(*output);
128 }
129 
131 static void
133 {
134  assert(e);
135  PackableStream stream(*e);
137  #if HAVE_DISKIO_MODULE_IPCIO
138  stream << "\n";
139  IpcIoFile::StatQueue(stream);
140  #endif
141  stream.flush();
142 }
143 
144 // XXX: new/delete operators need to be replaced with MEMPROXY_CLASS
145 // definitions but doing so exposes bug 4370, and maybe 4354 and 4355
146 void *
147 StoreEntry::operator new (size_t bytecount)
148 {
149  assert(bytecount == sizeof (StoreEntry));
150 
151  if (!pool) {
152  pool = memPoolCreate ("StoreEntry", bytecount);
153  }
154 
155  return pool->alloc();
156 }
157 
158 void
159 StoreEntry::operator delete (void *address)
160 {
161  pool->freeOne(address);
162 }
163 
164 bool
166 {
167  /* This object can be cached for a long time */
168  return !EBIT_TEST(flags, RELEASE_REQUEST) && setPublicKey(scope);
169 }
170 
171 void
172 StoreEntry::makePrivate(const bool shareable)
173 {
174  releaseRequest(shareable); /* delete object when not used */
175 }
176 
177 void
179 {
182  shareableWhenPrivate = false;
183 }
184 
185 bool
187 {
188  /* This object may be negatively cached */
189  if (makePublic()) {
190  negativeCache();
191  return true;
192  }
193  return false;
194 }
195 
196 size_t
198 {
199  if (!pool)
200  return 0;
201  return pool->getInUseCount();
202 }
203 
204 const char *
206 {
207  return storeKeyText((const cache_key *)key);
208 }
209 
210 #include "comm.h"
211 
212 void
213 StoreEntry::DeferReader(void *theContext, CommRead const &aRead)
214 {
215  StoreEntry *anEntry = (StoreEntry *)theContext;
216  anEntry->delayAwareRead(aRead.conn,
217  aRead.buf,
218  aRead.len,
219  aRead.callback);
220 }
221 
222 void
224 {
225  size_t amountToRead = bytesWanted(Range<size_t>(0, len));
226  /* sketch: readdeferer* = getdeferer.
227  * ->deferRead (fd, buf, len, callback, DelayAwareRead, this)
228  */
229 
230  if (amountToRead <= 0) {
231  assert (mem_obj);
233  return;
234  }
235 
236  if (fd_table[conn->fd].closing()) {
237  // Readers must have closing callbacks if they want to be notified. No
238  // readers appeared to care around 2009/12/14 as they skipped reading
239  // for other reasons. Closing may already be true at the delyaAwareRead
240  // call time or may happen while we wait after delayRead() above.
241  debugs(20, 3, "will not read from closing " << conn << " for " << callback);
242  return; // the read callback will never be called
243  }
244 
245  comm_read(conn, buf, amountToRead, callback);
246 }
247 
248 size_t
249 StoreEntry::bytesWanted (Range<size_t> const aRange, bool ignoreDelayPools) const
250 {
251  if (mem_obj == NULL)
252  return aRange.end;
253 
254 #if URL_CHECKSUM_DEBUG
255 
256  mem_obj->checkUrlChecksum();
257 
258 #endif
259 
261  return 0;
262 
263  return mem_obj->mostBytesWanted(aRange.end, ignoreDelayPools);
264 }
265 
266 bool
268 {
269  return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0);
270 }
271 
272 void
273 StoreEntry::setNoDelay(bool const newValue)
274 {
275  if (mem_obj)
276  mem_obj->setNoDelay(newValue);
277 }
278 
279 // XXX: Type names mislead. STORE_DISK_CLIENT actually means that we should
280 // open swapin file, aggressively trim memory, and ignore read-ahead gap.
281 // It does not mean we will read from disk exclusively (or at all!).
282 // STORE_MEM_CLIENT covers all other cases, including in-memory entries,
283 // newly created entries, and entries not backed by disk or memory cache.
284 // XXX: May create STORE_DISK_CLIENT with no disk caching configured.
285 // XXX: Collapsed clients cannot predict their type.
288 {
289  /* The needed offset isn't in memory
290  * XXX TODO: this is wrong for range requests
291  * as the needed offset may *not* be 0, AND
292  * offset 0 in the memory object is the HTTP headers.
293  */
294 
295  assert(mem_obj);
296 
297  if (mem_obj->inmem_lo)
298  return STORE_DISK_CLIENT;
299 
300  if (EBIT_TEST(flags, ENTRY_ABORTED)) {
301  /* I don't think we should be adding clients to aborted entries */
302  debugs(20, DBG_IMPORTANT, "storeClientType: adding to ENTRY_ABORTED entry");
303  return STORE_MEM_CLIENT;
304  }
305 
306  if (swapoutFailed())
307  return STORE_MEM_CLIENT;
308 
309  if (store_status == STORE_OK) {
310  /* the object has completed. */
311 
312  if (mem_obj->inmem_lo == 0 && !isEmpty()) {
313  if (swappedOut()) {
314  debugs(20,7, HERE << mem_obj << " lo: " << mem_obj->inmem_lo << " hi: " << mem_obj->endOffset() << " size: " << mem_obj->object_sz);
315  if (mem_obj->endOffset() == mem_obj->object_sz) {
316  /* hot object fully swapped in (XXX: or swapped out?) */
317  return STORE_MEM_CLIENT;
318  }
319  } else {
320  /* Memory-only, or currently being swapped out */
321  return STORE_MEM_CLIENT;
322  }
323  }
324  return STORE_DISK_CLIENT;
325  }
326 
327  /* here and past, entry is STORE_PENDING */
328  /*
329  * If this is the first client, let it be the mem client
330  */
331  if (mem_obj->nclients == 1)
332  return STORE_MEM_CLIENT;
333 
334  /*
335  * If there is no disk file to open yet, we must make this a
336  * mem client. If we can't open the swapin file before writing
337  * to the client, there is no guarantee that we will be able
338  * to open it later when we really need it.
339  */
340  if (swap_status == SWAPOUT_NONE)
341  return STORE_MEM_CLIENT;
342 
343  /*
344  * otherwise, make subsequent clients read from disk so they
345  * can not delay the first, and vice-versa.
346  */
347  return STORE_DISK_CLIENT;
348 }
349 
351  mem_obj(NULL),
352  timestamp(-1),
353  lastref(-1),
354  expires(-1),
355  lastModified_(-1),
356  swap_file_sz(0),
357  refcount(0),
358  flags(0),
359  swap_filen(-1),
360  swap_dirn(-1),
361  mem_status(NOT_IN_MEMORY),
362  ping_status(PING_NONE),
363  store_status(STORE_PENDING),
364  swap_status(SWAPOUT_NONE),
365  lock_count(0),
366  shareableWhenPrivate(false)
367 {
368  debugs(20, 5, "StoreEntry constructed, this=" << this);
369 }
370 
372 {
373  debugs(20, 5, "StoreEntry destructed, this=" << this);
374 }
375 
376 #if USE_ADAPTATION
377 void
379 {
380  if (!deferredProducer)
381  deferredProducer = producer;
382  else
383  debugs(20, 5, "Deferred producer call is already set to: " <<
384  *deferredProducer << ", requested call: " << *producer);
385 }
386 
387 void
389 {
390  if (deferredProducer != NULL) {
393  }
394 }
395 #endif
396 
397 void
399 {
400  debugs(20, 3, mem_obj << " in " << *this);
401 
402  // Store::Root() is FATALly missing during shutdown
403  if (hasTransients() && !shutting_down)
405  if (hasMemStore() && !shutting_down)
406  Store::Root().memoryDisconnect(*this);
407 
408  if (auto memObj = mem_obj) {
410  mem_obj = NULL;
411  delete memObj;
412  }
413 }
414 
415 void
417 {
418  debugs(20, 3, HERE << "destroyStoreEntry: destroying " << data);
419  StoreEntry *e = static_cast<StoreEntry *>(static_cast<hash_link *>(data));
420  assert(e != NULL);
421 
422  // Store::Root() is FATALly missing during shutdown
423  if (e->hasDisk() && !shutting_down)
424  e->disk().disconnect(*e);
425 
426  e->destroyMemObject();
427 
428  e->hashDelete();
429 
430  assert(e->key == NULL);
431 
432  delete e;
433 }
434 
435 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
436 
437 void
439 {
440  debugs(20, 3, "StoreEntry::hashInsert: Inserting Entry " << *this << " key '" << storeKeyText(someKey) << "'");
441  assert(!key);
442  key = storeKeyDup(someKey);
443  hash_join(store_table, this);
444 }
445 
446 void
448 {
449  if (key) { // some test cases do not create keys and do not hashInsert()
451  storeKeyFree((const cache_key *)key);
452  key = NULL;
453  }
454 }
455 
456 /* -------------------------------------------------------------------------- */
457 
458 void
459 StoreEntry::lock(const char *context)
460 {
461  ++lock_count;
462  debugs(20, 3, context << " locked key " << getMD5Text() << ' ' << *this);
463 }
464 
465 void
467 {
469 }
470 
471 void
472 StoreEntry::releaseRequest(const bool shareable)
473 {
474  debugs(20, 3, shareable << ' ' << *this);
475  if (!shareable)
476  shareableWhenPrivate = false; // may already be false
478  return;
479  setPrivateKey(shareable, true);
480 }
481 
482 int
483 StoreEntry::unlock(const char *context)
484 {
485  debugs(20, 3, (context ? context : "somebody") <<
486  " unlocking key " << getMD5Text() << ' ' << *this);
487  assert(lock_count > 0);
488  --lock_count;
489 
490  if (lock_count)
491  return (int) lock_count;
492 
493  abandon(context);
494  return 0;
495 }
496 
499 void
500 StoreEntry::doAbandon(const char *context)
501 {
502  debugs(20, 5, *this << " via " << (context ? context : "somebody"));
503  assert(!locked());
504  assert(storePendingNClients(this) == 0);
505 
506  // Both aborted local writers and aborted local readers (of remote writers)
507  // are STORE_PENDING, but aborted readers should never release().
509  (store_status == STORE_PENDING && !Store::Root().transientsReader(*this))) {
510  this->release();
511  return;
512  }
513 
515  debugs(20, DBG_IMPORTANT, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE");
516 
517  Store::Root().handleIdleEntry(*this); // may delete us
518 }
519 
520 void
522 {
523  assert (aClient);
525 }
526 
527 void
529 {
530  assert (aClient);
532 }
533 
534 void
535 StoreEntry::getPublic (StoreClient *aClient, const char *uri, const HttpRequestMethod& method)
536 {
537  assert (aClient);
538  aClient->created(storeGetPublic(uri, method));
539 }
540 
541 StoreEntry *
542 storeGetPublic(const char *uri, const HttpRequestMethod& method)
543 {
544  return Store::Root().find(storeKeyPublic(uri, method));
545 }
546 
547 StoreEntry *
549 {
550  return Store::Root().find(storeKeyPublicByRequestMethod(req, method, keyScope));
551 }
552 
553 StoreEntry *
555 {
556  StoreEntry *e = storeGetPublicByRequestMethod(req, req->method, keyScope);
557 
558  if (e == NULL && req->method == Http::METHOD_HEAD)
559  /* We can generate a HEAD reply from a cached GET object */
561 
562  return e;
563 }
564 
565 static int
567 {
568  static int key_counter = 0;
569 
570  if (++key_counter < 0)
571  key_counter = 1;
572 
573  return key_counter;
574 }
575 
576 /* RBC 20050104 AFAICT this should become simpler:
577  * rather than reinserting with a special key it should be marked
578  * as 'released' and then cleaned up when refcounting indicates.
579  * the StoreHashIndex could well implement its 'released' in the
580  * current manner.
581  * Also, clean log writing should skip over ia,t
582  * Otherwise, we need a 'remove from the index but not the store
583  * concept'.
584  */
585 void
586 StoreEntry::setPrivateKey(const bool shareable, const bool permanent)
587 {
588  debugs(20, 3, shareable << permanent << ' ' << *this);
589  if (permanent)
590  EBIT_SET(flags, RELEASE_REQUEST); // may already be set
591  if (!shareable)
592  shareableWhenPrivate = false; // may already be false
593 
595  return;
596 
597  if (key) {
598  Store::Root().evictCached(*this); // all caches/workers will know
599  hashDelete();
600  }
601 
602  if (mem_obj && mem_obj->hasUris())
603  mem_obj->id = getKeyCounter();
604  const cache_key *newkey = storeKeyPrivate();
605 
606  assert(hash_lookup(store_table, newkey) == NULL);
608  shareableWhenPrivate = shareable;
609  hashInsert(newkey);
610 }
611 
612 bool
614 {
615  debugs(20, 3, *this);
616  if (key && !EBIT_TEST(flags, KEY_PRIVATE))
617  return true; // already public
618 
619  assert(mem_obj);
620 
621  /*
622  * We can't make RELEASE_REQUEST objects public. Depending on
623  * when RELEASE_REQUEST gets set, we might not be swapping out
624  * the object. If we're not swapping out, then subsequent
625  * store clients won't be able to access object data which has
626  * been freed from memory.
627  *
628  * If RELEASE_REQUEST is set, setPublicKey() should not be called.
629  */
630 #if MORE_DEBUG_OUTPUT
631 
633  debugs(20, DBG_IMPORTANT, "assertion failed: RELEASE key " << key << ", url " << mem_obj->url);
634 
635 #endif
636 
638 
639  try {
640  EntryGuard newVaryMarker(adjustVary(), "setPublicKey+failure");
641  const cache_key *pubKey = calcPublicKey(scope);
642  Store::Root().addWriting(this, pubKey);
643  forcePublicKey(pubKey);
644  newVaryMarker.unlockAndReset("setPublicKey+success");
645  return true;
646  } catch (const std::exception &ex) {
647  debugs(20, 2, "for " << *this << " failed: " << ex.what());
648  }
649  return false;
650 }
651 
652 void
654 {
655  if (!key || EBIT_TEST(flags, KEY_PRIVATE))
656  return; // probably the old public key was deleted or made private
657 
658  // TODO: adjustVary() when collapsed revalidation supports that
659 
660  const cache_key *newKey = calcPublicKey(ksDefault);
661  if (!storeKeyHashCmp(key, newKey))
662  return; // probably another collapsed revalidation beat us to this change
663 
664  forcePublicKey(newKey);
665 }
666 
669 void
671 {
672  debugs(20, 3, storeKeyText(newkey) << " for " << *this);
673  assert(mem_obj);
674 
675  if (StoreEntry *e2 = (StoreEntry *)hash_lookup(store_table, newkey)) {
676  assert(e2 != this);
677  debugs(20, 3, "releasing clashing " << *e2);
678  e2->release(true);
679  }
680 
681  if (key)
682  hashDelete();
683 
684  clearPrivate();
685 
686  assert(mem_obj->hasUris());
687  hashInsert(newkey);
688 
689  if (hasDisk())
691 }
692 
695 const cache_key *
697 {
698  assert(mem_obj);
699  return mem_obj->request ? storeKeyPublicByRequest(mem_obj->request.getRaw(), keyScope) :
700  storeKeyPublic(mem_obj->storeId(), mem_obj->method, keyScope);
701 }
702 
708 StoreEntry *
710 {
711  assert(mem_obj);
712 
713  if (!mem_obj->request)
714  return nullptr;
715 
717  const auto &reply = mem_obj->freshestReply();
718 
719  if (mem_obj->vary_headers.isEmpty()) {
720  /* First handle the case where the object no longer varies */
721  request->vary_headers.clear();
722  } else {
723  if (!request->vary_headers.isEmpty() && request->vary_headers.cmp(mem_obj->vary_headers) != 0) {
724  /* Oops.. the variance has changed. Kill the base object
725  * to record the new variance key
726  */
727  request->vary_headers.clear(); /* free old "bad" variance key */
729  pe->release(true);
730  }
731 
732  /* Make sure the request knows the variance status */
733  if (request->vary_headers.isEmpty())
734  request->vary_headers = httpMakeVaryMark(request.getRaw(), &reply);
735  }
736 
737  // TODO: storeGetPublic() calls below may create unlocked entries.
738  // We should add/use storeHas() API or lock/unlock those entries.
740  /* Create "vary" base object */
742  // XXX: storeCreateEntry() already tries to make `pe` public under
743  // certain conditions. If those conditions do not apply to Vary markers,
744  // then refactor to call storeCreatePureEntry() above. Otherwise,
745  // refactor to simply check whether `pe` is already public below.
746  if (!pe->makePublic()) {
747  pe->unlock("StoreEntry::adjustVary+failed_makePublic");
748  throw TexcHere("failed to make Vary marker public");
749  }
750  /* We are allowed to do this typecast */
751  const HttpReplyPointer rep(new HttpReply);
752  rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
753  auto vary = reply.header.getList(Http::HdrType::VARY);
754 
755  if (vary.size()) {
756  /* Again, we own this structure layout */
757  rep->header.putStr(Http::HdrType::VARY, vary.termedBuf());
758  vary.clean();
759  }
760 
761 #if X_ACCELERATOR_VARY
762  vary = reply.header.getList(Http::HdrType::HDR_X_ACCELERATOR_VARY);
763 
764  if (vary.size() > 0) {
765  /* Again, we own this structure layout */
766  rep->header.putStr(Http::HdrType::HDR_X_ACCELERATOR_VARY, vary.termedBuf());
767  vary.clean();
768  }
769 
770 #endif
771  pe->replaceHttpReply(rep, false); // no write until timestampsSet()
772 
773  pe->timestampsSet();
774 
775  pe->startWriting(); // after timestampsSet()
776 
777  pe->complete();
778 
779  return pe;
780  }
781  return nullptr;
782 }
783 
784 StoreEntry *
785 storeCreatePureEntry(const char *url, const char *log_url, const HttpRequestMethod& method)
786 {
787  StoreEntry *e = NULL;
788  debugs(20, 3, "storeCreateEntry: '" << url << "'");
789 
790  e = new StoreEntry();
791  e->createMemObject(url, log_url, method);
792 
794  e->refcount = 0;
795  e->lastref = squid_curtime;
796  e->timestamp = -1; /* set in StoreEntry::timestampsSet() */
797  e->ping_status = PING_NONE;
799  return e;
800 }
801 
802 StoreEntry *
803 storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod& method)
804 {
806  e->lock("storeCreateEntry");
807 
808  if (!neighbors_do_private_keys && flags.hierarchical && flags.cachable && e->setPublicKey())
809  return e;
810 
811  e->setPrivateKey(false, !flags.cachable);
812  return e;
813 }
814 
815 /* Mark object as expired */
816 void
818 {
819  debugs(20, 3, "StoreEntry::expireNow: '" << getMD5Text() << "'");
821 }
822 
823 void
825 {
826  assert(mem_obj != NULL);
827  /* This assert will change when we teach the store to update */
828  PROF_start(StoreEntry_write);
830 
831  // XXX: caller uses content offset, but we also store headers
832  writeBuffer.offset += mem_obj->baseReply().hdr_sz;
833 
834  debugs(20, 5, "storeWrite: writing " << writeBuffer.length << " bytes for '" << getMD5Text() << "'");
835  PROF_stop(StoreEntry_write);
836  storeGetMemSpace(writeBuffer.length);
837  mem_obj->write(writeBuffer);
838 
840  debugs(20, 3, "allow Store clients to get entry content after buffering too much for " << *this);
842  }
843 
844  invokeHandlers();
845 }
846 
847 /* Append incoming data from a primary server to an entry. */
848 void
849 StoreEntry::append(char const *buf, int len)
850 {
851  assert(mem_obj != NULL);
852  assert(len >= 0);
854 
855  StoreIOBuffer tempBuffer;
856  tempBuffer.data = (char *)buf;
857  tempBuffer.length = len;
858  /*
859  * XXX sigh, offset might be < 0 here, but it gets "corrected"
860  * later. This offset crap is such a mess.
861  */
862  tempBuffer.offset = mem_obj->endOffset() - mem_obj->baseReply().hdr_sz;
863  write(tempBuffer);
864 }
865 
866 void
867 StoreEntry::vappendf(const char *fmt, va_list vargs)
868 {
869  LOCAL_ARRAY(char, buf, 4096);
870  *buf = 0;
871  int x;
872 
873  va_list ap;
874  /* Fix of bug 753r. The value of vargs is undefined
875  * after vsnprintf() returns. Make a copy of vargs
876  * in case we loop around and call vsnprintf() again.
877  */
878  va_copy(ap,vargs);
879  errno = 0;
880  if ((x = vsnprintf(buf, sizeof(buf), fmt, ap)) < 0) {
881  fatal(xstrerr(errno));
882  return;
883  }
884  va_end(ap);
885 
886  if (x < static_cast<int>(sizeof(buf))) {
887  append(buf, x);
888  return;
889  }
890 
891  // okay, do it the slow way.
892  char *buf2 = new char[x+1];
893  int y = vsnprintf(buf2, x+1, fmt, vargs);
894  assert(y >= 0 && y == x);
895  append(buf2, y);
896  delete[] buf2;
897 }
898 
899 // deprecated. use StoreEntry::appendf() instead.
900 void
901 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
902 {
903  va_list args;
904  va_start(args, fmt);
905  e->vappendf(fmt, args);
906  va_end(args);
907 }
908 
909 // deprecated. use StoreEntry::appendf() instead.
910 void
911 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
912 {
913  e->vappendf(fmt, vargs);
914 }
915 
917 
918  struct {
919  int non_get;
922  int too_big;
928  } no;
929 
930  struct {
931  int Default;
932  } yes;
934 
935 int
937 {
938  if (Config.max_open_disk_fds == 0)
939  return 0;
940 
942  return 1;
943 
944  return 0;
945 }
946 
947 int
949 {
951  return 0;
952 
953  if (STORE_OK == store_status)
954  if (mem_obj->object_sz >= 0 &&
956  return 1;
957 
958  const auto clen = mem().baseReply().content_length;
959  if (clen >= 0 && clen < Config.Store.minObjectSize)
960  return 1;
961  return 0;
962 }
963 
964 bool
966 {
968  return true;
969 
970  const auto clen = mem_obj->baseReply().content_length;
971  return (clen >= 0 && clen > store_maxobjsize);
972 }
973 
974 // TODO: move "too many open..." checks outside -- we are called too early/late
975 bool
977 {
978  // XXX: This method is used for both memory and disk caches, but some
979  // checks are specific to disk caches. Move them to mayStartSwapOut().
980 
981  // XXX: This method may be called several times, sometimes with different
982  // outcomes, making store_check_cachable_hist counters misleading.
983 
984  // check this first to optimize handling of repeated calls for uncachables
986  debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
988  return 0; // avoid rerequesting release below
989  }
990 
991 #if CACHE_ALL_METHODS
992 
993  if (mem_obj->method != Http::METHOD_GET) {
994  debugs(20, 2, "StoreEntry::checkCachable: NO: non-GET method");
996  } else
997 #endif
999  debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
1001  } else if (!mem_obj) {
1002  // XXX: In bug 4131, we forgetHit() without mem_obj, so we need
1003  // this segfault protection, but how can we get such a HIT?
1004  debugs(20, 2, "StoreEntry::checkCachable: NO: missing parts: " << *this);
1006  } else if (checkTooBig()) {
1007  debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
1009  } else if (checkTooSmall()) {
1010  debugs(20, 2, "StoreEntry::checkCachable: NO: too small");
1012  } else if (EBIT_TEST(flags, KEY_PRIVATE)) {
1013  debugs(20, 3, "StoreEntry::checkCachable: NO: private key");
1015  } else if (hasDisk()) {
1016  /*
1017  * the remaining cases are only relevant if we haven't
1018  * started swapping out the object yet.
1019  */
1020  return 1;
1021  } else if (storeTooManyDiskFilesOpen()) {
1022  debugs(20, 2, "StoreEntry::checkCachable: NO: too many disk files open");
1024  } else if (fdNFree() < RESERVED_FD) {
1025  debugs(20, 2, "StoreEntry::checkCachable: NO: too many FD's open");
1027  } else {
1029  return 1;
1030  }
1031 
1032  releaseRequest();
1033  return 0;
1034 }
1035 
1036 void
1038 {
1039  storeAppendPrintf(sentry, "Category\t Count\n");
1040 
1041 #if CACHE_ALL_METHODS
1042 
1043  storeAppendPrintf(sentry, "no.non_get\t%d\n",
1045 #endif
1046 
1047  storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
1049  storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
1051  storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
1052  0); // TODO: Remove this backward compatibility hack.
1053  storeAppendPrintf(sentry, "no.missing_parts\t%d\n",
1055  storeAppendPrintf(sentry, "no.too_big\t%d\n",
1057  storeAppendPrintf(sentry, "no.too_small\t%d\n",
1059  storeAppendPrintf(sentry, "no.private_key\t%d\n",
1061  storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
1063  storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
1065  storeAppendPrintf(sentry, "yes.default\t%d\n",
1067 }
1068 
1069 void
1070 StoreEntry::lengthWentBad(const char *reason)
1071 {
1072  debugs(20, 3, "because " << reason << ": " << *this);
1074  releaseRequest();
1075 }
1076 
1077 void
1079 {
1080  debugs(20, 3, "storeComplete: '" << getMD5Text() << "'");
1081 
1082  // To preserve forwarding retries, call FwdState::complete() instead.
1084 
1085  if (store_status != STORE_PENDING) {
1086  /*
1087  * if we're not STORE_PENDING, then probably we got aborted
1088  * and there should be NO clients on this entry
1089  */
1091  assert(mem_obj->nclients == 0);
1092  return;
1093  }
1094 
1096 
1098 
1100 
1102  lengthWentBad("!validLength() in complete()");
1103 
1104 #if USE_CACHE_DIGESTS
1105  if (mem_obj->request)
1107 
1108 #endif
1109  /*
1110  * We used to call invokeHandlers, then storeSwapOut. However,
1111  * Madhukar Reddy <myreddy@persistence.com> reported that
1112  * responses without content length would sometimes get released
1113  * in client_side, thinking that the response is incomplete.
1114  */
1115  invokeHandlers();
1116 }
1117 
1118 /*
1119  * Someone wants to abort this transfer. Set the reason in the
1120  * request structure, call the callback and mark the
1121  * entry for releasing
1122  */
1123 void
1125 {
1128  assert(mem_obj != NULL);
1129  debugs(20, 6, "storeAbort: " << getMD5Text());
1130 
1131  lock("StoreEntry::abort"); /* lock while aborting */
1132  negativeCache();
1133 
1134  releaseRequest();
1135 
1137 
1138  // allow the Store clients to be told about the problem
1140 
1142 
1144 
1145  /* Notify the server side */
1146 
1147  if (mem_obj->abortCallback) {
1149  mem_obj->abortCallback = nullptr;
1150  }
1151 
1152  /* XXX Should we reverse these two, so that there is no
1153  * unneeded disk swapping triggered?
1154  */
1155  /* Notify the client side */
1156  invokeHandlers();
1157 
1158  // abort swap out, invalidating what was created so far (release follows)
1160 
1161  unlock("StoreEntry::abort"); /* unlock */
1162 }
1163 
1167 void
1169 {
1171  if (!shutting_down) // Store::Root() is FATALly missing during shutdown
1174 }
1175 
1176 /* thunk through to Store::Root().maintain(). Note that this would be better still
1177  * if registered against the root store itself, but that requires more complex
1178  * update logic - bigger fish to fry first. Long term each store when
1179  * it becomes active will self register
1180  */
1181 void
1183 {
1184  Store::Root().maintain();
1185 
1186  /* Reregister a maintain event .. */
1187  eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1);
1188 
1189 }
1190 
1191 /* The maximum objects to scan for maintain storage space */
1192 #define MAINTAIN_MAX_SCAN 1024
1193 #define MAINTAIN_MAX_REMOVE 64
1194 
1195 void
1196 StoreEntry::release(const bool shareable)
1197 {
1198  PROF_start(storeRelease);
1199  debugs(20, 3, shareable << ' ' << *this << ' ' << getMD5Text());
1200  /* If, for any reason we can't discard this object because of an
1201  * outstanding request, mark it for pending release */
1202 
1203  if (locked()) {
1204  releaseRequest(shareable);
1205  PROF_stop(storeRelease);
1206  return;
1207  }
1208 
1210  /* TODO: Teach disk stores to handle releases during rebuild instead. */
1211 
1212  // lock the entry until rebuilding is done
1213  lock("storeLateRelease");
1214  releaseRequest(shareable);
1215  LateReleaseStack.push(this);
1216  PROF_stop(storeRelease);
1217  return;
1218  }
1219 
1220  storeLog(STORE_LOG_RELEASE, this);
1221  Store::Root().evictCached(*this);
1222  destroyStoreEntry(static_cast<hash_link *>(this));
1223  PROF_stop(storeRelease);
1224 }
1225 
1226 static void
1228 {
1229  StoreEntry *e;
1230  static int n = 0;
1231 
1233  eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1234  return;
1235  }
1236 
1237  // TODO: this works but looks unelegant.
1238  for (int i = 0; i < 10; ++i) {
1239  if (LateReleaseStack.empty()) {
1240  debugs(20, DBG_IMPORTANT, "storeLateRelease: released " << n << " objects");
1241  return;
1242  } else {
1243  e = LateReleaseStack.top();
1244  LateReleaseStack.pop();
1245  }
1246 
1247  e->unlock("storeLateRelease");
1248  ++n;
1249  }
1250 
1251  eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
1252 }
1253 
1257 bool
1259 {
1260  int64_t diff;
1261  assert(mem_obj != NULL);
1262  const auto reply = &mem_obj->baseReply();
1263  debugs(20, 3, "storeEntryValidLength: Checking '" << getMD5Text() << "'");
1264  debugs(20, 5, "storeEntryValidLength: object_len = " <<
1265  objectLen());
1266  debugs(20, 5, "storeEntryValidLength: hdr_sz = " << reply->hdr_sz);
1267  debugs(20, 5, "storeEntryValidLength: content_length = " << reply->content_length);
1268 
1269  if (reply->content_length < 0) {
1270  debugs(20, 5, "storeEntryValidLength: Unspecified content length: " << getMD5Text());
1271  return 1;
1272  }
1273 
1274  if (reply->hdr_sz == 0) {
1275  debugs(20, 5, "storeEntryValidLength: Zero header size: " << getMD5Text());
1276  return 1;
1277  }
1278 
1279  if (mem_obj->method == Http::METHOD_HEAD) {
1280  debugs(20, 5, "storeEntryValidLength: HEAD request: " << getMD5Text());
1281  return 1;
1282  }
1283 
1284  if (reply->sline.status() == Http::scNotModified)
1285  return 1;
1286 
1287  if (reply->sline.status() == Http::scNoContent)
1288  return 1;
1289 
1290  diff = reply->hdr_sz + reply->content_length - objectLen();
1291 
1292  if (diff == 0)
1293  return 1;
1294 
1295  debugs(20, 3, "storeEntryValidLength: " << (diff < 0 ? -diff : diff) << " bytes too " << (diff < 0 ? "big" : "small") <<"; '" << getMD5Text() << "'" );
1296 
1297  return 0;
1298 }
1299 
1300 static void
1302 {
1303  Mgr::RegisterAction("storedir", "Store Directory Stats", Store::Stats, 0, 1);
1304  Mgr::RegisterAction("store_io", "Store IO Interface Stats", &Mgr::StoreIoAction::Create, 0, 1);
1305  Mgr::RegisterAction("store_check_cachable_stats", "storeCheckCachable() Stats",
1306  storeCheckCachableStats, 0, 1);
1307  Mgr::RegisterAction("store_queues", "SMP Transients and Caching Queues", StatQueues, 0, 1);
1308 }
1309 
1310 void
1312 {
1313  storeKeyInit();
1315  storeDigestInit();
1316  storeLogOpen();
1317  eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1318  Store::Root().init();
1320 
1322 }
1323 
1324 void
1326 {
1328 }
1329 
1330 bool
1332 {
1333  if (!checkCachable())
1334  return 0;
1335 
1336  if (mem_obj == NULL)
1337  return 0;
1338 
1339  if (mem_obj->data_hdr.size() == 0)
1340  return 0;
1341 
1342  if (mem_obj->inmem_lo != 0)
1343  return 0;
1344 
1346  return 0;
1347 
1348  return 1;
1349 }
1350 
1351 int
1353 {
1355  return 0;
1356 
1357  if (expires <= squid_curtime)
1358  return 0;
1359 
1360  if (store_status != STORE_OK)
1361  return 0;
1362 
1363  return 1;
1364 }
1365 
1372 void
1374 {
1375  // XXX: should make the default for expires 0 instead of -1
1376  // so we can distinguish "Expires: -1" from nothing.
1377  if (expires <= 0)
1378 #if USE_HTTP_VIOLATIONS
1380 #else
1382 #endif
1383  if (expires > squid_curtime) {
1385  debugs(20, 6, "expires = " << expires << " +" << (expires-squid_curtime) << ' ' << *this);
1386  }
1387 }
1388 
1389 void
1391 {
1393 #if USE_CACHE_DIGESTS
1394  delete store_digest;
1395 #endif
1396  store_digest = NULL;
1397 }
1398 
1399 int
1400 expiresMoreThan(time_t expires, time_t when)
1401 {
1402  if (expires < 0) /* No Expires given */
1403  return 1;
1404 
1405  return (expires > (squid_curtime + when));
1406 }
1407 
1408 int
1410 {
1412  return 0;
1413 
1415  if (expires <= squid_curtime)
1416  return 0;
1417 
1419  return 0;
1420 
1421  // now check that the entry has a cache backing or is collapsed
1422  if (hasDisk()) // backed by a disk cache
1423  return 1;
1424 
1425  if (swappingOut()) // will be backed by a disk cache
1426  return 1;
1427 
1428  if (!mem_obj) // not backed by a memory cache and not collapsed
1429  return 0;
1430 
1431  // StoreEntry::storeClientType() assumes DISK_CLIENT here, but there is no
1432  // disk cache backing that store_client constructor will assert. XXX: This
1433  // is wrong for range requests (that could feed off nibbled memory) and for
1434  // entries backed by the shared memory cache (that could, in theory, get
1435  // nibbled bytes from that cache, but there is no such "memoryIn" code).
1436  if (mem_obj->inmem_lo) // in memory cache, but got nibbled at
1437  return 0;
1438 
1439  // The following check is correct but useless at this position. TODO: Move
1440  // it up when the shared memory cache can either replenish locally nibbled
1441  // bytes or, better, does not use local RAM copy at all.
1442  // if (mem_obj->memCache.index >= 0) // backed by a shared memory cache
1443  // return 1;
1444 
1445  return 1;
1446 }
1447 
1448 bool
1450 {
1451  debugs(20, 7, *this << " had " << describeTimestamps());
1452 
1453  // TODO: Remove change-reducing "&" before the official commit.
1454  const auto reply = &mem().freshestReply();
1455 
1456  time_t served_date = reply->date;
1457  int age = reply->header.getInt(Http::HdrType::AGE);
1458  /* Compute the timestamp, mimicking RFC2616 section 13.2.3. */
1459  /* make sure that 0 <= served_date <= squid_curtime */
1460 
1461  if (served_date < 0 || served_date > squid_curtime)
1462  served_date = squid_curtime;
1463 
1464  /* Bug 1791:
1465  * If the returned Date: is more than 24 hours older than
1466  * the squid_curtime, then one of us needs to use NTP to set our
1467  * clock. We'll pretend that our clock is right.
1468  */
1469  else if (served_date < (squid_curtime - 24 * 60 * 60) )
1470  served_date = squid_curtime;
1471 
1472  /*
1473  * Compensate with Age header if origin server clock is ahead
1474  * of us and there is a cache in between us and the origin
1475  * server. But DONT compensate if the age value is larger than
1476  * squid_curtime because it results in a negative served_date.
1477  */
1478  if (age > squid_curtime - served_date)
1479  if (squid_curtime > age)
1480  served_date = squid_curtime - age;
1481 
1482  // compensate for Squid-to-server and server-to-Squid delays
1483  if (mem_obj && mem_obj->request) {
1484  struct timeval responseTime;
1485  if (mem_obj->request->hier.peerResponseTime(responseTime))
1486  served_date -= responseTime.tv_sec;
1487  }
1488 
1489  time_t exp = 0;
1490  if (reply->expires > 0 && reply->date > -1)
1491  exp = served_date + (reply->expires - reply->date);
1492  else
1493  exp = reply->expires;
1494 
1495  if (timestamp == served_date && expires == exp) {
1496  // if the reply lacks LMT, then we now know that our effective
1497  // LMT (i.e., timestamp) will stay the same, otherwise, old and
1498  // new modification times must match
1499  if (reply->last_modified < 0 || reply->last_modified == lastModified())
1500  return false; // nothing has changed
1501  }
1502 
1503  expires = exp;
1504 
1505  lastModified_ = reply->last_modified;
1506 
1507  timestamp = served_date;
1508 
1509  debugs(20, 5, *this << " has " << describeTimestamps());
1510  return true;
1511 }
1512 
1513 bool
1515 {
1516  assert(mem_obj);
1517  assert(e304.mem_obj);
1518 
1519  // update reply before calling timestampsSet() below
1520  const auto &oldReply = mem_obj->freshestReply();
1521  const auto updatedReply = oldReply.recreateOnNotModified(e304.mem_obj->baseReply());
1522  if (updatedReply) // HTTP 304 brought in new information
1523  mem_obj->updateReply(*updatedReply);
1524  // else continue to use the previous update, if any
1525 
1526  if (!timestampsSet() && !updatedReply)
1527  return false;
1528 
1529  // Keep the old mem_obj->vary_headers; see HttpHeader::skipUpdateHeader().
1530 
1531  debugs(20, 5, "updated basics in " << *this << " with " << e304);
1532  mem_obj->appliedUpdates = true; // helps in triage; may already be true
1533  return true;
1534 }
1535 
1536 void
1538 {
1539  assert(mem_obj);
1542 }
1543 
1544 void
1546 {
1547  assert(mem_obj);
1548  if (mem_obj->abortCallback) {
1549  mem_obj->abortCallback->cancel(reason);
1550  mem_obj->abortCallback = nullptr;
1551  }
1552 }
1553 
1554 void
1555 StoreEntry::dump(int l) const
1556 {
1557  debugs(20, l, "StoreEntry->key: " << getMD5Text());
1558  debugs(20, l, "StoreEntry->next: " << next);
1559  debugs(20, l, "StoreEntry->mem_obj: " << mem_obj);
1560  debugs(20, l, "StoreEntry->timestamp: " << timestamp);
1561  debugs(20, l, "StoreEntry->lastref: " << lastref);
1562  debugs(20, l, "StoreEntry->expires: " << expires);
1563  debugs(20, l, "StoreEntry->lastModified_: " << lastModified_);
1564  debugs(20, l, "StoreEntry->swap_file_sz: " << swap_file_sz);
1565  debugs(20, l, "StoreEntry->refcount: " << refcount);
1566  debugs(20, l, "StoreEntry->flags: " << storeEntryFlags(this));
1567  debugs(20, l, "StoreEntry->swap_dirn: " << swap_dirn);
1568  debugs(20, l, "StoreEntry->swap_filen: " << swap_filen);
1569  debugs(20, l, "StoreEntry->lock_count: " << lock_count);
1570  debugs(20, l, "StoreEntry->mem_status: " << mem_status);
1571  debugs(20, l, "StoreEntry->ping_status: " << ping_status);
1572  debugs(20, l, "StoreEntry->store_status: " << store_status);
1573  debugs(20, l, "StoreEntry->swap_status: " << swap_status);
1574 }
1575 
1576 /*
1577  * NOTE, this function assumes only two mem states
1578  */
1579 void
1581 {
1582  if (new_status == mem_status)
1583  return;
1584 
1585  // are we using a shared memory cache?
1586  if (MemStore::Enabled()) {
1587  // This method was designed to update replacement policy, not to
1588  // actually purge something from the memory cache (TODO: rename?).
1589  // Shared memory cache does not have a policy that needs updates.
1590  mem_status = new_status;
1591  return;
1592  }
1593 
1594  assert(mem_obj != NULL);
1595 
1596  if (new_status == IN_MEMORY) {
1597  assert(mem_obj->inmem_lo == 0);
1598 
1599  if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1600  debugs(20, 4, "not inserting special " << *this << " into policy");
1601  } else {
1602  mem_policy->Add(mem_policy, this, &mem_obj->repl);
1603  debugs(20, 4, "inserted " << *this << " key: " << getMD5Text());
1604  }
1605 
1606  ++hot_obj_count; // TODO: maintain for the shared hot cache as well
1607  } else {
1608  if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1609  debugs(20, 4, "not removing special " << *this << " from policy");
1610  } else {
1612  debugs(20, 4, "removed " << *this);
1613  }
1614 
1615  --hot_obj_count;
1616  }
1617 
1618  mem_status = new_status;
1619 }
1620 
1621 const char *
1623 {
1624  if (mem_obj == NULL)
1625  return "[null_mem_obj]";
1626  else
1627  return mem_obj->storeId();
1628 }
1629 
1630 void
1632 {
1633  assert(!mem_obj);
1634  mem_obj = new MemObject();
1635 }
1636 
1637 void
1638 StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl, const HttpRequestMethod &aMethod)
1639 {
1640  assert(!mem_obj);
1641  ensureMemObject(aUrl, aLogUrl, aMethod);
1642 }
1643 
1644 void
1645 StoreEntry::ensureMemObject(const char *aUrl, const char *aLogUrl, const HttpRequestMethod &aMethod)
1646 {
1647  if (!mem_obj)
1648  mem_obj = new MemObject();
1649  mem_obj->setUris(aUrl, aLogUrl, aMethod);
1650 }
1651 
1656 void
1658 {
1660 }
1661 
1667 void
1669 {
1670  if (EBIT_TEST(flags, DELAY_SENDING)) {
1672  invokeHandlers();
1673  }
1674 }
1675 
1676 void
1678 {
1679  debugs(20, 3, url());
1680  mem().reset();
1681  expires = lastModified_ = timestamp = -1;
1682 }
1683 
1684 /*
1685  * storeFsInit
1686  *
1687  * This routine calls the SETUP routine for each fs type.
1688  * I don't know where the best place for this is, and I'm not going to shuffle
1689  * around large chunks of code right now (that can be done once its working.)
1690  */
1691 void
1693 {
1694  storeReplSetup();
1695 }
1696 
1697 /*
1698  * called to add another store removal policy module
1699  */
1700 void
1701 storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
1702 {
1703  int i;
1704 
1705  /* find the number of currently known repl types */
1706  for (i = 0; storerepl_list && storerepl_list[i].typestr; ++i) {
1707  if (strcmp(storerepl_list[i].typestr, type) == 0) {
1708  debugs(20, DBG_IMPORTANT, "WARNING: Trying to load store replacement policy " << type << " twice.");
1709  return;
1710  }
1711  }
1712 
1713  /* add the new type */
1714  storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
1715 
1716  memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
1717 
1719 
1720  storerepl_list[i].create = create;
1721 }
1722 
1723 /*
1724  * Create a removal policy instance
1725  */
1726 RemovalPolicy *
1728 {
1729  storerepl_entry_t *r;
1730 
1731  for (r = storerepl_list; r && r->typestr; ++r) {
1732  if (strcmp(r->typestr, settings->type) == 0)
1733  return r->create(settings->args);
1734  }
1735 
1736  debugs(20, DBG_IMPORTANT, "ERROR: Unknown policy " << settings->type);
1737  debugs(20, DBG_IMPORTANT, "ERROR: Be sure to have set cache_replacement_policy");
1738  debugs(20, DBG_IMPORTANT, "ERROR: and memory_replacement_policy in squid.conf!");
1739  fatalf("ERROR: Unknown policy %s\n", settings->type);
1740  return NULL; /* NOTREACHED */
1741 }
1742 
1743 #if 0
1744 void
1746 {
1747  if (e->swap_file_number == filn)
1748  return;
1749 
1750  if (filn < 0) {
1751  assert(-1 == filn);
1752  storeDirMapBitReset(e->swap_file_number);
1753  storeDirLRUDelete(e);
1754  e->swap_file_number = -1;
1755  } else {
1756  assert(-1 == e->swap_file_number);
1757  storeDirMapBitSet(e->swap_file_number = filn);
1758  storeDirLRUAdd(e);
1759  }
1760 }
1761 
1762 #endif
1763 
1764 void
1766 {
1767  lock("StoreEntry::storeErrorResponse");
1768  buffer();
1770  flush();
1771  complete();
1772  negativeCache();
1773  releaseRequest(false); // if it is safe to negatively cache, sharing is OK
1774  unlock("StoreEntry::storeErrorResponse");
1775 }
1776 
1777 /*
1778  * Replace a store entry with
1779  * a new reply. This eats the reply.
1780  */
1781 void
1782 StoreEntry::replaceHttpReply(const HttpReplyPointer &rep, const bool andStartWriting)
1783 {
1784  debugs(20, 3, "StoreEntry::replaceHttpReply: " << url());
1785 
1786  if (!mem_obj) {
1787  debugs(20, DBG_CRITICAL, "Attempt to replace object with no in-memory representation");
1788  return;
1789  }
1790 
1791  mem_obj->replaceBaseReply(rep);
1792 
1793  if (andStartWriting)
1794  startWriting();
1795 }
1796 
1797 void
1799 {
1800  /* TODO: when we store headers separately remove the header portion */
1801  /* TODO: mark the length of the headers ? */
1802  /* We ONLY want the headers */
1803  assert (isEmpty());
1804  assert(mem_obj);
1805 
1806  // Per MemObject replies definitions, we can only write our base reply.
1807  // Currently, all callers replaceHttpReply() first, so there is no updated
1808  // reply here anyway. Eventually, we may need to support the
1809  // updateOnNotModified(),startWriting() sequence as well.
1811  const auto rep = &mem_obj->baseReply();
1812 
1813  buffer();
1814  rep->packHeadersUsingSlowPacker(*this);
1816 
1817  rep->body.packInto(this);
1818  flush();
1819 
1820  // The entry headers are written, new clients
1821  // should not collapse anymore.
1822  if (hittingRequiresCollapsing()) {
1823  setCollapsingRequirement(false);
1825  }
1826 }
1827 
1828 char const *
1830 {
1831  StoreMeta *tlv_list = storeSwapMetaBuild(this);
1832  int swap_hdr_sz;
1833  char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
1834  storeSwapTLVFree(tlv_list);
1835  assert (swap_hdr_sz >= 0);
1836  length = static_cast<size_t>(swap_hdr_sz);
1837  return result;
1838 }
1839 
1846 void
1848 {
1849  if (mem_obj && !Store::Root().transientsReader(*this) && // this worker is responsible
1850  hasTransients() && // other workers may be interested
1851  !hasMemStore() && // rejected by the shared memory cache
1853  debugs(20, 7, "cannot be shared: " << *this);
1854  if (!shutting_down) // Store::Root() is FATALly missing during shutdown
1855  Store::Root().stopSharing(*this);
1856  }
1857 }
1858 
1859 void
1861 {
1863 }
1864 
1865 void
1867 {
1868  // Abandon our transient entry if neither shared memory nor disk wants it.
1869  assert(mem_obj);
1870  mem_obj->swapout.decision = decision;
1872 }
1873 
1874 void
1875 StoreEntry::trimMemory(const bool preserveSwappable)
1876 {
1877  /*
1878  * DPW 2007-05-09
1879  * Bug #1943. We must not let go any data for IN_MEMORY
1880  * objects. We have to wait until the mem_status changes.
1881  */
1882  if (mem_status == IN_MEMORY)
1883  return;
1884 
1886  return; // cannot trim because we do not load them again
1887 
1888  if (preserveSwappable)
1890  else
1892 
1893  debugs(88, 7, *this << " inmem_lo=" << mem_obj->inmem_lo);
1894 }
1895 
1896 bool
1897 StoreEntry::modifiedSince(const time_t ims, const int imslen) const
1898 {
1899  const time_t mod_time = lastModified();
1900 
1901  debugs(88, 3, "modifiedSince: '" << url() << "'");
1902 
1903  debugs(88, 3, "modifiedSince: mod_time = " << mod_time);
1904 
1905  if (mod_time < 0)
1906  return true;
1907 
1908  assert(imslen < 0); // TODO: Either remove imslen or support it properly.
1909 
1910  if (mod_time > ims) {
1911  debugs(88, 3, "--> YES: entry newer than client");
1912  return true;
1913  } else if (mod_time < ims) {
1914  debugs(88, 3, "--> NO: entry older than client");
1915  return false;
1916  } else {
1917  debugs(88, 3, "--> NO: same LMT");
1918  return false;
1919  }
1920 }
1921 
1922 bool
1924 {
1925  if (const auto reply = hasFreshestReply()) {
1926  etag = reply->header.getETag(Http::HdrType::ETAG);
1927  if (etag.str)
1928  return true;
1929  }
1930  return false;
1931 }
1932 
1933 bool
1935 {
1936  const String reqETags = request.header.getList(Http::HdrType::IF_MATCH);
1937  return hasOneOfEtags(reqETags, false);
1938 }
1939 
1940 bool
1942 {
1943  const String reqETags = request.header.getList(Http::HdrType::IF_NONE_MATCH);
1944  // weak comparison is allowed only for HEAD or full-body GET requests
1945  const bool allowWeakMatch = !request.flags.isRanged &&
1947  return hasOneOfEtags(reqETags, allowWeakMatch);
1948 }
1949 
1951 bool
1952 StoreEntry::hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
1953 {
1954  const auto repETag = mem().freshestReply().header.getETag(Http::HdrType::ETAG);
1955  if (!repETag.str) {
1956  static SBuf asterisk("*", 1);
1957  return strListIsMember(&reqETags, asterisk, ',');
1958  }
1959 
1960  bool matched = false;
1961  const char *pos = NULL;
1962  const char *item;
1963  int ilen;
1964  while (!matched && strListGetItem(&reqETags, ',', &item, &ilen, &pos)) {
1965  if (!strncmp(item, "*", ilen))
1966  matched = true;
1967  else {
1968  String str;
1969  str.append(item, ilen);
1970  ETag reqETag;
1971  if (etagParseInit(&reqETag, str.termedBuf())) {
1972  matched = allowWeakMatch ? etagIsWeakEqual(repETag, reqETag) :
1973  etagIsStrongEqual(repETag, reqETag);
1974  }
1975  }
1976  }
1977  return matched;
1978 }
1979 
1980 Store::Disk &
1982 {
1983  assert(hasDisk());
1985  assert(sd);
1986  return *sd;
1987 }
1988 
1989 bool
1990 StoreEntry::hasDisk(const sdirno dirn, const sfileno filen) const
1991 {
1992  checkDisk();
1993  if (dirn < 0 && filen < 0)
1994  return swap_dirn >= 0;
1995  Must(dirn >= 0);
1996  const bool matchingDisk = (swap_dirn == dirn);
1997  return filen < 0 ? matchingDisk : (matchingDisk && swap_filen == filen);
1998 }
1999 
2000 void
2001 StoreEntry::attachToDisk(const sdirno dirn, const sfileno fno, const swap_status_t status)
2002 {
2003  debugs(88, 3, "attaching entry with key " << getMD5Text() << " : " <<
2004  swapStatusStr[status] << " " << dirn << " " <<
2005  std::hex << std::setw(8) << std::setfill('0') <<
2006  std::uppercase << fno);
2007  checkDisk();
2008  swap_dirn = dirn;
2009  swap_filen = fno;
2010  swap_status = status;
2011  checkDisk();
2012 }
2013 
2014 void
2016 {
2017  swap_dirn = -1;
2018  swap_filen = -1;
2020 }
2021 
2022 void
2024 {
2025  try {
2026  if (swap_dirn < 0) {
2027  Must(swap_filen < 0);
2029  } else {
2030  Must(swap_filen >= 0);
2032  if (swapoutFailed()) {
2034  } else {
2035  Must(swappingOut() || swappedOut());
2036  }
2037  }
2038  } catch (...) {
2039  debugs(88, DBG_IMPORTANT, "ERROR: inconsistent disk entry state " <<
2040  *this << "; problem: " << CurrentException);
2041  throw;
2042  }
2043 }
2044 
2045 /*
2046  * return true if the entry is in a state where
2047  * it can accept more data (ie with write() method)
2048  */
2049 bool
2051 {
2052  if (STORE_PENDING != store_status)
2053  return false;
2054 
2056  return false;
2057 
2058  return true;
2059 }
2060 
2061 const char *
2063 {
2064  LOCAL_ARRAY(char, buf, 256);
2065  snprintf(buf, 256, "LV:%-9d LU:%-9d LM:%-9d EX:%-9d",
2066  static_cast<int>(timestamp),
2067  static_cast<int>(lastref),
2068  static_cast<int>(lastModified_),
2069  static_cast<int>(expires));
2070  return buf;
2071 }
2072 
2073 void
2075 {
2076  if (required)
2078  else
2080 }
2081 
2082 static std::ostream &
2083 operator <<(std::ostream &os, const Store::IoStatus &io)
2084 {
2085  switch (io) {
2086  case Store::ioUndecided:
2087  os << 'u';
2088  break;
2089  case Store::ioReading:
2090  os << 'r';
2091  break;
2092  case Store::ioWriting:
2093  os << 'w';
2094  break;
2095  case Store::ioDone:
2096  os << 'o';
2097  break;
2098  }
2099  return os;
2100 }
2101 
2102 std::ostream &operator <<(std::ostream &os, const StoreEntry &e)
2103 {
2104  os << "e:";
2105 
2106  if (e.hasTransients()) {
2107  const auto &xitTable = e.mem_obj->xitTable;
2108  os << 't' << xitTable.io << xitTable.index;
2109  }
2110 
2111  if (e.hasMemStore()) {
2112  const auto &memCache = e.mem_obj->memCache;
2113  os << 'm' << memCache.io << memCache.index << '@' << memCache.offset;
2114  }
2115 
2116  // Do not use e.hasDisk() here because its checkDisk() call may calls us.
2117  if (e.swap_filen > -1 || e.swap_dirn > -1)
2118  os << 'd' << e.swap_filen << '@' << e.swap_dirn;
2119 
2120  os << '=';
2121 
2122  // print only non-default status values, using unique letters
2123  if (e.mem_status != NOT_IN_MEMORY ||
2124  e.store_status != STORE_PENDING ||
2125  e.swap_status != SWAPOUT_NONE ||
2126  e.ping_status != PING_NONE) {
2127  if (e.mem_status != NOT_IN_MEMORY) os << 'm';
2128  if (e.store_status != STORE_PENDING) os << 's';
2129  if (e.swap_status != SWAPOUT_NONE) os << 'w' << e.swap_status;
2130  if (e.ping_status != PING_NONE) os << 'p' << e.ping_status;
2131  }
2132 
2133  // print only set flags, using unique letters
2134  if (e.flags) {
2135  if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) os << 'S';
2136  if (EBIT_TEST(e.flags, ENTRY_REVALIDATE_ALWAYS)) os << 'R';
2137  if (EBIT_TEST(e.flags, DELAY_SENDING)) os << 'P';
2138  if (EBIT_TEST(e.flags, RELEASE_REQUEST)) os << 'X';
2139  if (EBIT_TEST(e.flags, REFRESH_REQUEST)) os << 'F';
2140  if (EBIT_TEST(e.flags, ENTRY_REVALIDATE_STALE)) os << 'E';
2141  if (EBIT_TEST(e.flags, KEY_PRIVATE)) {
2142  os << 'I';
2143  if (e.shareableWhenPrivate)
2144  os << 'H';
2145  }
2146  if (EBIT_TEST(e.flags, ENTRY_FWD_HDR_WAIT)) os << 'W';
2147  if (EBIT_TEST(e.flags, ENTRY_NEGCACHED)) os << 'N';
2148  if (EBIT_TEST(e.flags, ENTRY_VALIDATED)) os << 'V';
2149  if (EBIT_TEST(e.flags, ENTRY_BAD_LENGTH)) os << 'L';
2150  if (EBIT_TEST(e.flags, ENTRY_ABORTED)) os << 'A';
2151  if (EBIT_TEST(e.flags, ENTRY_REQUIRES_COLLAPSING)) os << 'C';
2152  }
2153 
2154  return os << '/' << &e << '*' << e.locks();
2155 }
2156 
2157 void
2159 {
2161  entry_->releaseRequest(false);
2163  });
2164 }
2165 
#define EBIT_CLR(flag, bit)
Definition: defines.h:106
bool cancel(const char *reason)
Definition: AsyncCall.cc:56
int hdr_sz
Definition: Message.h:82
void fatal(const char *message)
Definition: fatal.cc:28
const cache_key * storeKeyPrivate()
const char * xstrerr(int error)
Definition: xstrerror.cc:83
void makePrivate(const bool shareable)
Definition: store.cc:172
REMOVALPOLICYCREATE * create
Definition: store.cc:105
char method[16]
Definition: tcp-banger2.c:115
void memoryDisconnect(StoreEntry &)
disassociates the entry from the memory cache, preserving cached data
Definition: Controller.cc:629
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:309
Definition: store.cc:103
HASHCMP storeKeyHashCmp
void Maintain(void *unused)
Definition: store.cc:1182
virtual void buffer()
Definition: store.cc:1657
void abandon(const char *context)
Definition: Store.h:279
#define INDEXSD(i)
Definition: SquidConfig.h:71
int checkTooSmall()
Definition: store.cc:948
time_t timestamp
Definition: Store.h:216
@ HDR_X_ACCELERATOR_VARY
const cache_key * storeKeyPublicByRequest(HttpRequest *request, const KeyScope keyScope)
@ METHOD_HEAD
Definition: MethodType.h:28
IoStatus
cache "I/O" direction and status
Definition: forward.h:40
const char * typestr
Definition: store.cc:104
int64_t minObjectSize
Definition: SquidConfig.h:270
Definition: ETag.h:17
virtual ~StoreEntry()
Definition: store.cc:371
store_client_t storeClientType() const
Definition: store.cc:287
bool validLength() const
Definition: store.cc:1258
void storeReplAdd(const char *type, REMOVALPOLICYCREATE *create)
Definition: store.cc:1701
bool updateOnNotModified(const StoreEntry &e304)
Definition: store.cc:1514
AsyncCall::Pointer callback
Definition: CommRead.h:28
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1645
const char * storeId() const
Definition: MemObject.cc:54
void unregisterAbortCallback(const char *reason)
Definition: store.cc:1545
virtual void created(StoreEntry *)=0
void releaseRequest(const bool shareable=false)
Definition: store.cc:472
StoreEntry * adjustVary()
Definition: store.cc:709
static OBJH storeCheckCachableStats
Definition: store.cc:114
@ ENTRY_REVALIDATE_STALE
Definition: enums.h:100
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod const KeyScope scope const char const RequestFlags const HttpRequestMethod const char * logUrl
Definition: stub_store.cc:110
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:2023
manages a single cache_dir
Definition: Disk.h:21
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
unsigned char cache_key
Store key.
Definition: forward.h:29
static bool Enabled()
whether Squid is correctly configured to use a shared memory cache
Definition: MemStore.h:68
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:165
HttpHeader header
Definition: Message.h:75
#define EBIT_SET(flag, bit)
Definition: defines.h:105
StoreEntry()
Definition: store.cc:350
int memory_cache_first
Definition: SquidConfig.h:341
bool isEmpty() const
Definition: SBuf.h:420
virtual void flush()
Definition: store.cc:1668
MemObject * mem_obj
Definition: Store.h:213
const cache_key * storeKeyPublic(const char *url, const HttpRequestMethod &method, const KeyScope keyScope)
mem_hdr data_hdr
Definition: MemObject.h:138
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:430
@ DELAY_SENDING
Definition: enums.h:97
void clearPublicKeyScope()
Definition: store.cc:653
C end
Definition: Range.h:25
void destroyStoreEntry(void *data)
Definition: store.cc:416
const char * url() const
Definition: store.cc:1622
void trimUnSwappable()
Definition: MemObject.cc:411
void storeInit(void)
Definition: store.cc:1311
void createMemObject()
Definition: store.cc:1631
MemObject & mem()
Definition: Store.h:52
static void getPublicByRequest(StoreClient *aClient, HttpRequest *request)
Definition: store.cc:528
bool shareableWhenPrivate
Definition: Store.h:323
int etagParseInit(ETag *etag, const char *str)
Definition: ETag.cc:29
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:901
void lock(const char *context)
Definition: store.cc:459
void FreeMemory()
undo Init()
Definition: Controller.cc:951
void transientsAbandonmentCheck()
Definition: store.cc:1847
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
int64_t offset
Definition: StoreIOBuffer.h:55
SQUIDCEXTERN void hash_remove_link(hash_table *, hash_link *)
Definition: hash.cc:224
@ KEY_PRIVATE
Definition: enums.h:102
Definition: SBuf.h:86
virtual void init() override
Definition: Controller.cc:59
void detachFromDisk()
Definition: store.cc:2015
void onException() noexcept
Definition: store.cc:2158
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:466
void setNoDelay(bool const newValue)
Definition: MemObject.cc:444
Io io
current I/O state
Definition: MemObject.h:173
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:586
const char * str
quoted-string
Definition: ETag.h:20
@ REFRESH_REQUEST
Definition: enums.h:99
static std::ostream & operator<<(std::ostream &os, const Store::IoStatus &io)
Definition: store.cc:2083
@ ENTRY_NEGCACHED
Definition: enums.h:112
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:613
Entry * entry_
the guarded Entry or nil
Definition: Store.h:382
bool hierarchical
Definition: RequestFlags.h:34
bool etagIsStrongEqual(const ETag &tag1, const ETag &tag2)
whether etags are strong-equal
Definition: ETag.cc:49
const char * memStatusStr[]
Definition: store.cc:74
int aborted_requests
Definition: StatCounters.h:152
int type
Definition: errorpage.cc:152
C * getRaw() const
Definition: RefCount.h:80
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:220
uint16_t flags
Definition: Store.h:224
struct _store_check_cachable_hist store_check_cachable_hist
void storeGetMemSpace(int size)
Definition: store.cc:1168
bool etagIsWeakEqual(const ETag &tag1, const ETag &tag2)
whether etags are weak-equal
Definition: ETag.cc:55
int hot_obj_count
size_t size() const
Definition: stmem.cc:408
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:264
static storerepl_entry_t * storerepl_list
Definition: store.cc:108
bool isAccepting() const
Definition: store.cc:2050
time_t expires
Definition: Store.h:218
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:427
Io io
current I/O state
Definition: MemObject.h:184
char * storeSwapMetaPack(tlv *tlv_list, int *length)
int64_t endOffset() const
Definition: MemObject.cc:235
static void storeRegisterWithCacheManager(void)
Definition: store.cc:1301
int64_t objectLen() const
Definition: Store.h:252
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1782
#define DBG_CRITICAL
Definition: Debug.h:45
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:55
void startWriting()
Definition: store.cc:1798
void storeSwapFileNumberSet(StoreEntry *e, sfileno filn)
SQUIDCEXTERN void hash_join(hash_table *, hash_link *)
Definition: hash.cc:132
@ SWAP_LOG_ADD
Definition: swap_log_op.h:14
RemovalPolicy * mem_policy
Definition: MemObject.cc:45
#define PROF_stop(probename)
Definition: Profiler.h:63
void storeFsInit(void)
Definition: store.cc:1692
int store_open_disk_fd
void reset()
Definition: store.cc:1677
#define DBG_IMPORTANT
Definition: Debug.h:46
virtual void evictCached(StoreEntry &) override
Definition: Controller.cc:502
StoreEntry * storeGetPublicByRequest(HttpRequest *req, const KeyScope keyScope)
Definition: store.cc:554
void destroyMemObject()
Definition: store.cc:398
int64_t inmem_lo
Definition: MemObject.h:139
int len
Definition: CommRead.h:27
static void getPublic(StoreClient *aClient, const char *uri, const HttpRequestMethod &method)
Definition: store.cc:535
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:612
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *req, const HttpRequestMethod &method, const KeyScope keyScope)
Definition: store.cc:548
#define TexcHere(msg)
legacy convenience macro; it is not difficult to type Here() now
Definition: TextException.h:55
void doAbandon(const char *context)
Definition: store.cc:500
void invokeHandlers()
void replaceBaseReply(const HttpReplyPointer &r)
Definition: MemObject.cc:143
void storeReplSetup(void)
const char * getSerialisedMetaData(size_t &length) const
Definition: store.cc:1829
static DeferredRead::DeferrableRead DeferReader
Definition: Store.h:45
void addWriting(StoreEntry *, const cache_key *)
Definition: Controller.cc:783
@ ENTRY_BAD_LENGTH
Definition: enums.h:114
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:65
Definition: Range.h:18
enum _mem_status_t mem_status_t
@ ENTRY_SPECIAL
Definition: enums.h:84
int validToSend() const
Definition: store.cc:1409
void updateLimits()
slowly calculate (and cache) hi/lo watermarks and similar limits
Definition: Controller.cc:198
@ ENTRY_FWD_HDR_WAIT
Definition: enums.h:111
@ STORE_MEM_CLIENT
Definition: enums.h:73
void comm_read(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer &callback)
Definition: Read.h:59
static std::stack< StoreEntry * > LateReleaseStack
Definition: store.cc:120
void storeFreeMemory(void)
Definition: store.cc:1390
A StoreEntry::getPublic*() caller.
Definition: StoreClient.h:24
void transientsDisconnect(StoreEntry &)
disassociates the entry from the intransit table
Definition: Controller.cc:665
Decision decision
current decision state
Definition: MemObject.h:156
StoreEntry * find(const cache_key *)
Definition: Controller.cc:356
virtual void disconnect(StoreEntry &)
called when the entry is about to forget its association with cache_dir
Definition: Disk.h:71
int size
Definition: ModDevPoll.cc:77
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1860
struct timeval store_complete_stop
void OBJH(StoreEntry *)
Definition: forward.h:44
#define NULL
Definition: types.h:166
RemovalPolicySettings * memPolicy
Definition: SquidConfig.h:97
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1941
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
SBuf vary_headers
Definition: MemObject.h:204
@ IN_MEMORY
Definition: enums.h:36
@ ENTRY_VALIDATED
Definition: enums.h:113
const HttpReply & baseReply() const
Definition: MemObject.h:59
void abort()
Definition: store.cc:1124
static void StatQueues(StoreEntry *e)
reports the current state of Store-related queues
Definition: store.cc:132
store_client_t
Definition: enums.h:71
void append(char const *buf, int len)
Definition: String.cc:161
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1952
void stopSharing(StoreEntry &)
stop any current (and prevent any future) SMP sharing of the given entry
Definition: Controller.cc:637
int max_open_disk_fds
Definition: SquidConfig.h:460
@ ENTRY_REVALIDATE_ALWAYS
Definition: enums.h:85
tlv * storeSwapMetaBuild(const StoreEntry *e)
uint16_t refcount
Definition: Store.h:223
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1990
sdirno swap_dirn
Definition: Store.h:230
@ STORE_DISK_CLIENT
Definition: enums.h:74
#define INT_MAX
Definition: types.h:76
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2074
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:157
cache_key * storeKeyDup(const cache_key *key)
ping_status_t ping_status
Definition: Store.h:234
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:151
@ ioDone
Definition: forward.h:40
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:205
void const char HLPCB void * data
Definition: stub_helper.cc:16
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
bool peerResponseTime(struct timeval &responseTime)
Definition: access_log.cc:316
static void StatQueue(std::ostream &)
prints IPC message queue state; suitable for cache manager reports
int unlock(const char *context)
Definition: store.cc:483
void expireNow()
Definition: store.cc:817
const char * storeEntryFlags(const StoreEntry *entry)
Definition: stat.cc:276
static size_t inUseCount()
Definition: store.cc:197
HttpRequestMethod method
Definition: MemObject.h:137
store_status_t store_status
Definition: Store.h:236
struct _store_check_cachable_hist::@146 yes
int neighbors_do_private_keys
int conn
the current server connection FD
Definition: Transport.cc:26
#define assert(EX)
Definition: assert.h:19
const char * swapStatusStr[]
Definition: store.cc:90
void Stats(StoreEntry *output)
Definition: store.cc:124
@ NOT_IN_MEMORY
Definition: enums.h:35
void fatalf(const char *fmt,...)
Definition: fatal.cc:68
HierarchyLogEntry hier
Definition: HttpRequest.h:157
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1866
const char * pingStatusStr[]
Definition: store.cc:79
RemovalPolicy * createRemovalPolicy(RemovalPolicySettings *settings)
Definition: store.cc:1727
int checkNegativeHit() const
Definition: store.cc:1352
virtual void stat(StoreEntry &) const override
Definition: Controller.cc:141
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:128
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
@ STORE_LOG_RELEASE
Definition: enums.h:159
@ PING_NONE
Has not considered whether to send ICP queries to peers yet.
Definition: enums.h:41
void write(StoreIOBuffer)
Definition: store.cc:824
bool checkCachable()
Definition: store.cc:976
void(* Remove)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node)
Definition: RemovalPolicy.h:47
hash_table * store_table
bool cacheNegatively()
Definition: store.cc:186
mem_status_t mem_status
Definition: Store.h:232
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:327
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:372
void dump(int debug_lvl) const
Definition: store.cc:1555
@ STORE_OK
Definition: enums.h:50
RefCount< HttpReply > HttpReplyPointer
Definition: forward.h:34
static int store_dirs_rebuilding
the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
Definition: Controller.h:139
time_t squid_curtime
Definition: stub_time.cc:17
void transientsClearCollapsingRequirement(StoreEntry &e)
removes collapsing requirement (for future hits)
Definition: Controller.cc:672
static MemAllocator * pool
Definition: Store.h:314
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:249
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:803
signed_int32_t sfileno
Definition: forward.h:22
int64_t content_length
Definition: Message.h:84
int64_t store_maxobjsize
time_t date
Definition: HttpReply.h:44
struct SquidConfig::@112 onoff
void storeDirSwapLog(const StoreEntry *e, int op)
Definition: Disks.cc:733
void storeConfigure(void)
Definition: store.cc:1325
@ scNotModified
Definition: StatusCode.h:40
void EVH(void *)
Definition: event.h:19
char * buf
Definition: CommRead.h:26
void setMemStatus(mem_status_t)
Definition: store.cc:1580
ETag getETag(Http::HdrType id) const
Definition: HttpHeader.cc:1322
swap_status_t swap_status
Definition: Store.h:238
@ ksDefault
Definition: store_key_md5.h:19
void negativeCache()
Definition: store.cc:1373
#define fd_table
Definition: fde.h:189
void complete()
Definition: store.cc:1078
bool appliedUpdates
Definition: MemObject.h:89
static void getPublicByRequestMethod(StoreClient *aClient, HttpRequest *request, const HttpRequestMethod &method)
Definition: store.cc:521
signed int sdirno
Definition: forward.h:23
void markEndOfReplyHeaders()
sets baseReply().hdr_sz (i.e. written reply headers size) to endOffset()
Definition: MemObject.cc:241
HttpRequestMethod method
Definition: HttpRequest.h:114
void storeKeyFree(const cache_key *key)
KeyScope
Definition: store_key_md5.h:18
@ ENTRY_ABORTED
Definition: enums.h:115
const char * termedBuf() const
Definition: SquidString.h:91
struct timeval current_time
Definition: stub_time.cc:15
@ ENTRY_REQUIRES_COLLAPSING
Definition: enums.h:118
void(* Add)(RemovalPolicy *policy, StoreEntry *entry, RemovalPolicyNode *node)
Definition: RemovalPolicy.h:46
#define memPoolCreate
Definition: Pool.h:325
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:155
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:388
bool checkDeferRead(int fd) const
Definition: store.cc:267
void swapOutFileClose(int how)
void delayRead(DeferredRead const &)
Definition: MemObject.cc:457
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:696
bool timestampsSet()
Definition: store.cc:1449
void storeSwapTLVFree(tlv *n)
@ scNoContent
Definition: StatusCode.h:30
void storeAppendVPrintf(StoreEntry *e, const char *fmt, va_list vargs)
Definition: store.cc:911
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:936
const char * storeStatusStr[]
Definition: store.cc:85
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: Controller.cc:93
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:71
@ ioUndecided
Definition: forward.h:40
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1025
void hashDelete()
Definition: store.cc:447
int64_t object_sz
Definition: MemObject.h:198
bool isEmpty() const
Definition: Store.h:67
void trimSwappable()
Definition: MemObject.cc:386
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod & method
Definition: stub_store.cc:108
void handleIdleEntry(StoreEntry &)
called when the entry is no longer needed by any transaction
Definition: Controller.cc:679
int fdNFree(void)
Definition: fd.cc:295
int RESERVED_FD
int expiresMoreThan(time_t expires, time_t when)
Definition: store.cc:1400
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
void storeRebuildStart(void)
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
Definition: store.cc:223
void storeLog(int tag, const StoreEntry *e)
Definition: store_log.cc:38
uint64_t swap_file_sz
Definition: Store.h:222
void IPH * handler
Definition: stub_ipcache.cc:16
void const char HLPCB * callback
Definition: stub_helper.cc:16
const char * storeKeyText(const cache_key *key)
CacheDigest * store_digest
virtual void vappendf(const char *, va_list)
Definition: store.cc:867
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:203
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
Comm::ConnectionPointer conn
Definition: CommRead.h:25
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:542
void release(const bool shareable=false)
Definition: store.cc:1196
static void StatQueue(std::ostream &)
prints IPC message queue state; suitable for cache manager reports
Definition: IpcIoFile.cc:526
virtual int getInUseCount()=0
const cache_key * storeKeyPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope)
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1981
const HttpReply & freshestReply() const
Definition: MemObject.h:67
static Pointer Create(const CommandPointer &cmd)
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:670
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:98
int shutting_down
void hashInsert(const cache_key *)
Definition: store.cc:438
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1331
StoreEntry * storeCreatePureEntry(const char *url, const char *log_url, const HttpRequestMethod &method)
Definition: store.cc:785
bool checkTooBig() const
Definition: store.cc:965
void freeMemorySpace(const int spaceRequired)
Definition: Controller.cc:542
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:2001
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:849
struct _store_check_cachable_hist::@145 no
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1875
void setNoDelay(bool const)
Definition: store.cc:273
struct SquidConfig::@110 Store
@ writerGone
failure: caller left before swapping out everything
Definition: StoreIOState.h:71
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:378
void setHeaders(Http::StatusCode status, const char *reason, const char *ctype, int64_t clen, time_t lmt, time_t expires)
Definition: HttpReply.cc:168
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:208
int locked() const
Definition: Store.h:136
static EVH storeLateRelease
Definition: store.cc:115
#define SWALLOW_EXCEPTIONS(code)
Definition: TextException.h:73
RemovalPolicy * REMOVALPOLICYCREATE(wordlist *args)
Definition: RemovalPolicy.h:80
@ scOkay
Definition: StatusCode.h:26
@ ioWriting
Definition: forward.h:40
RemovalPolicyNode repl
Definition: MemObject.h:196
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1070
void registerAbortCallback(const AsyncCall::Pointer &)
notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
Definition: store.cc:1537
void storeLogOpen(void)
Definition: store_log.cc:123
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:124
@ METHOD_GET
Definition: MethodType.h:25
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:126
AsyncCall::Pointer abortCallback
used for notifying StoreEntry writers about 3rd-party initiated aborts
Definition: MemObject.h:195
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1897
SQUIDCEXTERN hash_link * hash_lookup(hash_table *, const void *)
Definition: hash.cc:147
void updateReply(const HttpReply &r)
(re)sets updated reply;
Definition: MemObject.h:84
void reset()
Definition: MemObject.cc:285
#define false
Definition: GnuRegex.c:233
unsigned short lock_count
Definition: Store.h:316
void * xrealloc(void *s, size_t sz)
Definition: xalloc.cc:137
static int getKeyCounter(void)
Definition: store.cc:566
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1934
@ ioReading
Definition: forward.h:40
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1923
const char * context_
default unlock() message
Definition: Store.h:383
const HttpReply * hasFreshestReply() const
Definition: Store.h:58
@ SWAPOUT_NONE
Definition: enums.h:58
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
void storeKeyInit(void)
const char * describeTimestamps() const
Definition: store.cc:2062
const HttpReplyPointer & updatedReply() const
Definition: MemObject.h:63
void clearPrivate()
Definition: store.cc:178
HttpRequestPointer request
Definition: MemObject.h:188
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:77
time_t lastref
Definition: Store.h:217
int nclients
Definition: MemObject.h:146
const char * getMD5Text() const
Definition: store.cc:205
void const char * buf
Definition: stub_helper.cc:16
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:109
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1765
time_t negativeTtl
Definition: SquidConfig.h:99
void storeDigestInit(void)
#define PROF_start(probename)
Definition: Profiler.h:62
class SquidConfig Config
Definition: SquidConfig.cc:12
int storePendingNClients(const StoreEntry *e)
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:86
StatCounters statCounter
Definition: StatCounters.cc:12
time_t lastModified() const
Definition: Store.h:170
@ STORE_PENDING
Definition: enums.h:51
SwapOut swapout
Definition: MemObject.h:159
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:228
Pointer recreateOnNotModified(const HttpReply &reply304) const
Definition: HttpReply.cc:263
Controller & Root()
safely access controller singleton
Definition: Controller.cc:938

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors