Store.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_STORE_H
10 #define SQUID_STORE_H
11 
12 #include "base/DelayedAsyncCalls.h"
13 #include "base/Packable.h"
14 #include "base/Range.h"
15 #include "base/RefCount.h"
16 #include "comm/forward.h"
17 #include "hash.h"
18 #include "http/forward.h"
19 #include "http/RequestMethod.h"
20 #include "HttpReply.h"
21 #include "MemObject.h"
22 #include "RemovalPolicy.h"
23 #include "store/Controller.h"
24 #include "store/forward.h"
25 #include "store_key_md5.h"
26 #include "StoreIOBuffer.h"
27 #include "StoreStats.h"
28 
29 #if USE_SQUID_ESI
30 #include "esi/Element.h"
31 #endif
32 
33 #include <ostream>
34 
35 class AsyncCall;
36 class HttpRequest;
37 class RequestFlags;
38 
40 
41 class StoreEntry : public hash_link, public Packable
42 {
43 
44 public:
45  bool checkDeferRead(int fd) const;
46 
47  const char *getMD5Text() const;
48  StoreEntry();
49  virtual ~StoreEntry();
50 
51  MemObject &mem() { assert(mem_obj); return *mem_obj; }
52  const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
53 
57  const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
58 
59  void write(StoreIOBuffer);
60 
66  bool isEmpty() const { return mem().endOffset() == 0; }
67  bool isAccepting() const;
68  size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
69 
72  void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply);
73 
76  void completeTruncated(const char *whyWeConsiderTheReplyTruncated);
77 
79  void complete();
80 
83  const char *getSerialisedMetaData(size_t &length) const;
85  void storeErrorResponse(HttpReply *reply);
86  void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
87  void startWriting();
88  bool mayStartSwapOut();
90  void trimMemory(const bool preserveSwappable);
91 
92  // called when a decision to cache in memory has been made
93  void memOutDecision(const bool willCacheInRam);
94  // called when a decision to cache on disk has been made
95  void swapOutDecision(const MemObject::SwapOut::Decision &decision);
96 
97  void abort();
98  bool makePublic(const KeyScope keyScope = ksDefault);
99  void makePrivate(const bool shareable);
103  void clearPrivate();
104  bool setPublicKey(const KeyScope keyScope = ksDefault);
108  void clearPublicKeyScope();
109 
111  const cache_key *publicKey() const {
112  return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
113  reinterpret_cast<const cache_key*>(key): // may be nil
114  nullptr;
115  }
116 
120  void setPrivateKey(const bool shareable, const bool permanent);
121 
122  void expireNow();
125  void releaseRequest(const bool shareable = false);
126  void negativeCache();
127  bool cacheNegatively(); // TODO: why both negativeCache() and cacheNegatively() ?
128  void invokeHandlers();
129  void cacheInMemory();
130  void swapOut();
132  bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
134  bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
136  bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
137  void swapOutFileClose(int how);
138  const char *url() const;
142  bool checkCachable();
143  int checkNegativeHit() const;
144  int locked() const { return lock_count; }
145  int validToSend() const;
146  bool memoryCachable();
147 
150  void createMemObject();
151 
153  void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
154 
156  void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
157 
158  void dump(int debug_lvl) const;
159  void hashDelete();
160  void hashInsert(const cache_key *);
163  void reset();
165  bool timestampsSet();
169  void unregisterAbortCallback(const char *reason);
170  void destroyMemObject();
171  int checkTooSmall();
172 
173  void setNoDelay (bool const);
174  void lastModified(const time_t when) { lastModified_ = when; }
176  time_t lastModified() const {
177  // may still return -1 if timestamp is not set
178  return lastModified_ < 0 ? timestamp : lastModified_;
179  }
181  const char *describeTimestamps() const;
182  // TODO: consider removing currently unsupported imslen parameter
183  bool modifiedSince(const time_t ims, const int imslen = -1) const;
185  bool hasIfMatchEtag(const HttpRequest &request) const;
187  bool hasIfNoneMatchEtag(const HttpRequest &request) const;
189  bool hasEtag(ETag &etag) const;
190 
194  bool updateOnNotModified(const StoreEntry &e304);
195 
197  Store::Disk &disk() const;
200  bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
203  void attachToDisk(const sdirno, const sfileno, const swap_status_t);
206  void detachFromDisk();
207 
209  bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
211  bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
212 
215 
217  void setCollapsingRequirement(const bool required);
218 
221  /* START OF ON-DISK STORE_META_STD TLV field */
222  time_t timestamp;
223  time_t lastref;
224  time_t expires;
225 private:
226  time_t lastModified_;
227 public:
228  uint64_t swap_file_sz;
229  uint16_t refcount;
230  uint16_t flags;
231  /* END OF ON-DISK STORE_META_STD */
232 
234  sfileno swap_filen:25; // keep in sync with SwapFilenMax
235 
237 
239 
241 
243 
245 
246 public:
247  static size_t inUseCount();
248 
249  void *operator new(size_t byteCount);
250  void operator delete(void *address);
251 #if USE_SQUID_ESI
252 
254 #endif
255  int64_t objectLen() const { return mem().object_sz; }
256  int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
257 
260  void lock(const char *context);
261 
264  int unlock(const char *context);
265 
267  int locks() const { return static_cast<int>(lock_count); }
268 
270  void touch();
271 
276  void release(const bool shareable = false);
277 
282  void abandon(const char *context) { if (!locked()) doAbandon(context); }
283 
286  bool mayStartHitting() const {
288  }
289 
290 #if USE_ADAPTATION
291  void deferProducer(const AsyncCall::Pointer &producer);
294  void kickProducer();
295 #endif
296 
297  /* Packable API */
298  virtual void append(char const *, int);
299  virtual void vappendf(const char *, va_list);
300  virtual void buffer();
301  virtual void flush();
302 
303 protected:
305 
308  void checkDisk() const;
309 
310 private:
311  void doAbandon(const char *context);
312  bool checkTooBig() const;
313  void forcePublicKey(const cache_key *newkey);
315  const cache_key *calcPublicKey(const KeyScope keyScope);
316 
318  void lengthWentBad(const char *reason);
319 
321 
322  unsigned short lock_count; /* Assume < 65536! */
323 
330 
331 #if USE_ADAPTATION
334 #endif
335 
336  bool validLength() const;
337  bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
338 
339  friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
340 };
341 
342 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
343 
345 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
346 
347 namespace Store {
348 
351 class EntryGuard {
352 public:
355  EntryGuard(Entry *entry, const char *context):
356  entry_(entry), context_(context) {
357  assert(!entry_ || entry_->locked());
358  }
359 
361  if (entry_) {
362  // something went wrong -- the caller did not unlockAndReset() us
363  onException();
364  }
365  }
366 
367  EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
368 
371  Entry *get() {
372  return entry_;
373  }
374 
378  void unlockAndReset(const char *resetContext = nullptr) {
379  if (entry_) {
380  entry_->unlock(resetContext ? resetContext : context_);
381  entry_ = nullptr;
382  }
383  }
384 
385 private:
386  void onException() noexcept;
387 
389  const char *context_;
390 };
391 
392 void Stats(StoreEntry *output);
393 void Maintain(void *unused);
394 }; // namespace Store
395 
398 
400 const char *storeEntryFlags(const StoreEntry *);
401 
404 
406 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
407 
410 
413 
416 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
417 
420 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
421 
423 void storeInit(void);
424 
426 void storeConfigure(void);
427 
429 void storeFreeMemory(void);
430 
432 int expiresMoreThan(time_t, time_t);
433 
435 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
436 
438 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
439 
441 int storeTooManyDiskFilesOpen(void);
442 
445 
448 
450 void storeFsInit(void);
451 
453 void storeFsDone(void);
454 
456 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
457 
462 extern FREE destroyStoreEntry;
463 
465 void storeGetMemSpace(int size);
466 
467 #endif /* SQUID_STORE_H */
468 
int hdr_sz
Definition: Message.h:82
void makePrivate(const bool shareable)
Definition: store.cc:171
void Maintain(void *unused)
Definition: store.cc:1108
virtual void buffer()
Definition: store.cc:1579
StoreEntry * storeGetPublicByRequest(HttpRequest *request, const KeyScope keyScope=ksDefault)
Definition: store.cc:491
void abandon(const char *context)
Definition: Store.h:282
int checkTooSmall()
Definition: store.cc:876
time_t timestamp
Definition: Store.h:222
Definition: ETag.h:18
virtual ~StoreEntry()
Definition: store.cc:332
store_client_t storeClientType() const
Definition: store.cc:248
bool validLength() const
Definition: store.cc:1180
bool updateOnNotModified(const StoreEntry &e304)
Definition: store.cc:1436
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1567
void unregisterAbortCallback(const char *reason)
Definition: store.cc:1467
void releaseRequest(const bool shareable=false)
Definition: store.cc:433
StoreEntry * adjustVary()
Definition: store.cc:640
void storeGetMemSpace(int size)
Definition: store.cc:1096
const cache_key * publicKey() const
Definition: Store.h:111
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:1924
void FREE(void *)
Definition: forward.h:36
manages a single cache_dir
Definition: Disk.h:22
unsigned char cache_key
Store key.
Definition: forward.h:29
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:164
StoreEntry()
Definition: store.cc:311
virtual void flush()
Definition: store.cc:1590
MemObject * mem_obj
Definition: Store.h:219
void clearPublicKeyScope()
Definition: store.cc:584
const char * url() const
Definition: store.cc:1544
size_t storeEntryInUse()
void storeInit(void)
Definition: store.cc:1233
void createMemObject()
Definition: store.cc:1553
@ KEY_PRIVATE
Definition: enums.h:102
MemObject & mem()
Definition: Store.h:51
bool shareableWhenPrivate
Definition: Store.h:329
void lock(const char *context)
Definition: store.cc:420
void transientsAbandonmentCheck()
Definition: store.cc:1748
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
void detachFromDisk()
Definition: store.cc:1916
void onException() noexcept
Definition: store.cc:2059
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:427
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:523
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:550
Entry * entry_
the guarded Entry or nil
Definition: Store.h:388
Definition: cbdata.cc:60
void storeAppendVPrintf(StoreEntry *, const char *, va_list ap)
Definition: store.cc:840
@ SWAPOUT_WRITING
Definition: enums.h:61
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:226
uint16_t flags
Definition: Store.h:230
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:267
void storeReplAdd(const char *, REMOVALPOLICYCREATE *)
Definition: store.cc:1623
bool isAccepting() const
Definition: store.cc:1951
time_t expires
Definition: Store.h:224
int64_t endOffset() const
Definition: MemObject.cc:216
int64_t objectLen() const
Definition: Store.h:255
@ SWAPOUT_DONE
Definition: enums.h:64
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1683
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:55
Store::EntryGuard EntryGuard
Definition: Store.h:304
void startWriting()
Definition: store.cc:1699
void storeSwapFileNumberSet(StoreEntry *e, sfileno filn)
static const char * storeId(const int i)
Definition: testRock.cc:180
void reset()
Definition: store.cc:1599
std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2003
bool mayStartSwapOut()
whether we may start writing to disk (now or in the future)
void destroyMemObject()
Definition: store.cc:359
StoreIoStats store_io_stats
Definition: store_io.cc:16
#define PRINTF_FORMAT_ARG2
void doAbandon(const char *context)
Definition: store.cc:461
void invokeHandlers()
void storeConfigure(void)
Definition: store.cc:1247
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope=ksDefault)
Definition: store.cc:485
Definition: Range.h:19
enum _mem_status_t mem_status_t
int validToSend() const
Definition: store.cc:1331
int size
Definition: ModDevPoll.cc:75
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1761
store_status_t
Definition: enums.h:49
bool mayStartHitting() const
Definition: Store.h:286
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1842
const HttpReply & baseReply() const
Definition: MemObject.h:59
const char * storeEntryFlags(const StoreEntry *)
Definition: stat.cc:270
void abort()
Definition: store.cc:1052
store_client_t
Definition: enums.h:71
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1853
uint16_t refcount
Definition: Store.h:229
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1891
void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2
Definition: store.cc:830
friend std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2003
sdirno swap_dirn
Definition: Store.h:236
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:1975
int64_t contentLen() const
Definition: Store.h:256
@ ENTRY_REQUIRES_COLLAPSING
Definition: enums.h:118
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
ping_status_t ping_status
Definition: Store.h:240
const char * getSerialisedMetaData(size_t &length) const
Definition: store.cc:1730
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:211
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
int unlock(const char *context)
Definition: store.cc:444
void expireNow()
Definition: store.cc:748
static size_t inUseCount()
Definition: store.cc:196
store_status_t store_status
Definition: Store.h:242
StoreEntry * storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod &)
Definition: store.cc:716
#define assert(EX)
Definition: assert.h:19
void Stats(StoreEntry *output)
Definition: store.cc:123
FREE destroyStoreEntry
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1767
int checkNegativeHit() const
Definition: store.cc:1274
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:136
void write(StoreIOBuffer)
Definition: store.cc:755
bool checkCachable()
Definition: store.cc:904
bool cacheNegatively()
Definition: store.cc:185
mem_status_t mem_status
Definition: Store.h:238
Store statistics related to low-level I/O.
Definition: StoreStats.h:58
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:333
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:378
void dump(int debug_lvl) const
Definition: store.cc:1477
static MemAllocator * pool
Definition: Store.h:320
void storeHeapPositionUpdate(StoreEntry *, SwapDir *)
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:210
void swapOut()
signed_int32_t sfileno
Definition: forward.h:22
int expiresMoreThan(time_t, time_t)
Definition: store.cc:1322
void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply)
Definition: store.cc:992
StoreEntry * storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod &)
Definition: store.cc:734
void setMemStatus(mem_status_t)
Definition: store.cc:1502
swap_status_t swap_status
Definition: Store.h:244
@ ksDefault
Definition: store_key_md5.h:19
ESIElement::Pointer cachedESITree
Definition: Store.h:253
void negativeCache()
Definition: store.cc:1295
void complete()
Definition: store.cc:1006
void cacheInMemory()
start or continue storing in memory cache
signed int sdirno
Definition: forward.h:23
EntryGuard(Entry *entry, const char *context)
Definition: Store.h:355
KeyScope
Definition: store_key_md5.h:18
void storeFsInit(void)
Definition: store.cc:1614
int32_t index
entry position inside the memory cache
Definition: MemObject.h:181
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:479
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:155
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:349
bool checkDeferRead(int fd) const
Definition: store.cc:228
ping_status_t
Definition: enums.h:39
void swapOutFileClose(int how)
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:627
bool timestampsSet()
Definition: store.cc:1371
void hashDelete()
Definition: store.cc:408
int64_t object_sz
Definition: MemObject.h:198
@ SWAPOUT_FAILED
Definition: enums.h:68
bool isEmpty() const
Definition: Store.h:66
const MemObject & mem() const
Definition: Store.h:52
uint64_t swap_file_sz
Definition: Store.h:228
void storeEntryReplaceObject(StoreEntry *, HttpReply *)
virtual void vappendf(const char *, va_list)
Definition: store.cc:796
void storeFsDone(void)
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:209
void release(const bool shareable=false)
Definition: store.cc:1122
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1882
const HttpReply & freshestReply() const
Definition: MemObject.h:67
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:601
void hashInsert(const cache_key *)
Definition: store.cc:399
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1253
bool checkTooBig() const
Definition: store.cc:893
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:864
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:1902
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:778
void lastModified(const time_t when)
Definition: Store.h:174
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1776
void setNoDelay(bool const)
Definition: store.cc:234
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:339
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:214
int locked() const
Definition: Store.h:144
RemovalPolicy * REMOVALPOLICYCREATE(wordlist *args)
Definition: RemovalPolicy.h:80
EntryGuard(EntryGuard &&)=delete
void(* STOREGETCLIENT)(StoreEntry *, void *cbdata)
Definition: Store.h:345
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:984
void registerAbortCallback(const AsyncCall::Pointer &)
notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
Definition: store.cc:1459
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:132
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:134
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1798
Entry * get()
Definition: Store.h:371
unsigned short lock_count
Definition: Store.h:322
void storeFreeMemory(void)
Definition: store.cc:1312
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1835
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1824
const char * context_
default unlock() message
Definition: Store.h:389
const HttpReply * hasFreshestReply() const
Definition: Store.h:57
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
const char * describeTimestamps() const
Definition: store.cc:1963
void clearPrivate()
Definition: store.cc:177
void completeTruncated(const char *whyWeConsiderTheReplyTruncated)
Definition: store.cc:999
time_t lastref
Definition: Store.h:223
const char * getMD5Text() const
Definition: store.cc:204
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1666
RemovalPolicyNode repl
Definition: Store.h:220
time_t lastModified() const
Definition: Store.h:176
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:234

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors