Store.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_STORE_H
10 #define SQUID_STORE_H
11 
12 #include "base/Packable.h"
13 #include "base/Range.h"
14 #include "base/RefCount.h"
15 #include "comm/forward.h"
16 #include "CommRead.h"
17 #include "hash.h"
18 #include "http/forward.h"
19 #include "http/RequestMethod.h"
20 #include "HttpReply.h"
21 #include "MemObject.h"
22 #include "RemovalPolicy.h"
23 #include "store/Controller.h"
24 #include "store/forward.h"
25 #include "store_key_md5.h"
26 #include "StoreIOBuffer.h"
27 #include "StoreStats.h"
28 
29 #if USE_SQUID_ESI
30 #include "esi/Element.h"
31 #endif
32 
33 #include <ostream>
34 
35 class AsyncCall;
36 class HttpRequest;
37 class RequestFlags;
38 
40 
41 class StoreEntry : public hash_link, public Packable
42 {
43 
44 public:
46  bool checkDeferRead(int fd) const;
47 
48  const char *getMD5Text() const;
49  StoreEntry();
50  virtual ~StoreEntry();
51 
52  MemObject &mem() { assert(mem_obj); return *mem_obj; }
53  const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
54 
58  const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
59 
60  void write(StoreIOBuffer);
61 
67  bool isEmpty() const { return mem().endOffset() == 0; }
68  bool isAccepting() const;
69  size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
70 
73  void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply);
74 
77  void completeTruncated(const char *whyWeConsiderTheReplyTruncated);
78 
80  void complete();
81 
84  const char *getSerialisedMetaData(size_t &length) const;
86  void storeErrorResponse(HttpReply *reply);
87  void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
88  void startWriting();
89  bool mayStartSwapOut();
91  void trimMemory(const bool preserveSwappable);
92 
93  // called when a decision to cache in memory has been made
94  void memOutDecision(const bool willCacheInRam);
95  // called when a decision to cache on disk has been made
96  void swapOutDecision(const MemObject::SwapOut::Decision &decision);
97 
98  void abort();
99  bool makePublic(const KeyScope keyScope = ksDefault);
100  void makePrivate(const bool shareable);
104  void clearPrivate();
105  bool setPublicKey(const KeyScope keyScope = ksDefault);
109  void clearPublicKeyScope();
110 
112  const cache_key *publicKey() const {
113  return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
114  reinterpret_cast<const cache_key*>(key): // may be nil
115  nullptr;
116  }
117 
121  void setPrivateKey(const bool shareable, const bool permanent);
122 
123  void expireNow();
126  void releaseRequest(const bool shareable = false);
127  void negativeCache();
128  bool cacheNegatively(); // TODO: why both negativeCache() and cacheNegatively() ?
129  void invokeHandlers();
130  void cacheInMemory();
131  void swapOut();
133  bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
135  bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
137  bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
138  void swapOutFileClose(int how);
139  const char *url() const;
143  bool checkCachable();
144  int checkNegativeHit() const;
145  int locked() const { return lock_count; }
146  int validToSend() const;
147  bool memoryCachable();
148 
151  void createMemObject();
152 
154  void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
155 
157  void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
158 
159  void dump(int debug_lvl) const;
160  void hashDelete();
161  void hashInsert(const cache_key *);
164  void reset();
166  bool timestampsSet();
170  void unregisterAbortCallback(const char *reason);
171  void destroyMemObject();
172  int checkTooSmall();
173 
174  void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
175 
176  void setNoDelay (bool const);
177  void lastModified(const time_t when) { lastModified_ = when; }
179  time_t lastModified() const {
180  // may still return -1 if timestamp is not set
181  return lastModified_ < 0 ? timestamp : lastModified_;
182  }
184  const char *describeTimestamps() const;
185  // TODO: consider removing currently unsupported imslen parameter
186  bool modifiedSince(const time_t ims, const int imslen = -1) const;
188  bool hasIfMatchEtag(const HttpRequest &request) const;
190  bool hasIfNoneMatchEtag(const HttpRequest &request) const;
192  bool hasEtag(ETag &etag) const;
193 
197  bool updateOnNotModified(const StoreEntry &e304);
198 
200  Store::Disk &disk() const;
203  bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
206  void attachToDisk(const sdirno, const sfileno, const swap_status_t);
209  void detachFromDisk();
210 
212  bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
214  bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
215 
218 
220  void setCollapsingRequirement(const bool required);
221 
224  /* START OF ON-DISK STORE_META_STD TLV field */
225  time_t timestamp;
226  time_t lastref;
227  time_t expires;
228 private:
229  time_t lastModified_;
230 public:
231  uint64_t swap_file_sz;
232  uint16_t refcount;
233  uint16_t flags;
234  /* END OF ON-DISK STORE_META_STD */
235 
237  sfileno swap_filen:25; // keep in sync with SwapFilenMax
238 
240 
242 
244 
246 
248 
249 public:
250  static size_t inUseCount();
251 
252  void *operator new(size_t byteCount);
253  void operator delete(void *address);
254 #if USE_SQUID_ESI
255 
257 #endif
258  int64_t objectLen() const { return mem().object_sz; }
259  int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
260 
263  void lock(const char *context);
264 
267  int unlock(const char *context);
268 
270  int locks() const { return static_cast<int>(lock_count); }
271 
273  void touch();
274 
279  void release(const bool shareable = false);
280 
285  void abandon(const char *context) { if (!locked()) doAbandon(context); }
286 
289  bool mayStartHitting() const {
291  }
292 
293 #if USE_ADAPTATION
294  void deferProducer(const AsyncCall::Pointer &producer);
297  void kickProducer();
298 #endif
299 
300  /* Packable API */
301  virtual void append(char const *, int);
302  virtual void vappendf(const char *, va_list);
303  virtual void buffer();
304  virtual void flush();
305 
306 protected:
308 
311  void checkDisk() const;
312 
313 private:
314  void doAbandon(const char *context);
315  bool checkTooBig() const;
316  void forcePublicKey(const cache_key *newkey);
318  const cache_key *calcPublicKey(const KeyScope keyScope);
319 
321  void lengthWentBad(const char *reason);
322 
324 
325  unsigned short lock_count; /* Assume < 65536! */
326 
333 
334 #if USE_ADAPTATION
337 #endif
338 
339  bool validLength() const;
340  bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
341 
342  friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
343 };
344 
345 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
346 
348 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
349 
350 namespace Store {
351 
354 class EntryGuard {
355 public:
358  EntryGuard(Entry *entry, const char *context):
359  entry_(entry), context_(context) {
360  assert(!entry_ || entry_->locked());
361  }
362 
364  if (entry_) {
365  // something went wrong -- the caller did not unlockAndReset() us
366  onException();
367  }
368  }
369 
370  EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
371 
374  Entry *get() {
375  return entry_;
376  }
377 
381  void unlockAndReset(const char *resetContext = nullptr) {
382  if (entry_) {
383  entry_->unlock(resetContext ? resetContext : context_);
384  entry_ = nullptr;
385  }
386  }
387 
388 private:
389  void onException() noexcept;
390 
392  const char *context_;
393 };
394 
395 void Stats(StoreEntry *output);
396 void Maintain(void *unused);
397 }; // namespace Store
398 
401 
403 const char *storeEntryFlags(const StoreEntry *);
404 
407 
409 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
410 
413 
416 
419 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
420 
423 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
424 
426 void storeInit(void);
427 
429 void storeConfigure(void);
430 
432 void storeFreeMemory(void);
433 
435 int expiresMoreThan(time_t, time_t);
436 
438 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
439 
441 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
442 
444 int storeTooManyDiskFilesOpen(void);
445 
448 
451 
453 void storeFsInit(void);
454 
456 void storeFsDone(void);
457 
459 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
460 
465 extern FREE destroyStoreEntry;
466 
468 void storeGetMemSpace(int size);
469 
470 #endif /* SQUID_STORE_H */
471 
int hdr_sz
Definition: Message.h:82
void makePrivate(const bool shareable)
Definition: store.cc:173
void Maintain(void *unused)
Definition: store.cc:1152
virtual void buffer()
Definition: store.cc:1627
StoreEntry * storeGetPublicByRequest(HttpRequest *request, const KeyScope keyScope=ksDefault)
Definition: store.cc:531
void abandon(const char *context)
Definition: Store.h:285
int checkTooSmall()
Definition: store.cc:918
time_t timestamp
Definition: Store.h:225
Definition: ETag.h:18
virtual ~StoreEntry()
Definition: store.cc:372
store_client_t storeClientType() const
Definition: store.cc:288
bool validLength() const
Definition: store.cc:1228
bool updateOnNotModified(const StoreEntry &e304)
Definition: store.cc:1484
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1615
void unregisterAbortCallback(const char *reason)
Definition: store.cc:1515
void releaseRequest(const bool shareable=false)
Definition: store.cc:473
StoreEntry * adjustVary()
Definition: store.cc:680
void storeGetMemSpace(int size)
Definition: store.cc:1138
const cache_key * publicKey() const
Definition: Store.h:112
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:1972
void FREE(void *)
Definition: forward.h:36
manages a single cache_dir
Definition: Disk.h:22
unsigned char cache_key
Store key.
Definition: forward.h:29
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:166
StoreEntry()
Definition: store.cc:351
virtual void flush()
Definition: store.cc:1638
MemObject * mem_obj
Definition: Store.h:222
void clearPublicKeyScope()
Definition: store.cc:624
const char * url() const
Definition: store.cc:1592
size_t storeEntryInUse()
void storeInit(void)
Definition: store.cc:1281
void createMemObject()
Definition: store.cc:1601
@ KEY_PRIVATE
Definition: enums.h:102
MemObject & mem()
Definition: Store.h:52
bool shareableWhenPrivate
Definition: Store.h:332
void lock(const char *context)
Definition: store.cc:460
void transientsAbandonmentCheck()
Definition: store.cc:1796
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
void detachFromDisk()
Definition: store.cc:1964
void onException() noexcept
Definition: store.cc:2107
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:467
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:563
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:590
Entry * entry_
the guarded Entry or nil
Definition: Store.h:391
Definition: cbdata.cc:60
void storeAppendVPrintf(StoreEntry *, const char *, va_list ap)
Definition: store.cc:882
@ SWAPOUT_WRITING
Definition: enums.h:61
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:229
uint16_t flags
Definition: Store.h:233
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:270
void storeReplAdd(const char *, REMOVALPOLICYCREATE *)
Definition: store.cc:1671
bool isAccepting() const
Definition: store.cc:1999
time_t expires
Definition: Store.h:227
int64_t endOffset() const
Definition: MemObject.cc:219
int64_t objectLen() const
Definition: Store.h:258
@ SWAPOUT_DONE
Definition: enums.h:64
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1731
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:55
Store::EntryGuard EntryGuard
Definition: Store.h:307
void startWriting()
Definition: store.cc:1747
void storeSwapFileNumberSet(StoreEntry *e, sfileno filn)
static const char * storeId(const int i)
Definition: testRock.cc:180
void reset()
Definition: store.cc:1647
std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2051
bool mayStartSwapOut()
whether we may start writing to disk (now or in the future)
void destroyMemObject()
Definition: store.cc:399
StoreIoStats store_io_stats
Definition: store_io.cc:16
#define PRINTF_FORMAT_ARG2
void doAbandon(const char *context)
Definition: store.cc:501
void invokeHandlers()
void storeConfigure(void)
Definition: store.cc:1295
static DeferredRead::DeferrableRead DeferReader
Definition: Store.h:45
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope=ksDefault)
Definition: store.cc:525
Definition: Range.h:19
enum _mem_status_t mem_status_t
int validToSend() const
Definition: store.cc:1379
int size
Definition: ModDevPoll.cc:77
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1809
store_status_t
Definition: enums.h:49
bool mayStartHitting() const
Definition: Store.h:289
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1890
const HttpReply & baseReply() const
Definition: MemObject.h:59
const char * storeEntryFlags(const StoreEntry *)
Definition: stat.cc:273
void abort()
Definition: store.cc:1094
store_client_t
Definition: enums.h:71
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1901
void DeferrableRead(void *context, CommRead const &)
Definition: CommRead.h:35
uint16_t refcount
Definition: Store.h:232
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1939
void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2
Definition: store.cc:872
friend std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2051
sdirno swap_dirn
Definition: Store.h:239
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2023
int64_t contentLen() const
Definition: Store.h:259
@ ENTRY_REQUIRES_COLLAPSING
Definition: enums.h:118
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
ping_status_t ping_status
Definition: Store.h:243
const char * getSerialisedMetaData(size_t &length) const
Definition: store.cc:1778
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:214
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
int unlock(const char *context)
Definition: store.cc:484
void expireNow()
Definition: store.cc:788
static size_t inUseCount()
Definition: store.cc:198
store_status_t store_status
Definition: Store.h:245
StoreEntry * storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod &)
Definition: store.cc:756
int conn
the current server connection FD
Definition: Transport.cc:26
#define assert(EX)
Definition: assert.h:19
void Stats(StoreEntry *output)
Definition: store.cc:125
FREE destroyStoreEntry
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1815
int checkNegativeHit() const
Definition: store.cc:1322
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:137
void write(StoreIOBuffer)
Definition: store.cc:795
bool checkCachable()
Definition: store.cc:946
bool cacheNegatively()
Definition: store.cc:187
mem_status_t mem_status
Definition: Store.h:241
Store statistics related to low-level I/O.
Definition: StoreStats.h:58
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:336
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:381
void dump(int debug_lvl) const
Definition: store.cc:1525
static MemAllocator * pool
Definition: Store.h:323
void storeHeapPositionUpdate(StoreEntry *, SwapDir *)
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:250
void swapOut()
signed_int32_t sfileno
Definition: forward.h:22
int expiresMoreThan(time_t, time_t)
Definition: store.cc:1370
void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply)
Definition: store.cc:1034
StoreEntry * storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod &)
Definition: store.cc:774
void setMemStatus(mem_status_t)
Definition: store.cc:1550
swap_status_t swap_status
Definition: Store.h:247
@ ksDefault
Definition: store_key_md5.h:19
ESIElement::Pointer cachedESITree
Definition: Store.h:256
void negativeCache()
Definition: store.cc:1343
void complete()
Definition: store.cc:1048
void cacheInMemory()
start or continue storing in memory cache
signed int sdirno
Definition: forward.h:23
EntryGuard(Entry *entry, const char *context)
Definition: Store.h:358
KeyScope
Definition: store_key_md5.h:18
void storeFsInit(void)
Definition: store.cc:1662
int32_t index
entry position inside the memory cache
Definition: MemObject.h:181
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:519
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:155
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:389
bool checkDeferRead(int fd) const
Definition: store.cc:268
ping_status_t
Definition: enums.h:39
void swapOutFileClose(int how)
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:667
bool timestampsSet()
Definition: store.cc:1419
void hashDelete()
Definition: store.cc:448
int64_t object_sz
Definition: MemObject.h:198
@ SWAPOUT_FAILED
Definition: enums.h:68
bool isEmpty() const
Definition: Store.h:67
const MemObject & mem() const
Definition: Store.h:53
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
Definition: store.cc:224
uint64_t swap_file_sz
Definition: Store.h:231
void storeEntryReplaceObject(StoreEntry *, HttpReply *)
virtual void vappendf(const char *, va_list)
Definition: store.cc:838
void storeFsDone(void)
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:212
void release(const bool shareable=false)
Definition: store.cc:1166
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1930
const HttpReply & freshestReply() const
Definition: MemObject.h:67
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:641
void hashInsert(const cache_key *)
Definition: store.cc:439
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1301
bool checkTooBig() const
Definition: store.cc:935
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:906
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:1950
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:820
void lastModified(const time_t when)
Definition: Store.h:177
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1824
void setNoDelay(bool const)
Definition: store.cc:274
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:379
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:217
int locked() const
Definition: Store.h:145
RemovalPolicy * REMOVALPOLICYCREATE(wordlist *args)
Definition: RemovalPolicy.h:80
EntryGuard(EntryGuard &&)=delete
void(* STOREGETCLIENT)(StoreEntry *, void *cbdata)
Definition: Store.h:348
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1026
void registerAbortCallback(const AsyncCall::Pointer &)
notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
Definition: store.cc:1507
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:133
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:135
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1846
Entry * get()
Definition: Store.h:374
unsigned short lock_count
Definition: Store.h:325
void storeFreeMemory(void)
Definition: store.cc:1360
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1883
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1872
const char * context_
default unlock() message
Definition: Store.h:392
const HttpReply * hasFreshestReply() const
Definition: Store.h:58
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
const char * describeTimestamps() const
Definition: store.cc:2011
void clearPrivate()
Definition: store.cc:179
void completeTruncated(const char *whyWeConsiderTheReplyTruncated)
Definition: store.cc:1041
time_t lastref
Definition: Store.h:226
const char * getMD5Text() const
Definition: store.cc:206
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1714
RemovalPolicyNode repl
Definition: Store.h:223
time_t lastModified() const
Definition: Store.h:179
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:237

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors