Store.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_STORE_H
10 #define SQUID_STORE_H
11 
12 #include "base/Packable.h"
13 #include "base/Range.h"
14 #include "base/RefCount.h"
15 #include "comm/forward.h"
16 #include "CommRead.h"
17 #include "hash.h"
18 #include "http/forward.h"
19 #include "http/RequestMethod.h"
20 #include "HttpReply.h"
21 #include "MemObject.h"
22 #include "RemovalPolicy.h"
23 #include "store/Controller.h"
24 #include "store/forward.h"
25 #include "store_key_md5.h"
26 #include "StoreIOBuffer.h"
27 #include "StoreStats.h"
28 
29 #if USE_SQUID_ESI
30 #include "esi/Element.h"
31 #endif
32 
33 #include <ostream>
34 
35 class AsyncCall;
36 class HttpRequest;
37 class RequestFlags;
38 
40 
41 class StoreEntry : public hash_link, public Packable
42 {
43 
44 public:
46  bool checkDeferRead(int fd) const;
47 
48  const char *getMD5Text() const;
49  StoreEntry();
50  virtual ~StoreEntry();
51 
52  MemObject &mem() { assert(mem_obj); return *mem_obj; }
53  const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
54 
58  const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
59 
60  void write(StoreIOBuffer);
61 
67  bool isEmpty() const { return mem().endOffset() == 0; }
68  bool isAccepting() const;
69  size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
71  void lengthWentBad(const char *reason);
72  void complete();
75  const char *getSerialisedMetaData(size_t &length) const;
77  void storeErrorResponse(HttpReply *reply);
78  void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
79  void startWriting();
80  bool mayStartSwapOut();
82  void trimMemory(const bool preserveSwappable);
83 
84  // called when a decision to cache in memory has been made
85  void memOutDecision(const bool willCacheInRam);
86  // called when a decision to cache on disk has been made
87  void swapOutDecision(const MemObject::SwapOut::Decision &decision);
88 
89  void abort();
90  bool makePublic(const KeyScope keyScope = ksDefault);
91  void makePrivate(const bool shareable);
95  void clearPrivate();
96  bool setPublicKey(const KeyScope keyScope = ksDefault);
100  void clearPublicKeyScope();
101 
103  const cache_key *publicKey() const {
104  return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
105  reinterpret_cast<const cache_key*>(key): // may be nil
106  nullptr;
107  }
108 
112  void setPrivateKey(const bool shareable, const bool permanent);
113 
114  void expireNow();
117  void releaseRequest(const bool shareable = false);
118  void negativeCache();
119  bool cacheNegatively(); // TODO: why both negativeCache() and cacheNegatively() ?
120  void invokeHandlers();
121  void cacheInMemory();
122  void swapOut();
124  bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
126  bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
128  bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
129  void swapOutFileClose(int how);
130  const char *url() const;
134  bool checkCachable();
135  int checkNegativeHit() const;
136  int locked() const { return lock_count; }
137  int validToSend() const;
138  bool memoryCachable();
139 
142  void createMemObject();
143 
145  void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
146 
148  void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
149 
150  void dump(int debug_lvl) const;
151  void hashDelete();
152  void hashInsert(const cache_key *);
155  void reset();
157  bool timestampsSet();
161  void unregisterAbortCallback(const char *reason);
162  void destroyMemObject();
163  int checkTooSmall();
164 
166 
167  void setNoDelay (bool const);
168  void lastModified(const time_t when) { lastModified_ = when; }
170  time_t lastModified() const {
171  // may still return -1 if timestamp is not set
172  return lastModified_ < 0 ? timestamp : lastModified_;
173  }
175  const char *describeTimestamps() const;
176  // TODO: consider removing currently unsupported imslen parameter
177  bool modifiedSince(const time_t ims, const int imslen = -1) const;
179  bool hasIfMatchEtag(const HttpRequest &request) const;
181  bool hasIfNoneMatchEtag(const HttpRequest &request) const;
183  bool hasEtag(ETag &etag) const;
184 
188  bool updateOnNotModified(const StoreEntry &e304);
189 
191  Store::Disk &disk() const;
194  bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
197  void attachToDisk(const sdirno, const sfileno, const swap_status_t);
200  void detachFromDisk();
201 
203  bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
205  bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
206 
209 
211  void setCollapsingRequirement(const bool required);
212 
215  /* START OF ON-DISK STORE_META_STD TLV field */
216  time_t timestamp;
217  time_t lastref;
218  time_t expires;
219 private:
220  time_t lastModified_;
221 public:
222  uint64_t swap_file_sz;
223  uint16_t refcount;
224  uint16_t flags;
225  /* END OF ON-DISK STORE_META_STD */
226 
228  sfileno swap_filen:25; // keep in sync with SwapFilenMax
229 
231 
233 
235 
237 
239 
240 public:
241  static size_t inUseCount();
242 
243  void *operator new(size_t byteCount);
244  void operator delete(void *address);
245 #if USE_SQUID_ESI
246 
248 #endif
249  int64_t objectLen() const { return mem().object_sz; }
250  int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
251 
254  void lock(const char *context);
255 
258  int unlock(const char *context);
259 
261  int locks() const { return static_cast<int>(lock_count); }
262 
264  void touch();
265 
270  void release(const bool shareable = false);
271 
276  void abandon(const char *context) { if (!locked()) doAbandon(context); }
277 
280  bool mayStartHitting() const {
282  }
283 
284 #if USE_ADAPTATION
285  void deferProducer(const AsyncCall::Pointer &producer);
288  void kickProducer();
289 #endif
290 
291  /* Packable API */
292  virtual void append(char const *, int);
293  virtual void vappendf(const char *, va_list);
294  virtual void buffer();
295  virtual void flush();
296 
297 protected:
299 
302  void checkDisk() const;
303 
304 private:
305  void doAbandon(const char *context);
306  bool checkTooBig() const;
307  void forcePublicKey(const cache_key *newkey);
309  const cache_key *calcPublicKey(const KeyScope keyScope);
310 
312 
313  unsigned short lock_count; /* Assume < 65536! */
314 
321 
322 #if USE_ADAPTATION
325 #endif
326 
327  bool validLength() const;
328  bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
329 
330  friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
331 };
332 
333 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
334 
336 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
337 
338 namespace Store {
339 
342 class EntryGuard {
343 public:
346  EntryGuard(Entry *entry, const char *context):
347  entry_(entry), context_(context) {
348  assert(!entry_ || entry_->locked());
349  }
350 
352  if (entry_) {
353  // something went wrong -- the caller did not unlockAndReset() us
354  onException();
355  }
356  }
357 
358  EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
359 
362  Entry *get() {
363  return entry_;
364  }
365 
369  void unlockAndReset(const char *resetContext = nullptr) {
370  if (entry_) {
371  entry_->unlock(resetContext ? resetContext : context_);
372  entry_ = nullptr;
373  }
374  }
375 
376 private:
377  void onException() noexcept;
378 
380  const char *context_;
381 };
382 
383 void Stats(StoreEntry *output);
384 void Maintain(void *unused);
385 }; // namespace Store
386 
389 
391 const char *storeEntryFlags(const StoreEntry *);
392 
395 
397 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
398 
401 
404 
407 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
408 
411 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
412 
414 void storeInit(void);
415 
417 void storeConfigure(void);
418 
420 void storeFreeMemory(void);
421 
423 int expiresMoreThan(time_t, time_t);
424 
426 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
427 
429 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
430 
432 int storeTooManyDiskFilesOpen(void);
433 
436 
439 
441 void storeFsInit(void);
442 
444 void storeFsDone(void);
445 
447 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
448 
453 extern FREE destroyStoreEntry;
454 
456 void storeGetMemSpace(int size);
457 
458 #endif /* SQUID_STORE_H */
459 
int hdr_sz
Definition: Message.h:82
void makePrivate(const bool shareable)
Definition: store.cc:172
void Maintain(void *unused)
Definition: store.cc:1161
virtual void buffer()
Definition: store.cc:1636
StoreEntry * storeGetPublicByRequest(HttpRequest *request, const KeyScope keyScope=ksDefault)
Definition: store.cc:533
void abandon(const char *context)
Definition: Store.h:276
int checkTooSmall()
Definition: store.cc:927
time_t timestamp
Definition: Store.h:216
Definition: ETag.h:18
virtual ~StoreEntry()
Definition: store.cc:371
store_client_t storeClientType() const
Definition: store.cc:287
bool validLength() const
Definition: store.cc:1237
bool updateOnNotModified(const StoreEntry &e304)
Definition: store.cc:1493
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1624
void unregisterAbortCallback(const char *reason)
Definition: store.cc:1524
void releaseRequest(const bool shareable=false)
Definition: store.cc:472
StoreEntry * adjustVary()
Definition: store.cc:688
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod const KeyScope scope const char const RequestFlags const HttpRequestMethod const char * logUrl
Definition: stub_store.cc:107
void storeGetMemSpace(int size)
Definition: store.cc:1147
const cache_key * publicKey() const
Definition: Store.h:103
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:2002
void FREE(void *)
Definition: forward.h:36
manages a single cache_dir
Definition: Disk.h:22
unsigned char cache_key
Store key.
Definition: forward.h:29
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:165
StoreEntry()
Definition: store.cc:350
virtual void flush()
Definition: store.cc:1647
MemObject * mem_obj
Definition: Store.h:213
void clearPublicKeyScope()
Definition: store.cc:632
const char * url() const
Definition: store.cc:1601
size_t storeEntryInUse()
void storeInit(void)
Definition: store.cc:1290
void createMemObject()
Definition: store.cc:1610
MemObject & mem()
Definition: Store.h:52
bool shareableWhenPrivate
Definition: Store.h:320
void lock(const char *context)
Definition: store.cc:459
void transientsAbandonmentCheck()
Definition: store.cc:1826
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
@ KEY_PRIVATE
Definition: enums.h:102
void detachFromDisk()
Definition: store.cc:1994
void onException() noexcept
Definition: store.cc:2137
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:466
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:565
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:592
Entry * entry_
the guarded Entry or nil
Definition: Store.h:379
Definition: cbdata.cc:61
void storeAppendVPrintf(StoreEntry *, const char *, va_list ap)
Definition: store.cc:890
@ SWAPOUT_WRITING
Definition: enums.h:61
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:220
uint16_t flags
Definition: Store.h:224
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:261
void storeReplAdd(const char *, REMOVALPOLICYCREATE *)
Definition: store.cc:1680
bool isAccepting() const
Definition: store.cc:2029
time_t expires
Definition: Store.h:218
int64_t endOffset() const
Definition: MemObject.cc:235
int64_t objectLen() const
Definition: Store.h:249
@ SWAPOUT_DONE
Definition: enums.h:64
void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting=true)
Definition: store.cc:1761
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:55
Store::EntryGuard EntryGuard
Definition: Store.h:298
void startWriting()
Definition: store.cc:1777
void storeSwapFileNumberSet(StoreEntry *e, sfileno filn)
static const char * storeId(const int i)
Definition: testRock.cc:180
void reset()
Definition: store.cc:1656
std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2081
bool mayStartSwapOut()
whether we may start writing to disk (now or in the future)
void destroyMemObject()
Definition: store.cc:398
StoreIoStats store_io_stats
Definition: store_io.cc:16
#define PRINTF_FORMAT_ARG2
void doAbandon(const char *context)
Definition: store.cc:500
void invokeHandlers()
void storeConfigure(void)
Definition: store.cc:1304
static DeferredRead::DeferrableRead DeferReader
Definition: Store.h:45
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope=ksDefault)
Definition: store.cc:527
Definition: Range.h:19
enum _mem_status_t mem_status_t
int validToSend() const
Definition: store.cc:1388
int size
Definition: ModDevPoll.cc:77
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1839
store_status_t
Definition: enums.h:49
bool mayStartHitting() const
Definition: Store.h:280
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1920
const HttpReply & baseReply() const
Definition: MemObject.h:59
const char * storeEntryFlags(const StoreEntry *)
Definition: stat.cc:276
void abort()
Definition: store.cc:1103
store_client_t
Definition: enums.h:71
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1931
void DeferrableRead(void *context, CommRead const &)
Definition: CommRead.h:35
uint16_t refcount
Definition: Store.h:223
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1969
void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2
Definition: store.cc:880
friend std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2081
sdirno swap_dirn
Definition: Store.h:230
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2053
int64_t contentLen() const
Definition: Store.h:250
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
ping_status_t ping_status
Definition: Store.h:234
const char * getSerialisedMetaData(size_t &length) const
Definition: store.cc:1808
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:205
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
int unlock(const char *context)
Definition: store.cc:483
void expireNow()
Definition: store.cc:796
static size_t inUseCount()
Definition: store.cc:197
store_status_t store_status
Definition: Store.h:236
StoreEntry * storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod &)
Definition: store.cc:764
int conn
the current server connection FD
Definition: Transport.cc:26
#define assert(EX)
Definition: assert.h:19
void Stats(StoreEntry *output)
Definition: store.cc:124
FREE destroyStoreEntry
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1845
int checkNegativeHit() const
Definition: store.cc:1331
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:128
void write(StoreIOBuffer)
Definition: store.cc:803
bool checkCachable()
Definition: store.cc:955
bool cacheNegatively()
Definition: store.cc:186
mem_status_t mem_status
Definition: Store.h:232
Store statistics related to low-level I/O.
Definition: StoreStats.h:58
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:324
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:369
void dump(int debug_lvl) const
Definition: store.cc:1534
static MemAllocator * pool
Definition: Store.h:311
void storeHeapPositionUpdate(StoreEntry *, SwapDir *)
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:249
void swapOut()
signed_int32_t sfileno
Definition: forward.h:22
int expiresMoreThan(time_t, time_t)
Definition: store.cc:1379
StoreEntry * storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod &)
Definition: store.cc:782
void setMemStatus(mem_status_t)
Definition: store.cc:1559
swap_status_t swap_status
Definition: Store.h:238
@ ksDefault
Definition: store_key_md5.h:19
ESIElement::Pointer cachedESITree
Definition: Store.h:247
void negativeCache()
Definition: store.cc:1352
void complete()
Definition: store.cc:1057
void cacheInMemory()
start or continue storing in memory cache
signed int sdirno
Definition: forward.h:23
EntryGuard(Entry *entry, const char *context)
Definition: Store.h:346
KeyScope
Definition: store_key_md5.h:18
void storeFsInit(void)
Definition: store.cc:1671
@ ENTRY_REQUIRES_COLLAPSING
Definition: enums.h:118
int32_t index
entry position inside the memory cache
Definition: MemObject.h:181
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:521
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:155
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:388
bool checkDeferRead(int fd) const
Definition: store.cc:267
ping_status_t
Definition: enums.h:39
void swapOutFileClose(int how)
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:675
bool timestampsSet()
Definition: store.cc:1428
void hashDelete()
Definition: store.cc:447
int64_t object_sz
Definition: MemObject.h:198
@ SWAPOUT_FAILED
Definition: enums.h:68
bool isEmpty() const
Definition: Store.h:67
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod & method
Definition: stub_store.cc:105
const MemObject & mem() const
Definition: Store.h:53
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
Definition: store.cc:223
uint64_t swap_file_sz
Definition: Store.h:222
void const char HLPCB * callback
Definition: stub_helper.cc:16
void storeEntryReplaceObject(StoreEntry *, HttpReply *)
virtual void vappendf(const char *, va_list)
Definition: store.cc:846
void storeFsDone(void)
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:203
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
void release(const bool shareable=false)
Definition: store.cc:1175
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1960
const HttpReply & freshestReply() const
Definition: MemObject.h:67
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:649
void hashInsert(const cache_key *)
Definition: store.cc:438
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1310
bool checkTooBig() const
Definition: store.cc:944
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:915
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:1980
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:828
void lastModified(const time_t when)
Definition: Store.h:168
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1854
void setNoDelay(bool const)
Definition: store.cc:273
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:378
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:208
int locked() const
Definition: Store.h:136
RemovalPolicy * REMOVALPOLICYCREATE(wordlist *args)
Definition: RemovalPolicy.h:80
EntryGuard(EntryGuard &&)=delete
void(* STOREGETCLIENT)(StoreEntry *, void *cbdata)
Definition: Store.h:336
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1049
void registerAbortCallback(const AsyncCall::Pointer &)
notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
Definition: store.cc:1516
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:124
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:126
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1876
Entry * get()
Definition: Store.h:362
unsigned short lock_count
Definition: Store.h:313
void storeFreeMemory(void)
Definition: store.cc:1369
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1913
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1902
const char * context_
default unlock() message
Definition: Store.h:380
const HttpReply * hasFreshestReply() const
Definition: Store.h:58
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
const char * describeTimestamps() const
Definition: store.cc:2041
void clearPrivate()
Definition: store.cc:178
time_t lastref
Definition: Store.h:217
const char * getMD5Text() const
Definition: store.cc:205
void const char * buf
Definition: stub_helper.cc:16
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1744
RemovalPolicyNode repl
Definition: Store.h:214
time_t lastModified() const
Definition: Store.h:170
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:228

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors