Store.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_STORE_H
10 #define SQUID_STORE_H
11 
12 #include "base/Packable.h"
13 #include "base/Range.h"
14 #include "base/RefCount.h"
15 #include "comm/forward.h"
16 #include "CommRead.h"
17 #include "hash.h"
18 #include "http/forward.h"
19 #include "http/RequestMethod.h"
20 #include "HttpReply.h"
21 #include "MemObject.h"
22 #include "RemovalPolicy.h"
23 #include "store/Controller.h"
24 #include "store/forward.h"
25 #include "store_key_md5.h"
26 #include "StoreIOBuffer.h"
27 #include "StoreStats.h"
28 
29 #if USE_SQUID_ESI
30 #include "esi/Element.h"
31 #endif
32 
33 #include <ostream>
34 
35 class AsyncCall;
36 class HttpRequest;
37 class RequestFlags;
38 
40 
41 class StoreEntry : public hash_link, public Packable
42 {
43 
44 public:
46  bool checkDeferRead(int fd) const;
47 
48  const char *getMD5Text() const;
49  StoreEntry();
50  virtual ~StoreEntry();
51 
52  HttpReply const *getReply() const;
53  void write(StoreIOBuffer);
54 
60  bool isEmpty() const {
61  assert (mem_obj);
62  return mem_obj->endOffset() == 0;
63  }
64  bool isAccepting() const;
65  size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
67  void lengthWentBad(const char *reason);
68  void complete();
70  char const *getSerialisedMetaData();
72  void storeErrorResponse(HttpReply *reply);
73  void replaceHttpReply(HttpReply *, bool andStartWriting = true);
74  void startWriting();
75  bool mayStartSwapOut();
77  void trimMemory(const bool preserveSwappable);
78 
79  // called when a decision to cache in memory has been made
80  void memOutDecision(const bool willCacheInRam);
81  // called when a decision to cache on disk has been made
82  void swapOutDecision(const MemObject::SwapOut::Decision &decision);
83 
84  void abort();
85  bool makePublic(const KeyScope keyScope = ksDefault);
86  void makePrivate(const bool shareable);
90  void clearPrivate();
91  bool setPublicKey(const KeyScope keyScope = ksDefault);
95  void clearPublicKeyScope();
96 
98  const cache_key *publicKey() const {
99  return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
100  reinterpret_cast<const cache_key*>(key): // may be nil
101  nullptr;
102  }
103 
107  void setPrivateKey(const bool shareable, const bool permanent);
108 
109  void expireNow();
112  void releaseRequest(const bool shareable = false);
113  void negativeCache();
114  bool cacheNegatively();
115  void invokeHandlers();
116  void cacheInMemory();
117  void swapOut();
119  bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
121  bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
123  bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
124  void swapOutFileClose(int how);
125  const char *url() const;
129  bool checkCachable();
130  int checkNegativeHit() const;
131  int locked() const;
132  int validToSend() const;
133  bool memoryCachable();
134 
137  void createMemObject();
138 
140  void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
141 
143  void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
144 
145  void dump(int debug_lvl) const;
146  void hashDelete();
147  void hashInsert(const cache_key *);
148  void registerAbort(STABH * cb, void *);
149  void reset();
151  bool timestampsSet();
152  void unregisterAbort();
153  void destroyMemObject();
154  int checkTooSmall();
155 
157 
158  void setNoDelay (bool const);
159  void lastModified(const time_t when) { lastModified_ = when; }
161  time_t lastModified() const {
162  // may still return -1 if timestamp is not set
163  return lastModified_ < 0 ? timestamp : lastModified_;
164  }
166  const char *describeTimestamps() const;
167  // TODO: consider removing currently unsupported imslen parameter
168  bool modifiedSince(const time_t ims, const int imslen = -1) const;
170  bool hasIfMatchEtag(const HttpRequest &request) const;
172  bool hasIfNoneMatchEtag(const HttpRequest &request) const;
174  bool hasEtag(ETag &etag) const;
175 
177  Store::Disk &disk() const;
180  bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
183  void attachToDisk(const sdirno, const sfileno, const swap_status_t);
186  void detachFromDisk();
187 
189  bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
191  bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
192 
195 
197  void setCollapsingRequirement(const bool required);
198 
201  /* START OF ON-DISK STORE_META_STD TLV field */
202  time_t timestamp;
203  time_t lastref;
204  time_t expires;
205 private:
206  time_t lastModified_;
207 public:
208  uint64_t swap_file_sz;
209  uint16_t refcount;
210  uint16_t flags;
211  /* END OF ON-DISK STORE_META_STD */
212 
214  sfileno swap_filen:25; // keep in sync with SwapFilenMax
215 
217 
219 
221 
223 
225 
226 public:
227  static size_t inUseCount();
228  static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
229  static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
230  static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
231 
232  void *operator new(size_t byteCount);
233  void operator delete(void *address);
234 #if USE_SQUID_ESI
235 
237 #endif
238  int64_t objectLen() const;
239  int64_t contentLen() const;
240 
243  void lock(const char *context);
244 
247  int unlock(const char *context);
248 
250  int locks() const { return static_cast<int>(lock_count); }
251 
253  void touch();
254 
259  void release(const bool shareable = false);
260 
265  void abandon(const char *context) { if (!locked()) doAbandon(context); }
266 
269  bool mayStartHitting() const {
270  return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate;
271  }
272 
273 #if USE_ADAPTATION
274  void deferProducer(const AsyncCall::Pointer &producer);
277  void kickProducer();
278 #endif
279 
280  /* Packable API */
281  virtual void append(char const *, int);
282  virtual void vappendf(const char *, va_list);
283  virtual void buffer();
284  virtual void flush();
285 
286 protected:
288 
291  void checkDisk() const;
292 
293 private:
294  void doAbandon(const char *context);
295  bool checkTooBig() const;
296  void forcePublicKey(const cache_key *newkey);
297  StoreEntry *adjustVary();
298  const cache_key *calcPublicKey(const KeyScope keyScope);
299 
301 
302  unsigned short lock_count; /* Assume < 65536! */
303 
310 
311 #if USE_ADAPTATION
314 #endif
315 
316  bool validLength() const;
317  bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
318 
319  friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
320 };
321 
322 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
323 
325 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
326 
327 namespace Store {
328 
331 class EntryGuard {
332 public:
335  EntryGuard(Entry *entry, const char *context):
336  entry_(entry), context_(context) {
337  assert(!entry_ || entry_->locked());
338  }
339 
341  if (entry_) {
342  // something went wrong -- the caller did not unlockAndReset() us
343  onException();
344  }
345  }
346 
347  EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
348 
351  Entry *get() {
352  return entry_;
353  }
354 
358  void unlockAndReset(const char *resetContext = nullptr) {
359  if (entry_) {
360  entry_->unlock(resetContext ? resetContext : context_);
361  entry_ = nullptr;
362  }
363  }
364 
365 private:
366  void onException() noexcept;
367 
369  const char *context_;
370 };
371 
372 void Stats(StoreEntry *output);
373 void Maintain(void *unused);
374 }; // namespace Store
375 
377 size_t storeEntryInUse();
378 
380 const char *storeEntryFlags(const StoreEntry *);
381 
383 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
384 
386 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
387 
389 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
390 
393 
396 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
397 
400 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
401 
403 void storeInit(void);
404 
406 void storeConfigure(void);
407 
409 void storeFreeMemory(void);
410 
412 int expiresMoreThan(time_t, time_t);
413 
415 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
416 
418 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
419 
421 int storeTooManyDiskFilesOpen(void);
422 
424 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
425 
427 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
428 
430 void storeFsInit(void);
431 
433 void storeFsDone(void);
434 
436 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
437 
442 extern FREE destroyStoreEntry;
443 
445 void storeGetMemSpace(int size);
446 
447 #endif /* SQUID_STORE_H */
448 
const char * getMD5Text() const
Definition: store.cc:185
sdirno swap_dirn
Definition: Store.h:216
bool shareableWhenPrivate
Definition: Store.h:309
static MemAllocator * pool
Definition: Store.h:300
void detachFromDisk()
Definition: store.cc:2028
void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2
Definition: store.cc:881
void makePrivate(const bool shareable)
Definition: store.cc:152
#define assert(EX)
Definition: assert.h:17
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:522
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1899
void const char HLPCB * callback
Definition: stub_helper.cc:16
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1936
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:446
static void getPublicByRequestMethod(StoreClient *aClient, HttpRequest *request, const HttpRequestMethod &method)
Definition: store.cc:501
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:189
store_status_t
Definition: enums.h:45
StoreEntry * storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod &)
Definition: store.cc:764
void release(const bool shareable=false)
Definition: store.cc:1195
int unlock(const char *context)
Definition: store.cc:463
time_t lastModified() const
Definition: Store.h:161
Definition: ETag.h:17
Definition: cbdata.cc:60
EntryGuard(Entry *entry, const char *context)
Definition: Store.h:335
void storeEntryReplaceObject(StoreEntry *, HttpReply *)
bool timestampsSet()
Definition: store.cc:1460
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:2003
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:250
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:829
unsigned char cache_key
Store key.
Definition: forward.h:29
int64_t endOffset() const
Definition: MemObject.cc:218
int64_t objectLen() const
Definition: store.cc:1661
char const * getSerialisedMetaData()
Definition: store.cc:1831
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
static DeferredRead::DeferrableRead DeferReader
Definition: Store.h:45
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:2014
virtual ~StoreEntry()
Definition: store.cc:351
virtual void buffer()
Definition: store.cc:1641
void lastModified(const time_t when)
Definition: Store.h:159
void expireNow()
Definition: store.cc:796
void Stats(StoreEntry *output)
Definition: store.cc:118
static size_t inUseCount()
Definition: store.cc:177
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope=ksDefault)
Definition: store.cc:528
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1629
int locked() const
Definition: store.cc:1255
Definition: Range.h:18
int validToSend() const
Definition: store.cc:1420
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:191
virtual void flush()
Definition: store.cc:1652
HttpReply const * getReply() const
Definition: store.cc:1676
bool isAccepting() const
Definition: store.cc:2063
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:214
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
Definition: store.cc:203
void storeFsDone(void)
void FREE(void *)
Definition: forward.h:36
int conn
the current server connection FD
Definition: Transport.cc:26
time_t expires
Definition: Store.h:204
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:650
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1862
#define PRINTF_FORMAT_ARG2
void destroyMemObject()
Definition: store.cc:378
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:151
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:140
void doAbandon(const char *context)
Definition: store.cc:480
int32_t index
entry position inside the memory cache
Definition: MemObject.h:146
time_t timestamp
Definition: Store.h:202
void replaceHttpReply(HttpReply *, bool andStartWriting=true)
Definition: store.cc:1788
void storeInit(void)
Definition: store.cc:1325
ping_status_t
Definition: enums.h:39
static void getPublicByRequest(StoreClient *aClient, HttpRequest *request)
Definition: store.cc:508
void storeReplAdd(const char *, REMOVALPOLICYCREATE *)
Definition: store.cc:1707
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:137
void complete()
Definition: store.cc:1064
store_client_t storeClientType() const
Definition: store.cc:267
static void getPublic(StoreClient *aClient, const char *uri, const HttpRequestMethod &method)
Definition: store.cc:515
void invokeHandlers()
StoreEntry * storeGetPublicByRequest(HttpRequest *request, const KeyScope keyScope=ksDefault)
Definition: store.cc:534
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:119
void hashDelete()
Definition: store.cc:427
void storeSwapFileNumberSet(StoreEntry *e, sfileno filn)
uint64_t swap_file_sz
Definition: Store.h:208
ESIElement::Pointer cachedESITree
Definition: Store.h:236
ping_status_t ping_status
Definition: Store.h:220
void hashInsert(const cache_key *)
Definition: store.cc:418
store_client_t
Definition: enums.h:67
StoreEntry()
Definition: store.cc:330
void storeFsInit(void)
Definition: store.cc:1698
void storeConfigure(void)
Definition: store.cc:1339
const char * url() const
Definition: store.cc:1606
void abort()
Definition: store.cc:1113
size_t storeEntryInUse()
bool checkCachable()
Definition: store.cc:958
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:358
mem_status_t mem_status
Definition: Store.h:218
StoreEntry * storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod &)
Definition: store.cc:782
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:917
bool isEmpty() const
Definition: Store.h:60
void storeGetMemSpace(int size)
Definition: store.cc:1167
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:593
void abandon(const char *context)
Definition: Store.h:265
Store statistics related to low-level I/O.
Definition: StoreStats.h:57
void releaseRequest(const bool shareable=false)
Definition: store.cc:452
uint16_t flags
Definition: Store.h:210
virtual void vappendf(const char *, va_list)
Definition: store.cc:847
void setNoDelay(bool const)
Definition: store.cc:253
RemovalPolicyNode repl
Definition: Store.h:200
void cacheInMemory()
start or continue storing in memory cache
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:194
signed_int32_t sfileno
Definition: forward.h:22
MemObject * mem_obj
Definition: Store.h:199
int64_t contentLen() const
Definition: store.cc:1668
friend std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2115
int checkTooSmall()
Definition: store.cc:929
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
uint16_t refcount
Definition: Store.h:209
void STABH(void *)
Definition: MemObject.h:28
void const char * buf
Definition: stub_helper.cc:16
void DeferrableRead(void *context, CommRead const &)
Definition: CommRead.h:35
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1056
void Maintain(void *unused)
Definition: store.cc:1181
void storeFreeMemory(void)
Definition: store.cc:1401
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:121
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:313
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod const KeyScope scope const char const RequestFlags const HttpRequestMethod const char * logUrl
Definition: stub_store.cc:114
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2087
signed int sdirno
Definition: forward.h:23
unsigned short lock_count
Definition: Store.h:302
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:676
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:145
void dump(int debug_lvl) const
Definition: store.cc:1539
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1994
store_status_t store_status
Definition: Store.h:222
static const char * storeId(const int i)
Definition: testRock.cc:177
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1954
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1868
int checkNegativeHit() const
Definition: store.cc:1366
const cache_key * publicKey() const
Definition: Store.h:98
void swapOutFileClose(int how)
bool mayStartHitting() const
Definition: Store.h:269
Store::EntryGuard EntryGuard
Definition: Store.h:287
bool cacheNegatively()
Definition: store.cc:166
void write(StoreIOBuffer)
Definition: store.cc:803
void(* STOREGETCLIENT)(StoreEntry *, void *cbdata)
Definition: Store.h:325
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1345
const char * context_
default unlock() message
Definition: Store.h:369
bool mayStartSwapOut()
whether we may start writing to disk (now or in the future)
bool swapoutFailed() const
whether we failed to write this entry to disk
Definition: Store.h:123
bool checkTooBig() const
Definition: store.cc:945
bool checkDeferRead(int fd) const
Definition: store.cc:247
KeyScope
Definition: store_key_md5.h:18
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:51
StoreIoStats store_io_stats
Definition: store_io.cc:16
size_t HttpReply *STUB StoreEntry const KeyScope scope const HttpRequestMethod & method
Definition: stub_store.cc:112
const char * storeEntryFlags(const StoreEntry *)
Definition: stat.cc:276
StoreEntry * adjustVary()
Definition: store.cc:689
Entry * entry_
the guarded Entry or nil
Definition: Store.h:368
swap_status_t swap_status
Definition: Store.h:224
void storeAppendVPrintf(StoreEntry *, const char *, va_list ap)
Definition: store.cc:891
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:120
void registerAbort(STABH *cb, void *)
Definition: store.cc:1520
int expiresMoreThan(time_t, time_t)
Definition: store.cc:1411
void clearPublicKeyScope()
Definition: store.cc:633
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1771
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:358
void clearPrivate()
Definition: store.cc:158
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:206
void startWriting()
Definition: store.cc:1804
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1965
void unregisterAbort()
Definition: store.cc:1529
time_t lastref
Definition: Store.h:203
enum _mem_status_t mem_status_t
void storeHeapPositionUpdate(StoreEntry *, SwapDir *)
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1947
void swapOut()
void reset()
Definition: store.cc:1682
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:566
void transientsAbandonmentCheck()
Definition: store.cc:1849
bool validLength() const
Definition: store.cc:1272
const char * describeTimestamps() const
Definition: store.cc:2075
void negativeCache()
Definition: store.cc:1387
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:2036
A StoreEntry::getPublic*() caller.
Definition: StoreClient.h:23
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1877
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:229
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:368
manages a single cache_dir
Definition: Disk.h:21
void setMemStatus(mem_status_t)
Definition: store.cc:1564
void lock(const char *context)
Definition: store.cc:439
void createMemObject()
Definition: store.cc:1615

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors