Store.h
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 #ifndef SQUID_STORE_H
10 #define SQUID_STORE_H
11 
12 #include "base/Packable.h"
13 #include "base/Range.h"
14 #include "base/RefCount.h"
15 #include "comm/forward.h"
16 #include "CommRead.h"
17 #include "hash.h"
18 #include "http/forward.h"
19 #include "http/RequestMethod.h"
20 #include "HttpReply.h"
21 #include "MemObject.h"
22 #include "RemovalPolicy.h"
23 #include "store/Controller.h"
24 #include "store/forward.h"
25 #include "store_key_md5.h"
26 #include "StoreIOBuffer.h"
27 #include "StoreStats.h"
28 
29 #if USE_SQUID_ESI
30 #include "esi/Element.h"
31 #endif
32 
33 #include <ostream>
34 
35 class AsyncCall;
36 class HttpRequest;
37 class RequestFlags;
38 
40 
41 class StoreEntry : public hash_link, public Packable
42 {
43 
44 public:
46  bool checkDeferRead(int fd) const;
47 
48  const char *getMD5Text() const;
49  StoreEntry();
50  virtual ~StoreEntry();
51 
52  HttpReply const *getReply() const;
53  void write(StoreIOBuffer);
54 
60  bool isEmpty() const {
61  assert (mem_obj);
62  return mem_obj->endOffset() == 0;
63  }
64  bool isAccepting() const;
65  size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
67  void lengthWentBad(const char *reason);
68  void complete();
70  char const *getSerialisedMetaData();
72  void storeErrorResponse(HttpReply *reply);
73  void replaceHttpReply(HttpReply *, bool andStartWriting = true);
74  void startWriting();
75  bool mayStartSwapOut();
77  void trimMemory(const bool preserveSwappable);
78 
79  // called when a decision to cache in memory has been made
80  void memOutDecision(const bool willCacheInRam);
81  // called when a decision to cache on disk has been made
82  void swapOutDecision(const MemObject::SwapOut::Decision &decision);
83 
84  void abort();
85  bool makePublic(const KeyScope keyScope = ksDefault);
86  void makePrivate(const bool shareable);
90  void clearPrivate();
91  bool setPublicKey(const KeyScope keyScope = ksDefault);
95  void clearPublicKeyScope();
96 
98  const cache_key *publicKey() const {
99  return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
100  reinterpret_cast<const cache_key*>(key): // may be nil
101  nullptr;
102  }
103 
107  void setPrivateKey(const bool shareable, const bool permanent);
108 
109  void expireNow();
112  void releaseRequest(const bool shareable = false);
113  void negativeCache();
114  bool cacheNegatively();
115  void invokeHandlers();
116  void cacheInMemory();
117  void swapOut();
119  bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
121  bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
122  void swapOutFileClose(int how);
123  const char *url() const;
127  bool checkCachable();
128  int checkNegativeHit() const;
129  int locked() const;
130  int validToSend() const;
131  bool memoryCachable();
132 
135  void createMemObject();
136 
138  void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
139 
141  void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
142 
143  void dump(int debug_lvl) const;
144  void hashDelete();
145  void hashInsert(const cache_key *);
146  void registerAbort(STABH * cb, void *);
147  void reset();
149  bool timestampsSet();
150  void unregisterAbort();
151  void destroyMemObject();
152  int checkTooSmall();
153 
155 
156  void setNoDelay (bool const);
157  void lastModified(const time_t when) { lastModified_ = when; }
159  time_t lastModified() const {
160  // may still return -1 if timestamp is not set
161  return lastModified_ < 0 ? timestamp : lastModified_;
162  }
164  const char *describeTimestamps() const;
165  // TODO: consider removing currently unsupported imslen parameter
166  bool modifiedSince(const time_t ims, const int imslen = -1) const;
168  bool hasIfMatchEtag(const HttpRequest &request) const;
170  bool hasIfNoneMatchEtag(const HttpRequest &request) const;
172  bool hasEtag(ETag &etag) const;
173 
175  Store::Disk &disk() const;
178  bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
181  void attachToDisk(const sdirno, const sfileno, const swap_status_t);
184  void detachFromDisk();
185 
187  bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
189  bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
190 
193 
195  void setCollapsingRequirement(const bool required);
196 
199  /* START OF ON-DISK STORE_META_STD TLV field */
200  time_t timestamp;
201  time_t lastref;
202  time_t expires;
203 private:
204  time_t lastModified_;
205 public:
206  uint64_t swap_file_sz;
207  uint16_t refcount;
208  uint16_t flags;
209  /* END OF ON-DISK STORE_META_STD */
210 
212  sfileno swap_filen:25; // keep in sync with SwapFilenMax
213 
215 
217 
219 
221 
223 
224 public:
225  static size_t inUseCount();
226  static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
227  static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
228  static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
229 
230  void *operator new(size_t byteCount);
231  void operator delete(void *address);
232 #if USE_SQUID_ESI
233 
235 #endif
236  int64_t objectLen() const;
237  int64_t contentLen() const;
238 
241  void lock(const char *context);
242 
245  int unlock(const char *context);
246 
248  int locks() const { return static_cast<int>(lock_count); }
249 
251  void touch();
252 
257  void release(const bool shareable = false);
258 
263  void abandon(const char *context) { if (!locked()) doAbandon(context); }
264 
267  bool mayStartHitting() const {
269  }
270 
271 #if USE_ADAPTATION
272  void deferProducer(const AsyncCall::Pointer &producer);
275  void kickProducer();
276 #endif
277 
278  /* Packable API */
279  virtual void append(char const *, int);
280  virtual void vappendf(const char *, va_list);
281  virtual void buffer();
282  virtual void flush();
283 
284 protected:
286 
289  void checkDisk() const;
290 
291 private:
292  void doAbandon(const char *context);
293  bool checkTooBig() const;
294  void forcePublicKey(const cache_key *newkey);
295  StoreEntry *adjustVary();
296  const cache_key *calcPublicKey(const KeyScope keyScope);
297 
299 
300  unsigned short lock_count; /* Assume < 65536! */
301 
308 
309 #if USE_ADAPTATION
312 #endif
313 
314  bool validLength() const;
315  bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
316 
317  friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
318 };
319 
320 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
321 
323 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
324 
325 namespace Store {
326 
329 class EntryGuard {
330 public:
333  EntryGuard(Entry *entry, const char *context):
334  entry_(entry), context_(context) {
335  assert(!entry_ || entry_->locked());
336  }
337 
339  if (entry_) {
340  // something went wrong -- the caller did not unlockAndReset() us
341  onException();
342  }
343  }
344 
345  EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
346 
349  Entry *get() {
350  return entry_;
351  }
352 
356  void unlockAndReset(const char *resetContext = nullptr) {
357  if (entry_) {
358  entry_->unlock(resetContext ? resetContext : context_);
359  entry_ = nullptr;
360  }
361  }
362 
363 private:
364  void onException() noexcept;
365 
366  Entry *entry_;
367  const char *context_;
368 };
369 
370 void Stats(StoreEntry *output);
371 void Maintain(void *unused);
372 }; // namespace Store
373 
375 size_t storeEntryInUse();
376 
378 const char *storeEntryFlags(const StoreEntry *);
379 
381 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
382 
384 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
385 
387 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
388 
390 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault);
391 
394 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
395 
398 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
399 
401 void storeInit(void);
402 
404 void storeConfigure(void);
405 
407 void storeFreeMemory(void);
408 
410 int expiresMoreThan(time_t, time_t);
411 
413 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
414 
416 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
417 
419 int storeTooManyDiskFilesOpen(void);
420 
422 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
423 
425 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
426 
428 void storeFsInit(void);
429 
431 void storeFsDone(void);
432 
434 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
435 
440 extern FREE destroyStoreEntry;
441 
443 void storeGetMemSpace(int size);
444 
445 #endif /* SQUID_STORE_H */
446 
sdirno swap_dirn
Definition: Store.h:214
bool shareableWhenPrivate
Definition: Store.h:307
static MemAllocator * pool
Definition: Store.h:298
void detachFromDisk()
Definition: store.cc:2022
void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2
Definition: store.cc:875
bool isAccepting() const
Definition: store.cc:2047
void makePrivate(const bool shareable)
Definition: store.cc:151
#define assert(EX)
Definition: assert.h:17
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:516
void const char HLPCB * callback
Definition: stub_helper.cc:16
void touch()
update last reference timestamp and related Store metadata
Definition: store.cc:440
static void getPublicByRequestMethod(StoreClient *aClient, HttpRequest *request, const HttpRequestMethod &method)
Definition: store.cc:495
store_status_t
Definition: enums.h:45
int validToSend() const
Definition: store.cc:1414
StoreEntry * storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod &)
Definition: store.cc:758
void release(const bool shareable=false)
Definition: store.cc:1189
int unlock(const char *context)
Definition: store.cc:457
Definition: ETag.h:17
Definition: cbdata.cc:60
void(* STOREGETCLIENT)(StoreEntry *, void *cbdata)
Definition: Store.h:323
int locks() const
returns a local concurrent use counter, for debugging
Definition: Store.h:248
int64_t contentLen() const
Definition: store.cc:1662
EntryGuard(Entry *entry, const char *context)
Definition: Store.h:333
void storeEntryReplaceObject(StoreEntry *, HttpReply *)
bool timestampsSet()
Definition: store.cc:1454
virtual void append(char const *, int)
Appends a c-string to existing packed data.
Definition: store.cc:823
unsigned char cache_key
Store key.
Definition: forward.h:29
int64_t endOffset() const
Definition: MemObject.cc:218
static DeferredRead::DeferrableRead DeferReader
Definition: Store.h:45
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:187
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:2008
virtual ~StoreEntry()
Definition: store.cc:345
virtual void buffer()
Definition: store.cc:1635
void lastModified(const time_t when)
Definition: Store.h:157
void expireNow()
Definition: store.cc:790
void Stats(StoreEntry *output)
Definition: store.cc:117
static size_t inUseCount()
Definition: store.cc:176
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *request, const HttpRequestMethod &method, const KeyScope keyScope=ksDefault)
Definition: store.cc:522
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1623
Definition: Range.h:18
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1997
virtual void flush()
Definition: store.cc:1646
bool swappingOut() const
whether we are in the process of writing this entry to disk
Definition: Store.h:119
std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2099
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:212
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
Definition: store.cc:202
void storeFsDone(void)
void FREE(void *)
Definition: forward.h:36
int conn
the current server connection FD
Definition: Transport.cc:26
time_t expires
Definition: Store.h:202
void forcePublicKey(const cache_key *newkey)
Definition: store.cc:644
void memOutDecision(const bool willCacheInRam)
Definition: store.cc:1856
void destroyMemObject()
Definition: store.cc:372
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:151
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:140
void doAbandon(const char *context)
Definition: store.cc:474
int32_t index
entry position inside the memory cache
Definition: MemObject.h:146
time_t timestamp
Definition: Store.h:200
void replaceHttpReply(HttpReply *, bool andStartWriting=true)
Definition: store.cc:1782
void storeInit(void)
Definition: store.cc:1319
ping_status_t
Definition: enums.h:39
static void getPublicByRequest(StoreClient *aClient, HttpRequest *request)
Definition: store.cc:502
void storeReplAdd(const char *, REMOVALPOLICYCREATE *)
Definition: store.cc:1701
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:137
bool hasIfMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-Match etags
Definition: store.cc:1941
void complete()
Definition: store.cc:1058
void onException() noexcept
Definition: store.cc:2154
bool validLength() const
Definition: store.cc:1266
static void getPublic(StoreClient *aClient, const char *uri, const HttpRequestMethod &method)
Definition: store.cc:509
void invokeHandlers()
char const * getSerialisedMetaData()
Definition: store.cc:1825
StoreEntry * storeGetPublicByRequest(HttpRequest *request, const KeyScope keyScope=ksDefault)
Definition: store.cc:528
void hashDelete()
Definition: store.cc:421
int locked() const
Definition: store.cc:1249
void storeSwapFileNumberSet(StoreEntry *e, sfileno filn)
uint64_t swap_file_sz
Definition: Store.h:206
ESIElement::Pointer cachedESITree
Definition: Store.h:234
ping_status_t ping_status
Definition: Store.h:218
void hashInsert(const cache_key *)
Definition: store.cc:412
bool checkDeferRead(int fd) const
Definition: store.cc:246
store_client_t
Definition: enums.h:63
StoreEntry()
Definition: store.cc:324
bool isEmpty() const
Definition: Store.h:60
const char * describeTimestamps() const
Definition: store.cc:2059
void storeFsInit(void)
Definition: store.cc:1692
void storeConfigure(void)
Definition: store.cc:1333
int64_t objectLen() const
Definition: store.cc:1655
void abort()
Definition: store.cc:1107
time_t lastModified() const
Definition: Store.h:159
bool checkCachable()
Definition: store.cc:952
void unlockAndReset(const char *resetContext=nullptr)
Definition: Store.h:356
HttpReply const * getReply() const
Definition: store.cc:1670
mem_status_t mem_status
Definition: Store.h:216
StoreEntry * storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod &)
Definition: store.cc:776
bool hasMemStore() const
whether there is a corresponding locked shared memory table entry
Definition: Store.h:189
bool modifiedSince(const time_t ims, const int imslen=-1) const
Definition: store.cc:1893
int storeTooManyDiskFilesOpen(void)
Definition: store.cc:911
void storeGetMemSpace(int size)
Definition: store.cc:1161
bool setPublicKey(const KeyScope keyScope=ksDefault)
Definition: store.cc:587
void abandon(const char *context)
Definition: Store.h:263
void dump(int debug_lvl) const
Definition: store.cc:1533
int checkNegativeHit() const
Definition: store.cc:1360
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
whether at least one of the request ETags matches entity ETag
Definition: store.cc:1959
Store statistics related to low-level I/O.
Definition: StoreStats.h:58
void releaseRequest(const bool shareable=false)
Definition: store.cc:446
uint16_t flags
Definition: Store.h:208
virtual void vappendf(const char *, va_list)
Definition: store.cc:841
void setNoDelay(bool const)
Definition: store.cc:252
RemovalPolicyNode repl
Definition: Store.h:198
void cacheInMemory()
start or continue storing in memory cache
signed_int32_t sfileno
Definition: forward.h:22
MemObject * mem_obj
Definition: Store.h:197
friend std::ostream & operator<<(std::ostream &os, const StoreEntry &e)
Definition: store.cc:2099
int checkTooSmall()
Definition: store.cc:923
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
const char * url() const
Definition: store.cc:1600
bool mayStartHitting() const
Definition: Store.h:267
uint16_t refcount
Definition: Store.h:207
void STABH(void *)
Definition: MemObject.h:28
void const char * buf
Definition: stub_helper.cc:16
void DeferrableRead(void *context, CommRead const &)
Definition: CommRead.h:35
void lengthWentBad(const char *reason)
flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
Definition: store.cc:1050
void Maintain(void *unused)
Definition: store.cc:1175
void storeFreeMemory(void)
Definition: store.cc:1395
AsyncCall::Pointer deferredProducer
producer callback registered with deferProducer
Definition: Store.h:311
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2071
signed int sdirno
Definition: forward.h:23
unsigned short lock_count
Definition: Store.h:300
const cache_key * calcPublicKey(const KeyScope keyScope)
Definition: store.cc:670
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:144
const char * getMD5Text() const
Definition: store.cc:184
store_status_t store_status
Definition: Store.h:220
void checkDisk() const
does nothing except throwing if disk-associated data members are inconsistent
Definition: store.cc:2030
void swapOutDecision(const MemObject::SwapOut::Decision &decision)
Definition: store.cc:1862
void swapOutFileClose(int how)
Store::EntryGuard EntryGuard
Definition: Store.h:285
bool cacheNegatively()
Definition: store.cc:165
void write(StoreIOBuffer)
Definition: store.cc:797
Store::Disk & disk() const
the disk this entry is [being] cached on; asserts for entries w/o a disk
Definition: store.cc:1988
bool memoryCachable()
checkCachable() and can be cached in memory
Definition: store.cc:1339
bool mayStartSwapOut()
whether we may start writing to disk (now or in the future)
KeyScope
Definition: store_key_md5.h:18
swap_status_t
StoreEntry relationship with a disk cache.
Definition: enums.h:51
StoreIoStats store_io_stats
Definition: store_io.cc:16
const cache_key * publicKey() const
Definition: Store.h:98
const char * storeEntryFlags(const StoreEntry *)
Definition: stat.cc:276
bool hittingRequiresCollapsing() const
whether this entry can feed collapsed requests and only them
Definition: Store.h:192
StoreEntry * adjustVary()
Definition: store.cc:683
Entry * entry_
the guarded Entry or nil
Definition: Store.h:366
swap_status_t swap_status
Definition: Store.h:222
void storeAppendVPrintf(StoreEntry *, const char *, va_list ap)
Definition: store.cc:885
Decision
Decision states for StoreEntry::swapoutPossible() and related code.
Definition: MemObject.h:120
void registerAbort(STABH *cb, void *)
Definition: store.cc:1514
int expiresMoreThan(time_t, time_t)
Definition: store.cc:1405
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:228
void clearPublicKeyScope()
Definition: store.cc:627
void storeErrorResponse(HttpReply *reply)
Store a prepared error response. MemObject locks the reply object.
Definition: store.cc:1765
bool checkTooBig() const
Definition: store.cc:939
void deferProducer(const AsyncCall::Pointer &producer)
call back producer when more buffer space is available
Definition: store.cc:352
void clearPrivate()
Definition: store.cc:157
time_t lastModified_
received Last-Modified value or -1; use lastModified()
Definition: Store.h:204
bool hasEtag(ETag &etag) const
whether this entry has an ETag; if yes, puts ETag value into parameter
Definition: store.cc:1930
bool hasIfNoneMatchEtag(const HttpRequest &request) const
has ETag matching at least one of the If-None-Match etags
Definition: store.cc:1948
void startWriting()
Definition: store.cc:1798
void unregisterAbort()
Definition: store.cc:1523
time_t lastref
Definition: Store.h:201
enum _mem_status_t mem_status_t
store_client_t storeClientType() const
Definition: store.cc:264
void storeHeapPositionUpdate(StoreEntry *, SwapDir *)
void swapOut()
void reset()
Definition: store.cc:1676
void setPrivateKey(const bool shareable, const bool permanent)
Definition: store.cc:560
void transientsAbandonmentCheck()
Definition: store.cc:1843
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:121
void negativeCache()
Definition: store.cc:1381
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
A StoreEntry::getPublic*() caller.
Definition: StoreClient.h:23
void trimMemory(const bool preserveSwappable)
Definition: store.cc:1871
void kickProducer()
calls back producer registered with deferProducer
Definition: store.cc:362
manages a single cache_dir
Definition: Disk.h:21
void setMemStatus(mem_status_t)
Definition: store.cc:1558
void lock(const char *context)
Definition: store.cc:433
void createMemObject()
Definition: store.cc:1609

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors