Transients.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 20 Storage Manager */
10 
11 #include "squid.h"
12 #include "base/RunnersRegistry.h"
13 #include "CollapsedForwarding.h"
14 #include "HttpReply.h"
15 #include "ipc/mem/Page.h"
16 #include "ipc/mem/Pages.h"
17 #include "MemObject.h"
18 #include "mime_header.h"
19 #include "SquidConfig.h"
20 #include "SquidMath.h"
21 #include "StoreStats.h"
22 #include "tools.h"
23 #include "Transients.h"
24 
25 #include <limits>
26 
28 static const SBuf MapLabel("transients_map");
29 
31 {
32 }
33 
35 {
36  delete map;
37  delete locals;
38 }
39 
40 void
42 {
43  assert(Enabled());
44  const int64_t entryLimit = EntryLimit();
45  assert(entryLimit > 0);
46 
47  Must(!map);
48  map = new TransientsMap(MapLabel);
49  map->cleaner = this;
50  map->disableHitValidation(); // Transients lacks slices to validate
51 
52  locals = new Locals(entryLimit, 0);
53 }
54 
55 void
57 {
58 #if TRANSIENT_STATS_SUPPORTED
59  const size_t pageSize = Ipc::Mem::PageSize();
60 
61  stats.mem.shared = true;
62  stats.mem.capacity =
64  stats.mem.size =
66  stats.mem.count = currentCount();
67 #endif
68 }
69 
70 void
72 {
73  storeAppendPrintf(&e, "\n\nTransient Objects\n");
74 
75  storeAppendPrintf(&e, "Maximum Size: %.0f KB\n", maxSize()/1024.0);
76  storeAppendPrintf(&e, "Current Size: %.2f KB %.2f%%\n",
77  currentSize() / 1024.0,
79 
80  if (map) {
81  const int limit = map->entryLimit();
82  storeAppendPrintf(&e, "Maximum entries: %9d\n", limit);
83  if (limit > 0) {
84  storeAppendPrintf(&e, "Current entries: %" PRId64 " %.2f%%\n",
85  currentCount(), (100.0 * currentCount() / limit));
86  }
87  }
88 }
89 
90 void
92 {
93  // no lazy garbage collection needed
94 }
95 
96 uint64_t
98 {
99  return 0; // XXX: irrelevant, but Store parent forces us to implement this
100 }
101 
102 uint64_t
104 {
105  // Squid currently does not limit the total size of all transient objects
107 }
108 
109 uint64_t
111 {
112  // TODO: we do not get enough information to calculate this
113  // StoreEntry should update associated stores when its size changes
114  return 0;
115 }
116 
117 uint64_t
119 {
120  return map ? map->entryCount() : 0;
121 }
122 
123 int64_t
125 {
126  // Squid currently does not limit the size of a transient object
128 }
129 
130 void
132 {
133  // no replacement policy (but the cache(s) storing the entry may have one)
134 }
135 
136 bool
138 {
139  // no need to keep e in the global store_table for us; we have our own map
140  return false;
141 }
142 
143 StoreEntry *
145 {
146  if (!map)
147  return NULL;
148 
149  sfileno index;
150  const Ipc::StoreMapAnchor *anchor = map->openForReading(key, index);
151  if (!anchor)
152  return NULL;
153 
154  // If we already have a local entry, the store_table should have found it.
155  // Since it did not, the local entry key must have changed from public to
156  // private. We still need to keep the private entry around for syncing as
157  // its clients depend on it, but we should not allow new clients to join.
158  if (StoreEntry *oldE = locals->at(index)) {
159  debugs(20, 3, "not joining private " << *oldE);
160  assert(EBIT_TEST(oldE->flags, KEY_PRIVATE));
162  return nullptr;
163  }
164 
165  // store hadWriter before checking ENTRY_REQUIRES_COLLAPSING to avoid racing
166  // the writer that clears that flag and then leaves
167  const auto hadWriter = map->peekAtWriter(index);
168  if (!hadWriter && EBIT_TEST(anchor->basics.flags, ENTRY_REQUIRES_COLLAPSING)) {
169  debugs(20, 3, "not joining abandoned entry " << index);
171  return nullptr;
172  }
173 
174  StoreEntry *e = new StoreEntry();
175  e->createMemObject();
176  anchorEntry(*e, index, *anchor);
177 
178  // keep read lock to receive updates from others
179  return e;
180 }
181 
182 StoreEntry *
184 {
185  if (!map)
186  return NULL;
187 
188  if (StoreEntry *oldE = locals->at(index)) {
189  debugs(20, 5, "found " << *oldE << " at " << index << " in " << MapLabel);
190  assert(oldE->mem_obj && oldE->mem_obj->xitTable.index == index);
191  return oldE;
192  }
193 
194  debugs(20, 3, "no entry at " << index << " in " << MapLabel);
195  return NULL;
196 }
197 
198 void
200 {
201  assert(map);
202  assert(e.hasTransients());
203  assert(isWriter(e));
204  const auto idx = e.mem_obj->xitTable.index;
205  auto &anchor = map->writeableEntry(idx);
206  if (EBIT_TEST(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING)) {
207  EBIT_CLR(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING);
209  }
210 }
211 
212 void
214 {
215  if (!e->hasTransients()) {
216  addEntry(e, key, direction);
217  assert(e->hasTransients());
218  }
219 
220  const auto index = e->mem_obj->xitTable.index;
221  if (const auto old = locals->at(index)) {
222  assert(old == e);
223  } else {
224  // We do not lock e because we do not want to prevent its destruction;
225  // e is tied to us via mem_obj so we will know when it is destructed.
226  locals->at(index) = e;
227  }
228 }
229 
231 void
233 {
234  assert(e);
235  assert(e->mem_obj);
236  assert(!e->hasTransients());
237 
238  Must(map); // configured to track transients
239 
240  if (direction == Store::ioWriting)
241  return addWriterEntry(*e, key);
242 
243  assert(direction == Store::ioReading);
244  addReaderEntry(*e, key);
245 }
246 
248 void
250 {
251  sfileno index = 0;
252  const auto anchor = map->openForWriting(key, index);
253  if (!anchor)
254  throw TextException("writer collision", Here());
255 
256  // set ASAP in hope to unlock the slot if something throws
257  // and to provide index to such methods as hasWriter()
258  auto &xitTable = e.mem_obj->xitTable;
259  xitTable.index = index;
260  xitTable.io = Store::ioWriting;
261 
262  anchor->set(e, key);
263  // allow reading and receive remote DELETE events, but do not switch to
264  // the reading lock because transientReaders() callers want true readers
265  map->startAppending(index);
266 }
267 
270 void
272 {
273  sfileno index = 0;
274  const auto anchor = map->openOrCreateForReading(key, index, e);
275  if (!anchor)
276  throw TextException("reader collision", Here());
277 
278  anchorEntry(e, index, *anchor);
279  // keep the entry locked (for reading) to receive remote DELETE events
280 }
281 
283 void
285 {
286  // set ASAP in hope to unlock the slot if something throws
287  // and to provide index to such methods as hasWriter()
288  auto &xitTable = e.mem_obj->xitTable;
289  xitTable.index = index;
290  xitTable.io = Store::ioReading;
291 
292  const auto hadWriter = hasWriter(e); // before computing collapsingRequired
293  anchor.exportInto(e);
294  const bool collapsingRequired = EBIT_TEST(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING);
295  assert(!collapsingRequired || hadWriter);
296  e.setCollapsingRequirement(collapsingRequired);
297 }
298 
299 bool
301 {
302  if (!e.hasTransients())
303  return false;
304  return map->peekAtWriter(e.mem_obj->xitTable.index);
305 }
306 
307 void
309 {
310  // TODO: we should probably find the entry being deleted and abort it
311 }
312 
313 void
314 Transients::status(const StoreEntry &entry, Transients::EntryStatus &entryStatus) const
315 {
316  assert(map);
317  assert(entry.hasTransients());
318  const auto idx = entry.mem_obj->xitTable.index;
319  const auto &anchor = isWriter(entry) ?
320  map->writeableEntry(idx) : map->readableEntry(idx);
321  entryStatus.abortedByWriter = anchor.writerHalted;
322  entryStatus.waitingToBeFreed = anchor.waitingToBeFreed;
323  entryStatus.collapsed = EBIT_TEST(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING);
324 }
325 
326 void
328 {
329  assert(e.hasTransients());
330  assert(isWriter(e));
333 }
334 
335 int
337 {
338  if (e.hasTransients()) {
339  assert(map);
341  }
342  return 0;
343 }
344 
345 void
347 {
348  debugs(20, 5, e);
349  if (e.hasTransients()) {
350  const auto index = e.mem_obj->xitTable.index;
351  if (map->freeEntry(index)) {
352  // Delay syncCollapsed(index) which may end `e` wait for updates.
353  // Calling it directly/here creates complex reentrant call chains.
355  }
356  } // else nothing to do because e must be private
357 }
358 
359 void
361 {
362  if (!map)
363  return;
364 
365  const sfileno index = map->fileNoByKey(key);
366  if (map->freeEntry(index))
367  CollapsedForwarding::Broadcast(index, true);
368 }
369 
370 void
372 {
373  debugs(20, 5, entry);
374  if (entry.hasTransients()) {
375  auto &xitTable = entry.mem_obj->xitTable;
376  assert(map);
377  if (isWriter(entry)) {
378  map->abortWriting(xitTable.index);
379  } else {
380  assert(isReader(entry));
381  map->closeForReadingAndFreeIdle(xitTable.index);
382  }
383  locals->at(xitTable.index) = nullptr;
384  xitTable.index = -1;
385  xitTable.io = Store::ioDone;
386  }
387 }
388 
390 int64_t
392 {
393  return (UsingSmp() && Store::Controller::SmpAware()) ?
395 }
396 
397 bool
399 {
400  assert(map);
401  return map->markedForDeletion(key);
402 }
403 
404 bool
406 {
407  return e.mem_obj && e.mem_obj->xitTable.io == Store::ioReading;
408 }
409 
410 bool
412 {
413  return e.mem_obj && e.mem_obj->xitTable.io == Store::ioWriting;
414 }
415 
418 {
419 public:
420  /* RegisteredRunner API */
421  virtual void useConfig();
422  virtual ~TransientsRr();
423 
424 protected:
425  virtual void create();
426 
427 private:
429 };
430 
432 
433 void
435 {
438 }
439 
440 void
442 {
443  const int64_t entryLimit = Transients::EntryLimit();
444  if (entryLimit <= 0)
445  return; // no SMP configured or a misconfiguration
446 
447  Must(!mapOwner);
448  mapOwner = TransientsMap::Init(MapLabel, entryLimit);
449 }
450 
452 {
453  delete mapOwner;
454 }
455 
#define EBIT_CLR(flag, bit)
Definition: defines.h:106
Anchor & writeableEntry(const AnchorId anchorId)
writeable anchor for the entry created by openForWriting()
Definition: StoreMap.cc:237
class Ping::pingStats_ stats
void monitorIo(StoreEntry *, const cache_key *, const Store::IoStatus)
Definition: Transients.cc:213
#define Here()
source code location of the caller
Definition: Here.h:15
initializes shared memory segment used by Transients
Definition: Transients.cc:417
IoStatus
cache "I/O" direction and status
Definition: forward.h:40
void startAppending(const sfileno fileno)
restrict opened for writing entry to appending operations; allow reads
Definition: StoreMap.cc:191
virtual void getStats(StoreInfoStats &stats) const override
collect statistics
Definition: Transients.cc:56
TransientsMap * map
shared packed info indexed by Store keys, for creating new StoreEntries
Definition: Transients.h:107
unsigned char cache_key
Store key.
Definition: forward.h:29
Locals * locals
Definition: Transients.h:112
MemObject * mem_obj
Definition: Store.h:213
bool waitingToBeFreed
whether the entry was marked for deletion
Definition: Transients.h:34
static bool SmpAware()
whether there are any SMP-aware storages
Definition: Controller.cc:920
void createMemObject()
Definition: store.cc:1631
virtual uint64_t currentSize() const override
current size
Definition: Transients.cc:110
void anchorEntry(StoreEntry &, const sfileno, const Ipc::StoreMapAnchor &)
fills (recently created) StoreEntry with information currently in Transients
Definition: Transients.cc:284
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:901
void disconnect(StoreEntry &)
the caller is done writing or reading the given entry
Definition: Transients.cc:371
@ KEY_PRIVATE
Definition: enums.h:102
Definition: SBuf.h:86
int64_t shared_transient_entries_limit
Definition: SquidConfig.h:351
Io io
current I/O state
Definition: MemObject.h:173
const A & max(A const &lhs, A const &rhs)
virtual ~TransientsRr()
Definition: Transients.cc:451
ReadWriteLock lock
protects slot data below
Definition: StoreMap.h:80
virtual uint64_t maxSize() const override
Definition: Transients.cc:103
std::atomic< uint32_t > readers
number of reading users
Definition: ReadWriteLock.h:49
virtual uint64_t currentCount() const override
the total number of objects stored right now
Definition: Transients.cc:118
static Owner * Init(const SBuf &path, const int slotLimit)
initialize shared memory
Definition: StoreMap.cc:42
bool abortedByWriter
whether the entry was aborted
Definition: Transients.h:33
shared entry metadata, used for synchronization
Definition: Transients.h:30
virtual void init() override
Definition: Transients.cc:41
double doublePercent(const double, const double)
Definition: SquidMath.cc:25
const Anchor * openForReading(const cache_key *const key, sfileno &fileno)
opens entry (identified by key) for reading, increments read level
Definition: StoreMap.cc:440
void status(const StoreEntry &e, EntryStatus &entryStatus) const
copies current shared entry metadata into entryStatus
Definition: Transients.cc:314
struct Ipc::StoreMapAnchor::Basics basics
static bool Enabled()
Can we create and initialize Transients?
Definition: Transients.h:94
#define NULL
Definition: types.h:166
const Anchor & readableEntry(const AnchorId anchorId) const
readable anchor for the entry created by openForReading()
Definition: StoreMap.cc:244
virtual ~Transients()
Definition: Transients.cc:34
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
int entryLimit() const
maximum entryCount() possible
Definition: StoreMap.cc:733
int32_t StoreMapSliceId
Definition: StoreMap.h:24
bool isWriter(const StoreEntry &) const
whether the entry is in "writing to Transients" I/O state
Definition: Transients.cc:411
void addWriterEntry(StoreEntry &, const cache_key *)
addEntry() helper used for cache entry creators/writers
Definition: Transients.cc:249
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2074
std::vector< StoreEntry * > Locals
Definition: Transients.h:109
virtual void noteFreeMapSlice(const Ipc::StoreMapSliceId sliceId) override
adjust slice-linked state before a locked Readable slice is erased
Definition: Transients.cc:308
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
@ ioDone
Definition: forward.h:40
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
virtual StoreEntry * get(const cache_key *) override
Definition: Transients.cc:144
bool hasWriter(const StoreEntry &)
whether we or somebody else is in the "writing to Transients" I/O state
Definition: Transients.cc:300
#define assert(EX)
Definition: assert.h:19
YesNoNone memShared
whether the memory cache is shared among workers
Definition: SquidConfig.h:86
size_t PageLevel()
approximate total number of shared memory pages used now
Definition: Pages.cc:80
int entryCount() const
number of writeable and readable entries
Definition: StoreMap.cc:739
void addEntry(StoreEntry *, const cache_key *, const Store::IoStatus)
creates a new Transients entry
Definition: Transients.cc:232
bool configured() const
Definition: YesNoNone.h:67
StoreEntry * findCollapsed(const sfileno xitIndex)
return a local, previously collapsed entry
Definition: Transients.cc:183
virtual void reference(StoreEntry &e) override
somebody needs this entry (many cache replacement policies need to know)
Definition: Transients.cc:131
Anchor * openForWriting(const cache_key *const key, sfileno &fileno)
Definition: StoreMap.cc:140
signed_int32_t sfileno
Definition: forward.h:22
bool freeEntry(const sfileno)
Definition: StoreMap.cc:313
virtual uint64_t minSize() const override
the minimum size the store will shrink to via normal housekeeping
Definition: Transients.cc:97
bool markedForDeletion(const cache_key *) const
Definition: Transients.cc:398
void completeWriting(const StoreEntry &e)
called when the in-transit entry has been successfully cached
Definition: Transients.cc:327
const Anchor * openOrCreateForReading(const cache_key *, sfileno &, const StoreEntry &)
openForReading() but creates a new entry if there is no old one
Definition: StoreMap.cc:103
void exportInto(StoreEntry &) const
load StoreEntry basics that were previously stored with set()
Definition: StoreMap.cc:979
bool isReader(const StoreEntry &) const
whether the entry is in "reading from Transients" I/O state
Definition: Transients.cc:405
@ ENTRY_REQUIRES_COLLAPSING
Definition: enums.h:118
const Anchor * peekAtWriter(const sfileno fileno) const
Definition: StoreMap.cc:297
size_t PageLimit()
the total number of shared memory pages that can be in use at any time
Definition: Pages.cc:55
virtual bool dereference(StoreEntry &e) override
Definition: Transients.cc:137
void addReaderEntry(StoreEntry &, const cache_key *)
Definition: Transients.cc:271
StoreMapCleaner * cleaner
notified before a readable entry is freed
Definition: StoreMap.h:342
an std::runtime_error with thrower location info
Definition: TextException.h:19
void disableHitValidation()
Definition: StoreMap.h:327
bool markedForDeletion(const cache_key *const)
Definition: StoreMap.cc:355
const Anchor & peekAtEntry(const sfileno fileno) const
Definition: StoreMap.cc:307
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: Transients.cc:91
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
void closeForReadingAndFreeIdle(const sfileno fileno)
same as closeForReading() but also frees the entry if it is unlocked
Definition: StoreMap.cc:506
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:203
bool collapsed
whether the entry allows collapsing
Definition: Transients.h:35
static int64_t EntryLimit()
calculates maximum number of entries we need to store and map
Definition: Transients.cc:391
#define PRId64
Definition: types.h:110
TransientsMap::Owner * mapOwner
Definition: Transients.cc:428
void abortWriting(const sfileno fileno)
stop writing the entry, freeing its slot for others to use if possible
Definition: StoreMap.cc:251
RunnerRegistrationEntry(TransientsRr)
size_t PageSize()
returns page size in bytes; all pages are assumed to be the same size
Definition: Pages.cc:28
static void Broadcast(const StoreEntry &e, const bool includingThisWorker=false)
notify other workers about changes in entry state (e.g., new data)
static const SBuf MapLabel("transients_map")
shared memory segment path to use for Transients map
Ipc::StoreMap TransientsMap
Definition: Transients.h:20
@ ioWriting
Definition: forward.h:40
bool UsingSmp()
Whether there should be more than one worker process running.
Definition: tools.cc:658
virtual void useConfig()
Definition: Segment.cc:376
virtual void evictCached(StoreEntry &) override
Definition: Transients.cc:346
virtual int64_t maxObjectSize() const override
the maximum size of a storable object; -1 if unlimited
Definition: Transients.cc:124
virtual void create()
called when the runner should create a new memory segment
Definition: Transients.cc:441
@ ioReading
Definition: forward.h:40
virtual void stat(StoreEntry &e) const override
Definition: Transients.cc:71
void clearCollapsingRequirement(const StoreEntry &e)
removes collapsing requirement (for future hits)
Definition: Transients.cc:199
sfileno fileNoByKey(const cache_key *const key) const
computes map entry anchor position for a given entry key
Definition: StoreMap.cc:912
aggregates anchor and slice owners for Init() caller convenience
Definition: StoreMap.h:213
High-level store statistics used by mgr:info action. Used inside PODs!
Definition: StoreStats.h:13
virtual void useConfig()
Definition: Transients.cc:434
void switchWritingToReading(const sfileno fileno)
stop writing (or updating) the locked entry and start reading it
Definition: StoreMap.cc:211
class SquidConfig Config
Definition: SquidConfig.cc:12
virtual void evictIfFound(const cache_key *) override
Definition: Transients.cc:360
int readers(const StoreEntry &e) const
number of entry readers some time ago
Definition: Transients.cc:336

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors