Transients.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 20 Storage Manager */
10 
11 #include "squid.h"
12 #include "base/RunnersRegistry.h"
13 #include "CollapsedForwarding.h"
14 #include "HttpReply.h"
15 #include "ipc/mem/Page.h"
16 #include "ipc/mem/Pages.h"
17 #include "MemObject.h"
18 #include "mime_header.h"
19 #include "SquidConfig.h"
20 #include "SquidMath.h"
21 #include "StoreStats.h"
22 #include "tools.h"
23 #include "Transients.h"
24 
25 #include <limits>
26 
28 static const SBuf MapLabel("transients_map");
29 
31 {
32 }
33 
35 {
36  delete map;
37  delete locals;
38 }
39 
40 void
42 {
43  assert(Enabled());
44  const int64_t entryLimit = EntryLimit();
45  assert(entryLimit > 0);
46 
47  Must(!map);
48  map = new TransientsMap(MapLabel);
49  map->cleaner = this;
50 
51  locals = new Locals(entryLimit, 0);
52 }
53 
54 void
56 {
57 #if TRANSIENT_STATS_SUPPORTED
58  const size_t pageSize = Ipc::Mem::PageSize();
59 
60  stats.mem.shared = true;
61  stats.mem.capacity =
63  stats.mem.size =
65  stats.mem.count = currentCount();
66 #endif
67 }
68 
69 void
71 {
72  storeAppendPrintf(&e, "\n\nTransient Objects\n");
73 
74  storeAppendPrintf(&e, "Maximum Size: %.0f KB\n", maxSize()/1024.0);
75  storeAppendPrintf(&e, "Current Size: %.2f KB %.2f%%\n",
76  currentSize() / 1024.0,
78 
79  if (map) {
80  const int limit = map->entryLimit();
81  storeAppendPrintf(&e, "Maximum entries: %9d\n", limit);
82  if (limit > 0) {
83  storeAppendPrintf(&e, "Current entries: %" PRId64 " %.2f%%\n",
84  currentCount(), (100.0 * currentCount() / limit));
85  }
86  }
87 }
88 
89 void
91 {
92  // no lazy garbage collection needed
93 }
94 
95 uint64_t
97 {
98  return 0; // XXX: irrelevant, but Store parent forces us to implement this
99 }
100 
101 uint64_t
103 {
104  // Squid currently does not limit the total size of all transient objects
106 }
107 
108 uint64_t
110 {
111  // TODO: we do not get enough information to calculate this
112  // StoreEntry should update associated stores when its size changes
113  return 0;
114 }
115 
116 uint64_t
118 {
119  return map ? map->entryCount() : 0;
120 }
121 
122 int64_t
124 {
125  // Squid currently does not limit the size of a transient object
127 }
128 
129 void
131 {
132  // no replacement policy (but the cache(s) storing the entry may have one)
133 }
134 
135 bool
137 {
138  // no need to keep e in the global store_table for us; we have our own map
139  return false;
140 }
141 
142 StoreEntry *
144 {
145  if (!map)
146  return NULL;
147 
148  sfileno index;
149  const Ipc::StoreMapAnchor *anchor = map->openForReading(key, index);
150  if (!anchor)
151  return NULL;
152 
153  // If we already have a local entry, the store_table should have found it.
154  // Since it did not, the local entry key must have changed from public to
155  // private. We still need to keep the private entry around for syncing as
156  // its clients depend on it, but we should not allow new clients to join.
157  if (StoreEntry *oldE = locals->at(index)) {
158  debugs(20, 3, "not joining private " << *oldE);
159  assert(EBIT_TEST(oldE->flags, KEY_PRIVATE));
160  map->closeForReading(index);
161  return nullptr;
162  }
163 
164  StoreEntry *e = new StoreEntry();
165  e->createMemObject();
166  e->mem_obj->xitTable.index = index;
168  anchor->exportInto(*e);
169  const bool collapsingRequired = EBIT_TEST(anchor->basics.flags, ENTRY_REQUIRES_COLLAPSING);
170  e->setCollapsingRequirement(collapsingRequired);
171  // keep read lock to receive updates from others
172  return e;
173 }
174 
175 StoreEntry *
177 {
178  if (!map)
179  return NULL;
180 
181  if (StoreEntry *oldE = locals->at(index)) {
182  debugs(20, 5, "found " << *oldE << " at " << index << " in " << MapLabel);
183  assert(oldE->mem_obj && oldE->mem_obj->xitTable.index == index);
184  return oldE;
185  }
186 
187  debugs(20, 3, "no entry at " << index << " in " << MapLabel);
188  return NULL;
189 }
190 
191 void
193 {
194  assert(map);
195  assert(e.hasTransients());
196  assert(isWriter(e));
197  const auto idx = e.mem_obj->xitTable.index;
198  auto &anchor = map->writeableEntry(idx);
199  if (EBIT_TEST(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING)) {
200  EBIT_CLR(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING);
202  }
203 }
204 
205 void
207 {
208  if (!e->hasTransients()) {
209  addEntry(e, key, direction);
210  assert(e->hasTransients());
211  }
212 
213  const auto index = e->mem_obj->xitTable.index;
214  if (const auto old = locals->at(index)) {
215  assert(old == e);
216  } else {
217  // We do not lock e because we do not want to prevent its destruction;
218  // e is tied to us via mem_obj so we will know when it is destructed.
219  locals->at(index) = e;
220  }
221 }
222 
224 void
226 {
227  assert(e);
228  assert(e->mem_obj);
229  assert(!e->hasTransients());
230 
231  Must(map); // configured to track transients
232 
233  sfileno index = 0;
234  Ipc::StoreMapAnchor *slot = map->openForWriting(key, index);
235  Must(slot); // no writer collisions
236 
237  // set ASAP in hope to unlock the slot if something throws
238  e->mem_obj->xitTable.index = index;
240 
241  slot->set(*e, key);
242  if (direction == Store::ioWriting) {
243  // allow reading and receive remote DELETE events, but do not switch to
244  // the reading lock because transientReaders() callers want true readers
245  map->startAppending(index);
246  } else {
247  assert(direction == Store::ioReading);
248  // keep the entry locked (for reading) to receive remote DELETE events
249  map->switchWritingToReading(index);
251  }
252 }
253 
254 void
256 {
257  // TODO: we should probably find the entry being deleted and abort it
258 }
259 
260 void
261 Transients::status(const StoreEntry &entry, Transients::EntryStatus &entryStatus) const
262 {
263  assert(map);
264  assert(entry.hasTransients());
265  const auto idx = entry.mem_obj->xitTable.index;
266  const auto &anchor = isWriter(entry) ?
267  map->writeableEntry(idx) : map->readableEntry(idx);
268  entryStatus.abortedByWriter = anchor.writerHalted;
269  entryStatus.waitingToBeFreed = anchor.waitingToBeFreed;
270  entryStatus.collapsed = EBIT_TEST(anchor.basics.flags, ENTRY_REQUIRES_COLLAPSING);
271 }
272 
273 void
275 {
276  assert(e.hasTransients());
277  assert(isWriter(e));
280 }
281 
282 int
284 {
285  if (e.hasTransients()) {
286  assert(map);
288  }
289  return 0;
290 }
291 
292 void
294 {
295  debugs(20, 5, e);
296  if (e.hasTransients()) {
297  const auto index = e.mem_obj->xitTable.index;
298  if (map->freeEntry(index)) {
299  // Delay syncCollapsed(index) which may end `e` wait for updates.
300  // Calling it directly/here creates complex reentrant call chains.
302  }
303  } // else nothing to do because e must be private
304 }
305 
306 void
308 {
309  if (!map)
310  return;
311 
312  const sfileno index = map->fileNoByKey(key);
313  if (map->freeEntry(index))
314  CollapsedForwarding::Broadcast(index, true);
315 }
316 
317 void
319 {
320  debugs(20, 5, entry);
321  if (entry.hasTransients()) {
322  auto &xitTable = entry.mem_obj->xitTable;
323  assert(map);
324  if (isWriter(entry)) {
325  map->abortWriting(xitTable.index);
326  } else {
327  assert(isReader(entry));
328  map->closeForReading(xitTable.index);
329  }
330  locals->at(xitTable.index) = nullptr;
331  xitTable.index = -1;
332  xitTable.io = Store::ioDone;
333  }
334 }
335 
337 int64_t
339 {
340  return (UsingSmp() && Store::Controller::SmpAware()) ?
342 }
343 
344 bool
346 {
347  assert(map);
348  return map->markedForDeletion(key);
349 }
350 
351 bool
353 {
354  return e.mem_obj && e.mem_obj->xitTable.io == Store::ioReading;
355 }
356 
357 bool
359 {
360  return e.mem_obj && e.mem_obj->xitTable.io == Store::ioWriting;
361 }
362 
365 {
366 public:
367  /* RegisteredRunner API */
368  virtual void useConfig();
369  virtual ~TransientsRr();
370 
371 protected:
372  virtual void create();
373 
374 private:
375  TransientsMap::Owner *mapOwner = nullptr;
376 };
377 
379 
380 void
382 {
385 }
386 
387 void
389 {
390  const int64_t entryLimit = Transients::EntryLimit();
391  if (entryLimit <= 0)
392  return; // no SMP configured or a misconfiguration
393 
394  Must(!mapOwner);
395  mapOwner = TransientsMap::Init(MapLabel, entryLimit);
396 }
397 
399 {
400  delete mapOwner;
401 }
402 
bool freeEntry(const sfileno)
Definition: StoreMap.cc:263
Io io
current I/O state
Definition: MemObject.h:138
Locals * locals
Definition: Transients.h:107
#define assert(EX)
Definition: assert.h:17
struct Ipc::StoreMapAnchor::Basics basics
int readers(const StoreEntry &e) const
number of entry readers some time ago
Definition: Transients.cc:283
virtual void init() override
Definition: Transients.cc:41
bool hasTransients() const
whether there is a corresponding locked transients table entry
Definition: Store.h:187
double size
bytes currently in use
Definition: StoreStats.h:20
Definition: SBuf.h:86
virtual uint64_t maxSize() const override
Definition: Transients.cc:102
unsigned char cache_key
Store key.
Definition: forward.h:29
bool markedForDeletion(const cache_key *const)
Definition: StoreMap.cc:305
const Anchor & peekAtEntry(const sfileno fileno) const
only works on locked entries; returns the corresponding Anchor
Definition: StoreMap.cc:257
void switchWritingToReading(const sfileno fileno)
stop writing (or updating) the locked entry and start reading it
Definition: StoreMap.cc:169
#define PRId64
Definition: types.h:110
int entryCount() const
number of writeable and readable entries
Definition: StoreMap.cc:658
std::vector< StoreEntry * > Locals
Definition: Transients.h:104
YesNoNone memShared
whether the memory cache is shared among workers
Definition: SquidConfig.h:83
class Ping::pingStats_ stats
virtual void evictCached(StoreEntry &) override
Definition: Transients.cc:293
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
virtual bool dereference(StoreEntry &e) override
Definition: Transients.cc:136
bool abortedByWriter
whether the entry was aborted
Definition: Transients.h:33
StoreMapCleaner * cleaner
notified before a readable entry is freed
Definition: StoreMap.h:319
void clearCollapsingRequirement(const StoreEntry &e)
removes collapsing requirement (for future hits)
Definition: Transients.cc:192
static void Broadcast(const StoreEntry &e, const bool includingThisWorker=false)
notify other workers about changes in entry state (e.g., new data)
static Owner * Init(const SBuf &path, const int slotLimit)
initialize shared memory
Definition: StoreMap.cc:38
initializes shared memory segment used by Transients
Definition: Transients.cc:364
High-level store statistics used by mgr:info action. Used inside PODs!
Definition: StoreStats.h:13
StoreEntry * findCollapsed(const sfileno xitIndex)
return a local, previously collapsed entry
Definition: Transients.cc:176
RunnerRegistrationEntry(TransientsRr)
double count
number of cached objects
Definition: StoreStats.h:21
virtual void evictIfFound(const cache_key *) override
Definition: Transients.cc:307
A const & max(A const &lhs, A const &rhs)
virtual ~Transients()
Definition: Transients.cc:34
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:140
static const SBuf MapLabel("transients_map")
shared memory segment path to use for Transients map
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:137
bool isWriter(const StoreEntry &) const
whether the entry is in "writing to Transients" I/O state
Definition: Transients.cc:358
double capacity
the size limit
Definition: StoreStats.h:22
double doublePercent(const double, const double)
Definition: SquidMath.cc:25
Mem mem
all cache_dirs stats
Definition: StoreStats.h:49
virtual ~TransientsRr()
Definition: Transients.cc:398
int entryLimit() const
maximum entryCount() possible
Definition: StoreMap.cc:652
TransientsMap * map
shared packed info indexed by Store keys, for creating new StoreEntries
Definition: Transients.h:102
Ipc::StoreMap TransientsMap
Definition: Transients.h:20
virtual int64_t maxObjectSize() const override
the maximum size of a storable object; -1 if unlimited
Definition: Transients.cc:123
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
void startAppending(const sfileno fileno)
restrict opened for writing entry to appending operations; allow reads
Definition: StoreMap.cc:149
virtual void noteFreeMapSlice(const Ipc::StoreMapSliceId sliceId) override
adjust slice-linked state before a locked Readable slice is erased
Definition: Transients.cc:255
bool collapsed
whether the entry allows collapsing
Definition: Transients.h:35
virtual StoreEntry * get(const cache_key *) override
Definition: Transients.cc:143
#define EBIT_CLR(flag, bit)
Definition: defines.h:106
IoStatus
cache "I/O" direction and status
Definition: forward.h:40
const Anchor & readableEntry(const AnchorId anchorId) const
readable anchor for the entry created by openForReading()
Definition: StoreMap.cc:202
void disconnect(StoreEntry &)
the caller is done writing or reading the given entry
Definition: Transients.cc:318
int32_t StoreMapSliceId
Definition: StoreMap.h:24
static bool SmpAware()
whether there are any SMP-aware storages
Definition: Controller.cc:859
Anchor & writeableEntry(const AnchorId anchorId)
writeable anchor for the entry created by openForWriting()
Definition: StoreMap.cc:195
virtual void reference(StoreEntry &e) override
somebody needs this entry (many cache replacement policies need to know)
Definition: Transients.cc:130
void abortWriting(const sfileno fileno)
stop writing the entry, freeing its slot for others to use if possible
Definition: StoreMap.cc:209
void monitorIo(StoreEntry *, const cache_key *, const Store::IoStatus)
Definition: Transients.cc:206
int64_t shared_transient_entries_limit
Definition: SquidConfig.h:348
void completeWriting(const StoreEntry &e)
called when the in-transit entry has been successfully cached
Definition: Transients.cc:274
virtual void getStats(StoreInfoStats &stats) const override
collect statistics
Definition: Transients.cc:55
bool configured() const
Definition: YesNoNone.h:67
signed_int32_t sfileno
Definition: forward.h:22
MemObject * mem_obj
Definition: Store.h:197
virtual void create() override
create system resources needed for this store to operate in the future
Definition: Transients.h:65
bool UsingSmp()
Whether there should be more than one worker process running.
Definition: tools.cc:658
virtual uint64_t currentCount() const override
the total number of objects stored right now
Definition: Transients.cc:117
bool shared
whether memory cache is shared among workers
Definition: StoreStats.h:42
void setCollapsingRequirement(const bool required)
allow or forbid collapsed requests feeding
Definition: store.cc:2071
bool isReader(const StoreEntry &) const
whether the entry is in "reading from Transients" I/O state
Definition: Transients.cc:352
size_t PageSize()
returns page size in bytes; all pages are assumed to be the same size
Definition: Pages.cc:28
virtual uint64_t currentSize() const override
current size
Definition: Transients.cc:109
const Anchor * openForReading(const cache_key *const key, sfileno &fileno)
opens entry (identified by key) for reading, increments read level
Definition: StoreMap.cc:383
static int64_t EntryLimit()
calculates maximum number of entries we need to store and map
Definition: Transients.cc:338
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: Transients.cc:90
shared entry metadata, used for synchronization
Definition: Transients.h:30
virtual uint64_t minSize() const override
the minimum size the store will shrink to via normal housekeeping
Definition: Transients.cc:96
bool markedForDeletion(const cache_key *) const
Definition: Transients.cc:345
virtual void useConfig()
Definition: Segment.cc:376
aggregates anchor and slice owners for Init() caller convenience
Definition: StoreMap.h:210
size_t PageLevel()
approximate total number of shared memory pages used now
Definition: Pages.cc:80
sfileno fileNoByKey(const cache_key *const key) const
computes map entry anchor position for a given entry key
Definition: StoreMap.cc:730
virtual void stat(StoreEntry &e) const override
Definition: Transients.cc:70
void set(const StoreEntry &anEntry, const cache_key *aKey=nullptr)
store StoreEntry key and basics for an inode slot
Definition: StoreMap.cc:777
std::atomic< uint32_t > readers
number of reading users
Definition: ReadWriteLock.h:45
virtual void create()
called when the runner should create a new memory segment
Definition: Transients.cc:388
ReadWriteLock lock
protects slot data below
Definition: StoreMap.h:77
bool waitingToBeFreed
whether the entry was marked for deletion
Definition: Transients.h:34
Anchor * openForWriting(const cache_key *const key, sfileno &fileno)
Definition: StoreMap.cc:98
void status(const StoreEntry &e, EntryStatus &entryStatus) const
copies current shared entry metadata into entryStatus
Definition: Transients.cc:261
void addEntry(StoreEntry *, const cache_key *, const Store::IoStatus)
creates a new Transients entry
Definition: Transients.cc:225
void closeForReading(const sfileno fileno)
closes open entry after reading, decrements read level
Definition: StoreMap.cc:430
static bool Enabled()
Can we create and initialize Transients?
Definition: Transients.h:92
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:875
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
virtual void useConfig()
Definition: Transients.cc:381
size_t PageLimit()
the total number of shared memory pages that can be in use at any time
Definition: Pages.cc:55
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
void exportInto(StoreEntry &) const
load StoreEntry basics that were previously stored with set()
Definition: StoreMap.cc:797
void createMemObject()
Definition: store.cc:1609

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors