MemObject.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 19 Store Memory Primitives */
10 
11 #include "squid.h"
12 #include "comm/Connection.h"
13 #include "Generic.h"
14 #include "globals.h"
15 #include "HttpReply.h"
16 #include "MemBuf.h"
17 #include "MemObject.h"
18 #include "profiler/Profiler.h"
19 #include "SquidConfig.h"
20 #include "Store.h"
21 #include "StoreClient.h"
22 
23 #if USE_DELAY_POOLS
24 #include "DelayPools.h"
25 #endif
26 
27 /* TODO: make this global or private */
28 #if URL_CHECKSUM_DEBUG
29 static unsigned int url_checksum(const char *url);
30 unsigned int
31 url_checksum(const char *url)
32 {
33  unsigned int ck;
34  SquidMD5_CTX M;
35  static unsigned char digest[16];
36  SquidMD5Init(&M);
37  SquidMD5Update(&M, (unsigned char *) url, strlen(url));
38  SquidMD5Final(digest, &M);
39  memcpy(&ck, digest, sizeof(ck));
40  return ck;
41 }
42 
43 #endif
44 
46 
47 size_t
49 {
50  return Pool().inUseCount();
51 }
52 
53 const char *
55 {
56  if (!storeId_.size()) {
57  debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value");
58  dump();
59  storeId_ = "[unknown_URI]";
60  }
61  return storeId_.termedBuf();
62 }
63 
64 const char *
66 {
67  return logUri_.size() ? logUri_.termedBuf() : storeId();
68 }
69 
70 bool
72 {
73  return storeId_.size();
74 }
75 
76 void
77 MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
78 {
79  if (hasUris())
80  return;
81 
82  storeId_ = aStoreId;
83  debugs(88, 3, this << " storeId: " << storeId_);
84 
85  // fast pointer comparison for a common storeCreateEntry(url,url,...) case
86  if (!aLogUri || aLogUri == aStoreId)
87  logUri_.clean(); // use storeId_ by default to minimize copying
88  else
89  logUri_ = aLogUri;
90 
91  method = aMethod;
92 
93 #if URL_CHECKSUM_DEBUG
94  chksum = url_checksum(urlXXX());
95 #endif
96 }
97 
99 {
100  debugs(20, 3, "MemObject constructed, this=" << this);
101  ping_reply_callback = nullptr;
102  memset(&start_ping, 0, sizeof(start_ping));
103  reply_ = new HttpReply;
104 }
105 
107 {
108  debugs(20, 3, "MemObject destructed, this=" << this);
109  const Ctx ctx = ctx_enter(hasUris() ? urlXXX() : "[unknown_ctx]");
110 
111 #if URL_CHECKSUM_DEBUG
112  checkUrlChecksum();
113 #endif
114 
115  if (!shutting_down) { // Store::Root() is FATALly missing during shutdown
116  assert(xitTable.index < 0);
117  assert(memCache.index < 0);
118  assert(swapout.sio == NULL);
119  }
120 
122 
123  ctx_exit(ctx); /* must exit before we free mem->url */
124 }
125 
126 HttpReply &
128 {
130  return *reply_;
131 }
132 
133 void
135 {
136  assert(r);
137  reply_ = r;
138  updatedReply_ = nullptr;
139 }
140 
141 void
142 MemObject::write(const StoreIOBuffer &writeBuffer)
143 {
144  PROF_start(MemObject_write);
145  debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
146 
147  /* We don't separate out mime headers yet, so ensure that the first
148  * write is at offset 0 - where they start
149  */
150  assert (data_hdr.endOffset() || writeBuffer.offset == 0);
151 
152  assert (data_hdr.write (writeBuffer));
153  PROF_stop(MemObject_write);
154 }
155 
156 void
158 {
159  data_hdr.dump();
160 
161  debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping);
162  debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
163  debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
164  debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
165  debugs(20, DBG_IMPORTANT, "MemObject->reply: " << reply_);
166  debugs(20, DBG_IMPORTANT, "MemObject->updatedReply: " << updatedReply_);
167  debugs(20, DBG_IMPORTANT, "MemObject->appliedUpdates: " << appliedUpdates);
168  debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
169  debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
170  debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
171 }
172 
173 struct LowestMemReader : public unary_function<store_client, void> {
174  LowestMemReader(int64_t seed):current(seed) {}
175 
176  void operator() (store_client const &x) {
178  current = x.copyInto.offset;
179  }
180 
181  int64_t current;
182 };
183 
184 struct StoreClientStats : public unary_function<store_client, void> {
185  StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
186 
187  void operator()(store_client const &x) {
188  x.dumpStats(where, index);
189  ++index;
190  }
191 
193  size_t index;
194 };
195 
196 void
198 {
199  mb->appendf("\t" SQUIDSBUFPH " %s\n", SQUIDSBUFPRINT(method.image()), logUri());
200  if (!vary_headers.isEmpty())
201  mb->appendf("\tvary_headers: " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(vary_headers));
202  mb->appendf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
203  mb->appendf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
204  mb->appendf("\tswapout: %" PRId64 " bytes queued\n", swapout.queue_offset);
205 
206  if (swapout.sio.getRaw())
207  mb->appendf("\tswapout: %" PRId64 " bytes written\n", (int64_t) swapout.sio->offset());
208 
209  if (xitTable.index >= 0)
210  mb->appendf("\ttransient index: %d state: %d\n", xitTable.index, xitTable.io);
211  if (memCache.index >= 0)
212  mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset);
213  if (object_sz >= 0)
214  mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz);
215 
216  StoreClientStats statsVisitor(mb);
217 
218  for_each<StoreClientStats>(clients, statsVisitor);
219 }
220 
221 int64_t
223 {
224  return data_hdr.endOffset();
225 }
226 
227 void
229 {
230  const int hdr_sz = endOffset();
231  assert(hdr_sz >= 0);
232  assert(reply_);
233  reply_->hdr_sz = hdr_sz;
234 }
235 
236 int64_t
238 {
239  if (object_sz < 0)
240  return endOffset();
241 
242  return object_sz;
243 }
244 
245 int64_t
247 {
248  if (object_sz >= 0) {
249  debugs(20, 7, object_sz << " frozen by complete()");
250  return object_sz;
251  }
252 
253  const auto hdr_sz = baseReply().hdr_sz;
254 
255  // Cannot predict future length using an empty/unset or HTTP/0 reply.
256  // For any HTTP/1 reply, hdr_sz is positive -- status-line cannot be empty.
257  if (hdr_sz <= 0)
258  return -1;
259 
260  const auto clen = baseReply().bodySize(method);
261  if (clen < 0) {
262  debugs(20, 7, "unknown; hdr: " << hdr_sz);
263  return -1;
264  }
265 
266  const auto messageSize = clen + hdr_sz;
267  debugs(20, 7, messageSize << " hdr: " << hdr_sz << " clen: " << clen);
268  return messageSize;
269 }
270 
271 void
273 {
274  assert(swapout.sio == NULL);
276  inmem_lo = 0;
277  /* Should we check for clients? */
278  assert(reply_);
279  reply_->reset();
280  updatedReply_ = nullptr;
281  appliedUpdates = false;
282 }
283 
284 int64_t
286 {
287  LowestMemReader lowest (endOffset() + 1);
288 
289  for_each <LowestMemReader>(clients, lowest);
290 
291  return lowest.current;
292 }
293 
294 /* XXX: This is wrong. It breaks *badly* on range combining */
295 bool
297 {
298  const auto savedHttpHeaders = baseReply().hdr_sz;
299  const bool canRead = endOffset() - savedHttpHeaders <
301 
302  if (!canRead) {
303  debugs(19, 5, "no: " << endOffset() << '-' << savedHttpHeaders <<
304  " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
305  }
306 
307  return canRead;
308 }
309 
310 void
312 {
313  ++nclients;
314  dlinkAdd(aClient, &aClient->node, &clients);
315 }
316 
317 #if URL_CHECKSUM_DEBUG
318 void
319 MemObject::checkUrlChecksum () const
320 {
321  assert(chksum == url_checksum(urlXXX()));
322 }
323 
324 #endif
325 
326 /*
327  * How much of the object data is on the disk?
328  */
329 int64_t
331 {
332  /*
333  * NOTE: storeOffset() represents the disk file size,
334  * not the amount of object data on disk.
335  *
336  * If we don't have at least 'swap_hdr_sz' bytes
337  * then none of the object data is on disk.
338  *
339  * This should still be safe if swap_hdr_sz == 0,
340  * meaning we haven't even opened the swapout file
341  * yet.
342  */
343 
344  if (swapout.sio.getRaw() == NULL)
345  return 0;
346 
347  int64_t nwritten = swapout.sio->offset();
348 
349  if (nwritten <= (int64_t)swap_hdr_sz)
350  return 0;
351 
352  return (nwritten - swap_hdr_sz);
353 }
354 
355 int64_t
357 {
358  /*
359  * Careful. lowest_offset can be greater than endOffset(), such
360  * as in the case of a range request.
361  */
362  int64_t lowest_offset = lowestMemReaderOffset();
363 
364  if (endOffset() < lowest_offset ||
365  endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
366  (swap && !Config.onoff.memory_cache_first))
367  return lowest_offset;
368 
369  return inmem_lo;
370 }
371 
372 void
374 {
375  int64_t new_mem_lo = policyLowestOffsetToKeep(1);
376  /*
377  * We should only free up to what we know has been written
378  * to disk, not what has been queued for writing. Otherwise
379  * there will be a chunk of the data which is not in memory
380  * and is not yet on disk.
381  * The -1 makes sure the page isn't freed until storeSwapOut has
382  * walked to the next page.
383  */
384  int64_t on_disk;
385 
386  if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
387  new_mem_lo = on_disk - 1;
388 
389  if (new_mem_lo == -1)
390  new_mem_lo = 0; /* the above might become -1 */
391 
392  data_hdr.freeDataUpto(new_mem_lo);
393 
394  inmem_lo = new_mem_lo;
395 }
396 
397 void
399 {
400  if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
401  assert (new_mem_lo > 0);
402  data_hdr.freeDataUpto(new_mem_lo);
403  inmem_lo = new_mem_lo;
404  } // else we should not trim anything at this time
405 }
406 
407 bool
409 {
411  /* XXX : make this higher level */
412  debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
413  return result;
414 }
415 
416 int
417 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
418 {
419 #if USE_DELAY_POOLS
420  if (!ignoreDelayPools) {
421  /* identify delay id with largest allowance */
422  DelayId largestAllowance = mostBytesAllowed ();
423  return largestAllowance.bytesWanted(0, max);
424  }
425 #endif
426 
427  return max;
428 }
429 
430 void
431 MemObject::setNoDelay(bool const newValue)
432 {
433 #if USE_DELAY_POOLS
434 
435  for (dlink_node *node = clients.head; node; node = node->next) {
436  store_client *sc = (store_client *) node->data;
437  sc->delayId.setNoDelay(newValue);
438  }
439 
440 #endif
441 }
442 
443 void
445 {
446 #if USE_DELAY_POOLS
447  if (readAheadPolicyCanRead()) {
448  if (DelayId mostAllowedId = mostBytesAllowed()) {
449  mostAllowedId.delayRead(aRead);
450  return;
451  }
452  }
453 #endif
454  deferredReads.delayRead(aRead);
455 }
456 
457 void
459 {
461 }
462 
463 #if USE_DELAY_POOLS
464 DelayId
466 {
467  int j;
468  int jmax = -1;
469  DelayId result;
470 
471  for (dlink_node *node = clients.head; node; node = node->next) {
472  store_client *sc = (store_client *) node->data;
473 
474  j = sc->delayId.bytesWanted(0, sc->copyInto.length);
475 
476  if (j > jmax) {
477  jmax = j;
478  result = sc->delayId;
479  }
480  }
481 
482  return result;
483 }
484 
485 #endif
486 
487 int64_t
489 {
490  return endOffset() - swapout.queue_offset;
491 }
492 
int64_t freeDataUpto(int64_t)
Definition: stmem.cc:82
int64_t queue_offset
number of bytes sent to SwapDir for writing
Definition: MemObject.h:151
int hdr_sz
Definition: Message.h:82
Definition: parse.c:104
HttpReply & adjustableBaseReply()
Definition: MemObject.cc:127
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:296
LowestMemReader(int64_t seed)
Definition: MemObject.cc:174
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
StoreClientStats(MemBuf *anEntry)
Definition: MemObject.cc:185
SQUIDCEXTERN void SquidMD5Init(struct SquidMD5Context *context)
Definition: md5.c:73
int inUseCount() const
void delayRead(DeferredRead const &)
Definition: comm.cc:1713
const char * storeId() const
Definition: MemObject.cc:54
String storeId_
StoreId for our entry (usually request URI)
Definition: MemObject.h:213
int Ctx
Definition: Debug.h:37
int memory_cache_first
Definition: SquidConfig.h:342
bool isEmpty() const
Definition: SBuf.h:424
mem_hdr data_hdr
Definition: MemObject.h:138
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:417
struct node * next
Definition: parse.c:105
void trimUnSwappable()
Definition: MemObject.cc:398
struct timeval start_ping
Definition: MemObject.h:190
int bytesWanted(int min, int max) const
Definition: DelayId.cc:130
DeferredReadManager deferredReads
Definition: MemObject.h:216
int64_t offset
bytes written/read to/from the memory cache so far
Definition: MemObject.h:182
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:375
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
int64_t offset
Definition: StoreIOBuffer.h:55
StoreIOState::Pointer sio
Definition: MemObject.h:152
void setNoDelay(bool const newValue)
Definition: MemObject.cc:431
Io io
current I/O state
Definition: MemObject.h:173
int64_t expectedReplySize() const
Definition: MemObject.cc:246
DelayId mostBytesAllowed() const
Definition: MemObject.cc:465
bool isContiguous() const
Definition: MemObject.cc:408
C * getRaw() const
Definition: RefCount.h:80
static Mem::AllocatorProxy & Pool()
Definition: MemObject.h:34
const SBuf & image() const
bool write(StoreIOBuffer const &)
Definition: stmem.cc:342
Io io
current I/O state
Definition: MemObject.h:184
int64_t endOffset() const
Definition: MemObject.cc:222
SQUIDCEXTERN void SquidMD5Final(uint8_t digest[16], struct SquidMD5Context *context)
int64_t availableForSwapOut() const
buffered bytes we have not swapped out yet
Definition: MemObject.cc:488
RemovalPolicy * mem_policy
Definition: MemObject.cc:45
#define PROF_stop(probename)
Definition: Profiler.h:63
#define DBG_IMPORTANT
Definition: Debug.h:46
int64_t inmem_lo
Definition: MemObject.h:139
A const & max(A const &lhs, A const &rhs)
void replaceBaseReply(const HttpReplyPointer &r)
Definition: MemObject.cc:134
int64_t endOffset() const
Definition: stmem.cc:46
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:65
Definition: Range.h:19
void operator()(store_client const &x)
Definition: MemObject.cc:176
HttpReplyPointer reply_
Definition: MemObject.h:210
struct SquidConfig::@113 onoff
struct SquidConfig::@111 Store
#define NULL
Definition: types.h:166
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:128
SBuf vary_headers
Definition: MemObject.h:204
const HttpReply & baseReply() const
Definition: MemObject.h:59
bool memReaderHasLowerOffset(int64_t) const
Definition: store_client.cc:90
Definition: MemBuf.h:24
int64_t policyLowestOffsetToKeep(bool swap) const
Definition: MemObject.cc:356
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
int64_t current
Definition: MemObject.cc:181
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:142
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
void kickReads(int const count)
Definition: comm.cc:1771
off_t offset() const
Definition: StoreIOState.h:60
HttpRequestMethod method
Definition: MemObject.h:137
#define assert(EX)
Definition: assert.h:19
char const * termedBuf() const
Definition: SquidString.h:91
void dump() const
Definition: MemObject.cc:157
void dump() const
Definition: stmem.cc:401
int64_t size() const
Definition: MemObject.cc:237
static size_t inUseCount()
Definition: MemObject.cc:48
static int sc[16]
Definition: smbdes.c:121
void freeContent()
Definition: stmem.cc:60
size_t maxInMemObjSize
Definition: SquidConfig.h:272
MemBuf * where
Definition: MemObject.cc:192
String logUri_
URI used for logging (usually request URI)
Definition: MemObject.h:214
Ctx ctx_enter(const char *descr)
Definition: debug.cc:721
StoreIOBuffer copyInto
Definition: StoreClient.h:94
bool appliedUpdates
Definition: MemObject.h:89
void markEndOfReplyHeaders()
sets baseReply().hdr_sz (i.e. written reply headers size) to endOffset()
Definition: MemObject.cc:228
bool hasContigousContentRange(Range< int64_t > const &range) const
Definition: stmem.cc:284
int32_t index
entry position inside the memory cache
Definition: MemObject.h:181
dlink_node node
Definition: StoreClient.h:92
void delayRead(DeferredRead const &)
Definition: MemObject.cc:444
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:71
void dumpStats(MemBuf *output, int clientNumber) const
size_type size() const
Definition: SquidString.h:72
int64_t object_sz
Definition: MemObject.h:198
void trimSwappable()
Definition: MemObject.cc:373
void addClient(store_client *)
Definition: MemObject.cc:311
HttpReplyPointer updatedReply_
Definition: MemObject.h:211
#define PRId64
Definition: types.h:110
SQUIDCEXTERN void SquidMD5Update(struct SquidMD5Context *context, const void *buf, unsigned len)
Definition: md5.c:89
int shutting_down
void kickReads()
Definition: MemObject.cc:458
int64_t readAheadGap
Definition: SquidConfig.h:95
virtual void reset()
Definition: HttpReply.cc:58
int64_t lowestMemReaderOffset() const
Definition: MemObject.cc:285
const char * urlXXX() const
Definition: MemObject.h:128
IRCB * ping_reply_callback
Definition: MemObject.h:191
void reset()
Definition: MemObject.cc:272
int64_t objectBytesOnDisk() const
Definition: MemObject.cc:330
void stat(MemBuf *mb) const
Definition: MemObject.cc:197
HttpRequestPointer request
Definition: MemObject.h:188
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:77
size_t swap_hdr_sz
Definition: MemObject.h:199
int nclients
Definition: MemObject.h:146
void operator()(store_client const &x)
Definition: MemObject.cc:187
void ctx_exit(Ctx ctx)
Definition: debug.cc:737
#define SQUIDSBUFPH
Definition: SBuf.h:31
#define PROF_start(probename)
Definition: Profiler.h:62
class SquidConfig Config
Definition: SquidConfig.cc:12
dlink_list clients
Definition: MemObject.h:140
void clean()
Definition: String.cc:125
SwapOut swapout
Definition: MemObject.h:159

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors