MemObject.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 19 Store Memory Primitives */
10 
11 #include "squid.h"
12 #include "comm/Connection.h"
13 #include "Generic.h"
14 #include "globals.h"
15 #include "HttpReply.h"
16 #include "MemBuf.h"
17 #include "MemObject.h"
18 #include "SquidConfig.h"
19 #include "Store.h"
20 #include "StoreClient.h"
21 
22 #if USE_DELAY_POOLS
23 #include "DelayPools.h"
24 #endif
25 
26 /* TODO: make this global or private */
27 #if URL_CHECKSUM_DEBUG
28 static unsigned int url_checksum(const char *url);
29 unsigned int
30 url_checksum(const char *url)
31 {
32  unsigned int ck;
33  SquidMD5_CTX M;
34  static unsigned char digest[16];
35  SquidMD5Init(&M);
36  SquidMD5Update(&M, (unsigned char *) url, strlen(url));
37  SquidMD5Final(digest, &M);
38  memcpy(&ck, digest, sizeof(ck));
39  return ck;
40 }
41 
42 #endif
43 
45 
46 size_t
48 {
49  return Pool().inUseCount();
50 }
51 
52 const char *
54 {
55  if (!storeId_.size()) {
56  debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value");
57  dump();
58  storeId_ = "[unknown_URI]";
59  }
60  return storeId_.termedBuf();
61 }
62 
63 const char *
65 {
66  return logUri_.size() ? logUri_.termedBuf() : storeId();
67 }
68 
69 bool
71 {
72  return storeId_.size();
73 }
74 
75 void
76 MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
77 {
78  if (hasUris())
79  return;
80 
81  storeId_ = aStoreId;
82  debugs(88, 3, this << " storeId: " << storeId_);
83 
84  // fast pointer comparison for a common storeCreateEntry(url,url,...) case
85  if (!aLogUri || aLogUri == aStoreId)
86  logUri_.clean(); // use storeId_ by default to minimize copying
87  else
88  logUri_ = aLogUri;
89 
90  method = aMethod;
91 
92 #if URL_CHECKSUM_DEBUG
93  chksum = url_checksum(urlXXX());
94 #endif
95 }
96 
98 {
99  debugs(20, 3, "MemObject constructed, this=" << this);
100  ping_reply_callback = nullptr;
101  memset(&start_ping, 0, sizeof(start_ping));
102  reply_ = new HttpReply;
103 }
104 
106 {
107  debugs(20, 3, "MemObject destructed, this=" << this);
108 
109 #if URL_CHECKSUM_DEBUG
110  checkUrlChecksum();
111 #endif
112 
113  if (!shutting_down) { // Store::Root() is FATALly missing during shutdown
114  assert(xitTable.index < 0);
115  assert(memCache.index < 0);
116  assert(swapout.sio == NULL);
117  }
118 
120 }
121 
122 HttpReply &
124 {
126  return *reply_;
127 }
128 
129 void
131 {
132  assert(r);
133  reply_ = r;
134  updatedReply_ = nullptr;
135 }
136 
137 void
138 MemObject::write(const StoreIOBuffer &writeBuffer)
139 {
140  debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
141 
142  /* We don't separate out mime headers yet, so ensure that the first
143  * write is at offset 0 - where they start
144  */
145  assert (data_hdr.endOffset() || writeBuffer.offset == 0);
146 
147  assert (data_hdr.write (writeBuffer));
148 }
149 
150 void
152 {
153  data_hdr.dump();
154 
155  debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping);
156  debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
157  debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
158  debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
159  debugs(20, DBG_IMPORTANT, "MemObject->reply: " << reply_);
160  debugs(20, DBG_IMPORTANT, "MemObject->updatedReply: " << updatedReply_);
161  debugs(20, DBG_IMPORTANT, "MemObject->appliedUpdates: " << appliedUpdates);
162  debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
163  debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
164  debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
165 }
166 
167 struct LowestMemReader : public unary_function<store_client, void> {
168  LowestMemReader(int64_t seed):current(seed) {}
169 
170  void operator() (store_client const &x) {
172  current = x.copyInto.offset;
173  }
174 
175  int64_t current;
176 };
177 
178 struct StoreClientStats : public unary_function<store_client, void> {
179  StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
180 
181  void operator()(store_client const &x) {
182  x.dumpStats(where, index);
183  ++index;
184  }
185 
187  size_t index;
188 };
189 
190 void
192 {
193  mb->appendf("\t" SQUIDSBUFPH " %s\n", SQUIDSBUFPRINT(method.image()), logUri());
194  if (!vary_headers.isEmpty())
195  mb->appendf("\tvary_headers: " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(vary_headers));
196  mb->appendf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
197  mb->appendf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
198  mb->appendf("\tswapout: %" PRId64 " bytes queued\n", swapout.queue_offset);
199 
200  if (swapout.sio.getRaw())
201  mb->appendf("\tswapout: %" PRId64 " bytes written\n", (int64_t) swapout.sio->offset());
202 
203  if (xitTable.index >= 0)
204  mb->appendf("\ttransient index: %d state: %d\n", xitTable.index, xitTable.io);
205  if (memCache.index >= 0)
206  mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset);
207  if (object_sz >= 0)
208  mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz);
209 
210  StoreClientStats statsVisitor(mb);
211 
212  for_each<StoreClientStats>(clients, statsVisitor);
213 }
214 
215 int64_t
217 {
218  return data_hdr.endOffset();
219 }
220 
221 void
223 {
224  const int hdr_sz = endOffset();
225  assert(hdr_sz >= 0);
226  assert(reply_);
227  reply_->hdr_sz = hdr_sz;
228 }
229 
230 int64_t
232 {
233  if (object_sz < 0)
234  return endOffset();
235 
236  return object_sz;
237 }
238 
239 int64_t
241 {
242  if (object_sz >= 0) {
243  debugs(20, 7, object_sz << " frozen by complete()");
244  return object_sz;
245  }
246 
247  const auto hdr_sz = baseReply().hdr_sz;
248 
249  // Cannot predict future length using an empty/unset or HTTP/0 reply.
250  // For any HTTP/1 reply, hdr_sz is positive -- status-line cannot be empty.
251  if (hdr_sz <= 0)
252  return -1;
253 
254  const auto clen = baseReply().bodySize(method);
255  if (clen < 0) {
256  debugs(20, 7, "unknown; hdr: " << hdr_sz);
257  return -1;
258  }
259 
260  const auto messageSize = clen + hdr_sz;
261  debugs(20, 7, messageSize << " hdr: " << hdr_sz << " clen: " << clen);
262  return messageSize;
263 }
264 
265 void
267 {
268  assert(swapout.sio == NULL);
270  inmem_lo = 0;
271  /* Should we check for clients? */
272  assert(reply_);
273  reply_->reset();
274  updatedReply_ = nullptr;
275  appliedUpdates = false;
276 }
277 
278 int64_t
280 {
281  LowestMemReader lowest (endOffset() + 1);
282 
283  for_each <LowestMemReader>(clients, lowest);
284 
285  return lowest.current;
286 }
287 
288 /* XXX: This is wrong. It breaks *badly* on range combining */
289 bool
291 {
292  const auto savedHttpHeaders = baseReply().hdr_sz;
293  const bool canRead = endOffset() - savedHttpHeaders <
295 
296  if (!canRead) {
297  debugs(19, 5, "no: " << endOffset() << '-' << savedHttpHeaders <<
298  " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
299  }
300 
301  return canRead;
302 }
303 
304 void
306 {
307  ++nclients;
308  dlinkAdd(aClient, &aClient->node, &clients);
309 }
310 
311 #if URL_CHECKSUM_DEBUG
312 void
313 MemObject::checkUrlChecksum () const
314 {
315  assert(chksum == url_checksum(urlXXX()));
316 }
317 
318 #endif
319 
320 /*
321  * How much of the object data is on the disk?
322  */
323 int64_t
325 {
326  /*
327  * NOTE: storeOffset() represents the disk file size,
328  * not the amount of object data on disk.
329  *
330  * If we don't have at least 'swap_hdr_sz' bytes
331  * then none of the object data is on disk.
332  *
333  * This should still be safe if swap_hdr_sz == 0,
334  * meaning we haven't even opened the swapout file
335  * yet.
336  */
337 
338  if (swapout.sio.getRaw() == NULL)
339  return 0;
340 
341  int64_t nwritten = swapout.sio->offset();
342 
343  if (nwritten <= (int64_t)swap_hdr_sz)
344  return 0;
345 
346  return (nwritten - swap_hdr_sz);
347 }
348 
349 int64_t
351 {
352  /*
353  * Careful. lowest_offset can be greater than endOffset(), such
354  * as in the case of a range request.
355  */
356  int64_t lowest_offset = lowestMemReaderOffset();
357 
358  if (endOffset() < lowest_offset ||
359  endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
360  (swap && !Config.onoff.memory_cache_first))
361  return lowest_offset;
362 
363  return inmem_lo;
364 }
365 
366 void
368 {
369  int64_t new_mem_lo = policyLowestOffsetToKeep(1);
370  /*
371  * We should only free up to what we know has been written
372  * to disk, not what has been queued for writing. Otherwise
373  * there will be a chunk of the data which is not in memory
374  * and is not yet on disk.
375  * The -1 makes sure the page isn't freed until storeSwapOut has
376  * walked to the next page.
377  */
378  int64_t on_disk;
379 
380  if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
381  new_mem_lo = on_disk - 1;
382 
383  if (new_mem_lo == -1)
384  new_mem_lo = 0; /* the above might become -1 */
385 
386  data_hdr.freeDataUpto(new_mem_lo);
387 
388  inmem_lo = new_mem_lo;
389 }
390 
391 void
393 {
394  if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
395  assert (new_mem_lo > 0);
396  data_hdr.freeDataUpto(new_mem_lo);
397  inmem_lo = new_mem_lo;
398  } // else we should not trim anything at this time
399 }
400 
401 bool
403 {
405  /* XXX : make this higher level */
406  debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
407  return result;
408 }
409 
410 int
411 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
412 {
413 #if USE_DELAY_POOLS
414  if (!ignoreDelayPools) {
415  /* identify delay id with largest allowance */
416  DelayId largestAllowance = mostBytesAllowed ();
417  return largestAllowance.bytesWanted(0, max);
418  }
419 #else
420  (void)ignoreDelayPools;
421 #endif
422 
423  return max;
424 }
425 
426 void
427 MemObject::setNoDelay(bool const newValue)
428 {
429 #if USE_DELAY_POOLS
430 
431  for (dlink_node *node = clients.head; node; node = node->next) {
432  store_client *sc = (store_client *) node->data;
433  sc->delayId.setNoDelay(newValue);
434  }
435 #else
436  (void)newValue;
437 #endif
438 }
439 
440 void
442 {
443 #if USE_DELAY_POOLS
444  if (readAheadPolicyCanRead()) {
445  if (DelayId mostAllowedId = mostBytesAllowed()) {
446  mostAllowedId.delayRead(aRead);
447  return;
448  }
449  }
450 #endif
451  deferredReads.delayRead(aRead);
452 }
453 
454 void
456 {
458 }
459 
460 #if USE_DELAY_POOLS
461 DelayId
463 {
464  int j;
465  int jmax = -1;
466  DelayId result;
467 
468  for (dlink_node *node = clients.head; node; node = node->next) {
469  store_client *sc = (store_client *) node->data;
470 
471  j = sc->delayId.bytesWanted(0, sc->copyInto.length);
472 
473  if (j > jmax) {
474  jmax = j;
475  result = sc->delayId;
476  }
477  }
478 
479  return result;
480 }
481 
482 #endif
483 
484 int64_t
486 {
487  return endOffset() - swapout.queue_offset;
488 }
489 
int64_t freeDataUpto(int64_t)
Definition: stmem.cc:81
int64_t queue_offset
number of bytes sent to SwapDir for writing
Definition: MemObject.h:151
int hdr_sz
Definition: Message.h:82
Definition: parse.c:104
HttpReply & adjustableBaseReply()
Definition: MemObject.cc:123
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:290
LowestMemReader(int64_t seed)
Definition: MemObject.cc:168
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
StoreClientStats(MemBuf *anEntry)
Definition: MemObject.cc:179
SQUIDCEXTERN void SquidMD5Init(struct SquidMD5Context *context)
Definition: md5.c:73
int inUseCount() const
void delayRead(DeferredRead const &)
Definition: comm.cc:1669
const char * storeId() const
Definition: MemObject.cc:53
String storeId_
StoreId for our entry (usually request URI)
Definition: MemObject.h:213
int memory_cache_first
Definition: SquidConfig.h:342
bool isEmpty() const
Definition: SBuf.h:424
mem_hdr data_hdr
Definition: MemObject.h:138
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:411
struct node * next
Definition: parse.c:105
void trimUnSwappable()
Definition: MemObject.cc:392
struct timeval start_ping
Definition: MemObject.h:190
int bytesWanted(int min, int max) const
Definition: DelayId.cc:130
DeferredReadManager deferredReads
Definition: MemObject.h:216
int64_t offset
bytes written/read to/from the memory cache so far
Definition: MemObject.h:182
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:375
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
int64_t offset
Definition: StoreIOBuffer.h:55
StoreIOState::Pointer sio
Definition: MemObject.h:152
void setNoDelay(bool const newValue)
Definition: MemObject.cc:427
Io io
current I/O state
Definition: MemObject.h:173
int64_t expectedReplySize() const
Definition: MemObject.cc:240
DelayId mostBytesAllowed() const
Definition: MemObject.cc:462
bool isContiguous() const
Definition: MemObject.cc:402
C * getRaw() const
Definition: RefCount.h:80
static Mem::AllocatorProxy & Pool()
Definition: MemObject.h:34
const SBuf & image() const
bool write(StoreIOBuffer const &)
Definition: stmem.cc:341
Io io
current I/O state
Definition: MemObject.h:184
int64_t endOffset() const
Definition: MemObject.cc:216
SQUIDCEXTERN void SquidMD5Final(uint8_t digest[16], struct SquidMD5Context *context)
int64_t availableForSwapOut() const
buffered bytes we have not swapped out yet
Definition: MemObject.cc:485
RemovalPolicy * mem_policy
Definition: MemObject.cc:44
#define DBG_IMPORTANT
Definition: Debug.h:41
int64_t inmem_lo
Definition: MemObject.h:139
A const & max(A const &lhs, A const &rhs)
void replaceBaseReply(const HttpReplyPointer &r)
Definition: MemObject.cc:130
int64_t endOffset() const
Definition: stmem.cc:45
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:64
Definition: Range.h:19
void operator()(store_client const &x)
Definition: MemObject.cc:170
HttpReplyPointer reply_
Definition: MemObject.h:210
#define NULL
Definition: types.h:166
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
SBuf vary_headers
Definition: MemObject.h:204
const HttpReply & baseReply() const
Definition: MemObject.h:59
bool memReaderHasLowerOffset(int64_t) const
Definition: store_client.cc:89
Definition: MemBuf.h:24
int64_t policyLowestOffsetToKeep(bool swap) const
Definition: MemObject.cc:350
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
int64_t current
Definition: MemObject.cc:175
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:138
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
void kickReads(int const count)
Definition: comm.cc:1731
off_t offset() const
Definition: StoreIOState.h:60
HttpRequestMethod method
Definition: MemObject.h:137
#define assert(EX)
Definition: assert.h:19
char const * termedBuf() const
Definition: SquidString.h:92
void dump() const
Definition: MemObject.cc:151
void dump() const
Definition: stmem.cc:397
int64_t size() const
Definition: MemObject.cc:231
static size_t inUseCount()
Definition: MemObject.cc:47
static int sc[16]
Definition: smbdes.c:121
void freeContent()
Definition: stmem.cc:59
size_t maxInMemObjSize
Definition: SquidConfig.h:273
MemBuf * where
Definition: MemObject.cc:186
String logUri_
URI used for logging (usually request URI)
Definition: MemObject.h:214
StoreIOBuffer copyInto
Definition: StoreClient.h:94
bool appliedUpdates
Definition: MemObject.h:89
void markEndOfReplyHeaders()
sets baseReply().hdr_sz (i.e. written reply headers size) to endOffset()
Definition: MemObject.cc:222
bool hasContigousContentRange(Range< int64_t > const &range) const
Definition: stmem.cc:283
int32_t index
entry position inside the memory cache
Definition: MemObject.h:181
dlink_node node
Definition: StoreClient.h:92
struct SquidConfig::@111 onoff
void delayRead(DeferredRead const &)
Definition: MemObject.cc:441
struct SquidConfig::@109 Store
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:70
void dumpStats(MemBuf *output, int clientNumber) const
size_type size() const
Definition: SquidString.h:73
int64_t object_sz
Definition: MemObject.h:198
void trimSwappable()
Definition: MemObject.cc:367
void addClient(store_client *)
Definition: MemObject.cc:305
HttpReplyPointer updatedReply_
Definition: MemObject.h:211
#define PRId64
Definition: types.h:110
SQUIDCEXTERN void SquidMD5Update(struct SquidMD5Context *context, const void *buf, unsigned len)
Definition: md5.c:89
int shutting_down
void kickReads()
Definition: MemObject.cc:455
int64_t readAheadGap
Definition: SquidConfig.h:96
virtual void reset()
Definition: HttpReply.cc:58
int64_t lowestMemReaderOffset() const
Definition: MemObject.cc:279
const char * urlXXX() const
Definition: MemObject.h:128
IRCB * ping_reply_callback
Definition: MemObject.h:191
void reset()
Definition: MemObject.cc:266
int64_t objectBytesOnDisk() const
Definition: MemObject.cc:324
void stat(MemBuf *mb) const
Definition: MemObject.cc:191
HttpRequestPointer request
Definition: MemObject.h:188
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:76
size_t swap_hdr_sz
Definition: MemObject.h:199
int nclients
Definition: MemObject.h:146
void operator()(store_client const &x)
Definition: MemObject.cc:181
#define SQUIDSBUFPH
Definition: SBuf.h:31
class SquidConfig Config
Definition: SquidConfig.cc:12
dlink_list clients
Definition: MemObject.h:140
void clean()
Definition: String.cc:120
SwapOut swapout
Definition: MemObject.h:159

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors