MemObject.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 19 Store Memory Primitives */
10 
11 #include "squid.h"
12 #include "comm/Connection.h"
13 #include "Generic.h"
14 #include "globals.h"
15 #include "HttpReply.h"
16 #include "MemBuf.h"
17 #include "MemObject.h"
18 #include "profiler/Profiler.h"
19 #include "SquidConfig.h"
20 #include "Store.h"
21 #include "StoreClient.h"
22 
23 #if USE_DELAY_POOLS
24 #include "DelayPools.h"
25 #endif
26 
27 /* TODO: make this global or private */
28 #if URL_CHECKSUM_DEBUG
29 static unsigned int url_checksum(const char *url);
30 unsigned int
31 url_checksum(const char *url)
32 {
33  unsigned int ck;
34  SquidMD5_CTX M;
35  static unsigned char digest[16];
36  SquidMD5Init(&M);
37  SquidMD5Update(&M, (unsigned char *) url, strlen(url));
38  SquidMD5Final(digest, &M);
39  memcpy(&ck, digest, sizeof(ck));
40  return ck;
41 }
42 
43 #endif
44 
45 RemovalPolicy * mem_policy = NULL;
46 
47 size_t
49 {
50  return Pool().inUseCount();
51 }
52 
53 const char *
55 {
56  if (!storeId_.size()) {
57  debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value");
58  dump();
59  storeId_ = "[unknown_URI]";
60  }
61  return storeId_.termedBuf();
62 }
63 
64 const char *
66 {
67  return logUri_.size() ? logUri_.termedBuf() : storeId();
68 }
69 
70 bool
72 {
73  return storeId_.size();
74 }
75 
76 void
77 MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
78 {
79  if (hasUris())
80  return;
81 
82  storeId_ = aStoreId;
83  debugs(88, 3, this << " storeId: " << storeId_);
84 
85  // fast pointer comparison for a common storeCreateEntry(url,url,...) case
86  if (!aLogUri || aLogUri == aStoreId)
87  logUri_.clean(); // use storeId_ by default to minimize copying
88  else
89  logUri_ = aLogUri;
90 
91  method = aMethod;
92 
93 #if URL_CHECKSUM_DEBUG
94  chksum = url_checksum(urlXXX());
95 #endif
96 }
97 
99  inmem_lo(0),
100  nclients(0),
101  ping_reply_callback(nullptr),
102  ircb_data(nullptr),
103  id(0),
104  object_sz(-1),
105  swap_hdr_sz(0),
106 #if URL_CHECKSUM_DEBUG
107  chksum(0),
108 #endif
109  vary_headers(nullptr)
110 {
111  debugs(20, 3, "new MemObject " << this);
112  memset(&start_ping, 0, sizeof(start_ping));
113  memset(&abort, 0, sizeof(abort));
114  reply_ = new HttpReply;
115 }
116 
118 {
119  debugs(20, 3, "del MemObject " << this);
120  const Ctx ctx = ctx_enter(hasUris() ? urlXXX() : "[unknown_ctx]");
121 
122 #if URL_CHECKSUM_DEBUG
123  checkUrlChecksum();
124 #endif
125 
126  if (!shutting_down) { // Store::Root() is FATALly missing during shutdown
127  assert(xitTable.index < 0);
128  assert(memCache.index < 0);
129  assert(swapout.sio == NULL);
130  }
131 
133 
134 #if 0
135  /*
136  * There is no way to abort FD-less clients, so they might
137  * still have mem->clients set.
138  */
139  assert(clients.head == NULL);
140 
141 #endif
142 
143  ctx_exit(ctx); /* must exit before we free mem->url */
144 }
145 
146 void
147 MemObject::write(const StoreIOBuffer &writeBuffer)
148 {
149  PROF_start(MemObject_write);
150  debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
151 
152  /* We don't separate out mime headers yet, so ensure that the first
153  * write is at offset 0 - where they start
154  */
155  assert (data_hdr.endOffset() || writeBuffer.offset == 0);
156 
157  assert (data_hdr.write (writeBuffer));
158  PROF_stop(MemObject_write);
159 }
160 
161 void
163 {
164  data_hdr.dump();
165 #if 0
166  /* do we want this one? */
167  debugs(20, DBG_IMPORTANT, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0));
168 #endif
169 
170  debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping.tv_sec << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec);
171  debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
172  debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
173  debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
174  debugs(20, DBG_IMPORTANT, "MemObject->reply: " << reply_);
175  debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
176  debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
177  debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
178 }
179 
180 struct LowestMemReader : public unary_function<store_client, void> {
181  LowestMemReader(int64_t seed):current(seed) {}
182 
183  void operator() (store_client const &x) {
185  current = x.copyInto.offset;
186  }
187 
188  int64_t current;
189 };
190 
191 struct StoreClientStats : public unary_function<store_client, void> {
192  StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
193 
194  void operator()(store_client const &x) {
195  x.dumpStats(where, index);
196  ++index;
197  }
198 
200  size_t index;
201 };
202 
203 void
205 {
206  mb->appendf("\t" SQUIDSBUFPH " %s\n", SQUIDSBUFPRINT(method.image()), logUri());
207  if (!vary_headers.isEmpty())
208  mb->appendf("\tvary_headers: " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(vary_headers));
209  mb->appendf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
210  mb->appendf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
211  mb->appendf("\tswapout: %" PRId64 " bytes queued\n", swapout.queue_offset);
212 
213  if (swapout.sio.getRaw())
214  mb->appendf("\tswapout: %" PRId64 " bytes written\n", (int64_t) swapout.sio->offset());
215 
216  if (xitTable.index >= 0)
217  mb->appendf("\ttransient index: %d state: %d\n", xitTable.index, xitTable.io);
218  if (memCache.index >= 0)
219  mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset);
220  if (object_sz >= 0)
221  mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz);
222 
223  StoreClientStats statsVisitor(mb);
224 
225  for_each<StoreClientStats>(clients, statsVisitor);
226 }
227 
228 int64_t
230 {
231  return data_hdr.endOffset();
232 }
233 
234 void
236 {
237  const int hdr_sz = endOffset();
238  assert(hdr_sz >= 0);
239  assert(reply_);
240  reply_->hdr_sz = hdr_sz;
241 }
242 
243 int64_t
245 {
246  if (object_sz < 0)
247  return endOffset();
248 
249  return object_sz;
250 }
251 
252 int64_t
254 {
255  debugs(20, 7, "object_sz: " << object_sz);
256  if (object_sz >= 0) // complete() has been called; we know the exact answer
257  return object_sz;
258 
259  if (reply_) {
260  const int64_t clen = reply_->bodySize(method);
261  debugs(20, 7, "clen: " << clen);
262  if (clen >= 0 && reply_->hdr_sz > 0) // yuck: Http::Message sets hdr_sz to 0
263  return clen + reply_->hdr_sz;
264  }
265 
266  return -1; // not enough information to predict
267 }
268 
269 void
271 {
272  assert(swapout.sio == NULL);
274  inmem_lo = 0;
275  /* Should we check for clients? */
276  if (reply_)
277  reply_->reset();
278 }
279 
280 int64_t
282 {
283  LowestMemReader lowest (endOffset() + 1);
284 
285  for_each <LowestMemReader>(clients, lowest);
286 
287  return lowest.current;
288 }
289 
290 /* XXX: This is wrong. It breaks *badly* on range combining */
291 bool
293 {
294  const bool canRead = endOffset() - getReply()->hdr_sz <
296 
297  if (!canRead) {
298  debugs(19, 9, "no: " << endOffset() << '-' << getReply()->hdr_sz <<
299  " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
300  }
301 
302  return canRead;
303 }
304 
305 void
307 {
308  ++nclients;
309  dlinkAdd(aClient, &aClient->node, &clients);
310 }
311 
312 #if URL_CHECKSUM_DEBUG
313 void
314 MemObject::checkUrlChecksum () const
315 {
316  assert(chksum == url_checksum(urlXXX()));
317 }
318 
319 #endif
320 
321 /*
322  * How much of the object data is on the disk?
323  */
324 int64_t
326 {
327  /*
328  * NOTE: storeOffset() represents the disk file size,
329  * not the amount of object data on disk.
330  *
331  * If we don't have at least 'swap_hdr_sz' bytes
332  * then none of the object data is on disk.
333  *
334  * This should still be safe if swap_hdr_sz == 0,
335  * meaning we haven't even opened the swapout file
336  * yet.
337  */
338 
339  if (swapout.sio.getRaw() == NULL)
340  return 0;
341 
342  int64_t nwritten = swapout.sio->offset();
343 
344  if (nwritten <= (int64_t)swap_hdr_sz)
345  return 0;
346 
347  return (nwritten - swap_hdr_sz);
348 }
349 
350 int64_t
352 {
353  /*
354  * Careful. lowest_offset can be greater than endOffset(), such
355  * as in the case of a range request.
356  */
357  int64_t lowest_offset = lowestMemReaderOffset();
358 
359  if (endOffset() < lowest_offset ||
360  endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
361  (swap && !Config.onoff.memory_cache_first))
362  return lowest_offset;
363 
364  return inmem_lo;
365 }
366 
367 void
369 {
370  int64_t new_mem_lo = policyLowestOffsetToKeep(1);
371  /*
372  * We should only free up to what we know has been written
373  * to disk, not what has been queued for writing. Otherwise
374  * there will be a chunk of the data which is not in memory
375  * and is not yet on disk.
376  * The -1 makes sure the page isn't freed until storeSwapOut has
377  * walked to the next page.
378  */
379  int64_t on_disk;
380 
381  if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
382  new_mem_lo = on_disk - 1;
383 
384  if (new_mem_lo == -1)
385  new_mem_lo = 0; /* the above might become -1 */
386 
387  data_hdr.freeDataUpto(new_mem_lo);
388 
389  inmem_lo = new_mem_lo;
390 }
391 
392 void
394 {
395  if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
396  assert (new_mem_lo > 0);
397  data_hdr.freeDataUpto(new_mem_lo);
398  inmem_lo = new_mem_lo;
399  } // else we should not trim anything at this time
400 }
401 
402 bool
404 {
406  /* XXX : make this higher level */
407  debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
408  return result;
409 }
410 
411 int
412 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
413 {
414 #if USE_DELAY_POOLS
415  if (!ignoreDelayPools) {
416  /* identify delay id with largest allowance */
417  DelayId largestAllowance = mostBytesAllowed ();
418  return largestAllowance.bytesWanted(0, max);
419  }
420 #endif
421 
422  return max;
423 }
424 
425 void
426 MemObject::setNoDelay(bool const newValue)
427 {
428 #if USE_DELAY_POOLS
429 
430  for (dlink_node *node = clients.head; node; node = node->next) {
431  store_client *sc = (store_client *) node->data;
432  sc->delayId.setNoDelay(newValue);
433  }
434 
435 #endif
436 }
437 
438 void
440 {
441 #if USE_DELAY_POOLS
442  if (readAheadPolicyCanRead()) {
443  if (DelayId mostAllowedId = mostBytesAllowed()) {
444  mostAllowedId.delayRead(aRead);
445  return;
446  }
447  }
448 #endif
449  deferredReads.delayRead(aRead);
450 }
451 
452 void
454 {
456 }
457 
458 #if USE_DELAY_POOLS
459 DelayId
461 {
462  int j;
463  int jmax = -1;
464  DelayId result;
465 
466  for (dlink_node *node = clients.head; node; node = node->next) {
467  store_client *sc = (store_client *) node->data;
468 #if 0
469  /* This test is invalid because the client may be writing data
470  * and thus will want data immediately.
471  * If we include the test, there is a race condition when too much
472  * data is read - if all sc's are writing when a read is scheduled.
473  * XXX: fixme.
474  */
475 
476  if (!sc->callbackPending())
477  /* not waiting for more data */
478  continue;
479 
480 #endif
481 
482  j = sc->delayId.bytesWanted(0, sc->copyInto.length);
483 
484  if (j > jmax) {
485  jmax = j;
486  result = sc->delayId;
487  }
488  }
489 
490  return result;
491 }
492 
493 #endif
494 
495 int64_t
497 {
498  return endOffset() - swapout.queue_offset;
499 }
500 
int64_t endOffset() const
Definition: stmem.cc:46
int64_t freeDataUpto(int64_t)
Definition: stmem.cc:82
Io io
current I/O state
Definition: MemObject.h:142
#define assert(EX)
Definition: assert.h:17
void kickReads()
Definition: MemObject.cc:453
size_t maxInMemObjSize
Definition: SquidConfig.h:264
static int sc[16]
Definition: smbdes.c:121
struct MemObject::@87 abort
int64_t size() const
Definition: MemObject.cc:244
MemBuf * where
Definition: MemObject.cc:199
String storeId_
StoreId for our entry (usually request URI)
Definition: MemObject.h:188
size_t swap_hdr_sz
Definition: MemObject.h:174
int hdr_sz
Definition: Message.h:81
void operator()(store_client const &x)
Definition: MemObject.cc:183
struct SquidConfig::@113 onoff
int inUseCount() const
int64_t endOffset() const
Definition: MemObject.cc:229
#define PRId64
Definition: types.h:110
static Mem::AllocatorProxy & Pool()
Definition: MemObject.h:35
void dump() const
Definition: stmem.cc:401
void ctx_exit(Ctx ctx)
Definition: debug.cc:727
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:361
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:77
Definition: Range.h:18
Io io
current I/O state
Definition: MemObject.h:155
off_t offset() const
Definition: StoreIOState.h:60
DeferredReadManager deferredReads
Definition: MemObject.h:191
static size_t inUseCount()
Definition: MemObject.cc:48
bool isEmpty() const
Definition: SBuf.h:420
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:65
struct timeval start_ping
Definition: MemObject.h:163
size_type size() const
Definition: SquidString.h:72
bool memReaderHasLowerOffset(int64_t) const
Definition: store_client.cc:81
A const & max(A const &lhs, A const &rhs)
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:157
SQUIDCEXTERN void SquidMD5Update(struct SquidMD5Context *context, const void *buf, unsigned len)
Definition: md5.c:89
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:144
void trimUnSwappable()
Definition: MemObject.cc:393
int32_t index
entry position inside the memory cache
Definition: MemObject.h:152
int64_t lowestMemReaderOffset() const
Definition: MemObject.cc:281
StoreIOState::Pointer sio
Definition: MemObject.h:119
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:292
dlink_node node
Definition: StoreClient.h:92
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:141
void setNoDelay(bool const)
Definition: DelayId.cc:122
LowestMemReader(int64_t seed)
Definition: MemObject.cc:181
int shutting_down
Definition: testAddress.cc:36
void delayRead(DeferredRead const &)
Definition: MemObject.cc:439
int64_t offset
Definition: StoreIOBuffer.h:55
Definition: parse.c:104
int64_t readAheadGap
Definition: SquidConfig.h:88
void stat(MemBuf *mb) const
Definition: MemObject.cc:204
HttpReplyPointer reply_
Definition: MemObject.h:186
bool write(StoreIOBuffer const &)
Definition: stmem.cc:342
int64_t object_sz
Definition: MemObject.h:173
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
const char * storeId() const
Definition: MemObject.cc:54
HttpRequestMethod method
Definition: MemObject.h:102
#define DBG_IMPORTANT
Definition: Debug.h:46
char const * termedBuf() const
Definition: SquidString.h:91
int memory_cache_first
Definition: SquidConfig.h:335
int64_t policyLowestOffsetToKeep(bool swap) const
Definition: MemObject.cc:351
void operator()(store_client const &x)
Definition: MemObject.cc:194
StoreIOBuffer copyInto
Definition: StoreClient.h:94
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:71
int64_t objectBytesOnDisk() const
Definition: MemObject.cc:325
String logUri_
URI used for logging (usually request URI)
Definition: MemObject.h:189
void delayRead(DeferredRead const &)
Definition: comm.cc:1723
Ctx ctx_enter(const char *descr)
Definition: debug.cc:711
void markEndOfReplyHeaders()
Definition: MemObject.cc:235
DelayId mostBytesAllowed() const
Definition: MemObject.cc:460
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:147
void dumpStats(MemBuf *output, int clientNumber) const
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:412
StoreClientStats(MemBuf *anEntry)
Definition: MemObject.cc:192
void kickReads(int const count)
Definition: comm.cc:1781
SwapOut swapout
Definition: MemObject.h:126
SBuf vary_headers
Definition: MemObject.h:180
const SBuf & image() const
struct SquidConfig::@111 Store
int64_t queue_offset
number of bytes sent to SwapDir for writing
Definition: MemObject.h:118
#define PROF_start(probename)
Definition: Profiler.h:62
void reset()
Definition: MemObject.cc:270
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
dlink_list clients
Definition: MemObject.h:105
Definition: MemBuf.h:23
struct node * next
Definition: parse.c:105
int nclients
Definition: MemObject.h:111
bool hasContigousContentRange(Range< int64_t > const &range) const
Definition: stmem.cc:284
virtual void reset()
Definition: HttpReply.cc:57
const HttpReplyPointer & getReply() const
Definition: MemObject.h:57
SQUIDCEXTERN void SquidMD5Final(uint8_t digest[16], struct SquidMD5Context *context)
DelayId delayId
Definition: StoreClient.h:88
void clean()
Definition: String.cc:125
#define PROF_stop(probename)
Definition: Profiler.h:63
void dump() const
Definition: MemObject.cc:162
#define SQUIDSBUFPH
Definition: SBuf.h:31
mem_hdr data_hdr
Definition: MemObject.h:103
void trimSwappable()
Definition: MemObject.cc:368
bool isContiguous() const
Definition: MemObject.cc:403
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
int64_t availableForSwapOut() const
buffered bytes we have not swapped out yet
Definition: MemObject.cc:496
SQUIDCEXTERN void SquidMD5Init(struct SquidMD5Context *context)
Definition: md5.c:73
HttpRequestPointer request
Definition: MemObject.h:161
int64_t current
Definition: MemObject.cc:188
C * getRaw() const
Definition: RefCount.h:74
class SquidConfig Config
Definition: SquidConfig.cc:12
void freeContent()
Definition: stmem.cc:60
void setNoDelay(bool const newValue)
Definition: MemObject.cc:426
#define NULL
Definition: types.h:166
void addClient(store_client *)
Definition: MemObject.cc:306
int64_t expectedReplySize() const
Definition: MemObject.cc:253
const char * urlXXX() const
Definition: MemObject.h:93
int64_t inmem_lo
Definition: MemObject.h:104
int Ctx
Definition: Debug.h:37
int64_t offset
bytes written/read to/from the memory cache so far
Definition: MemObject.h:153
int bytesWanted(int min, int max) const
Definition: DelayId.cc:132

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors