MemObject.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 19 Store Memory Primitives */
10 
11 #include "squid.h"
12 #include "comm/Connection.h"
13 #include "Generic.h"
14 #include "globals.h"
15 #include "HttpReply.h"
16 #include "MemBuf.h"
17 #include "MemObject.h"
18 #include "profiler/Profiler.h"
19 #include "SquidConfig.h"
20 #include "Store.h"
21 #include "StoreClient.h"
22 
23 #if USE_DELAY_POOLS
24 #include "DelayPools.h"
25 #endif
26 
27 /* TODO: make this global or private */
28 #if URL_CHECKSUM_DEBUG
29 static unsigned int url_checksum(const char *url);
30 unsigned int
31 url_checksum(const char *url)
32 {
33  unsigned int ck;
34  SquidMD5_CTX M;
35  static unsigned char digest[16];
36  SquidMD5Init(&M);
37  SquidMD5Update(&M, (unsigned char *) url, strlen(url));
38  SquidMD5Final(digest, &M);
39  memcpy(&ck, digest, sizeof(ck));
40  return ck;
41 }
42 
43 #endif
44 
45 RemovalPolicy * mem_policy = NULL;
46 
47 size_t
49 {
50  return Pool().inUseCount();
51 }
52 
53 const char *
55 {
56  if (!storeId_.size()) {
57  debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value");
58  dump();
59  storeId_ = "[unknown_URI]";
60  }
61  return storeId_.termedBuf();
62 }
63 
64 const char *
66 {
67  return logUri_.size() ? logUri_.termedBuf() : storeId();
68 }
69 
70 bool
72 {
73  return storeId_.size();
74 }
75 
76 void
77 MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
78 {
79  if (hasUris())
80  return;
81 
82  storeId_ = aStoreId;
83  debugs(88, 3, this << " storeId: " << storeId_);
84 
85  // fast pointer comparison for a common storeCreateEntry(url,url,...) case
86  if (!aLogUri || aLogUri == aStoreId)
87  logUri_.clean(); // use storeId_ by default to minimize copying
88  else
89  logUri_ = aLogUri;
90 
91  method = aMethod;
92 
93 #if URL_CHECKSUM_DEBUG
94  chksum = url_checksum(urlXXX());
95 #endif
96 }
97 
99 {
100  debugs(20, 3, "MemObject constructed, this=" << this);
101  ping_reply_callback = nullptr;
102  memset(&start_ping, 0, sizeof(start_ping));
103  reply_ = new HttpReply;
104 }
105 
107 {
108  debugs(20, 3, "MemObject destructed, this=" << this);
109  const Ctx ctx = ctx_enter(hasUris() ? urlXXX() : "[unknown_ctx]");
110 
111 #if URL_CHECKSUM_DEBUG
112  checkUrlChecksum();
113 #endif
114 
115  if (!shutting_down) { // Store::Root() is FATALly missing during shutdown
116  assert(xitTable.index < 0);
117  assert(memCache.index < 0);
118  assert(swapout.sio == NULL);
119  }
120 
122 
123 #if 0
124  /*
125  * There is no way to abort FD-less clients, so they might
126  * still have mem->clients set.
127  */
128  assert(clients.head == NULL);
129 
130 #endif
131 
132  ctx_exit(ctx); /* must exit before we free mem->url */
133 }
134 
135 void
136 MemObject::write(const StoreIOBuffer &writeBuffer)
137 {
138  PROF_start(MemObject_write);
139  debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
140 
141  /* We don't separate out mime headers yet, so ensure that the first
142  * write is at offset 0 - where they start
143  */
144  assert (data_hdr.endOffset() || writeBuffer.offset == 0);
145 
146  assert (data_hdr.write (writeBuffer));
147  PROF_stop(MemObject_write);
148 }
149 
150 void
152 {
153  data_hdr.dump();
154 #if 0
155  /* do we want this one? */
156  debugs(20, DBG_IMPORTANT, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0));
157 #endif
158 
159  debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping.tv_sec << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec);
160  debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
161  debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
162  debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
163  debugs(20, DBG_IMPORTANT, "MemObject->reply: " << reply_);
164  debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
165  debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
166  debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
167 }
168 
169 struct LowestMemReader : public unary_function<store_client, void> {
170  LowestMemReader(int64_t seed):current(seed) {}
171 
172  void operator() (store_client const &x) {
174  current = x.copyInto.offset;
175  }
176 
177  int64_t current;
178 };
179 
180 struct StoreClientStats : public unary_function<store_client, void> {
181  StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
182 
183  void operator()(store_client const &x) {
184  x.dumpStats(where, index);
185  ++index;
186  }
187 
189  size_t index;
190 };
191 
192 void
194 {
195  mb->appendf("\t" SQUIDSBUFPH " %s\n", SQUIDSBUFPRINT(method.image()), logUri());
196  if (!vary_headers.isEmpty())
197  mb->appendf("\tvary_headers: " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(vary_headers));
198  mb->appendf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
199  mb->appendf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
200  mb->appendf("\tswapout: %" PRId64 " bytes queued\n", swapout.queue_offset);
201 
202  if (swapout.sio.getRaw())
203  mb->appendf("\tswapout: %" PRId64 " bytes written\n", (int64_t) swapout.sio->offset());
204 
205  if (xitTable.index >= 0)
206  mb->appendf("\ttransient index: %d state: %d\n", xitTable.index, xitTable.io);
207  if (memCache.index >= 0)
208  mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset);
209  if (object_sz >= 0)
210  mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz);
211 
212  StoreClientStats statsVisitor(mb);
213 
214  for_each<StoreClientStats>(clients, statsVisitor);
215 }
216 
217 int64_t
219 {
220  return data_hdr.endOffset();
221 }
222 
223 void
225 {
226  const int hdr_sz = endOffset();
227  assert(hdr_sz >= 0);
228  assert(reply_);
229  reply_->hdr_sz = hdr_sz;
230 }
231 
232 int64_t
234 {
235  if (object_sz < 0)
236  return endOffset();
237 
238  return object_sz;
239 }
240 
241 int64_t
243 {
244  debugs(20, 7, "object_sz: " << object_sz);
245  if (object_sz >= 0) // complete() has been called; we know the exact answer
246  return object_sz;
247 
248  if (reply_) {
249  const int64_t clen = reply_->bodySize(method);
250  debugs(20, 7, "clen: " << clen);
251  if (clen >= 0 && reply_->hdr_sz > 0) // yuck: Http::Message sets hdr_sz to 0
252  return clen + reply_->hdr_sz;
253  }
254 
255  return -1; // not enough information to predict
256 }
257 
258 void
260 {
261  assert(swapout.sio == NULL);
263  inmem_lo = 0;
264  /* Should we check for clients? */
265  if (reply_)
266  reply_->reset();
267 }
268 
269 int64_t
271 {
272  LowestMemReader lowest (endOffset() + 1);
273 
274  for_each <LowestMemReader>(clients, lowest);
275 
276  return lowest.current;
277 }
278 
279 /* XXX: This is wrong. It breaks *badly* on range combining */
280 bool
282 {
283  const bool canRead = endOffset() - getReply()->hdr_sz <
285 
286  if (!canRead) {
287  debugs(19, 9, "no: " << endOffset() << '-' << getReply()->hdr_sz <<
288  " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
289  }
290 
291  return canRead;
292 }
293 
294 void
296 {
297  ++nclients;
298  dlinkAdd(aClient, &aClient->node, &clients);
299 }
300 
301 #if URL_CHECKSUM_DEBUG
302 void
303 MemObject::checkUrlChecksum () const
304 {
305  assert(chksum == url_checksum(urlXXX()));
306 }
307 
308 #endif
309 
310 /*
311  * How much of the object data is on the disk?
312  */
313 int64_t
315 {
316  /*
317  * NOTE: storeOffset() represents the disk file size,
318  * not the amount of object data on disk.
319  *
320  * If we don't have at least 'swap_hdr_sz' bytes
321  * then none of the object data is on disk.
322  *
323  * This should still be safe if swap_hdr_sz == 0,
324  * meaning we haven't even opened the swapout file
325  * yet.
326  */
327 
328  if (swapout.sio.getRaw() == NULL)
329  return 0;
330 
331  int64_t nwritten = swapout.sio->offset();
332 
333  if (nwritten <= (int64_t)swap_hdr_sz)
334  return 0;
335 
336  return (nwritten - swap_hdr_sz);
337 }
338 
339 int64_t
341 {
342  /*
343  * Careful. lowest_offset can be greater than endOffset(), such
344  * as in the case of a range request.
345  */
346  int64_t lowest_offset = lowestMemReaderOffset();
347 
348  if (endOffset() < lowest_offset ||
349  endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
350  (swap && !Config.onoff.memory_cache_first))
351  return lowest_offset;
352 
353  return inmem_lo;
354 }
355 
356 void
358 {
359  int64_t new_mem_lo = policyLowestOffsetToKeep(1);
360  /*
361  * We should only free up to what we know has been written
362  * to disk, not what has been queued for writing. Otherwise
363  * there will be a chunk of the data which is not in memory
364  * and is not yet on disk.
365  * The -1 makes sure the page isn't freed until storeSwapOut has
366  * walked to the next page.
367  */
368  int64_t on_disk;
369 
370  if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
371  new_mem_lo = on_disk - 1;
372 
373  if (new_mem_lo == -1)
374  new_mem_lo = 0; /* the above might become -1 */
375 
376  data_hdr.freeDataUpto(new_mem_lo);
377 
378  inmem_lo = new_mem_lo;
379 }
380 
381 void
383 {
384  if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
385  assert (new_mem_lo > 0);
386  data_hdr.freeDataUpto(new_mem_lo);
387  inmem_lo = new_mem_lo;
388  } // else we should not trim anything at this time
389 }
390 
391 bool
393 {
395  /* XXX : make this higher level */
396  debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
397  return result;
398 }
399 
400 int
401 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
402 {
403 #if USE_DELAY_POOLS
404  if (!ignoreDelayPools) {
405  /* identify delay id with largest allowance */
406  DelayId largestAllowance = mostBytesAllowed ();
407  return largestAllowance.bytesWanted(0, max);
408  }
409 #endif
410 
411  return max;
412 }
413 
414 void
415 MemObject::setNoDelay(bool const newValue)
416 {
417 #if USE_DELAY_POOLS
418 
419  for (dlink_node *node = clients.head; node; node = node->next) {
420  store_client *sc = (store_client *) node->data;
421  sc->delayId.setNoDelay(newValue);
422  }
423 
424 #endif
425 }
426 
427 void
429 {
430 #if USE_DELAY_POOLS
431  if (readAheadPolicyCanRead()) {
432  if (DelayId mostAllowedId = mostBytesAllowed()) {
433  mostAllowedId.delayRead(aRead);
434  return;
435  }
436  }
437 #endif
438  deferredReads.delayRead(aRead);
439 }
440 
441 void
443 {
445 }
446 
447 #if USE_DELAY_POOLS
448 DelayId
450 {
451  int j;
452  int jmax = -1;
453  DelayId result;
454 
455  for (dlink_node *node = clients.head; node; node = node->next) {
456  store_client *sc = (store_client *) node->data;
457 #if 0
458  /* This test is invalid because the client may be writing data
459  * and thus will want data immediately.
460  * If we include the test, there is a race condition when too much
461  * data is read - if all sc's are writing when a read is scheduled.
462  * XXX: fixme.
463  */
464 
465  if (!sc->callbackPending())
466  /* not waiting for more data */
467  continue;
468 
469 #endif
470 
471  j = sc->delayId.bytesWanted(0, sc->copyInto.length);
472 
473  if (j > jmax) {
474  jmax = j;
475  result = sc->delayId;
476  }
477  }
478 
479  return result;
480 }
481 
482 #endif
483 
484 int64_t
486 {
487  return endOffset() - swapout.queue_offset;
488 }
489 
int64_t endOffset() const
Definition: stmem.cc:46
int64_t freeDataUpto(int64_t)
Definition: stmem.cc:82
Io io
current I/O state
Definition: MemObject.h:138
#define assert(EX)
Definition: assert.h:17
void kickReads()
Definition: MemObject.cc:442
size_t maxInMemObjSize
Definition: SquidConfig.h:268
static int sc[16]
Definition: smbdes.c:121
int64_t size() const
Definition: MemObject.cc:233
MemBuf * where
Definition: MemObject.cc:188
String storeId_
StoreId for our entry (usually request URI)
Definition: MemObject.h:182
size_t swap_hdr_sz
Definition: MemObject.h:169
int hdr_sz
Definition: Message.h:81
void operator()(store_client const &x)
Definition: MemObject.cc:172
int inUseCount() const
int64_t endOffset() const
Definition: MemObject.cc:218
#define PRId64
Definition: types.h:110
static Mem::AllocatorProxy & Pool()
Definition: MemObject.h:35
void dump() const
Definition: stmem.cc:401
void ctx_exit(Ctx ctx)
Definition: debug.cc:731
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:371
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:77
Definition: Range.h:18
Io io
current I/O state
Definition: MemObject.h:149
off_t offset() const
Definition: StoreIOState.h:60
DeferredReadManager deferredReads
Definition: MemObject.h:185
static size_t inUseCount()
Definition: MemObject.cc:48
struct SquidConfig::@110 Store
bool isEmpty() const
Definition: SBuf.h:420
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:65
struct timeval start_ping
Definition: MemObject.h:157
size_type size() const
Definition: SquidString.h:72
bool memReaderHasLowerOffset(int64_t) const
Definition: store_client.cc:96
A const & max(A const &lhs, A const &rhs)
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:151
SQUIDCEXTERN void SquidMD5Update(struct SquidMD5Context *context, const void *buf, unsigned len)
Definition: md5.c:89
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:140
void trimUnSwappable()
Definition: MemObject.cc:382
int32_t index
entry position inside the memory cache
Definition: MemObject.h:146
int64_t lowestMemReaderOffset() const
Definition: MemObject.cc:270
StoreIOState::Pointer sio
Definition: MemObject.h:117
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:281
dlink_node node
Definition: StoreClient.h:99
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:137
void setNoDelay(bool const)
Definition: DelayId.cc:122
LowestMemReader(int64_t seed)
Definition: MemObject.cc:170
int shutting_down
Definition: testAddress.cc:36
void delayRead(DeferredRead const &)
Definition: MemObject.cc:428
int64_t offset
Definition: StoreIOBuffer.h:55
Definition: parse.c:104
int64_t readAheadGap
Definition: SquidConfig.h:92
void stat(MemBuf *mb) const
Definition: MemObject.cc:193
HttpReplyPointer reply_
Definition: MemObject.h:180
bool write(StoreIOBuffer const &)
Definition: stmem.cc:342
int64_t object_sz
Definition: MemObject.h:168
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
const char * storeId() const
Definition: MemObject.cc:54
HttpRequestMethod method
Definition: MemObject.h:102
#define DBG_IMPORTANT
Definition: Debug.h:46
char const * termedBuf() const
Definition: SquidString.h:91
int memory_cache_first
Definition: SquidConfig.h:339
int64_t policyLowestOffsetToKeep(bool swap) const
Definition: MemObject.cc:340
void operator()(store_client const &x)
Definition: MemObject.cc:183
StoreIOBuffer copyInto
Definition: StoreClient.h:101
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:71
int64_t objectBytesOnDisk() const
Definition: MemObject.cc:314
String logUri_
URI used for logging (usually request URI)
Definition: MemObject.h:183
void delayRead(DeferredRead const &)
Definition: comm.cc:1723
Ctx ctx_enter(const char *descr)
Definition: debug.cc:715
void markEndOfReplyHeaders()
Definition: MemObject.cc:224
DelayId mostBytesAllowed() const
Definition: MemObject.cc:449
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:136
void dumpStats(MemBuf *output, int clientNumber) const
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:401
struct SquidConfig::@112 onoff
StoreClientStats(MemBuf *anEntry)
Definition: MemObject.cc:181
void kickReads(int const count)
Definition: comm.cc:1781
SwapOut swapout
Definition: MemObject.h:124
SBuf vary_headers
Definition: MemObject.h:174
const SBuf & image() const
int64_t queue_offset
number of bytes sent to SwapDir for writing
Definition: MemObject.h:116
#define PROF_start(probename)
Definition: Profiler.h:62
void reset()
Definition: MemObject.cc:259
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
dlink_list clients
Definition: MemObject.h:105
Definition: MemBuf.h:23
struct node * next
Definition: parse.c:105
int nclients
Definition: MemObject.h:111
bool hasContigousContentRange(Range< int64_t > const &range) const
Definition: stmem.cc:284
virtual void reset()
Definition: HttpReply.cc:58
const HttpReplyPointer & getReply() const
Definition: MemObject.h:57
SQUIDCEXTERN void SquidMD5Final(uint8_t digest[16], struct SquidMD5Context *context)
DelayId delayId
Definition: StoreClient.h:95
void clean()
Definition: String.cc:125
#define PROF_stop(probename)
Definition: Profiler.h:63
void dump() const
Definition: MemObject.cc:151
#define SQUIDSBUFPH
Definition: SBuf.h:31
mem_hdr data_hdr
Definition: MemObject.h:103
void trimSwappable()
Definition: MemObject.cc:357
bool isContiguous() const
Definition: MemObject.cc:392
IRCB * ping_reply_callback
Definition: MemObject.h:158
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
int64_t availableForSwapOut() const
buffered bytes we have not swapped out yet
Definition: MemObject.cc:485
SQUIDCEXTERN void SquidMD5Init(struct SquidMD5Context *context)
Definition: md5.c:73
HttpRequestPointer request
Definition: MemObject.h:155
int64_t current
Definition: MemObject.cc:177
C * getRaw() const
Definition: RefCount.h:74
class SquidConfig Config
Definition: SquidConfig.cc:12
void freeContent()
Definition: stmem.cc:60
void setNoDelay(bool const newValue)
Definition: MemObject.cc:415
#define NULL
Definition: types.h:166
void addClient(store_client *)
Definition: MemObject.cc:295
int64_t expectedReplySize() const
Definition: MemObject.cc:242
const char * urlXXX() const
Definition: MemObject.h:93
int64_t inmem_lo
Definition: MemObject.h:104
int Ctx
Definition: Debug.h:37
int64_t offset
bytes written/read to/from the memory cache so far
Definition: MemObject.h:147
int bytesWanted(int min, int max) const
Definition: DelayId.cc:132

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors