MemObject.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 19 Store Memory Primitives */
10
11#include "squid.h"
12#include "comm/Connection.h"
13#include "Generic.h"
14#include "globals.h"
15#include "HttpReply.h"
16#include "MemBuf.h"
17#include "MemObject.h"
18#include "SquidConfig.h"
19#include "Store.h"
20#include "StoreClient.h"
21
22#if USE_DELAY_POOLS
23#include "DelayPools.h"
24#endif
25
26/* TODO: make this global or private */
27#if URL_CHECKSUM_DEBUG
28static unsigned int url_checksum(const char *url);
29unsigned int
30url_checksum(const char *url)
31{
32 unsigned int ck;
34 static unsigned char digest[16];
35 SquidMD5Init(&M);
36 SquidMD5Update(&M, (unsigned char *) url, strlen(url));
37 SquidMD5Final(digest, &M);
38 memcpy(&ck, digest, sizeof(ck));
39 return ck;
40}
41
42#endif
43
45
46size_t
48{
49 return Pool().inUseCount();
50}
51
52const char *
54{
55 if (!storeId_.size()) {
56 debugs(20, DBG_IMPORTANT, "ERROR: Squid BUG: Missing MemObject::storeId value");
57 dump();
58 storeId_ = "[unknown_URI]";
59 }
60 return storeId_.termedBuf();
61}
62
63const char *
65{
66 return logUri_.size() ? logUri_.termedBuf() : storeId();
67}
68
69bool
71{
72 return storeId_.size();
73}
74
75void
76MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
77{
78 if (hasUris())
79 return;
80
81 storeId_ = aStoreId;
82 debugs(88, 3, this << " storeId: " << storeId_);
83
84 // fast pointer comparison for a common storeCreateEntry(url,url,...) case
85 if (!aLogUri || aLogUri == aStoreId)
86 logUri_.clean(); // use storeId_ by default to minimize copying
87 else
88 logUri_ = aLogUri;
89
90 method = aMethod;
91
92#if URL_CHECKSUM_DEBUG
93 chksum = url_checksum(urlXXX());
94#endif
95}
96
98{
99 debugs(20, 3, "MemObject constructed, this=" << this);
100 ping_reply_callback = nullptr;
101 memset(&start_ping, 0, sizeof(start_ping));
102 reply_ = new HttpReply;
103}
104
106{
107 debugs(20, 3, "MemObject destructed, this=" << this);
108
109#if URL_CHECKSUM_DEBUG
110 checkUrlChecksum();
111#endif
112
113 assert(xitTable.index < 0);
114 assert(memCache.index < 0);
115 assert(swapout.sio == nullptr);
116
118}
119
120HttpReply &
122{
124 return *reply_;
125}
126
127void
129{
130 assert(r);
131 reply_ = r;
132 updatedReply_ = nullptr;
133}
134
135void
137{
138 debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
139
140 /* We don't separate out mime headers yet, so ensure that the first
141 * write is at offset 0 - where they start
142 */
143 assert (data_hdr.endOffset() || writeBuffer.offset == 0);
144
145 assert (data_hdr.write (writeBuffer));
146}
147
148void
150{
151 data_hdr.dump();
152
153 debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping);
154 debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
155 debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
156 debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
157 debugs(20, DBG_IMPORTANT, "MemObject->reply: " << reply_);
158 debugs(20, DBG_IMPORTANT, "MemObject->updatedReply: " << updatedReply_);
159 debugs(20, DBG_IMPORTANT, "MemObject->appliedUpdates: " << appliedUpdates);
160 debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
161 debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
162 debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
163}
164
165struct LowestMemReader : public unary_function<store_client, void> {
166 LowestMemReader(int64_t seed):current(seed) {}
167
168 void operator() (store_client const &x) {
169 if (x.getType() == STORE_MEM_CLIENT)
171 }
172
173 int64_t current;
174};
175
176struct StoreClientStats : public unary_function<store_client, void> {
177 StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
178
179 void operator()(store_client const &x) {
181 ++index;
182 }
183
185 size_t index;
186};
187
188void
190{
191 mb->appendf("\t" SQUIDSBUFPH " %s\n", SQUIDSBUFPRINT(method.image()), logUri());
192 if (!vary_headers.isEmpty())
193 mb->appendf("\tvary_headers: " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(vary_headers));
194 mb->appendf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
195 mb->appendf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
196 mb->appendf("\tswapout: %" PRId64 " bytes queued\n", swapout.queue_offset);
197
198 if (swapout.sio.getRaw())
199 mb->appendf("\tswapout: %" PRId64 " bytes written\n", (int64_t) swapout.sio->offset());
200
201 if (xitTable.index >= 0)
202 mb->appendf("\ttransient index: %d state: %d\n", xitTable.index, xitTable.io);
203 if (memCache.index >= 0)
204 mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset);
205 if (object_sz >= 0)
206 mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz);
207
208 StoreClientStats statsVisitor(mb);
209
210 for_each<StoreClientStats>(clients, statsVisitor);
211}
212
213int64_t
215{
216 return data_hdr.endOffset();
217}
218
219void
221{
222 const int hdr_sz = endOffset();
223 assert(hdr_sz >= 0);
224 assert(reply_);
225 reply_->hdr_sz = hdr_sz;
226}
227
228int64_t
230{
231 if (object_sz < 0)
232 return endOffset();
233
234 return object_sz;
235}
236
237int64_t
239{
240 if (object_sz >= 0) {
241 debugs(20, 7, object_sz << " frozen by complete()");
242 return object_sz;
243 }
244
245 const auto hdr_sz = baseReply().hdr_sz;
246
247 // Cannot predict future length using an empty/unset or HTTP/0 reply.
248 // For any HTTP/1 reply, hdr_sz is positive -- status-line cannot be empty.
249 if (hdr_sz <= 0)
250 return -1;
251
252 const auto clen = baseReply().bodySize(method);
253 if (clen < 0) {
254 debugs(20, 7, "unknown; hdr: " << hdr_sz);
255 return -1;
256 }
257
258 const auto messageSize = clen + hdr_sz;
259 debugs(20, 7, messageSize << " hdr: " << hdr_sz << " clen: " << clen);
260 return messageSize;
261}
262
263void
265{
266 assert(swapout.sio == nullptr);
268 inmem_lo = 0;
269 /* Should we check for clients? */
270 assert(reply_);
271 reply_->reset();
272 updatedReply_ = nullptr;
273 appliedUpdates = false;
274}
275
276int64_t
278{
279 LowestMemReader lowest (endOffset() + 1);
280
281 for_each <LowestMemReader>(clients, lowest);
282
283 return lowest.current;
284}
285
286/* XXX: This is wrong. It breaks *badly* on range combining */
287bool
289{
290 const auto savedHttpHeaders = baseReply().hdr_sz;
291 const bool canRead = endOffset() - savedHttpHeaders <
293
294 if (!canRead) {
295 debugs(19, 5, "no: " << endOffset() << '-' << savedHttpHeaders <<
296 " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
297 }
298
299 return canRead;
300}
301
302void
304{
305 ++nclients;
306 dlinkAdd(aClient, &aClient->node, &clients);
307}
308
309#if URL_CHECKSUM_DEBUG
310void
311MemObject::checkUrlChecksum () const
312{
313 assert(chksum == url_checksum(urlXXX()));
314}
315
316#endif
317
318/*
319 * How much of the object data is on the disk?
320 */
321int64_t
323{
324 /*
325 * NOTE: storeOffset() represents the disk file size,
326 * not the amount of object data on disk.
327 *
328 * If we don't have at least 'swap_hdr_sz' bytes
329 * then none of the object data is on disk.
330 *
331 * This should still be safe if swap_hdr_sz == 0,
332 * meaning we haven't even opened the swapout file
333 * yet.
334 */
335
336 if (swapout.sio.getRaw() == nullptr)
337 return 0;
338
339 int64_t nwritten = swapout.sio->offset();
340
341 if (nwritten <= (int64_t)swap_hdr_sz)
342 return 0;
343
344 return (nwritten - swap_hdr_sz);
345}
346
347int64_t
349{
350 /*
351 * Careful. lowest_offset can be greater than endOffset(), such
352 * as in the case of a range request.
353 */
354 int64_t lowest_offset = lowestMemReaderOffset();
355
356 if (endOffset() < lowest_offset ||
359 return lowest_offset;
360
361 return inmem_lo;
362}
363
364void
366{
367 int64_t new_mem_lo = policyLowestOffsetToKeep(1);
368 /*
369 * We should only free up to what we know has been written
370 * to disk, not what has been queued for writing. Otherwise
371 * there will be a chunk of the data which is not in memory
372 * and is not yet on disk.
373 * The -1 makes sure the page isn't freed until storeSwapOut has
374 * walked to the next page.
375 */
376 int64_t on_disk;
377
378 if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
379 new_mem_lo = on_disk - 1;
380
381 if (new_mem_lo == -1)
382 new_mem_lo = 0; /* the above might become -1 */
383
384 data_hdr.freeDataUpto(new_mem_lo);
385
386 inmem_lo = new_mem_lo;
387}
388
389void
391{
392 if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
393 assert (new_mem_lo > 0);
394 data_hdr.freeDataUpto(new_mem_lo);
395 inmem_lo = new_mem_lo;
396 } // else we should not trim anything at this time
397}
398
399bool
401{
403 /* XXX : make this higher level */
404 debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
405 return result;
406}
407
408int
409MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
410{
411#if USE_DELAY_POOLS
412 if (!ignoreDelayPools) {
413 /* identify delay id with largest allowance */
414 DelayId largestAllowance = mostBytesAllowed ();
415 return largestAllowance.bytesWanted(0, max);
416 }
417#else
418 (void)ignoreDelayPools;
419#endif
420
421 return max;
422}
423
424void
425MemObject::setNoDelay(bool const newValue)
426{
427#if USE_DELAY_POOLS
428
429 for (dlink_node *node = clients.head; node; node = node->next) {
430 store_client *sc = (store_client *) node->data;
431 sc->delayId.setNoDelay(newValue);
432 }
433#else
434 (void)newValue;
435#endif
436}
437
438void
440{
441#if USE_DELAY_POOLS
443 if (DelayId mostAllowedId = mostBytesAllowed()) {
444 mostAllowedId.delayRead(aRead);
445 return;
446 }
447 }
448#endif
449 deferredReads.delay(aRead);
450}
451
452void
454{
456}
457
458#if USE_DELAY_POOLS
461{
462 int j;
463 int jmax = -1;
464 DelayId result;
465
466 for (dlink_node *node = clients.head; node; node = node->next) {
467 store_client *sc = (store_client *) node->data;
468
469 j = sc->bytesWanted();
470
471 if (j > jmax) {
472 jmax = j;
473 result = sc->delayId;
474 }
475 }
476
477 return result;
478}
479
480#endif
481
482int64_t
484{
485 return endOffset() - swapout.queue_offset;
486}
487
RemovalPolicy * mem_policy
Definition: MemObject.cc:44
#define SQUIDSBUFPH
Definition: SBuf.h:31
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
class SquidConfig Config
Definition: SquidConfig.cc:12
#define assert(EX)
Definition: assert.h:19
int bytesWanted(int min, int max) const
Definition: DelayId.cc:130
void schedule()
schedules and forgets all async calls previously stored by delay()
void delay(const AsyncCallPointer &)
stores the given call to schedule it at schedule() or destruction time
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:374
virtual void reset()
Definition: HttpReply.cc:57
const SBuf & image() const
int hdr_sz
Definition: Message.h:81
Definition: MemBuf.h:24
int32_t index
entry position inside the memory cache
Definition: MemObject.h:181
Io io
current I/O state
Definition: MemObject.h:184
int64_t offset
bytes written/read to/from the memory cache so far
Definition: MemObject.h:182
int64_t queue_offset
number of bytes sent to SwapDir for writing
Definition: MemObject.h:151
StoreIOState::Pointer sio
Definition: MemObject.h:152
int32_t index
entry position inside the in-transit table
Definition: MemObject.h:172
Io io
current I/O state
Definition: MemObject.h:173
void replaceBaseReply(const HttpReplyPointer &r)
Definition: MemObject.cc:128
String storeId_
StoreId for our entry (usually request URI)
Definition: MemObject.h:213
bool appliedUpdates
Definition: MemObject.h:89
int64_t lowestMemReaderOffset() const
Definition: MemObject.cc:277
DelayId mostBytesAllowed() const
Definition: MemObject.cc:460
IRCB * ping_reply_callback
Definition: MemObject.h:191
int64_t size() const
Definition: MemObject.cc:229
const char * urlXXX() const
Definition: MemObject.h:128
int nclients
Definition: MemObject.h:146
static size_t inUseCount()
Definition: MemObject.cc:47
SwapOut swapout
Definition: MemObject.h:159
HttpRequestMethod method
Definition: MemObject.h:137
int64_t availableForSwapOut() const
buffered bytes we have not swapped out yet
Definition: MemObject.cc:483
HttpRequestPointer request
Definition: MemObject.h:188
void trimSwappable()
Definition: MemObject.cc:365
int64_t policyLowestOffsetToKeep(bool swap) const
Definition: MemObject.cc:348
int64_t objectBytesOnDisk() const
Definition: MemObject.cc:322
HttpReplyPointer reply_
Definition: MemObject.h:210
void setNoDelay(bool const newValue)
Definition: MemObject.cc:425
void reset()
Definition: MemObject.cc:264
void trimUnSwappable()
Definition: MemObject.cc:390
void addClient(store_client *)
Definition: MemObject.cc:303
XitTable xitTable
current [shared] memory caching state for the entry
Definition: MemObject.h:175
int64_t expectedReplySize() const
Definition: MemObject.cc:238
SBuf vary_headers
Definition: MemObject.h:204
dlink_list clients
Definition: MemObject.h:140
String logUri_
URI used for logging (usually request URI)
Definition: MemObject.h:214
mem_hdr data_hdr
Definition: MemObject.h:138
void dump() const
Definition: MemObject.cc:149
struct timeval start_ping
Definition: MemObject.h:190
void markEndOfReplyHeaders()
sets baseReply().hdr_sz (i.e. written reply headers size) to endOffset()
Definition: MemObject.cc:220
void delayRead(const AsyncCallPointer &)
Definition: MemObject.cc:439
void write(const StoreIOBuffer &buf)
Definition: MemObject.cc:136
int64_t inmem_lo
Definition: MemObject.h:139
size_t swap_hdr_sz
Definition: MemObject.h:199
int mostBytesWanted(int max, bool ignoreDelayPools) const
Definition: MemObject.cc:409
MemCache memCache
current [shared] memory caching state for the entry
Definition: MemObject.h:186
int64_t endOffset() const
Definition: MemObject.cc:214
void setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
Definition: MemObject.cc:76
const char * storeId() const
Definition: MemObject.cc:53
const HttpReply & baseReply() const
Definition: MemObject.h:59
HttpReply & adjustableBaseReply()
Definition: MemObject.cc:121
bool hasUris() const
whether setUris() has been called
Definition: MemObject.cc:70
const char * logUri() const
client request URI used for logging; storeId() by default
Definition: MemObject.cc:64
DelayedAsyncCalls deferredReads
Definition: MemObject.h:216
HttpReplyPointer updatedReply_
Definition: MemObject.h:211
void kickReads()
Definition: MemObject.cc:453
bool readAheadPolicyCanRead() const
Definition: MemObject.cc:288
bool isContiguous() const
Definition: MemObject.cc:400
void stat(MemBuf *mb) const
Definition: MemObject.cc:189
int64_t object_sz
Definition: MemObject.h:198
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
Definition: Range.h:19
C * getRaw() const
Definition: RefCount.h:80
bool isEmpty() const
Definition: SBuf.h:431
int64_t readAheadGap
Definition: SquidConfig.h:96
struct SquidConfig::@111 onoff
int memory_cache_first
Definition: SquidConfig.h:333
size_t maxInMemObjSize
Definition: SquidConfig.h:266
struct SquidConfig::@109 Store
int64_t offset
Definition: StoreIOBuffer.h:55
off_t offset() const
Definition: StoreIOState.h:60
void clean()
Definition: String.cc:118
char const * termedBuf() const
Definition: SquidString.h:92
size_type size() const
Definition: SquidString.h:73
void dump() const
Definition: stmem.cc:397
void freeContent()
Definition: stmem.cc:59
int64_t freeDataUpto(int64_t)
Definition: stmem.cc:81
bool write(StoreIOBuffer const &)
Definition: stmem.cc:341
int64_t endOffset() const
Definition: stmem.cc:45
bool hasContigousContentRange(Range< int64_t > const &range) const
Definition: stmem.cc:283
int getType() const
Definition: store_client.cc:88
dlink_node node
Definition: StoreClient.h:123
int64_t readOffset() const
Definition: StoreClient.h:72
void dumpStats(MemBuf *output, int clientNumber) const
A const & max(A const &lhs, A const &rhs)
A const & min(A const &lhs, A const &rhs)
#define DBG_IMPORTANT
Definition: Stream.h:41
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
@ STORE_MEM_CLIENT
Definition: enums.h:73
SQUIDCEXTERN void SquidMD5Init(struct SquidMD5Context *context)
Definition: md5.c:73
SQUIDCEXTERN void SquidMD5Update(struct SquidMD5Context *context, const void *buf, unsigned len)
Definition: md5.c:89
SQUIDCEXTERN void SquidMD5Final(uint8_t digest[16], struct SquidMD5Context *context)
static int sc[16]
Definition: smbdes.c:121
LowestMemReader(int64_t seed)
Definition: MemObject.cc:166
int64_t current
Definition: MemObject.cc:173
void operator()(store_client const &x)
Definition: MemObject.cc:168
MemBuf * where
Definition: MemObject.cc:184
void operator()(store_client const &x)
Definition: MemObject.cc:179
StoreClientStats(MemBuf *anEntry)
Definition: MemObject.cc:177
Definition: parse.c:104
struct node * next
Definition: parse.c:105
#define PRId64
Definition: types.h:110

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors