peer_digest.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 72 Peer Digest Routines */
10
11#include "squid.h"
12#if USE_CACHE_DIGESTS
13#include "base/IoManip.h"
14#include "CacheDigest.h"
15#include "CachePeer.h"
16#include "event.h"
17#include "FwdState.h"
18#include "globals.h"
19#include "HttpReply.h"
20#include "HttpRequest.h"
21#include "internal.h"
22#include "MemObject.h"
23#include "mime_header.h"
24#include "neighbors.h"
25#include "PeerDigest.h"
26#include "Store.h"
27#include "store_key_md5.h"
28#include "StoreClient.h"
29#include "tools.h"
30#include "util.h"
31
32/* local types */
33
34/* local prototypes */
35static time_t peerDigestIncDelay(const PeerDigest * pd);
36static time_t peerDigestNewDelay(const StoreEntry * e);
37static void peerDigestSetCheck(PeerDigest * pd, time_t delay);
39static void peerDigestRequest(PeerDigest * pd);
41static int peerDigestFetchReply(void *, char *, ssize_t);
42int peerDigestSwapInCBlock(void *, char *, ssize_t);
43int peerDigestSwapInMask(void *, char *, ssize_t);
44static int peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name);
45static void peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason);
46static void peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason);
47static void peerDigestReqFinish(DigestFetchState * fetch, char *buf, int, int, int, const char *reason, int err);
48static void peerDigestPDFinish(DigestFetchState * fetch, int pcb_valid, int err);
49static void peerDigestFetchFinish(DigestFetchState * fetch, int err);
50static void peerDigestFetchSetStats(DigestFetchState * fetch);
51static int peerDigestSetCBlock(PeerDigest * pd, const char *buf);
52static int peerDigestUseful(const PeerDigest * pd);
53
54/* local constants */
55Version const CacheDigestVer = { 5, 3 };
56
57#define StoreDigestCBlockSize sizeof(StoreDigestCBlock)
58
59/* min interval for requesting digests from a given peer */
60static const time_t PeerDigestReqMinGap = 5 * 60; /* seconds */
61/* min interval for requesting digests (cumulative request stream) */
62static const time_t GlobDigestReqMinGap = 1 * 60; /* seconds */
63
64/* local vars */
65
66static time_t pd_last_req_time = 0; /* last call to Check */
67
69 peer(p),
70 host(peer->host) // if peer disappears, we will know its name
71{
72 times.initialized = squid_curtime;
73}
74
76
78
80 pd(cbdataReference(aPd)),
81 entry(nullptr),
82 old_entry(nullptr),
83 sc(nullptr),
84 old_sc(nullptr),
85 request(req),
86 offset(0),
87 mask_offset(0),
88 start_time(squid_curtime),
89 resp_time(0),
90 expires(0),
91 bufofs(0),
93{
95
96 sent.msg = 0;
97 sent.bytes = 0;
98
99 recv.msg = 0;
100 recv.bytes = 0;
101
102 *buf = 0;
103}
104
106{
107 /* unlock everything */
108 storeUnregister(sc, entry, this);
109
110 entry->unlock("DigestFetchState destructed");
111 entry = nullptr;
112
114
115 assert(pd == nullptr);
116}
117
119{
120 if (times.next_check && eventFind(peerDigestCheck, this))
122 delete cd;
123 // req_result pointer is not owned by us
124}
125
126/* called by peer to indicate that somebody actually needs this digest */
127void
129{
130 assert(pd);
131 assert(!pd->flags.needed);
132 assert(!pd->cd);
133
134 pd->flags.needed = true;
136 peerDigestSetCheck(pd, 0); /* check asap */
137}
138
139/* increment retry delay [after an unsuccessful attempt] */
140static time_t
142{
143 assert(pd);
144 return pd->times.retry_delay > 0 ?
145 2 * pd->times.retry_delay : /* exponential backoff */
146 PeerDigestReqMinGap; /* minimal delay */
147}
148
149/* artificially increases Expires: setting to avoid race conditions
150 * returns the delay till that [increased] expiration time */
151static time_t
153{
154 assert(e);
155
156 if (e->expires > 0)
158
159 return PeerDigestReqMinGap;
160}
161
162/* registers next digest verification */
163static void
165{
166 eventAdd("peerDigestCheck", peerDigestCheck, pd, (double) delay, 1);
167 pd->times.next_check = squid_curtime + delay;
168 debugs(72, 3, "peerDigestSetCheck: will check peer " << pd->host << " in " << delay << " secs");
169}
170
171/*
172 * called when peer is about to disappear or have already disappeared
173 */
174void
176{
177 if (pd->flags.requested) {
178 debugs(72, 2, "peerDigest: peer " << pd->host << " gone, will destroy after fetch.");
179 /* do nothing now, the fetching chain will notice and take action */
180 } else {
181 debugs(72, 2, "peerDigest: peer " << pd->host << " is gone, destroying now.");
182 delete pd;
183 }
184}
185
186/* callback for eventAdd() (with peer digest locked)
187 * request new digest if our copy is too old or if we lack one;
188 * schedule next check otherwise */
189static void
191{
192 PeerDigest *pd = (PeerDigest *)data;
193 time_t req_time;
194
195 assert(!pd->flags.requested);
196
197 pd->times.next_check = 0; /* unknown */
198
199 if (pd->peer.set() && !pd->peer.valid()) {
201 return;
202 }
203
204 debugs(72, 3, "cache_peer " << RawPointer(pd->peer).orNil());
205 debugs(72, 3, "peerDigestCheck: time: " << squid_curtime <<
206 ", last received: " << (long int) pd->times.received << " (" <<
207 std::showpos << (int) (squid_curtime - pd->times.received) << ")");
208
209 /* decide when we should send the request:
210 * request now unless too close to other requests */
211 req_time = squid_curtime;
212
213 /* per-peer limit */
214
215 if (req_time - pd->times.received < PeerDigestReqMinGap) {
216 debugs(72, 2, "peerDigestCheck: " << pd->host <<
217 ", avoiding close peer requests (" <<
218 (int) (req_time - pd->times.received) << " < " <<
219 (int) PeerDigestReqMinGap << " secs).");
220
221 req_time = pd->times.received + PeerDigestReqMinGap;
222 }
223
224 /* global limit */
225 if (req_time - pd_last_req_time < GlobDigestReqMinGap) {
226 debugs(72, 2, "peerDigestCheck: " << pd->host <<
227 ", avoiding close requests (" <<
228 (int) (req_time - pd_last_req_time) << " < " <<
229 (int) GlobDigestReqMinGap << " secs).");
230
232 }
233
234 if (req_time <= squid_curtime)
235 peerDigestRequest(pd); /* will set pd->flags.requested */
236 else
237 peerDigestSetCheck(pd, req_time - squid_curtime);
238}
239
240/* ask store for a digest */
241static void
243{
244 const auto p = pd->peer.get(); // TODO: Replace with a reference.
245 StoreEntry *e, *old_e;
246 char *url = nullptr;
247 HttpRequest *req;
248 StoreIOBuffer tempBuffer;
249
250 pd->req_result = nullptr;
251 pd->flags.requested = true;
252
253 /* compute future request components */
254
255 if (p->digest_url)
256 url = xstrdup(p->digest_url);
257 else
258 url = xstrdup(internalRemoteUri(p->secure.encryptTransport, p->host, p->http_port, "/squid-internal-periodic/", SBuf(StoreDigestFileName)));
259 debugs(72, 2, url);
260
261 const auto mx = MasterXaction::MakePortless<XactionInitiator::initCacheDigest>();
262 req = HttpRequest::FromUrlXXX(url, mx);
263
264 assert(req);
265
266 /* add custom headers */
267 assert(!req->header.len);
268
270
271 req->header.putStr(Http::HdrType::ACCEPT, "text/html");
272
273 if (p->login &&
274 p->login[0] != '*' &&
275 strcmp(p->login, "PASS") != 0 &&
276 strcmp(p->login, "PASSTHRU") != 0 &&
277 strncmp(p->login, "NEGOTIATE",9) != 0 &&
278 strcmp(p->login, "PROXYPASS") != 0) {
279 req->url.userInfo(SBuf(p->login)); // XXX: performance regression make peer login SBuf as well.
280 }
281 /* create fetch state structure */
282 DigestFetchState *fetch = new DigestFetchState(pd, req);
283
284 /* update timestamps */
287 req->flags.cachable.support(); // prevent RELEASE_REQUEST in storeCreateEntry()
288
289 /* the rest is based on clientReplyContext::processExpired() */
290 req->flags.refresh = true;
291
292 old_e = fetch->old_entry = storeGetPublicByRequest(req);
293
294 // XXX: Missing a hittingRequiresCollapsing() && startCollapsingOn() check.
295 if (old_e) {
296 debugs(72, 5, "found old " << *old_e);
297
298 old_e->lock("peerDigestRequest");
299 old_e->ensureMemObject(url, url, req->method);
300
301 fetch->old_sc = storeClientListAdd(old_e, fetch);
302 }
303
304 e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method);
305 debugs(72, 5, "created " << *e);
307 fetch->sc = storeClientListAdd(e, fetch);
308 /* set lastmod to trigger IMS request if possible */
309
310 // TODO: Also check for fetch->pd->cd presence as a precondition for sending
311 // IMS requests because peerDigestFetchReply() does not accept 304 responses
312 // without an in-memory cache digest.
313 if (old_e)
314 e->lastModified(old_e->lastModified());
315
316 /* push towards peer cache */
318
319 tempBuffer.offset = 0;
320
321 tempBuffer.length = SM_PAGE_SIZE;
322
323 tempBuffer.data = fetch->buf;
324
325 storeClientCopy(fetch->sc, e, tempBuffer,
326 peerDigestHandleReply, fetch);
327
328 safe_free(url);
329}
330
331/* Handle the data copying .. */
332
333/*
334 * This routine handles the copy data and then redirects the
335 * copy to a bunch of subfunctions depending upon the copy state.
336 * It also tracks the buffer offset and "seen", since I'm actually
337 * not interested in rewriting everything to suit my little idea.
338 */
339static void
340peerDigestHandleReply(void *data, StoreIOBuffer receivedData)
341{
342 DigestFetchState *fetch = (DigestFetchState *)data;
343 int retsize = -1;
344 digest_read_state_t prevstate;
345 int newsize;
346
347 if (receivedData.flags.error) {
348 peerDigestFetchAbort(fetch, fetch->buf, "failure loading digest reply from Store");
349 return;
350 }
351
352 assert(fetch->pd && receivedData.data);
353 /* The existing code assumes that the received pointer is
354 * where we asked the data to be put
355 */
356 assert(fetch->buf + fetch->bufofs == receivedData.data);
357
358 /* Update the buffer size */
359 fetch->bufofs += receivedData.length;
360
361 assert(fetch->bufofs <= SM_PAGE_SIZE);
362
363 /* If we've fetched enough, return */
364
365 if (peerDigestFetchedEnough(fetch, fetch->buf, fetch->bufofs, "peerDigestHandleReply"))
366 return;
367
368 /* Call the right function based on the state */
369 /* (Those functions will update the state if needed) */
370
371 /* Give us a temporary reference. Some of the calls we make may
372 * try to destroy the fetch structure, and we like to know if they
373 * do
374 */
375 CbcPointer<DigestFetchState> tmpLock = fetch;
376
377 /* Repeat this loop until we're out of data OR the state changes */
378 /* (So keep going if the state has changed and we still have data */
379 do {
380 prevstate = fetch->state;
381
382 switch (fetch->state) {
383
385 retsize = peerDigestFetchReply(fetch, fetch->buf, fetch->bufofs);
386 break;
387
389 retsize = peerDigestSwapInCBlock(fetch, fetch->buf, fetch->bufofs);
390 break;
391
392 case DIGEST_READ_MASK:
393 retsize = peerDigestSwapInMask(fetch, fetch->buf, fetch->bufofs);
394 break;
395
396 case DIGEST_READ_NONE:
397 break;
398
399 case DIGEST_READ_DONE:
400 return;
401 break;
402
403 default:
404 fatal("Bad digest transfer mode!\n");
405 }
406
407 if (retsize < 0)
408 return;
409
410 /*
411 * The returned size indicates how much of the buffer was read -
412 * so move the remainder of the buffer to the beginning
413 * and update the bufofs / bufsize
414 */
415 newsize = fetch->bufofs - retsize;
416
417 memmove(fetch->buf, fetch->buf + retsize, fetch->bufofs - newsize);
418
419 fetch->bufofs = newsize;
420
421 } while (cbdataReferenceValid(fetch) && prevstate != fetch->state && fetch->bufofs > 0);
422
423 // Check for EOF here, thus giving the parser one extra run. We could avoid this overhead by
424 // checking at the beginning of this function. However, in this case, we would have to require
425 // that the parser does not regard EOF as a special condition (it is true now but may change
426 // in the future).
427 if (fetch->sc->atEof()) {
428 peerDigestFetchAbort(fetch, fetch->buf, "premature end of digest reply");
429 return;
430 }
431
432 /* Update the copy offset */
433 fetch->offset += receivedData.length;
434
435 /* Schedule another copy */
436 if (cbdataReferenceValid(fetch)) {
437 StoreIOBuffer tempBuffer;
438 tempBuffer.offset = fetch->offset;
439 tempBuffer.length = SM_PAGE_SIZE - fetch->bufofs;
440 tempBuffer.data = fetch->buf + fetch->bufofs;
441 storeClientCopy(fetch->sc, fetch->entry, tempBuffer,
442 peerDigestHandleReply, fetch);
443 }
444}
445
447static int
448peerDigestFetchReply(void *data, char *buf, ssize_t size)
449{
450 DigestFetchState *fetch = (DigestFetchState *)data;
451 PeerDigest *pd = fetch->pd;
452 assert(pd && buf);
453 assert(!fetch->offset);
454
455 assert(fetch->state == DIGEST_READ_REPLY);
456
457 if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestFetchReply"))
458 return -1;
459
460 {
461 const auto &reply = fetch->entry->mem().freshestReply();
462 const auto status = reply.sline.status();
463 assert(status != Http::scNone);
464 debugs(72, 3, "peerDigestFetchReply: " << pd->host << " status: " << status <<
465 ", expires: " << (long int) reply.expires << " (" << std::showpos <<
466 (int) (reply.expires - squid_curtime) << ")");
467
468 /* this "if" is based on clientHandleIMSReply() */
469
470 if (status == Http::scNotModified) {
471 /* our old entry is fine */
472 assert(fetch->old_entry);
473
474 if (!fetch->old_entry->mem_obj->request)
475 fetch->old_entry->mem_obj->request = fetch->entry->mem_obj->request;
476
478
479 if (!Store::Root().updateOnNotModified(fetch->old_entry, *fetch->entry)) {
480 peerDigestFetchAbort(fetch, buf, "header update failure after a 304 response");
481 return -1;
482 }
483
484 /* get rid of 304 reply */
485 storeUnregister(fetch->sc, fetch->entry, fetch);
486
487 fetch->entry->unlock("peerDigestFetchReply 304");
488
489 fetch->entry = fetch->old_entry;
490
491 fetch->old_entry = nullptr;
492
493 /* preserve request -- we need its size to update counters */
494 /* requestUnlink(r); */
495 /* fetch->entry->mem_obj->request = nullptr; */
496
497 if (!fetch->pd->cd) {
498 peerDigestFetchAbort(fetch, buf, "304 without the old in-memory digest");
499 return -1;
500 }
501
502 // stay with the old in-memory digest
503 peerDigestFetchStop(fetch, buf, "Not modified");
504 fetch->state = DIGEST_READ_DONE;
505 } else if (status == Http::scOkay) {
506 /* get rid of old entry if any */
507
508 if (fetch->old_entry) {
509 debugs(72, 3, "peerDigestFetchReply: got new digest, releasing old one");
510 storeUnregister(fetch->old_sc, fetch->old_entry, fetch);
511 fetch->old_entry->releaseRequest();
512 fetch->old_entry->unlock("peerDigestFetchReply 200");
513 fetch->old_entry = nullptr;
514 }
515
516 fetch->state = DIGEST_READ_CBLOCK;
517 } else {
518 /* some kind of a bug */
519 peerDigestFetchAbort(fetch, buf, reply.sline.reason());
520 return -1; /* XXX -1 will abort stuff in ReadReply! */
521 }
522 }
523
524 return 0; // we consumed/used no buffered bytes
525}
526
527int
528peerDigestSwapInCBlock(void *data, char *buf, ssize_t size)
529{
530 DigestFetchState *fetch = (DigestFetchState *)data;
531
533
534 if (peerDigestFetchedEnough(fetch, buf, size, "peerDigestSwapInCBlock"))
535 return -1;
536
537 if (size >= (ssize_t)StoreDigestCBlockSize) {
538 PeerDigest *pd = fetch->pd;
539
540 assert(pd);
541 assert(fetch->entry->mem_obj);
542
543 if (peerDigestSetCBlock(pd, buf)) {
544 /* XXX: soon we will have variable header size */
545 /* switch to CD buffer and fetch digest guts */
546 buf = nullptr;
547 assert(pd->cd->mask);
548 fetch->state = DIGEST_READ_MASK;
550 } else {
551 peerDigestFetchAbort(fetch, buf, "invalid digest cblock");
552 return -1;
553 }
554 }
555
556 /* need more data, do we have space? */
557 if (size >= SM_PAGE_SIZE) {
558 peerDigestFetchAbort(fetch, buf, "digest cblock too big");
559 return -1;
560 }
561
562 return 0; /* We need more data */
563}
564
565int
566peerDigestSwapInMask(void *data, char *buf, ssize_t size)
567{
568 DigestFetchState *fetch = (DigestFetchState *)data;
569 PeerDigest *pd;
570
571 pd = fetch->pd;
572 assert(pd->cd && pd->cd->mask);
573
574 /*
575 * NOTENOTENOTENOTENOTE: buf doesn't point to pd->cd->mask anymore!
576 * we need to do the copy ourselves!
577 */
578 memcpy(pd->cd->mask + fetch->mask_offset, buf, size);
579
580 /* NOTE! buf points to the middle of pd->cd->mask! */
581
582 if (peerDigestFetchedEnough(fetch, nullptr, size, "peerDigestSwapInMask"))
583 return -1;
584
585 fetch->mask_offset += size;
586
587 if (fetch->mask_offset >= pd->cd->mask_size) {
588 debugs(72, 2, "peerDigestSwapInMask: Done! Got " <<
589 fetch->mask_offset << ", expected " << pd->cd->mask_size);
590 assert(fetch->mask_offset == pd->cd->mask_size);
591 assert(peerDigestFetchedEnough(fetch, nullptr, 0, "peerDigestSwapInMask"));
592 return -1; /* XXX! */
593 }
594
595 /* We always read everything, so return size */
596 return size;
597}
598
599static int
600peerDigestFetchedEnough(DigestFetchState * fetch, char *buf, ssize_t size, const char *step_name)
601{
602 static const SBuf hostUnknown("<unknown>"); // peer host (if any)
603 SBuf host = hostUnknown;
604
605 PeerDigest *pd = nullptr;
606 const char *reason = nullptr; /* reason for completion */
607 const char *no_bug = nullptr; /* successful completion if set */
608 const int pdcb_valid = cbdataReferenceValid(fetch->pd);
609 const int pcb_valid = pdcb_valid && fetch->pd->peer.valid();
610
611 /* test possible exiting conditions (the same for most steps!)
612 * cases marked with '?!' should not happen */
613
614 if (!reason) {
615 if (!pdcb_valid || !(pd = fetch->pd))
616 reason = "peer digest disappeared?!";
617 else
618 host = pd->host;
619 }
620
621 debugs(72, 6, step_name << ": peer " << host << ", offset: " <<
622 fetch->offset << " size: " << size << ".");
623
624 /* continue checking (with pd and host known and valid) */
625
626 if (!reason) {
627 if (!pd->peer)
628 reason = "peer disappeared";
629 else if (size < 0)
630 reason = "swap failure";
631 else if (!fetch->entry)
632 reason = "swap aborted?!";
633 else if (EBIT_TEST(fetch->entry->flags, ENTRY_ABORTED))
634 reason = "swap aborted";
635 }
636
637 /* continue checking (maybe-successful eof case) */
638 if (!reason && !size) {
639 if (!pd->cd)
640 reason = "null digest?!";
641 else if (fetch->mask_offset != pd->cd->mask_size)
642 reason = "premature end of digest?!";
643 else if (!peerDigestUseful(pd))
644 reason = "useless digest";
645 else
646 reason = no_bug = "success";
647 }
648
649 /* finish if we have a reason */
650 if (reason) {
651 const int level = strstr(reason, "?!") ? 1 : 3;
652 debugs(72, level, "" << step_name << ": peer " << host << ", exiting after '" << reason << "'");
653 peerDigestReqFinish(fetch, buf,
654 1, pdcb_valid, pcb_valid, reason, !no_bug);
655 } else {
656 /* paranoid check */
657 assert(pdcb_valid && pcb_valid);
658 }
659
660 return reason != nullptr;
661}
662
663/* call this when all callback data is valid and fetch must be stopped but
664 * no error has occurred (e.g. we received 304 reply and reuse old digest) */
665static void
666peerDigestFetchStop(DigestFetchState * fetch, char *buf, const char *reason)
667{
668 assert(reason);
669 debugs(72, 2, "peerDigestFetchStop: peer " << fetch->pd->host << ", reason: " << reason);
670 peerDigestReqFinish(fetch, buf, 1, 1, 1, reason, 0);
671}
672
673/* call this when all callback data is valid but something bad happened */
674static void
675peerDigestFetchAbort(DigestFetchState * fetch, char *buf, const char *reason)
676{
677 assert(reason);
678 debugs(72, 2, "peerDigestFetchAbort: peer " << fetch->pd->host << ", reason: " << reason);
679 peerDigestReqFinish(fetch, buf, 1, 1, 1, reason, 1);
680}
681
682/* complete the digest transfer, update stats, unlock/release everything */
683static void
684peerDigestReqFinish(DigestFetchState * fetch, char * /* buf */,
685 int fcb_valid, int pdcb_valid, int pcb_valid,
686 const char *reason, int err)
687{
688 assert(reason);
689
690 /* must go before peerDigestPDFinish */
691
692 if (pdcb_valid) {
693 fetch->pd->flags.requested = false;
694 fetch->pd->req_result = reason;
695 }
696
697 /* schedule next check if peer is still out there */
698 if (pcb_valid) {
699 PeerDigest *pd = fetch->pd;
700
701 if (err) {
704 } else {
705 pd->times.retry_delay = 0;
707 }
708 }
709
710 /* note: order is significant */
711 if (fcb_valid)
713
714 if (pdcb_valid)
715 peerDigestPDFinish(fetch, pcb_valid, err);
716
717 if (fcb_valid)
718 peerDigestFetchFinish(fetch, err);
719}
720
721/* destroys digest if peer disappeared
722 * must be called only when fetch and pd cbdata are valid */
723static void
724peerDigestPDFinish(DigestFetchState * fetch, int pcb_valid, int err)
725{
726 PeerDigest *pd = fetch->pd;
727 const auto host = pd->host;
729 pd->times.req_delay = fetch->resp_time;
730 pd->stats.sent.kbytes += fetch->sent.bytes;
731 pd->stats.recv.kbytes += fetch->recv.bytes;
732 pd->stats.sent.msgs += fetch->sent.msg;
733 pd->stats.recv.msgs += fetch->recv.msg;
734
735 if (err) {
736 debugs(72, DBG_IMPORTANT, "" << (pcb_valid ? "temporary " : "" ) << "disabling (" << pd->req_result << ") digest from " << host);
737
738 delete pd->cd;
739 pd->cd = nullptr;
740
741 pd->flags.usable = false;
742
743 if (!pcb_valid)
745 } else {
746 assert(pcb_valid);
747
748 pd->flags.usable = true;
749
750 /* XXX: ugly condition, but how? */
751
752 if (fetch->entry->store_status == STORE_OK)
753 debugs(72, 2, "re-used old digest from " << host);
754 else
755 debugs(72, 2, "received valid digest from " << host);
756 }
757
758 cbdataReferenceDone(fetch->pd);
759}
760
761/* free fetch state structures
762 * must be called only when fetch cbdata is valid */
763static void
765{
766 assert(fetch->entry && fetch->request);
767
768 if (fetch->old_entry) {
769 debugs(72, 3, "peerDigestFetchFinish: deleting old entry");
770 storeUnregister(fetch->old_sc, fetch->old_entry, fetch);
771 fetch->old_entry->releaseRequest();
772 fetch->old_entry->unlock("peerDigestFetchFinish old");
773 fetch->old_entry = nullptr;
774 }
775
776 /* update global stats */
779 statCounter.cd.msgs_sent += fetch->sent.msg;
780 statCounter.cd.msgs_recv += fetch->recv.msg;
781
782 delete fetch;
783}
784
785/* calculate fetch stats after completion */
786static void
788{
789 MemObject *mem;
790 assert(fetch->entry && fetch->request);
791
792 mem = fetch->entry->mem_obj;
793 assert(mem);
794
795 /* XXX: outgoing numbers are not precise */
796 /* XXX: we must distinguish between 304 hits and misses here */
797 fetch->sent.bytes = fetch->request->prefixLen();
798 /* XXX: this is slightly wrong: we don't KNOW that the entire memobject
799 * was fetched. We only know how big it is
800 */
801 fetch->recv.bytes = mem->size();
802 fetch->sent.msg = fetch->recv.msg = 1;
803 fetch->expires = fetch->entry->expires;
804 fetch->resp_time = squid_curtime - fetch->start_time;
805
806 debugs(72, 3, "peerDigestFetchFinish: recv " << fetch->recv.bytes <<
807 " bytes in " << (int) fetch->resp_time << " secs");
808
809 debugs(72, 3, "peerDigestFetchFinish: expires: " <<
810 (long int) fetch->expires << " (" << std::showpos <<
811 (int) (fetch->expires - squid_curtime) << "), lmt: " <<
812 std::noshowpos << (long int) fetch->entry->lastModified() << " (" <<
813 std::showpos << (int) (fetch->entry->lastModified() - squid_curtime) <<
814 ")");
815
816}
817
818static int
819peerDigestSetCBlock(PeerDigest * pd, const char *buf)
820{
821 StoreDigestCBlock cblock;
822 int freed_size = 0;
823 const auto host = pd->host;
824
825 memcpy(&cblock, buf, sizeof(cblock));
826 /* network -> host conversions */
827 cblock.ver.current = ntohs(cblock.ver.current);
828 cblock.ver.required = ntohs(cblock.ver.required);
829 cblock.capacity = ntohl(cblock.capacity);
830 cblock.count = ntohl(cblock.count);
831 cblock.del_count = ntohl(cblock.del_count);
832 cblock.mask_size = ntohl(cblock.mask_size);
833 debugs(72, 2, "got digest cblock from " << host << "; ver: " <<
834 (int) cblock.ver.current << " (req: " << (int) cblock.ver.required <<
835 ")");
836
837 debugs(72, 2, "\t size: " <<
838 cblock.mask_size << " bytes, e-cnt: " <<
839 cblock.count << ", e-util: " <<
840 xpercentInt(cblock.count, cblock.capacity) << "%" );
841 /* check version requirements (both ways) */
842
843 if (cblock.ver.required > CacheDigestVer.current) {
844 debugs(72, DBG_IMPORTANT, "" << host << " digest requires version " <<
845 cblock.ver.required << "; have: " << CacheDigestVer.current);
846
847 return 0;
848 }
849
850 if (cblock.ver.current < CacheDigestVer.required) {
851 debugs(72, DBG_IMPORTANT, "" << host << " digest is version " <<
852 cblock.ver.current << "; we require: " <<
854
855 return 0;
856 }
857
858 /* check consistency */
859 if (cblock.ver.required > cblock.ver.current ||
860 cblock.mask_size <= 0 || cblock.capacity <= 0 ||
861 cblock.bits_per_entry <= 0 || cblock.hash_func_count <= 0) {
862 debugs(72, DBG_CRITICAL, "" << host << " digest cblock is corrupted.");
863 return 0;
864 }
865
866 /* check consistency further */
867 if ((size_t)cblock.mask_size != CacheDigest::CalcMaskSize(cblock.capacity, cblock.bits_per_entry)) {
868 debugs(72, DBG_CRITICAL, host << " digest cblock is corrupted " <<
869 "(mask size mismatch: " << cblock.mask_size << " ? " <<
871 << ").");
872 return 0;
873 }
874
875 /* there are some things we cannot do yet */
877 debugs(72, DBG_CRITICAL, "ERROR: " << host << " digest: unsupported #hash functions: " <<
878 cblock.hash_func_count << " ? " << CacheDigestHashFuncCount << ".");
879 return 0;
880 }
881
882 /*
883 * no cblock bugs below this point
884 */
885 /* check size changes */
886 if (pd->cd && cblock.mask_size != (ssize_t)pd->cd->mask_size) {
887 debugs(72, 2, host << " digest changed size: " << cblock.mask_size <<
888 " -> " << pd->cd->mask_size);
889 freed_size = pd->cd->mask_size;
890 delete pd->cd;
891 pd->cd = nullptr;
892 }
893
894 if (!pd->cd) {
895 debugs(72, 2, "creating " << host << " digest; size: " << cblock.mask_size << " (" <<
896 std::showpos << (int) (cblock.mask_size - freed_size) << ") bytes");
897 pd->cd = new CacheDigest(cblock.capacity, cblock.bits_per_entry);
898
899 if (cblock.mask_size >= freed_size)
900 statCounter.cd.memory += (cblock.mask_size - freed_size);
901 }
902
903 assert(pd->cd);
904 /* these assignments leave us in an inconsistent state until we finish reading the digest */
905 pd->cd->count = cblock.count;
906 pd->cd->del_count = cblock.del_count;
907 return 1;
908}
909
910static int
912{
913 /* TODO: we should calculate the prob of a false hit instead of bit util */
914 const auto bit_util = pd->cd->usedMaskPercent();
915
916 if (bit_util > 65.0) {
917 debugs(72, DBG_CRITICAL, "WARNING: " << pd->host <<
918 " peer digest has too many bits on (" << bit_util << "%).");
919 return 0;
920 }
921
922 return 1;
923}
924
925static int
926saneDiff(time_t diff)
927{
928 return abs((int) diff) > squid_curtime / 2 ? 0 : diff;
929}
930
931void
933{
934#define f2s(flag) (pd->flags.flag ? "yes" : "no")
935#define appendTime(tm) storeAppendPrintf(e, "%s\t %10ld\t %+d\t %+d\n", \
936 ""#tm, (long int)pd->times.tm, \
937 saneDiff(pd->times.tm - squid_curtime), \
938 saneDiff(pd->times.tm - pd->times.initialized))
939
940 assert(pd);
941
942 auto host = pd->host;
943 storeAppendPrintf(e, "\npeer digest from " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(host));
944
946
947 storeAppendPrintf(e, "\nevent\t timestamp\t secs from now\t secs from init\n");
948 appendTime(initialized);
949 appendTime(needed);
950 appendTime(requested);
951 appendTime(received);
952 appendTime(next_check);
953
954 storeAppendPrintf(e, "peer digest state:\n");
955 storeAppendPrintf(e, "\tneeded: %3s, usable: %3s, requested: %3s\n",
956 f2s(needed), f2s(usable), f2s(requested));
957 storeAppendPrintf(e, "\n\tlast retry delay: %d secs\n",
958 (int) pd->times.retry_delay);
959 storeAppendPrintf(e, "\tlast request response time: %d secs\n",
960 (int) pd->times.req_delay);
961 storeAppendPrintf(e, "\tlast request result: %s\n",
962 pd->req_result ? pd->req_result : "(none)");
963
964 storeAppendPrintf(e, "\npeer digest traffic:\n");
965 storeAppendPrintf(e, "\trequests sent: %d, volume: %d KB\n",
966 pd->stats.sent.msgs, (int) pd->stats.sent.kbytes.kb);
967 storeAppendPrintf(e, "\treplies recv: %d, volume: %d KB\n",
968 pd->stats.recv.msgs, (int) pd->stats.recv.kbytes.kb);
969
970 storeAppendPrintf(e, "\npeer digest structure:\n");
971
972 if (pd->cd)
973 cacheDigestReport(pd->cd, host, e);
974 else
975 storeAppendPrintf(e, "\tno in-memory copy\n");
976}
977
978#endif
979
void cacheDigestGuessStatsReport(const CacheDigestGuessStats *stats, StoreEntry *sentry, const SBuf &label)
Definition: CacheDigest.cc:209
void cacheDigestReport(CacheDigest *cd, const SBuf &label, StoreEntry *e)
Definition: CacheDigest.cc:245
RawPointerT< Pointer > RawPointer(const char *label, const Pointer &ptr)
convenience wrapper for creating RawPointerT<> objects
Definition: IoManip.h:43
int size
Definition: ModDevPoll.cc:75
time_t squid_curtime
Definition: stub_libtime.cc:20
#define SQUIDSBUFPH
Definition: SBuf.h:31
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
StatCounters statCounter
Definition: StatCounters.cc:12
void(void *, StoreIOBuffer) STCB
Definition: StoreClient.h:32
#define assert(EX)
Definition: assert.h:17
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:265
#define cbdataReferenceDone(var)
Definition: cbdata.h:352
#define cbdataReference(var)
Definition: cbdata.h:343
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:320
void userInfo(const SBuf &s)
Definition: Uri.h:79
size_t kb
Definition: ByteCounter.h:25
char * mask
Definition: CacheDigest.h:58
double usedMaskPercent() const
percentage of mask bits which are used
Definition: CacheDigest.cc:183
static uint32_t CalcMaskSize(uint64_t cap, uint8_t bpe)
Definition: CacheDigest.cc:273
uint64_t del_count
Definition: CacheDigest.h:56
uint64_t count
Definition: CacheDigest.h:55
uint32_t mask_size
Definition: CacheDigest.h:59
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
Cbc * get() const
a temporary valid raw Cbc pointer or NULL
Definition: CbcPointer.h:159
bool set() const
was set but may be invalid
Definition: CbcPointer.h:40
store_client * sc
Definition: PeerDigest.h:55
StoreEntry * entry
Definition: PeerDigest.h:53
PeerDigest * pd
Definition: PeerDigest.h:52
HttpRequest * request
Definition: PeerDigest.h:57
struct DigestFetchState::@83 sent
struct DigestFetchState::@83 recv
char buf[SM_PAGE_SIZE]
Definition: PeerDigest.h:69
store_client * old_sc
Definition: PeerDigest.h:56
ssize_t bufofs
Definition: PeerDigest.h:70
time_t resp_time
Definition: PeerDigest.h:61
time_t start_time
Definition: PeerDigest.h:60
DigestFetchState(PeerDigest *, HttpRequest *)
Definition: peer_digest.cc:79
digest_read_state_t state
Definition: PeerDigest.h:71
uint32_t mask_offset
Definition: PeerDigest.h:59
StoreEntry * old_entry
Definition: PeerDigest.h:54
static void fwdStart(const Comm::ConnectionPointer &client, StoreEntry *, HttpRequest *)
Same as Start() but no master xaction info (AccessLogEntry) available.
Definition: FwdState.cc:412
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:996
Http::StatusLine sline
Definition: HttpReply.h:56
HttpRequestMethod method
Definition: HttpRequest.h:114
int prefixLen() const
Definition: HttpRequest.cc:369
static HttpRequest * FromUrlXXX(const char *url, const MasterXaction::Pointer &, const HttpRequestMethod &method=Http::METHOD_GET)
Definition: HttpRequest.cc:528
RequestFlags flags
Definition: HttpRequest.h:141
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
HttpHeader header
Definition: Message.h:74
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
int64_t size() const
Definition: MemObject.cc:229
HttpRequestPointer request
Definition: MemObject.h:212
const HttpReply & freshestReply() const
Definition: MemObject.h:68
struct PeerDigest::@86::@87 recv
time_t retry_delay
Definition: PeerDigest.h:98
ByteCounter kbytes
Definition: PeerDigest.h:111
bool requested
Definition: PeerDigest.h:90
CacheDigestGuessStats guess
Definition: PeerDigest.h:106
bool usable
Definition: PeerDigest.h:89
time_t next_check
Definition: PeerDigest.h:97
time_t received
Definition: PeerDigest.h:101
CbcPointer< CachePeer > peer
pointer back to peer structure, argh
Definition: PeerDigest.h:82
struct PeerDigest::@86 stats
PeerDigest(CachePeer *)
Definition: peer_digest.cc:68
bool needed
Definition: PeerDigest.h:88
struct PeerDigest::@85 times
CacheDigest * cd
Definition: PeerDigest.h:83
time_t req_delay
Definition: PeerDigest.h:100
const char * req_result
Definition: PeerDigest.h:85
struct PeerDigest::@86::@87 sent
const SBuf host
copy of peer->host
Definition: PeerDigest.h:84
struct PeerDigest::@84 flags
SupportOrVeto cachable
whether the response may be stored in the cache
Definition: RequestFlags.h:35
Definition: SBuf.h:94
struct StatCounters::@128 cd
ByteCounter kbytes_sent
Definition: StatCounters.h:76
ByteCounter kbytes_recv
Definition: StatCounters.h:79
ByteCounter memory
Definition: StatCounters.h:105
unsigned char bits_per_entry
Definition: PeerDigest.h:34
unsigned char hash_func_count
Definition: PeerDigest.h:35
uint16_t flags
Definition: Store.h:232
MemObject & mem()
Definition: Store.h:51
void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod)
initialize mem_obj (if needed) and set URIs/method (if missing)
Definition: store.cc:1575
int unlock(const char *context)
Definition: store.cc:455
time_t expires
Definition: Store.h:226
void lastModified(const time_t when)
Definition: Store.h:176
void lock(const char *context)
Definition: store.cc:431
MemObject * mem_obj
Definition: Store.h:221
store_status_t store_status
Definition: Store.h:244
void releaseRequest(const bool shareable=false)
Definition: store.cc:444
unsigned error
Definition: StoreIOBuffer.h:55
int64_t offset
Definition: StoreIOBuffer.h:58
struct StoreIOBuffer::@141 flags
short int required
Definition: PeerDigest.h:21
short int current
Definition: PeerDigest.h:20
bool atEof() const
Definition: StoreClient.h:109
#define DBG_IMPORTANT
Definition: Stream.h:38
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:194
#define DBG_CRITICAL
Definition: Stream.h:37
#define SM_PAGE_SIZE
Definition: defines.h:65
#define EBIT_TEST(flag, bit)
Definition: defines.h:69
digest_read_state_t
Definition: enums.h:202
@ DIGEST_READ_DONE
Definition: enums.h:207
@ DIGEST_READ_NONE
Definition: enums.h:203
@ DIGEST_READ_CBLOCK
Definition: enums.h:205
@ DIGEST_READ_REPLY
Definition: enums.h:204
@ DIGEST_READ_MASK
Definition: enums.h:206
@ KEY_PRIVATE
Definition: enums.h:102
@ ENTRY_ABORTED
Definition: enums.h:115
@ STORE_OK
Definition: enums.h:50
int eventFind(EVH *func, void *arg)
Definition: event.cc:145
void eventDelete(EVH *func, void *arg)
Definition: event.cc:127
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:107
void EVH(void *)
Definition: event.h:18
void fatal(const char *message)
Definition: fatal.cc:28
int CacheDigestHashFuncCount
const char * StoreDigestMimeStr
const char * StoreDigestFileName
void HTTPMSGUNLOCK(M *&a)
Definition: Message.h:150
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:161
char * internalRemoteUri(bool encrypt, const char *host, unsigned short port, const char *dir, const SBuf &name)
Definition: internal.cc:96
@ scNone
Definition: StatusCode.h:21
@ scNotModified
Definition: StatusCode.h:40
@ scOkay
Definition: StatusCode.h:26
Controller & Root()
safely access controller singleton
Definition: Controller.cc:938
#define xstrdup
#define StoreDigestCBlockSize
Definition: peer_digest.cc:57
#define appendTime(tm)
int peerDigestSwapInCBlock(void *, char *, ssize_t)
Definition: peer_digest.cc:528
static time_t peerDigestIncDelay(const PeerDigest *pd)
Definition: peer_digest.cc:141
static EVH peerDigestCheck
Definition: peer_digest.cc:38
static void peerDigestSetCheck(PeerDigest *pd, time_t delay)
Definition: peer_digest.cc:164
static STCB peerDigestHandleReply
Definition: peer_digest.cc:40
static void peerDigestPDFinish(DigestFetchState *fetch, int pcb_valid, int err)
Definition: peer_digest.cc:724
static void peerDigestRequest(PeerDigest *pd)
Definition: peer_digest.cc:242
#define f2s(flag)
void peerDigestStatsReport(const PeerDigest *pd, StoreEntry *e)
Definition: peer_digest.cc:932
static const time_t GlobDigestReqMinGap
Definition: peer_digest.cc:62
static void peerDigestFetchSetStats(DigestFetchState *fetch)
Definition: peer_digest.cc:787
static int peerDigestFetchedEnough(DigestFetchState *fetch, char *buf, ssize_t size, const char *step_name)
Definition: peer_digest.cc:600
static void peerDigestFetchStop(DigestFetchState *fetch, char *buf, const char *reason)
Definition: peer_digest.cc:666
static int peerDigestSetCBlock(PeerDigest *pd, const char *buf)
Definition: peer_digest.cc:819
static void peerDigestReqFinish(DigestFetchState *fetch, char *buf, int, int, int, const char *reason, int err)
Definition: peer_digest.cc:684
void peerDigestNotePeerGone(PeerDigest *pd)
Definition: peer_digest.cc:175
static void peerDigestFetchAbort(DigestFetchState *fetch, char *buf, const char *reason)
Definition: peer_digest.cc:675
static time_t peerDigestNewDelay(const StoreEntry *e)
Definition: peer_digest.cc:152
static int peerDigestFetchReply(void *, char *, ssize_t)
handle HTTP response headers in the initial storeClientCopy() response
Definition: peer_digest.cc:448
void peerDigestNeeded(PeerDigest *pd)
Definition: peer_digest.cc:128
Version const CacheDigestVer
Definition: peer_digest.cc:55
static int saneDiff(time_t diff)
Definition: peer_digest.cc:926
static time_t pd_last_req_time
Definition: peer_digest.cc:66
static int peerDigestUseful(const PeerDigest *pd)
Definition: peer_digest.cc:911
int peerDigestSwapInMask(void *, char *, ssize_t)
Definition: peer_digest.cc:566
static void peerDigestFetchFinish(DigestFetchState *fetch, int err)
Definition: peer_digest.cc:764
static const time_t PeerDigestReqMinGap
Definition: peer_digest.cc:60
static int sc[16]
Definition: smbdes.c:121
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:841
StoreEntry * storeGetPublicByRequest(HttpRequest *req, const KeyScope keyScope)
Definition: store.cc:502
StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod &method)
Definition: store.cc:745
int storeUnregister(store_client *sc, StoreEntry *e, void *data)
void storeClientCopy(store_client *sc, StoreEntry *e, StoreIOBuffer copyInto, STCB *callback, void *data)
store_client * storeClientListAdd(StoreEntry *e, void *data)
SQUIDCEXTERN int xpercentInt(double part, double whole)
Definition: util.c:46
#define safe_free(x)
Definition: xalloc.h:73

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors