store_rebuild.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 20 Store Rebuild Routines */
10 
11 #include "squid.h"
12 #include "event.h"
13 #include "globals.h"
14 #include "md5.h"
15 #include "SquidConfig.h"
16 #include "SquidTime.h"
17 #include "StatCounters.h"
18 #include "Store.h"
19 #include "store/Disk.h"
20 #include "store_digest.h"
21 #include "store_key_md5.h"
22 #include "store_rebuild.h"
23 #include "StoreSearch.h"
24 // for tvSubDsec() which should be in SquidTime.h
25 #include "util.h"
26 
27 #include <cerrno>
28 
30 
31 static struct timeval rebuild_start;
32 static void storeCleanup(void *);
33 
34 typedef struct {
35  /* total number of "swap.state" entries that will be read */
36  int total;
37  /* number of entries read so far */
38  int scanned;
40 
41 static store_rebuild_progress *RebuildProgress = NULL;
42 
43 static int
45 {
46  SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
47  return (SD->doubleCheck(*e));
48 }
49 
50 static void
51 storeCleanup(void *)
52 {
53  static int store_errors = 0;
54  static StoreSearchPointer currentSearch;
55  static int validated = 0;
56  static int seen = 0;
57 
58  if (currentSearch == NULL || currentSearch->isDone())
59  currentSearch = Store::Root().search();
60 
61  size_t statCount = 500;
62 
63  // TODO: Avoid the loop (and ENTRY_VALIDATED) unless opt_store_doublecheck.
64  while (statCount-- && !currentSearch->isDone() && currentSearch->next()) {
65  StoreEntry *e;
66 
67  e = currentSearch->currentItem();
68 
69  ++seen;
70 
72  continue;
73 
74  /*
75  * Calling StoreEntry->release() has no effect because we're
76  * still in 'store_rebuilding' state
77  */
78  if (e->swap_filen < 0)
79  continue;
80 
83  ++store_errors;
84 
86 
87  /*
88  * Only set the file bit if we know its a valid entry
89  * otherwise, set it in the validation procedure
90  */
91 
92  if ((++validated & 0x3FFFF) == 0)
93  /* TODO format the int with with a stream operator */
94  debugs(20, DBG_IMPORTANT, " " << validated << " Entries Validated so far.");
95  }
96 
97  if (currentSearch->isDone()) {
98  debugs(20, 2, "Seen: " << seen << " entries");
99  debugs(20, DBG_IMPORTANT, " Completed Validation Procedure");
100  debugs(20, DBG_IMPORTANT, " Validated " << validated << " Entries");
101  debugs(20, DBG_IMPORTANT, " store_swap_size = " << Store::Root().currentSize() / 1024.0 << " KB");
104 
105  if (opt_store_doublecheck && store_errors) {
106  fatalf("Quitting after finding %d cache index inconsistencies. " \
107  "Removing cache index will force its slow rebuild. " \
108  "Removing -S will let Squid start with an inconsistent " \
109  "cache index (at your own risk).\n", store_errors);
110  }
111 
112  if (store_digest)
114 
115  currentSearch = NULL;
116  } else
117  eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
118 }
119 
120 /* meta data recreated from disk image in swap directory */
121 void
122 
124 {
125  double dt;
126  counts.objcount += dc->objcount;
127  counts.expcount += dc->expcount;
128  counts.scancount += dc->scancount;
129  counts.clashcount += dc->clashcount;
130  counts.dupcount += dc->dupcount;
131  counts.cancelcount += dc->cancelcount;
132  counts.invalid += dc->invalid;
133  counts.badflags += dc->badflags;
134  counts.bad_log_op += dc->bad_log_op;
135  counts.zero_object_sz += dc->zero_object_sz;
136  /*
137  * When store_dirs_rebuilding == 1, it means we are done reading
138  * or scanning all cache_dirs. Now report the stats and start
139  * the validation (storeCleanup()) thread.
140  */
141 
143  return;
144 
146 
147  debugs(20, DBG_IMPORTANT, "Finished rebuilding storage from disk.");
148  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.scancount << " Entries scanned");
149  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.invalid << " Invalid entries.");
150  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.badflags << " With invalid flags.");
151  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.objcount << " Objects loaded.");
152  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.expcount << " Objects expired.");
153  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.cancelcount << " Objects cancelled.");
154  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.dupcount << " Duplicate URLs purged.");
155  debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.clashcount << " Swapfile clashes avoided.");
156  debugs(20, DBG_IMPORTANT, " Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
157  ((double) counts.objcount / (dt > 0.0 ? dt : 1.0)) << " objects/sec).");
158  debugs(20, DBG_IMPORTANT, "Beginning Validation Procedure");
159 
160  eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
161 
162  xfree(RebuildProgress);
163 
164  RebuildProgress = NULL;
165 }
166 
167 /*
168  * this is ugly. We don't actually start any rebuild threads here,
169  * but only initialize counters, etc. The rebuild threads are
170  * actually started by the filesystem "fooDirInit" function.
171  */
172 void
174 {
175  memset(&counts, '\0', sizeof(counts));
177  /*
178  * Note: store_dirs_rebuilding is initialized to 1.
179  *
180  * When we parse the configuration and construct each swap dir,
181  * the construction of that raises the rebuild count.
182  *
183  * This prevents us from trying to write clean logs until we
184  * finished rebuilding - including after a reconfiguration that opens an
185  * existing swapdir. The corresponding decrement * occurs in
186  * storeCleanup(), when it is finished.
187  */
189  sizeof(store_rebuild_progress));
190 }
191 
192 /*
193  * A fs-specific rebuild procedure periodically reports its
194  * progress.
195  */
196 void
197 storeRebuildProgress(int sd_index, int total, int sofar)
198 {
199  static time_t last_report = 0;
200  double n = 0.0;
201  double d = 0.0;
202 
203  if (sd_index < 0)
204  return;
205 
206  if (sd_index >= Config.cacheSwap.n_configured)
207  return;
208 
209  if (NULL == RebuildProgress)
210  return;
211 
212  RebuildProgress[sd_index].total = total;
213 
214  RebuildProgress[sd_index].scanned = sofar;
215 
216  if (squid_curtime - last_report < 15)
217  return;
218 
219  for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; ++sd_index) {
220  n += (double) RebuildProgress[sd_index].scanned;
221  d += (double) RebuildProgress[sd_index].total;
222  }
223 
224  debugs(20, DBG_IMPORTANT, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
225  last_report = squid_curtime;
226 }
227 
228 #include "fde.h"
229 #include "Generic.h"
230 #include "StoreMeta.h"
231 #include "StoreMetaUnpacker.h"
232 
233 struct InitStoreEntry : public unary_function<StoreMeta, void> {
234  InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
235 
236  void operator()(StoreMeta const &x) {
237  switch (x.getType()) {
238 
239  case STORE_META_KEY:
241  memcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH);
242  break;
243 
244  case STORE_META_STD:
245  struct old_metahdr {
246  time_t timestamp;
247  time_t lastref;
248  time_t expires;
249  time_t lastmod;
250  size_t swap_file_sz;
251  uint16_t refcount;
252  uint16_t flags;
253  } *tmp;
254  tmp = (struct old_metahdr *)x.value;
256  what->timestamp = tmp->timestamp;
257  what->lastref = tmp->lastref;
258  what->expires = tmp->expires;
259  what->lastModified(tmp->lastmod);
260  what->swap_file_sz = tmp->swap_file_sz;
261  what->refcount = tmp->refcount;
262  what->flags = tmp->flags;
263  break;
264 
265  case STORE_META_STD_LFS:
267  memcpy(&what->timestamp, x.value, STORE_HDR_METASIZE);
268  break;
269 
270  default:
271  break;
272  }
273  }
274 
277 };
278 
279 bool
281 {
282  if (fd < 0)
283  return false;
284 
285  assert(buf.hasSpace()); // caller must allocate
286 
287  const int len = FD_READ_METHOD(fd, buf.space(), buf.spaceSize());
288  ++ statCounter.syscalls.disk.reads;
289  if (len < 0) {
290  const int xerrno = errno;
291  debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << diskIndex << "]: " <<
292  "Ignoring cached entry after meta data read failure: " << xstrerr(xerrno));
293  return false;
294  }
295 
296  buf.appended(len);
297  return true;
298 }
299 
300 bool
303  uint64_t expectedSize)
304 {
305  int swap_hdr_len = 0;
306  StoreMetaUnpacker aBuilder(buf.content(), buf.contentSize(), &swap_hdr_len);
307  if (aBuilder.isBufferZero()) {
308  debugs(47,5, HERE << "skipping empty record.");
309  return false;
310  }
311 
312  StoreMeta *tlv_list = nullptr;
313  try {
314  tlv_list = aBuilder.createStoreMeta();
315  } catch (const std::exception &e) {
316  debugs(47, DBG_IMPORTANT, "WARNING: Ignoring store entry because " << e.what());
317  return false;
318  }
319  assert(tlv_list);
320 
321  // TODO: consume parsed metadata?
322 
323  debugs(47,7, "successful swap meta unpacking; swap_file_sz=" << tmpe.swap_file_sz);
324  memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
325 
326  InitStoreEntry visitor(&tmpe, key);
327  for_each(*tlv_list, visitor);
328  storeSwapTLVFree(tlv_list);
329  tlv_list = NULL;
330 
331  if (storeKeyNull(key)) {
332  debugs(47, DBG_IMPORTANT, "WARNING: Ignoring keyless cache entry");
333  return false;
334  }
335 
336  tmpe.key = key;
337  /* check sizes */
338 
339  if (expectedSize > 0) {
340  if (tmpe.swap_file_sz == 0) {
341  tmpe.swap_file_sz = expectedSize;
342  } else if (tmpe.swap_file_sz == (uint64_t)(expectedSize - swap_hdr_len)) {
343  tmpe.swap_file_sz = expectedSize;
344  } else if (tmpe.swap_file_sz != expectedSize) {
345  debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry due to a " <<
346  "SIZE MISMATCH " << tmpe.swap_file_sz << "!=" << expectedSize);
347  return false;
348  }
349  } else if (tmpe.swap_file_sz <= 0) {
350  // if caller cannot handle unknown sizes, it must check after the call.
351  debugs(47, 7, "unknown size: " << tmpe);
352  }
353 
354  if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
355  ++ stats.badflags;
356  return false;
357  }
358 
359  return true;
360 }
361 
362 bool
364 {
365  /* this needs to become
366  * 1) unpack url
367  * 2) make synthetic request with headers ?? or otherwise search
368  * for a matching object in the store
369  * TODO FIXME change to new async api
370  * TODO FIXME I think there is a race condition here with the
371  * async api :
372  * store A reads in object foo, searchs for it, and finds nothing.
373  * store B reads in object foo, searchs for it, finds nothing.
374  * store A gets called back with nothing, so registers the object
375  * store B gets called back with nothing, so registers the object,
376  * which will conflict when the in core index gets around to scanning
377  * store B.
378  *
379  * this suggests that rather than searching for duplicates, the
380  * index rebuild should just assume its the most recent accurate
381  * store entry and whoever indexes the stores handles duplicates.
382  */
383  if (StoreEntry *e = Store::Root().get(key)) {
384 
385  if (e->lastref >= tmpe.lastref) {
386  /* key already exists, old entry is newer */
387  /* keep old, ignore new */
388  ++stats.dupcount;
389 
390  // For some stores, get() creates/unpacks a store entry. Signal
391  // such stores that we will no longer use the get() result:
392  e->lock("storeRebuildKeepEntry");
393  e->unlock("storeRebuildKeepEntry");
394 
395  return false;
396  } else {
397  /* URL already exists, this swapfile not being used */
398  /* junk old, load new */
399  e->release(); /* release old entry */
400  ++stats.dupcount;
401  }
402  }
403 
404  return true;
405 }
406 
sdirno swap_dirn
Definition: Store.h:179
bool hasSpace() const
Definition: MemBuf.h:72
virtual void next(void(callback)(void *cbdata), void *cbdata)=0
StoreSearch * search()
Definition: Controller.cc:205
static int store_dirs_rebuilding
the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
Definition: Controller.h:90
#define STORE_HDR_METASIZE
Definition: defines.h:172
StatCounters statCounter
Definition: StatCounters.cc:12
#define assert(EX)
Definition: assert.h:17
mb_size_t spaceSize() const
Definition: MemBuf.cc:167
void storeRebuildComplete(StoreRebuildData *dc)
virtual StoreEntry * currentItem()=0
bool storeRebuildParseEntry(MemBuf &buf, StoreEntry &tmpe, cache_key *key, StoreRebuildData &stats, uint64_t expectedSize)
parses entry buffer and validates entry metadata; fills e on success
virtual bool isDone() const =0
unsigned char cache_key
Store key.
Definition: forward.h:29
#define xcalloc
Definition: membanger.c:57
void operator()(StoreMeta const &x)
void lastModified(const time_t when)
Definition: Store.h:142
StoreEntry * what
class Ping::pingStats_ stats
cache_key * index
void * value
Definition: StoreMeta.h:133
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:177
Controller & Root()
safely access controller singleton
Definition: Controller.cc:619
time_t expires
Definition: Store.h:167
void storeDigestNoteStoreReady(void)
struct timeval current_time
Definition: stub_time.cc:15
time_t squid_curtime
Definition: stub_time.cc:17
time_t timestamp
Definition: Store.h:165
CacheDigest * store_digest
void fatalf(const char *fmt,...)
Definition: fatal.cc:79
T & for_each(L const &head, T &visitor)
Definition: Generic.h:23
static void storeCleanup(void *)
uint64_t swap_file_sz
Definition: Store.h:171
const char * xstrerr(int error)
Definition: xstrerror.cc:83
bool storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key, StoreRebuildData &stats)
checks whether the loaded entry should be kept; updates counters
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
struct StatCounters::@136 syscalls
char * space()
returns buffer after data; does not check space existence
Definition: MemBuf.h:57
#define DBG_IMPORTANT
Definition: Debug.h:45
static StoreRebuildData counts
#define STORE_META_KEY
Definition: defines.h:165
#define SQUID_MD5_DIGEST_LENGTH
Definition: md5.h:66
struct StatCounters::@136::@139 disk
uint16_t flags
Definition: Store.h:173
void storeSwapTLVFree(tlv *n)
int length
Definition: StoreMeta.h:132
int opt_store_doublecheck
#define INDEXSD(i)
Definition: SquidConfig.h:64
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
char * content()
start of the added data
Definition: MemBuf.h:41
uint16_t refcount
Definition: Store.h:172
void appended(mb_size_t sz)
updates content size after external append
Definition: MemBuf.cc:244
void const char * buf
Definition: stub_helper.cc:16
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:147
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:109
#define EBIT_SET(flag, bit)
Definition: defines.h:105
virtual char getType() const =0
#define FD_READ_METHOD(fd, buf, len)
Definition: fde.h:161
int storeKeyNull(const cache_key *key)
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:418
Definition: MemBuf.h:23
void const cache_key * key
void storeRebuildStart(void)
void storeRebuildProgress(int sd_index, int total, int sofar)
time_t lastref
Definition: Store.h:166
virtual bool doubleCheck(StoreEntry &)
Definition: Disk.cc:46
InitStoreEntry(StoreEntry *anEntry, cache_key *aKey)
#define xfree
bool storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf, StoreRebuildData &)
loads entry from disk; fills supplied memory buffer on success
#define STORE_HDR_METASIZE_OLD
Definition: defines.h:173
static struct timeval rebuild_start
static int storeCleanupDoubleCheck(StoreEntry *e)
#define EBIT_TEST(flag, bit)
Definition: defines.h:107
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
SQUIDCEXTERN double tvSubDsec(struct timeval, struct timeval)
Definition: util.c:46
manages a single cache_dir
Definition: Disk.h:21

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors