UFSSwapDir.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 47 Store Directory Routines */
10 
11 #define CLEAN_BUF_SZ 16384
12 
13 #include "squid.h"
14 #include "cache_cf.h"
15 #include "ConfigOption.h"
16 #include "DiskIO/DiskIOModule.h"
17 #include "DiskIO/DiskIOStrategy.h"
18 #include "fde.h"
19 #include "FileMap.h"
20 #include "fs_io.h"
21 #include "globals.h"
22 #include "Parsing.h"
23 #include "RebuildState.h"
24 #include "SquidConfig.h"
25 #include "SquidMath.h"
26 #include "StatCounters.h"
27 #include "store_key_md5.h"
28 #include "StoreSearchUFS.h"
29 #include "StoreSwapLogData.h"
30 #include "tools.h"
31 #include "UFSSwapDir.h"
32 
33 #include <cerrno>
34 #include <cmath>
35 #include <random>
36 #if HAVE_SYS_STAT_H
37 #include <sys/stat.h>
38 #endif
39 
42 
44 {
45 
46 public:
47  UFSCleanLog(SwapDir *aSwapDir) : sd(aSwapDir) {}
48 
50  virtual const StoreEntry *nextEntry();
51 
53  virtual void write(StoreEntry const &);
54 
58  char *outbuf = nullptr;
59  off_t outbuf_offset = 0;
60  int fd = -1;
62  SwapDir *sd = nullptr;
63 };
64 
65 const StoreEntry *
67 {
68  const StoreEntry *entry = NULL;
69 
70  if (walker)
71  entry = walker->Next(walker);
72 
73  return entry;
74 }
75 
76 void
78 {
80  static size_t ss = sizeof(StoreSwapLogData);
81  s.op = (char) SWAP_LOG_ADD;
82  s.swap_filen = e.swap_filen;
83  s.timestamp = e.timestamp;
84  s.lastref = e.lastref;
85  s.expires = e.expires;
86  s.lastmod = e.lastModified();
88  s.refcount = e.refcount;
89  s.flags = e.flags;
90  memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH);
91  s.finalize();
92  memcpy(outbuf + outbuf_offset, &s, ss);
93  outbuf_offset += ss;
94  /* buffered write */
95 
96  if (outbuf_offset + ss >= CLEAN_BUF_SZ) {
98  int xerrno = errno;
99  /* XXX This error handling should probably move up to the caller */
100  debugs(50, DBG_CRITICAL, MYNAME << newLog << ": write: " << xstrerr(xerrno));
101  debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
102  file_close(fd);
103  fd = -1;
104  unlink(newLog.c_str());
105  sd->cleanLog = NULL;
106  delete this;
107  return;
108  }
109 
110  outbuf_offset = 0;
111  }
112 }
113 
114 bool
115 Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
116 {
117  if (!SwapDir::canStore(e, diskSpaceNeeded, load))
118  return false;
119 
120  if (IO->shedLoad())
121  return false;
122 
123  load = IO->load();
124  return true;
125 }
126 
127 static void
128 FreeObject(void *address)
129 {
130  StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address);
131  delete anObject;
132 }
133 
134 static int
135 rev_int_sort(const void *A, const void *B)
136 {
137  const int *i1 = (const int *)A;
138  const int *i2 = (const int *)B;
139  return *i2 - *i1;
140 }
141 
142 void
144 {
145  int i = GetInteger();
146  if (i <= 0)
147  fatal("UFSSwapDir::parseSizeL1L2: invalid size value");
148 
149  const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes
150 
151  /* just reconfigure it */
152  if (reconfiguring) {
153  if (size == maxSize())
154  debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB");
155  else
156  debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB");
157  }
158 
159  max_size = size;
160 
161  l1 = GetInteger();
162 
163  if (l1 <= 0)
164  fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value");
165 
166  l2 = GetInteger();
167 
168  if (l2 <= 0)
169  fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value");
170 }
171 
172 void
174 {
175  parseSizeL1L2();
176  parseOptions(1);
177 }
178 
179 void
180 Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath)
181 {
182  index = anIndex;
183  path = xstrdup(aPath);
184 
185  parseSizeL1L2();
186 
187  /* Initialise replacement policy stuff */
189 
190  parseOptions(0);
191 }
192 
193 void
195 {
196  DiskIOStrategy *anIO = module->createStrategy();
197  safe_free(ioType);
198  ioType = xstrdup(module->type());
199 
200  delete IO->io;
201  IO->io = anIO;
202  /* Change the IO Options */
203 
204  if (currentIOOptions && currentIOOptions->options.size() > 2) {
205  delete currentIOOptions->options.back();
206  currentIOOptions->options.pop_back();
207  }
208 
209  /* TODO: factor out these 4 lines */
210  ConfigOption *ioOptions = IO->io->getOptionTree();
211 
212  if (currentIOOptions && ioOptions)
213  currentIOOptions->options.push_back(ioOptions);
214 }
215 
216 bool
217 Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig)
218 {
219  if (strcmp(option, "IOEngine") != 0)
220  return false;
221 
222  if (isaReconfig)
223  /* silently ignore this */
224  return true;
225 
226  if (!value) {
227  self_destruct();
228  return false;
229  }
230 
231  DiskIOModule *module = DiskIOModule::Find(value);
232 
233  if (!module) {
234  self_destruct();
235  return false;
236  }
237 
238  changeIO(module);
239 
240  return true;
241 }
242 
243 void
245 {
246  storeAppendPrintf(e, " IOEngine=%s", ioType);
247 }
248 
249 ConfigOption *
251 {
252  ConfigOption *parentResult = SwapDir::getOptionTree();
253 
254  if (currentIOOptions == NULL)
255  currentIOOptions = new ConfigOptionVector();
256 
257  currentIOOptions->options.push_back(parentResult);
258 
259  currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump));
260 
261  if (ConfigOption *ioOptions = IO->io->getOptionTree())
262  currentIOOptions->options.push_back(ioOptions);
263 
264  ConfigOption* result = currentIOOptions;
265 
266  currentIOOptions = NULL;
267 
268  return result;
269 }
270 
271 void
273 {
274  debugs(47, 3, "Initialising UFS SwapDir engine.");
275  /* Parsing must be finished by now - force to NULL, don't delete */
276  currentIOOptions = NULL;
277  static int started_clean_event = 0;
278  static const char *errmsg =
279  "\tFailed to verify one of the swap directories, Check cache.log\n"
280  "\tfor details. Run 'squid -z' to create swap directories\n"
281  "\tif needed, or if running Squid for the first time.";
282  IO->init();
283 
284  if (verifyCacheDirs())
285  fatal(errmsg);
286 
287  openLog();
288 
289  rebuild();
290 
291  if (!started_clean_event) {
292  eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1);
293  started_clean_event = 1;
294  }
295 
296  (void) fsBlockSize(path, &fs.blksize);
297 }
298 
299 void
301 {
302  debugs(47, 3, "Creating swap space in " << path);
303  createDirectory(path, 0);
304  createSwapSubDirs();
305 }
306 
307 Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) :
308  SwapDir(aType),
309  IO(NULL),
310  fsdata(NULL),
311  map(new FileMap()),
312  suggest(0),
313  l1(16),
314  l2(256),
315  swaplog_fd(-1),
316  currentIOOptions(new ConfigOptionVector()),
317  ioType(xstrdup(anIOType)),
318  cur_size(0),
319  n_disk_objects(0),
320  rebuilding_(false)
321 {
322  /* modulename is only set to disk modules that are built, by configure,
323  * so the Find call should never return NULL here.
324  */
325  IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy());
326 }
327 
329 {
330  if (swaplog_fd > -1) {
331  file_close(swaplog_fd);
332  swaplog_fd = -1;
333  }
334  xfree(ioType);
335  delete map;
336  delete IO;
337  delete currentIOOptions;
338 }
339 
340 void
342 {
343  debugs(47, DBG_CRITICAL, "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
344  debugs(47, DBG_CRITICAL, "PATH " << fullPath(e.swap_filen, NULL) );
345  e.dump(0);
346 }
347 
348 bool
350 {
351 
352  struct stat sb;
353 
354  if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) {
355  debugs(47, DBG_CRITICAL, "WARNING: Missing swap file");
356  dumpEntry(e);
357  return true;
358  }
359 
360  if ((off_t)e.swap_file_sz != sb.st_size) {
361  debugs(47, DBG_CRITICAL, "WARNING: Size Mismatch. Entry size: "
362  << e.swap_file_sz << ", file size: " << sb.st_size);
363  dumpEntry(e);
364  return true;
365  }
366 
367  return false;
368 }
369 
370 void
372 {
373  int totl_kb = 0;
374  int free_kb = 0;
375  int totl_in = 0;
376  int free_in = 0;
377  int x;
378  storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1);
379  storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2);
380  storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10);
381  storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0);
382  storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
383  Math::doublePercent(currentSize(), maxSize()));
384  storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
385  map->numFilesInMap(), map->capacity(),
386  Math::intPercent(map->numFilesInMap(), map->capacity()));
387  x = fsStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
388 
389  if (0 == x) {
390  storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
391  totl_kb - free_kb,
392  totl_kb,
393  Math::intPercent(totl_kb - free_kb, totl_kb));
394  storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
395  totl_in - free_in,
396  totl_in,
397  Math::intPercent(totl_in - free_in, totl_in));
398  }
399 
400  storeAppendPrintf(&sentry, "Flags:");
401 
402  if (flags.selected)
403  storeAppendPrintf(&sentry, " SELECTED");
404 
405  if (flags.read_only)
406  storeAppendPrintf(&sentry, " READ-ONLY");
407 
408  storeAppendPrintf(&sentry, "\n");
409 
410  IO->statfs(sentry);
411 }
412 
413 void
415 {
416  /* TODO: possible options for improvement;
417  *
418  * Note that too much aggression here is not good. It means that disk
419  * controller is getting a long queue of removals to act on, along
420  * with its regular I/O queue, and that client traffic is 'paused'
421  * and growing the network I/O queue as well while the scan happens.
422  * Possibly bad knock-on effects as Squid catches up on all that.
423  *
424  * Bug 2448 may have been a sign of what can wrong. At the least it
425  * provides a test case for aggression effects in overflow conditions.
426  *
427  * - base removal limit on space saved, instead of count ?
428  *
429  * - base removal rate on a traffic speed counter ?
430  * as the purge took up more time out of the second it would grow to
431  * a graceful full pause
432  *
433  * - pass out a value to cause another event to be scheduled immediately
434  * instead of waiting a whole second more ?
435  * knock on; schedule less if all caches are under low-water
436  *
437  * - admin configurable removal rate or count ?
438  * the current numbers are arbitrary, config helps with experimental
439  * trials and future-proofing the install base.
440  * we also have this indirectly by shifting the relative positions
441  * of low-, high- water and the total capacity limit.
442  */
443 
444  // minSize() is swap_low_watermark in bytes
445  const uint64_t lowWaterSz = minSize();
446 
447  if (currentSize() < lowWaterSz) {
448  debugs(47, 5, "space still available in " << path);
449  return;
450  }
451 
452  /* We can't delete objects while rebuilding swap */
453  /* XXX each store should start maintaining as it comes online. */
455  // suppress the warnings, except once each minute
456  static int64_t lastWarn = 0;
457  int warnLevel = 3;
458  if (lastWarn+60 < squid_curtime) {
459  lastWarn = squid_curtime;
460  warnLevel = DBG_IMPORTANT;
461  }
462  debugs(47, warnLevel, StoreController::store_dirs_rebuilding << " cache_dir still rebuilding. Skip GC for " << path);
463  return;
464  }
465 
466  // maxSize() is cache_dir total size in bytes
467  const uint64_t highWaterSz = ((maxSize() * Config.Swap.highWaterMark) / 100);
468 
469  // f is percentage of 'gap' filled between low- and high-water.
470  // Used to reduced purge rate when between water markers, and
471  // to multiply it more aggressively the further above high-water
472  // it reaches. But in a graceful linear growth curve.
473  double f = 1.0;
474  if (highWaterSz > lowWaterSz) {
475  // might be equal. n/0 is bad.
476  f = (double) (currentSize() - lowWaterSz) / (highWaterSz - lowWaterSz);
477  }
478 
479  // how deep to look for a single object that can be removed
480  int max_scan = (int) (f * 400.0 + 100.0);
481 
482  // try to purge only this many objects this cycle.
483  int max_remove = (int) (f * 300.0 + 20.0);
484 
485  /*
486  * This is kinda cheap, but so we need this priority hack?
487  */
488  debugs(47, 3, "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove);
489 
490  RemovalPurgeWalker *walker = repl->PurgeInit(repl, max_scan);
491 
492  int removed = 0;
493  // only purge while above low-water
494  while (currentSize() >= lowWaterSz) {
495 
496  // stop if we reached max removals for this cycle,
497  // Bug 2448 may be from this not clearing enough,
498  // but it predates the current algorithm so not sure
499  if (removed >= max_remove)
500  break;
501 
502  StoreEntry *e = walker->Next(walker);
503 
504  // stop if all objects are locked / in-use,
505  // or the cache is empty
506  if (!e)
507  break; /* no more objects */
508 
509  ++removed;
510 
511  e->release(true);
512  }
513 
514  walker->Done(walker);
515  debugs(47, (removed ? 2 : 3), path <<
516  " removed " << removed << "/" << max_remove << " f=" <<
517  std::setprecision(4) << f << " max_scan=" << max_scan);
518 
519  // what if cache is still over the high watermark ?
520  // Store::Maintain() schedules another purge in 1 second.
521 }
522 
523 void
525 {
526  debugs(47, 3, "referencing " << &e << " " <<
527  e.swap_dirn << "/" << e.swap_filen);
528 
529  if (repl->Referenced)
530  repl->Referenced(repl, &e, &e.repl);
531 }
532 
533 bool
535 {
536  debugs(47, 3, "dereferencing " << &e << " " <<
537  e.swap_dirn << "/" << e.swap_filen);
538 
539  if (repl->Dereferenced)
540  repl->Dereferenced(repl, &e, &e.repl);
541 
542  return true; // keep e in the global store_table
543 }
544 
546 Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
547 {
548  return IO->create (this, &e, file_callback, aCallback, callback_data);
549 }
550 
552 Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
553 {
554  return IO->open (this, &e, file_callback, aCallback, callback_data);
555 }
556 
557 int
559 {
560  return map->testBit(filn);
561 }
562 
563 void
565 {
566  map->setBit(filn);
567 }
568 
569 void
571 {
572  /*
573  * We have to test the bit before calling clearBit as
574  * it doesn't do bounds checking and blindly assumes
575  * filn is a valid file number, but it might not be because
576  * the map is dynamic in size. Also clearing an already clear
577  * bit puts the map counter of-of-whack.
578  */
579 
580  if (map->testBit(filn))
581  map->clearBit(filn);
582 }
583 
584 int
586 {
587  int fn;
588  fn = map->allocate(suggest);
589  map->setBit(fn);
590  suggest = fn + 1;
591  return fn;
592 }
593 
594 char *
596 {
597  LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
598  assert(0 <= subdirn && subdirn < l1);
599  snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn);
600  return fullfilename;
601 }
602 
603 int
604 Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist)
605 {
606  int created = 0;
607 
608  struct stat st;
609  getCurrentTime();
610 
611  if (0 == ::stat(aPath, &st)) {
612  if (S_ISDIR(st.st_mode)) {
613  debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists");
614  } else {
615  fatalf("Swap directory %s is not a directory.", aPath);
616  }
617  } else if (0 == mkdir(aPath, 0755)) {
618  debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created");
619  created = 1;
620  } else {
621  int xerrno = errno;
622  fatalf("Failed to make swap directory %s: %s", aPath, xstrerr(xerrno));
623  }
624 
625  return created;
626 }
627 
628 bool
630 {
631 
632  struct stat sb;
633 
634  if (::stat(aPath, &sb) < 0) {
635  int xerrno = errno;
636  debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerr(xerrno));
637  return false;
638  }
639 
640  if (S_ISDIR(sb.st_mode) == 0) {
641  debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory");
642  return false;
643  }
644 
645  return true;
646 }
647 
648 bool
650 {
651  if (!pathIsDirectory(path))
652  return true;
653 
654  for (int j = 0; j < l1; ++j) {
655  char const *aPath = swapSubDir(j);
656 
657  if (!pathIsDirectory(aPath))
658  return true;
659  }
660 
661  return false;
662 }
663 
664 void
666 {
667  LOCAL_ARRAY(char, name, MAXPATHLEN);
668 
669  for (int i = 0; i < l1; ++i) {
670  snprintf(name, MAXPATHLEN, "%s/%02X", path, i);
671 
672  int should_exist;
673 
674  if (createDirectory(name, 0))
675  should_exist = 0;
676  else
677  should_exist = 1;
678 
679  debugs(47, DBG_IMPORTANT, "Making directories in " << name);
680 
681  for (int k = 0; k < l2; ++k) {
682  snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k);
683  createDirectory(name, should_exist);
684  }
685  }
686 }
687 
688 SBuf
689 Fs::Ufs::UFSSwapDir::logFile(char const *ext) const
690 {
691  SBuf lpath;
692 
693  if (Config.Log.swap) {
694  static char pathtmp[MAXPATHLEN];
695  char *pathtmp2 = xstrncpy(pathtmp, path, MAXPATHLEN - 64);
696 
697  // replace all '/' with '.'
698  while ((pathtmp2 = strchr(pathtmp2, '/')))
699  *pathtmp2 = '.';
700 
701  // remove any trailing '.' characters
702  int pos = strlen(pathtmp);
703  while (pos && pathtmp[pos-1] == '.')
704  pathtmp[--pos] = '\0';
705 
706  // remove any prefix '.' characters
707  for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2);
708  // replace a '%s' (if any) in the config string
709  // with the resulting pathtmp2 string
710  lpath.appendf(Config.Log.swap, pathtmp2);
711 
712  // is pathtmp2 was NOT injected, append numeric file extension
713  if (lpath.cmp(Config.Log.swap) == 0) {
714  lpath.append(".", 1);
715  lpath.appendf("%02d", index);
716  }
717  } else {
718  lpath.append(path);
719  lpath.append("/swap.state", 11);
720  }
721 
722  lpath.append(ext); // may be nil, that is okay.
723 
724  return lpath;
725 }
726 
727 void
729 {
730  if (!IamWorkerProcess())
731  return;
732 
733  assert(NumberOfUFSDirs || !UFSDirToGlobalDirMapping);
734  ++NumberOfUFSDirs;
735  assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
736 
737  if (rebuilding_) { // we did not close the temporary log used for rebuilding
738  assert(swaplog_fd >= 0);
739  return;
740  }
741 
742  SBuf logPath(logFile());
743  swaplog_fd = file_open(logPath.c_str(), O_WRONLY | O_CREAT | O_BINARY);
744 
745  if (swaplog_fd < 0) {
746  int xerrno = errno;
747  debugs(50, DBG_IMPORTANT, "ERROR: opening swap log " << logPath << ": " << xstrerr(xerrno));
748  fatal("UFSSwapDir::openLog: Failed to open swap log.");
749  }
750 
751  debugs(50, 3, "Cache Dir #" << index << " log opened on FD " << swaplog_fd);
752 }
753 
754 void
756 {
757  if (swaplog_fd < 0) /* not open */
758  return;
759 
760  --NumberOfUFSDirs;
761  assert(NumberOfUFSDirs >= 0);
762  if (!NumberOfUFSDirs)
763  safe_free(UFSDirToGlobalDirMapping);
764 
765  if (rebuilding_) // we cannot close the temporary log used for rebuilding
766  return;
767 
768  file_close(swaplog_fd);
769 
770  debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd);
771 
772  swaplog_fd = -1;
773 }
774 
775 bool
777 {
778  return anInt < l1;
779 }
780 
781 bool
783 {
784  return anInt < l2;
785 }
786 
787 StoreEntry *
789  sfileno file_number,
790  uint64_t swap_file_sz,
791  time_t expires,
792  time_t timestamp,
793  time_t lastref,
794  time_t lastmod,
795  uint32_t refcount,
796  uint16_t newFlags,
797  int)
798 {
799  StoreEntry *e = NULL;
800  debugs(47, 5, storeKeyText(key) <<
801  ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number);
802  /* if you call this you'd better be sure file_number is not
803  * already in use! */
804  e = new StoreEntry();
805  e->store_status = STORE_OK;
807  e->attachToDisk(index, file_number, SWAPOUT_DONE);
808  e->swap_file_sz = swap_file_sz;
809  e->lastref = lastref;
810  e->timestamp = timestamp;
811  e->expires = expires;
812  e->lastModified(lastmod);
813  e->refcount = refcount;
814  e->flags = newFlags;
815  e->ping_status = PING_NONE;
817  mapBitSet(e->swap_filen);
818  cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz);
819  ++n_disk_objects;
820  e->hashInsert(key);
821  replacementAdd (e);
822  return e;
823 }
824 
825 void
827 {
828  eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1);
829 }
830 
831 void
833 {
834  assert(rebuilding_);
835  rebuilding_ = false;
836 
837  SBuf swaplog_path(logFile()); // where the swaplog should be
838  SBuf tmp_path(logFile(".new"));
839 
840  file_close(swaplog_fd);
841 
842  if (!FileRename(tmp_path, swaplog_path)) {
843  fatalf("Failed to rename log file " SQUIDSBUFPH " to " SQUIDSBUFPH, SQUIDSBUFPRINT(tmp_path), SQUIDSBUFPRINT(swaplog_path));
844  }
845 
846  int fd = file_open(swaplog_path.c_str(), O_WRONLY | O_CREAT | O_BINARY);
847 
848  if (fd < 0) {
849  int xerrno = errno;
850  debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerr(xerrno));
851  fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
852  }
853 
854  swaplog_fd = fd;
855  debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd);
856 }
857 
858 FILE *
859 Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag)
860 {
861  assert(!rebuilding_);
862 
863  SBuf swaplog_path(logFile());
864  SBuf clean_path(logFile(".last-clean"));
865  SBuf new_path(logFile(".new"));
866 
867  struct stat log_sb;
868 
869  struct stat clean_sb;
870 
871  if (::stat(swaplog_path.c_str(), &log_sb) < 0) {
872  debugs(47, DBG_IMPORTANT, "Cache Dir #" << index << ": No log file");
873  return NULL;
874  }
875 
876  *zero_flag = log_sb.st_size == 0 ? 1 : 0;
877  /* close the existing write-only FD */
878 
879  if (swaplog_fd >= 0)
880  file_close(swaplog_fd);
881 
882  /* open a write-only FD for the new log */
883  int fd = file_open(new_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
884  if (fd < 0) {
885  int xerrno = errno;
886  debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerr(xerrno));
887  fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(new_path));
888  }
889 
890  swaplog_fd = fd;
891  rebuilding_ = true;
892 
893  {
894  const StoreSwapLogHeader header;
895  MemBuf buf;
896  buf.init(header.record_size, header.record_size);
897  buf.append(reinterpret_cast<const char*>(&header), sizeof(header));
898  // Pad to keep in sync with UFSSwapDir::writeCleanStart().
899  memset(buf.space(), 0, header.gapSize());
900  buf.appended(header.gapSize());
901  file_write(swaplog_fd, -1, buf.content(), buf.contentSize(),
902  NULL, NULL, buf.freeFunc());
903  }
904 
905  /* open a read-only stream of the old log */
906  FILE *fp = fopen(swaplog_path.c_str(), "rb");
907  if (!fp) {
908  int xerrno = errno;
909  debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerr(xerrno));
910  fatalf("Failed to open swap log for reading " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
911  }
912 
913  memset(&clean_sb, '\0', sizeof(struct stat));
914 
915  if (::stat(clean_path.c_str(), &clean_sb) < 0)
916  *clean_flag = 0;
917  else if (clean_sb.st_mtime < log_sb.st_mtime)
918  *clean_flag = 0;
919  else
920  *clean_flag = 1;
921 
922  safeunlink(clean_path.c_str(), 1);
923 
924  return fp;
925 }
926 
927 /*
928  * Begin the process to write clean cache state. For AUFS this means
929  * opening some log files and allocating write buffers. Return 0 if
930  * we succeed, and assign the 'func' and 'data' return pointers.
931  */
932 int
934 {
935  UFSCleanLog *state = new UFSCleanLog(this);
936  StoreSwapLogHeader header;
937 #if HAVE_FCHMOD
938 
939  struct stat sb;
940 #endif
941 
942  cleanLog = NULL;
943  state->cur = logFile();
944  state->newLog = logFile(".clean");
945  state->fd = file_open(state->newLog.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
946 
947  if (state->fd < 0) {
948  delete state;
949  return -1;
950  }
951 
952  state->cln = state->cur;
953  state->cln.append(".last-clean");
954  state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1);
955  state->outbuf_offset = 0;
956  /*copy the header */
957  memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader));
958  // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog().
959  memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize());
960  state->outbuf_offset += header.record_size;
961 
962  state->walker = repl->WalkInit(repl);
963  ::unlink(state->cln.c_str());
964  debugs(47, 3, "opened " << state->newLog << ", FD " << state->fd);
965 #if HAVE_FCHMOD
966 
967  if (::stat(state->cur.c_str(), &sb) == 0)
968  fchmod(state->fd, sb.st_mode);
969 
970 #endif
971 
972  cleanLog = state;
973  return 0;
974 }
975 
976 void
978 {
979  UFSCleanLog *state = (UFSCleanLog *)cleanLog;
980  int fd;
981 
982  if (NULL == state)
983  return;
984 
985  if (state->fd < 0)
986  return;
987 
988  state->walker->Done(state->walker);
989 
990  if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
991  int xerrno = errno;
992  debugs(50, DBG_CRITICAL, MYNAME << state->newLog << ": write: " << xstrerr(xerrno));
993  debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
994  file_close(state->fd);
995  state->fd = -1;
996  ::unlink(state->newLog.c_str());
997  }
998 
999  safe_free(state->outbuf);
1000  /*
1001  * You can't rename open files on Microsoft "operating systems"
1002  * so we have to close before renaming.
1003  */
1004  closeLog();
1005  /* save the fd value for a later test */
1006  fd = state->fd;
1007  /* rename */
1008 
1009  if (state->fd >= 0) {
1010 #if _SQUID_OS2_ || _SQUID_WINDOWS_
1011  file_close(state->fd);
1012  state->fd = -1;
1013 #endif
1014 
1015  FileRename(state->newLog, state->cur);
1016  // TODO handle rename errors
1017  }
1018 
1019  /* touch a timestamp file if we're not still validating */
1021  (void) 0;
1022  else if (fd < 0)
1023  (void) 0;
1024  else
1025  file_close(file_open(state->cln.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
1026 
1027  /* close */
1028  if (state->fd >= 0)
1029  file_close(state->fd);
1030 
1031  state->fd = -1;
1032 
1033  delete state;
1034 
1035  cleanLog = NULL;
1036 }
1037 
1039 int
1041 {
1042  static int swap_index = 0;
1043  int i;
1044  int j = 0;
1045  int n = 0;
1046 
1047  if (!NumberOfUFSDirs)
1048  return 0; // probably in the middle of reconfiguration
1049 
1050  if (NULL == UFSDirToGlobalDirMapping) {
1051  SwapDir *sd;
1052  /*
1053  * Initialize the little array that translates UFS cache_dir
1054  * number into the Config.cacheSwap.swapDirs array index.
1055  */
1056  UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping));
1057 
1058  for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) {
1059  /* This is bogus, the controller should just clean each instance once */
1060  sd = dynamic_cast <SwapDir *>(INDEXSD(i));
1061 
1062  if (!UFSSwapDir::IsUFSDir(sd))
1063  continue;
1064 
1065  UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd);
1066 
1067  assert (usd);
1068 
1069  UFSDirToGlobalDirMapping[n] = i;
1070  ++n;
1071 
1072  j += (usd->l1 * usd->l2);
1073  }
1074 
1075  assert(n == NumberOfUFSDirs);
1076  /*
1077  * Start the commonUfsDirClean() swap_index with a random
1078  * value. j equals the total number of UFS level 2
1079  * swap directories
1080  */
1081  std::mt19937 mt(static_cast<uint32_t>(getCurrentTime() & 0xFFFFFFFF));
1082  xuniform_int_distribution<> dist(0, j);
1083  swap_index = dist(mt);
1084  }
1085 
1086  /* if the rebuild is finished, start cleaning directories. */
1088  n = DirClean(swap_index);
1089  ++swap_index;
1090  }
1091 
1092  return n;
1093 }
1094 
1095 void
1097 {
1098  const int n = HandleCleanEvent();
1099  eventAdd("storeDirClean", CleanEvent, NULL,
1100  15.0 * exp(-0.25 * n), 1);
1101 }
1102 
1103 bool
1105 {
1106  UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd);
1107  return (mySD != 0) ;
1108 }
1109 
1110 /*
1111  * XXX: this is broken - it assumes all cache dirs use the same
1112  * l1 and l2 scheme. -RBC 20021215. Partial fix is in place -
1113  * if not UFSSwapDir return 0;
1114  */
1115 bool
1117 {
1118  int D1, D2;
1119  int L1, L2;
1120  int filn = fn;
1122  assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0))));
1123  UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0));
1124 
1125  if (!sd)
1126  return 0;
1127 
1128  L1 = sd->l1;
1129 
1130  L2 = sd->l2;
1131 
1132  D1 = ((filn / L2) / L2) % L1;
1133 
1134  if (F1 != D1)
1135  return 0;
1136 
1137  D2 = (filn / L2) % L2;
1138 
1139  if (F2 != D2)
1140  return 0;
1141 
1142  return 1;
1143 }
1144 
1145 int
1147 {
1148  if (filn < 0)
1149  return 0;
1150 
1151  /*
1152  * If flag is set it means out-of-range file number should
1153  * be considered invalid.
1154  */
1155  if (flag)
1156  if (filn > map->capacity())
1157  return 0;
1158 
1159  return 1;
1160 }
1161 
1162 void
1164 {
1165  debugs(79, 3, "unlinking fileno " << std::setfill('0') <<
1166  std::hex << std::uppercase << std::setw(8) << f << " '" <<
1167  fullPath(f,NULL) << "'");
1168  /* commonUfsDirMapBitReset(this, f); */
1169  IO->unlinkFile(fullPath(f,NULL));
1170 }
1171 
1172 bool
1174 {
1175  // unlinkd may be useful only in workers
1176  return IamWorkerProcess() && IO->io->unlinkdUseful();
1177 }
1178 
1179 void
1181 {
1182  debugs(79, 3, e);
1183  if (e.locked()) // somebody else may still be using this file
1184  return; // nothing to do: our get() always returns nil
1185 
1186  if (!e.hasDisk())
1187  return; // see evictIfFound()
1188 
1189  // Since these fields grow only after swap out ends successfully,
1190  // do not decrement them for e.swappingOut() and e.swapoutFailed().
1191  if (e.swappedOut()) {
1192  cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
1193  --n_disk_objects;
1194  }
1195  replacementRemove(&e);
1196  mapBitReset(e.swap_filen);
1198  e.detachFromDisk();
1199 }
1200 
1201 void
1203 {
1204  // UFS disk entries always have (attached) StoreEntries so if we got here,
1205  // the entry is not cached on disk and there is nothing for us to do.
1206 }
1207 
1208 void
1210 {
1211  debugs(47, 4, "added node " << e << " to dir " << index);
1212  repl->Add(repl, e, &e->repl);
1213 }
1214 
1215 void
1217 {
1218  assert(e->hasDisk());
1219 
1220  SwapDirPointer SD = INDEXSD(e->swap_dirn);
1221 
1222  assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
1223 
1224  debugs(47, 4, "remove node " << e << " from dir " << index);
1225 
1226  repl->Remove(repl, e, &e->repl);
1227 }
1228 
1229 void
1231 {
1232  storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2);
1233  dumpOptions(&entry);
1234 }
1235 
1236 char *
1237 Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const
1238 {
1239  LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
1240  int L1 = l1;
1241  int L2 = l2;
1242 
1243  if (!fullpath)
1244  fullpath = fullfilename;
1245 
1246  fullpath[0] = '\0';
1247 
1248  snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X",
1249  path,
1250  ((filn / L2) / L2) % L1,
1251  (filn / L2) % L2,
1252  filn);
1253 
1254  return fullpath;
1255 }
1256 
1257 int
1259 {
1260  return IO->callback();
1261 }
1262 
1263 void
1265 {
1266  IO->sync();
1267 }
1268 
1269 void
1271 {
1272  cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz);
1273  ++n_disk_objects;
1274 }
1275 
1276 void
1278 {
1279  debugs(47, 5, entry);
1280  // rely on the expected eventual StoreEntry::release(), evictCached(), or
1281  // a similar call to call unlink(), detachFromDisk(), etc. for the entry.
1282 }
1283 
1284 void
1286 {
1287  if (swaplog_fd < 0) {
1288  debugs(36, 5, "cannot log " << e << " in the middle of reconfiguration");
1289  return;
1290  }
1291 
1293  s->op = (char) op;
1294  s->swap_filen = e.swap_filen;
1295  s->timestamp = e.timestamp;
1296  s->lastref = e.lastref;
1297  s->expires = e.expires;
1298  s->lastmod = e.lastModified();
1299  s->swap_file_sz = e.swap_file_sz;
1300  s->refcount = e.refcount;
1301  s->flags = e.flags;
1302  memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH);
1303  s->finalize();
1304  file_write(swaplog_fd,
1305  -1,
1306  s,
1307  sizeof(StoreSwapLogData),
1308  NULL,
1309  NULL,
1310  FreeObject);
1311 }
1312 
1313 int
1315 {
1316  DIR *dir_pointer = NULL;
1317  int files[20];
1318  int swapfileno;
1319  int fn; /* same as swapfileno, but with dirn bits set */
1320  int n = 0;
1321  int k = 0;
1322  int N0, N1, N2;
1323  int D0, D1, D2;
1324  UFSSwapDir *SD;
1325  N0 = NumberOfUFSDirs;
1326  D0 = UFSDirToGlobalDirMapping[swap_index % N0];
1327  SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0));
1328  assert (SD);
1329  N1 = SD->l1;
1330  D1 = (swap_index / N0) % N1;
1331  N2 = SD->l2;
1332  D2 = ((swap_index / N0) / N1) % N2;
1333 
1334  SBuf p1;
1335  p1.appendf("%s/%02X/%02X", SD->path, D1, D2);
1336  debugs(36, 3, "Cleaning directory " << p1);
1337  dir_pointer = opendir(p1.c_str());
1338 
1339  if (!dir_pointer) {
1340  int xerrno = errno;
1341  if (xerrno == ENOENT) {
1342  debugs(36, DBG_CRITICAL, MYNAME << "WARNING: Creating " << p1);
1343  if (mkdir(p1.c_str(), 0777) == 0)
1344  return 0;
1345  }
1346 
1347  debugs(50, DBG_CRITICAL, MYNAME << p1 << ": " << xstrerr(xerrno));
1348  safeunlink(p1.c_str(), 1);
1349  return 0;
1350  }
1351 
1352  dirent_t *de;
1353  while ((de = readdir(dir_pointer)) != NULL && k < 20) {
1354  if (sscanf(de->d_name, "%X", &swapfileno) != 1)
1355  continue;
1356 
1357  fn = swapfileno; /* XXX should remove this cruft ! */
1358 
1359  if (SD->validFileno(fn, 1))
1360  if (SD->mapBitTest(fn))
1361  if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2))
1362  continue;
1363 
1364  files[k] = swapfileno;
1365  ++k;
1366  }
1367 
1368  closedir(dir_pointer);
1369 
1370  if (k == 0)
1371  return 0;
1372 
1373  qsort(files, k, sizeof(int), rev_int_sort);
1374 
1375  if (k > 10)
1376  k = 10;
1377 
1378  for (n = 0; n < k; ++n) {
1379  debugs(36, 3, "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]);
1380  SBuf p2(p1);
1381  p2.appendf("/%08X", files[n]);
1382  safeunlink(p2.c_str(), 0);
1384  }
1385 
1386  debugs(36, 3, "Cleaned " << k << " unused files from " << p1);
1387  return k;
1388 }
1389 
#define false
Definition: GnuRegex.c:233
int size
Definition: ModDevPoll.cc:75
int GetInteger(void)
Definition: Parsing.cc:148
time_t squid_curtime
Definition: stub_libtime.cc:20
RemovalPolicy * createRemovalPolicy(RemovalPolicySettings *settings)
Definition: store.cc:1649
#define SQUIDSBUFPH
Definition: SBuf.h:31
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
class SquidConfig Config
Definition: SquidConfig.cc:12
#define INDEXSD(i)
Definition: SquidConfig.h:72
StatCounters statCounter
Definition: StatCounters.cc:12
static int rev_int_sort(const void *A, const void *B)
Definition: UFSSwapDir.cc:135
#define CLEAN_BUF_SZ
Definition: UFSSwapDir.cc:11
static void FreeObject(void *address)
Definition: UFSSwapDir.cc:128
#define assert(EX)
Definition: assert.h:19
void self_destruct(void)
Definition: cache_cf.cc:276
static DiskIOModule * Find(char const *type)
virtual char const * type() const =0
virtual DiskIOStrategy * createStrategy()=0
static EVH RebuildStep
Definition: RebuildState.h:29
virtual int load()
Definition: UFSStrategy.cc:25
virtual bool shedLoad()
Definition: UFSStrategy.cc:19
virtual void openLog() override
Definition: UFSSwapDir.cc:728
bool pathIsDirectory(const char *path) const
Definition: UFSSwapDir.cc:629
virtual void sync() override
prepare for shutdown
Definition: UFSSwapDir.cc:1264
static EVH CleanEvent
Definition: UFSSwapDir.h:130
virtual void finalizeSwapoutFailure(StoreEntry &) override
abort the failed swapout that has been already noticed by Store
Definition: UFSSwapDir.cc:1277
virtual bool unlinkdUseful() const override
whether SwapDir may benefit from unlinkd
Definition: UFSSwapDir.cc:1173
virtual int callback() override
called once every main loop iteration; TODO: Move to UFS code.
Definition: UFSSwapDir.cc:1258
void replacementAdd(StoreEntry *e)
Definition: UFSSwapDir.cc:1209
virtual void evictIfFound(const cache_key *) override
Definition: UFSSwapDir.cc:1202
UFSSwapDir(char const *aType, const char *aModuleType)
Definition: UFSSwapDir.cc:307
StoreEntry * addDiskRestore(const cache_key *key, sfileno file_number, uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, time_t lastmod, uint32_t refcount, uint16_t flags, int clean)
Definition: UFSSwapDir.cc:788
void mapBitReset(sfileno filn)
Definition: UFSSwapDir.cc:570
void optionIODump(StoreEntry *e) const
Definition: UFSSwapDir.cc:244
int createDirectory(const char *path, int)
Definition: UFSSwapDir.cc:604
void changeIO(DiskIOModule *)
Definition: UFSSwapDir.cc:194
virtual void reconfigure() override
Definition: UFSSwapDir.cc:173
virtual void finalizeSwapoutSuccess(const StoreEntry &) override
finalize the successful swapout that has been already noticed by Store
Definition: UFSSwapDir.cc:1270
virtual void parse(int index, char *path) override
Definition: UFSSwapDir.cc:180
virtual void statfs(StoreEntry &) const override
Definition: UFSSwapDir.cc:371
static int DirClean(int swap_index)
Definition: UFSSwapDir.cc:1314
SBuf logFile(char const *ext=nullptr) const
Definition: UFSSwapDir.cc:689
FILE * openTmpSwapLog(int *clean_flag, int *zero_flag)
Definition: UFSSwapDir.cc:859
virtual void create() override
create system resources needed for this store to operate in the future
Definition: UFSSwapDir.cc:300
void mapBitSet(sfileno filn)
Definition: UFSSwapDir.cc:564
virtual void evictCached(StoreEntry &) override
Definition: UFSSwapDir.cc:1180
virtual void reference(StoreEntry &) override
somebody needs this entry (many cache replacement policies need to know)
Definition: UFSSwapDir.cc:524
char * swapSubDir(int subdirn) const
Definition: UFSSwapDir.cc:595
static int * UFSDirToGlobalDirMapping
Definition: UFSSwapDir.h:127
virtual void init() override
Definition: UFSSwapDir.cc:272
int validFileno(sfileno filn, int flag) const
Definition: UFSSwapDir.cc:1146
void replacementRemove(StoreEntry *e)
Definition: UFSSwapDir.cc:1216
int mapBitTest(sfileno filn)
Definition: UFSSwapDir.cc:558
Fs::Ufs::UFSStrategy * IO
Definition: UFSSwapDir.h:83
void dumpEntry(StoreEntry &) const
Definition: UFSSwapDir.cc:341
virtual ~UFSSwapDir()
Definition: UFSSwapDir.cc:328
virtual bool dereference(StoreEntry &) override
Definition: UFSSwapDir.cc:534
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override
Definition: UFSSwapDir.cc:546
virtual bool doubleCheck(StoreEntry &) override
Definition: UFSSwapDir.cc:349
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: UFSSwapDir.cc:414
static bool IsUFSDir(SwapDir *sd)
Definition: UFSSwapDir.cc:1104
virtual ConfigOption * getOptionTree() const override
Definition: UFSSwapDir.cc:250
static int NumberOfUFSDirs
Definition: UFSSwapDir.h:126
virtual void logEntry(const StoreEntry &e, int op) const override
Definition: UFSSwapDir.cc:1285
virtual void dump(StoreEntry &) const override
Definition: UFSSwapDir.cc:1230
static bool FilenoBelongsHere(int fn, int cachedir, int level1dir, int level2dir)
Definition: UFSSwapDir.cc:1116
char * fullPath(sfileno, char *) const
Definition: UFSSwapDir.cc:1237
virtual int writeCleanStart() override
Definition: UFSSwapDir.cc:933
virtual void closeLog() override
Definition: UFSSwapDir.cc:755
static int HandleCleanEvent()
safely cleans a few unused files if possible
Definition: UFSSwapDir.cc:1040
void unlinkFile(sfileno f)
Definition: UFSSwapDir.cc:1163
bool validL1(int) const
Definition: UFSSwapDir.cc:776
bool optionIOParse(char const *option, const char *value, int reconfiguring)
Definition: UFSSwapDir.cc:217
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const override
check whether we can store the entry; if we can, report current load
Definition: UFSSwapDir.cc:115
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override
Definition: UFSSwapDir.cc:552
bool validL2(int) const
Definition: UFSSwapDir.cc:782
virtual void writeCleanDone() override
Definition: UFSSwapDir.cc:977
Definition: MemBuf.h:24
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:209
FREE * freeFunc()
Definition: MemBuf.cc:303
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
char * content()
start of the added data
Definition: MemBuf.h:41
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
void appended(mb_size_t sz)
updates content size after external append
Definition: MemBuf.cc:226
char * space()
returns buffer after data; does not check space existence
Definition: MemBuf.h:57
C * getRaw() const
Definition: RefCount.h:80
const StoreEntry *(* Next)(RemovalPolicyWalker *walker)
Definition: RemovalPolicy.h:62
void(* Done)(RemovalPolicyWalker *walker)
Definition: RemovalPolicy.h:63
StoreEntry *(* Next)(RemovalPurgeWalker *walker)
Definition: RemovalPolicy.h:74
void(* Done)(RemovalPurgeWalker *walker)
Definition: RemovalPolicy.h:75
Definition: SBuf.h:94
const char * c_str()
Definition: SBuf.cc:516
int cmp(const SBuf &S, const size_type n) const
shorthand version for compare()
Definition: SBuf.h:275
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:229
char * swap
Definition: SquidConfig.h:183
RemovalPolicySettings * replPolicy
Definition: SquidConfig.h:97
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:429
struct SquidConfig::@94 Swap
struct SquidConfig::@101 Log
int highWaterMark
Definition: SquidConfig.h:83
struct StatCounters::@136 swap
void hashInsert(const cache_key *)
Definition: store.cc:399
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:134
uint16_t flags
Definition: Store.h:230
sdirno swap_dirn
Definition: Store.h:236
int locked() const
Definition: Store.h:144
void dump(int debug_lvl) const
Definition: store.cc:1477
time_t expires
Definition: Store.h:224
void lastModified(const time_t when)
Definition: Store.h:174
void release(const bool shareable=false)
Definition: store.cc:1122
void detachFromDisk()
Definition: store.cc:1916
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1891
time_t timestamp
Definition: Store.h:222
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:234
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:1902
RemovalPolicyNode repl
Definition: Store.h:220
ping_status_t ping_status
Definition: Store.h:240
store_status_t store_status
Definition: Store.h:242
time_t lastref
Definition: Store.h:223
uint64_t swap_file_sz
Definition: Store.h:228
uint16_t refcount
Definition: Store.h:229
void setMemStatus(mem_status_t)
Definition: store.cc:1502
void STFNCB(void *their_data, int errflag, StoreIOState::Pointer self)
Definition: StoreIOState.h:41
void STIOCB(void *their_data, int errflag, StoreIOState::Pointer self)
Definition: StoreIOState.h:51
unsigned char key[SQUID_MD5_DIGEST_LENGTH]
SwappedTime timestamp
void finalize()
call this before storing the log entry
size_t gapSize() const
number of bytes after the log header before the first log entry
static int store_dirs_rebuilding
the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
Definition: Controller.h:139
manages a single cache_dir
Definition: Disk.h:22
char * path
Definition: Disk.h:102
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const =0
check whether we can store the entry; if we can, report current load
Definition: Disk.cc:164
CleanLog * cleanLog
Definition: Disk.h:140
virtual ConfigOption * getOptionTree() const
Definition: Disk.cc:258
char * outbuf
Definition: UFSSwapDir.cc:58
off_t outbuf_offset
Definition: UFSSwapDir.cc:59
virtual const StoreEntry * nextEntry()
Get the next entry that is a candidate for clean log writing.
Definition: UFSSwapDir.cc:66
SwapDir * sd
Definition: UFSSwapDir.cc:62
UFSCleanLog(SwapDir *aSwapDir)
Definition: UFSSwapDir.cc:47
virtual void write(StoreEntry const &)
"write" an entry to the clean log file.
Definition: UFSSwapDir.cc:77
RemovalPolicyWalker * walker
Definition: UFSSwapDir.cc:61
#define dirent_t
Definition: compat_shared.h:71
#define MYNAME
Definition: Stream.h:238
#define DBG_IMPORTANT
Definition: Stream.h:41
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
#define DBG_CRITICAL
Definition: Stream.h:40
#define EBIT_CLR(flag, bit)
Definition: defines.h:68
#define O_BINARY
Definition: defines.h:143
@ NOT_IN_MEMORY
Definition: enums.h:35
@ PING_NONE
Has not considered whether to send ICP queries to peers yet.
Definition: enums.h:41
@ SWAPOUT_DONE
Definition: enums.h:64
@ STORE_OK
Definition: enums.h:50
@ ENTRY_VALIDATED
Definition: enums.h:113
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:107
void fatal(const char *message)
Definition: fatal.cc:28
void fatalf(const char *fmt,...)
Definition: fatal.cc:68
int FD_WRITE_METHOD(int fd, const char *buf, int len)
Definition: fde.h:200
int file_open(const char *path, int mode)
Definition: fs_io.cc:45
bool FileRename(const SBuf &from, const SBuf &to)
Definition: fs_io.cc:480
void safeunlink(const char *s, int quiet)
Definition: fs_io.cc:469
int fsBlockSize(const char *path, int *blksize)
Definition: fs_io.cc:503
void file_close(int fd)
Definition: fs_io.cc:73
int fsStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
Definition: fs_io.cc:527
void file_write(int fd, off_t file_offset, void const *ptr_to_buf, int len, DWCB *handle, void *handle_data, FREE *free_func)
Definition: fs_io.cc:326
int n_disk_objects
int reconfiguring
static uint32 A
Definition: md4.c:43
static uint32 B
Definition: md4.c:43
#define F1(x, y, z)
Definition: md5.c:166
#define F2(x, y, z)
Definition: md5.c:167
#define SQUID_MD5_DIGEST_LENGTH
Definition: md5.h:66
int intPercent(const int a, const int b)
Definition: SquidMath.cc:13
double doublePercent(const double, const double)
Definition: SquidMath.cc:25
#define xfree
#define xstrdup
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:75
static struct stat sb
Definition: squidclient.cc:71
#define MAXPATHLEN
Definition: stdio.h:62
unsigned char cache_key
Store key.
Definition: forward.h:29
signed_int32_t sfileno
Definition: forward.h:22
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:830
const char * storeKeyText(const cache_key *key)
void EVH void double
Definition: stub_event.cc:16
int unsigned int
Definition: stub_fd.cc:19
time_t getCurrentTime() STUB_RETVAL(0) int tvSubUsec(struct timeval
bool IamWorkerProcess()
whether the current process handles HTTP transactions and such
Definition: stub_tools.cc:47
@ SWAP_LOG_ADD
Definition: swap_log_op.h:14
#define NULL
Definition: types.h:166
#define PRIu64
Definition: types.h:120
void * xcalloc(size_t n, size_t sz)
Definition: xalloc.cc:71
#define safe_free(x)
Definition: xalloc.h:73
const char * xstrerr(int error)
Definition: xstrerror.cc:83
char * xstrncpy(char *dst, const char *src, size_t n)
Definition: xstring.cc:37

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors