UFSSwapDir.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 47 Store Directory Routines */
10 
11 #define CLEAN_BUF_SZ 16384
12 
13 #include "squid.h"
14 #include "cache_cf.h"
15 #include "ConfigOption.h"
16 #include "DiskIO/DiskIOModule.h"
17 #include "DiskIO/DiskIOStrategy.h"
18 #include "fde.h"
19 #include "FileMap.h"
20 #include "fs_io.h"
21 #include "globals.h"
22 #include "Parsing.h"
23 #include "RebuildState.h"
24 #include "SquidConfig.h"
25 #include "SquidMath.h"
26 #include "SquidTime.h"
27 #include "StatCounters.h"
28 #include "store_key_md5.h"
29 #include "StoreSearchUFS.h"
30 #include "StoreSwapLogData.h"
31 #include "tools.h"
32 #include "UFSSwapDir.h"
33 
34 #include <cerrno>
35 #include <cmath>
36 #include <random>
37 #if HAVE_SYS_STAT_H
38 #include <sys/stat.h>
39 #endif
40 
43 
45 {
46 
47 public:
48  UFSCleanLog(SwapDir *aSwapDir) : sd(aSwapDir) {}
49 
51  virtual const StoreEntry *nextEntry();
52 
54  virtual void write(StoreEntry const &);
55 
59  char *outbuf = nullptr;
60  off_t outbuf_offset = 0;
61  int fd = -1;
63  SwapDir *sd = nullptr;
64 };
65 
66 const StoreEntry *
68 {
69  const StoreEntry *entry = NULL;
70 
71  if (walker)
72  entry = walker->Next(walker);
73 
74  return entry;
75 }
76 
77 void
79 {
81  static size_t ss = sizeof(StoreSwapLogData);
82  s.op = (char) SWAP_LOG_ADD;
83  s.swap_filen = e.swap_filen;
84  s.timestamp = e.timestamp;
85  s.lastref = e.lastref;
86  s.expires = e.expires;
87  s.lastmod = e.lastModified();
89  s.refcount = e.refcount;
90  s.flags = e.flags;
91  memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH);
92  s.finalize();
93  memcpy(outbuf + outbuf_offset, &s, ss);
94  outbuf_offset += ss;
95  /* buffered write */
96 
97  if (outbuf_offset + ss >= CLEAN_BUF_SZ) {
99  int xerrno = errno;
100  /* XXX This error handling should probably move up to the caller */
101  debugs(50, DBG_CRITICAL, MYNAME << newLog << ": write: " << xstrerr(xerrno));
102  debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
103  file_close(fd);
104  fd = -1;
105  unlink(newLog.c_str());
106  sd->cleanLog = NULL;
107  delete this;
108  return;
109  }
110 
111  outbuf_offset = 0;
112  }
113 }
114 
115 bool
116 Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
117 {
118  if (!SwapDir::canStore(e, diskSpaceNeeded, load))
119  return false;
120 
121  if (IO->shedLoad())
122  return false;
123 
124  load = IO->load();
125  return true;
126 }
127 
128 static void
129 FreeObject(void *address)
130 {
131  StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address);
132  delete anObject;
133 }
134 
135 static int
136 rev_int_sort(const void *A, const void *B)
137 {
138  const int *i1 = (const int *)A;
139  const int *i2 = (const int *)B;
140  return *i2 - *i1;
141 }
142 
143 void
145 {
146  int i = GetInteger();
147  if (i <= 0)
148  fatal("UFSSwapDir::parseSizeL1L2: invalid size value");
149 
150  const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes
151 
152  /* just reconfigure it */
153  if (reconfiguring) {
154  if (size == maxSize())
155  debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB");
156  else
157  debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB");
158  }
159 
160  max_size = size;
161 
162  l1 = GetInteger();
163 
164  if (l1 <= 0)
165  fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value");
166 
167  l2 = GetInteger();
168 
169  if (l2 <= 0)
170  fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value");
171 }
172 
173 void
175 {
176  parseSizeL1L2();
177  parseOptions(1);
178 }
179 
180 void
181 Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath)
182 {
183  index = anIndex;
184  path = xstrdup(aPath);
185 
186  parseSizeL1L2();
187 
188  /* Initialise replacement policy stuff */
190 
191  parseOptions(0);
192 }
193 
194 void
196 {
197  DiskIOStrategy *anIO = module->createStrategy();
198  safe_free(ioType);
199  ioType = xstrdup(module->type());
200 
201  delete IO->io;
202  IO->io = anIO;
203  /* Change the IO Options */
204 
205  if (currentIOOptions && currentIOOptions->options.size() > 2) {
206  delete currentIOOptions->options.back();
207  currentIOOptions->options.pop_back();
208  }
209 
210  /* TODO: factor out these 4 lines */
211  ConfigOption *ioOptions = IO->io->getOptionTree();
212 
213  if (currentIOOptions && ioOptions)
214  currentIOOptions->options.push_back(ioOptions);
215 }
216 
217 bool
218 Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig)
219 {
220  if (strcmp(option, "IOEngine") != 0)
221  return false;
222 
223  if (isaReconfig)
224  /* silently ignore this */
225  return true;
226 
227  if (!value) {
228  self_destruct();
229  return false;
230  }
231 
232  DiskIOModule *module = DiskIOModule::Find(value);
233 
234  if (!module) {
235  self_destruct();
236  return false;
237  }
238 
239  changeIO(module);
240 
241  return true;
242 }
243 
244 void
246 {
247  storeAppendPrintf(e, " IOEngine=%s", ioType);
248 }
249 
250 ConfigOption *
252 {
253  ConfigOption *parentResult = SwapDir::getOptionTree();
254 
255  if (currentIOOptions == NULL)
256  currentIOOptions = new ConfigOptionVector();
257 
258  currentIOOptions->options.push_back(parentResult);
259 
260  currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump));
261 
262  if (ConfigOption *ioOptions = IO->io->getOptionTree())
263  currentIOOptions->options.push_back(ioOptions);
264 
265  ConfigOption* result = currentIOOptions;
266 
267  currentIOOptions = NULL;
268 
269  return result;
270 }
271 
272 void
274 {
275  debugs(47, 3, HERE << "Initialising UFS SwapDir engine.");
276  /* Parsing must be finished by now - force to NULL, don't delete */
277  currentIOOptions = NULL;
278  static int started_clean_event = 0;
279  static const char *errmsg =
280  "\tFailed to verify one of the swap directories, Check cache.log\n"
281  "\tfor details. Run 'squid -z' to create swap directories\n"
282  "\tif needed, or if running Squid for the first time.";
283  IO->init();
284 
285  if (verifyCacheDirs())
286  fatal(errmsg);
287 
288  openLog();
289 
290  rebuild();
291 
292  if (!started_clean_event) {
293  eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1);
294  started_clean_event = 1;
295  }
296 
297  (void) fsBlockSize(path, &fs.blksize);
298 }
299 
300 void
302 {
303  debugs(47, 3, "Creating swap space in " << path);
304  createDirectory(path, 0);
305  createSwapSubDirs();
306 }
307 
308 Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) :
309  SwapDir(aType),
310  IO(NULL),
311  fsdata(NULL),
312  map(new FileMap()),
313  suggest(0),
314  l1(16),
315  l2(256),
316  swaplog_fd(-1),
317  currentIOOptions(new ConfigOptionVector()),
318  ioType(xstrdup(anIOType)),
319  cur_size(0),
320  n_disk_objects(0),
321  rebuilding_(false)
322 {
323  /* modulename is only set to disk modules that are built, by configure,
324  * so the Find call should never return NULL here.
325  */
326  IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy());
327 }
328 
330 {
331  if (swaplog_fd > -1) {
332  file_close(swaplog_fd);
333  swaplog_fd = -1;
334  }
335  xfree(ioType);
336  delete map;
337  delete IO;
338  delete currentIOOptions;
339 }
340 
341 void
343 {
344  debugs(47, DBG_CRITICAL, HERE << "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
345  debugs(47, DBG_CRITICAL, HERE << "PATH " << fullPath(e.swap_filen, NULL) );
346  e.dump(0);
347 }
348 
349 bool
351 {
352 
353  struct stat sb;
354 
355  if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) {
356  debugs(47, DBG_CRITICAL, HERE << "WARNING: Missing swap file");
357  dumpEntry(e);
358  return true;
359  }
360 
361  if ((off_t)e.swap_file_sz != sb.st_size) {
362  debugs(47, DBG_CRITICAL, HERE << "WARNING: Size Mismatch. Entry size: "
363  << e.swap_file_sz << ", file size: " << sb.st_size);
364  dumpEntry(e);
365  return true;
366  }
367 
368  return false;
369 }
370 
371 void
373 {
374  int totl_kb = 0;
375  int free_kb = 0;
376  int totl_in = 0;
377  int free_in = 0;
378  int x;
379  storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1);
380  storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2);
381  storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10);
382  storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0);
383  storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
384  Math::doublePercent(currentSize(), maxSize()));
385  storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
386  map->numFilesInMap(), map->capacity(),
387  Math::intPercent(map->numFilesInMap(), map->capacity()));
388  x = fsStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
389 
390  if (0 == x) {
391  storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
392  totl_kb - free_kb,
393  totl_kb,
394  Math::intPercent(totl_kb - free_kb, totl_kb));
395  storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
396  totl_in - free_in,
397  totl_in,
398  Math::intPercent(totl_in - free_in, totl_in));
399  }
400 
401  storeAppendPrintf(&sentry, "Flags:");
402 
403  if (flags.selected)
404  storeAppendPrintf(&sentry, " SELECTED");
405 
406  if (flags.read_only)
407  storeAppendPrintf(&sentry, " READ-ONLY");
408 
409  storeAppendPrintf(&sentry, "\n");
410 
411  IO->statfs(sentry);
412 }
413 
414 void
416 {
417  /* TODO: possible options for improvement;
418  *
419  * Note that too much aggression here is not good. It means that disk
420  * controller is getting a long queue of removals to act on, along
421  * with its regular I/O queue, and that client traffic is 'paused'
422  * and growing the network I/O queue as well while the scan happens.
423  * Possibly bad knock-on effects as Squid catches up on all that.
424  *
425  * Bug 2448 may have been a sign of what can wrong. At the least it
426  * provides a test case for aggression effects in overflow conditions.
427  *
428  * - base removal limit on space saved, instead of count ?
429  *
430  * - base removal rate on a traffic speed counter ?
431  * as the purge took up more time out of the second it would grow to
432  * a graceful full pause
433  *
434  * - pass out a value to cause another event to be scheduled immediately
435  * instead of waiting a whole second more ?
436  * knock on; schedule less if all caches are under low-water
437  *
438  * - admin configurable removal rate or count ?
439  * the current numbers are arbitrary, config helps with experimental
440  * trials and future-proofing the install base.
441  * we also have this indirectly by shifting the relative positions
442  * of low-, high- water and the total capacity limit.
443  */
444 
445  // minSize() is swap_low_watermark in bytes
446  const uint64_t lowWaterSz = minSize();
447 
448  if (currentSize() < lowWaterSz) {
449  debugs(47, 5, "space still available in " << path);
450  return;
451  }
452 
453  /* We can't delete objects while rebuilding swap */
454  /* XXX each store should start maintaining as it comes online. */
456  // suppress the warnings, except once each minute
457  static int64_t lastWarn = 0;
458  int warnLevel = 3;
459  if (lastWarn+60 < squid_curtime) {
460  lastWarn = squid_curtime;
461  warnLevel = DBG_IMPORTANT;
462  }
463  debugs(47, warnLevel, StoreController::store_dirs_rebuilding << " cache_dir still rebuilding. Skip GC for " << path);
464  return;
465  }
466 
467  // maxSize() is cache_dir total size in bytes
468  const uint64_t highWaterSz = ((maxSize() * Config.Swap.highWaterMark) / 100);
469 
470  // f is percentage of 'gap' filled between low- and high-water.
471  // Used to reduced purge rate when between water markers, and
472  // to multiply it more agressively the further above high-water
473  // it reaches. But in a graceful linear growth curve.
474  double f = 1.0;
475  if (highWaterSz > lowWaterSz) {
476  // might be equal. n/0 is bad.
477  f = (double) (currentSize() - lowWaterSz) / (highWaterSz - lowWaterSz);
478  }
479 
480  // how deep to look for a single object that can be removed
481  int max_scan = (int) (f * 400.0 + 100.0);
482 
483  // try to purge only this many objects this cycle.
484  int max_remove = (int) (f * 300.0 + 20.0);
485 
486  /*
487  * This is kinda cheap, but so we need this priority hack?
488  */
489  debugs(47, 3, "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove);
490 
491  RemovalPurgeWalker *walker = repl->PurgeInit(repl, max_scan);
492 
493  int removed = 0;
494  // only purge while above low-water
495  while (currentSize() >= lowWaterSz) {
496 
497  // stop if we reached max removals for this cycle,
498  // Bug 2448 may be from this not clearing enough,
499  // but it predates the current algorithm so not sure
500  if (removed >= max_remove)
501  break;
502 
503  StoreEntry *e = walker->Next(walker);
504 
505  // stop if all objects are locked / in-use,
506  // or the cache is empty
507  if (!e)
508  break; /* no more objects */
509 
510  ++removed;
511 
512  e->release(true);
513  }
514 
515  walker->Done(walker);
516  debugs(47, (removed ? 2 : 3), path <<
517  " removed " << removed << "/" << max_remove << " f=" <<
518  std::setprecision(4) << f << " max_scan=" << max_scan);
519 
520  // what if cache is still over the high watermark ?
521  // Store::Maintain() schedules another purge in 1 second.
522 }
523 
524 void
526 {
527  debugs(47, 3, HERE << "referencing " << &e << " " <<
528  e.swap_dirn << "/" << e.swap_filen);
529 
530  if (repl->Referenced)
531  repl->Referenced(repl, &e, &e.repl);
532 }
533 
534 bool
536 {
537  debugs(47, 3, HERE << "dereferencing " << &e << " " <<
538  e.swap_dirn << "/" << e.swap_filen);
539 
540  if (repl->Dereferenced)
541  repl->Dereferenced(repl, &e, &e.repl);
542 
543  return true; // keep e in the global store_table
544 }
545 
547 Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
548 {
549  return IO->create (this, &e, file_callback, aCallback, callback_data);
550 }
551 
553 Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
554 {
555  return IO->open (this, &e, file_callback, aCallback, callback_data);
556 }
557 
558 int
560 {
561  return map->testBit(filn);
562 }
563 
564 void
566 {
567  map->setBit(filn);
568 }
569 
570 void
572 {
573  /*
574  * We have to test the bit before calling clearBit as
575  * it doesn't do bounds checking and blindly assumes
576  * filn is a valid file number, but it might not be because
577  * the map is dynamic in size. Also clearing an already clear
578  * bit puts the map counter of-of-whack.
579  */
580 
581  if (map->testBit(filn))
582  map->clearBit(filn);
583 }
584 
585 int
587 {
588  int fn;
589  fn = map->allocate(suggest);
590  map->setBit(fn);
591  suggest = fn + 1;
592  return fn;
593 }
594 
595 char *
597 {
598  LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
599  assert(0 <= subdirn && subdirn < l1);
600  snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn);
601  return fullfilename;
602 }
603 
604 int
605 Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist)
606 {
607  int created = 0;
608 
609  struct stat st;
610  getCurrentTime();
611 
612  if (0 == ::stat(aPath, &st)) {
613  if (S_ISDIR(st.st_mode)) {
614  debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists");
615  } else {
616  fatalf("Swap directory %s is not a directory.", aPath);
617  }
618  } else if (0 == mkdir(aPath, 0755)) {
619  debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created");
620  created = 1;
621  } else {
622  int xerrno = errno;
623  fatalf("Failed to make swap directory %s: %s", aPath, xstrerr(xerrno));
624  }
625 
626  return created;
627 }
628 
629 bool
631 {
632 
633  struct stat sb;
634 
635  if (::stat(aPath, &sb) < 0) {
636  int xerrno = errno;
637  debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerr(xerrno));
638  return false;
639  }
640 
641  if (S_ISDIR(sb.st_mode) == 0) {
642  debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory");
643  return false;
644  }
645 
646  return true;
647 }
648 
649 bool
651 {
652  if (!pathIsDirectory(path))
653  return true;
654 
655  for (int j = 0; j < l1; ++j) {
656  char const *aPath = swapSubDir(j);
657 
658  if (!pathIsDirectory(aPath))
659  return true;
660  }
661 
662  return false;
663 }
664 
665 void
667 {
668  LOCAL_ARRAY(char, name, MAXPATHLEN);
669 
670  for (int i = 0; i < l1; ++i) {
671  snprintf(name, MAXPATHLEN, "%s/%02X", path, i);
672 
673  int should_exist;
674 
675  if (createDirectory(name, 0))
676  should_exist = 0;
677  else
678  should_exist = 1;
679 
680  debugs(47, DBG_IMPORTANT, "Making directories in " << name);
681 
682  for (int k = 0; k < l2; ++k) {
683  snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k);
684  createDirectory(name, should_exist);
685  }
686  }
687 }
688 
689 SBuf
690 Fs::Ufs::UFSSwapDir::logFile(char const *ext) const
691 {
692  SBuf lpath;
693 
694  if (Config.Log.swap) {
695  static char pathtmp[MAXPATHLEN];
696  char *pathtmp2 = xstrncpy(pathtmp, path, MAXPATHLEN - 64);
697 
698  // replace all '/' with '.'
699  while ((pathtmp2 = strchr(pathtmp2, '/')))
700  *pathtmp2 = '.';
701 
702  // remove any trailing '.' characters
703  int pos = strlen(pathtmp);
704  while (pos && pathtmp[pos-1] == '.')
705  pathtmp[--pos] = '\0';
706 
707  // remove any prefix '.' characters
708  for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2);
709  // replace a '%s' (if any) in the config string
710  // with the resulting pathtmp2 string
711  lpath.appendf(Config.Log.swap, pathtmp2);
712 
713  // is pathtmp2 was NOT injected, append numeric file extension
714  if (lpath.cmp(Config.Log.swap) == 0) {
715  lpath.append(".", 1);
716  lpath.appendf("%02d", index);
717  }
718  } else {
719  lpath.append(path);
720  lpath.append("/swap.state", 11);
721  }
722 
723  lpath.append(ext); // may be nil, that is okay.
724 
725  return lpath;
726 }
727 
728 void
730 {
731  assert(NumberOfUFSDirs || !UFSDirToGlobalDirMapping);
732  ++NumberOfUFSDirs;
733  assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
734 
735  if (rebuilding_) { // we did not close the temporary log used for rebuilding
736  assert(swaplog_fd >= 0);
737  return;
738  }
739 
740  SBuf logPath(logFile());
741  swaplog_fd = file_open(logPath.c_str(), O_WRONLY | O_CREAT | O_BINARY);
742 
743  if (swaplog_fd < 0) {
744  int xerrno = errno;
745  debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerr(xerrno));
746  fatal("UFSSwapDir::openLog: Failed to open swap log.");
747  }
748 
749  debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd);
750 }
751 
752 void
754 {
755  if (swaplog_fd < 0) /* not open */
756  return;
757 
758  --NumberOfUFSDirs;
759  assert(NumberOfUFSDirs >= 0);
760  if (!NumberOfUFSDirs)
761  safe_free(UFSDirToGlobalDirMapping);
762 
763  if (rebuilding_) // we cannot close the temporary log used for rebuilding
764  return;
765 
766  file_close(swaplog_fd);
767 
768  debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd);
769 
770  swaplog_fd = -1;
771 }
772 
773 bool
775 {
776  return anInt < l1;
777 }
778 
779 bool
781 {
782  return anInt < l2;
783 }
784 
785 StoreEntry *
787  sfileno file_number,
788  uint64_t swap_file_sz,
789  time_t expires,
790  time_t timestamp,
791  time_t lastref,
792  time_t lastmod,
793  uint32_t refcount,
794  uint16_t newFlags,
795  int)
796 {
797  StoreEntry *e = NULL;
798  debugs(47, 5, HERE << storeKeyText(key) <<
799  ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number);
800  /* if you call this you'd better be sure file_number is not
801  * already in use! */
802  e = new StoreEntry();
803  e->store_status = STORE_OK;
805  e->attachToDisk(index, file_number, SWAPOUT_DONE);
806  e->swap_file_sz = swap_file_sz;
807  e->lastref = lastref;
808  e->timestamp = timestamp;
809  e->expires = expires;
810  e->lastModified(lastmod);
811  e->refcount = refcount;
812  e->flags = newFlags;
813  e->ping_status = PING_NONE;
815  mapBitSet(e->swap_filen);
816  cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz);
817  ++n_disk_objects;
818  e->hashInsert(key);
819  replacementAdd (e);
820  return e;
821 }
822 
823 void
825 {
827  eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1);
828 }
829 
830 void
832 {
833  assert(rebuilding_);
834  rebuilding_ = false;
835 
836  SBuf swaplog_path(logFile()); // where the swaplog should be
837  SBuf tmp_path(logFile(".new"));
838 
839  file_close(swaplog_fd);
840 
841  if (!FileRename(tmp_path, swaplog_path)) {
842  fatalf("Failed to rename log file " SQUIDSBUFPH " to " SQUIDSBUFPH, SQUIDSBUFPRINT(tmp_path), SQUIDSBUFPRINT(swaplog_path));
843  }
844 
845  int fd = file_open(swaplog_path.c_str(), O_WRONLY | O_CREAT | O_BINARY);
846 
847  if (fd < 0) {
848  int xerrno = errno;
849  debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerr(xerrno));
850  fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
851  }
852 
853  swaplog_fd = fd;
854  debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd);
855 }
856 
857 FILE *
858 Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag)
859 {
860  assert(!rebuilding_);
861 
862  SBuf swaplog_path(logFile());
863  SBuf clean_path(logFile(".last-clean"));
864  SBuf new_path(logFile(".new"));
865 
866  struct stat log_sb;
867 
868  struct stat clean_sb;
869 
870  if (::stat(swaplog_path.c_str(), &log_sb) < 0) {
871  debugs(47, DBG_IMPORTANT, "Cache Dir #" << index << ": No log file");
872  return NULL;
873  }
874 
875  *zero_flag = log_sb.st_size == 0 ? 1 : 0;
876  /* close the existing write-only FD */
877 
878  if (swaplog_fd >= 0)
879  file_close(swaplog_fd);
880 
881  /* open a write-only FD for the new log */
882  int fd = file_open(new_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
883  if (fd < 0) {
884  int xerrno = errno;
885  debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerr(xerrno));
886  fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(new_path));
887  }
888 
889  swaplog_fd = fd;
890  rebuilding_ = true;
891 
892  {
893  const StoreSwapLogHeader header;
894  MemBuf buf;
895  buf.init(header.record_size, header.record_size);
896  buf.append(reinterpret_cast<const char*>(&header), sizeof(header));
897  // Pad to keep in sync with UFSSwapDir::writeCleanStart().
898  memset(buf.space(), 0, header.gapSize());
899  buf.appended(header.gapSize());
900  file_write(swaplog_fd, -1, buf.content(), buf.contentSize(),
901  NULL, NULL, buf.freeFunc());
902  }
903 
904  /* open a read-only stream of the old log */
905  FILE *fp = fopen(swaplog_path.c_str(), "rb");
906  if (!fp) {
907  int xerrno = errno;
908  debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerr(xerrno));
909  fatalf("Failed to open swap log for reading " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
910  }
911 
912  memset(&clean_sb, '\0', sizeof(struct stat));
913 
914  if (::stat(clean_path.c_str(), &clean_sb) < 0)
915  *clean_flag = 0;
916  else if (clean_sb.st_mtime < log_sb.st_mtime)
917  *clean_flag = 0;
918  else
919  *clean_flag = 1;
920 
921  safeunlink(clean_path.c_str(), 1);
922 
923  return fp;
924 }
925 
926 /*
927  * Begin the process to write clean cache state. For AUFS this means
928  * opening some log files and allocating write buffers. Return 0 if
929  * we succeed, and assign the 'func' and 'data' return pointers.
930  */
931 int
933 {
934  UFSCleanLog *state = new UFSCleanLog(this);
935  StoreSwapLogHeader header;
936 #if HAVE_FCHMOD
937 
938  struct stat sb;
939 #endif
940 
941  cleanLog = NULL;
942  state->cur = logFile();
943  state->newLog = logFile(".clean");
944  state->fd = file_open(state->newLog.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
945 
946  if (state->fd < 0) {
947  delete state;
948  return -1;
949  }
950 
951  state->cln = state->cur;
952  state->cln.append(".last-clean");
953  state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1);
954  state->outbuf_offset = 0;
955  /*copy the header */
956  memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader));
957  // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog().
958  memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize());
959  state->outbuf_offset += header.record_size;
960 
961  state->walker = repl->WalkInit(repl);
962  ::unlink(state->cln.c_str());
963  debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd);
964 #if HAVE_FCHMOD
965 
966  if (::stat(state->cur.c_str(), &sb) == 0)
967  fchmod(state->fd, sb.st_mode);
968 
969 #endif
970 
971  cleanLog = state;
972  return 0;
973 }
974 
975 void
977 {
978  UFSCleanLog *state = (UFSCleanLog *)cleanLog;
979  int fd;
980 
981  if (NULL == state)
982  return;
983 
984  if (state->fd < 0)
985  return;
986 
987  state->walker->Done(state->walker);
988 
989  if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
990  int xerrno = errno;
991  debugs(50, DBG_CRITICAL, MYNAME << state->newLog << ": write: " << xstrerr(xerrno));
992  debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
993  file_close(state->fd);
994  state->fd = -1;
995  ::unlink(state->newLog.c_str());
996  }
997 
998  safe_free(state->outbuf);
999  /*
1000  * You can't rename open files on Microsoft "operating systems"
1001  * so we have to close before renaming.
1002  */
1003  closeLog();
1004  /* save the fd value for a later test */
1005  fd = state->fd;
1006  /* rename */
1007 
1008  if (state->fd >= 0) {
1009 #if _SQUID_OS2_ || _SQUID_WINDOWS_
1010  file_close(state->fd);
1011  state->fd = -1;
1012 #endif
1013 
1014  FileRename(state->newLog, state->cur);
1015  // TODO handle rename errors
1016  }
1017 
1018  /* touch a timestamp file if we're not still validating */
1020  (void) 0;
1021  else if (fd < 0)
1022  (void) 0;
1023  else
1024  file_close(file_open(state->cln.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
1025 
1026  /* close */
1027  if (state->fd >= 0)
1028  file_close(state->fd);
1029 
1030  state->fd = -1;
1031 
1032  delete state;
1033 
1034  cleanLog = NULL;
1035 }
1036 
1038 int
1040 {
1041  static int swap_index = 0;
1042  int i;
1043  int j = 0;
1044  int n = 0;
1045 
1046  if (!NumberOfUFSDirs)
1047  return 0; // probably in the middle of reconfiguration
1048 
1049  if (NULL == UFSDirToGlobalDirMapping) {
1050  SwapDir *sd;
1051  /*
1052  * Initialize the little array that translates UFS cache_dir
1053  * number into the Config.cacheSwap.swapDirs array index.
1054  */
1055  UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping));
1056 
1057  for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) {
1058  /* This is bogus, the controller should just clean each instance once */
1059  sd = dynamic_cast <SwapDir *>(INDEXSD(i));
1060 
1061  if (!UFSSwapDir::IsUFSDir(sd))
1062  continue;
1063 
1064  UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd);
1065 
1066  assert (usd);
1067 
1068  UFSDirToGlobalDirMapping[n] = i;
1069  ++n;
1070 
1071  j += (usd->l1 * usd->l2);
1072  }
1073 
1074  assert(n == NumberOfUFSDirs);
1075  /*
1076  * Start the commonUfsDirClean() swap_index with a random
1077  * value. j equals the total number of UFS level 2
1078  * swap directories
1079  */
1080  std::mt19937 mt(static_cast<uint32_t>(getCurrentTime() & 0xFFFFFFFF));
1081  xuniform_int_distribution<> dist(0, j);
1082  swap_index = dist(mt);
1083  }
1084 
1085  /* if the rebuild is finished, start cleaning directories. */
1087  n = DirClean(swap_index);
1088  ++swap_index;
1089  }
1090 
1091  return n;
1092 }
1093 
1094 void
1096 {
1097  const int n = HandleCleanEvent();
1098  eventAdd("storeDirClean", CleanEvent, NULL,
1099  15.0 * exp(-0.25 * n), 1);
1100 }
1101 
1102 bool
1104 {
1105  UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd);
1106  return (mySD != 0) ;
1107 }
1108 
1109 /*
1110  * XXX: this is broken - it assumes all cache dirs use the same
1111  * l1 and l2 scheme. -RBC 20021215. Partial fix is in place -
1112  * if not UFSSwapDir return 0;
1113  */
1114 bool
1116 {
1117  int D1, D2;
1118  int L1, L2;
1119  int filn = fn;
1121  assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0))));
1122  UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0));
1123 
1124  if (!sd)
1125  return 0;
1126 
1127  L1 = sd->l1;
1128 
1129  L2 = sd->l2;
1130 
1131  D1 = ((filn / L2) / L2) % L1;
1132 
1133  if (F1 != D1)
1134  return 0;
1135 
1136  D2 = (filn / L2) % L2;
1137 
1138  if (F2 != D2)
1139  return 0;
1140 
1141  return 1;
1142 }
1143 
1144 int
1146 {
1147  if (filn < 0)
1148  return 0;
1149 
1150  /*
1151  * If flag is set it means out-of-range file number should
1152  * be considered invalid.
1153  */
1154  if (flag)
1155  if (filn > map->capacity())
1156  return 0;
1157 
1158  return 1;
1159 }
1160 
1161 void
1163 {
1164  debugs(79, 3, HERE << "unlinking fileno " << std::setfill('0') <<
1165  std::hex << std::uppercase << std::setw(8) << f << " '" <<
1166  fullPath(f,NULL) << "'");
1167  /* commonUfsDirMapBitReset(this, f); */
1168  IO->unlinkFile(fullPath(f,NULL));
1169 }
1170 
1171 bool
1173 {
1174  // unlinkd may be useful only in workers
1175  return IamWorkerProcess() && IO->io->unlinkdUseful();
1176 }
1177 
1178 void
1180 {
1181  debugs(79, 3, e);
1182  if (e.locked()) // somebody else may still be using this file
1183  return; // nothing to do: our get() always returns nil
1184 
1185  if (!e.hasDisk())
1186  return; // see evictIfFound()
1187 
1188  if (e.swappedOut()) {
1189  cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
1190  --n_disk_objects;
1191  }
1192  replacementRemove(&e);
1193  mapBitReset(e.swap_filen);
1195  e.detachFromDisk();
1196 }
1197 
1198 void
1200 {
1201  // UFS disk entries always have (attached) StoreEntries so if we got here,
1202  // the entry is not cached on disk and there is nothing for us to do.
1203 }
1204 
1205 void
1207 {
1208  debugs(47, 4, HERE << "added node " << e << " to dir " << index);
1209  repl->Add(repl, e, &e->repl);
1210 }
1211 
1212 void
1214 {
1215  assert(e->hasDisk());
1216 
1217  SwapDirPointer SD = INDEXSD(e->swap_dirn);
1218 
1219  assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
1220 
1221  debugs(47, 4, HERE << "remove node " << e << " from dir " << index);
1222 
1223  repl->Remove(repl, e, &e->repl);
1224 }
1225 
1226 void
1228 {
1229  storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2);
1230  dumpOptions(&entry);
1231 }
1232 
1233 char *
1234 Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const
1235 {
1236  LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
1237  int L1 = l1;
1238  int L2 = l2;
1239 
1240  if (!fullpath)
1241  fullpath = fullfilename;
1242 
1243  fullpath[0] = '\0';
1244 
1245  snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X",
1246  path,
1247  ((filn / L2) / L2) % L1,
1248  (filn / L2) % L2,
1249  filn);
1250 
1251  return fullpath;
1252 }
1253 
1254 int
1256 {
1257  return IO->callback();
1258 }
1259 
1260 void
1262 {
1263  IO->sync();
1264 }
1265 
1266 void
1268 {
1269  cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz);
1270  ++n_disk_objects;
1271 }
1272 
1273 void
1275 {
1276  debugs(47, 5, entry);
1277  // rely on the expected subsequent StoreEntry::release(), evictCached(), or
1278  // a similar call to call unlink(), detachFromDisk(), etc. for the entry.
1279 }
1280 
1281 void
1283 {
1284  if (swaplog_fd < 0) {
1285  debugs(36, 5, "cannot log " << e << " in the middle of reconfiguration");
1286  return;
1287  }
1288 
1290  s->op = (char) op;
1291  s->swap_filen = e.swap_filen;
1292  s->timestamp = e.timestamp;
1293  s->lastref = e.lastref;
1294  s->expires = e.expires;
1295  s->lastmod = e.lastModified();
1296  s->swap_file_sz = e.swap_file_sz;
1297  s->refcount = e.refcount;
1298  s->flags = e.flags;
1299  memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH);
1300  s->finalize();
1301  file_write(swaplog_fd,
1302  -1,
1303  s,
1304  sizeof(StoreSwapLogData),
1305  NULL,
1306  NULL,
1307  FreeObject);
1308 }
1309 
1310 int
1312 {
1313  DIR *dir_pointer = NULL;
1314  int files[20];
1315  int swapfileno;
1316  int fn; /* same as swapfileno, but with dirn bits set */
1317  int n = 0;
1318  int k = 0;
1319  int N0, N1, N2;
1320  int D0, D1, D2;
1321  UFSSwapDir *SD;
1322  N0 = NumberOfUFSDirs;
1323  D0 = UFSDirToGlobalDirMapping[swap_index % N0];
1324  SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0));
1325  assert (SD);
1326  N1 = SD->l1;
1327  D1 = (swap_index / N0) % N1;
1328  N2 = SD->l2;
1329  D2 = ((swap_index / N0) / N1) % N2;
1330 
1331  SBuf p1;
1332  p1.appendf("%s/%02X/%02X", SD->path, D1, D2);
1333  debugs(36, 3, HERE << "Cleaning directory " << p1);
1334  dir_pointer = opendir(p1.c_str());
1335 
1336  if (!dir_pointer) {
1337  int xerrno = errno;
1338  if (xerrno == ENOENT) {
1339  debugs(36, DBG_CRITICAL, MYNAME << "WARNING: Creating " << p1);
1340  if (mkdir(p1.c_str(), 0777) == 0)
1341  return 0;
1342  }
1343 
1344  debugs(50, DBG_CRITICAL, MYNAME << p1 << ": " << xstrerr(xerrno));
1345  safeunlink(p1.c_str(), 1);
1346  return 0;
1347  }
1348 
1349  dirent_t *de;
1350  while ((de = readdir(dir_pointer)) != NULL && k < 20) {
1351  if (sscanf(de->d_name, "%X", &swapfileno) != 1)
1352  continue;
1353 
1354  fn = swapfileno; /* XXX should remove this cruft ! */
1355 
1356  if (SD->validFileno(fn, 1))
1357  if (SD->mapBitTest(fn))
1358  if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2))
1359  continue;
1360 
1361  files[k] = swapfileno;
1362  ++k;
1363  }
1364 
1365  closedir(dir_pointer);
1366 
1367  if (k == 0)
1368  return 0;
1369 
1370  qsort(files, k, sizeof(int), rev_int_sort);
1371 
1372  if (k > 10)
1373  k = 10;
1374 
1375  for (n = 0; n < k; ++n) {
1376  debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]);
1377  SBuf p2(p1);
1378  p2.appendf("/%08X", files[n]);
1379  safeunlink(p2.c_str(), 0);
1381  }
1382 
1383  debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1);
1384  return k;
1385 }
1386 
RemovalPolicy * createRemovalPolicy(RemovalPolicySettings *settings)
Definition: store.cc:1727
CleanLog * cleanLog
Definition: Disk.h:140
FREE * freeFunc()
Definition: MemBuf.cc:312
sdirno swap_dirn
Definition: Store.h:214
StoreEntry *(* Next)(RemovalPurgeWalker *walker)
Definition: RemovalPolicy.h:74
void detachFromDisk()
Definition: store.cc:2022
int GetInteger(void)
Definition: Parsing.cc:142
static int store_dirs_rebuilding
the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
Definition: Controller.h:139
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const override
check whether we can store the entry; if we can, report current load
Definition: UFSSwapDir.cc:116
StatCounters statCounter
Definition: StatCounters.cc:12
char * swap
Definition: SquidConfig.h:179
#define assert(EX)
Definition: assert.h:17
virtual char const * type() const =0
virtual ConfigOption * getOptionTree() const override
Definition: UFSSwapDir.cc:251
SBuf & appendf(const char *fmt,...)
Definition: SBuf.cc:239
virtual void writeCleanDone() override
Definition: UFSSwapDir.cc:976
virtual void evictIfFound(const cache_key *) override
Definition: UFSSwapDir.cc:1199
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: UFSSwapDir.cc:415
void finalize()
call this before storing the log entry
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:216
virtual DiskIOStrategy * createStrategy()=0
void release(const bool shareable=false)
Definition: store.cc:1189
void safeunlink(const char *s, int quiet)
Definition: fs_io.cc:497
int createDirectory(const char *path, int)
Definition: UFSSwapDir.cc:605
virtual bool unlinkdUseful() const override
whether SwapDir may benefit from unlinkd
Definition: UFSSwapDir.cc:1172
void(* Done)(RemovalPurgeWalker *walker)
Definition: RemovalPolicy.h:75
Definition: SBuf.h:86
static uint32 B
Definition: md4.c:43
unsigned char cache_key
Store key.
Definition: forward.h:29
#define xcalloc
Definition: membanger.c:57
virtual void reference(StoreEntry &) override
somebody needs this entry (many cache replacement policies need to know)
Definition: UFSSwapDir.cc:525
void self_destruct(void)
Definition: cache_cf.cc:257
virtual bool doubleCheck(StoreEntry &) override
Definition: UFSSwapDir.cc:350
char * fn
Definition: membanger.c:36
int i
Definition: membanger.c:49
#define xstrdup
SBuf & append(const SBuf &S)
Definition: SBuf.cc:195
void attachToDisk(const sdirno, const sfileno, const swap_status_t)
Definition: store.cc:2008
#define MAXPATHLEN
Definition: stdio.h:62
virtual void closeLog() override
Definition: UFSSwapDir.cc:753
void lastModified(const time_t when)
Definition: Store.h:157
#define safe_free(x)
Definition: xalloc.h:73
const char * storeKeyText(const cache_key *key)
UFSSwapDir(char const *aType, const char *aModuleType)
Definition: UFSSwapDir.cc:308
bool hasDisk(const sdirno dirn=-1, const sfileno filen=-1) const
Definition: store.cc:1997
static struct stat sb
Definition: squidclient.cc:70
int file_open(const char *path, int mode)
Definition: fs_io.cc:46
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:212
#define DBG_CRITICAL
Definition: Debug.h:45
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override
Definition: UFSSwapDir.cc:553
time_t expires
Definition: Store.h:202
StoreEntry * addDiskRestore(const cache_key *key, sfileno file_number, uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, time_t lastmod, uint32_t refcount, uint16_t flags, int clean)
Definition: UFSSwapDir.cc:786
virtual int load()
Definition: UFSStrategy.cc:25
virtual void write(StoreEntry const &)
&quot;write&quot; an entry to the clean log file.
Definition: UFSSwapDir.cc:78
bool validL1(int) const
Definition: UFSSwapDir.cc:774
virtual int writeCleanStart() override
Definition: UFSSwapDir.cc:932
bool IamWorkerProcess()
whether the current process handles HTTP transactions and such
Definition: stub_tools.cc:49
time_t squid_curtime
Definition: stub_time.cc:17
time_t timestamp
Definition: Store.h:200
void replacementAdd(StoreEntry *e)
Definition: UFSSwapDir.cc:1206
char * path
Definition: Disk.h:102
struct StatCounters::@137 swap
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:96
void fatalf(const char *fmt,...)
Definition: fatal.cc:68
double doublePercent(const double, const double)
Definition: SquidMath.cc:25
int locked() const
Definition: store.cc:1249
uint64_t swap_file_sz
Definition: Store.h:206
static DiskIOModule * Find(char const *type)
const char * xstrerr(int error)
Definition: xstrerror.cc:83
ping_status_t ping_status
Definition: Store.h:218
void hashInsert(const cache_key *)
Definition: store.cc:412
char * outbuf
Definition: UFSSwapDir.cc:59
int n_disk_objects
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
void file_write(int fd, off_t file_offset, void const *ptr_to_buf, int len, DWCB *handle, void *handle_data, FREE *free_func)
Definition: fs_io.cc:344
int fsBlockSize(const char *path, int *blksize)
Definition: fs_io.cc:531
unsigned char key[SQUID_MD5_DIGEST_LENGTH]
char * space()
returns buffer after data; does not check space existence
Definition: MemBuf.h:57
virtual const StoreEntry * nextEntry()
Get the next entry that is a candidate for clean log writing.
Definition: UFSSwapDir.cc:67
#define DBG_IMPORTANT
Definition: Debug.h:46
virtual bool dereference(StoreEntry &) override
Definition: UFSSwapDir.cc:535
static int HandleCleanEvent()
safely cleans a few unused files if possible
Definition: UFSSwapDir.cc:1039
struct SquidConfig::@103 Log
void optionIODump(StoreEntry *e) const
Definition: UFSSwapDir.cc:245
#define SQUID_MD5_DIGEST_LENGTH
Definition: md5.h:66
#define EBIT_CLR(flag, bit)
Definition: defines.h:106
virtual void dump(StoreEntry &) const override
Definition: UFSSwapDir.cc:1227
void STFNCB(void *their_data, int errflag, StoreIOState::Pointer self)
Definition: StoreIOState.h:41
FILE * fp
Definition: membanger.c:35
RemovalPolicyWalker * walker
Definition: UFSSwapDir.cc:62
static bool IsUFSDir(SwapDir *sd)
Definition: UFSSwapDir.cc:1103
void unlinkFile(sfileno f)
Definition: UFSSwapDir.cc:1162
static EVH RebuildStep
Definition: RebuildState.h:29
virtual void sync() override
prepare for shutdown
Definition: UFSSwapDir.cc:1261
SwappedTime timestamp
char * xstrncpy(char *dst, const char *src, size_t n)
Definition: xstring.cc:37
static void FreeObject(void *address)
Definition: UFSSwapDir.cc:129
void fatal(const char *message)
Definition: fatal.cc:28
void dump(int debug_lvl) const
Definition: store.cc:1533
virtual void finalizeSwapoutFailure(StoreEntry &) override
abort the failed swapout that has been already noticed by Store
Definition: UFSSwapDir.cc:1274
char * swapSubDir(int subdirn) const
Definition: UFSSwapDir.cc:596
static int NumberOfUFSDirs
Definition: UFSSwapDir.h:126
const char * c_str()
Definition: SBuf.cc:526
uint16_t flags
Definition: Store.h:208
virtual bool shedLoad()
Definition: UFSStrategy.cc:19
void replacementRemove(StoreEntry *e)
Definition: UFSSwapDir.cc:1213
RemovalPolicyNode repl
Definition: Store.h:198
bool pathIsDirectory(const char *path) const
Definition: UFSSwapDir.cc:630
signed_int32_t sfileno
Definition: forward.h:22
virtual void evictCached(StoreEntry &) override
Definition: UFSSwapDir.cc:1179
size_t gapSize() const
number of bytes after the log header before the first log entry
#define LOCAL_ARRAY(type, name, size)
Definition: leakcheck.h:18
bool FileRename(const SBuf &from, const SBuf &to)
Definition: fs_io.cc:508
#define INDEXSD(i)
Definition: SquidConfig.h:68
char * content()
start of the added data
Definition: MemBuf.h:41
uint16_t refcount
Definition: Store.h:207
void appended(mb_size_t sz)
updates content size after external append
Definition: MemBuf.cc:235
void const char * buf
Definition: stub_helper.cc:16
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:153
static uint32 A
Definition: md4.c:43
SwapDir * sd
Definition: UFSSwapDir.cc:63
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
#define CLEAN_BUF_SZ
Definition: UFSSwapDir.cc:11
void mapBitSet(sfileno filn)
Definition: UFSSwapDir.cc:565
Definition: enums.h:46
bool SIGHDLR int STUB void int
Definition: stub_tools.cc:68
bool validL2(int) const
Definition: UFSSwapDir.cc:780
#define dirent_t
Definition: compat_shared.h:71
void STIOCB(void *their_data, int errflag, StoreIOState::Pointer self)
Definition: StoreIOState.h:51
time_t getCurrentTime(void)
Get current time.
int highWaterMark
Definition: SquidConfig.h:79
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:109
#define PRIu64
Definition: types.h:120
off_t outbuf_offset
Definition: UFSSwapDir.cc:60
store_status_t store_status
Definition: Store.h:220
#define MYNAME
Definition: Debug.h:166
int validFileno(sfileno filn, int flag) const
Definition: UFSSwapDir.cc:1145
char * fullPath(sfileno, char *) const
Definition: UFSSwapDir.cc:1234
void dumpEntry(StoreEntry &) const
Definition: UFSSwapDir.cc:342
int mapBitTest(sfileno filn)
Definition: UFSSwapDir.cc:559
static bool FilenoBelongsHere(int fn, int cachedir, int level1dir, int level2dir)
Definition: UFSSwapDir.cc:1115
static int rev_int_sort(const void *A, const void *B)
Definition: UFSSwapDir.cc:136
RemovalPolicySettings * replPolicy
Definition: SquidConfig.h:93
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:422
static int * UFSDirToGlobalDirMapping
Definition: UFSSwapDir.h:127
int intPercent(const int a, const int b)
Definition: SquidMath.cc:13
int fsStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
Definition: fs_io.cc:555
virtual void statfs(StoreEntry &) const override
Definition: UFSSwapDir.cc:372
const StoreEntry *(* Next)(RemovalPolicyWalker *walker)
Definition: RemovalPolicy.h:62
#define F1(x, y, z)
Definition: md5.c:166
virtual void reconfigure() override
Definition: UFSSwapDir.cc:174
int cmp(const SBuf &S, const size_type n) const
shorthand version for compare()
Definition: SBuf.h:264
void mapBitReset(sfileno filn)
Definition: UFSSwapDir.cc:571
virtual void parse(int index, char *path) override
Definition: UFSSwapDir.cc:181
Definition: MemBuf.h:23
virtual ~UFSSwapDir()
Definition: UFSSwapDir.cc:329
static EVH CleanEvent
Definition: UFSSwapDir.h:130
virtual void logEntry(const StoreEntry &e, int op) const override
Definition: UFSSwapDir.cc:1282
virtual ConfigOption * getOptionTree() const
Definition: Disk.cc:258
SBuf logFile(char const *ext=nullptr) const
Definition: UFSSwapDir.cc:690
static int DirClean(int swap_index)
Definition: UFSSwapDir.cc:1311
time_t lastref
Definition: Store.h:201
Fs::Ufs::UFSStrategy * IO
Definition: UFSSwapDir.h:83
virtual void openLog() override
Definition: UFSSwapDir.cc:729
#define O_BINARY
Definition: defines.h:204
#define SQUIDSBUFPH
Definition: SBuf.h:31
#define FD_WRITE_METHOD(fd, buf, len)
Definition: fde.h:162
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const =0
check whether we can store the entry; if we can, report current load
Definition: Disk.cc:164
virtual void init() override
Definition: UFSSwapDir.cc:273
#define xfree
void file_close(int fd)
Definition: fs_io.cc:76
UFSCleanLog(SwapDir *aSwapDir)
Definition: UFSSwapDir.cc:48
void changeIO(DiskIOModule *)
Definition: UFSSwapDir.cc:195
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
bool swappedOut() const
whether the entire entry is now on disk (possibly marked for deletion)
Definition: Store.h:121
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:875
struct SquidConfig::@96 Swap
virtual int callback() override
called once every main loop iteration; TODO: Move to UFS code.
Definition: UFSSwapDir.cc:1255
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override
Definition: UFSSwapDir.cc:547
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
bool optionIOParse(char const *option, const char *value, int reconfiguring)
Definition: UFSSwapDir.cc:218
void(* Done)(RemovalPolicyWalker *walker)
Definition: RemovalPolicy.h:63
int reconfiguring
virtual void create() override
create system resources needed for this store to operate in the future
Definition: UFSSwapDir.cc:301
int size
Definition: ModDevPoll.cc:77
virtual void finalizeSwapoutSuccess(const StoreEntry &) override
finalize the successful swapout that has been already noticed by Store
Definition: UFSSwapDir.cc:1267
#define F2(x, y, z)
Definition: md5.c:167
#define false
Definition: GnuRegex.c:233
manages a single cache_dir
Definition: Disk.h:21
void setMemStatus(mem_status_t)
Definition: store.cc:1558
FILE * openTmpSwapLog(int *clean_flag, int *zero_flag)
Definition: UFSSwapDir.cc:858

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors