UFSSwapDir.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 47 Store Directory Routines */
10 
11 #define CLEAN_BUF_SZ 16384
12 
13 #include "squid.h"
14 #include "cache_cf.h"
15 #include "ConfigOption.h"
16 #include "DiskIO/DiskIOModule.h"
17 #include "DiskIO/DiskIOStrategy.h"
18 #include "fde.h"
19 #include "FileMap.h"
20 #include "fs_io.h"
21 #include "globals.h"
22 #include "Parsing.h"
23 #include "RebuildState.h"
24 #include "SquidConfig.h"
25 #include "SquidMath.h"
26 #include "SquidTime.h"
27 #include "StatCounters.h"
28 #include "store_key_md5.h"
29 #include "StoreSearchUFS.h"
30 #include "StoreSwapLogData.h"
31 #include "tools.h"
32 #include "UFSSwapDir.h"
33 
34 #include <cerrno>
35 #include <cmath>
36 #include <random>
37 #if HAVE_SYS_STAT_H
38 #include <sys/stat.h>
39 #endif
40 
43 
45 {
46 
47 public:
51  virtual const StoreEntry *nextEntry();
54  virtual void write(StoreEntry const &);
55  char *cur;
56  char *newLog;
57  char *cln;
58  char *outbuf;
60  int fd;
63 };
64 
66  cur(NULL), newLog(NULL), cln(NULL), outbuf(NULL),
67  outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir)
68 {}
69 
70 const StoreEntry *
72 {
73  const StoreEntry *entry = NULL;
74 
75  if (walker)
76  entry = walker->Next(walker);
77 
78  return entry;
79 }
80 
81 void
83 {
85  static size_t ss = sizeof(StoreSwapLogData);
86  s.op = (char) SWAP_LOG_ADD;
87  s.swap_filen = e.swap_filen;
88  s.timestamp = e.timestamp;
89  s.lastref = e.lastref;
90  s.expires = e.expires;
91  s.lastmod = e.lastModified();
93  s.refcount = e.refcount;
94  s.flags = e.flags;
95  memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH);
96  s.finalize();
97  memcpy(outbuf + outbuf_offset, &s, ss);
98  outbuf_offset += ss;
99  /* buffered write */
100 
101  if (outbuf_offset + ss >= CLEAN_BUF_SZ) {
102  if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) {
103  int xerrno = errno;
104  /* XXX This error handling should probably move up to the caller */
105  debugs(50, DBG_CRITICAL, MYNAME << newLog << ": write: " << xstrerr(xerrno));
106  debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
107  file_close(fd);
108  fd = -1;
109  unlink(newLog);
110  sd->cleanLog = NULL;
111  delete this;
112  return;
113  }
114 
115  outbuf_offset = 0;
116  }
117 }
118 
119 bool
120 Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
121 {
122  if (!SwapDir::canStore(e, diskSpaceNeeded, load))
123  return false;
124 
125  if (IO->shedLoad())
126  return false;
127 
128  load = IO->load();
129  return true;
130 }
131 
132 static void
133 FreeObject(void *address)
134 {
135  StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address);
136  delete anObject;
137 }
138 
139 static int
140 rev_int_sort(const void *A, const void *B)
141 {
142  const int *i1 = (const int *)A;
143  const int *i2 = (const int *)B;
144  return *i2 - *i1;
145 }
146 
147 void
149 {
150  int i = GetInteger();
151  if (i <= 0)
152  fatal("UFSSwapDir::parseSizeL1L2: invalid size value");
153 
154  const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes
155 
156  /* just reconfigure it */
157  if (reconfiguring) {
158  if (size == maxSize())
159  debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB");
160  else
161  debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB");
162  }
163 
164  max_size = size;
165 
166  l1 = GetInteger();
167 
168  if (l1 <= 0)
169  fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value");
170 
171  l2 = GetInteger();
172 
173  if (l2 <= 0)
174  fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value");
175 }
176 
177 void
179 {
180  parseSizeL1L2();
181  parseOptions(1);
182 }
183 
184 void
185 Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath)
186 {
187  index = anIndex;
188  path = xstrdup(aPath);
189 
190  parseSizeL1L2();
191 
192  /* Initialise replacement policy stuff */
194 
195  parseOptions(0);
196 }
197 
198 void
200 {
201  DiskIOStrategy *anIO = module->createStrategy();
202  safe_free(ioType);
203  ioType = xstrdup(module->type());
204 
205  delete IO->io;
206  IO->io = anIO;
207  /* Change the IO Options */
208 
209  if (currentIOOptions && currentIOOptions->options.size() > 2) {
210  delete currentIOOptions->options.back();
211  currentIOOptions->options.pop_back();
212  }
213 
214  /* TODO: factor out these 4 lines */
215  ConfigOption *ioOptions = IO->io->getOptionTree();
216 
217  if (currentIOOptions && ioOptions)
218  currentIOOptions->options.push_back(ioOptions);
219 }
220 
221 bool
222 Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig)
223 {
224  if (strcmp(option, "IOEngine") != 0)
225  return false;
226 
227  if (isaReconfig)
228  /* silently ignore this */
229  return true;
230 
231  if (!value) {
232  self_destruct();
233  return false;
234  }
235 
236  DiskIOModule *module = DiskIOModule::Find(value);
237 
238  if (!module) {
239  self_destruct();
240  return false;
241  }
242 
243  changeIO(module);
244 
245  return true;
246 }
247 
248 void
250 {
251  storeAppendPrintf(e, " IOEngine=%s", ioType);
252 }
253 
254 ConfigOption *
256 {
257  ConfigOption *parentResult = SwapDir::getOptionTree();
258 
259  if (currentIOOptions == NULL)
260  currentIOOptions = new ConfigOptionVector();
261 
262  currentIOOptions->options.push_back(parentResult);
263 
264  currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump));
265 
266  if (ConfigOption *ioOptions = IO->io->getOptionTree())
267  currentIOOptions->options.push_back(ioOptions);
268 
269  ConfigOption* result = currentIOOptions;
270 
271  currentIOOptions = NULL;
272 
273  return result;
274 }
275 
276 void
278 {
279  debugs(47, 3, HERE << "Initialising UFS SwapDir engine.");
280  /* Parsing must be finished by now - force to NULL, don't delete */
281  currentIOOptions = NULL;
282  static int started_clean_event = 0;
283  static const char *errmsg =
284  "\tFailed to verify one of the swap directories, Check cache.log\n"
285  "\tfor details. Run 'squid -z' to create swap directories\n"
286  "\tif needed, or if running Squid for the first time.";
287  IO->init();
288 
289  if (verifyCacheDirs())
290  fatal(errmsg);
291 
292  openLog();
293 
294  rebuild();
295 
296  if (!started_clean_event) {
297  eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1);
298  started_clean_event = 1;
299  }
300 
301  (void) fsBlockSize(path, &fs.blksize);
302 }
303 
304 void
306 {
307  debugs(47, 3, "Creating swap space in " << path);
308  createDirectory(path, 0);
309  createSwapSubDirs();
310 }
311 
312 Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) :
313  SwapDir(aType),
314  IO(NULL),
315  fsdata(NULL),
316  map(new FileMap()),
317  suggest(0),
318  l1(16),
319  l2(256),
320  swaplog_fd(-1),
321  currentIOOptions(new ConfigOptionVector()),
322  ioType(xstrdup(anIOType)),
323  cur_size(0),
324  n_disk_objects(0),
325  rebuilding_(false)
326 {
327  /* modulename is only set to disk modules that are built, by configure,
328  * so the Find call should never return NULL here.
329  */
330  IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy());
331 }
332 
334 {
335  if (swaplog_fd > -1) {
336  file_close(swaplog_fd);
337  swaplog_fd = -1;
338  }
339  xfree(ioType);
340  delete map;
341  delete IO;
342  delete currentIOOptions;
343 }
344 
345 void
347 {
348  debugs(47, DBG_CRITICAL, HERE << "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
349  debugs(47, DBG_CRITICAL, HERE << "PATH " << fullPath(e.swap_filen, NULL) );
350  e.dump(0);
351 }
352 
353 bool
355 {
356 
357  struct stat sb;
358 
359  if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) {
360  debugs(47, DBG_CRITICAL, HERE << "WARNING: Missing swap file");
361  dumpEntry(e);
362  return true;
363  }
364 
365  if ((off_t)e.swap_file_sz != sb.st_size) {
366  debugs(47, DBG_CRITICAL, HERE << "WARNING: Size Mismatch. Entry size: "
367  << e.swap_file_sz << ", file size: " << sb.st_size);
368  dumpEntry(e);
369  return true;
370  }
371 
372  return false;
373 }
374 
375 void
377 {
378  int totl_kb = 0;
379  int free_kb = 0;
380  int totl_in = 0;
381  int free_in = 0;
382  int x;
383  storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1);
384  storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2);
385  storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10);
386  storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0);
387  storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
388  Math::doublePercent(currentSize(), maxSize()));
389  storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
390  map->numFilesInMap(), map->capacity(),
391  Math::intPercent(map->numFilesInMap(), map->capacity()));
392  x = fsStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
393 
394  if (0 == x) {
395  storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
396  totl_kb - free_kb,
397  totl_kb,
398  Math::intPercent(totl_kb - free_kb, totl_kb));
399  storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
400  totl_in - free_in,
401  totl_in,
402  Math::intPercent(totl_in - free_in, totl_in));
403  }
404 
405  storeAppendPrintf(&sentry, "Flags:");
406 
407  if (flags.selected)
408  storeAppendPrintf(&sentry, " SELECTED");
409 
410  if (flags.read_only)
411  storeAppendPrintf(&sentry, " READ-ONLY");
412 
413  storeAppendPrintf(&sentry, "\n");
414 
415  IO->statfs(sentry);
416 }
417 
418 void
420 {
421  /* TODO: possible options for improvement;
422  *
423  * Note that too much aggression here is not good. It means that disk
424  * controller is getting a long queue of removals to act on, along
425  * with its regular I/O queue, and that client traffic is 'paused'
426  * and growing the network I/O queue as well while the scan happens.
427  * Possibly bad knock-on effects as Squid catches up on all that.
428  *
429  * Bug 2448 may have been a sign of what can wrong. At the least it
430  * provides a test case for aggression effects in overflow conditions.
431  *
432  * - base removal limit on space saved, instead of count ?
433  *
434  * - base removal rate on a traffic speed counter ?
435  * as the purge took up more time out of the second it would grow to
436  * a graceful full pause
437  *
438  * - pass out a value to cause another event to be scheduled immediately
439  * instead of waiting a whole second more ?
440  * knock on; schedule less if all caches are under low-water
441  *
442  * - admin configurable removal rate or count ?
443  * the current numbers are arbitrary, config helps with experimental
444  * trials and future-proofing the install base.
445  * we also have this indirectly by shifting the relative positions
446  * of low-, high- water and the total capacity limit.
447  */
448 
449  // minSize() is swap_low_watermark in bytes
450  const uint64_t lowWaterSz = minSize();
451 
452  if (currentSize() < lowWaterSz) {
453  debugs(47, 5, "space still available in " << path);
454  return;
455  }
456 
457  /* We can't delete objects while rebuilding swap */
458  /* XXX each store should start maintaining as it comes online. */
460  // suppress the warnings, except once each minute
461  static int64_t lastWarn = 0;
462  int warnLevel = 3;
463  if (lastWarn+60 < squid_curtime) {
464  lastWarn = squid_curtime;
465  warnLevel = DBG_IMPORTANT;
466  }
467  debugs(47, warnLevel, StoreController::store_dirs_rebuilding << " cache_dir still rebuilding. Skip GC for " << path);
468  return;
469  }
470 
471  // maxSize() is cache_dir total size in bytes
472  const uint64_t highWaterSz = ((maxSize() * Config.Swap.highWaterMark) / 100);
473 
474  // f is percentage of 'gap' filled between low- and high-water.
475  // Used to reduced purge rate when between water markers, and
476  // to multiply it more agressively the further above high-water
477  // it reaches. But in a graceful linear growth curve.
478  double f = 1.0;
479  if (highWaterSz > lowWaterSz) {
480  // might be equal. n/0 is bad.
481  f = (double) (currentSize() - lowWaterSz) / (highWaterSz - lowWaterSz);
482  }
483 
484  // how deep to look for a single object that can be removed
485  int max_scan = (int) (f * 400.0 + 100.0);
486 
487  // try to purge only this many objects this cycle.
488  int max_remove = (int) (f * 300.0 + 20.0);
489 
490  /*
491  * This is kinda cheap, but so we need this priority hack?
492  */
493  debugs(47, 3, "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove);
494 
495  RemovalPurgeWalker *walker = repl->PurgeInit(repl, max_scan);
496 
497  int removed = 0;
498  // only purge while above low-water
499  while (currentSize() >= lowWaterSz) {
500 
501  // stop if we reached max removals for this cycle,
502  // Bug 2448 may be from this not clearing enough,
503  // but it predates the current algorithm so not sure
504  if (removed >= max_remove)
505  break;
506 
507  StoreEntry *e = walker->Next(walker);
508 
509  // stop if all objects are locked / in-use,
510  // or the cache is empty
511  if (!e)
512  break; /* no more objects */
513 
514  ++removed;
515 
516  e->release();
517  }
518 
519  walker->Done(walker);
520  debugs(47, (removed ? 2 : 3), path <<
521  " removed " << removed << "/" << max_remove << " f=" <<
522  std::setprecision(4) << f << " max_scan=" << max_scan);
523 
524  // what if cache is still over the high watermark ?
525  // Store::Maintain() schedules another purge in 1 second.
526 }
527 
528 void
530 {
531  debugs(47, 3, HERE << "referencing " << &e << " " <<
532  e.swap_dirn << "/" << e.swap_filen);
533 
534  if (repl->Referenced)
535  repl->Referenced(repl, &e, &e.repl);
536 }
537 
538 bool
540 {
541  debugs(47, 3, HERE << "dereferencing " << &e << " " <<
542  e.swap_dirn << "/" << e.swap_filen);
543 
544  if (repl->Dereferenced)
545  repl->Dereferenced(repl, &e, &e.repl);
546 
547  return true; // keep e in the global store_table
548 }
549 
551 Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
552 {
553  return IO->create (this, &e, file_callback, aCallback, callback_data);
554 }
555 
557 Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
558 {
559  return IO->open (this, &e, file_callback, aCallback, callback_data);
560 }
561 
562 int
564 {
565  return map->testBit(filn);
566 }
567 
568 void
570 {
571  map->setBit(filn);
572 }
573 
574 void
576 {
577  /*
578  * We have to test the bit before calling clearBit as
579  * it doesn't do bounds checking and blindly assumes
580  * filn is a valid file number, but it might not be because
581  * the map is dynamic in size. Also clearing an already clear
582  * bit puts the map counter of-of-whack.
583  */
584 
585  if (map->testBit(filn))
586  map->clearBit(filn);
587 }
588 
589 int
591 {
592  int fn;
593  fn = map->allocate(suggest);
594  map->setBit(fn);
595  suggest = fn + 1;
596  return fn;
597 }
598 
599 char *
601 {
602  LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
603  assert(0 <= subdirn && subdirn < l1);
604  snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn);
605  return fullfilename;
606 }
607 
608 int
609 Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist)
610 {
611  int created = 0;
612 
613  struct stat st;
614  getCurrentTime();
615 
616  if (0 == ::stat(aPath, &st)) {
617  if (S_ISDIR(st.st_mode)) {
618  debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists");
619  } else {
620  fatalf("Swap directory %s is not a directory.", aPath);
621  }
622  } else if (0 == mkdir(aPath, 0755)) {
623  debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created");
624  created = 1;
625  } else {
626  int xerrno = errno;
627  fatalf("Failed to make swap directory %s: %s", aPath, xstrerr(xerrno));
628  }
629 
630  return created;
631 }
632 
633 bool
635 {
636 
637  struct stat sb;
638 
639  if (::stat(aPath, &sb) < 0) {
640  int xerrno = errno;
641  debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerr(xerrno));
642  return false;
643  }
644 
645  if (S_ISDIR(sb.st_mode) == 0) {
646  debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory");
647  return false;
648  }
649 
650  return true;
651 }
652 
653 bool
655 {
656  if (!pathIsDirectory(path))
657  return true;
658 
659  for (int j = 0; j < l1; ++j) {
660  char const *aPath = swapSubDir(j);
661 
662  if (!pathIsDirectory(aPath))
663  return true;
664  }
665 
666  return false;
667 }
668 
669 void
671 {
672  LOCAL_ARRAY(char, name, MAXPATHLEN);
673 
674  for (int i = 0; i < l1; ++i) {
675  snprintf(name, MAXPATHLEN, "%s/%02X", path, i);
676 
677  int should_exist;
678 
679  if (createDirectory(name, 0))
680  should_exist = 0;
681  else
682  should_exist = 1;
683 
684  debugs(47, DBG_IMPORTANT, "Making directories in " << name);
685 
686  for (int k = 0; k < l2; ++k) {
687  snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k);
688  createDirectory(name, should_exist);
689  }
690  }
691 }
692 
693 char *
694 Fs::Ufs::UFSSwapDir::logFile(char const *ext) const
695 {
696  LOCAL_ARRAY(char, lpath, MAXPATHLEN);
697  LOCAL_ARRAY(char, pathtmp, MAXPATHLEN);
698  LOCAL_ARRAY(char, digit, 32);
699  char *pathtmp2;
700 
701  if (Config.Log.swap) {
702  xstrncpy(pathtmp, path, MAXPATHLEN - 64);
703  pathtmp2 = pathtmp;
704 
705  while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
706  *pathtmp2 = '.';
707 
708  while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
709  pathtmp[strlen(pathtmp) - 1] = '\0';
710 
711  for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2);
712  snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
713 
714  if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) {
715  strcat(lpath, ".");
716  snprintf(digit, 32, "%02d", index);
717  strncat(lpath, digit, 3);
718  }
719  } else {
720  xstrncpy(lpath, path, MAXPATHLEN - 64);
721  strcat(lpath, "/swap.state");
722  }
723 
724  if (ext)
725  strncat(lpath, ext, 16);
726 
727  return lpath;
728 }
729 
730 void
732 {
733  assert(NumberOfUFSDirs || !UFSDirToGlobalDirMapping);
734  ++NumberOfUFSDirs;
735  assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
736 
737  if (rebuilding_) { // we did not close the temporary log used for rebuilding
738  assert(swaplog_fd >= 0);
739  return;
740  }
741 
742  char *logPath;
743  logPath = logFile();
744  swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY);
745 
746  if (swaplog_fd < 0) {
747  int xerrno = errno;
748  debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerr(xerrno));
749  fatal("UFSSwapDir::openLog: Failed to open swap log.");
750  }
751 
752  debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd);
753 }
754 
755 void
757 {
758  if (swaplog_fd < 0) /* not open */
759  return;
760 
761  --NumberOfUFSDirs;
762  assert(NumberOfUFSDirs >= 0);
763  if (!NumberOfUFSDirs)
764  safe_free(UFSDirToGlobalDirMapping);
765 
766  if (rebuilding_) // we cannot close the temporary log used for rebuilding
767  return;
768 
769  file_close(swaplog_fd);
770 
771  debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd);
772 
773  swaplog_fd = -1;
774 }
775 
776 bool
778 {
779  return anInt < l1;
780 }
781 
782 bool
784 {
785  return anInt < l2;
786 }
787 
788 StoreEntry *
790  sfileno file_number,
791  uint64_t swap_file_sz,
792  time_t expires,
793  time_t timestamp,
794  time_t lastref,
795  time_t lastmod,
796  uint32_t refcount,
797  uint16_t newFlags,
798  int)
799 {
800  StoreEntry *e = NULL;
801  debugs(47, 5, HERE << storeKeyText(key) <<
802  ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number);
803  /* if you call this you'd better be sure file_number is not
804  * already in use! */
805  e = new StoreEntry();
806  e->store_status = STORE_OK;
809  e->swap_filen = file_number;
810  e->swap_dirn = index;
811  e->swap_file_sz = swap_file_sz;
812  e->lastref = lastref;
813  e->timestamp = timestamp;
814  e->expires = expires;
815  e->lastModified(lastmod);
816  e->refcount = refcount;
817  e->flags = newFlags;
819  e->clearPrivate();
820  e->ping_status = PING_NONE;
822  mapBitSet(e->swap_filen);
823  cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz);
824  ++n_disk_objects;
825  e->hashInsert(key); /* do it after we clear KEY_PRIVATE */
826  replacementAdd (e);
827  return e;
828 }
829 
830 void
832 {
833  debugs(47, 5, HERE << *e);
834  replacementRemove(e); // checks swap_dirn so do it before we invalidate it
835  // Do not unlink the file as it might be used by a subsequent entry.
836  mapBitReset(e->swap_filen);
837  e->swap_filen = -1;
838  e->swap_dirn = -1;
839  cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz);
840  --n_disk_objects;
841 }
842 
843 void
845 {
847  eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1);
848 }
849 
850 void
852 {
853  assert(rebuilding_);
854  rebuilding_ = false;
855 
856  char *swaplog_path = xstrdup(logFile(NULL)); // where the swaplog should be
857  char *tmp_path = xstrdup(logFile(".new")); // the temporary file we have generated
858  int fd;
859  file_close(swaplog_fd);
860 
861  if (xrename(tmp_path, swaplog_path) < 0) {
862  fatalf("Failed to rename log file %s to %s", tmp_path, swaplog_path);
863  }
864 
865  fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);
866 
867  if (fd < 0) {
868  int xerrno = errno;
869  debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerr(xerrno));
870  fatalf("Failed to open swap log %s", swaplog_path);
871  }
872 
873  xfree(swaplog_path);
874  xfree(tmp_path);
875  swaplog_fd = fd;
876  debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd);
877 }
878 
879 FILE *
880 Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag)
881 {
882  assert(!rebuilding_);
883 
884  char *swaplog_path = xstrdup(logFile(NULL));
885  char *clean_path = xstrdup(logFile(".last-clean"));
886  char *new_path = xstrdup(logFile(".new"));
887 
888  struct stat log_sb;
889 
890  struct stat clean_sb;
891  FILE *fp;
892  int fd;
893 
894  if (::stat(swaplog_path, &log_sb) < 0) {
895  debugs(47, DBG_IMPORTANT, "Cache Dir #" << index << ": No log file");
896  safe_free(swaplog_path);
897  safe_free(clean_path);
898  safe_free(new_path);
899  return NULL;
900  }
901 
902  *zero_flag = log_sb.st_size == 0 ? 1 : 0;
903  /* close the existing write-only FD */
904 
905  if (swaplog_fd >= 0)
906  file_close(swaplog_fd);
907 
908  /* open a write-only FD for the new log */
909  fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
910 
911  if (fd < 0) {
912  int xerrno = errno;
913  debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerr(xerrno));
914  fatalf("Failed to open swap log %s", new_path);
915  }
916 
917  swaplog_fd = fd;
918  rebuilding_ = true;
919 
920  {
921  const StoreSwapLogHeader header;
922  MemBuf buf;
923  buf.init(header.record_size, header.record_size);
924  buf.append(reinterpret_cast<const char*>(&header), sizeof(header));
925  // Pad to keep in sync with UFSSwapDir::writeCleanStart().
926  memset(buf.space(), 0, header.gapSize());
927  buf.appended(header.gapSize());
928  file_write(swaplog_fd, -1, buf.content(), buf.contentSize(),
929  NULL, NULL, buf.freeFunc());
930  }
931 
932  /* open a read-only stream of the old log */
933  fp = fopen(swaplog_path, "rb");
934 
935  if (!fp) {
936  int xerrno = errno;
937  debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerr(xerrno));
938  fatalf("Failed to open swap log for reading %s", swaplog_path);
939  }
940 
941  memset(&clean_sb, '\0', sizeof(struct stat));
942 
943  if (::stat(clean_path, &clean_sb) < 0)
944  *clean_flag = 0;
945  else if (clean_sb.st_mtime < log_sb.st_mtime)
946  *clean_flag = 0;
947  else
948  *clean_flag = 1;
949 
950  safeunlink(clean_path, 1);
951 
952  safe_free(swaplog_path);
953 
954  safe_free(clean_path);
955 
956  safe_free(new_path);
957 
958  return fp;
959 }
960 
961 /*
962  * Begin the process to write clean cache state. For AUFS this means
963  * opening some log files and allocating write buffers. Return 0 if
964  * we succeed, and assign the 'func' and 'data' return pointers.
965  */
966 int
968 {
969  UFSCleanLog *state = new UFSCleanLog(this);
970  StoreSwapLogHeader header;
971 #if HAVE_FCHMOD
972 
973  struct stat sb;
974 #endif
975 
976  cleanLog = NULL;
977  state->newLog = xstrdup(logFile(".clean"));
978  state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
979 
980  if (state->fd < 0) {
981  xfree(state->newLog);
982  delete state;
983  return -1;
984  }
985 
986  state->cur = xstrdup(logFile(NULL));
987  state->cln = xstrdup(logFile(".last-clean"));
988  state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1);
989  state->outbuf_offset = 0;
990  /*copy the header */
991  memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader));
992  // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog().
993  memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize());
994  state->outbuf_offset += header.record_size;
995 
996  state->walker = repl->WalkInit(repl);
997  ::unlink(state->cln);
998  debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd);
999 #if HAVE_FCHMOD
1000 
1001  if (::stat(state->cur, &sb) == 0)
1002  fchmod(state->fd, sb.st_mode);
1003 
1004 #endif
1005 
1006  cleanLog = state;
1007  return 0;
1008 }
1009 
1010 void
1012 {
1013  UFSCleanLog *state = (UFSCleanLog *)cleanLog;
1014  int fd;
1015 
1016  if (NULL == state)
1017  return;
1018 
1019  if (state->fd < 0)
1020  return;
1021 
1022  state->walker->Done(state->walker);
1023 
1024  if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
1025  int xerrno = errno;
1026  debugs(50, DBG_CRITICAL, MYNAME << state->newLog << ": write: " << xstrerr(xerrno));
1027  debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
1028  file_close(state->fd);
1029  state->fd = -1;
1030  ::unlink(state->newLog);
1031  }
1032 
1033  safe_free(state->outbuf);
1034  /*
1035  * You can't rename open files on Microsoft "operating systems"
1036  * so we have to close before renaming.
1037  */
1038  closeLog();
1039  /* save the fd value for a later test */
1040  fd = state->fd;
1041  /* rename */
1042 
1043  if (state->fd >= 0) {
1044 #if _SQUID_OS2_ || _SQUID_WINDOWS_
1045  file_close(state->fd);
1046  state->fd = -1;
1047 #endif
1048 
1049  xrename(state->newLog, state->cur);
1050  }
1051 
1052  /* touch a timestamp file if we're not still validating */
1054  (void) 0;
1055  else if (fd < 0)
1056  (void) 0;
1057  else
1058  file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
1059 
1060  /* close */
1061  safe_free(state->cur);
1062 
1063  safe_free(state->newLog);
1064 
1065  safe_free(state->cln);
1066 
1067  if (state->fd >= 0)
1068  file_close(state->fd);
1069 
1070  state->fd = -1;
1071 
1072  delete state;
1073 
1074  cleanLog = NULL;
1075 }
1076 
1078 int
1080 {
1081  static int swap_index = 0;
1082  int i;
1083  int j = 0;
1084  int n = 0;
1085 
1086  if (!NumberOfUFSDirs)
1087  return 0; // probably in the middle of reconfiguration
1088 
1089  if (NULL == UFSDirToGlobalDirMapping) {
1090  SwapDir *sd;
1091  /*
1092  * Initialize the little array that translates UFS cache_dir
1093  * number into the Config.cacheSwap.swapDirs array index.
1094  */
1095  UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping));
1096 
1097  for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) {
1098  /* This is bogus, the controller should just clean each instance once */
1099  sd = dynamic_cast <SwapDir *>(INDEXSD(i));
1100 
1101  if (!UFSSwapDir::IsUFSDir(sd))
1102  continue;
1103 
1104  UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd);
1105 
1106  assert (usd);
1107 
1108  UFSDirToGlobalDirMapping[n] = i;
1109  ++n;
1110 
1111  j += (usd->l1 * usd->l2);
1112  }
1113 
1114  assert(n == NumberOfUFSDirs);
1115  /*
1116  * Start the commonUfsDirClean() swap_index with a random
1117  * value. j equals the total number of UFS level 2
1118  * swap directories
1119  */
1120  std::mt19937 mt(static_cast<uint32_t>(getCurrentTime() & 0xFFFFFFFF));
1121  xuniform_int_distribution<> dist(0, j);
1122  swap_index = dist(mt);
1123  }
1124 
1125  /* if the rebuild is finished, start cleaning directories. */
1127  n = DirClean(swap_index);
1128  ++swap_index;
1129  }
1130 
1131  return n;
1132 }
1133 
1134 void
1136 {
1137  const int n = HandleCleanEvent();
1138  eventAdd("storeDirClean", CleanEvent, NULL,
1139  15.0 * exp(-0.25 * n), 1);
1140 }
1141 
1142 bool
1144 {
1145  UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd);
1146  return (mySD != 0) ;
1147 }
1148 
1149 /*
1150  * XXX: this is broken - it assumes all cache dirs use the same
1151  * l1 and l2 scheme. -RBC 20021215. Partial fix is in place -
1152  * if not UFSSwapDir return 0;
1153  */
1154 bool
1156 {
1157  int D1, D2;
1158  int L1, L2;
1159  int filn = fn;
1161  assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0))));
1162  UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0));
1163 
1164  if (!sd)
1165  return 0;
1166 
1167  L1 = sd->l1;
1168 
1169  L2 = sd->l2;
1170 
1171  D1 = ((filn / L2) / L2) % L1;
1172 
1173  if (F1 != D1)
1174  return 0;
1175 
1176  D2 = (filn / L2) % L2;
1177 
1178  if (F2 != D2)
1179  return 0;
1180 
1181  return 1;
1182 }
1183 
1184 int
1186 {
1187  if (filn < 0)
1188  return 0;
1189 
1190  /*
1191  * If flag is set it means out-of-range file number should
1192  * be considered invalid.
1193  */
1194  if (flag)
1195  if (filn > map->capacity())
1196  return 0;
1197 
1198  return 1;
1199 }
1200 
1201 void
1203 {
1204  debugs(79, 3, HERE << "unlinking fileno " << std::setfill('0') <<
1205  std::hex << std::uppercase << std::setw(8) << f << " '" <<
1206  fullPath(f,NULL) << "'");
1207  /* commonUfsDirMapBitReset(this, f); */
1208  IO->unlinkFile(fullPath(f,NULL));
1209 }
1210 
1211 bool
1213 {
1214  // unlinkd may be useful only in workers
1215  return IamWorkerProcess() && IO->io->unlinkdUseful();
1216 }
1217 
1218 void
1220 {
1221  debugs(79, 3, HERE << "dirno " << index << ", fileno "<<
1222  std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
1223  if (e.swap_status == SWAPOUT_DONE) {
1224  cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
1225  --n_disk_objects;
1226  }
1227  replacementRemove(&e);
1228  mapBitReset(e.swap_filen);
1230  e.swap_filen = -1;
1231  e.swap_dirn = -1;
1233 }
1234 
1235 void
1237 {
1238  debugs(47, 4, HERE << "added node " << e << " to dir " << index);
1239  repl->Add(repl, e, &e->repl);
1240 }
1241 
1242 void
1244 {
1245  if (e->swap_dirn < 0)
1246  return;
1247 
1248  SwapDirPointer SD = INDEXSD(e->swap_dirn);
1249 
1250  assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
1251 
1252  debugs(47, 4, HERE << "remove node " << e << " from dir " << index);
1253 
1254  repl->Remove(repl, e, &e->repl);
1255 }
1256 
1257 void
1259 {
1260  storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2);
1261  dumpOptions(&entry);
1262 }
1263 
1264 char *
1265 Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const
1266 {
1267  LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
1268  int L1 = l1;
1269  int L2 = l2;
1270 
1271  if (!fullpath)
1272  fullpath = fullfilename;
1273 
1274  fullpath[0] = '\0';
1275 
1276  snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X",
1277  path,
1278  ((filn / L2) / L2) % L1,
1279  (filn / L2) % L2,
1280  filn);
1281 
1282  return fullpath;
1283 }
1284 
1285 int
1287 {
1288  return IO->callback();
1289 }
1290 
1291 void
1293 {
1294  IO->sync();
1295 }
1296 
1297 void
1299 {
1300  cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz);
1301  ++n_disk_objects;
1302 }
1303 
1304 void
1306 {
1307  if (swaplog_fd < 0) {
1308  debugs(36, 5, "cannot log " << e << " in the middle of reconfiguration");
1309  return;
1310  }
1311 
1313  s->op = (char) op;
1314  s->swap_filen = e.swap_filen;
1315  s->timestamp = e.timestamp;
1316  s->lastref = e.lastref;
1317  s->expires = e.expires;
1318  s->lastmod = e.lastModified();
1319  s->swap_file_sz = e.swap_file_sz;
1320  s->refcount = e.refcount;
1321  s->flags = e.flags;
1322  memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH);
1323  s->finalize();
1324  file_write(swaplog_fd,
1325  -1,
1326  s,
1327  sizeof(StoreSwapLogData),
1328  NULL,
1329  NULL,
1330  FreeObject);
1331 }
1332 
1333 int
1335 {
1336  DIR *dir_pointer = NULL;
1337 
1338  LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
1339  LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
1340 
1341  int files[20];
1342  int swapfileno;
1343  int fn; /* same as swapfileno, but with dirn bits set */
1344  int n = 0;
1345  int k = 0;
1346  int N0, N1, N2;
1347  int D0, D1, D2;
1348  UFSSwapDir *SD;
1349  N0 = NumberOfUFSDirs;
1350  D0 = UFSDirToGlobalDirMapping[swap_index % N0];
1351  SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0));
1352  assert (SD);
1353  N1 = SD->l1;
1354  D1 = (swap_index / N0) % N1;
1355  N2 = SD->l2;
1356  D2 = ((swap_index / N0) / N1) % N2;
1357  snprintf(p1, MAXPATHLEN, "%s/%02X/%02X",
1358  SD->path, D1, D2);
1359  debugs(36, 3, HERE << "Cleaning directory " << p1);
1360  dir_pointer = opendir(p1);
1361 
1362  if (!dir_pointer) {
1363  int xerrno = errno;
1364  if (xerrno == ENOENT) {
1365  debugs(36, DBG_CRITICAL, MYNAME << "WARNING: Creating " << p1);
1366  if (mkdir(p1, 0777) == 0)
1367  return 0;
1368  }
1369 
1370  debugs(50, DBG_CRITICAL, MYNAME << p1 << ": " << xstrerr(xerrno));
1371  safeunlink(p1, 1);
1372  return 0;
1373  }
1374 
1375  dirent_t *de;
1376  while ((de = readdir(dir_pointer)) != NULL && k < 20) {
1377  if (sscanf(de->d_name, "%X", &swapfileno) != 1)
1378  continue;
1379 
1380  fn = swapfileno; /* XXX should remove this cruft ! */
1381 
1382  if (SD->validFileno(fn, 1))
1383  if (SD->mapBitTest(fn))
1384  if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2))
1385  continue;
1386 
1387  files[k] = swapfileno;
1388  ++k;
1389  }
1390 
1391  closedir(dir_pointer);
1392 
1393  if (k == 0)
1394  return 0;
1395 
1396  qsort(files, k, sizeof(int), rev_int_sort);
1397 
1398  if (k > 10)
1399  k = 10;
1400 
1401  for (n = 0; n < k; ++n) {
1402  debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]);
1403  snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
1404  safeunlink(p2, 0);
1406  }
1407 
1408  debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1);
1409  return k;
1410 }
1411 
RemovalPolicy * createRemovalPolicy(RemovalPolicySettings *settings)
Definition: store.cc:1799
CleanLog * cleanLog
Definition: Disk.h:134
FREE * freeFunc()
Definition: MemBuf.cc:328
sdirno swap_dirn
Definition: Store.h:179
StoreEntry *(* Next)(RemovalPurgeWalker *walker)
Definition: RemovalPolicy.h:74
int GetInteger(void)
Definition: Parsing.cc:142
static int store_dirs_rebuilding
the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
Definition: Controller.h:90
char * logFile(char const *ext=NULL) const
Definition: UFSSwapDir.cc:694
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const override
check whether we can store the entry; if we can, report current load
Definition: UFSSwapDir.cc:120
StatCounters statCounter
Definition: StatCounters.cc:12
char * swap
Definition: SquidConfig.h:175
#define assert(EX)
Definition: assert.h:17
virtual char const * type() const =0
virtual ConfigOption * getOptionTree() const override
Definition: UFSSwapDir.cc:255
virtual void writeCleanDone() override
Definition: UFSSwapDir.cc:1011
virtual void maintain() override
perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
Definition: UFSSwapDir.cc:419
void finalize()
call this before storing the log entry
virtual void append(const char *c, int sz)
Definition: MemBuf.cc:225
virtual DiskIOStrategy * createStrategy()=0
virtual void release(const bool shareable=false)
Definition: store.cc:1250
void safeunlink(const char *s, int quiet)
Definition: fs_io.cc:497
int createDirectory(const char *path, int)
Definition: UFSSwapDir.cc:609
virtual bool unlinkdUseful() const override
whether SwapDir may benefit from unlinkd
Definition: UFSSwapDir.cc:1212
void(* Done)(RemovalPurgeWalker *walker)
Definition: RemovalPolicy.h:75
static uint32 B
Definition: md4.c:43
unsigned char cache_key
Store key.
Definition: forward.h:29
#define xcalloc
Definition: membanger.c:57
virtual void reference(StoreEntry &) override
somebody needs this entry (many cache replacement policies need to know)
Definition: UFSSwapDir.cc:529
void self_destruct(void)
Definition: cache_cf.cc:255
virtual bool doubleCheck(StoreEntry &) override
Definition: UFSSwapDir.cc:354
char * fn
Definition: membanger.c:36
int i
Definition: membanger.c:49
#define xstrdup
#define MAXPATHLEN
Definition: stdio.h:62
virtual void closeLog() override
Definition: UFSSwapDir.cc:756
void lastModified(const time_t when)
Definition: Store.h:142
#define safe_free(x)
Definition: xalloc.h:73
const char * storeKeyText(const cache_key *key)
UFSSwapDir(char const *aType, const char *aModuleType)
Definition: UFSSwapDir.cc:312
int file_open(const char *path, int mode)
Definition: fs_io.cc:46
char * cln
Definition: UFSSwapDir.cc:57
sfileno swap_filen
unique ID inside a cache_dir for swapped out entries; -1 for others
Definition: Store.h:177
#define DBG_CRITICAL
Definition: Debug.h:44
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override
Definition: UFSSwapDir.cc:557
int xrename(const char *from, const char *to)
Definition: fs_io.cc:513
time_t expires
Definition: Store.h:167
StoreEntry * addDiskRestore(const cache_key *key, sfileno file_number, uint64_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, time_t lastmod, uint32_t refcount, uint16_t flags, int clean)
Definition: UFSSwapDir.cc:789
virtual int load()
Definition: UFSStrategy.cc:25
virtual void write(StoreEntry const &)
Definition: UFSSwapDir.cc:82
bool validL1(int) const
Definition: UFSSwapDir.cc:777
virtual int writeCleanStart() override
Definition: UFSSwapDir.cc:967
bool IamWorkerProcess()
whether the current process handles HTTP transactions and such
Definition: stub_tools.cc:49
time_t squid_curtime
Definition: stub_time.cc:17
time_t timestamp
Definition: Store.h:165
void replacementAdd(StoreEntry *e)
Definition: UFSSwapDir.cc:1236
char * path
Definition: Disk.h:96
struct StatCounters::@137 swap
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:105
void fatalf(const char *fmt,...)
Definition: fatal.cc:79
double doublePercent(const double, const double)
Definition: SquidMath.cc:25
uint64_t swap_file_sz
Definition: Store.h:171
static DiskIOModule * Find(char const *type)
const char * xstrerr(int error)
Definition: xstrerror.cc:83
ping_status_t ping_status
Definition: Store.h:183
void hashInsert(const cache_key *)
Definition: store.cc:413
char * outbuf
Definition: UFSSwapDir.cc:58
int n_disk_objects
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
void file_write(int fd, off_t file_offset, void const *ptr_to_buf, int len, DWCB *handle, void *handle_data, FREE *free_func)
Definition: fs_io.cc:344
int fsBlockSize(const char *path, int *blksize)
Definition: fs_io.cc:530
unsigned char key[SQUID_MD5_DIGEST_LENGTH]
char * space()
returns buffer after data; does not check space existence
Definition: MemBuf.h:57
virtual const StoreEntry * nextEntry()
Definition: UFSSwapDir.cc:71
#define DBG_IMPORTANT
Definition: Debug.h:45
virtual bool dereference(StoreEntry &) override
Definition: UFSSwapDir.cc:539
static int HandleCleanEvent()
safely cleans a few unused files if possible
Definition: UFSSwapDir.cc:1079
struct SquidConfig::@103 Log
void optionIODump(StoreEntry *e) const
Definition: UFSSwapDir.cc:249
#define SQUID_MD5_DIGEST_LENGTH
Definition: md5.h:66
#define EBIT_CLR(flag, bit)
Definition: defines.h:106
virtual void dump(StoreEntry &) const override
Definition: UFSSwapDir.cc:1258
void STFNCB(void *their_data, int errflag, StoreIOState::Pointer self)
Definition: StoreIOState.h:41
FILE * fp
Definition: membanger.c:35
RemovalPolicyWalker * walker
Definition: UFSSwapDir.cc:61
static bool IsUFSDir(SwapDir *sd)
Definition: UFSSwapDir.cc:1143
void unlinkFile(sfileno f)
Definition: UFSSwapDir.cc:1202
static EVH RebuildStep
Definition: RebuildState.h:29
virtual void sync() override
prepare for shutdown
Definition: UFSSwapDir.cc:1292
virtual void swappedOut(const StoreEntry &e) override
called when entry swap out is complete
Definition: UFSSwapDir.cc:1298
SwappedTime timestamp
char * xstrncpy(char *dst, const char *src, size_t n)
Definition: xstring.cc:37
static void FreeObject(void *address)
Definition: UFSSwapDir.cc:133
void fatal(const char *message)
Definition: fatal.cc:39
void dump(int debug_lvl) const
Definition: store.cc:1605
char * swapSubDir(int subdirn) const
Definition: UFSSwapDir.cc:600
static int NumberOfUFSDirs
Definition: UFSSwapDir.h:124
uint16_t flags
Definition: Store.h:173
virtual bool shedLoad()
Definition: UFSStrategy.cc:19
void replacementRemove(StoreEntry *e)
Definition: UFSSwapDir.cc:1243
RemovalPolicyNode repl
Definition: Store.h:163
bool pathIsDirectory(const char *path) const
Definition: UFSSwapDir.cc:634
signed_int32_t sfileno
Definition: forward.h:22
size_t gapSize() const
number of bytes after the log header before the first log entry
#define LOCAL_ARRAY(type, name, size)
Definition: leakcheck.h:18
int cur
Definition: ModDevPoll.cc:76
#define INDEXSD(i)
Definition: SquidConfig.h:64
char * content()
start of the added data
Definition: MemBuf.h:41
uint16_t refcount
Definition: Store.h:172
void appended(mb_size_t sz)
updates content size after external append
Definition: MemBuf.cc:244
void const char * buf
Definition: stub_helper.cc:16
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:147
static uint32 A
Definition: md4.c:43
SwapDir * sd
Definition: UFSSwapDir.cc:62
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
#define CLEAN_BUF_SZ
Definition: UFSSwapDir.cc:11
void mapBitSet(sfileno filn)
Definition: UFSSwapDir.cc:569
Definition: enums.h:46
bool SIGHDLR int STUB void int
Definition: stub_tools.cc:68
bool validL2(int) const
Definition: UFSSwapDir.cc:783
#define dirent_t
Definition: compat_shared.h:71
void STIOCB(void *their_data, int errflag, StoreIOState::Pointer self)
Definition: StoreIOState.h:51
time_t getCurrentTime(void)
Get current time.
int highWaterMark
Definition: SquidConfig.h:75
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:109
#define PRIu64
Definition: types.h:120
off_t outbuf_offset
Definition: UFSSwapDir.cc:59
store_status_t store_status
Definition: Store.h:185
#define MYNAME
Definition: Debug.h:160
int validFileno(sfileno filn, int flag) const
Definition: UFSSwapDir.cc:1185
char * fullPath(sfileno, char *) const
Definition: UFSSwapDir.cc:1265
void dumpEntry(StoreEntry &) const
Definition: UFSSwapDir.cc:346
int mapBitTest(sfileno filn)
Definition: UFSSwapDir.cc:563
static bool FilenoBelongsHere(int fn, int cachedir, int level1dir, int level2dir)
Definition: UFSSwapDir.cc:1155
static int rev_int_sort(const void *A, const void *B)
Definition: UFSSwapDir.cc:140
RemovalPolicySettings * replPolicy
Definition: SquidConfig.h:89
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:418
static int * UFSDirToGlobalDirMapping
Definition: UFSSwapDir.h:125
int intPercent(const int a, const int b)
Definition: SquidMath.cc:13
int fsStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
Definition: fs_io.cc:554
virtual void statfs(StoreEntry &) const override
Definition: UFSSwapDir.cc:376
const StoreEntry *(* Next)(RemovalPolicyWalker *walker)
Definition: RemovalPolicy.h:62
#define F1(x, y, z)
Definition: md5.c:166
virtual void reconfigure() override
Definition: UFSSwapDir.cc:178
void mapBitReset(sfileno filn)
Definition: UFSSwapDir.cc:575
virtual void parse(int index, char *path) override
Definition: UFSSwapDir.cc:185
Definition: MemBuf.h:23
virtual ~UFSSwapDir()
Definition: UFSSwapDir.cc:333
void const cache_key * key
static EVH CleanEvent
Definition: UFSSwapDir.h:128
virtual void logEntry(const StoreEntry &e, int op) const override
Definition: UFSSwapDir.cc:1305
swap_status_t swap_status
Definition: Store.h:187
virtual ConfigOption * getOptionTree() const
Definition: Disk.cc:258
void undoAddDiskRestore(StoreEntry *e)
Undo the effects of UFSSwapDir::addDiskRestore().
Definition: UFSSwapDir.cc:831
void clearPrivate()
Definition: store.cc:158
static int DirClean(int swap_index)
Definition: UFSSwapDir.cc:1334
char * newLog
Definition: UFSSwapDir.cc:56
time_t lastref
Definition: Store.h:166
Fs::Ufs::UFSStrategy * IO
Definition: UFSSwapDir.h:79
virtual void openLog() override
Definition: UFSSwapDir.cc:731
#define O_BINARY
Definition: defines.h:204
#define FD_WRITE_METHOD(fd, buf, len)
Definition: fde.h:162
virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const =0
check whether we can store the entry; if we can, report current load
Definition: Disk.cc:164
virtual void init() override
Definition: UFSSwapDir.cc:277
#define xfree
char * cur
Definition: UFSSwapDir.cc:55
void file_close(int fd)
Definition: fs_io.cc:76
void changeIO(DiskIOModule *)
Definition: UFSSwapDir.cc:199
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:904
struct SquidConfig::@96 Swap
UFSCleanLog(SwapDir *)
Definition: UFSSwapDir.cc:65
virtual int callback() override
called once every main loop iteration; TODO: Move to UFS code.
Definition: UFSSwapDir.cc:1286
C * getRaw() const
Definition: RefCount.h:74
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override
Definition: UFSSwapDir.cc:551
class SquidConfig Config
Definition: SquidConfig.cc:12
#define NULL
Definition: types.h:166
bool optionIOParse(char const *option, const char *value, int reconfiguring)
Definition: UFSSwapDir.cc:222
void(* Done)(RemovalPolicyWalker *walker)
Definition: RemovalPolicy.h:63
virtual void unlink(StoreEntry &) override
remove the entry from the store
Definition: UFSSwapDir.cc:1219
int reconfiguring
virtual void create() override
create system resources needed for this store to operate in the future
Definition: UFSSwapDir.cc:305
int size
Definition: ModDevPoll.cc:77
#define F2(x, y, z)
Definition: md5.c:167
#define false
Definition: GnuRegex.c:233
manages a single cache_dir
Definition: Disk.h:21
void setMemStatus(mem_status_t)
Definition: store.cc:1630
FILE * openTmpSwapLog(int *clean_flag, int *zero_flag)
Definition: UFSSwapDir.cc:880

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors