IpcIoFile.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 47 Store Directory Routines */
10 
11 #include "squid.h"
12 #include "base/CodeContext.h"
13 #include "base/RunnersRegistry.h"
14 #include "base/TextException.h"
15 #include "DiskIO/IORequestor.h"
16 #include "DiskIO/IpcIo/IpcIoFile.h"
17 #include "DiskIO/ReadRequest.h"
18 #include "DiskIO/WriteRequest.h"
19 #include "fd.h"
20 #include "fs_io.h"
21 #include "globals.h"
22 #include "ipc/mem/Pages.h"
23 #include "ipc/Messages.h"
24 #include "ipc/Port.h"
25 #include "ipc/Queue.h"
26 #include "ipc/StrandCoord.h"
27 #include "ipc/StrandSearch.h"
28 #include "ipc/UdsOp.h"
29 #include "sbuf/SBuf.h"
30 #include "SquidConfig.h"
31 #include "SquidTime.h"
32 #include "StatCounters.h"
33 #include "tools.h"
34 
35 #include <cerrno>
36 
38 
40 static const char *const ShmLabel = "io_file";
44 // TODO: make configurable or compute from squid.conf settings if possible
45 static const int QueueCapacity = 1024;
46 
47 const double IpcIoFile::Timeout = 7; // seconds; XXX: ALL,9 may require more
50 std::unique_ptr<IpcIoFile::Queue> IpcIoFile::queue;
51 
53 
54 static bool DiskerOpen(const SBuf &path, int flags, mode_t mode);
55 static void DiskerClose(const SBuf &path);
56 
58 struct SipcIo {
59  SipcIo(int aWorker, const IpcIoMsg &aMsg, int aDisker):
60  worker(aWorker), msg(aMsg), disker(aDisker) {}
61 
62  int worker;
63  const IpcIoMsg &msg;
64  int disker;
65 };
66 
67 static std::ostream &
68 operator <<(std::ostream &os, const SipcIo &sio)
69 {
70  return os << "ipcIo" << sio.worker << '.' << sio.msg.requestId <<
71  sio.msg.command << sio.disker;
72 }
73 
74 /* IpcIo::Command */
75 
76 std::ostream &
77 operator <<(std::ostream &os, const IpcIo::Command command)
78 {
79  switch (command) {
80  case IpcIo::cmdNone:
81  return os << '-';
82  case IpcIo::cmdOpen:
83  return os << 'o';
84  case IpcIo::cmdRead:
85  return os << 'r';
86  case IpcIo::cmdWrite:
87  return os << 'w';
88  }
89  // unreachable code
90  return os << static_cast<int>(command);
91 }
92 
93 /* IpcIoFile */
94 
95 IpcIoFile::IpcIoFile(char const *aDb):
96  dbName(aDb),
97  myPid(getpid()),
98  diskId(-1),
99  error_(false),
100  lastRequestId(0),
101  olderRequests(&requestMap1), newerRequests(&requestMap2),
102  timeoutCheckScheduled(false)
103 {
104  assert(myPid >= 0);
105 }
106 
108 {
110  if (diskId >= 0) {
111  const auto i = IpcIoFiles.find(diskId);
112  Must(i != IpcIoFiles.end());
113  Must(i->second == this);
114  IpcIoFiles.erase(i);
115  }
116  });
117 }
118 
119 void
121 {
122  DiskFile::configure(cfg);
123  config = cfg;
124 }
125 
126 void
127 IpcIoFile::open(int flags, mode_t mode, RefCount<IORequestor> callback)
128 {
129  ioRequestor = callback;
130  Must(diskId < 0); // we do not know our disker yet
131 
132  if (!queue.get())
134 
135  if (IamDiskProcess()) {
136  error_ = !DiskerOpen(SBuf(dbName.termedBuf()), flags, mode);
137  if (error_)
138  return;
139 
141  const bool inserted =
142  IpcIoFiles.insert(std::make_pair(diskId, this)).second;
143  Must(inserted);
144 
145  queue->localRateLimit().store(config.ioRate);
146 
148 
150  return;
151  }
152 
154  Ipc::TypedMsgHdr msg;
155  request.pack(msg);
157 
158  WaitingForOpen.push_back(this);
159 
160  eventAdd("IpcIoFile::OpenTimeout", &IpcIoFile::OpenTimeout,
161  this, Timeout, 0, false); // "this" pointer is used as id
162 }
163 
164 void
166 {
167  Must(diskId < 0); // we do not know our disker yet
168 
169  if (!response) {
170  debugs(79, DBG_IMPORTANT, "ERROR: " << dbName << " communication " <<
171  "channel establishment timeout");
172  error_ = true;
173  } else {
174  diskId = response->strand.kidId;
175  if (diskId >= 0) {
176  const bool inserted =
177  IpcIoFiles.insert(std::make_pair(diskId, this)).second;
178  Must(inserted);
179  } else {
180  error_ = true;
181  debugs(79, DBG_IMPORTANT, "ERROR: no disker claimed " <<
182  "responsibility for " << dbName);
183  }
184  }
185 
187 }
188 
193 void
195 {
196  assert(false); // check
197  /* We use the same logic path for open */
198  open(flags, mode, callback);
199 }
200 
201 void
203 {
204  assert(ioRequestor != NULL);
205 
206  if (IamDiskProcess())
208  // XXX: else nothing to do?
209 
211 }
212 
213 bool
215 {
216  return diskId >= 0 && !error_ && canWait();
217 }
218 
219 bool
221 {
222  return diskId >= 0 && !error_ && canWait();
223 }
224 
225 bool
227 {
228  return error_;
229 }
230 
231 void
233 {
234  debugs(79,3, HERE << "(disker" << diskId << ", " << readRequest->len << ", " <<
235  readRequest->offset << ")");
236 
237  assert(ioRequestor != NULL);
238  assert(readRequest->offset >= 0);
239  Must(!error_);
240 
241  //assert(minOffset < 0 || minOffset <= readRequest->offset);
242  //assert(maxOffset < 0 || readRequest->offset + readRequest->len <= (uint64_t)maxOffset);
243 
244  IpcIoPendingRequest *const pending = new IpcIoPendingRequest(this);
245  pending->readRequest = readRequest;
246  push(pending);
247 }
248 
249 void
251  IpcIoMsg *const response)
252 {
253  bool ioError = false;
254  if (!response) {
255  debugs(79, 3, HERE << "error: timeout");
256  ioError = true; // I/O timeout does not warrant setting error_?
257  } else {
258  if (response->xerrno) {
259  debugs(79, DBG_IMPORTANT, "ERROR: " << dbName << " read: " <<
260  xstrerr(response->xerrno));
261  ioError = error_ = true;
262  } else if (!response->page) {
263  debugs(79, DBG_IMPORTANT, "ERROR: " << dbName << " read ran " <<
264  "out of shared memory pages");
265  ioError = true;
266  } else {
267  const char *const buf = Ipc::Mem::PagePointer(response->page);
268  memcpy(readRequest->buf, buf, response->len);
269  }
270 
271  Ipc::Mem::PutPage(response->page);
272  }
273 
274  const ssize_t rlen = ioError ? -1 : (ssize_t)readRequest->len;
275  const int errflag = ioError ? DISK_ERROR :DISK_OK;
276  ioRequestor->readCompleted(readRequest->buf, rlen, errflag, readRequest);
277 }
278 
279 void
281 {
282  debugs(79,3, HERE << "(disker" << diskId << ", " << writeRequest->len << ", " <<
283  writeRequest->offset << ")");
284 
285  assert(ioRequestor != NULL);
286  assert(writeRequest->len > 0); // TODO: work around mmap failures on zero-len?
287  assert(writeRequest->offset >= 0);
288  Must(!error_);
289 
290  //assert(minOffset < 0 || minOffset <= writeRequest->offset);
291  //assert(maxOffset < 0 || writeRequest->offset + writeRequest->len <= (uint64_t)maxOffset);
292 
293  IpcIoPendingRequest *const pending = new IpcIoPendingRequest(this);
294  pending->writeRequest = writeRequest;
295  push(pending);
296 }
297 
298 void
300  const IpcIoMsg *const response)
301 {
302  bool ioError = false;
303  if (!response) {
304  debugs(79, 3, "disker " << diskId << " timeout");
305  ioError = true; // I/O timeout does not warrant setting error_?
306  } else if (response->xerrno) {
307  debugs(79, DBG_IMPORTANT, "ERROR: disker " << diskId <<
308  " error writing " << writeRequest->len << " bytes at " <<
309  writeRequest->offset << ": " << xstrerr(response->xerrno) <<
310  "; this worker will stop using " << dbName);
311  ioError = error_ = true;
312  } else if (response->len != writeRequest->len) {
313  debugs(79, DBG_IMPORTANT, "ERROR: disker " << diskId << " wrote " <<
314  response->len << " instead of " << writeRequest->len <<
315  " bytes (offset " << writeRequest->offset << "); " <<
316  "this worker will stop using " << dbName);
317  error_ = true;
318  }
319 
320  if (writeRequest->free_func)
321  (writeRequest->free_func)(const_cast<char*>(writeRequest->buf)); // broken API?
322 
323  if (!ioError) {
324  debugs(79,5, HERE << "wrote " << writeRequest->len << " to disker" <<
325  diskId << " at " << writeRequest->offset);
326  }
327 
328  const ssize_t rlen = ioError ? 0 : (ssize_t)writeRequest->len;
329  const int errflag = ioError ? DISK_ERROR :DISK_OK;
330  ioRequestor->writeCompleted(errflag, rlen, writeRequest);
331 }
332 
333 bool
335 {
336  return !olderRequests->empty() || !newerRequests->empty();
337 }
338 
340 void
341 IpcIoFile::trackPendingRequest(const unsigned int id, IpcIoPendingRequest *const pending)
342 {
343  const std::pair<RequestMap::iterator,bool> result =
344  newerRequests->insert(std::make_pair(id, pending));
345  Must(result.second); // failures means that id was not unique
348 }
349 
351 void
353 {
354  // prevent queue overflows: check for responses to earlier requests
355  // warning: this call may result in indirect push() recursion
356  CallService(nullptr, [] {
357  HandleResponses("before push");
358  });
359 
360  debugs(47, 7, HERE);
361  Must(diskId >= 0);
362  Must(pending);
363  Must(pending->readRequest || pending->writeRequest);
364 
365  IpcIoMsg ipcIo;
366  try {
367  if (++lastRequestId == 0) // don't use zero value as requestId
368  ++lastRequestId;
369  ipcIo.requestId = lastRequestId;
370  ipcIo.start = current_time;
371  ipcIo.workerPid = myPid;
372  if (pending->readRequest) {
373  ipcIo.command = IpcIo::cmdRead;
374  ipcIo.offset = pending->readRequest->offset;
375  ipcIo.len = pending->readRequest->len;
376  } else { // pending->writeRequest
377  Must(pending->writeRequest->len <= Ipc::Mem::PageSize());
379  ipcIo.len = 0;
380  throw TexcHere("run out of shared memory pages for IPC I/O");
381  }
382  ipcIo.command = IpcIo::cmdWrite;
383  ipcIo.offset = pending->writeRequest->offset;
384  ipcIo.len = pending->writeRequest->len;
385  char *const buf = Ipc::Mem::PagePointer(ipcIo.page);
386  memcpy(buf, pending->writeRequest->buf, ipcIo.len); // optimize away
387  }
388 
389  debugs(47, 7, HERE << "pushing " << SipcIo(KidIdentifier, ipcIo, diskId));
390 
391  // protect DiskerHandleRequest() from pop queue overflow
394 
395  if (queue->push(diskId, ipcIo))
396  Notify(diskId); // must notify disker
397  trackPendingRequest(ipcIo.requestId, pending);
398  } catch (const Queue::Full &) {
399  debugs(47, DBG_IMPORTANT, "ERROR: worker I/O push queue for " <<
400  dbName << " overflow: " <<
401  SipcIo(KidIdentifier, ipcIo, diskId)); // TODO: report queue len
402  // TODO: grow queue size
403  if (ipcIo.page)
404  Ipc::Mem::PutPage(ipcIo.page);
405 
406  pending->completeIo(NULL);
407  delete pending;
408  } catch (const TextException &e) {
409  debugs(47, DBG_IMPORTANT, "ERROR: " << dbName << " exception: " << e.what());
410  pending->completeIo(NULL);
411  delete pending;
412  }
413 }
414 
416 bool
418 {
419  if (!config.ioTimeout)
420  return true; // no timeout specified
421 
422  IpcIoMsg oldestIo;
423  if (!queue->findOldest(diskId, oldestIo) || oldestIo.start.tv_sec <= 0)
424  return true; // we cannot estimate expected wait time; assume it is OK
425 
426  const int oldestWait = tvSubMsec(oldestIo.start, current_time);
427 
428  int rateWait = -1; // time in millisecons
429  const int ioRate = queue->rateLimit(diskId).load();
430  if (ioRate > 0) {
431  // if there are N requests pending, the new one will wait at
432  // least N/max-swap-rate seconds
433  rateWait = static_cast<int>(1e3 * queue->outSize(diskId) / ioRate);
434  // adjust N/max-swap-rate value based on the queue "balance"
435  // member, in case we have been borrowing time against future
436  // I/O already
437  rateWait += queue->balance(diskId);
438  }
439 
440  const int expectedWait = max(oldestWait, rateWait);
441  if (expectedWait < 0 ||
442  static_cast<time_msec_t>(expectedWait) < config.ioTimeout)
443  return true; // expected wait time is acceptable
444 
445  debugs(47,2, HERE << "cannot wait: " << expectedWait <<
446  " oldest: " << SipcIo(KidIdentifier, oldestIo, diskId));
447  return false; // do not want to wait that long
448 }
449 
451 void
453 {
454  debugs(47, 7, HERE << "coordinator response to open request");
455  for (IpcIoFileList::iterator i = WaitingForOpen.begin();
456  i != WaitingForOpen.end(); ++i) {
457  if (response.strand.tag == (*i)->dbName) {
458  (*i)->openCompleted(&response);
459  WaitingForOpen.erase(i);
460  return;
461  }
462  }
463 
464  debugs(47, 4, HERE << "LATE disker response to open for " <<
465  response.strand.tag);
466  // nothing we can do about it; completeIo() has been called already
467 }
468 
469 void
470 IpcIoFile::HandleResponses(const char *const when)
471 {
472  debugs(47, 4, HERE << "popping all " << when);
473  IpcIoMsg ipcIo;
474  // get all responses we can: since we are not pushing, this will stop
475  int diskId;
476  while (queue->pop(diskId, ipcIo)) {
477  const IpcIoFilesMap::const_iterator i = IpcIoFiles.find(diskId);
478  Must(i != IpcIoFiles.end()); // TODO: warn but continue
479  i->second->handleResponse(ipcIo);
480  }
481 }
482 
483 void
485 {
486  const int requestId = ipcIo.requestId;
487 
488  Must(requestId);
489  if (IpcIoPendingRequest *const pending = dequeueRequest(requestId)) {
490  CallBack(pending->codeContext, [&] {
491  debugs(47, 7, "popped disker response to " << SipcIo(KidIdentifier, ipcIo, diskId));
492  if (myPid == ipcIo.workerPid)
493  pending->completeIo(&ipcIo);
494  else
495  debugs(47, 5, "ignoring response meant for our predecessor PID: " << ipcIo.workerPid);
496  delete pending; // XXX: leaking if throwing
497  });
498  } else {
499  debugs(47, 4, "LATE disker response to " << SipcIo(KidIdentifier, ipcIo, diskId));
500  // nothing we can do about it; completeIo() has been called already
501  }
502 }
503 
504 void
505 IpcIoFile::Notify(const int peerId)
506 {
507  // TODO: Count and report the total number of notifications, pops, pushes.
508  debugs(47, 7, HERE << "kid" << peerId);
509  Ipc::TypedMsgHdr msg;
510  msg.setType(Ipc::mtIpcIoNotification); // TODO: add proper message type?
511  msg.putInt(KidIdentifier);
512  const String addr = Ipc::Port::MakeAddr(Ipc::strandAddrLabel, peerId);
513  Ipc::SendMessage(addr, msg);
514 }
515 
516 void
518 {
519  const int from = msg.getInt();
520  debugs(47, 7, HERE << "from " << from);
521  queue->clearReaderSignal(from);
522  if (IamDiskProcess())
524  else
525  HandleResponses("after notification");
526 }
527 
528 void
529 IpcIoFile::StatQueue(std::ostream &os)
530 {
531  if (queue.get()) {
532  os << "SMP disk I/O queues:\n";
533  queue->stat<IpcIoMsg>(os);
534  }
535 }
536 
538 void
539 IpcIoFile::OpenTimeout(void *const param)
540 {
541  Must(param);
542  // the pointer is used for comparison only and not dereferenced
543  const IpcIoFile *const ipcIoFile =
544  reinterpret_cast<const IpcIoFile *>(param);
545  for (IpcIoFileList::iterator i = WaitingForOpen.begin();
546  i != WaitingForOpen.end(); ++i) {
547  if (*i == ipcIoFile) {
548  (*i)->openCompleted(NULL);
549  WaitingForOpen.erase(i);
550  break;
551  }
552  }
553 }
554 
556 void
557 IpcIoFile::CheckTimeouts(void *const param)
558 {
559  Must(param);
560  const int diskId = reinterpret_cast<uintptr_t>(param);
561  debugs(47, 7, HERE << "diskId=" << diskId);
562  const IpcIoFilesMap::const_iterator i = IpcIoFiles.find(diskId);
563  if (i != IpcIoFiles.end())
564  i->second->checkTimeouts();
565 }
566 
567 void
569 {
570  timeoutCheckScheduled = false;
571 
572  // last chance to recover in case a notification message was lost, etc.
573  const RequestMap::size_type timeoutsBefore = olderRequests->size();
574  HandleResponses("before timeout");
575  const RequestMap::size_type timeoutsNow = olderRequests->size();
576 
577  if (timeoutsBefore > timeoutsNow) { // some requests were rescued
578  // notification message lost or significantly delayed?
579  debugs(47, DBG_IMPORTANT, "WARNING: communication with " << dbName <<
580  " may be too slow or disrupted for about " <<
581  Timeout << "s; rescued " << (timeoutsBefore - timeoutsNow) <<
582  " out of " << timeoutsBefore << " I/Os");
583  }
584 
585  if (timeoutsNow) {
586  debugs(47, DBG_IMPORTANT, "WARNING: abandoning " <<
587  timeoutsNow << ' ' << dbName << " I/Os after at least " <<
588  Timeout << "s timeout");
589  }
590 
591  // any old request would have timed out by now
592  typedef RequestMap::const_iterator RMCI;
593  for (RMCI i = olderRequests->begin(); i != olderRequests->end(); ++i) {
594  IpcIoPendingRequest *const pending = i->second;
595  CallBack(pending->codeContext, [&] {
596  const auto requestId = i->first;
597  debugs(47, 7, "disker timeout; ipcIo" << KidIdentifier << '.' << requestId);
598  pending->completeIo(nullptr); // no response
599  delete pending; // XXX: leaking if throwing
600  });
601  }
602  olderRequests->clear();
603 
604  swap(olderRequests, newerRequests); // switches pointers around
605  if (!olderRequests->empty() && !timeoutCheckScheduled)
607 }
608 
610 void
612 {
613  // We may be running in an I/O requestor CodeContext, but are scheduling
614  // one-for-all CheckTimeouts() that is not specific to any request.
615  CallService(nullptr, [&] {
616  // we check all older requests at once so some may be wait for 2*Timeout
617  eventAdd("IpcIoFile::CheckTimeouts", &IpcIoFile::CheckTimeouts,
618  reinterpret_cast<void *>(diskId), Timeout, 0, false);
619  timeoutCheckScheduled = true;
620  });
621 }
622 
625 IpcIoFile::dequeueRequest(const unsigned int requestId)
626 {
627  Must(requestId != 0);
628 
629  RequestMap *map = NULL;
630  RequestMap::iterator i = requestMap1.find(requestId);
631 
632  if (i != requestMap1.end())
633  map = &requestMap1;
634  else {
635  i = requestMap2.find(requestId);
636  if (i != requestMap2.end())
637  map = &requestMap2;
638  }
639 
640  if (!map) // not found in both maps
641  return NULL;
642 
643  IpcIoPendingRequest *pending = i->second;
644  map->erase(i);
645  return pending;
646 }
647 
648 int
650 {
651  assert(false); // not supported; TODO: remove this method from API
652  return -1;
653 }
654 
655 /* IpcIoMsg */
656 
658  requestId(0),
659  offset(0),
660  len(0),
661  workerPid(-1), // Unix-like systems use process IDs starting from 0
662  command(IpcIo::cmdNone),
663  xerrno(0)
664 {
665  start.tv_sec = 0;
666  start.tv_usec = 0;
667 }
668 
669 void
670 IpcIoMsg::stat(std::ostream &os)
671 {
672  timeval elapsedTime;
673  tvSub(elapsedTime, start, current_time);
674  os << "id: " << requestId <<
675  ", offset: " << offset <<
676  ", size: " << len <<
677  ", workerPid: " << workerPid <<
678  ", page: " << page <<
679  ", command: " << command <<
680  ", start: " << start <<
681  ", elapsed: " << elapsedTime <<
682  ", errno: " << xerrno;
683 }
684 
685 /* IpcIoPendingRequest */
686 
688  file(aFile),
689  readRequest(nullptr),
690  writeRequest(nullptr),
691  codeContext(CodeContext::Current())
692 {
693 }
694 
695 void
697 {
698  if (readRequest)
699  file->readCompleted(readRequest, response);
700  else if (writeRequest)
701  file->writeCompleted(writeRequest, response);
702  else {
703  Must(!response); // only timeouts are handled here
704  file->openCompleted(NULL);
705  }
706 }
707 
708 /* XXX: disker code that should probably be moved elsewhere */
709 
710 static SBuf DbName;
711 static int TheFile = -1;
712 
713 static void
715 {
717  ipcIo.len = 0;
718  debugs(47,2, HERE << "run out of shared memory pages for IPC I/O");
719  return;
720  }
721 
722  char *const buf = Ipc::Mem::PagePointer(ipcIo.page);
723  const ssize_t read = pread(TheFile, buf, min(ipcIo.len, Ipc::Mem::PageSize()), ipcIo.offset);
724  ++statCounter.syscalls.disk.reads;
725  fd_bytes(TheFile, read, FD_READ);
726 
727  if (read >= 0) {
728  ipcIo.xerrno = 0;
729  const size_t len = static_cast<size_t>(read); // safe because read > 0
730  debugs(47,8, HERE << "disker" << KidIdentifier << " read " <<
731  (len == ipcIo.len ? "all " : "just ") << read);
732  ipcIo.len = len;
733  } else {
734  ipcIo.xerrno = errno;
735  ipcIo.len = 0;
736  debugs(47,5, HERE << "disker" << KidIdentifier << " read error: " <<
737  ipcIo.xerrno);
738  }
739 }
740 
743 static void
745 {
746  const char *buf = Ipc::Mem::PagePointer(ipcIo.page);
747  size_t toWrite = min(ipcIo.len, Ipc::Mem::PageSize());
748  size_t wroteSoFar = 0;
749  off_t offset = ipcIo.offset;
750  // Partial writes to disk do happen. It is unlikely that the caller can
751  // handle partial writes by doing something other than writing leftovers
752  // again, so we try to write them ourselves to minimize overheads.
753  const int attemptLimit = 10;
754  for (int attempts = 1; attempts <= attemptLimit; ++attempts) {
755  const ssize_t result = pwrite(TheFile, buf, toWrite, offset);
756  ++statCounter.syscalls.disk.writes;
757  fd_bytes(TheFile, result, FD_WRITE);
758 
759  if (result < 0) {
760  ipcIo.xerrno = errno;
761  assert(ipcIo.xerrno);
762  debugs(47, DBG_IMPORTANT, "ERROR: " << DbName << " failure" <<
763  " writing " << toWrite << '/' << ipcIo.len <<
764  " at " << ipcIo.offset << '+' << wroteSoFar <<
765  " on " << attempts << " try: " << xstrerr(ipcIo.xerrno));
766  ipcIo.len = wroteSoFar;
767  return; // bail on error
768  }
769 
770  const size_t wroteNow = static_cast<size_t>(result); // result >= 0
771  ipcIo.xerrno = 0;
772 
773  debugs(47,3, "disker" << KidIdentifier << " wrote " <<
774  (wroteNow >= toWrite ? "all " : "just ") << wroteNow <<
775  " out of " << toWrite << '/' << ipcIo.len << " at " <<
776  ipcIo.offset << '+' << wroteSoFar << " on " << attempts <<
777  " try");
778 
779  wroteSoFar += wroteNow;
780 
781  if (wroteNow >= toWrite) {
782  ipcIo.xerrno = 0;
783  ipcIo.len = wroteSoFar;
784  return; // wrote everything there was to write
785  }
786 
787  buf += wroteNow;
788  offset += wroteNow;
789  toWrite -= wroteNow;
790  }
791 
792  debugs(47, DBG_IMPORTANT, "ERROR: " << DbName << " exhausted all " <<
793  attemptLimit << " attempts while writing " <<
794  toWrite << '/' << ipcIo.len << " at " << ipcIo.offset << '+' <<
795  wroteSoFar);
796  return; // not a fatal I/O error, unless the caller treats it as such
797 }
798 
799 static void
801 {
802  diskerWriteAttempts(ipcIo); // may fail
803  Ipc::Mem::PutPage(ipcIo.page);
804 }
805 
806 void
808 {
809  debugs(47, 7, HERE << "resuming handling requests after " <<
810  static_cast<const char *>(source));
813 }
814 
815 bool
817 {
818  const int ioRate = queue->localRateLimit().load();
819  const double maxRate = ioRate/1e3; // req/ms
820 
821  // do we need to enforce configured I/O rate?
822  if (maxRate <= 0)
823  return false;
824 
825  // is there an I/O request we could potentially delay?
826  int kidId;
827  IpcIoMsg ipcIo;
828  if (!queue->peek(kidId, ipcIo)) {
829  // unlike pop(), peek() is not reliable and does not block reader
830  // so we must proceed with pop() even if it is likely to fail
831  return false;
832  }
833 
834  static timeval LastIo = current_time;
835 
836  const double ioDuration = 1.0 / maxRate; // ideal distance between two I/Os
837  // do not accumulate more than 100ms or 100 I/Os, whichever is smaller
838  const int64_t maxImbalance = min(static_cast<int64_t>(100), static_cast<int64_t>(100 * ioDuration));
839 
840  const double credit = ioDuration; // what the last I/O should have cost us
841  const double debit = tvSubMsec(LastIo, current_time); // actual distance from the last I/O
842  LastIo = current_time;
843 
844  Ipc::QueueReader::Balance &balance = queue->localBalance();
845  balance += static_cast<int64_t>(credit - debit);
846 
847  debugs(47, 7, HERE << "rate limiting balance: " << balance << " after +" << credit << " -" << debit);
848 
849  if (ipcIo.command == IpcIo::cmdWrite && balance > maxImbalance) {
850  // if the next request is (likely) write and we accumulated
851  // too much time for future slow I/Os, then shed accumulated
852  // time to keep just half of the excess
853  const int64_t toSpend = balance - maxImbalance/2;
854 
855  if (toSpend/1e3 > Timeout)
856  debugs(47, DBG_IMPORTANT, "WARNING: " << DbName << " delays " <<
857  "I/O requests for " << (toSpend/1e3) << " seconds " <<
858  "to obey " << ioRate << "/sec rate limit");
859 
860  debugs(47, 3, HERE << "rate limiting by " << toSpend << " ms to get" <<
861  (1e3*maxRate) << "/sec rate");
862  eventAdd("IpcIoFile::DiskerHandleMoreRequests",
864  const_cast<char*>("rate limiting"),
865  toSpend/1e3, 0, false);
867  return true;
868  } else if (balance < -maxImbalance) {
869  // do not owe "too much" to avoid "too large" bursts of I/O
870  balance = -maxImbalance;
871  }
872 
873  return false;
874 }
875 
876 void
878 {
879  // Balance our desire to maximize the number of concurrent I/O requests
880  // (reordred by OS to minimize seek time) with a requirement to
881  // send 1st-I/O notification messages, process Coordinator events, etc.
882  const int maxSpentMsec = 10; // keep small: most RAM I/Os are under 1ms
883  const timeval loopStart = current_time;
884 
885  int popped = 0;
886  int workerId = 0;
887  IpcIoMsg ipcIo;
888  while (!WaitBeforePop() && queue->pop(workerId, ipcIo)) {
889  ++popped;
890 
891  // at least one I/O per call is guaranteed if the queue is not empty
892  DiskerHandleRequest(workerId, ipcIo);
893 
894  getCurrentTime();
895  const double elapsedMsec = tvSubMsec(loopStart, current_time);
896  if (elapsedMsec > maxSpentMsec || elapsedMsec < 0) {
898  // the gap must be positive for select(2) to be given a chance
899  const double minBreakSecs = 0.001;
900  eventAdd("IpcIoFile::DiskerHandleMoreRequests",
902  const_cast<char*>("long I/O loop"),
903  minBreakSecs, 0, false);
905  }
906  debugs(47, 3, HERE << "pausing after " << popped << " I/Os in " <<
907  elapsedMsec << "ms; " << (elapsedMsec/popped) << "ms per I/O");
908  break;
909  }
910  }
911 
912  // TODO: consider using O_DIRECT with "elevator" optimization where we pop
913  // requests first, then reorder the popped requests to optimize seek time,
914  // then do I/O, then take a break, and come back for the next set of I/O
915  // requests.
916 }
917 
919 void
920 IpcIoFile::DiskerHandleRequest(const int workerId, IpcIoMsg &ipcIo)
921 {
922  if (ipcIo.command != IpcIo::cmdRead && ipcIo.command != IpcIo::cmdWrite) {
923  debugs(0, DBG_CRITICAL, "ERROR: " << DbName <<
924  " should not receive " << ipcIo.command <<
925  " ipcIo" << workerId << '.' << ipcIo.requestId);
926  return;
927  }
928 
929  debugs(47,5, HERE << "disker" << KidIdentifier <<
930  (ipcIo.command == IpcIo::cmdRead ? " reads " : " writes ") <<
931  ipcIo.len << " at " << ipcIo.offset <<
932  " ipcIo" << workerId << '.' << ipcIo.requestId);
933 
934  const auto workerPid = ipcIo.workerPid;
935  assert(workerPid >= 0);
936 
937  if (ipcIo.command == IpcIo::cmdRead)
938  diskerRead(ipcIo);
939  else // ipcIo.command == IpcIo::cmdWrite
940  diskerWrite(ipcIo);
941 
942  assert(ipcIo.workerPid == workerPid);
943 
944  debugs(47, 7, HERE << "pushing " << SipcIo(workerId, ipcIo, KidIdentifier));
945 
946  try {
947  if (queue->push(workerId, ipcIo))
948  Notify(workerId); // must notify worker
949  } catch (const Queue::Full &) {
950  // The worker pop queue should not overflow because the worker can
951  // push only if pendingRequests() is less than QueueCapacity.
952  debugs(47, DBG_IMPORTANT, "BUG: Worker I/O pop queue for " <<
953  DbName << " overflow: " <<
954  SipcIo(workerId, ipcIo, KidIdentifier)); // TODO: report queue len
955 
956  // the I/O request we could not push will timeout
957  }
958 }
959 
960 static bool
961 DiskerOpen(const SBuf &path, int flags, mode_t)
962 {
963  assert(TheFile < 0);
964 
965  DbName = path;
966  TheFile = file_open(DbName.c_str(), flags);
967 
968  if (TheFile < 0) {
969  const int xerrno = errno;
970  debugs(47, DBG_CRITICAL, "ERROR: cannot open " << DbName << ": " <<
971  xstrerr(xerrno));
972  return false;
973  }
974 
976  debugs(79,3, "rock db opened " << DbName << ": FD " << TheFile);
977  return true;
978 }
979 
980 static void
981 DiskerClose(const SBuf &path)
982 {
983  if (TheFile >= 0) {
985  debugs(79,3, HERE << "rock db closed " << path << ": FD " << TheFile);
986  TheFile = -1;
988  }
989  DbName.clear();
990 }
991 
995 {
996 public:
997  /* RegisteredRunner API */
999  virtual ~IpcIoRr();
1000  virtual void claimMemoryNeeds();
1001 
1002 protected:
1003  /* Ipc::Mem::RegisteredRunner API */
1004  virtual void create();
1005 
1006 private:
1008 };
1009 
1011 
1012 void
1014 {
1015  const int itemsCount = Ipc::FewToFewBiQueue::MaxItemsCount(
1017  // the maximum number of shared I/O pages is approximately the
1018  // number of queue slots, we add a fudge factor to that to account
1019  // for corner cases where I/O pages are created before queue
1020  // limits are checked or destroyed long after the I/O is dequeued
1022  static_cast<int>(itemsCount * 1.1));
1023 }
1024 
1025 void
1027 {
1028  if (Config.cacheSwap.n_strands <= 0)
1029  return;
1030 
1031  Must(!owner);
1034  1 + Config.workers, sizeof(IpcIoMsg),
1035  QueueCapacity);
1036 }
1037 
1039 {
1040  delete owner;
1041 }
1042 
static String MakeAddr(const char *proccessLabel, int id)
calculates IPC message address for strand id of processLabel type
Definition: Port.cc:52
const char * xstrerr(int error)
Definition: xstrerror.cc:83
bool timeoutCheckScheduled
we expect a CheckTimeouts() call
Definition: IpcIoFile.h:147
size_t pendingRequests() const
Definition: IpcIoFile.h:114
static void diskerWrite(IpcIoMsg &ipcIo)
Definition: IpcIoFile.cc:800
ReadRequest * readRequest
set if this is a read requests
Definition: IpcIoFile.h:176
virtual bool canRead() const
Definition: IpcIoFile.cc:214
static SBuf DbName
full db file name
Definition: IpcIoFile.cc:710
Command
what kind of I/O the disker needs to do or have done
Definition: IpcIoFile.h:33
IpcIoPendingRequest(const IpcIoFile::Pointer &aFile)
Definition: IpcIoFile.cc:687
void setType(int aType)
sets message type; use MessageType enum
Definition: TypedMsgHdr.cc:100
static bool DiskerOpen(const SBuf &path, int flags, mode_t mode)
Definition: IpcIoFile.cc:961
RefCount< IORequestor > ioRequestor
Definition: IpcIoFile.h:135
StrandCoord strand
messageType-specific coordinates (e.g., sender)
Definition: StrandCoord.h:52
friend class IpcIoPendingRequest
Definition: IpcIoFile.h:101
int KidIdentifier
RequestMap * olderRequests
older requests (map1 or map2)
Definition: IpcIoFile.h:145
int ioRate
shape I/O request stream to approach that many per second
Definition: DiskFile.h:36
virtual void configure(const Config &)
notes supported configuration options; kids must call this first
Definition: DiskFile.h:42
int kidId
internal Squid process number
Definition: StrandCoord.h:31
@ mtIpcIoNotification
Definition: Messages.h:31
const char strandAddrLabel[]
strand's listening address unique label
Definition: Port.cc:23
off_t offset
Definition: IpcIoFile.h:51
struct StatCounters::@136::@140 disk
static std::ostream & operator<<(std::ostream &os, const SipcIo &sio)
Definition: IpcIoFile.cc:68
Definition: SBuf.h:87
void readCompleted(ReadRequest *readRequest, IpcIoMsg *const response)
Definition: IpcIoFile.cc:250
virtual bool ioInProgress() const
Definition: IpcIoFile.cc:334
IpcIoPendingRequest * dequeueRequest(const unsigned int requestId)
returns and forgets the right IpcIoFile pending request
Definition: IpcIoFile.cc:625
virtual ~IpcIoFile()
Definition: IpcIoFile.cc:107
static void NotifyCoordinator(MessageType, const char *tag)
creates and sends StrandMessage to Coordinator
Definition: StrandCoord.cc:62
std::map< unsigned int, IpcIoPendingRequest * > RequestMap
maps requestId to the handleResponse callback
Definition: IpcIoFile.h:142
CBDATA_CLASS_INIT(IpcIoFile)
static void diskerRead(IpcIoMsg &ipcIo)
Definition: IpcIoFile.cc:714
void trackPendingRequest(const unsigned int id, IpcIoPendingRequest *const pending)
track a new pending request
Definition: IpcIoFile.cc:341
CodeContext::Pointer codeContext
requestor's context
Definition: IpcIoFile.h:179
Ipc::FewToFewBiQueue::Owner * owner
Definition: IpcIoFile.cc:1007
virtual ~IpcIoRr()
Definition: IpcIoFile.cc:1038
void checkTimeouts()
Definition: IpcIoFile.cc:568
Store::DiskConfig cacheSwap
Definition: SquidConfig.h:430
bool IamWorkerProcess()
whether the current process handles HTTP transactions and such
Definition: stub_tools.cc:47
#define DBG_CRITICAL
Definition: Debug.h:40
std::list< Pointer > IpcIoFileList
Definition: IpcIoFile.h:151
void CallBack(const CodeContext::Pointer &callbackContext, Fun &&callback)
Definition: CodeContext.h:112
void file_close(int fd)
Definition: fs_io.cc:73
static void diskerWriteAttempts(IpcIoMsg &ipcIo)
Definition: IpcIoFile.cc:744
time_t getCurrentTime(void)
Get current time.
int store_open_disk_fd
static void Notify(const int peerId)
Definition: IpcIoFile.cc:505
#define DBG_IMPORTANT
Definition: Debug.h:41
static IpcIoFileList WaitingForOpen
pending open requests
Definition: IpcIoFile.h:152
void clear()
Definition: SBuf.cc:175
A const & max(A const &lhs, A const &rhs)
#define DISK_OK
Definition: defines.h:27
@ cmdWrite
Definition: IpcIoFile.h:33
virtual void open(int flags, mode_t mode, RefCount< IORequestor > callback)
Definition: IpcIoFile.cc:127
#define TexcHere(msg)
legacy convenience macro; it is not difficult to type Here() now
Definition: TextException.h:58
static void HandleOpenResponse(const Ipc::StrandMessage &)
handle open response from coordinator
Definition: IpcIoFile.cc:452
Ipc::Mem::PageId page
Definition: IpcIoFile.h:53
void writeCompleted(WriteRequest *writeRequest, const IpcIoMsg *const response)
Definition: IpcIoFile.cc:299
@ cmdOpen
Definition: IpcIoFile.h:33
struct StatCounters::@136 syscalls
static String CoordinatorAddr()
get the IPC message address for coordinator process
Definition: Port.cc:65
bool error_
whether we have seen at least one I/O error (XXX)
Definition: IpcIoFile.h:137
static std::unique_ptr< Queue > queue
IPC queue.
Definition: IpcIoFile.h:159
void putInt(int n)
store an integer
Definition: TypedMsgHdr.cc:119
#define NULL
Definition: types.h:166
IpcIo wrapper for debugs() streams; XXX: find a better class name.
Definition: IpcIoFile.cc:58
virtual void ioCompletedNotification()=0
int xerrno
I/O error code or zero.
Definition: IpcIoFile.h:59
int disker
Definition: IpcIoFile.cc:64
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:123
void NotePageNeed(const int purpose, const int count)
claim the need for a number of pages for a given purpose
Definition: Pages.cc:72
bool IamDiskProcess() STUB_RETVAL_NOP(false) bool InDaemonMode() STUB_RETVAL_NOP(false) bool UsingSmp() STUB_RETVAL_NOP(false) bool IamCoordinatorProcess() STUB_RETVAL(false) bool IamPrimaryProcess() STUB_RETVAL(false) int NumberOfKids() STUB_RETVAL(0) void setMaxFD(void) STUB void setSystemLimits(void) STUB void squid_signal(int
whether the current process is dedicated to managing a cache_dir
virtual bool canWrite() const
Definition: IpcIoFile.cc:220
void push(IpcIoPendingRequest *const pending)
push an I/O request to disker
Definition: IpcIoFile.cc:352
struct timeval start
when the I/O request was converted to IpcIoMsg
Definition: IpcIoFile.h:57
static void OpenTimeout(void *const param)
handles open request timeout
Definition: IpcIoFile.cc:539
static void HandleResponses(const char *const when)
Definition: IpcIoFile.cc:470
int n_strands
number of disk processes required to support all cache_dirs
Definition: SquidConfig.h:70
static bool DiskerHandleMoreRequestsScheduled
whether we are waiting for an event to handle still queued I/O requests
Definition: IpcIoFile.h:162
virtual bool error() const
Definition: IpcIoFile.cc:226
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:152
static void DiskerHandleRequest(const int workerId, IpcIoMsg &ipcIo)
called when disker receives an I/O request
Definition: IpcIoFile.cc:920
static int TheFile
db file descriptor
Definition: IpcIoFile.cc:711
static const int QueueCapacity
Definition: IpcIoFile.cc:45
Ipc::FewToFewBiQueue Queue
Definition: IpcIoFile.h:158
unsigned int requestId
unique for requestor; matches request w/ response
Definition: IpcIoFile.h:49
static int MaxItemsCount(const int groupASize, const int groupBSize, const int capacity)
maximum number of items in the queue
Definition: Queue.cc:240
virtual int getFD() const
Definition: IpcIoFile.cc:649
int tvSubMsec(struct timeval, struct timeval)
Definition: time.cc:37
void handleResponse(IpcIoMsg &ipcIo)
Definition: IpcIoFile.cc:484
void SendMessage(const String &toAddress, const TypedMsgHdr &message)
Definition: UdsOp.cc:188
#define assert(EX)
Definition: assert.h:19
char const * termedBuf() const
Definition: SquidString.h:92
int worker
Definition: IpcIoFile.cc:62
virtual void create(int flags, mode_t mode, RefCount< IORequestor > callback)
Definition: IpcIoFile.cc:194
int getInt() const
load an integer
Definition: TypedMsgHdr.cc:111
virtual void close()
Definition: IpcIoFile.cc:202
void stat(std::ostream &)
prints message parameters; suitable for cache manager reports
Definition: IpcIoFile.cc:670
WriteRequest * writeRequest
set if this is a write request
Definition: IpcIoFile.h:177
void PutPage(PageId &page)
makes identified page available as a free page to future GetPage() callers
Definition: Pages.cc:41
void scheduleTimeoutCheck()
prepare to check for timeouts in a little while
Definition: IpcIoFile.cc:611
size_t len
Definition: ReadRequest.h:26
const char * c_str()
Definition: SBuf.cc:516
asynchronous strand search request
Definition: StrandSearch.h:22
RequestMap requestMap1
older (or newer) pending requests
Definition: IpcIoFile.h:143
IpcIo::Command command
what disker is supposed to do or did
Definition: IpcIoFile.h:56
IpcIoFile(char const *aDb)
Definition: IpcIoFile.cc:95
static void HandleNotification(const Ipc::TypedMsgHdr &msg)
handle queue push notifications from worker or disker
Definition: IpcIoFile.cc:517
FREE * free_func
Definition: WriteRequest.h:28
std::map< int, IpcIoFile * > IpcIoFilesMap
Definition: IpcIoFile.h:155
virtual void write(WriteRequest *)
Definition: IpcIoFile.cc:280
@ cmdRead
Definition: IpcIoFile.h:33
void fd_bytes(int fd, int len, unsigned int type)
Definition: fd.cc:227
const pid_t myPid
optimization: cached process ID of our process
Definition: IpcIoFile.h:133
virtual void configure(const Config &cfg)
notes supported configuration options; kids must call this first
Definition: IpcIoFile.cc:120
String tag
optional unique well-known key (e.g., cache_dir path)
Definition: StrandCoord.h:34
static const double Timeout
timeout value in seconds
Definition: IpcIoFile.h:149
char const * buf
Definition: WriteRequest.h:25
virtual void writeCompleted(int errflag, size_t len, RefCount< WriteRequest >)=0
char * buf
Definition: ReadRequest.h:24
an IPC message carrying StrandCoord
Definition: StrandCoord.h:39
pid_t workerPid
the process ID of the I/O requestor
Definition: IpcIoFile.h:54
@ mtRegisterStrand
notifies about our strand existence
Definition: Messages.h:22
@ FD_READ
Definition: enums.h:23
static const char *const ShmLabel
shared memory segment path to use for IpcIoFile maps
Definition: IpcIoFile.cc:40
struct timeval current_time
Definition: stub_time.cc:15
char * PagePointer(const PageId &page)
converts page handler into a temporary writeable shared memory pointer
Definition: Pages.cc:48
DiskFile::Config config
supported configuration options
Definition: IpcIoFile.h:98
keeps original I/O request parameters while disker is handling the request
Definition: IpcIoFile.h:167
generally useful configuration options supported by some children
Definition: DiskFile.h:28
RequestMap requestMap2
newer (or older) pending requests
Definition: IpcIoFile.h:144
AtomicSignedMsec Balance
Definition: Queue.h:59
virtual void claimMemoryNeeds()
Definition: IpcIoFile.cc:1013
bool GetPage(const PageId::Purpose purpose, PageId &page)
sets page ID and returns true unless no free pages are found
Definition: Pages.cc:34
virtual const char * what() const override
an std::runtime_error with thrower location info
Definition: TextException.h:20
static Owner * Init(const String &id, const int groupASize, const int groupAIdOffset, const int groupBSize, const int groupBIdOffset, const unsigned int maxItemSize, const int capacity)
Definition: Queue.cc:221
void openCompleted(const Ipc::StrandMessage *)
Definition: IpcIoFile.cc:165
static bool WaitBeforePop()
Definition: IpcIoFile.cc:816
static void DiskerHandleMoreRequests(void *)
Definition: IpcIoFile.cc:807
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:73
unsigned short mode_t
Definition: types.h:150
struct msghdr with a known type, fixed-size I/O and control buffers
Definition: TypedMsgHdr.h:33
RunnerRegistrationEntry(IpcIoRr)
void tvSub(struct timeval &res, struct timeval const &t1, struct timeval const &t2)
Definition: time.cc:44
static void StatQueue(std::ostream &)
prints IPC message queue state; suitable for cache manager reports
Definition: IpcIoFile.cc:529
virtual void read(ReadRequest *)
Definition: IpcIoFile.cc:232
void completeIo(IpcIoMsg *const response)
called when response is received and, with a nil response, on timeouts
Definition: IpcIoFile.cc:696
virtual void closeCompleted()=0
#define DISK_ERROR
Definition: defines.h:28
@ FD_WRITE
Definition: enums.h:24
virtual void create()
called when the runner should create a new memory segment
Definition: IpcIoFile.cc:1026
const IpcIoFile::Pointer file
the file object waiting for the response
Definition: IpcIoFile.h:175
size_t PageSize()
returns page size in bytes; all pages are assumed to be the same size
Definition: Pages.cc:28
size_t len
Definition: IpcIoFile.h:52
RequestMap * newerRequests
newer requests (map2 or map1)
Definition: IpcIoFile.h:146
time_msec_t ioTimeout
canRead/Write should return false if expected I/O delay exceeds it
Definition: DiskFile.h:33
int file_open(const char *path, int mode)
Definition: fs_io.cc:45
#define SWALLOW_EXCEPTIONS(code)
Definition: TextException.h:77
void CallService(const CodeContext::Pointer &serviceContext, Fun &&service)
Definition: CodeContext.h:129
SipcIo(int aWorker, const IpcIoMsg &aMsg, int aDisker)
Definition: IpcIoFile.cc:59
static IpcIoFilesMap IpcIoFiles
Definition: IpcIoFile.h:156
converts DiskIO requests to IPC queue messages
Definition: IpcIoFile.h:41
int diskId
the kid ID of the disker we talk to
Definition: IpcIoFile.h:134
@ cmdNone
Definition: IpcIoFile.h:33
#define false
Definition: GnuRegex.c:233
bool canWait() const
whether we think there is enough time to complete the I/O
Definition: IpcIoFile.cc:417
static void CheckTimeouts(void *const param)
IpcIoFile::checkTimeouts wrapper.
Definition: IpcIoFile.cc:557
A const & min(A const &lhs, A const &rhs)
unsigned int lastRequestId
last requestId used
Definition: IpcIoFile.h:139
struct _request * request(char *urlin)
Definition: tcp-banger2.c:291
const IpcIoMsg & msg
Definition: IpcIoFile.cc:63
off_t offset
Definition: ReadRequest.h:25
uint64_t time_msec_t
Definition: SquidTime.h:21
virtual void readCompleted(const char *buf, int len, int errflag, RefCount< ReadRequest >)=0
void eventAdd(const char *name, EVH *func, void *arg, double when, int weight, bool cbdata)
Definition: event.cc:108
static void DiskerClose(const SBuf &path)
Definition: IpcIoFile.cc:981
class SquidConfig Config
Definition: SquidConfig.cc:12
StatCounters statCounter
Definition: StatCounters.cc:12
static void DiskerHandleRequests()
Definition: IpcIoFile.cc:877
const String dbName
the name of the file we are managing
Definition: IpcIoFile.h:132

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors