pconn.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 48 Persistent Connections */
10 
11 #include "squid.h"
12 #include "CachePeer.h"
13 #include "comm.h"
14 #include "comm/Connection.h"
15 #include "comm/Read.h"
16 #include "fd.h"
17 #include "fde.h"
18 #include "globals.h"
19 #include "mgr/Registration.h"
20 #include "neighbors.h"
21 #include "pconn.h"
22 #include "PeerPoolMgr.h"
23 #include "SquidConfig.h"
24 #include "Store.h"
25 
26 #define PCONN_FDS_SZ 8 /* pconn set size, increase for better memcache hit rate */
27 
28 //TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
29 PconnModule * PconnModule::instance = NULL;
31 
32 /* ========== IdleConnList ============================================ */
33 
34 IdleConnList::IdleConnList(const char *aKey, PconnPool *thePool) :
35  capacity_(PCONN_FDS_SZ),
36  size_(0),
37  parent_(thePool)
38 {
39  //Initialize hash_link members
40  key = xstrdup(aKey);
41  next = NULL;
42 
44 
46 
47 // TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
48 }
49 
51 {
52  if (parent_)
53  parent_->unlinkList(this);
54 
55  if (size_) {
56  parent_ = NULL; // prevent reentrant notifications and deletions
57  closeN(size_);
58  }
59 
60  delete[] theList_;
61 
62  xfree(key);
63 }
64 
71 int
73 {
74  for (int index = size_ - 1; index >= 0; --index) {
75  if (conn->fd == theList_[index]->fd) {
76  debugs(48, 3, HERE << "found " << conn << " at index " << index);
77  return index;
78  }
79  }
80 
81  debugs(48, 2, HERE << conn << " NOT FOUND!");
82  return -1;
83 }
84 
89 bool
91 {
92  if (index < 0 || index >= size_)
93  return false;
94 
95  // shuffle the remaining entries to fill the new gap.
96  for (; index < size_ - 1; ++index)
97  theList_[index] = theList_[index + 1];
98  theList_[--size_] = NULL;
99 
100  if (parent_) {
102  if (size_ == 0) {
103  debugs(48, 3, "deleting " << hashKeyStr(this));
104  delete this;
105  }
106  }
107 
108  return true;
109 }
110 
111 // almost a duplicate of removeFD. But drops multiple entries.
112 void
114 {
115  if (n < 1) {
116  debugs(48, 2, HERE << "Nothing to do.");
117  return;
118  } else if (n >= (size_t)size_) {
119  debugs(48, 2, HERE << "Closing all entries.");
120  while (size_ > 0) {
121  const Comm::ConnectionPointer conn = theList_[--size_];
122  theList_[size_] = NULL;
123  clearHandlers(conn);
124  conn->close();
125  if (parent_)
127  }
128  } else { //if (n < size_)
129  debugs(48, 2, HERE << "Closing " << n << " of " << size_ << " entries.");
130 
131  size_t index;
132  // ensure the first N entries are closed
133  for (index = 0; index < n; ++index) {
134  const Comm::ConnectionPointer conn = theList_[index];
135  theList_[index] = NULL;
136  clearHandlers(conn);
137  conn->close();
138  if (parent_)
140  }
141  // shuffle the list N down.
142  for (index = 0; index < (size_t)size_ - n; ++index) {
143  theList_[index] = theList_[index + n];
144  }
145  // ensure the last N entries are unset
146  while (index < ((size_t)size_)) {
147  theList_[index] = NULL;
148  ++index;
149  }
150  size_ -= n;
151  }
152 
153  if (parent_ && size_ == 0) {
154  debugs(48, 3, "deleting " << hashKeyStr(this));
155  delete this;
156  }
157 }
158 
159 void
161 {
162  debugs(48, 3, HERE << "removing close handler for " << conn);
163  comm_read_cancel(conn->fd, IdleConnList::Read, this);
164  commUnsetConnTimeout(conn);
165 }
166 
167 void
169 {
170  if (size_ == capacity_) {
171  debugs(48, 3, HERE << "growing idle Connection array");
172  capacity_ <<= 1;
173  const Comm::ConnectionPointer *oldList = theList_;
175  for (int index = 0; index < size_; ++index)
176  theList_[index] = oldList[index];
177 
178  delete[] oldList;
179  }
180 
181  if (parent_)
183 
184  theList_[size_] = conn;
185  ++size_;
186  AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
188  comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
189  AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
191  commSetConnTimeout(conn, conn->timeLeft(Config.Timeout.serverIdlePconn), timeoutCall);
192 }
193 
196 bool
198 {
199  const Comm::ConnectionPointer &conn = theList_[i];
200 
201  // connection already closed. useless.
202  if (!Comm::IsConnOpen(conn))
203  return false;
204 
205  // our connection early-read/close handler is scheduled to run already. unsafe
206  if (!COMMIO_FD_READCB(conn->fd)->active())
207  return false;
208 
209  return true;
210 }
211 
214 {
215  for (int i=size_-1; i>=0; --i) {
216 
217  if (!isAvailable(i))
218  continue;
219 
220  // our connection timeout handler is scheduled to run already. unsafe for now.
221  // TODO: cancel the pending timeout callback and allow re-use of the conn.
222  if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
223  continue;
224 
225  // finally, a match. pop and return it.
227  clearHandlers(result);
228  /* may delete this */
229  removeAt(i);
230  return result;
231  }
232 
233  return Comm::ConnectionPointer();
234 }
235 
236 /*
237  * XXX this routine isn't terribly efficient - if there's a pending
238  * read event (which signifies the fd will close in the next IO loop!)
239  * we ignore the FD and move onto the next one. This means, as an example,
240  * if we have a lot of FDs open to a very popular server and we get a bunch
241  * of requests JUST as they timeout (say, it shuts down) we'll be wasting
242  * quite a bit of CPU. Just keep it in mind.
243  */
246 {
247  assert(size_);
248 
249  // small optimization: do the constant bool tests only once.
250  const bool keyCheckAddr = !aKey->local.isAnyAddr();
251  const bool keyCheckPort = aKey->local.port() > 0;
252 
253  for (int i=size_-1; i>=0; --i) {
254 
255  if (!isAvailable(i))
256  continue;
257 
258  // local end port is required, but do not match.
259  if (keyCheckPort && aKey->local.port() != theList_[i]->local.port())
260  continue;
261 
262  // local address is required, but does not match.
263  if (keyCheckAddr && aKey->local.matchIPAddr(theList_[i]->local) != 0)
264  continue;
265 
266  // our connection timeout handler is scheduled to run already. unsafe for now.
267  // TODO: cancel the pending timeout callback and allow re-use of the conn.
268  if (fd_table[theList_[i]->fd].timeoutHandler == NULL)
269  continue;
270 
271  // finally, a match. pop and return it.
273  clearHandlers(result);
274  /* may delete this */
275  removeAt(i);
276  return result;
277  }
278 
279  return Comm::ConnectionPointer();
280 }
281 
282 /* might delete list */
283 void
285 {
286  const int index = findIndexOf(conn);
287  if (index >= 0) {
288  if (parent_)
289  parent_->notifyManager("idle conn closure");
290  clearHandlers(conn);
291  /* might delete this */
292  removeAt(index);
293  conn->close();
294  }
295 }
296 
297 void
298 IdleConnList::Read(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
299 {
300  debugs(48, 3, HERE << len << " bytes from " << conn);
301 
302  if (flag == Comm::ERR_CLOSING) {
303  debugs(48, 3, HERE << "Comm::ERR_CLOSING from " << conn);
304  /* Bail out on Comm::ERR_CLOSING - may happen when shutdown aborts our idle FD */
305  return;
306  }
307 
308  IdleConnList *list = (IdleConnList *) data;
309  /* may delete list/data */
310  list->findAndClose(conn);
311 }
312 
313 void
315 {
316  debugs(48, 3, HERE << io.conn);
317  IdleConnList *list = static_cast<IdleConnList *>(io.data);
318  /* may delete list/data */
319  list->findAndClose(io.conn);
320 }
321 
322 void
324 {
325  closeN(size_);
326 }
327 
328 /* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
329 
330 const char *
331 PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
332 {
333  LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
334 
335  destLink->remote.toUrl(buf, SQUIDHOSTNAMELEN * 3 + 10);
336  if (domain) {
337  const int used = strlen(buf);
338  snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
339  }
340 
341  debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
342  return buf;
343 }
344 
345 void
347 {
349  "%s persistent connection counts:\n"
350  "\n"
351  "\t Requests\t Connection Count\n"
352  "\t --------\t ----------------\n",
353  descr);
354 
355  for (int i = 0; i < PCONN_HIST_SZ; ++i) {
356  if (hist[i] == 0)
357  continue;
358 
359  storeAppendPrintf(e, "\t%d\t%d\n", i, hist[i]);
360  }
361 }
362 
363 void
365 {
366  hash_table *hid = table;
367  hash_first(hid);
368 
369  int i = 0;
370  for (hash_link *walker = hash_next(hid); walker; walker = hash_next(hid)) {
371  storeAppendPrintf(e, "\t item %d:\t%s\n", i, (char *)(walker->key));
372  ++i;
373  }
374 }
375 
376 /* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
377 
378 PconnPool::PconnPool(const char *aDescr, const CbcPointer<PeerPoolMgr> &aMgr):
379  table(NULL), descr(aDescr),
380  mgr(aMgr),
381  theCount(0)
382 {
383  int i;
384  table = hash_create((HASHCMP *) strcmp, 229, hash_string);
385 
386  for (i = 0; i < PCONN_HIST_SZ; ++i)
387  hist[i] = 0;
388 
389  PconnModule::GetInstance()->add(this);
390 }
391 
392 static void
393 DeleteIdleConnList(void *hashItem)
394 {
395  delete static_cast<IdleConnList*>(hashItem);
396 }
397 
399 {
403  descr = NULL;
404 }
405 
406 void
407 PconnPool::push(const Comm::ConnectionPointer &conn, const char *domain)
408 {
409  if (fdUsageHigh()) {
410  debugs(48, 3, HERE << "Not many unused FDs");
411  conn->close();
412  return;
413  } else if (shutting_down) {
414  conn->close();
415  debugs(48, 3, HERE << "Squid is shutting down. Refusing to do anything");
416  return;
417  }
418  // TODO: also close used pconns if we exceed peer max-conn limit
419 
420  const char *aKey = key(conn, domain);
421  IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
422 
423  if (list == NULL) {
424  list = new IdleConnList(aKey, this);
425  debugs(48, 3, "new IdleConnList for {" << hashKeyStr(list) << "}" );
426  hash_join(table, list);
427  } else {
428  debugs(48, 3, "found IdleConnList for {" << hashKeyStr(list) << "}" );
429  }
430 
431  list->push(conn);
433 
434  LOCAL_ARRAY(char, desc, FD_DESC_SZ);
435  snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
436  fd_note(conn->fd, desc);
437  debugs(48, 3, HERE << "pushed " << conn << " for " << aKey);
438 
439  // successful push notifications resume multi-connection opening sequence
440  notifyManager("push");
441 }
442 
444 PconnPool::pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
445 {
446 
447  const char * aKey = key(dest, domain);
448 
449  IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
450  if (list == NULL) {
451  debugs(48, 3, HERE << "lookup for key {" << aKey << "} failed.");
452  // failure notifications resume standby conn creation after fdUsageHigh
453  notifyManager("pop failure");
454  return Comm::ConnectionPointer();
455  } else {
456  debugs(48, 3, "found " << hashKeyStr(list) <<
457  (keepOpen ? " to use" : " to kill"));
458  }
459 
460  /* may delete list */
461  Comm::ConnectionPointer popped = list->findUseable(dest);
462  if (!keepOpen && Comm::IsConnOpen(popped))
463  popped->close();
464 
465  // successful pop notifications replenish standby connections pool
466  notifyManager("pop");
467  return popped;
468 }
469 
470 void
471 PconnPool::notifyManager(const char *reason)
472 {
473  if (mgr.valid())
474  PeerPoolMgr::Checkpoint(mgr, reason);
475 }
476 
477 void
479 {
480  hash_table *hid = table;
481  hash_first(hid);
482 
483  // close N connections, one per list, to treat all lists "fairly"
484  for (int i = 0; i < n && count(); ++i) {
485 
486  hash_link *current = hash_next(hid);
487  if (!current) {
488  hash_first(hid);
489  current = hash_next(hid);
490  Must(current); // must have one because the count() was positive
491  }
492 
493  // may delete current
494  static_cast<IdleConnList*>(current)->closeN(1);
495  }
496 }
497 
498 void
500 {
501  theCount -= list->count();
502  assert(theCount >= 0);
503  hash_remove_link(table, list);
504 }
505 
506 void
508 {
509  if (uses >= PCONN_HIST_SZ)
510  uses = PCONN_HIST_SZ - 1;
511 
512  ++hist[uses];
513 }
514 
515 /* ========== PconnModule ============================================ */
516 
517 /*
518  * This simple class exists only for the cache manager
519  */
520 
522 {
524 }
525 
526 PconnModule *
528 {
529  if (instance == NULL)
530  instance = new PconnModule;
531 
532  return instance;
533 }
534 
535 void
537 {
538  Mgr::RegisterAction("pconn",
539  "Persistent Connection Utilization Histograms",
540  DumpWrapper, 0, 1);
541 }
542 
543 void
545 {
546  pools.insert(aPool);
547 }
548 
549 void
551 {
552  pools.erase(aPool);
553 }
554 
555 void
557 {
558  typedef Pools::const_iterator PCI;
559  int i = 0; // TODO: Why number pools if they all have names?
560  for (PCI p = pools.begin(); p != pools.end(); ++p, ++i) {
561  // TODO: Let each pool dump itself the way it wants to.
562  storeAppendPrintf(e, "\n Pool %d Stats\n", i);
563  (*p)->dumpHist(e);
564  storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
565  (*p)->dumpHash(e);
566  }
567 }
568 
569 void
571 {
573 }
574 
int fdUsageHigh(void)
Definition: fd.cc:301
char fakeReadBuf_[4096]
Definition: pconn.h:96
void closeN(int n)
closes any n connections, regardless of their destination
Definition: pconn.cc:478
#define fd_table
Definition: fde.h:174
#define COMMIO_FD_READCB(fd)
Definition: IoCallback.h:78
static const char * key(const Comm::ConnectionPointer &destLink, const char *domain)
Definition: pconn.cc:331
unsigned short port() const
Definition: Address.cc:778
#define assert(EX)
Definition: assert.h:17
SQUIDCEXTERN void hash_first(hash_table *)
Definition: hash.cc:176
SQUIDCEXTERN void hashFreeMemory(hash_table *)
Definition: hash.cc:272
static void DeleteIdleConnList(void *hashItem)
Definition: pconn.cc:393
void noteUses(int uses)
Definition: pconn.cc:507
void fd_note(int fd, const char *s)
Definition: fd.cc:246
SQUIDCEXTERN HASHHASH hash_string
Definition: hash.h:45
~PconnPool()
Definition: pconn.cc:398
void registerWithCacheManager(void)
Definition: pconn.cc:536
int findIndexOf(const Comm::ConnectionPointer &conn) const
Definition: pconn.cc:72
int i
Definition: membanger.c:49
#define xstrdup
CbcPointer< PeerPoolMgr > mgr
optional pool manager (for notifications)
Definition: pconn.h:157
static IOCB Read
Definition: pconn.h:73
void remove(PconnPool *)
unregister and forget about this pool object
Definition: pconn.cc:550
bool comm_has_incomplete_write(int fd)
Definition: comm.cc:147
SQUIDCEXTERN void hash_join(hash_table *, hash_link *)
Definition: hash.cc:132
#define Must(condition)
Like assert() but throws an exception instead of aborting the process.
Definition: TextException.h:69
char * p
Definition: membanger.c:43
static void Checkpoint(const Pointer &mgr, const char *reason)
Definition: PeerPoolMgr.cc:261
int conn
the current server connection FD
Definition: Transport.cc:26
void closeN(size_t count)
Definition: pconn.cc:113
PconnModule()
Definition: pconn.cc:521
void dumpHash(StoreEntry *e) const
Definition: pconn.cc:364
time_t timeLeft(const time_t idleTimeout) const
Definition: Connection.cc:120
const char * descr
Definition: pconn.h:156
static PconnModule * instance
Definition: pconn.h:190
char * toUrl(char *buf, unsigned int len) const
Definition: Address.cc:884
void noteConnectionAdded()
Definition: pconn.h:144
int count() const
Definition: pconn.h:63
void push(const Comm::ConnectionPointer &serverConn, const char *domain)
Definition: pconn.cc:407
int shutting_down
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:24
void const char HLPCB void * data
Definition: stub_helper.cc:16
int commUnsetConnTimeout(const Comm::ConnectionPointer &conn)
Definition: comm.cc:578
Comm::ConnectionPointer findUseable(const Comm::ConnectionPointer &key)
Definition: pconn.cc:245
~IdleConnList()
Definition: pconn.cc:50
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Debug.h:124
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
#define FD_DESC_SZ
Definition: defines.h:46
SQUIDCEXTERN hash_link * hash_lookup(hash_table *, const void *)
Definition: hash.cc:147
virtual void endingShutdown()
Definition: pconn.cc:323
void comm_read(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer &callback)
Definition: Read.h:59
void push(const Comm::ConnectionPointer &conn)
Pass control of the connection to the idle list.
Definition: pconn.cc:168
void findAndClose(const Comm::ConnectionPointer &conn)
Definition: pconn.cc:284
Comm::ConnectionPointer pop()
get first conn which is not pending read fd.
Definition: pconn.cc:213
void noteConnectionRemoved()
Definition: pconn.h:145
void unlinkList(IdleConnList *list)
Definition: pconn.cc:499
int size_
Definition: pconn.h:87
void notifyManager(const char *reason)
Definition: pconn.cc:471
SQUIDCEXTERN hash_table * hash_create(HASHCMP *, int, HASHHASH *)
Definition: hash.cc:109
SQUIDCEXTERN void hashFreeItems(hash_table *, HASHFREE *)
Definition: hash.cc:256
RefCount< Comm::Connection > ConnectionPointer
Definition: forward.h:25
static CTCB Timeout
Definition: pconn.h:74
#define LOCAL_ARRAY(type, name, size)
Definition: leakcheck.h:18
bool isAnyAddr() const
Definition: Address.cc:170
void clearHandlers(const Comm::ConnectionPointer &conn)
Definition: pconn.cc:160
IdleConnList(const char *key, PconnPool *parent)
Definition: pconn.cc:34
int unsigned int const char *desc STUB void int len
Definition: stub_fd.cc:20
int count() const
Definition: pconn.h:143
void const char * buf
Definition: stub_helper.cc:16
std::ostream & HERE(std::ostream &s)
Definition: Debug.h:153
Flag
Definition: Flag.h:15
Ip::Address local
Definition: Connection.h:135
time_t serverIdlePconn
Definition: SquidConfig.h:114
void comm_read_cancel(int fd, IOCB *callback, void *data)
Definition: Read.cc:179
SQUIDCEXTERN hash_link * hash_next(hash_table *)
Definition: hash.cc:192
void dumpHist(StoreEntry *e) const
Definition: pconn.cc:346
PconnPool(const char *aDescription, const CbcPointer< PeerPoolMgr > &aMgr)
Definition: pconn.cc:378
Ip::Address remote
Definition: Connection.h:138
hash_table * table
Definition: pconn.h:155
bool isAvailable(int i) const
Definition: pconn.cc:197
SQUIDCEXTERN const char * hashKeyStr(const hash_link *)
Definition: hash.cc:317
int hist[PCONN_HIST_SZ]
Definition: pconn.h:154
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
struct SquidConfig::@97 Timeout
#define SQUIDHOSTNAMELEN
Definition: rfc2181.h:30
static PconnModule * GetInstance()
Definition: pconn.cc:527
int const char size_t
Definition: stub_liblog.cc:86
SQUIDCEXTERN void hash_remove_link(hash_table *, hash_link *)
Definition: hash.cc:224
PconnPool * parent_
Definition: pconn.h:94
CommCbFunPtrCallT< Dialer > * commCbCall(int debugSection, int debugLevel, const char *callName, const Dialer &dialer)
Definition: CommCalls.h:342
Pools pools
all live pools
Definition: pconn.h:188
int matchIPAddr(const Address &rhs) const
Definition: Address.cc:703
Comm::ConnectionPointer * theList_
Definition: pconn.h:82
void add(PconnPool *)
Definition: pconn.cc:544
bool removeAt(int index)
Definition: pconn.cc:90
#define PCONN_FDS_SZ
Definition: pconn.cc:26
static void DumpWrapper(StoreEntry *e)
Definition: pconn.cc:570
#define xfree
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:881
int commSetConnTimeout(const Comm::ConnectionPointer &conn, int timeout, AsyncCall::Pointer &callback)
Definition: comm.cc:552
int capacity_
Number of entries theList can currently hold without re-allocating (capacity).
Definition: pconn.h:85
#define PCONN_HIST_SZ
Definition: pconn.h:34
class SquidConfig Config
Definition: SquidConfig.cc:12
int theCount
the number of pooled connections
Definition: pconn.h:158
#define NULL
Definition: types.h:166
OBJH dump
Definition: pconn.h:184
int HASHCMP(const void *, const void *)
Definition: hash.h:13
Comm::ConnectionPointer pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
Definition: pconn.cc:444

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors