pconn.cc
Go to the documentation of this file.
1/*
2 * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 48 Persistent Connections */
10
11#include "squid.h"
12#include "CachePeer.h"
13#include "comm.h"
14#include "comm/Connection.h"
15#include "comm/Read.h"
16#include "fd.h"
17#include "fde.h"
18#include "globals.h"
19#include "mgr/Registration.h"
20#include "neighbors.h"
21#include "pconn.h"
22#include "PeerPoolMgr.h"
23#include "SquidConfig.h"
24#include "Store.h"
25
26#define PCONN_FDS_SZ 8 /* pconn set size, increase for better memcache hit rate */
27
28//TODO: re-attach to MemPools. WAS: static MemAllocator *pconn_fds_pool = NULL;
31
32/* ========== IdleConnList ============================================ */
33
34IdleConnList::IdleConnList(const char *aKey, PconnPool *thePool) :
35 capacity_(PCONN_FDS_SZ),
36 size_(0),
37 parent_(thePool)
38{
39 //Initialize hash_link members
40 key = xstrdup(aKey);
41 next = nullptr;
42
44
46
47// TODO: re-attach to MemPools. WAS: theList = (?? *)pconn_fds_pool->alloc();
48}
49
51{
52 if (parent_)
53 parent_->unlinkList(this);
54
55 if (size_) {
56 parent_ = nullptr; // prevent reentrant notifications and deletions
58 }
59
60 delete[] theList_;
61
62 xfree(key);
63}
64
71int
73{
74 for (int index = size_ - 1; index >= 0; --index) {
75 if (conn->fd == theList_[index]->fd) {
76 debugs(48, 3, "found " << conn << " at index " << index);
77 return index;
78 }
79 }
80
81 debugs(48, 2, conn << " NOT FOUND!");
82 return -1;
83}
84
89bool
91{
92 if (index < 0 || index >= size_)
93 return false;
94
95 // shuffle the remaining entries to fill the new gap.
96 for (; index < size_ - 1; ++index)
97 theList_[index] = theList_[index + 1];
98 theList_[--size_] = nullptr;
99
100 if (parent_) {
102 if (size_ == 0) {
103 debugs(48, 3, "deleting " << hashKeyStr(this));
104 delete this;
105 }
106 }
107
108 return true;
109}
110
111// almost a duplicate of removeFD. But drops multiple entries.
112void
114{
115 if (n < 1) {
116 debugs(48, 2, "Nothing to do.");
117 return;
118 } else if (n >= (size_t)size_) {
119 debugs(48, 2, "Closing all entries.");
120 while (size_ > 0) {
122 theList_[size_] = nullptr;
124 conn->close();
125 if (parent_)
127 }
128 } else { //if (n < size_)
129 debugs(48, 2, "Closing " << n << " of " << size_ << " entries.");
130
131 size_t index;
132 // ensure the first N entries are closed
133 for (index = 0; index < n; ++index) {
135 theList_[index] = nullptr;
137 conn->close();
138 if (parent_)
140 }
141 // shuffle the list N down.
142 for (index = 0; index < (size_t)size_ - n; ++index) {
143 theList_[index] = theList_[index + n];
144 }
145 // ensure the last N entries are unset
146 while (index < ((size_t)size_)) {
147 theList_[index] = nullptr;
148 ++index;
149 }
150 size_ -= n;
151 }
152
153 if (parent_ && size_ == 0) {
154 debugs(48, 3, "deleting " << hashKeyStr(this));
155 delete this;
156 }
157}
158
159void
161{
162 debugs(48, 3, "removing close handler for " << conn);
165}
166
167void
169{
170 if (size_ == capacity_) {
171 debugs(48, 3, "growing idle Connection array");
172 capacity_ <<= 1;
173 const Comm::ConnectionPointer *oldList = theList_;
175 for (int index = 0; index < size_; ++index)
176 theList_[index] = oldList[index];
177
178 delete[] oldList;
179 }
180
181 if (parent_)
183
185 ++size_;
186 AsyncCall::Pointer readCall = commCbCall(5,4, "IdleConnList::Read",
188 comm_read(conn, fakeReadBuf_, sizeof(fakeReadBuf_), readCall);
189 AsyncCall::Pointer timeoutCall = commCbCall(5,4, "IdleConnList::Timeout",
191 commSetConnTimeout(conn, conn->timeLeft(Config.Timeout.serverIdlePconn), timeoutCall);
192}
193
196bool
198{
200
201 // connection already closed. useless.
203 return false;
204
205 // our connection early-read/close handler is scheduled to run already. unsafe
206 if (!COMMIO_FD_READCB(conn->fd)->active())
207 return false;
208
209 return true;
210}
211
214{
215 for (int i=size_-1; i>=0; --i) {
216
217 if (!isAvailable(i))
218 continue;
219
220 // our connection timeout handler is scheduled to run already. unsafe for now.
221 // TODO: cancel the pending timeout callback and allow re-use of the conn.
222 if (fd_table[theList_[i]->fd].timeoutHandler == nullptr)
223 continue;
224
225 // finally, a match. pop and return it.
227 clearHandlers(result);
228 /* may delete this */
229 removeAt(i);
230 return result;
231 }
232
234}
235
236/*
237 * XXX this routine isn't terribly efficient - if there's a pending
238 * read event (which signifies the fd will close in the next IO loop!)
239 * we ignore the FD and move onto the next one. This means, as an example,
240 * if we have a lot of FDs open to a very popular server and we get a bunch
241 * of requests JUST as they timeout (say, it shuts down) we'll be wasting
242 * quite a bit of CPU. Just keep it in mind.
243 */
246{
247 assert(size_);
248
249 // small optimization: do the constant bool tests only once.
250 const bool keyCheckAddr = !aKey->local.isAnyAddr();
251 const bool keyCheckPort = aKey->local.port() > 0;
252
253 for (int i=size_-1; i>=0; --i) {
254
255 if (!isAvailable(i))
256 continue;
257
258 // local end port is required, but do not match.
259 if (keyCheckPort && aKey->local.port() != theList_[i]->local.port())
260 continue;
261
262 // local address is required, but does not match.
263 if (keyCheckAddr && aKey->local.matchIPAddr(theList_[i]->local) != 0)
264 continue;
265
266 // our connection timeout handler is scheduled to run already. unsafe for now.
267 // TODO: cancel the pending timeout callback and allow re-use of the conn.
268 if (fd_table[theList_[i]->fd].timeoutHandler == nullptr)
269 continue;
270
271 // finally, a match. pop and return it.
273 clearHandlers(result);
274 /* may delete this */
275 removeAt(i);
276 return result;
277 }
278
280}
281
282/* might delete list */
283void
285{
286 const int index = findIndexOf(conn);
287 if (index >= 0) {
288 if (parent_)
289 parent_->notifyManager("idle conn closure");
291 /* might delete this */
292 removeAt(index);
293 conn->close();
294 }
295}
296
297void
298IdleConnList::Read(const Comm::ConnectionPointer &conn, char *, size_t len, Comm::Flag flag, int, void *data)
299{
300 debugs(48, 3, len << " bytes from " << conn);
301
302 if (flag == Comm::ERR_CLOSING) {
303 debugs(48, 3, "Comm::ERR_CLOSING from " << conn);
304 /* Bail out on Comm::ERR_CLOSING - may happen when shutdown aborts our idle FD */
305 return;
306 }
307
308 IdleConnList *list = (IdleConnList *) data;
309 /* may delete list/data */
310 list->findAndClose(conn);
311}
312
313void
315{
316 debugs(48, 3, io.conn);
317 IdleConnList *list = static_cast<IdleConnList *>(io.data);
318 /* may delete list/data */
319 list->findAndClose(io.conn);
320}
321
322void
324{
325 closeN(size_);
326}
327
328/* ========== PconnPool PRIVATE FUNCTIONS ============================================ */
329
330const char *
331PconnPool::key(const Comm::ConnectionPointer &destLink, const char *domain)
332{
333 LOCAL_ARRAY(char, buf, SQUIDHOSTNAMELEN * 3 + 10);
334
335 destLink->remote.toUrl(buf, SQUIDHOSTNAMELEN * 3 + 10);
336
337 // when connecting through a cache_peer, ignore the final destination
338 if (destLink->getPeer())
339 domain = nullptr;
340
341 if (domain) {
342 const int used = strlen(buf);
343 snprintf(buf+used, SQUIDHOSTNAMELEN * 3 + 10-used, "/%s", domain);
344 }
345
346 debugs(48,6,"PconnPool::key(" << destLink << ", " << (domain?domain:"[no domain]") << ") is {" << buf << "}" );
347 return buf;
348}
349
350void
352{
354 "%s persistent connection counts:\n"
355 "\n"
356 "\t Requests\t Connection Count\n"
357 "\t --------\t ----------------\n",
358 descr);
359
360 for (int i = 0; i < PCONN_HIST_SZ; ++i) {
361 if (hist[i] == 0)
362 continue;
363
364 storeAppendPrintf(e, "\t%d\t%d\n", i, hist[i]);
365 }
366}
367
368void
370{
371 hash_table *hid = table;
372 hash_first(hid);
373
374 int i = 0;
375 for (hash_link *walker = hash_next(hid); walker; walker = hash_next(hid)) {
376 storeAppendPrintf(e, "\t item %d:\t%s\n", i, (char *)(walker->key));
377 ++i;
378 }
379}
380
381/* ========== PconnPool PUBLIC FUNCTIONS ============================================ */
382
383PconnPool::PconnPool(const char *aDescr, const CbcPointer<PeerPoolMgr> &aMgr):
384 table(nullptr), descr(aDescr),
385 mgr(aMgr),
386 theCount(0)
387{
388 int i;
389 table = hash_create((HASHCMP *) strcmp, 229, hash_string);
390
391 for (i = 0; i < PCONN_HIST_SZ; ++i)
392 hist[i] = 0;
393
395}
396
397static void
398DeleteIdleConnList(void *hashItem)
399{
400 delete static_cast<IdleConnList*>(hashItem);
401}
402
404{
408 descr = nullptr;
409}
410
411void
413{
414 if (fdUsageHigh()) {
415 debugs(48, 3, "Not many unused FDs");
416 conn->close();
417 return;
418 } else if (shutting_down) {
419 conn->close();
420 debugs(48, 3, "Squid is shutting down. Refusing to do anything");
421 return;
422 }
423 // TODO: also close used pconns if we exceed peer max-conn limit
424
425 const char *aKey = key(conn, domain);
426 IdleConnList *list = (IdleConnList *) hash_lookup(table, aKey);
427
428 if (list == nullptr) {
429 list = new IdleConnList(aKey, this);
430 debugs(48, 3, "new IdleConnList for {" << hashKeyStr(list) << "}" );
431 hash_join(table, list);
432 } else {
433 debugs(48, 3, "found IdleConnList for {" << hashKeyStr(list) << "}" );
434 }
435
436 list->push(conn);
438
439 LOCAL_ARRAY(char, desc, FD_DESC_SZ);
440 snprintf(desc, FD_DESC_SZ, "Idle server: %s", aKey);
441 fd_note(conn->fd, desc);
442 debugs(48, 3, "pushed " << conn << " for " << aKey);
443
444 // successful push notifications resume multi-connection opening sequence
445 notifyManager("push");
446}
447
449PconnPool::pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
450{
451 // always call shared pool first because we need to close an idle
452 // connection there if we have to use a standby connection.
453 if (const auto direct = popStored(dest, domain, keepOpen))
454 return direct;
455
456 // either there was no pconn to pop or this is not a retriable xaction
457 if (const auto peer = dest->getPeer()) {
458 if (peer->standby.pool)
459 return peer->standby.pool->popStored(dest, domain, true);
460 }
461
462 return nullptr;
463}
464
468PconnPool::popStored(const Comm::ConnectionPointer &dest, const char *domain, const bool keepOpen)
469{
470 const char * aKey = key(dest, domain);
471
472 IdleConnList *list = (IdleConnList *)hash_lookup(table, aKey);
473 if (list == nullptr) {
474 debugs(48, 3, "lookup for key {" << aKey << "} failed.");
475 // failure notifications resume standby conn creation after fdUsageHigh
476 notifyManager("pop lookup failure");
478 } else {
479 debugs(48, 3, "found " << hashKeyStr(list) <<
480 (keepOpen ? " to use" : " to kill"));
481 }
482
483 if (const auto popped = list->findUseable(dest)) { // may delete list
484 // successful pop notifications replenish standby connections pool
485 notifyManager("pop");
486
487 if (keepOpen)
488 return popped;
489
490 popped->close();
492 }
493
494 // failure notifications resume standby conn creation after fdUsageHigh
495 notifyManager("pop usability failure");
497}
498
499void
500PconnPool::notifyManager(const char *reason)
501{
502 if (mgr.valid())
504}
505
506void
508{
509 hash_table *hid = table;
510 hash_first(hid);
511
512 // close N connections, one per list, to treat all lists "fairly"
513 for (int i = 0; i < n && count(); ++i) {
514
515 hash_link *current = hash_next(hid);
516 if (!current) {
517 hash_first(hid);
518 current = hash_next(hid);
519 Must(current); // must have one because the count() was positive
520 }
521
522 // may delete current
523 static_cast<IdleConnList*>(current)->closeN(1);
524 }
525}
526
527void
529{
530 theCount -= list->count();
531 assert(theCount >= 0);
532 hash_remove_link(table, list);
533}
534
535void
537{
538 if (uses >= PCONN_HIST_SZ)
539 uses = PCONN_HIST_SZ - 1;
540
541 ++hist[uses];
542}
543
544/* ========== PconnModule ============================================ */
545
546/*
547 * This simple class exists only for the cache manager
548 */
549
551{
553}
554
557{
558 if (instance == nullptr)
559 instance = new PconnModule;
560
561 return instance;
562}
563
564void
566{
567 Mgr::RegisterAction("pconn",
568 "Persistent Connection Utilization Histograms",
569 DumpWrapper, 0, 1);
570}
571
572void
574{
575 pools.insert(aPool);
576}
577
578void
580{
581 pools.erase(aPool);
582}
583
584void
586{
587 typedef Pools::const_iterator PCI;
588 int i = 0; // TODO: Why number pools if they all have names?
589 for (PCI p = pools.begin(); p != pools.end(); ++p, ++i) {
590 // TODO: Let each pool dump itself the way it wants to.
591 storeAppendPrintf(e, "\n Pool %d Stats\n", i);
592 (*p)->dumpHist(e);
593 storeAppendPrintf(e, "\n Pool %d Hash Table\n",i);
594 (*p)->dumpHash(e);
595 }
596}
597
598void
600{
602}
603
CommCbFunPtrCallT< Dialer > * commCbCall(int debugSection, int debugLevel, const char *callName, const Dialer &dialer)
Definition: CommCalls.h:342
#define COMMIO_FD_READCB(fd)
Definition: IoCallback.h:78
void comm_read_cancel(int fd, IOCB *callback, void *data)
Definition: Read.cc:181
void comm_read(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer &callback)
Definition: Read.h:59
class SquidConfig Config
Definition: SquidConfig.cc:12
#define Must(condition)
Definition: TextException.h:71
int conn
the current server connection FD
Definition: Transport.cc:26
#define assert(EX)
Definition: assert.h:19
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:318
Cbc * valid() const
was set and is valid
Definition: CbcPointer.h:41
Comm::ConnectionPointer conn
Definition: CommCalls.h:85
CachePeer * getPeer() const
Definition: Connection.cc:124
Ip::Address remote
Definition: Connection.h:149
Ip::Address local
Definition: Connection.h:146
IdleConnList(const char *key, PconnPool *parent)
Definition: pconn.cc:34
PconnPool * parent_
Definition: pconn.h:92
void findAndClose(const Comm::ConnectionPointer &conn)
Definition: pconn.cc:284
bool removeAt(int index)
Definition: pconn.cc:90
Comm::ConnectionPointer pop()
get first conn which is not pending read fd.
Definition: pconn.cc:213
void clearHandlers(const Comm::ConnectionPointer &conn)
Definition: pconn.cc:160
bool isAvailable(int i) const
Definition: pconn.cc:197
virtual void endingShutdown()
Definition: pconn.cc:323
int count() const
Definition: pconn.h:61
int findIndexOf(const Comm::ConnectionPointer &conn) const
Definition: pconn.cc:72
Comm::ConnectionPointer findUseable(const Comm::ConnectionPointer &key)
Definition: pconn.cc:245
Comm::ConnectionPointer * theList_
Definition: pconn.h:80
void closeN(size_t count)
Definition: pconn.cc:113
void push(const Comm::ConnectionPointer &conn)
Pass control of the connection to the idle list.
Definition: pconn.cc:168
~IdleConnList()
Definition: pconn.cc:50
static CTCB Timeout
Definition: pconn.h:72
char fakeReadBuf_[4096]
Definition: pconn.h:94
int capacity_
Number of entries theList can currently hold without re-allocating (capacity).
Definition: pconn.h:83
static IOCB Read
Definition: pconn.h:71
int size_
Definition: pconn.h:85
int matchIPAddr(const Address &rhs) const
Definition: Address.cc:703
bool isAnyAddr() const
Definition: Address.cc:170
char * toUrl(char *buf, unsigned int len) const
Definition: Address.cc:874
unsigned short port() const
Definition: Address.cc:778
void remove(PconnPool *)
unregister and forget about this pool object
Definition: pconn.cc:579
Pools pools
all live pools
Definition: pconn.h:189
void add(PconnPool *)
Definition: pconn.cc:573
PconnModule()
Definition: pconn.cc:550
static PconnModule * instance
Definition: pconn.h:191
void registerWithCacheManager(void)
Definition: pconn.cc:565
OBJH dump
Definition: pconn.h:185
static PconnModule * GetInstance()
Definition: pconn.cc:556
static void DumpWrapper(StoreEntry *e)
Definition: pconn.cc:599
void closeN(int n)
closes any n connections, regardless of their destination
Definition: pconn.cc:507
int hist[PCONN_HIST_SZ]
Definition: pconn.h:155
PconnPool(const char *aDescription, const CbcPointer< PeerPoolMgr > &aMgr)
Definition: pconn.cc:383
void notifyManager(const char *reason)
Definition: pconn.cc:500
void dumpHash(StoreEntry *e) const
Definition: pconn.cc:369
hash_table * table
Definition: pconn.h:156
void unlinkList(IdleConnList *list)
Definition: pconn.cc:528
void noteConnectionAdded()
Definition: pconn.h:143
Comm::ConnectionPointer popStored(const Comm::ConnectionPointer &dest, const char *domain, const bool keepOpen)
Definition: pconn.cc:468
CbcPointer< PeerPoolMgr > mgr
optional pool manager (for notifications)
Definition: pconn.h:158
~PconnPool()
Definition: pconn.cc:403
void dumpHist(StoreEntry *e) const
Definition: pconn.cc:351
int count() const
Definition: pconn.h:142
const char * descr
Definition: pconn.h:157
static const char * key(const Comm::ConnectionPointer &destLink, const char *domain)
Definition: pconn.cc:331
void noteConnectionRemoved()
Definition: pconn.h:144
void push(const Comm::ConnectionPointer &serverConn, const char *domain)
Definition: pconn.cc:412
Comm::ConnectionPointer pop(const Comm::ConnectionPointer &dest, const char *domain, bool keepOpen)
Definition: pconn.cc:449
void noteUses(int uses)
Definition: pconn.cc:536
int theCount
the number of pooled connections
Definition: pconn.h:159
static void Checkpoint(const Pointer &mgr, const char *reason)
Definition: PeerPoolMgr.cc:241
time_t serverIdlePconn
Definition: SquidConfig.h:118
struct SquidConfig::@98 Timeout
int commSetConnTimeout(const Comm::ConnectionPointer &conn, int timeout, AsyncCall::Pointer &callback)
Definition: comm.cc:589
bool comm_has_incomplete_write(int fd)
Definition: comm.cc:150
int commUnsetConnTimeout(const Comm::ConnectionPointer &conn)
Definition: comm.cc:615
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:196
#define FD_DESC_SZ
Definition: defines.h:32
void fd_note(int fd, const char *s)
Definition: fd.cc:217
int fdUsageHigh(void)
Definition: fd.cc:272
#define fd_table
Definition: fde.h:189
int shutting_down
#define PCONN_HIST_SZ
Definition: pconn.h:32
SQUIDCEXTERN hash_table * hash_create(HASHCMP *, int, HASHHASH *)
Definition: hash.cc:108
SQUIDCEXTERN void hashFreeItems(hash_table *, HASHFREE *)
Definition: hash.cc:252
SQUIDCEXTERN void hash_join(hash_table *, hash_link *)
Definition: hash.cc:131
SQUIDCEXTERN void hash_first(hash_table *)
Definition: hash.cc:172
int HASHCMP(const void *, const void *)
Definition: hash.h:13
SQUIDCEXTERN HASHHASH hash_string
Definition: hash.h:45
SQUIDCEXTERN void hashFreeMemory(hash_table *)
Definition: hash.cc:268
SQUIDCEXTERN hash_link * hash_next(hash_table *)
Definition: hash.cc:188
SQUIDCEXTERN void hash_remove_link(hash_table *, hash_link *)
Definition: hash.cc:220
SQUIDCEXTERN const char * hashKeyStr(const hash_link *)
Definition: hash.cc:313
SQUIDCEXTERN hash_link * hash_lookup(hash_table *, const void *)
Definition: hash.cc:146
RefCount< Comm::Connection > ConnectionPointer
Definition: forward.h:28
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:27
Flag
Definition: Flag.h:15
@ ERR_CLOSING
Definition: Flag.h:25
void RegisterAction(char const *action, char const *desc, OBJH *handler, int pw_req_flag, int atomic)
Definition: Registration.cc:16
#define xfree
#define xstrdup
static void DeleteIdleConnList(void *hashItem)
Definition: pconn.cc:398
#define PCONN_FDS_SZ
Definition: pconn.cc:26
#define SQUIDHOSTNAMELEN
Definition: rfc2181.h:30
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:68
void storeAppendPrintf(StoreEntry *e, const char *fmt,...)
Definition: store.cc:828
int const char size_t
Definition: stub_liblog.cc:86

 

Introduction

Documentation

Support

Miscellaneous

Web Site Translations

Mirrors