Hi all,
The attached patch is working ok for me, allows > 1024 fds on Linux
using polled I/O. This is achieved by removing all references to fd_set
in the code if polled I/O is on.
For example, delay_pools.c uses fd_set as a convenience and this has
been replaced with an array of chars.
Has been running for quite some time under load. Remember to do ulimit
-n 2048 (or similar) before running configure :-)
Cheers,
Grahame
--
Grahame Bowland Email: grahame@ucs.uwa.edu.au
University Communications Services Phone: +61 8 9380 1175
The University of Western Australia Fax: +61 8 9380 1109
CRICOS: 00126G
? config.cache
? config.log
? config.status
? makefile
? stable6.diff
? auth_modules/Makefile
? auth_modules/LDAP/Makefile
? auth_modules/MSNT/Makefile
? auth_modules/NCSA/Makefile
? auth_modules/PAM/Makefile
? auth_modules/SMB/Makefile
? auth_modules/YP/Makefile
? auth_modules/getpwnam/Makefile
? contrib/Makefile
? errors/Makefile
? icons/Makefile
? include/autoconf.h
? include/config.h
? lib/Makefile
? scripts/Makefile
? scripts/RunAccel
? scripts/RunCache
? snmplib/Makefile
? src/Makefile
? src/cachemgr.cgi
? src/cf.data
? src/cf_gen
? src/cf_gen_defines.h
? src/cf_parser.c
? src/client
? src/globals.c
? src/repl_modules.c
? src/squid
? src/squid.conf
? src/store_modules.c
? src/string_arrays.c
? src/unlinkd
? src/fs/Makefile
? src/fs/aufs/Makefile
? src/fs/ufs/Makefile
? src/repl/Makefile
? src/repl/stamp
? src/repl/heap/Makefile
? src/repl/lru/Makefile
Index: src/comm.c
===================================================================
RCS file: /var/cvs/squid/src/comm.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- src/comm.c 15 Apr 2002 10:14:59 -0000 1.1.1.1
+++ src/comm.c 15 Apr 2002 11:17:40 -0000 1.2
@@ -1,6 +1,6 @@
/*
- * $Id: comm.c,v 1.1.1.1 2002/04/15 10:14:59 grahame Exp $
+ * $Id: comm.c,v 1.2 2002/04/15 11:17:40 grahame Exp $
*
* DEBUG: section 5 Socket Functions
* AUTHOR: Harvest Derived
@@ -640,12 +640,16 @@
if (type & COMM_SELECT_READ) {
F->read_handler = handler;
F->read_data = client_data;
+#if !HAVE_POLL
commUpdateReadBits(fd, handler);
+#endif
}
if (type & COMM_SELECT_WRITE) {
F->write_handler = handler;
F->write_data = client_data;
+#if !HAVE_POLL
commUpdateWriteBits(fd, handler);
+#endif
}
if (timeout)
F->timeout = squid_curtime + timeout;
Index: src/comm_select.c
===================================================================
RCS file: /var/cvs/squid/src/comm_select.c,v
retrieving revision 1.1.1.1
retrieving revision 1.3
diff -u -r1.1.1.1 -r1.3
--- src/comm_select.c 15 Apr 2002 10:14:59 -0000 1.1.1.1
+++ src/comm_select.c 15 Apr 2002 11:17:40 -0000 1.3
@@ -1,6 +1,6 @@
/*
- * $Id: comm_select.c,v 1.1.1.1 2002/04/15 10:14:59 grahame Exp $
+ * $Id: comm_select.c,v 1.3 2002/04/15 11:17:40 grahame Exp $
*
* DEBUG: section 5 Socket Functions
*
@@ -54,7 +54,9 @@
static int fdIsDns(int fd);
static int commDeferRead(int fd);
static void checkTimeouts(void);
+#if !HAVE_POLL
static OBJH commIncomingStats;
+#endif
#if HAVE_POLL
static int comm_check_incoming_poll_handlers(int nfds, int *fds);
static void comm_poll_dns_incoming(void);
@@ -63,11 +65,15 @@
static void comm_select_dns_incoming(void);
#endif
+#if !HAVE_POLL
static struct timeval zero_tv;
static fd_set global_readfds;
static fd_set global_writefds;
+#endif
+#if !HAVE_POLL
static int nreadfds;
static int nwritefds;
+#endif
/*
* Automatic tuning for incoming requests:
@@ -308,7 +314,7 @@
{
struct pollfd pfds[SQUID_MAXFD];
#if DELAY_POOLS
- fd_set slowfds;
+ char slowfds[SQUID_MAXFD];
#endif
PF *hdl = NULL;
int fd;
@@ -329,7 +335,7 @@
/* Handle any fs callbacks that need doing */
storeDirCallback();
#if DELAY_POOLS
- FD_ZERO(&slowfds);
+ bzero(slowfds, sizeof(slowfds));
#endif
if (commCheckICPIncoming)
comm_poll_icp_incoming();
@@ -354,7 +360,11 @@
#if DELAY_POOLS
case -1:
events |= POLLRDNORM;
- FD_SET(i, &slowfds);
+ if (i < SQUID_MAXFD) {
+ slowfds[i] = 1;
+ } else {
+ debug(5, 0) ("WARNING: tried to set FD %d > SQUID_MAXFD in slowfds.", i);
+ }
break;
#endif
default:
@@ -424,7 +434,7 @@
if (NULL == (hdl = F->read_handler))
(void) 0;
#if DELAY_POOLS
- else if (FD_ISSET(fd, &slowfds))
+ else if (slowfds[fd])
commAddSlowFd(fd);
#endif
else {
@@ -514,6 +524,7 @@
#else
+#if !HAVE_POLL
static int
comm_check_incoming_select_handlers(int nfds, int *fds)
{
@@ -572,6 +583,7 @@
}
return incame;
}
+#endif
static void
comm_select_icp_incoming(void)
@@ -632,7 +644,7 @@
fd_set readfds;
fd_set writefds;
#if DELAY_POOLS
- fd_set slowfds;
+ char slowfds[SQUID_MAXFD];
#endif
PF *hdl = NULL;
int fd;
@@ -657,7 +669,7 @@
getCurrentTime();
#endif
#if DELAY_POOLS
- FD_ZERO(&slowfds);
+ bzero(slowfds, sizeof(slowfds));
#endif
/* Handle any fs callbacks that need doing */
storeDirCallback();
@@ -692,7 +704,11 @@
break;
#if DELAY_POOLS
case -1:
- FD_SET(fd, &slowfds);
+ if (i < SQUID_MAXFD) {
+ slowfds[i] = 1;
+ } else {
+ debug(5, 0) ("WARNING: tried to set FD %d > SQUID_MAxFD in slowfds.");
+ }
break;
#endif
default:
@@ -789,7 +805,7 @@
if (NULL == (hdl = F->read_handler))
(void) 0;
#if DELAY_POOLS
- else if (FD_ISSET(fd, &slowfds))
+ else if (slowfds[fd])
commAddSlowFd(fd);
#endif
else {
@@ -913,6 +929,7 @@
statHistCount(&statCounter.comm_dns_incoming, nevents);
}
+#if !HAVE_POLL
void
comm_select_init(void)
{
@@ -925,6 +942,7 @@
FD_ZERO(&global_writefds);
nreadfds = nwritefds = 0;
}
+#endif
#if !HAVE_POLL
/*
@@ -1021,6 +1039,7 @@
}
}
+#if !HAVE_POLL
static void
commIncomingStats(StoreEntry * sentry)
{
@@ -1052,7 +1071,9 @@
#endif
statHistDump(&f->comm_http_incoming, sentry, statHistIntDumper);
}
+#endif
+#if !HAVE_POLL
void
commUpdateReadBits(int fd, PF * handler)
{
@@ -1064,7 +1085,9 @@
nreadfds--;
}
}
+#endif
+#if !HAVE_POLL
void
commUpdateWriteBits(int fd, PF * handler)
{
@@ -1076,6 +1099,7 @@
nwritefds--;
}
}
+#endif
/* Called by async-io or diskd to speed up the polling */
void
Index: src/delay_pools.c
===================================================================
RCS file: /var/cvs/squid/src/delay_pools.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- src/delay_pools.c 15 Apr 2002 10:14:59 -0000 1.1.1.1
+++ src/delay_pools.c 15 Apr 2002 10:33:21 -0000 1.2
@@ -1,6 +1,6 @@
/*
- * $Id: delay_pools.c,v 1.1.1.1 2002/04/15 10:14:59 grahame Exp $
+ * $Id: delay_pools.c,v 1.2 2002/04/15 10:33:21 grahame Exp $
*
* DEBUG: section 77 Delay Pools
* AUTHOR: David Luyer <david@luyer.net>
@@ -89,7 +89,7 @@
typedef union _delayPool delayPool;
static delayPool *delay_data = NULL;
-static fd_set delay_no_delay;
+static char delay_no_delay[SQUID_MAXFD];
static time_t delay_pools_last_update = 0;
static hash_table *delay_id_ptr_hash = NULL;
static long memory_used = 0;
@@ -134,7 +134,7 @@
delayPoolsInit(void)
{
delay_pools_last_update = getCurrentTime();
- FD_ZERO(&delay_no_delay);
+ bzero(delay_no_delay, sizeof(delay_no_delay));
cachemgrRegister("delay", "Delay Pool Levels", delayPoolStats, 0, 1);
}
@@ -283,19 +283,32 @@
void
delaySetNoDelay(int fd)
{
- FD_SET(fd, &delay_no_delay);
+ if (fd < SQUID_MAXFD) {
+ delay_no_delay[fd] = 1;
+ } else {
+ debug(77, 1) ("delaySetNoDelay: can't set fd %d, too high\n", fd);
+ }
}
void
delayClearNoDelay(int fd)
{
- FD_CLR(fd, &delay_no_delay);
+ if (fd < SQUID_MAXFD) {
+ delay_no_delay[fd] = 0;
+ } else {
+ debug(77, 1) ("delayClearNoDelay: can't clear fd %d, too high\n", fd);
+ }
}
int
delayIsNoDelay(int fd)
{
- return FD_ISSET(fd, &delay_no_delay);
+ if (fd < SQUID_MAXFD) {
+ return delay_no_delay[fd];
+ } else {
+ debug(77, 1) ("delayIsNoDelay: can't check fd %d, too high\n", fd);
+ return 0;
+ }
}
static delay_id
Index: src/fd.c
===================================================================
RCS file: /var/cvs/squid/src/fd.c,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- src/fd.c 15 Apr 2002 10:14:59 -0000 1.1.1.1
+++ src/fd.c 15 Apr 2002 11:17:40 -0000 1.2
@@ -1,6 +1,6 @@
/*
- * $Id: fd.c,v 1.1.1.1 2002/04/15 10:14:59 grahame Exp $
+ * $Id: fd.c,v 1.2 2002/04/15 11:17:40 grahame Exp $
*
* DEBUG: section 51 Filedescriptor Functions
* AUTHOR: Duane Wessels
@@ -84,8 +84,10 @@
F->flags.open = 0;
fdUpdateBiggest(fd, 0);
Number_FD--;
+#if !HAVE_POLL
commUpdateReadBits(fd, NULL);
commUpdateWriteBits(fd, NULL);
+#endif
memset(F, '\0', sizeof(fde));
F->timeout = 0;
}
Index: src/main.c
===================================================================
RCS file: /var/cvs/squid/src/main.c,v
retrieving revision 1.1.1.1
retrieving revision 1.3
diff -u -r1.1.1.1 -r1.3
--- src/main.c 15 Apr 2002 10:15:00 -0000 1.1.1.1
+++ src/main.c 15 Apr 2002 11:17:40 -0000 1.3
@@ -1,6 +1,6 @@
/*
- * $Id: main.c,v 1.1.1.1 2002/04/15 10:15:00 grahame Exp $
+ * $Id: main.c,v 1.3 2002/04/15 11:17:40 grahame Exp $
*
* DEBUG: section 1 Startup and Main Loop
* AUTHOR: Harvest Derived
@@ -477,6 +477,7 @@
CONFIG_HOST_TYPE);
debug(1, 1) ("Process ID %d\n", (int) getpid());
debug(1, 1) ("With %d file descriptors available\n", Squid_MaxFD);
+ debug(1, 1) ("(statically compiled for %d file descriptors\n", SQUID_MAXFD);
if (!configured_once)
disk_init(); /* disk_init must go before ipcache_init() */
@@ -574,8 +575,13 @@
mode_t oldmask;
debug_log = stderr;
+#if !HAVE_POLL
+/* the poll code should now never use fd_set, so we
+ * need only worry if using another method
+ */
if (FD_SETSIZE < Squid_MaxFD)
Squid_MaxFD = FD_SETSIZE;
+#endif
/* call mallopt() before anything else */
#if HAVE_MALLOPT
@@ -642,7 +648,9 @@
#if TEST_ACCESS
comm_init();
+#if !HAVE_POLL
comm_select_init();
+#endif
mainInitialize();
test_access();
return 0;
@@ -677,7 +685,9 @@
/* init comm module */
comm_init();
+#if !HAVE_POLL
comm_select_init();
+#endif
if (opt_no_daemon) {
/* we have to init fdstat here. */
Received on Sun Apr 21 2002 - 20:34:10 MDT
This archive was generated by hypermail pre-2.1.9 : Tue Dec 09 2003 - 16:15:17 MST