set timeout to readtimeout after receiving a chunk (bb #1540).

better checks and documentation for MaxQueue (bb #1521).

git-svn: trunk@5023

ChangeLog

@@ -1,3 +1,9 @@
+Mon Apr  6 12:02:38 EEST 2009 (edwin)
+-------------------------------------
+ * clamd/server-th.c, shared/optparser.c: set timeout to readtimeout
+ after receiving a chunk (bb #1540).  better checks and documentation
+ for MaxQueue (bb #1521).
+
 Fri Apr  3 15:30:34 CEST 2009 (tk)
 ----------------------------------
  * freshclam: short-term blacklisting of faulty mirrors;

clamd/server-th.c

@@ -603,13 +603,16 @@ static const unsigned char* parse_dispatch_cmd(client_conn_t *conn, struct fd_bu
 }
 
 //static const unsigned char* parse_dispatch_cmd(client_conn_t *conn, struct fd_buf *buf, size_t *ppos, int *error, const struct optstruct *opts, int readtimeout)
-static int handle_stream(client_conn_t *conn, struct fd_buf *buf, const struct optstruct *opts, int *error, size_t *ppos)
+static int handle_stream(client_conn_t *conn, struct fd_buf *buf, const struct optstruct *opts, int *error, size_t *ppos, int readtimeout)
 {
     int rc;
     size_t pos = *ppos;
     size_t cmdlen;
 
     logg("$mode == MODE_STREAM\n");
+    /* we received a chunk, set readtimeout */
+    time(&buf->timeout_at);
+    buf->timeout_at += readtimeout;
     if (!buf->chunksize) {
 	/* read chunksize */
 	if (buf->off >= 4) {
@@ -941,22 +944,40 @@ int recvloop_th(int *socketds, unsigned nsockets, struct cl_engine *engine, unsi
 
 #if !defined(C_WINDOWS) && defined(RLIMIT_NOFILE)
     if (getrlimit(RLIMIT_NOFILE, &rlim) == 0) {
-	if (max_queue*4 > rlim.rlim_cur) {
-	    max_queue = rlim.rlim_cur / 4;
-	    logg("^MaxQueue value too high, lowering to: %d\n", max_queue);
-	} else if (max_queue < 2*max_threads) {
-	    /* increase it but only if it doesn't exceed limit otherwise */
-	    int newmax = 2*max_threads;
-	    if (newmax*4 > rlim.rlim_cur)
-		newmax = rlim.rlim_cur/4;
-	    if (max_queue < newmax) {
-		max_queue = newmax;
-		logg("^MaxQueue is lower than twice MaxThreads, increasing to: %d\n", max_queue);
-	    }
+	/* don't warn if default value is too high, silently fix it */
+	unsigned warn = optget(opts, "MaxQueue")->active;
+	const unsigned clamdfiles = 6;
+	/* Condition to not run out of file descriptors:
+	 * MaxThreads * MaxRecursion + (MaxQueue - MaxThreads) + CLAMDFILES < RLIMIT_NOFILE 
+	 * CLAMDFILES is 6: 3 standard FD + logfile + 2 FD for reloading the DB
+	 * */
+	opt = optget(opts,"MaxRecursion");
+	unsigned maxrec = opt->numarg;
+	int max_max_queue = rlim.rlim_cur - maxrec * max_threads - clamdfiles + max_threads;
+	if (max_queue < max_threads) {
+	    max_queue = max_threads;
+	    if (warn)
+		logg("^MaxQueue value too low, increasing to: %d\n", max_queue);
+	}
+	if (max_max_queue < max_threads) {
+	    logg("^MaxThreads * MaxRecursion is too high: %d, open file descriptor limit is: %d\n",
+		 maxrec*max_threads, rlim.rlim_cur);
+	    max_max_queue = max_threads;
+	}
+	if (max_queue > max_max_queue) {
+	    max_queue = max_max_queue;
+	    if (warn)
+		logg("^MaxQueue value too high, lowering to: %d\n", max_queue);
+	} else if (max_queue < 2*max_threads && max_queue < max_max_queue) {
+	    max_queue = 2*max_threads;
+	    if (max_queue > max_max_queue)
+		max_queue = max_max_queue;
+	    /* always warn here */
+	    logg("^MaxQueue is lower than twice MaxThreads, increasing to: %d\n", max_queue);
 	}
     }
 #endif
-
+    logg("*MaxQueue set to: %d\n", max_queue);
     acceptdata.max_queue = max_queue;
 
     if(optget(opts, "ClamukoScanOnAccess")->enabled)
@@ -1156,7 +1177,7 @@ int recvloop_th(int *socketds, unsigned nsockets, struct cl_engine *engine, unsi
 			logg("$Garbage: %s\n", buf->buffer);
 			error = 1;
 		    } else if (buf->mode == MODE_STREAM) {
-			rc = handle_stream(&conn, buf, opts, &error, &pos);
+			rc = handle_stream(&conn, buf, opts, &error, &pos, readtimeout);
 			if (rc == -1)
 			    break;
 			else

shared/optparser.c

@@ -197,7 +197,7 @@ const struct clam_option clam_options[] = {
 
     { "ReadTimeout", NULL, 0, TYPE_NUMBER, MATCH_NUMBER, 120, NULL, 0, OPT_MILTER, "Waiting for data from clamd will timeout after this time (seconds).\nValue of 0 disables the timeout.", "300" },
 
-    { "MaxQueue", NULL, 0, TYPE_NUMBER, MATCH_NUMBER, 100, NULL, 0, OPT_CLAMD, "Maximum number of queued items (including those being processed)\nWARNING: you shouldn't increase this beyond 512, since you may run out of\nfile descriptors (usual max is 1024)\n", "200" },
+    { "MaxQueue", NULL, 0, TYPE_NUMBER, MATCH_NUMBER, 100, NULL, 0, OPT_CLAMD, "Maximum number of queued items (including those being processed by MaxThreads threads)\nIt is recommended to have this value at least twice MaxThreads if possible.\nWARNING: you shouldn't increase this too much to avoid running out  of file descriptors, the following condition should hold: MaxThreads*MaxRecursion + MaxQueue < RLIMIT_NOFILE (usual max is 1024)\n", "200" },
 
     { "IdleTimeout", NULL, 0, TYPE_NUMBER, MATCH_NUMBER, 30, NULL, 0, OPT_CLAMD, "This option specifies how long (in seconds) the process should wait\nfor a new job.", "60" },