From 8765287eb486fc83eafe5a2f29bfc9555b2d3166 Mon Sep 17 00:00:00 2001 From: Tomasz Kojm Date: Mon, 11 Sep 2006 23:10:31 +0000 Subject: [PATCH] add support for hardware acceleration in clamd git-svn: trunk@2239 --- clamav-devel/ChangeLog | 6 + clamav-devel/clamd/clamd.c | 25 +++- clamav-devel/clamd/scanner.c | 4 +- clamav-devel/clamd/scanner.h | 2 + clamav-devel/clamd/session.c | 204 ++++++++++++++++++++++++++ clamav-devel/clamd/session.h | 1 + clamav-devel/clamdscan/client.c | 32 +++- clamav-devel/docs/man/clamd.8.in | 3 + clamav-devel/docs/man/clamd.conf.5.in | 5 + clamav-devel/etc/clamd.conf | 4 + clamav-devel/shared/cfgparser.c | 1 + 11 files changed, 272 insertions(+), 15 deletions(-) diff --git a/clamav-devel/ChangeLog b/clamav-devel/ChangeLog index db4fb5780..db09d9b30 100644 --- a/clamav-devel/ChangeLog +++ b/clamav-devel/ChangeLog @@ -1,3 +1,9 @@ +Tue Sep 12 01:04:39 CEST 2006 (tk) +---------------------------------- + * clamd, clamdscan: add support for hardware acceleration + * etc/clamd.conf: add HardwareAcceleration option + * clamd: add MULTISCAN command (for scanning directories with multiple threads) + Sun Sep 10 22:40:20 BST 2006 (njh) ---------------------------------- * clamav-milter: Fix possible underrun in load balanced configurations diff --git a/clamav-devel/clamd/clamd.c b/clamav-devel/clamd/clamd.c index 6e90399bc..ddf241dc3 100644 --- a/clamav-devel/clamd/clamd.c +++ b/clamav-devel/clamd/clamd.c @@ -281,6 +281,14 @@ int main(int argc, char **argv) if(cfgopt(copt, "LeaveTemporaryFiles")->enabled) cl_settempdir(NULL, 1); + /* fork into background */ + if(!cfgopt(copt, "Foreground")->enabled) { + daemonize(); + if(!debug_mode) + chdir("/"); + } else + foreground = 1; + /* load the database(s) */ dbdir = cfgopt(copt, "DatabaseDirectory")->strarg; logg("#Reading databases from %s\n", dbdir); @@ -290,6 +298,15 @@ int main(int argc, char **argv) logg("Not loading phishing signatures.\n"); } + if(!cfgopt(copt, "HardwareAcceleration")->enabled) { +#ifdef HAVE_HWACCEL + dboptions |= CL_DB_HWACCEL; + logg("Enabling support for hardware acceleration.\n"); +#else + logg("^Support for hardware acceleration not compiled in.\n"); +#endif + } + if((ret = cl_load(dbdir, &root, &sigs, dboptions))) { logg("!%s\n", cl_strerror(ret)); logg_close(); @@ -312,14 +329,6 @@ int main(int argc, char **argv) return 1; } - /* fork into background */ - if(!cfgopt(copt, "Foreground")->enabled) { - daemonize(); - if(!debug_mode) - chdir("/"); - } else - foreground = 1; - if(tcpsock) { lsockets[nlsockets] = tcpserver(copt); if(lsockets[nlsockets] == -1) { diff --git a/clamav-devel/clamd/scanner.c b/clamav-devel/clamd/scanner.c index aae641b9b..69d642d68 100644 --- a/clamav-devel/clamd/scanner.c +++ b/clamav-devel/clamd/scanner.c @@ -70,7 +70,7 @@ dev_t procdev; /* /proc device */ # endif #endif -static int checksymlink(const char *path) +int checksymlink(const char *path) { struct stat statbuf; @@ -255,7 +255,7 @@ int scan(const char *filename, unsigned long int *scanned, const struct cl_node if(!ret) mdprintf(odesc, "%s: OK\n", filename); - mdprintf(odesc, "\n"); /* Terminate response with a blank line boundary */ + /* mdprintf(odesc, "\n"); */ /* Terminate response with a blank line boundary */ return ret; } diff --git a/clamav-devel/clamd/scanner.h b/clamav-devel/clamd/scanner.h index f18bc0176..f63bdcc89 100644 --- a/clamav-devel/clamd/scanner.h +++ b/clamav-devel/clamd/scanner.h @@ -31,4 +31,6 @@ int scanfd(const int fd, unsigned long int *scanned, const struct cl_node *root, int scanstream(int odesc, unsigned long int *scanned, const struct cl_node *root, const struct cl_limits *limits, int options, const struct cfgstruct *copt); +int checksymlink(const char *path); + #endif diff --git a/clamav-devel/clamd/session.c b/clamav-devel/clamd/session.c index 73f98d725..ea99461fd 100644 --- a/clamav-devel/clamd/session.c +++ b/clamav-devel/clamd/session.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -45,8 +46,166 @@ #include "server.h" #include "clamuko.h" #include "session.h" +#include "thrmgr.h" +#include "shared.h" static pthread_mutex_t ctime_mutex = PTHREAD_MUTEX_INITIALIZER; +extern int progexit; + +struct multi_tag { + int sd; + int options; + const struct cfgstruct *copt; + char *fname; + const struct cl_node *root; + const struct cl_limits *limits; +}; + +void multiscanfile(void *arg) +{ + struct multi_tag *tag = (struct multi_tag *) arg; + const char *virname; + sigset_t sigset; + int ret; + + + /* ignore all signals */ + sigfillset(&sigset); + pthread_sigmask(SIG_SETMASK, &sigset, NULL); + + ret = cl_scanfile(tag->fname, &virname, NULL, tag->root, tag->limits, tag->options); + + if(ret == CL_VIRUS) { + mdprintf(tag->sd, "%s: %s FOUND\n", tag->fname, virname); + logg("%s: %s FOUND\n", tag->fname, virname); + virusaction(tag->fname, virname, tag->copt); + } else if(ret != CL_CLEAN) { + mdprintf(tag->sd, "%s: %s ERROR\n", tag->fname, cl_strerror(ret)); + logg("%s: %s ERROR\n", tag->fname, cl_strerror(ret)); + } else if(logok) { + logg("%s: OK\n", tag->fname); + } + + free(tag->fname); + free(tag); + return; +} + +static int multiscan(const char *dirname, const struct cl_node *root, const struct cl_limits *limits, int options, const struct cfgstruct *copt, int odesc, unsigned int *reclev, threadpool_t *multi_pool) +{ + DIR *dd; + struct dirent *dent; +#if defined(HAVE_READDIR_R_3) || defined(HAVE_READDIR_R_2) + union { + struct dirent d; + char b[offsetof(struct dirent, d_name) + NAME_MAX + 1]; + } result; +#endif + struct stat statbuf; + char *fname; + int scanret = 0; + unsigned int maxdirrec = 0; + struct multi_tag *scandata; + + + maxdirrec = cfgopt(copt, "MaxDirectoryRecursion")->numarg; + if(maxdirrec) { + if(*reclev > maxdirrec) { + logg("*multiscan: Directory recursion limit exceeded at %s\n", dirname); + return 0; + } + (*reclev)++; + } + + if((dd = opendir(dirname)) != NULL) { +#ifdef HAVE_READDIR_R_3 + while(!readdir_r(dd, &result.d, &dent) && dent) { +#elif defined(HAVE_READDIR_R_2) + while((dent = (struct dirent *) readdir_r(dd, &result.d))) { +#else + while((dent = readdir(dd))) { +#endif + if (!is_fd_connected(odesc)) { + logg("multiscan: Client disconnected\n"); + closedir(dd); + return -1; + } + + if(progexit) { + closedir(dd); + return -1; + } + +#ifndef C_INTERIX + if(dent->d_ino) +#endif + { + if(strcmp(dent->d_name, ".") && strcmp(dent->d_name, "..")) { + /* build the full name */ + fname = (char *) mcalloc(strlen(dirname) + strlen(dent->d_name) + 2, sizeof(char)); + if(!fname) { + logg("!multiscan: Can't allocate memory for fname\n"); + closedir(dd); + return -1; + } + sprintf(fname, "%s/%s", dirname, dent->d_name); + + /* stat the file */ + if(lstat(fname, &statbuf) != -1) { + if((S_ISDIR(statbuf.st_mode) && !S_ISLNK(statbuf.st_mode)) || (S_ISLNK(statbuf.st_mode) && (checksymlink(fname) == 1) && cfgopt(copt, "FollowDirectorySymlinks")->enabled)) { + if(multiscan(fname, root, limits, options, copt, odesc, reclev, multi_pool) == -1) { + free(fname); + closedir(dd); + return -1; + } + } else { + if(S_ISREG(statbuf.st_mode) || (S_ISLNK(statbuf.st_mode) && (checksymlink(fname) == 2) && cfgopt(copt, "FollowFileSymlinks")->enabled)) { + +#ifdef C_LINUX + if(procdev && (statbuf.st_dev == procdev)) + scanret = CL_CLEAN; + else +#endif + { + scandata = (struct multi_tag *) mmalloc(sizeof(struct multi_tag)); + if(!scandata) { + logg("!multiscan: Can't allocate memory for scandata\n"); + free(fname); + closedir(dd); + return -1; + } + scandata->sd = odesc; + scandata->options = options; + scandata->copt = copt; + scandata->fname = fname; + scandata->root = root; + scandata->limits = limits; + if(!thrmgr_dispatch(multi_pool, scandata)) { + logg("!multiscan: thread dispatch failed for multi_pool (file %s)\n", fname); + mdprintf(odesc, "ERROR: Can't scan file %s\n", fname); + free(fname); + free(scandata); + closedir(dd); + return -1; + } + + while(!multi_pool->thr_idle) /* non-critical */ + usleep(200); + } + } + } + } + } + } + } + closedir(dd); + } else { + return -1; + } + + (*reclev)--; + return 0; +} int command(int desc, const struct cl_node *root, const struct cl_limits *limits, int options, const struct cfgstruct *copt, int timeout) { @@ -139,6 +298,51 @@ int command(int desc, const struct cl_node *root, const struct cl_limits *limits scanfd(fd, NULL, root, limits, options, copt, desc); close(fd); /* FIXME: should we close it here? */ + } else if(!strncmp(buff, CMD13, strlen(CMD13))) { /* MULTISCAN */ + threadpool_t *multi_pool; + int idletimeout = cfgopt(copt, "IdleTimeout")->numarg; + int max_threads = cfgopt(copt, "MaxThreads")->numarg; + int ret; + unsigned int reclev = 0; + const char *path = buff + strlen(CMD13) + 1; + const char *virname; + struct stat sb; + + if(stat(path, &sb) == -1) { + mdprintf(desc, "Can't stat file %s\n", path); + return -1; + } + + if(S_ISDIR(sb.st_mode)) { + if((multi_pool = thrmgr_new(max_threads, idletimeout, multiscanfile)) == NULL) { + logg("!thrmgr_new failed for multi_pool\n"); + mdprintf(desc, "ERROR: thrmgr_new failed for multi_pool\n"); + return -1; + } + + ret = multiscan(path, root, limits, options, copt, desc, &reclev, multi_pool); + thrmgr_destroy(multi_pool); + + if(ret < 0) + return -1; + + } else { + ret = cl_scanfile(path, &virname, NULL, root, limits, options); + + if(ret == CL_VIRUS) { + mdprintf(desc, "%s: %s FOUND\n", path, virname); + logg("%s: %s FOUND\n", path, virname); + virusaction(path, virname, copt); + } else if(ret != CL_CLEAN) { + mdprintf(desc, "%s: %s ERROR\n", path, cl_strerror(ret)); + logg("%s: %s ERROR\n", path, cl_strerror(ret)); + } else { + mdprintf(desc, "%s: OK\n", path); + if(logok) + logg("%s: OK\n", path); + } + } + } else { mdprintf(desc, "UNKNOWN COMMAND\n"); } diff --git a/clamav-devel/clamd/session.h b/clamav-devel/clamd/session.h index acdba306c..37cb02689 100644 --- a/clamav-devel/clamd/session.h +++ b/clamav-devel/clamd/session.h @@ -37,6 +37,7 @@ #define CMD10 "END" #define CMD11 "SHUTDOWN" #define CMD12 "FD" +#define CMD13 "MULTISCAN" #include "libclamav/clamav.h" #include "shared/cfgparser.h" diff --git a/clamav-devel/clamdscan/client.c b/clamav-devel/clamdscan/client.c index 50eb9d2b3..5e7f7be80 100644 --- a/clamav-devel/clamdscan/client.c +++ b/clamav-devel/clamdscan/client.c @@ -58,6 +58,7 @@ void move_infected(const char *filename, const struct optstruct *opt); int notremoved = 0, notmoved = 0; +static int hwaccel = 0; static int dsresult(int sockd, const struct optstruct *opt) { @@ -118,14 +119,14 @@ static int dsresult(int sockd, const struct optstruct *opt) return infected ? infected : (waserror ? -1 : 0); } -static int dsfile(int sockd, const char *filename, const struct optstruct *opt) +static int dsfile(int sockd, const char *scantype, const char *filename, const struct optstruct *opt) { int ret; char *scancmd; scancmd = mcalloc(strlen(filename) + 20, sizeof(char)); - sprintf(scancmd, "CONTSCAN %s", filename); + sprintf(scancmd, "%s %s", scantype, filename); if(write(sockd, scancmd, strlen(scancmd)) <= 0) { logg("^Can't write to the socket.\n"); @@ -341,6 +342,7 @@ static int dconnect(const struct optstruct *opt) if((sockd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { perror("socket()"); logg("^Can't create the socket.\n"); + freecfg(copt); return -1; } @@ -348,6 +350,7 @@ static int dconnect(const struct optstruct *opt) close(sockd); perror("connect()"); logg("^Can't connect to clamd.\n"); + freecfg(copt); return -1; } @@ -356,6 +359,7 @@ static int dconnect(const struct optstruct *opt) if((sockd = socket(SOCKET_INET, SOCK_STREAM, 0)) < 0) { perror("socket()"); logg("^Can't create the socket.\n"); + freecfg(copt); return -1; } @@ -367,6 +371,7 @@ static int dconnect(const struct optstruct *opt) close(sockd); perror("gethostbyname()"); logg("^Can't lookup clamd hostname.\n"); + freecfg(copt); return -1; } server2.sin_addr = *(struct in_addr *) he->h_addr_list[0]; @@ -376,14 +381,23 @@ static int dconnect(const struct optstruct *opt) close(sockd); perror("connect()"); logg("^Can't connect to clamd.\n"); + freecfg(copt); return -1; } } else { logg("^Clamd is not configured properly.\n"); + freecfg(copt); return -1; } +#ifdef HAVE_HWACCEL + if(cfgopt(copt, "HardwareAcceleration")->enabled) + hwaccel = 1; +#endif + + freecfg(copt); + return sockd; } @@ -392,12 +406,20 @@ int client(const struct optstruct *opt, int *infected) char cwd[200], *fullpath; int sockd, ret, errors = 0; struct stat sb; + const char *scantype; *infected = 0; - /* parse argument list */ + /* TODO: add a cmdline option to allow using MULTISCAN on systems + * without hardware accelerators (but with multiple CPUs) + */ + if(hwaccel) + scantype = "MULTISCAN"; + else + scantype = "CONTSCAN"; + /* parse argument list */ if(opt->filename == NULL || strlen(opt->filename) == 0) { /* scan current directory */ if(!getcwd(cwd, 200)) { @@ -408,7 +430,7 @@ int client(const struct optstruct *opt, int *infected) if((sockd = dconnect(opt)) < 0) return 2; - if((ret = dsfile(sockd, cwd, opt)) >= 0) + if((ret = dsfile(sockd, scantype, cwd, opt)) >= 0) *infected += ret; else errors++; @@ -466,7 +488,7 @@ int client(const struct optstruct *opt, int *infected) if((sockd = dconnect(opt)) < 0) return 2; - if((ret = dsfile(sockd, fullpath, opt)) >= 0) + if((ret = dsfile(sockd, scantype, fullpath, opt)) >= 0) *infected += ret; else errors++; diff --git a/clamav-devel/docs/man/clamd.8.in b/clamav-devel/docs/man/clamd.8.in index f8c09d4cd..0ced238ec 100644 --- a/clamav-devel/docs/man/clamd.8.in +++ b/clamav-devel/docs/man/clamd.8.in @@ -36,6 +36,9 @@ Scan a file or directory (recursively) with archive support disabled. A full pat \fBCONTSCAN file/directory\fR Scan a file or directory (recursively) with archive support enabled and continue scanning even when virus is found. A full path is required. .TP +\fBMULTISCAN file/directory\fR +Scan directories with multiple threads. +.TP \fBSTREAM\fR Scan stream \- on this command clamd will return "PORT number" and you can connect to that port and send a data to scan. .SH "OPTIONS" diff --git a/clamav-devel/docs/man/clamd.conf.5.in b/clamav-devel/docs/man/clamd.conf.5.in index c7369f59d..d175fe265 100644 --- a/clamav-devel/docs/man/clamd.conf.5.in +++ b/clamav-devel/docs/man/clamd.conf.5.in @@ -274,6 +274,11 @@ Mark archives as viruses (e.g RAR.ExceededFileSize, Zip.ExceededFilesLimit) if A .br Default: disabled .TP +\fBHardwareAcceleration\fR +Enable support for Sensory Networks' NodalCore hardware accelerator. +.br +Default: disabled +.TP \fBClamukoScanOnAccess\fR Enable Clamuko. Dazuko (/dev/dazuko) must be configured and running. .br diff --git a/clamav-devel/etc/clamd.conf b/clamav-devel/etc/clamd.conf index 7c8775183..71ea6334d 100644 --- a/clamav-devel/etc/clamd.conf +++ b/clamav-devel/etc/clamd.conf @@ -270,6 +270,10 @@ LocalSocket /tmp/clamd # Default: no #ArchiveBlockMax no +# Enable support for Sensory Networks' NodalCore hardware accelerator. +# Default: no +#HardwareAcceleration yes + ## ## Clamuko settings diff --git a/clamav-devel/shared/cfgparser.c b/clamav-devel/shared/cfgparser.c index 61f5950a3..dbda3ac46 100644 --- a/clamav-devel/shared/cfgparser.c +++ b/clamav-devel/shared/cfgparser.c @@ -82,6 +82,7 @@ struct cfgoption cfg_options[] = { {"AllowSupplementaryGroups", OPT_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_FRESHCLAM}, {"SelfCheck", OPT_NUM, 1800, NULL, 0, OPT_CLAMD}, {"VirusEvent", OPT_FULLSTR, -1, NULL, 0, OPT_CLAMD}, + {"HardwareAcceleration", OPT_BOOL, 0, NULL, 0, OPT_CLAMD}, {"ClamukoScanOnAccess", OPT_BOOL, 0, NULL, 0, OPT_CLAMD}, {"ClamukoScanOnOpen", OPT_BOOL, 0, NULL, 0, OPT_CLAMD}, {"ClamukoScanOnClose", OPT_BOOL, 0, NULL, 0, OPT_CLAMD},