enable prefiltering, and add to dconf.

Also downgrade some warnings to debug messages.

libclamav/dconf.c

@@ -110,6 +110,7 @@ static struct dconf_module modules[] = {
     { "OTHER",	    "CRYPTFF",	    OTHER_CONF_CRYPTFF,	    1 },
     { "OTHER",	    "DLP",	    OTHER_CONF_DLP,	    1 },
     { "OTHER",	    "MYDOOMLOG",    OTHER_CONF_MYDOOMLOG,   1 },
+    { "OTHER",      "PREFILTERING", OTHER_CONF_PREFILTERING,1 },
 
     { "PHISHING",   "ENGINE",       PHISHING_CONF_ENGINE,   1 },
     { "PHISHING",   "ENTCONV",      PHISHING_CONF_ENTCONV,  1 },

libclamav/dconf.h

@@ -99,6 +99,7 @@ struct cli_dconf {
 #define OTHER_CONF_CRYPTFF  0x10
 #define OTHER_CONF_DLP	    0x20
 #define OTHER_CONF_MYDOOMLOG	0x40
+#define OTHER_CONF_PREFILTERING 0x80
 
 /* Phishing flags */
 #define PHISHING_CONF_ENGINE   0x1

libclamav/filtering.c

@@ -228,7 +228,7 @@ int filter_add_static(struct filter *m, const unsigned char *pattern, unsigned l
 
 	assert(best_pos < len-1);
 	if (pattern[best_pos] == 0 && pattern[best_pos+1] == 0) {
-		cli_warnmsg("filter: subsignature begins with zero (static): %s\n", name);
+		cli_dbgmsg("!filter: subsignature begins with zero (static): %s\n", name);
 	}
 	pattern += best_pos;
 	len -= best_pos;
@@ -627,7 +627,7 @@ int  filter_add_acpatt(struct filter *m, const struct cli_ac_patt *pat)
 				unsigned char c0 = spec_ith_char(spec0, k0);
 				unsigned char c1 = spec_ith_char(spec1, k1);
 				if (!c0 && !c1 && !i) {
-					cli_warnmsg("filter: subsignature begins with zero: %s\n",pat->virname);
+					cli_dbgmsg("!filter: subsignature begins with zero: %s\n",pat->virname);
 				}
 				filter_set_atpos(m, i, c0 | (c1<<8));
 			}

libclamav/matcher-ac.c

@@ -360,11 +360,11 @@ int cli_ac_buildtrie(struct cli_matcher *root)
     }
 
     if (root->filter)
-	cli_warnmsg("Using filter for trie %d\n", root->type);
+	cli_dbgmsg("!Using filter for trie %d\n", root->type);
     return ac_maketrans(root);
 }
 
-int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth)
+int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth, uint8_t dconf_prefiltering)
 {
 #ifdef USE_MPOOL
     assert(root->mempool && "mempool must be initialized");
@@ -386,8 +386,7 @@ int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth)
     root->ac_mindepth = mindepth;
     root->ac_maxdepth = maxdepth;
 
-    /* TODO: dconf here ?*/
-    if (cli_mtargets[root->type].enable_prefiltering && 0) {/* Disabled for now */
+    if (cli_mtargets[root->type].enable_prefiltering && dconf_prefiltering) {
 	root->filter = mpool_malloc(root->mempool, sizeof(*root->filter));
 	if (!root->filter) {
 	    cli_errmsg("cli_ac_init: Can't allocate memory for ac_root->filter\n");

libclamav/matcher-ac.h

@@ -94,7 +94,7 @@ int cli_ac_chklsig(const char *expr, const char *end, uint32_t *lsigcnt, unsigne
 void cli_ac_freedata(struct cli_ac_data *data);
 int cli_ac_scanbuff(const unsigned char *buffer, uint32_t length, const char **virname, void **customdata, struct cli_ac_result **res, const struct cli_matcher *root, struct cli_ac_data *mdata, uint32_t offset, cli_file_t ftype, struct cli_matched_type **ftoffset, unsigned int mode, const cli_ctx *ctx);
 int cli_ac_buildtrie(struct cli_matcher *root);
-int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth);
+int cli_ac_init(struct cli_matcher *root, uint8_t mindepth, uint8_t maxdepth, uint8_t dconf_prefiltering);
 int cli_ac_caloff(const struct cli_matcher *root, struct cli_ac_data *data, fmap_t *map);
 void cli_ac_free(struct cli_matcher *root);
 int cli_ac_addsig(struct cli_matcher *root, const char *virname, const char *hexsig, uint32_t sigid, uint16_t parts, uint16_t partno, uint16_t rtype, uint16_t type, uint32_t mindist, uint32_t maxdist, const char *offset, const uint32_t *lsigid, unsigned int options);

libclamav/phish_domaincheck_db.c

@@ -56,7 +56,7 @@ int init_domainlist(struct cl_engine* engine)
 #ifdef USE_MPOOL
 		((struct regex_matcher*)engine->domainlist_matcher)->mempool = engine->mempool;
 #endif
-		return init_regex_list(engine->domainlist_matcher);
+		return init_regex_list(engine->domainlist_matcher, engine->dconf->other&OTHER_CONF_PREFILTERING);
 	}
 	else
 		return CL_ENULLARG;

libclamav/phish_whitelist.c

@@ -57,7 +57,7 @@ int init_whitelist(struct cl_engine* engine)
 #endif
 		if(!engine->whitelist_matcher)
 			return CL_EMEM;
-		return	init_regex_list(engine->whitelist_matcher);
+		return	init_regex_list(engine->whitelist_matcher, engine->dconf->other&OTHER_CONF_PREFILTERING);
 	}
 	else
 		return CL_ENULLARG;

libclamav/readdb.c

@@ -344,7 +344,7 @@ int cli_initroots(struct cl_engine *engine, unsigned int options)
 		root->ac_only = 1;
 
 	    cli_dbgmsg("Initialising AC pattern matcher of root[%d]\n", i);
-	    if((ret = cli_ac_init(root, engine->ac_mindepth, engine->ac_maxdepth))) {
+	    if((ret = cli_ac_init(root, engine->ac_mindepth, engine->ac_maxdepth, engine->dconf->other&OTHER_CONF_PREFILTERING))) {
 		/* no need to free previously allocated memory here */
 		cli_errmsg("cli_initroots: Can't initialise AC pattern matcher\n");
 		return ret;
@@ -817,7 +817,7 @@ static int cli_loadwdb(FILE *fs, struct cl_engine *engine, unsigned int options,
 	}
     }
 
-    if((ret = load_regex_matcher(engine->whitelist_matcher, fs, NULL, options, 1, dbio))) {
+    if((ret = load_regex_matcher(engine->whitelist_matcher, fs, NULL, options, 1, dbio, engine->dconf->other&OTHER_CONF_PREFILTERING))) {
 	return ret;
     }
 
@@ -838,7 +838,7 @@ static int cli_loadpdb(FILE *fs, struct cl_engine *engine, unsigned int *signo,
 	}
     }
 
-    if((ret = load_regex_matcher(engine->domainlist_matcher, fs, signo, options, 0, dbio))) {
+    if((ret = load_regex_matcher(engine->domainlist_matcher, fs, signo, options, 0, dbio, engine->dconf->other&OTHER_CONF_PREFILTERING))) {
 	return ret;
     }
 

libclamav/regex_list.c

@@ -251,7 +251,7 @@ int regex_list_match(struct regex_matcher* matcher,char* real_url,const char* di
 
 /* Initialization & loading */
 /* Initializes @matcher, allocating necesarry substructures */
-int init_regex_list(struct regex_matcher* matcher)
+int init_regex_list(struct regex_matcher* matcher, uint8_t dconf_prefiltering)
 {
 #ifdef USE_MPOOL
 	mpool_t *mp = matcher->mempool;
@@ -270,7 +270,7 @@ int init_regex_list(struct regex_matcher* matcher)
 	matcher->suffixes.mempool = mp;
 	assert(mp && "mempool must be initialized");
 #endif
-	if((rc = cli_ac_init(&matcher->suffixes, 2, 32))) {
+	if((rc = cli_ac_init(&matcher->suffixes, 2, 32, dconf_prefiltering))) {
 		return rc;
 	}
 #ifdef USE_MPOOL
@@ -385,7 +385,7 @@ static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl,
 
 
 /* Load patterns/regexes from file */
-int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio)
+int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio, uint8_t dconf_prefiltering)
 {
 	int rc,line=0,entry=0;
 	char buffer[FILEBUFF];
@@ -401,7 +401,7 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *sign
 
 	cli_dbgmsg("Loading regex_list\n");
 	if(!matcher->list_inited) {
-		rc = init_regex_list(matcher);
+		rc = init_regex_list(matcher, dconf_prefiltering);
 		if (!matcher->list_inited) {
 			cli_errmsg("Regex list failed to initialize!\n");
 			fatal_error(matcher);

libclamav/regex_list.h

@@ -60,8 +60,8 @@ struct regex_matcher {
 int cli_build_regex_list(struct regex_matcher* matcher);
 int regex_list_add_pattern(struct regex_matcher *matcher, char *pattern);
 int regex_list_match(struct regex_matcher* matcher, char* real_url,const char* display_url,const struct pre_fixup_info* pre_fixup, int hostOnly,const char **info, int is_whitelist);
-int init_regex_list(struct regex_matcher* matcher);
-int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio);
+int init_regex_list(struct regex_matcher* matcher, uint8_t dconf_prefiltering);
+int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *signo,unsigned int options,int is_whitelist,struct cli_dbio *dbio,uint8_t dconf_prefiltering);
 void regex_list_cleanup(struct regex_matcher* matcher);
 void regex_list_done(struct regex_matcher* matcher);
 int is_regex_ok(struct regex_matcher* matcher);

unit_tests/check_matchers.c

@@ -91,7 +91,7 @@ START_TEST (test_ac_scanbuff) {
 #ifdef USE_MPOOL
     root->mempool = mpool_create();
 #endif
-    ret = cli_ac_init(root, CLI_DEFAULT_AC_MINDEPTH, CLI_DEFAULT_AC_MAXDEPTH);
+    ret = cli_ac_init(root, CLI_DEFAULT_AC_MINDEPTH, CLI_DEFAULT_AC_MAXDEPTH, 1);
     fail_unless(ret == CL_SUCCESS, "cli_ac_init() failed");
 
 

unit_tests/check_regex.c

@@ -157,7 +157,7 @@ static void rsetup(void)
 #ifdef USE_MPOOL
 	matcher.mempool = mpool_create();
 #endif
-	rc = init_regex_list(&matcher);
+	rc = init_regex_list(&matcher, 1);
 	fail_unless(rc == 0, "init_regex_list");
 }
 
@@ -298,7 +298,7 @@ static void psetup_impl(int load2)
 	f = fdopen(open_testfile("input/daily.pdb"),"r");
 	fail_unless(!!f, "fopen daily.pdb");
 
-	rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL);
+	rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL, 1);
 	fail_unless(rc == 0, "load_regex_matcher");
 	fclose(f);
 
@@ -309,7 +309,7 @@ static void psetup_impl(int load2)
 		fail_unless(!!f, "fopen daily.gdb");
 
 		signo = 0;
-		rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL);
+		rc = load_regex_matcher(engine->domainlist_matcher,  f, &signo, 0, 0, NULL, 1);
 		fail_unless(rc == 0, "load_regex_matcher");
 		fclose(f);
 
@@ -322,7 +322,7 @@ static void psetup_impl(int load2)
 
 	f = fdopen(open_testfile("input/daily.wdb"),"r");
 	signo = 0;
-	rc = load_regex_matcher(engine->whitelist_matcher, f, &signo, 0, 1, NULL);
+	rc = load_regex_matcher(engine->whitelist_matcher, f, &signo, 0, 1, NULL, 1);
 	fail_unless(rc == 0,"load_regex_matcher");
 	fclose(f);