make use of hostkey prefix entries

git-svn: trunk@4929

ChangeLog

@@ -1,3 +1,9 @@
+Wed Mar 11 22:06:30 EET 2009 (edwin)
+------------------------------------
+ * libclamav/phishcheck.c, libclamav/regex_list.c,
+ libclamav/regex_list.h, unit_tests/check_regex.c,
+ unit_tests/input/daily.gdb: make use of hostkey prefix entries
+
 Wed Mar 11 21:27:32 EET 2009 (edwin)
 ------------------------------------
  * clamd/others.c, sigtool/Makefile.in: fix previous commit

libclamav/phishcheck.c

@@ -1172,7 +1172,7 @@ static int whitelist_check(const struct cl_engine* engine,struct url_check* urls
 	return whitelist_match(engine,urls->realLink.data,urls->displayLink.data,hostOnly);
 }
 
-static int hash_match(const struct regex_matcher *rlist, const char *host, size_t hlen, const char *path, size_t plen)
+static int hash_match(const struct regex_matcher *rlist, const char *host, size_t hlen, const char *path, size_t plen, int *prefix_matched)
 {
 	const char *virname;
 #if 0
@@ -1198,9 +1198,15 @@ static int hash_match(const struct regex_matcher *rlist, const char *host, size_
 		h[2*i+1] = hexchars[sha256_dig[i]&0xf];
 	    }
 	    h[64]='\0';
-	    cli_dbgmsg("Looking up hash %s for %s%s\n", h, host, path);
-	    if(SO_search(&rlist->sha256_filter, sha256_dig, 32) != -1 &&
-	       cli_bm_scanbuff(sha256_dig, 32, &virname, &rlist->sha256_hashes,0,0,-1) == CL_VIRUS) {
+	    cli_dbgmsg("Looking up hash %s for %s(%u)%s(%u)\n", h, host, hlen, path, plen);
+	    if (prefix_matched) {
+		if (cli_bm_scanbuff(sha256_dig, 4, &virname, &rlist->hostkey_prefix,0,0,-1) == CL_VIRUS) {
+		    cli_dbgmsg("prefix matched\n", virname);
+		    *prefix_matched = 1;
+		} else
+		    return CL_SUCCESS;
+	    }
+	    if (cli_bm_scanbuff(sha256_dig, 32, &virname, &rlist->sha256_hashes,0,0,-1) == CL_VIRUS) {
 		switch(*virname) {
 		    case '1':
 			return CL_PHISH_HASH1;
@@ -1316,10 +1322,11 @@ static int url_hash_match(const struct regex_matcher *rlist, const char *inurl,
 	size_t path_len;
 	size_t host_len;
 	char *p;
-	int rc;
+	int rc, prefix_matched=0;
 	const char *lp[COMPONENTS+1];
 	size_t pp[COMPONENTS+2];
 	char urlbuff[URL_MAX_LEN+3];/* htmlnorm truncates at 1024 bytes + terminating null + slash + host end null */
+	unsigned count;
 
 	if(!rlist || !rlist->sha256_hashes.bm_patterns) {
 		return CL_SUCCESS;
@@ -1358,15 +1365,27 @@ static int url_hash_match(const struct regex_matcher *rlist, const char *inurl,
 		}
 	} else
 		k = 1;
-
-	for(ji=j;ji < COMPONENTS+1; ji++) {
-		for(ki=0;ki < k; ki++) {
-			assert(pp[ki] <= path_len);
-			rc = hash_match(rlist, lp[ji], host_begin + host_len - lp[ji] + 1, path_begin, pp[ki]);
-			if(rc) {
-				return rc;
-			}
+	count = 0;
+	for(ki=k;ki > 0;) {
+	    --ki;
+	    for(ji=COMPONENTS+1;ji > j;) {
+		/* lookup last 2 and 3 components of host, as hostkey prefix,
+		 * if not matched, shortcircuit lookups */
+		int need_prefixmatch = (count<2 && !prefix_matched) &&
+				       rlist->hostkey_prefix.bm_patterns;
+		--ji;
+		assert(pp[ki] <= path_len);
+		rc = hash_match(rlist, lp[ji], host_begin + host_len - lp[ji] + 1, path_begin, pp[ki], 
+				need_prefixmatch ? &prefix_matched : NULL);
+		if(rc) {
+		    return rc;
+		}
+		count++;
+		if (count == 2 && !prefix_matched && rlist->hostkey_prefix.bm_patterns) {
+		    cli_dbgmsg("hostkey prefix not matched, short-circuiting lookups\n");
+		    return CL_SUCCESS;
 		}
+	    }
 	}
 	return CL_SUCCESS;
 }
@@ -1394,8 +1413,11 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url
 	}
 
 	if(( rc = url_hash_match(engine->domainlist_matcher, urls->realLink.data, strlen(urls->realLink.data)) )) {
+	    if (rc == CL_PHISH_CLEAN)
+		cli_dbgmsg("not analyzing, not a real url: %s\n", urls->realLink.data);
+	    else
 		cli_dbgmsg("Hash matched for: %s\n", urls->realLink.data);
-		return rc;
+	    return rc;
 	}
 
 	if((rc = cleanupURLs(urls))) {

libclamav/regex_list.c

@@ -372,12 +372,15 @@ int init_regex_list(struct regex_matcher* matcher)
 	}
 #ifdef USE_MPOOL
 	matcher->sha256_hashes.mempool = mp;
+	matcher->hostkey_prefix.mempool = mp;
 #endif
 	if((rc = cli_bm_init(&matcher->sha256_hashes))) {
 		return rc;
 	}
+	if((rc = cli_bm_init(&matcher->hostkey_prefix))) {
+		return rc;
+	}
 	SO_init(&matcher->filter);
-	SO_init(&matcher->sha256_filter);
 	return CL_SUCCESS;
 }
 
@@ -424,10 +427,11 @@ static int functionality_level_check(char* line)
 	}
 }
 
-static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl)
+static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl, int is_prefix)
 {
 	int rc;
 	struct cli_bm_patt *pat = mpool_calloc(matcher->mempool, 1, sizeof(*pat));
+	struct cli_matcher *bm;
 	if(!pat)
 		return CL_EMEM;
 	pat->pattern = (unsigned char*)cli_mpool_hex2str(matcher->mempool, pattern);
@@ -440,8 +444,14 @@ static int add_hash(struct regex_matcher *matcher, char* pattern, const char fl)
 		return CL_EMEM;
 	}
 	*pat->virname = fl;
-	SO_preprocess_add(&matcher->sha256_filter, pat->pattern, pat->length);
-	if((rc = cli_bm_addpatt(&matcher->sha256_hashes, pat))) {
+	if (is_prefix) {
+	    pat->length=4;
+	    bm = &matcher->hostkey_prefix;
+	} else {
+	    bm = &matcher->sha256_hashes;
+	}
+
+	if((rc = cli_bm_addpatt(bm, pat))) {
 		cli_errmsg("add_hash: failed to add BM pattern\n");
 		free(pat->pattern);
 		free(pat->virname);
@@ -542,15 +552,12 @@ int load_regex_matcher(struct regex_matcher* matcher,FILE* fd,unsigned int *sign
 				return rc==CL_EMEM ? CL_EMEM : CL_EMALFDB;
 		} else if (buffer[0] == 'S' && !is_whitelist) {
 			pattern[pattern_len] = '\0';
-			if(*pattern=='F' && pattern[1]==':') {
+			if((pattern[0]=='F' || pattern[0]=='P') && pattern[1]==':') {
 			    pattern += 2;
-			    if (( rc = add_hash(matcher, pattern, flags[0]) )) {
+			    if (( rc = add_hash(matcher, pattern, flags[0], pattern[-2] == 'P') )) {
 				cli_errmsg("Error loading at line: %d\n", line);
 				return rc==CL_EMEM ? CL_EMEM : CL_EMALFDB;
 			    }
-			} else if (*pattern=='P' && pattern[1]==':') {
-			    pattern += 2;
-			    /* TODO: hostkey prefix */
 			} else {
 			    cli_errmsg("Error loading line: %d, %c\n", line, *pattern);
 			    return CL_EMALFDB;
@@ -617,6 +624,7 @@ void regex_list_done(struct regex_matcher* matcher)
 		}
 		hashtab_free(&matcher->suffix_hash);
 		cli_bm_free(&matcher->sha256_hashes);
+		cli_bm_free(&matcher->hostkey_prefix);
 	}
 }
 

libclamav/regex_list.h

@@ -52,7 +52,7 @@ struct regex_matcher {
 	regex_t **all_pregs;
 	struct cli_matcher suffixes;
 	struct cli_matcher sha256_hashes;
-	struct filter sha256_filter;
+	struct cli_matcher hostkey_prefix;
 	struct filter filter;
 #ifdef USE_MPOOL
 	mpool_t *mempool;

unit_tests/check_regex.c

@@ -313,7 +313,7 @@ static void psetup_impl(int load2)
 		fail_unless(rc == 0, "load_regex_matcher");
 		fclose(f);
 
-		fail_unless_fmt(signo == 2, "Incorrect number of signatures: %u, expected %u", signo, 2);
+		fail_unless_fmt(signo == 4, "Incorrect number of signatures: %u, expected %u", signo, 4);
 	}
 	loaded_2 = load2;
 

unit_tests/input/daily.gdb

@@ -1,2 +1,4 @@
+S:P:d1b8a025
 S:F:d1b8a0251d7555d016b6468ae623e4b1e830c7efccc54966d09447a3d0a85c60
+S2:P:7f6fd541
 S2:F:7f6fd541e625e7bc5d5a64f166e47ecfe13735464a74d160b48265c162a71089