extract URLs from mail body (bb #1482).

git-svn: trunk@5014

ChangeLog

@@ -1,3 +1,8 @@
+Thu Apr  2 22:59:30 EEST 2009 (edwin)
+-------------------------------------
+ * libclamav/htmlnorm.c, libclamav/htmlnorm.h, libclamav/mbox.c,
+ libclamav/phishcheck.c: extract URLs from mail body (bb #1482).
+
 Thu Apr  2 19:30:19 CEST 2009 (tk)
 ----------------------------------
  * libclamav/cab.c: fix compiler warnings (bb#1494)

libclamav/htmlnorm.c

@@ -346,7 +346,7 @@ static void html_tag_arg_set(tag_arguments_t *tags, const char *tag, const char
 	}
 	return;
 }
-static void html_tag_arg_add(tag_arguments_t *tags,
+void html_tag_arg_add(tag_arguments_t *tags,
 		const char *tag, char *value)
 {
 	int len, i;

libclamav/htmlnorm.h

@@ -40,6 +40,7 @@ int html_normalise_mem(unsigned char *in_buff, off_t in_size, const char *dirnam
 int html_normalise_fd(int fd, const char *dirname, tag_arguments_t *hrefs, const struct cli_dconf* dconf);
 void html_tag_arg_free(tag_arguments_t *tags);
 int html_screnc_decode(int fd, const char *dirname);
+void html_tag_arg_add(tag_arguments_t *tags, const char *tag, char *value);
  
 #endif
 

libclamav/mbox.c

@@ -3962,6 +3962,36 @@ hrefs_done(blob *b, tag_arguments_t *hrefs)
 	html_tag_arg_free(hrefs);
 }
 
+/* extract URLs from static text */
+static void extract_text_urls(const unsigned char *mem, size_t len, tag_arguments_t *hrefs)
+{
+    char url[1024];
+    size_t off;
+    for (off=0;off + 10 < len;off++) {
+	/* check whether this is the start of a URL */
+	int32_t proto = cli_readint32(mem + off);
+	/* convert to lowercase */
+	proto |= 0x20202020;
+	/* 'http:', 'https:', or 'ftp:' in little-endian */
+	if ((proto == 0x70747468 &&
+	     (mem[off+4] == ':' || (mem[off+5] == 's' && mem[off+6] == ':')))
+	    || proto == 0x3a707466) {
+	    size_t url_len;
+	    for (url_len=4; off + url_len < len && url_len < (sizeof(url)-1); url_len++) {
+		unsigned char c = mem[off + url_len];
+		/* smart compilers will compile this if into
+		 * a single bt + jb instruction */
+		if (c == ' ' || c == '\n' || c == '\t')
+		    break;
+	    }
+	    memcpy(url, mem + off, url_len);
+	    url[url_len] = '\0';
+	    html_tag_arg_add(hrefs, "href", url);
+	    off += url_len;
+	}
+    }
+}
+
 /*
  * This used to be part of checkURLs, split out, because phishingScan needs it
  * too, and phishingScan might be used in situations where checkURLs is
@@ -3970,6 +4000,7 @@ hrefs_done(blob *b, tag_arguments_t *hrefs)
 static blob *
 getHrefs(message *m, tag_arguments_t *hrefs)
 {
+	unsigned char *mem;
 	blob *b = messageToBlob(m, 0);
 	size_t len;
 
@@ -3995,11 +4026,15 @@ getHrefs(message *m, tag_arguments_t *hrefs)
 	hrefs->contents = NULL;
 
 	cli_dbgmsg("getHrefs: calling html_normalise_mem\n");
-	if(!html_normalise_mem(blobGetData(b), (off_t)len, NULL, hrefs,m->ctx->dconf)) {
+	mem = blobGetData(b);
+	if(!html_normalise_mem(mem, (off_t)len, NULL, hrefs,m->ctx->dconf)) {
 		blobDestroy(b);
 		return NULL;
 	}
 	cli_dbgmsg("getHrefs: html_normalise_mem returned\n");
+	if (!hrefs->count && hrefs->scanContents) {
+	    extract_text_urls(mem, len, hrefs);
+	}
 
 	/* TODO: Do we need to call remove_html_comments? */
 	return b;

libclamav/phishcheck.c

@@ -146,9 +146,9 @@ static const char src_text[] = "src";
 static const char href_text[] = "href";
 static const char mailto[] = "mailto:";
 static const char mailto_proto[] = "mailto://";
-static const char https[]="https://";
-static const char http[]="http://";
-static const char ftp[] = "ftp://";
+static const char https[]="https:";
+static const char http[]="http:";
+static const char ftp[] = "ftp:";
 
 static const size_t href_text_len = sizeof(href_text);
 static const size_t src_text_len = sizeof(src_text);
@@ -774,8 +774,7 @@ int phishingScan(cli_ctx* ctx,tag_arguments_t* hrefs)
 	fclose(f);
 	return 0;
 #endif
-	for(i=0;i<hrefs->count;i++)
-		if(hrefs->contents[i]) {
+	for(i=0;i<hrefs->count;i++) {
 			struct url_check urls;
 			enum phish_status rc;
 			urls.flags	 = strncmp((char*)hrefs->tag[i],href_text,href_text_len)? (CL_PHISH_ALL_CHECKS&~CHECK_SSL): CL_PHISH_ALL_CHECKS;
@@ -841,10 +840,7 @@ int phishingScan(cli_ctx* ctx,tag_arguments_t* hrefs)
 					break;
 			}
 			return cli_found_possibly_unwanted(ctx);
-		}
-		else
-			if(strcmp((char*)hrefs->tag[i],"href"))
-					cli_dbgmsg("Phishcheck: href with no contents?\n");
+	}
 	return CL_CLEAN;
 }
 
@@ -1015,33 +1011,34 @@ static int isURL(char* URL, int accept_anyproto)
 	switch (URL[0]) {
 		case 'h':
 			if (strncmp(URL, https, https_len) == 0)
-				start = URL + https_len;
+				start = URL + https_len - 1;
 			else if (strncmp(URL, http, http_len) == 0)
-				start = URL + http_len;
+				start = URL + http_len - 1;
 			break;
 		case 'f':
 		       if (strncmp(URL, ftp, ftp_len) == 0)
-			       start = URL + ftp_len;
+			       start = URL + ftp_len - 1;
 		       break;
 		case 'm':
 		       if (strncmp(URL, mailto_proto, mailto_proto_len) == 0)
-			       start = URL + mailto_proto_len;
+			       start = URL + mailto_proto_len - 1;
 		       break;
 	}
-	if(start) {
-		if(start[0] == '\0')
-			return 0;/* empty URL */
+	if(start && start[1] == '/' && start[2] == '/') {
 		/* has a valid protocol, it is a URL */
 		return 1;
 	}
-	start = accept_anyproto ?  strchr(URL, ':') : NULL;
+	start = accept_anyproto ?  strchr(URL, ':') : start;
 	if(start) {
 		/* validate URI scheme */
 		if(validate_uri_ialpha(URL, start)) {
-			if(start[1] == '/' && start[2] == '/')
-				start += 3; /* skip :// */
-			else
+			/* skip :// */
+			if (start[1] == '/') {
+			    start += 2;
+			    if (*start == '/')
 				start++;
+			} else
+			    start++;
 		}
 		else
 			start = URL; /* scheme invalid */
@@ -1298,7 +1295,7 @@ int cli_url_canon(const char *inurl, size_t len, char *urlbuff, size_t dest_len,
 	/* determine end of hostname */
 	host_len = strcspn(host_begin, ":/?");
 	path_begin = host_begin + host_len;
-	if(host_len < len) {
+	if(host_len <= len) {
 		/* url without path, use a single / */
 		memmove(path_begin + 2, path_begin + 1, len - host_len);
 		*path_begin++ = '/';
@@ -1419,7 +1416,7 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url
 	int phishy=0, blacklisted=0;
 	const struct phishcheck* pchk = (const struct phishcheck*) engine->phishcheck;
 
-	if(!urls->realLink.data || urls->displayLink.data[0]=='\0')
+	if(!urls->realLink.data)
 		return CL_PHISH_CLEAN;
 
 	cli_dbgmsg("Phishcheck:Checking url %s->%s\n", urls->realLink.data,
@@ -1466,6 +1463,10 @@ static enum phish_status phishingCheck(const struct cl_engine* engine,struct url
 	if (blacklisted)
 	    return blacklisted;
 
+	if (urls->displayLink.data[0] == '\0') {
+	    return CL_PHISH_CLEAN;
+	}
+
 	url_check_init(&host_url);
 
 	if((rc = url_get_host(urls, &host_url, DOMAIN_DISPLAY, &phishy))) {