new hash matching

remotes/push_mirror/multihash
aCaB 15 years ago
parent 38d54c9204
commit 3faa97834c
  1. 2
      libclamav/Makefile.am
  2. 87
      libclamav/Makefile.in
  3. 37
      libclamav/matcher-hash.c
  4. 4
      libclamav/matcher-hash.h
  5. 135
      libclamav/matcher-md5.c
  6. 39
      libclamav/matcher-md5.h
  7. 14
      libclamav/matcher.c
  8. 8
      libclamav/matcher.h
  9. 10
      libclamav/others.h
  10. 19
      libclamav/pe.c
  11. 252
      libclamav/readdb.c

@ -142,8 +142,6 @@ libclamav_la_SOURCES = \
matcher-ac.h \
matcher-bm.c \
matcher-bm.h \
matcher-md5.c \
matcher-md5.h \
matcher-hash.c \
matcher-hash.h \
matcher.c \

@ -114,23 +114,22 @@ LTLIBRARIES = $(lib_LTLIBRARIES) $(noinst_LTLIBRARIES)
@ENABLE_LLVM_FALSE@am__DEPENDENCIES_2 = libclamav_nocxx.la
@ENABLE_LLVM_TRUE@am__DEPENDENCIES_2 = c++/libclamavcxx.la
am__libclamav_la_SOURCES_DIST = clamav.h matcher-ac.c matcher-ac.h \
matcher-bm.c matcher-bm.h matcher-md5.c matcher-md5.h \
matcher-hash.c matcher-hash.h matcher.c matcher.h others.c \
others.h readdb.c readdb.h cvd.c cvd.h dsig.c dsig.h \
scanners.c scanners.h textdet.c textdet.h filetypes.c \
filetypes.h filetypes_int.h rtf.c rtf.h blob.c blob.h mbox.c \
mbox.h message.c message.h table.c table.h text.c text.h \
ole2_extract.c ole2_extract.h vba_extract.c vba_extract.h \
cltypes.h msexpand.c msexpand.h pe.c pe.h pe_icons.c \
pe_icons.h disasm.c disasm.h disasm-common.h disasmpriv.h \
upx.c upx.h htmlnorm.c htmlnorm.h chmunpack.c chmunpack.h \
rebuildpe.c rebuildpe.h petite.c petite.h wwunpack.c \
wwunpack.h unsp.c unsp.h aspack.c aspack.h packlibs.c \
packlibs.h fsg.c fsg.h mew.c mew.h upack.c upack.h line.c \
line.h untar.c untar.h unzip.c unzip.h inflate64.c inflate64.h \
inffixed64.h inflate64_priv.h special.c special.h binhex.c \
binhex.h is_tar.c is_tar.h tnef.c tnef.h autoit.c autoit.h \
unarj.c unarj.h nsis/bzlib.c nsis/bzlib_private.h \
matcher-bm.c matcher-bm.h matcher-hash.c matcher-hash.h \
matcher.c matcher.h others.c others.h readdb.c readdb.h cvd.c \
cvd.h dsig.c dsig.h scanners.c scanners.h textdet.c textdet.h \
filetypes.c filetypes.h filetypes_int.h rtf.c rtf.h blob.c \
blob.h mbox.c mbox.h message.c message.h table.c table.h \
text.c text.h ole2_extract.c ole2_extract.h vba_extract.c \
vba_extract.h cltypes.h msexpand.c msexpand.h pe.c pe.h \
pe_icons.c pe_icons.h disasm.c disasm.h disasm-common.h \
disasmpriv.h upx.c upx.h htmlnorm.c htmlnorm.h chmunpack.c \
chmunpack.h rebuildpe.c rebuildpe.h petite.c petite.h \
wwunpack.c wwunpack.h unsp.c unsp.h aspack.c aspack.h \
packlibs.c packlibs.h fsg.c fsg.h mew.c mew.h upack.c upack.h \
line.c line.h untar.c untar.h unzip.c unzip.h inflate64.c \
inflate64.h inffixed64.h inflate64_priv.h special.c special.h \
binhex.c binhex.h is_tar.c is_tar.h tnef.c tnef.h autoit.c \
autoit.h unarj.c unarj.h nsis/bzlib.c nsis/bzlib_private.h \
nsis/nsis_bzlib.h nsis/nulsft.c nsis/nulsft.h nsis/infblock.c \
nsis/nsis_zconf.h nsis/nsis_zlib.h nsis/nsis_zutil.h pdf.c \
pdf.h spin.c spin.h yc.c yc.h elf.c elf.h execs.h sis.c sis.h \
@ -162,13 +161,12 @@ am__libclamav_la_SOURCES_DIST = clamav.h matcher-ac.c matcher-ac.h \
bignum.c bignum_class.h
@LINK_TOMMATH_FALSE@am__objects_1 = libclamav_la-bignum.lo
am_libclamav_la_OBJECTS = libclamav_la-matcher-ac.lo \
libclamav_la-matcher-bm.lo libclamav_la-matcher-md5.lo \
libclamav_la-matcher-hash.lo libclamav_la-matcher.lo \
libclamav_la-others.lo libclamav_la-readdb.lo \
libclamav_la-cvd.lo libclamav_la-dsig.lo \
libclamav_la-scanners.lo libclamav_la-textdet.lo \
libclamav_la-filetypes.lo libclamav_la-rtf.lo \
libclamav_la-blob.lo libclamav_la-mbox.lo \
libclamav_la-matcher-bm.lo libclamav_la-matcher-hash.lo \
libclamav_la-matcher.lo libclamav_la-others.lo \
libclamav_la-readdb.lo libclamav_la-cvd.lo \
libclamav_la-dsig.lo libclamav_la-scanners.lo \
libclamav_la-textdet.lo libclamav_la-filetypes.lo \
libclamav_la-rtf.lo libclamav_la-blob.lo libclamav_la-mbox.lo \
libclamav_la-message.lo libclamav_la-table.lo \
libclamav_la-text.lo libclamav_la-ole2_extract.lo \
libclamav_la-vba_extract.lo libclamav_la-msexpand.lo \
@ -621,22 +619,22 @@ libclamav_la_LDFLAGS = @TH_SAFE@ -version-info @LIBCLAMAV_VERSION@ \
-no-undefined $(am__append_6)
include_HEADERS = clamav.h
libclamav_la_SOURCES = clamav.h matcher-ac.c matcher-ac.h matcher-bm.c \
matcher-bm.h matcher-md5.c matcher-md5.h matcher-hash.c \
matcher-hash.h matcher.c matcher.h others.c others.h readdb.c \
readdb.h cvd.c cvd.h dsig.c dsig.h scanners.c scanners.h \
textdet.c textdet.h filetypes.c filetypes.h filetypes_int.h \
rtf.c rtf.h blob.c blob.h mbox.c mbox.h message.c message.h \
table.c table.h text.c text.h ole2_extract.c ole2_extract.h \
vba_extract.c vba_extract.h cltypes.h msexpand.c msexpand.h \
pe.c pe.h pe_icons.c pe_icons.h disasm.c disasm.h \
disasm-common.h disasmpriv.h upx.c upx.h htmlnorm.c htmlnorm.h \
chmunpack.c chmunpack.h rebuildpe.c rebuildpe.h petite.c \
petite.h wwunpack.c wwunpack.h unsp.c unsp.h aspack.c aspack.h \
packlibs.c packlibs.h fsg.c fsg.h mew.c mew.h upack.c upack.h \
line.c line.h untar.c untar.h unzip.c unzip.h inflate64.c \
inflate64.h inffixed64.h inflate64_priv.h special.c special.h \
binhex.c binhex.h is_tar.c is_tar.h tnef.c tnef.h autoit.c \
autoit.h unarj.c unarj.h nsis/bzlib.c nsis/bzlib_private.h \
matcher-bm.h matcher-hash.c matcher-hash.h matcher.c matcher.h \
others.c others.h readdb.c readdb.h cvd.c cvd.h dsig.c dsig.h \
scanners.c scanners.h textdet.c textdet.h filetypes.c \
filetypes.h filetypes_int.h rtf.c rtf.h blob.c blob.h mbox.c \
mbox.h message.c message.h table.c table.h text.c text.h \
ole2_extract.c ole2_extract.h vba_extract.c vba_extract.h \
cltypes.h msexpand.c msexpand.h pe.c pe.h pe_icons.c \
pe_icons.h disasm.c disasm.h disasm-common.h disasmpriv.h \
upx.c upx.h htmlnorm.c htmlnorm.h chmunpack.c chmunpack.h \
rebuildpe.c rebuildpe.h petite.c petite.h wwunpack.c \
wwunpack.h unsp.c unsp.h aspack.c aspack.h packlibs.c \
packlibs.h fsg.c fsg.h mew.c mew.h upack.c upack.h line.c \
line.h untar.c untar.h unzip.c unzip.h inflate64.c inflate64.h \
inffixed64.h inflate64_priv.h special.c special.h binhex.c \
binhex.h is_tar.c is_tar.h tnef.c tnef.h autoit.c autoit.h \
unarj.c unarj.h nsis/bzlib.c nsis/bzlib_private.h \
nsis/nsis_bzlib.h nsis/nulsft.c nsis/nulsft.h nsis/infblock.c \
nsis/nsis_zconf.h nsis/nsis_zlib.h nsis/nsis_zutil.h pdf.c \
pdf.h spin.c spin.h yc.c yc.h elf.c elf.h execs.h sis.c sis.h \
@ -840,7 +838,6 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-matcher-ac.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-matcher-bm.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-matcher-hash.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-matcher-md5.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-matcher.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-mbox.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libclamav_la-message.Plo@am__quote@
@ -938,14 +935,6 @@ libclamav_la-matcher-bm.lo: matcher-bm.c
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libclamav_la_CFLAGS) $(CFLAGS) -c -o libclamav_la-matcher-bm.lo `test -f 'matcher-bm.c' || echo '$(srcdir)/'`matcher-bm.c
libclamav_la-matcher-md5.lo: matcher-md5.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libclamav_la_CFLAGS) $(CFLAGS) -MT libclamav_la-matcher-md5.lo -MD -MP -MF $(DEPDIR)/libclamav_la-matcher-md5.Tpo -c -o libclamav_la-matcher-md5.lo `test -f 'matcher-md5.c' || echo '$(srcdir)/'`matcher-md5.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libclamav_la-matcher-md5.Tpo $(DEPDIR)/libclamav_la-matcher-md5.Plo
@am__fastdepCC_FALSE@ $(AM_V_CC) @AM_BACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ source='matcher-md5.c' object='libclamav_la-matcher-md5.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libclamav_la_CFLAGS) $(CFLAGS) -c -o libclamav_la-matcher-md5.lo `test -f 'matcher-md5.c' || echo '$(srcdir)/'`matcher-md5.c
libclamav_la-matcher-hash.lo: matcher-hash.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libclamav_la_CFLAGS) $(CFLAGS) -MT libclamav_la-matcher-hash.lo -MD -MP -MF $(DEPDIR)/libclamav_la-matcher-hash.Tpo -c -o libclamav_la-matcher-hash.lo `test -f 'matcher-hash.c' || echo '$(srcdir)/'`matcher-hash.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libclamav_la-matcher-hash.Tpo $(DEPDIR)/libclamav_la-matcher-hash.Plo

@ -66,10 +66,9 @@ int hm_addhash(struct cli_matcher *root, const char *hash, uint32_t size, const
hashlen /= 2;
ht = &root->hm.sizehashes[type];
if(!root->hm.htinint[type]) {
if(!root->hm.sizehashes[type].capacity) {
i = cli_htu32_init(ht, 64, root->mempool);
if(i) return i;
root->hm.htinint[type] = 1;
}
item = cli_htu32_find(ht, size);
@ -99,6 +98,7 @@ int hm_addhash(struct cli_matcher *root, const char *hash, uint32_t size, const
cli_errmsg("ht_add: failed to grow hash array to %u entries\n", szh->items);
szh->items=0;
mpool_free(root->mempool, szh->virusnames);
szh->virusnames = NULL;
return CL_EMEM;
}
@ -107,6 +107,7 @@ int hm_addhash(struct cli_matcher *root, const char *hash, uint32_t size, const
cli_errmsg("ht_add: failed to grow virusname array to %u entries\n", szh->items);
szh->items=0;
mpool_free(root->mempool, szh->hash_array);
szh->hash_array = NULL;
return CL_EMEM;
}
@ -182,7 +183,7 @@ void hm_flush(struct cli_matcher *root) {
struct cli_htu32 *ht = &root->hm.sizehashes[type];
const struct cli_htu32_element *item = NULL;
if(!root->hm.htinint[type])
if(!root->hm.sizehashes[type].capacity)
continue;
while((item = cli_htu32_next(ht, item))) {
@ -197,7 +198,7 @@ void hm_flush(struct cli_matcher *root) {
int cli_hm_have_size(const struct cli_matcher *root, enum CLI_HASH_TYPE type, uint32_t size) {
return (size && size != 0xffffffff && root && root->hm.htinint[type] && cli_htu32_find(&root->hm.sizehashes[type], size));
return (size && size != 0xffffffff && root && root->hm.sizehashes[type].capacity && cli_htu32_find(&root->hm.sizehashes[type], size));
}
int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type) {
@ -206,7 +207,7 @@ int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname
struct cli_sz_hash *szh;
size_t l, r;
if(!digest || !size || size == 0xffffffff || !root || !root->hm.htinint[type])
if(!digest || !size || size == 0xffffffff || !root || !root->hm.sizehashes[type].capacity)
return CL_CLEAN;
item = cli_htu32_find(&root->hm.sizehashes[type], size);
@ -236,3 +237,29 @@ int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname
}
return CL_CLEAN;
}
void hm_free(struct cli_matcher *root) {
enum CLI_HASH_TYPE type;
if(!root)
return;
for(type = CLI_HASH_MD5; type < CLI_HASH_AVAIL_TYPES; type++) {
struct cli_htu32 *ht = &root->hm.sizehashes[type];
const struct cli_htu32_element *item = NULL;
if(!root->hm.sizehashes[type].capacity)
continue;
while((item = cli_htu32_next(ht, item))) {
struct cli_sz_hash *szh = (struct cli_sz_hash *)item->data.as_ptr;
unsigned int keylen = hashlen[type];
mpool_free(root->mempool, szh->hash_array);
mpool_free(root->mempool, szh->virusnames);
mpool_free(root->mempool, szh);
}
cli_htu32_free(ht, root->mempool);
}
}

@ -38,7 +38,7 @@ enum CLI_HASH_TYPE {
};
struct cli_sz_hash {
uint8_t *hash_array; /* FIXME: make 256 entries? */
uint8_t *hash_array;
const char **virusnames;
uint32_t items;
};
@ -46,7 +46,6 @@ struct cli_sz_hash {
struct cli_hash_patt {
struct cli_htu32 sizehashes[CLI_HASH_AVAIL_TYPES];
int htinint[CLI_HASH_AVAIL_TYPES];
};
@ -54,5 +53,6 @@ int hm_addhash(struct cli_matcher *root, const char *hash, uint32_t size, const
void hm_flush(struct cli_matcher *root);
int cli_hm_scan(const unsigned char *digest, uint32_t size, const char **virname, const struct cli_matcher *root, enum CLI_HASH_TYPE type);
int cli_hm_have_size(const struct cli_matcher *root, enum CLI_HASH_TYPE type, uint32_t size);
void hm_free(struct cli_matcher *root);
#endif

@ -1,135 +0,0 @@
/*
* Copyright (C) 2007-2010 Sourcefire, Inc.
*
* Authors: Tomasz Kojm
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#if HAVE_CONFIG_H
#include "clamav-config.h"
#endif
#include <stdio.h>
#include "clamav.h"
#include "memory.h"
#include "mpool.h"
#include "others.h"
#include "cltypes.h"
#include "matcher.h"
#include "matcher-md5.h"
#define HASH(a,b,c) (211 * a + 37 * b + c)
int cli_md5m_addpatt(struct cli_matcher *root, struct cli_md5m_patt *patt)
{
unsigned int idx;
struct cli_md5m_patt *prev, *next = NULL;
idx = HASH(patt->md5[0], patt->md5[1], patt->md5[2]);
prev = next = root->md5tab[idx];
while(next) {
if(patt->md5[0] >= next->md5[0])
break;
prev = next;
next = next->next;
}
if(next == root->md5tab[idx]) {
patt->next = root->md5tab[idx];
root->md5tab[idx] = patt;
} else {
patt->next = prev->next;
prev->next = patt;
}
root->md5_patterns++;
return CL_SUCCESS;
}
int cli_md5m_init(struct cli_matcher *root)
{
#ifdef USE_MPOOL
if(!root->mempool) {
cli_errmsg("cli_md5m_init: mempool must be initialized\n");
return CL_EMEM;
}
#endif
if(!(root->md5tab = (struct cli_md5m_patt **) mpool_calloc(root->mempool, HASH(255, 255, 255) + 1, sizeof(struct cli_md5m_patt *)))) {
mpool_free(root->mempool, root->bm_shift);
return CL_EMEM;
}
return CL_SUCCESS;
}
void cli_md5m_free(struct cli_matcher *root)
{
struct cli_md5m_patt *patt, *prev;
unsigned int i, size = HASH(255, 255, 255) + 1;
if(root->md5tab) {
for(i = 0; i < size; i++) {
patt = root->md5tab[i];
while(patt) {
prev = patt;
patt = patt->next;
if(prev->virname)
mpool_free(root->mempool, prev->virname);
mpool_free(root->mempool, prev);
}
}
mpool_free(root->mempool, root->md5tab);
}
}
int cli_md5m_scan(const unsigned char *md5, uint32_t filesize, const char **virname, const struct cli_matcher *root)
{
unsigned int pchain = 0, idx;
struct cli_md5m_patt *p;
if(!root)
return CL_CLEAN;
idx = HASH(md5[0], md5[1], md5[2]);
p = root->md5tab[idx];
if(!p || (!p->next && p->filesize != filesize))
return CL_CLEAN;
while(p) {
if(p->md5[0] != md5[0]) {
if(pchain)
break;
p = p->next;
continue;
} else pchain = 1;
if(p->filesize != filesize) {
p = p->next;
continue;
}
if(!memcmp(p->md5, md5, 16)) {
if(virname)
*virname = p->virname;
return CL_VIRUS;
}
p = p->next;
}
return CL_CLEAN;
}

@ -1,39 +0,0 @@
/*
* Copyright (C) 2007-2010 Sourcefire, Inc.
*
* Authors: Tomasz Kojm
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef __MATCHER_MD5_H
#define __MATCHER_MD5_H
#include "matcher.h"
#include "cltypes.h"
struct cli_md5m_patt {
unsigned char md5[16];
uint32_t filesize;
char *virname;
struct cli_md5m_patt *next;
};
int cli_md5m_addpatt(struct cli_matcher *root, struct cli_md5m_patt *patt);
int cli_md5m_init(struct cli_matcher *root);
int cli_md5m_scan(const unsigned char *md5, uint32_t filesize, const char **virname, const struct cli_matcher *root);
void cli_md5m_free(struct cli_matcher *root);
#endif

@ -34,7 +34,6 @@
#include "others.h"
#include "matcher-ac.h"
#include "matcher-bm.h"
#include "matcher-md5.h"
#include "md5.h"
#include "filetypes.h"
#include "matcher.h"
@ -383,11 +382,6 @@ int cli_checkfp(unsigned char *digest, size_t size, cli_ctx *ctx)
unsigned int i;
const char *virname;
if(ctx->engine->md5_fp && cli_md5m_scan(digest, size, &virname, ctx->engine->md5_fp) == CL_VIRUS) {
cli_dbgmsg("cli_checkfp(): Found false positive detection (fp sig: %s)\n", virname);
return CL_CLEAN;
}
if(ctx->engine->hm_fp && cli_hm_scan(digest, size, &virname, ctx->engine->hm_fp, CLI_HASH_MD5) == CL_VIRUS) {
cli_dbgmsg("cli_checkfp(): Found false positive detection (fp sig: %s)\n", virname);
return CL_CLEAN;
@ -651,7 +645,7 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli
}
}
if(!refhash && !ftonly && (ctx->engine->md5_hdb || ctx->engine->hm_hdb))
if(!refhash && !ftonly && ctx->engine->hm_hdb)
cli_md5_init(&md5ctx);
while(offset < map->len) {
@ -696,7 +690,7 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli
type = ret;
}
if(!refhash && (ctx->engine->md5_hdb || ctx->engine->hm_hdb))
if(!refhash && ctx->engine->hm_hdb)
cli_md5_update(&md5ctx, buff + maxpatlen * (offset!=0), bytes - maxpatlen * (offset!=0));
}
@ -704,13 +698,11 @@ int cli_fmap_scandesc(cli_ctx *ctx, cli_file_t ftype, uint8_t ftonly, struct cli
offset += bytes - maxpatlen;
}
if(!ftonly && (ctx->engine->md5_hdb || ctx->engine->hm_hdb)) {
if(!ftonly && ctx->engine->hm_hdb) {
if(!refhash) {
cli_md5_final(digest, &md5ctx);
refhash = digest;
}
if(ctx->engine->md5_hdb && cli_md5m_scan(refhash, map->len, ctx->virname, ctx->engine->md5_hdb) == CL_VIRUS && cli_md5m_scan(refhash, map->len, NULL, ctx->engine->md5_fp) != CL_VIRUS)
ret = CL_VIRUS;
if(ctx->engine->hm_hdb && cli_hm_scan(refhash, map->len, ctx->virname, ctx->engine->hm_hdb, CLI_HASH_MD5) == CL_VIRUS && cli_hm_scan(refhash, map->len, NULL, ctx->engine->hm_fp, CLI_HASH_MD5) != CL_VIRUS)
ret = CL_VIRUS;
}

@ -28,7 +28,6 @@
#include "others.h"
#include "execs.h"
#include "cltypes.h"
#include "md5.h"
struct cli_target_info {
off_t fsize;
@ -39,7 +38,6 @@ struct cli_target_info {
#include "matcher-ac.h"
#include "matcher-bm.h"
#include "matcher-hash.h"
#include "hashtab.h"
#include "fmap.h"
#include "mpool.h"
@ -90,14 +88,10 @@ struct cli_matcher {
/* Extended Boyer-Moore */
uint8_t *bm_shift;
struct cli_bm_patt **bm_suffix, **bm_pattab;
struct cli_hashset md5_sizes_hs;
uint32_t *soff, soff_len; /* for PE section sigs */
uint32_t bm_offmode, bm_patterns, bm_reloff_num, bm_absoff_num;
/* MD5 */
struct cli_md5m_patt **md5tab;
uint32_t md5_patterns;
/* HASH */
struct cli_hash_patt hm;
/* Extended Aho-Corasick */

@ -208,16 +208,6 @@ struct cl_engine {
/* Roots table */
struct cli_matcher **root;
/* B-M matcher for standard MD5 sigs */
struct cli_matcher *md5_hdb;
/* B-M matcher for MD5 sigs for PE sections */
struct cli_matcher *md5_mdb;
/* B-M matcher for whitelist db */
struct cli_matcher *md5_fp;
/* hash matcher for standard MD5 sigs */
struct cli_matcher *hm_hdb;
/* hash matcher for MD5 sigs for PE sections */

@ -54,7 +54,7 @@
#include "mew.h"
#include "upack.h"
#include "matcher.h"
#include "matcher-md5.h"
#include "matcher-hash.h"
#include "disasm.h"
#include "special.h"
#include "ishield.h"
@ -1000,23 +1000,6 @@ int cli_scanpe(cli_ctx *ctx)
if(SCAN_ALGO && (DCONF & PE_CONF_POLIPOS) && !*sname && exe_sections[i].vsz > 40000 && exe_sections[i].vsz < 70000 && exe_sections[i].chr == 0xe0000060) polipos = i;
/* check MD5 section sigs */
md5_sect = ctx->engine->md5_mdb;
if((DCONF & PE_CONF_MD5SECT) && md5_sect) {
for(j = 0; j < md5_sect->soff_len && md5_sect->soff[j] <= exe_sections[i].rsz; j++) {
if(md5_sect->soff[j] == exe_sections[i].rsz) {
unsigned char md5_dig[16];
if(cli_md5sect(map, &exe_sections[i], md5_dig) && cli_md5m_scan(md5_dig, exe_sections[i].rsz, ctx->virname, ctx->engine->md5_mdb) == CL_VIRUS) {
if(cli_md5m_scan(md5_dig, fsize, NULL, ctx->engine->md5_fp) != CL_VIRUS) {
free(section_hdr);
free(exe_sections);
return CL_VIRUS;
}
}
break;
}
}
}
md5_sect = ctx->engine->hm_mdb;
if((DCONF & PE_CONF_MD5SECT) && md5_sect) {
unsigned char md5_dig[16];

@ -46,7 +46,6 @@
#endif
#include "matcher-ac.h"
#include "matcher-bm.h"
#include "matcher-md5.h"
#include "matcher-hash.h"
#include "matcher.h"
#include "others.h"
@ -1861,171 +1860,6 @@ static int cli_loadign(FILE *fs, struct cl_engine *engine, unsigned int options,
#define MD5_MDB 1
#define MD5_FP 2
static int cli_md5db_init(struct cl_engine *engine, unsigned int mode)
{
struct cli_matcher *bm = NULL;
int ret;
if(mode == MD5_HDB) {
bm = engine->md5_hdb = (struct cli_matcher *) mpool_calloc(engine->mempool, sizeof(struct cli_matcher), 1);
} else if(mode == MD5_MDB) {
bm = engine->md5_mdb = (struct cli_matcher *) mpool_calloc(engine->mempool, sizeof(struct cli_matcher), 1);
} else {
bm = engine->md5_fp = (struct cli_matcher *) mpool_calloc(engine->mempool, sizeof(struct cli_matcher), 1);
}
if(!bm)
return CL_EMEM;
#ifdef USE_MPOOL
bm->mempool = engine->mempool;
#endif
if((ret = cli_md5m_init(bm))) {
cli_errmsg("cli_md5db_init: Failed to initialize MD5 matcher\n");
return ret;
}
return CL_SUCCESS;
}
#define MD5_DB \
if(mode == MD5_HDB) \
db = engine->md5_hdb; \
else if(mode == MD5_MDB) \
db = engine->md5_mdb; \
else \
db = engine->md5_fp;
#define MD5_TOKENS 3
static int cli_loadmd5(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int mode, unsigned int options, struct cli_dbio *dbio, const char *dbname)
{
const char *tokens[MD5_TOKENS + 1];
char buffer[FILEBUFF], *buffer_cpy = NULL;
const char *pt;
unsigned char *md5;
int ret = CL_SUCCESS;
unsigned int size_field = 1, md5_field = 0, line = 0, sigs = 0, tokens_count;
struct cli_md5m_patt *new;
struct cli_matcher *db = NULL;
if(mode == MD5_MDB) {
size_field = 0;
md5_field = 1;
}
if(engine->ignored)
if(!(buffer_cpy = cli_malloc(FILEBUFF)))
return CL_EMEM;
while(cli_dbgets(buffer, FILEBUFF, fs, dbio)) {
line++;
cli_chomp(buffer);
if(engine->ignored)
strcpy(buffer_cpy, buffer);
tokens_count = cli_strtokenize(buffer, ':', MD5_TOKENS + 1, tokens);
if(tokens_count != MD5_TOKENS) {
ret = CL_EMALFDB;
break;
}
if(!cli_isnumber(tokens[size_field])) {
cli_errmsg("cli_loadmd5: Invalid value for the size field\n");
ret = CL_EMALFDB;
break;
}
pt = tokens[2]; /* virname */
if(engine->pua_cats && (options & CL_DB_PUA_MODE) && (options & (CL_DB_PUA_INCLUDE | CL_DB_PUA_EXCLUDE)))
if(cli_chkpua(pt, engine->pua_cats, options))
continue;
if(engine->ignored && cli_chkign(engine->ignored, pt, buffer_cpy))
continue;
if(engine->cb_sigload) {
const char *dot = strchr(dbname, '.');
if(!dot)
dot = dbname;
else
dot++;
if(engine->cb_sigload(dot, pt, engine->cb_sigload_ctx)) {
cli_dbgmsg("cli_loadmd5: skipping %s due to callback\n", pt);
continue;
}
}
new = (struct cli_md5m_patt *) mpool_calloc(engine->mempool, 1, sizeof(struct cli_md5m_patt));
if(!new) {
ret = CL_EMEM;
break;
}
pt = tokens[md5_field]; /* md5 */
if(strlen(pt) != 32 || !(md5 = (unsigned char *) cli_mpool_hex2str(engine->mempool, pt))) {
cli_errmsg("cli_loadmd5: Malformed MD5 string at line %u\n", line);
mpool_free(engine->mempool, new);
ret = CL_EMALFDB;
break;
}
memcpy(new->md5, md5, 16);
mpool_free(engine->mempool, md5);
new->filesize = atoi(tokens[size_field]);
new->virname = cli_mpool_virname(engine->mempool, tokens[2], options & CL_DB_OFFICIAL);
if(!new->virname) {
mpool_free(engine->mempool, new);
ret = CL_EMALFDB;
break;
}
MD5_DB;
if(!db && (ret = cli_md5db_init(engine, mode))) {
mpool_free(engine->mempool, new->virname);
mpool_free(engine->mempool, new);
break;
} else {
MD5_DB;
}
if((ret = cli_md5m_addpatt(db, new))) {
cli_errmsg("cli_loadmd5: Error adding BM pattern\n");
mpool_free(engine->mempool, new->virname);
mpool_free(engine->mempool, new);
break;
}
if(mode == MD5_MDB) { /* section MD5 */
if(!db->md5_sizes_hs.capacity) {
cli_hashset_init_pool(&db->md5_sizes_hs, 65536, 80, engine->mempool);
}
cli_hashset_addkey(&db->md5_sizes_hs, new->filesize);
}
sigs++;
}
if(engine->ignored)
free(buffer_cpy);
if(!line) {
cli_errmsg("cli_loadmd5: Empty database file\n");
return CL_EMALFDB;
}
if(ret) {
cli_errmsg("cli_loadmd5: Problem parsing database at line %u\n", line);
return ret;
}
if(signo)
*signo += sigs;
return CL_SUCCESS;
}
#define MD5_TOKENS 3
static int cli_loadhash(FILE *fs, struct cl_engine *engine, unsigned int *signo, unsigned int mode, unsigned int options, struct cli_dbio *dbio, const char *dbname)
{
@ -2506,24 +2340,22 @@ int cli_load(const char *filename, struct cl_engine *engine, unsigned int *signo
} else if(cli_strbcasestr(dbname, ".cld")) {
ret = cli_cvdload(fs, engine, signo, options, 1, filename);
} else if(cli_strbcasestr(dbname, ".hdb")) {
ret = cli_loadmd5(fs, engine, signo, MD5_HDB, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".hdu")) {
} else if(cli_strbcasestr(dbname, ".hdb") || cli_strbcasestr(dbname, ".hsb")) {
ret = cli_loadhash(fs, engine, signo, MD5_HDB, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".hdu") || cli_strbcasestr(dbname, ".hsu")) {
if(options & CL_DB_PUA)
ret = cli_loadmd5(fs, engine, signo, MD5_HDB, options | CL_DB_PUA_MODE, dbio, dbname);
ret = cli_loadhash(fs, engine, signo, MD5_HDB, options | CL_DB_PUA_MODE, dbio, dbname);
else
skipped = 1;
} else if(cli_strbcasestr(dbname, ".fp")) {
ret = cli_loadmd5(fs, engine, signo, MD5_FP, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".mdb")) {
ret = cli_loadmd5(fs, engine, signo, MD5_MDB, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".fp") || cli_strbcasestr(dbname, ".sfp")) {
ret = cli_loadhash(fs, engine, signo, MD5_FP, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".mdb") || cli_strbcasestr(dbname, ".msb")) {
ret = cli_loadhash(fs, engine, signo, MD5_MDB, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".mdu")) {
} else if(cli_strbcasestr(dbname, ".mdu") || cli_strbcasestr(dbname, ".msu")) {
if(options & CL_DB_PUA)
ret = cli_loadmd5(fs, engine, signo, MD5_MDB, options | CL_DB_PUA_MODE, dbio, dbname);
ret = cli_loadhash(fs, engine, signo, MD5_MDB, options | CL_DB_PUA_MODE, dbio, dbname);
else
skipped = 1;
@ -2585,24 +2417,6 @@ int cli_load(const char *filename, struct cl_engine *engine, unsigned int *signo
} else if(cli_strbcasestr(dbname, ".cdb")) {
ret = cli_loadcdb(fs, engine, signo, options, dbio);
} else if(cli_strbcasestr(dbname, ".hsb")) {
ret = cli_loadhash(fs, engine, signo, MD5_HDB, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".hsu")) {
if(options & CL_DB_PUA)
ret = cli_loadhash(fs, engine, signo, MD5_HDB, options | CL_DB_PUA_MODE, dbio, dbname);
else
skipped = 1;
} else if(cli_strbcasestr(dbname, ".sfp")) {
ret = cli_loadhash(fs, engine, signo, MD5_FP, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".msb")) {
ret = cli_loadhash(fs, engine, signo, MD5_MDB, options, dbio, dbname);
} else if(cli_strbcasestr(dbname, ".msu")) {
if(options & CL_DB_PUA)
ret = cli_loadhash(fs, engine, signo, MD5_MDB, options | CL_DB_PUA_MODE, dbio, dbname);
else
skipped = 1;
} else {
cli_dbgmsg("cli_load: unknown extension - assuming old database format\n");
ret = cli_loaddb(fs, engine, signo, options, dbio, dbname);
@ -3079,22 +2893,18 @@ int cl_engine_free(struct cl_engine *engine)
mpool_free(engine->mempool, engine->root);
}
if((root = engine->md5_hdb)) {
cli_md5m_free(root);
if((root = engine->hm_hdb)) {
hm_free(root);
mpool_free(engine->mempool, root);
}
if((root = engine->md5_mdb)) {
cli_md5m_free(root);
mpool_free(engine->mempool, root->soff);
if(root->md5_sizes_hs.capacity) {
cli_hashset_destroy(&root->md5_sizes_hs);
}
if((root = engine->hm_mdb)) {
hm_free(root);
mpool_free(engine->mempool, root);
}
if((root = engine->md5_fp)) {
cli_md5m_free(root);
if((root = engine->hm_fp)) {
hm_free(root);
mpool_free(engine->mempool, root);
}
@ -3179,29 +2989,6 @@ int cl_engine_free(struct cl_engine *engine)
return CL_SUCCESS;
}
static void cli_md5db_build(struct cli_matcher* root)
{
if(root && root->md5_sizes_hs.capacity) {
/* TODO: use hashset directly, instead of the array when matching*/
cli_dbgmsg("Converting hashset to array: %u entries\n", root->md5_sizes_hs.count);
#ifdef USE_MPOOL
{
uint32_t *mpoolht;
unsigned int mpoolhtsz = root->md5_sizes_hs.count * sizeof(*mpoolht);
root->soff = mpool_malloc(root->mempool, mpoolhtsz);
root->soff_len = cli_hashset_toarray(&root->md5_sizes_hs, &mpoolht);
memcpy(root->soff, mpoolht, mpoolhtsz);
free(mpoolht);
}
#else
root->soff_len = cli_hashset_toarray(&root->md5_sizes_hs, &root->soff);
#endif
cli_hashset_destroy(&root->md5_sizes_hs);
cli_qsort(root->soff, root->soff_len, sizeof(uint32_t), NULL);
}
}
int cl_engine_compile(struct cl_engine *engine)
{
unsigned int i;
@ -3223,12 +3010,6 @@ int cl_engine_compile(struct cl_engine *engine)
cli_dbgmsg("Matcher[%u]: %s: AC sigs: %u (reloff: %u, absoff: %u) BM sigs: %u (reloff: %u, absoff: %u) maxpatlen %u %s\n", i, cli_mtargets[i].name, root->ac_patterns, root->ac_reloff_num, root->ac_absoff_num, root->bm_patterns, root->bm_reloff_num, root->bm_absoff_num, root->maxpatlen, root->ac_only ? "(ac_only mode)" : "");
}
}
if(engine->md5_hdb)
cli_dbgmsg("MD5 sigs (files): %u\n", engine->md5_hdb->md5_patterns);
if(engine->md5_mdb)
cli_dbgmsg("MD5 sigs (PE sections): %u\n", engine->md5_mdb->md5_patterns);
if(engine->hm_hdb)
hm_flush(engine->hm_hdb);
@ -3244,7 +3025,6 @@ int cl_engine_compile(struct cl_engine *engine)
if((ret = cli_build_regex_list(engine->domainlist_matcher))) {
return ret;
}
cli_md5db_build(engine->md5_mdb);
if(engine->ignored) {
cli_bm_free(engine->ignored);
mpool_free(engine->mempool, engine->ignored);

Loading…
Cancel
Save