Add bytecode performance statistics

0.98.2
Steve Morgan 13 years ago
parent 729f973756
commit 54402320c0
  1. 5
      clambc/bcrun.c
  2. 3
      clamscan/clamscan.c
  3. 10
      clamscan/manager.c
  4. 3
      docs/man/clamscan.1.in
  5. 129
      libclamav/bytecode.c
  6. 54
      libclamav/bytecode.h
  7. 1
      libclamav/clamav.h
  8. 10
      libclamav/events.c
  9. 4
      libclamav/events.h
  10. 3
      libclamav/libclamav.map
  11. 4
      libclamav/readdb.c
  12. 6
      shared/optparser.c
  13. 4
      unit_tests/check_bytecode.c

@ -1,7 +1,7 @@
/*
* ClamAV bytecode handler tool.
*
* Copyright (C) 2009 Sourcefire, Inc.
* Copyright (C) 2009-2012 Sourcefire, Inc.
*
* Authors: Török Edvin
*
@ -306,7 +306,8 @@ int main(int argc, char *argv[])
bcs.all_bcs = bc;
bcs.count = 1;
rc = cli_bytecode_load(bc, f, NULL, optget(opts, "trust-bytecode")->enabled);
rc = cli_bytecode_load(bc, f, NULL, optget(opts, "trust-bytecode")->enabled,
optget(opts, "bytecode-statistics")->enabled);
if (rc != CL_SUCCESS) {
fprintf(stderr,"Unable to load bytecode: %s\n", cl_strerror(rc));
optfree(opts);

@ -1,5 +1,5 @@
/*
* Copyright (C) 2007-2009 Sourcefire, Inc.
* Copyright (C) 2007-2012 Sourcefire, Inc.
*
* Authors: Tomasz Kojm
*
@ -231,6 +231,7 @@ void help(void)
mprintf(" --bytecode[=yes(*)/no] Load bytecode from the database\n");
mprintf(" --bytecode-unsigned[=yes/no(*)] Load unsigned bytecode\n");
mprintf(" --bytecode-timeout=N Set bytecode timeout (in milliseconds)\n");
mprintf(" --bytecode-statistics[=yes/no(*)] Collect and print bytecode statistics\n");
mprintf(" --detect-pua[=yes/no(*)] Detect Possibly Unwanted Applications\n");
mprintf(" --exclude-pua=CAT Skip PUA sigs of category CAT\n");
mprintf(" --include-pua=CAT Load PUA sigs of category CAT\n");

@ -1,5 +1,5 @@
/*
* Copyright (C) 2007-2009 Sourcefire, Inc.
* Copyright (C) 2007-2012 Sourcefire, Inc.
*
* Authors: Tomasz Kojm
*
@ -635,6 +635,9 @@ int scanmanager(const struct optstruct *opts)
if(optget(opts, "bytecode-unsigned")->enabled)
dboptions |= CL_DB_BYTECODE_UNSIGNED;
if(optget(opts, "bytecode-statistics")->enabled)
dboptions |= CL_DB_BYTECODE_STATS;
if((opt = optget(opts,"bytecode-timeout"))->enabled)
cl_engine_set_num(engine, CL_ENGINE_BYTECODE_TIMEOUT, opt->numarg);
if((opt = optget(opts,"bytecode-mode"))->enabled) {
@ -938,6 +941,11 @@ int scanmanager(const struct optstruct *opts)
}
}
if(optget(opts, "bytecode-statistics")->enabled) {
cli_sigperf_print();
cli_sigperf_events_destroy();
}
/* free the engine */
cl_engine_free(engine);

@ -96,6 +96,9 @@ Allow loading bytecode from outside digitally signed .c[lv]d files.
\fB\-\-bytecode\-timeout=N\fR
Set bytecode timeout in milliseconds (default: 60000 = 60s)
.TP
\fB\-\-bytecode\-statistics[=yes/no(*)]\fR
Collect and print bytecode statistics.
.TP
\fB\-\-detect\-pua[=yes/no(*)]\fR
Detect Possibly Unwanted Applications.
.TP

@ -1,7 +1,7 @@
/*
* Load, and verify ClamAV bytecode.
*
* Copyright (C) 2009-2010 Sourcefire, Inc.
* Copyright (C) 2009-2012 Sourcefire, Inc.
*
* Authors: Török Edvin
*
@ -40,6 +40,13 @@
#include "builtin_bytecodes.h"
#include <string.h>
#define MAX_BC 64
#define BC_EVENTS_PER_SIG 2
#define MAX_BC_SIGEVENT_ID MAX_BC*BC_EVENTS_PER_SIG
cli_events_t * g_sigevents = NULL;
unsigned int g_sigid;
/* dummy values */
static const uint32_t nomatch[64] = {
0xdeadbeef, 0xdeaddead, 0xbeefdead, 0xdeaddead, 0xdeadbeef, 0, 0, 0,
@ -1409,7 +1416,112 @@ enum parse_state {
PARSE_SKIP
};
int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int trust)
struct sigperf_elem {
const char * bc_name;
uint64_t usecs;
unsigned long run_count;
unsigned long match_count;
};
static int sigelem_comp(const void * a, const void * b)
{
const struct sigperf_elem *ela = a;
const struct sigperf_elem *elb = b;
return elb->usecs/elb->run_count - ela->usecs/ela->run_count;
}
void cli_sigperf_print()
{
struct sigperf_elem stats[MAX_BC], *elem = stats;
int i, elems = 0, max_name_len = 0, name_len;
memset(stats, 0, sizeof(stats));
for (i=0;i<MAX_BC;i++) {
union ev_val val;
uint32_t count;
const char * name = cli_event_get_name(g_sigevents, i*BC_EVENTS_PER_SIG);
cli_event_get(g_sigevents, i*BC_EVENTS_PER_SIG, &val, &count);
if (!count) {
if (name)
cli_dbgmsg("No event triggered for %s\n", name);
continue;
}
name_len = strlen(name);
if (name_len > max_name_len)
max_name_len = name_len;
elem->bc_name = name?name:"\"noname\"";
elem->usecs = val.v_int;
elem->run_count = count;
cli_event_get(g_sigevents, i*BC_EVENTS_PER_SIG+1, &val, &count);
elem->match_count = count;
elem++, elems++;
}
qsort(stats, elems, sizeof(struct sigperf_elem), sigelem_comp);
elem = stats;
/* name runs matches microsecs avg */
cli_infomsg (NULL, "%-*s %*s %*s %*s %*s\n", max_name_len, "Bytecode name",
8, "#runs", 8, "#matches", 12, "usecs total", 9, "usecs avg");
cli_infomsg (NULL, "%-*s %*s %*s %*s %*s\n", max_name_len, "=============",
8, "=====", 8, "========", 12, "===========", 9, "=========");
while (elem->run_count) {
cli_infomsg (NULL, "%-*s %*lu %*lu %*lu %*.2f\n", max_name_len, elem->bc_name,
8, elem->run_count, 8, elem->match_count,
12, elem->usecs, 9, (double)elem->usecs/elem->run_count);
elem++;
}
}
static void sigperf_events_init(struct cli_bc *bc)
{
int ret;
if (!g_sigevents)
g_sigevents = cli_events_new(MAX_BC_SIGEVENT_ID);
if (!g_sigevents) {
cli_errmsg("No memory for events table\n");
return;
}
if (g_sigid > MAX_BC_SIGEVENT_ID - BC_EVENTS_PER_SIG - 1) {
cli_errmsg("sigperf_events_init: events table full. Increase MAX_BC\n");
return;
}
cli_dbgmsg("sigperf_events_init(): adding sig ids starting %u for %s\n", g_sigid, bc->lsig);
if (!bc->lsig) {
cli_dbgmsg("cli_event_define error for time event id %d\n", bc->sigtime_id);
return;
}
/* register time event */
bc->sigtime_id = g_sigid;
ret = cli_event_define(g_sigevents, g_sigid++, bc->lsig, ev_time, multiple_sum);
if (ret) {
cli_errmsg("sigperf_events_init: cli_event_define() error for time event id %d\n", bc->sigtime_id);
bc->sigtime_id = MAX_BC_SIGEVENT_ID+1;
return;
}
/* register match count */
bc->sigmatch_id = g_sigid;
ret = cli_event_define(g_sigevents, g_sigid++, bc->lsig, ev_int, multiple_sum);
if (ret) {
cli_errmsg("sigperf_events_init: cli_event_define() error for matches event id %d\n", bc->sigmatch_id);
bc->sigtime_id = MAX_BC_SIGEVENT_ID+1;
return;
}
}
void cli_sigperf_events_destroy()
{
cli_events_free(g_sigevents);
}
int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int trust, int sigperf)
{
unsigned row = 0, current_func = 0, bb=0;
char *buffer;
@ -1565,6 +1677,8 @@ int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int tru
}
free(buffer);
cli_dbgmsg("Parsed %d functions\n", current_func);
if (sigperf)
sigperf_events_init(bc);
if (current_func != bc->num_func && bc->state != bc_skip) {
cli_errmsg("Loaded less functions than declared: %u vs. %u\n",
current_func, bc->num_func);
@ -1652,6 +1766,7 @@ int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, stru
return CL_EBYTECODE_TESTFAIL;
}
}
cli_event_time_start(g_sigevents, bc->sigtime_id);
if (bc->state == bc_interp || test_mode) {
ctx->bc_events = interp_ev;
memset(&func, 0, sizeof(func));
@ -1705,6 +1820,9 @@ int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, stru
if (ctx->outfd)
cli_bcapi_extract_new(ctx, -1);
}
cli_event_time_stop(g_sigevents, bc->sigtime_id);
if (ctx->virname)
cli_event_count(g_sigevents, bc->sigmatch_id);
if (test_mode) {
unsigned interp_errors = cli_event_errors(interp_ev);
@ -1738,7 +1856,8 @@ int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, stru
ok = 0;
}
/*cli_event_debug(jit_ev, BCEV_EXEC_TIME);
cli_event_debug(interp_ev, BCEV_EXEC_TIME);*/
cli_event_debug(interp_ev, BCEV_EXEC_TIME);
cli_event_debug(g_sigevents, bc->sigtime_id);*/
if (!ok) {
cli_events_free(jit_ev);
cli_events_free(interp_ev);
@ -2398,7 +2517,7 @@ static int run_builtin_or_loaded(struct cli_all_bc *bcs, uint8_t kind, const cha
return CL_EMALFDB;
}
rc = cli_bytecode_load(bc, NULL, &dbio, 1);
rc = cli_bytecode_load(bc, NULL, &dbio, 1, 0);
if (rc) {
cli_errmsg("Failed to load builtin %s bytecode\n", desc);
free(bc);
@ -2692,7 +2811,7 @@ int cli_bytecode_runhook(cli_ctx *cctx, const struct cl_engine *engine, struct c
continue;
}
if (ctx->virname) {
cli_dbgmsg("Bytecode found virus: %s\n", ctx->virname);
cli_dbgmsg("Bytecode runhook found virus: %s\n", ctx->virname);
cli_append_virus(cctx, ctx->virname);
if (!(cctx->options & CL_SCAN_ALLMATCHES)) {
cli_bytecode_context_clear(ctx);

@ -1,7 +1,7 @@
/*
* Load, verify and execute ClamAV bytecode.
*
* Copyright (C) 2009-2010 Sourcefire, Inc.
* Copyright (C) 2009-2012 Sourcefire, Inc.
*
* Authors: Török Edvin
*
@ -47,29 +47,30 @@ enum bc_state {
};
struct cli_bc {
struct bytecode_metadata metadata;
unsigned id;
unsigned kind;
unsigned num_types;
unsigned num_func;
struct cli_bc_func *funcs;
struct cli_bc_type *types;
uint64_t **globals;
uint16_t *globaltys;
size_t num_globals;
enum bc_state state;
struct bitset_tag *uses_apis;
char *lsig;
char *vnameprefix;
char **vnames;
unsigned vnames_cnt;
uint16_t start_tid;
struct cli_bc_dbgnode *dbgnodes;
unsigned dbgnode_cnt;
unsigned hook_lsig_id;
unsigned trusted;
uint32_t numGlobalBytes;
uint8_t *globalBytes;
struct bytecode_metadata metadata;
unsigned id;
unsigned kind;
unsigned num_types;
unsigned num_func;
struct cli_bc_func *funcs;
struct cli_bc_type *types;
uint64_t **globals;
uint16_t *globaltys;
size_t num_globals;
enum bc_state state;
struct bitset_tag *uses_apis;
char *lsig;
char *vnameprefix;
char **vnames;
unsigned vnames_cnt;
uint16_t start_tid;
struct cli_bc_dbgnode *dbgnodes;
unsigned dbgnode_cnt;
unsigned hook_lsig_id;
unsigned trusted;
uint32_t numGlobalBytes;
uint8_t *globalBytes;
uint32_t sigtime_id, sigmatch_id;
};
struct cli_all_bc {
@ -107,7 +108,7 @@ extern int have_clamjit;
}
#endif
int cli_bytecode_init(struct cli_all_bc *allbc);
int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int security);
int cli_bytecode_load(struct cli_bc *bc, FILE *f, struct cli_dbio *dbio, int security, int sigperf);
int cli_bytecode_prepare2(struct cl_engine *engine, struct cli_all_bc *allbc, unsigned dconfmask);
int cli_bytecode_run(const struct cli_all_bc *bcs, const struct cli_bc *bc, struct cli_bc_ctx *ctx);
void cli_bytecode_destroy(struct cli_bc *bc);
@ -141,7 +142,8 @@ void cli_bytecode_context_set_trace(struct cli_bc_ctx*, unsigned level,
bc_dbg_callback_trace_op,
bc_dbg_callback_trace_val,
bc_dbg_callback_trace_ptr);
void cli_sigperf_print(void);
void cli_sigperf_events_destroy(void);
#ifdef __cplusplus
}
#endif

@ -120,6 +120,7 @@ typedef enum {
#define CL_DB_SIGNED 0x4000 /* internal */
#define CL_DB_BYTECODE_UNSIGNED 0x8000
#define CL_DB_UNSIGNED 0x10000 /* internal */
#define CL_DB_BYTECODE_STATS 0x20000
/* recommended db settings */
#define CL_DB_STDOPT (CL_DB_PHISHING | CL_DB_PHISHING_URLS | CL_DB_BYTECODE)

@ -1,7 +1,7 @@
/*
* (bytecode) events
*
* Copyright (C) 2010 Sourcefire, Inc.
* Copyright (C) 2010-2012 Sourcefire, Inc.
*
* Authors: Török Edvin
*
@ -138,6 +138,14 @@ static inline void ev_chain(cli_events_t *ctx, struct cli_event *ev, union ev_va
ev->count++;
}
const char * cli_event_get_name(cli_events_t *ctx, unsigned id)
{
struct cli_event *ev = get_event(ctx, id);
if (!ev)
return NULL;
return ev->name;
}
void cli_event_int(cli_events_t *ctx, unsigned id, uint64_t arg)
{
struct cli_event *ev = get_event(ctx, id);

@ -1,7 +1,7 @@
/*
* (bytecode) events
*
* Copyright (C) 2010 Sourcefire, Inc.
* Copyright (C) 2010-2012 Sourcefire, Inc.
*
* Authors: Török Edvin
*
@ -83,6 +83,8 @@ void cli_event_count(cli_events_t *ctx, unsigned id);
void cli_event_get(cli_events_t* ctx, unsigned id, union ev_val *val, uint32_t *count);
const char * cli_event_get_name(cli_events_t* ctx, unsigned id);
/* print all recorded events */
void cli_event_debug_all(cli_events_t *ctx);

@ -44,6 +44,9 @@ CLAMAV_PUBLIC {
};
CLAMAV_PRIVATE {
global:
cli_sigperf_print;
cli_sigperf_events_destroy;
cli_gettmpdir;
cli_strtok;
cli_strtokenize;

@ -1,5 +1,5 @@
/*
* Copyright (C) 2007-2010 Sourcefire, Inc.
* Copyright (C) 2007-2012 Sourcefire, Inc.
*
* Authors: Tomasz Kojm
*
@ -1487,7 +1487,7 @@ static int cli_loadcbc(FILE *fs, struct cl_engine *engine, unsigned int *signo,
security_trust = 0;
}
rc = cli_bytecode_load(bc, fs, dbio, security_trust);
rc = cli_bytecode_load(bc, fs, dbio, security_trust, options&CL_DB_BYTECODE_STATS);
/* read remainder of DB, needed because cvd.c checks that we read the entire
* file */
while (cli_dbgets(buf, sizeof(buf), fs, dbio)) {}

@ -1,5 +1,5 @@
/*
* Copyright (C) 2008-2009 Sourcefire, Inc.
* Copyright (C) 2008-2012 Sourcefire, Inc.
*
* Author: Tomasz Kojm <tkojm@clamav.net>
*
@ -276,7 +276,9 @@ const struct clam_option __clam_options[] = {
{ "BytecodeMode", "bytecode-mode", 0, TYPE_STRING, "^(Auto|ForceJIT|ForceInterpreter|Test)$", -1, "Auto", FLAG_REQUIRED, OPT_CLAMD | OPT_CLAMSCAN,
"Set bytecode execution mode.\nPossible values:\n\tAuto - automatically choose JIT if possible, fallback to interpreter\nForceJIT - always choose JIT, fail if not possible\nForceIntepreter - always choose interpreter\nTest - run with both JIT and interpreter and compare results. Make all failures fatal.","Auto"},
{ "DetectPUA", "detect-pua", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "Detect Potentially Unwanted Applications.", "yes" },
{ "BytecodeStatistics", "bytecode-statistics", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMSCAN, "Collect and print bytecode execution statistics.", "no" },
{ "DetectPUA", "detect-pua", 0, TYPE_BOOL, MATCH_BOOL, 0, NULL, 0, OPT_CLAMD | OPT_CLAMSCAN, "Detect Potentially Unwanted Applications.", "yes" },
{ "ExcludePUA", "exclude-pua", 0, TYPE_STRING, NULL, -1, NULL, FLAG_MULTIPLE, OPT_CLAMD | OPT_CLAMSCAN, "Exclude a specific PUA category. This directive can be used multiple times.\nSee http://www.clamav.net/support/pua for the complete list of PUA\ncategories.", "NetTool\nPWTool" },

@ -1,7 +1,7 @@
/*
* Unit tests for bytecode functions.
*
* Copyright (C) 2009 Sourcefire, Inc.
* Copyright (C) 2009-2012 Sourcefire, Inc.
*
* Authors: Török Edvin
*
@ -87,7 +87,7 @@ static void runtest(const char *file, uint64_t expected, int fail, int nojit,
bcs.all_bcs = &bc;
bcs.count = 1;
rc = cli_bytecode_load(&bc, f, NULL, 1);
rc = cli_bytecode_load(&bc, f, NULL, 1, 0);
fail_unless(rc == CL_SUCCESS, "cli_bytecode_load failed");
fclose(f);

Loading…
Cancel
Save