open-dsi
/
clamav
mirror of https://github.com/Cisco-Talos/clamav
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix YARA arena management, improve error reporting, clean up some code.

Browse Source
remotes/push_mirror/klin/altstr-yara^2
Steven Morgan 10 years ago
parent 0abc761f12
commit d03c18bed3
13 changed files with 248 additions and 286 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 38
      libclamav/others.c
  2. 6
      libclamav/others.h
  3. 44
      libclamav/readdb.c
  4. 6
      libclamav/yara_arena.c
  5. 118
      libclamav/yara_clam.h
  6. 2
      libclamav/yara_exec.c
  7. 277
      libclamav/yara_grammar.c
  8. 2
      libclamav/yara_grammar.h
  9. 1
      libclamav/yara_grammar.y
  10. 2
      libclamav/yara_hash.c
  11. 14
      libclamav/yara_lexer.c
  12. 14
      libclamav/yara_lexer.l
  13. 6
      libclamav/yara_parser.c

38
libclamav/others.c
Unescape View File

@ -436,6 +436,44 @@ struct cl_engine *cl_engine_new(void)
new->pcre_recmatch_limit = CLI_DEFAULT_PCRE_RECMATCH_LIMIT;
new->pcre_max_filesize = CLI_DEFAULT_PCRE_MAX_FILESIZE;
/* Initialize YARA */
if (ERROR_SUCCESS != yr_arena_create(1024, 0, &new->the_arena)) {
cli_errmsg("cli_engine_new: failed to create the YARA arena\n");
mpool_free(new->mempool, new->dconf);
mpool_free(new->mempool, new->root);
#ifdef USE_MPOOL
mpool_destroy(new->mempool);
#endif
free(new);
free(intel);
return NULL;
}
if (ERROR_SUCCESS != yr_hash_table_create(10007, &new->rules_table)) {
cli_errmsg("cli_engine_new: failed to create the YARA rules table\n");
yr_arena_destroy(new->the_arena);
mpool_free(new->mempool, new->dconf);
mpool_free(new->mempool, new->root);
#ifdef USE_MPOOL
mpool_destroy(new->mempool);
#endif
free(new);
free(intel);
return NULL;
}
if (ERROR_SUCCESS != yr_hash_table_create(10007, &new->objects_table)) {
cli_errmsg("cli_engine_new: failed to create the YARA objects table\n");
yr_hash_table_destroy(new->rules_table, NULL);
yr_arena_destroy(new->the_arena);
mpool_free(new->mempool, new->dconf);
mpool_free(new->mempool, new->root);
#ifdef USE_MPOOL
mpool_destroy(new->mempool);
#endif
free(new);
free(intel);
return NULL;
}
cli_dbgmsg("Initialized %s engine\n", cl_retver());
return new;
}

6
libclamav/others.h
Unescape View File

@ -53,6 +53,7 @@
#ifdef HAVE_JSON
#include "json.h"
#endif
#include "yara_clam.h"
#if HAVE_LIBXML2
#define CLAMAV_MIN_XMLREADER_FLAGS (XML_PARSE_NOERROR | XML_PARSE_NONET)
@ -356,6 +357,11 @@ struct cl_engine {
uint64_t pcre_match_limit;
uint64_t pcre_recmatch_limit;
uint64_t pcre_max_filesize;
/* YARA */
YR_ARENA * the_arena;
YR_HASH_TABLE * rules_table;
YR_HASH_TABLE * objects_table;
};
struct cl_settings {

44
libclamav/readdb.c
Unescape View File

@ -3774,10 +3774,6 @@ static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo,
STAILQ_INIT(&compiler.rule_q);
STAILQ_INIT(&compiler.current_rule_string_q);
rc = yr_hash_table_create(10007, &compiler.rules_table);
if (rc == ERROR_SUCCESS)
rc = yr_hash_table_create(10007, &compiler.objects_table);
if (rc == ERROR_SUCCESS)
rc = yr_arena_create(65536, 0, &compiler.sz_arena);
if (rc == ERROR_SUCCESS)
rc = yr_arena_create(65536, 0, &compiler.rules_arena);
@ -3792,17 +3788,18 @@ static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo,
compiler.loop_for_of_mem_offset = -1;
ns.name = "default";
compiler.current_namespace = &ns;
compiler.the_arena = engine->the_arena;
compiler.rules_table = engine->rules_table;
compiler.objects_table = engine->objects_table;
rc = yr_lex_parse_rules_file(fs, &compiler);
if (rc > 0) { /* rc = number of errors */
/* TODO - handle the various errors? */
cli_errmsg("cli_loadyara: failed to parse rules file %s, error count %i\n", dbname, rc);
yr_hash_table_destroy(compiler.rules_table, NULL);
yr_hash_table_destroy(compiler.objects_table, NULL);
// yr_arena_destroy(compiler.sz_arena);
// yr_arena_destroy(compiler.rules_arena);
yr_arena_destroy(compiler.sz_arena);
yr_arena_destroy(compiler.rules_arena);
yr_arena_destroy(compiler.code_arena);
// yr_arena_destroy(compiler.strings_arena);
yr_arena_destroy(compiler.strings_arena);
yr_arena_destroy(compiler.metas_arena);
#ifdef YARA_FINISHED
return CL_EMALFDB;
@ -3826,12 +3823,10 @@ static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo,
}
}
yr_hash_table_destroy(compiler.rules_table, NULL);
yr_hash_table_destroy(compiler.objects_table, NULL);
// yr_arena_destroy(compiler.sz_arena);
// yr_arena_destroy(compiler.rules_arena);
yr_arena_append(engine->the_arena, compiler.sz_arena);
yr_arena_append(engine->the_arena, compiler.rules_arena);
yr_arena_append(engine->the_arena, compiler.strings_arena);
yr_arena_destroy(compiler.code_arena);
// yr_arena_destroy(compiler.strings_arena);
yr_arena_destroy(compiler.metas_arena);
if(rc)
@ -4285,6 +4280,7 @@ int cl_load(const char *path, struct cl_engine *engine, unsigned int *signo, uns
cli_errmsg("cl_load(%s): Not supported database file type\n", path);
return CL_EOPEN;
}
#ifdef YARA_PROTO
if (yara_total) {
cli_yaramsg("$$$$$$$$$$$$ YARA $$$$$$$$$$$$\n");
@ -4553,9 +4549,6 @@ int cl_engine_free(struct cl_engine *engine)
for(j = 0; j < root->ac_lsigs; j++) {
if (root->ac_lsigtable[j]->type == CLI_LSIG_NORMAL)
mpool_free(engine->mempool, root->ac_lsigtable[j]->u.logic);
else if (root->ac_lsigtable[j]->type == CLI_YARA_NORMAL ||
root->ac_lsigtable[j]->type == CLI_YARA_NORMAL)
free(root->ac_lsigtable[j]->u.code_start);
FREE_TDB(root->ac_lsigtable[j]->tdb);
mpool_free(engine->mempool, root->ac_lsigtable[j]);
}
@ -4667,6 +4660,16 @@ int cl_engine_free(struct cl_engine *engine)
#ifdef USE_MPOOL
if(engine->mempool) mpool_destroy(engine->mempool);
#endif
if (engine->rules_table)
yr_hash_table_destroy(engine->rules_table, NULL);
if (engine->objects_table)
yr_hash_table_destroy(engine->objects_table, NULL);
if (engine->the_arena)
yr_arena_destroy(engine->the_arena);
free(engine);
return CL_SUCCESS;
}
@ -4680,6 +4683,13 @@ int cl_engine_compile(struct cl_engine *engine)
if(!engine)
return CL_ENULLARG;
/* Free YARA hash tables - only needed for parse and load */
if (engine->rules_table)
yr_hash_table_destroy(engine->rules_table, NULL);
if (engine->objects_table)
yr_hash_table_destroy(engine->objects_table, NULL);
engine->rules_table = engine->objects_table = NULL;
if(!engine->ftypes)
if((ret = cli_loadftm(NULL, engine, 0, 1, NULL)))
return ret;

6
libclamav/yara_arena.c
Unescape View File

@ -31,8 +31,8 @@ from files.
#include <time.h>
#include <stdint.h>
#include "yara_clam.h"
#include <yara_arena.h>
#include "yara_clam.h"
#if REAL_YARA
#include <yara/mem.h>
#include <yara/error.h>
@ -750,7 +750,7 @@ int yr_arena_write_string(
(void**) written_string);
}
#if REAL_YARA
//
// yr_arena_append
//
@ -777,7 +777,7 @@ int yr_arena_append(
return ERROR_SUCCESS;
}
#endif
#if REAL_YARA
//

118
libclamav/yara_clam.h
Unescape View File

@ -41,9 +41,9 @@ limitations under the License.
#define _YARA_CLAM_H_
#include "shared/queue.h"
#include "others.h"
#include "yara_arena.h"
#include "yara_hash.h"
#include "others.h"
/* From libyara/include/yara/types.h */
#define DECLARE_REFERENCE(type, name) \
@ -313,95 +313,6 @@ typedef struct _SIZED_STRING
#define RE_FLAGS_DOT_ALL 0x80
#define RE_FLAGS_NOT_AT_START 0x100
/* From libyara/include/yara/exec.h */
#define UNDEFINED 0xFFFABADAFABADAFFLL
#define IS_UNDEFINED(x) ((x) == UNDEFINED)
#define OP_HALT 255
#define OP_AND 1
#define OP_OR 2
#define OP_XOR 3
#define OP_NOT 4
#define OP_LT 5
#define OP_GT 6
#define OP_LE 7
#define OP_GE 8
#define OP_EQ 9
#define OP_NEQ 10
#define OP_SZ_EQ 11
#define OP_SZ_NEQ 12
#define OP_SZ_TO_BOOL 13
#define OP_ADD 14
#define OP_SUB 15
#define OP_MUL 16
#define OP_DIV 17
#define OP_MOD 18
#define OP_NEG 19
#define OP_SHL 20
#define OP_SHR 21
#define OP_PUSH 22
#define OP_POP 23
#define OP_CALL 24
#define OP_OBJ_LOAD 25
#define OP_OBJ_VALUE 26
#define OP_OBJ_FIELD 27
#define OP_INDEX_ARRAY 28
#define OP_STR_COUNT 29
#define OP_STR_FOUND 30
#define OP_STR_FOUND_AT 31
#define OP_STR_FOUND_IN 32
#define OP_STR_OFFSET 33
#define OP_OF 34
#define OP_PUSH_RULE 35
#define OP_MATCH_RULE 36
#define OP_INCR_M 37
#define OP_CLEAR_M 38
#define OP_ADD_M 39
#define OP_POP_M 40
#define OP_PUSH_M 41
#define OP_SWAPUNDEF 42
#define OP_JNUNDEF 43
#define OP_JLE 44
#define OP_FILESIZE 45
#define OP_ENTRYPOINT 46
#define OP_INT8 47
#define OP_INT16 48
#define OP_INT32 49
#define OP_UINT8 50
#define OP_UINT16 51
#define OP_UINT32 52
#define OP_CONTAINS 53
#define OP_MATCHES 54
#define OP_IMPORT 55
/*
typedef struct _YR_MATCH
{
int64_t offset;
int32_t length;
union {
uint8_t* data; // Confirmed matches use "data",
int32_t chain_length; // unconfirmed ones use "chain_length"
};
struct _YR_MATCH* prev;
struct _YR_MATCH* next;
} YR_MATCH;
typedef struct _YR_MATCHES
{
int32_t count;
DECLARE_REFERENCE(YR_MATCH*, head);
DECLARE_REFERENCE(YR_MATCH*, tail);
} YR_MATCHES;
*/
typedef struct _YR_META
{
int32_t type;
@ -564,30 +475,31 @@ typedef struct _yc_compiler {
int last_error_line;
int last_result;
YR_ARENA* sz_arena;
YR_ARENA* rules_arena;
YR_ARENA* strings_arena;
YR_ARENA* code_arena;
YR_ARENA* metas_arena;
YR_HASH_TABLE* rules_table;
YR_HASH_TABLE* objects_table;
YR_NAMESPACE* current_namespace;
yc_string* current_rule_strings;
YR_ARENA *sz_arena;
YR_ARENA *rules_arena;
YR_ARENA *strings_arena;
YR_ARENA *code_arena;
YR_ARENA *metas_arena;
YR_ARENA *the_arena;
YR_HASH_TABLE *rules_table;
YR_HASH_TABLE *objects_table;
YR_NAMESPACE *current_namespace;
yc_string *current_rule_strings;
uint32_t current_rule_flags;
uint32_t current_rule_clflags;
int8_t* loop_address[MAX_LOOP_NESTING];
char* loop_identifier[MAX_LOOP_NESTING];
int8_t *loop_address[MAX_LOOP_NESTING];
char *loop_identifier[MAX_LOOP_NESTING];
int loop_depth;
int loop_for_of_mem_offset;
char last_error_extra_info[MAX_COMPILER_ERROR_EXTRA_INFO];
char lex_buf[LEX_BUF_SIZE];
char* lex_buf_ptr;
char *lex_buf_ptr;
unsigned short lex_buf_len;
char * error_msg;
char *error_msg;
STAILQ_HEAD(rq, _yc_rule) rule_q;
STAILQ_HEAD(cs, _yc_string) current_rule_string_q;

2
libclamav/yara_exec.c
Unescape View File

@ -165,7 +165,7 @@ int yr_execute_code(
while(1)
{
// cli_errmsg("yara_exec: executing %i\n", *ip);
cli_dbgmsg("yara_exec: executing %i\n", *ip);
switch(*ip)
{
case OP_HALT:

277
libclamav/yara_grammar.c
View File

File diff suppressed because it is too large Load Diff

2
libclamav/yara_grammar.h
Unescape View File

@ -151,7 +151,7 @@ typedef union YYSTYPE
{
/* Line 2068 of yacc.c */
#line 213 "yara_grammar.y"
#line 214 "yara_grammar.y"
SIZED_STRING* sized_string;
char* c_string;

1
libclamav/yara_grammar.y
Unescape View File

@ -62,6 +62,7 @@ limitations under the License.
#include "libclamav/yara_grammar.h"
#include "libclamav/yara_lexer.h"
#include "libclamav/yara_parser.h"
#include "libclamav/yara_exec.h"
#endif
#define YYERROR_VERBOSE

2
libclamav/yara_hash.c
Unescape View File

@ -17,8 +17,8 @@ limitations under the License.
#include <stdint.h>
#include <string.h>
#include "yara_clam.h"
#include <yara_hash.h>
#include "yara_clam.h"
#if REAL_YARA
#include <yara/mem.h>
#include <yara/error.h>

14
libclamav/yara_lexer.c
Unescape View File

@ -3039,15 +3039,11 @@ void yyerror(
#else
compiler->errors++;
if (error_message != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", error_message);
else if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->error_msg);
else if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->last_error_extra_info);
else
cli_errmsg("yara_lexer:yyerror() error unknown\n");
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
cli_errmsg("yara_lexer:yyerror() error message: %s\n", error_message);
if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() compiler error message: %s\n", compiler->error_msg);
if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() error extra info: %s\n", compiler->last_error_extra_info);
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
if (compiler->error_line != 0)

14
libclamav/yara_lexer.l
Unescape View File

@ -705,15 +705,11 @@ void yyerror(
#else
compiler->errors++;
if (error_message != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", error_message);
else if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->error_msg);
else if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->last_error_extra_info);
else
cli_errmsg("yara_lexer:yyerror() error unknown\n");
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
cli_errmsg("yara_lexer:yyerror() error message: %s\n", error_message);
if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() compiler error message: %s\n", compiler->error_msg);
if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() error extra info: %s\n", compiler->last_error_extra_info);
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
if (compiler->error_line != 0)

6
libclamav/yara_parser.c
Unescape View File

@ -55,6 +55,7 @@ limitations under the License.
#include "yara_clam.h"
#include "yara_grammar.h"
#include "yara_lexer.h"
#include "yara_exec.h"
#include "others.h"
#endif
@ -811,8 +812,9 @@ int yr_parser_reduce_rule_declaration(
//Yara condition code will work OK as long as it is less than 64K.
//FAIL_ON_COMPILER_ERROR(yr_arena_coalesce(compiler->code_arena));
rule->code_start = yr_arena_base_address(compiler->code_arena);
compiler->code_arena->page_list_head->address = NULL;
yr_arena_destroy(compiler->code_arena);
yr_arena_append(compiler->the_arena, compiler->code_arena);
// compiler->code_arena->page_list_head->address = NULL;
// yr_arena_destroy(compiler->code_arena);
FAIL_ON_COMPILER_ERROR(yr_arena_create(65536, 0, &compiler->code_arena));
STAILQ_INSERT_TAIL(&compiler->rule_q, rule, link);
#endif

Write Preview
Loading…
Cancel
Save