diff --git a/Makefile.am b/Makefile.am index 956e5b81..8c054f32 100644 --- a/Makefile.am +++ b/Makefile.am @@ -44,6 +44,8 @@ test_bitmask_SOURCES = tests/test-bitmask.c test_bitmask_LDADD = libyara/.libs/libyara.a test_math_SOURCES = tests/test-math.c tests/util.c test_math_LDADD = libyara/.libs/libyara.a +test_stack_SOURCES = tests/test-stack.c +test_stack_LDADD = libyara/.libs/libyara.a TESTS = $(check_PROGRAMS) TESTS_ENVIRONMENT = TOP_SRCDIR=$(top_srcdir) @@ -56,7 +58,8 @@ check_PROGRAMS = test-alignment \ test-elf \ test-version \ test-bitmask \ - test-math + test-math \ + test-stack if POSIX # The -fsanitize=address option makes test-exception fail. Include the test @@ -64,20 +67,20 @@ if POSIX if !ADDRESS_SANITIZER check_PROGRAMS+=test-exception test_exception_SOURCES = tests/test-exception.c tests/util.c -test_exception_LDADD = -Llibyara/.libs -lyara +test_exception_LDADD = libyara/.libs/libyara.a endif endif if MACHO_MODULE check_PROGRAMS+=test-macho test_macho_SOURCES = tests/test-macho.c tests/util.c -test_macho_LDADD = -Llibyara/.libs -lyara +test_macho_LDADD = libyara/.libs/libyara.a endif if DEX_MODULE check_PROGRAMS+=test-dex test_dex_SOURCES = tests/test-dex.c tests/util.c -test_dex_LDADD = -Llibyara/.libs -lyara +test_dex_LDADD = libyara/.libs/libyara.a endif # man pages diff --git a/libyara/Makefile.am b/libyara/Makefile.am index 4c86ec3f..d47b868a 100644 --- a/libyara/Makefile.am +++ b/libyara/Makefile.am @@ -89,6 +89,7 @@ yarainclude_HEADERS = \ include/yara/scan.h \ include/yara/scanner.h \ include/yara/sizedstr.h \ + include/yara/stack.h \ include/yara/stopwatch.h \ include/yara/stream.h \ include/yara/strutils.h \ @@ -152,6 +153,7 @@ libyara_la_SOURCES = \ scan.c \ scanner.c \ sizedstr.c \ + stack.c \ stopwatch.c \ strutils.c \ stream.c \ diff --git a/libyara/ahocorasick.c b/libyara/ahocorasick.c index d16326d0..d2dfc7d1 100644 --- a/libyara/ahocorasick.c +++ b/libyara/ahocorasick.c @@ -503,7 +503,7 @@ static int _yr_ac_find_suitable_transition_table_slot( automaton->bitmask = (YR_BITMASK*) yr_realloc( automaton->bitmask, b_bytes_size * 2); - + if (automaton->t_table == NULL || automaton->m_table == NULL || automaton->bitmask == NULL) @@ -602,7 +602,7 @@ static int _yr_ac_build_transition_table( automaton->bitmask = (YR_BITMASK*) yr_calloc( YR_BITMASK_SIZE(automaton->tables_size), sizeof(YR_BITMASK)); - if (automaton->t_table == NULL || + if (automaton->t_table == NULL || automaton->m_table == NULL || automaton->bitmask == NULL) { @@ -848,13 +848,13 @@ int yr_ac_add_string( { state = automaton->root; - for (i = 0; i < atom->atom_length; i++) + for (i = 0; i < atom->atom.length; i++) { - next_state = _yr_ac_next_state(state, atom->atom[i]); + next_state = _yr_ac_next_state(state, atom->atom.bytes[i]); if (next_state == NULL) { - next_state = _yr_ac_state_create(state, atom->atom[i]); + next_state = _yr_ac_state_create(state, atom->atom.bytes[i]); if (next_state == NULL) return ERROR_INSUFFICIENT_MEMORY; diff --git a/libyara/atoms.c b/libyara/atoms.c index 41aa2fcd..111e7543 100644 --- a/libyara/atoms.c +++ b/libyara/atoms.c @@ -81,20 +81,14 @@ will end up using the "Look" atom alone, but in /a(bcd|efg)h/ atoms "bcd" and #include #include +#include #include #include #include #include #include #include - - -#define append_current_leaf_to_node(node) \ - if (atom_tree->current_leaf != NULL) \ - { \ - _yr_atoms_tree_node_append(node, atom_tree->current_leaf); \ - atom_tree->current_leaf = NULL; \ - } \ +#include // @@ -109,8 +103,7 @@ will end up using the "Look" atom alone, but in /a(bcd|efg)h/ atoms "bcd" and // // Args: // YR_ATOMS_CONFIG* config - Pointer to YR_ATOMS_CONFIG struct. -// uint8_t* atom - Pointer to the atom's bytes. -// int atom_length - Atom's length. +// YR_ATOM* atom - Pointer to YR_ATOM struct. // // Returns: // An integer indicating the atom's quality @@ -118,54 +111,120 @@ will end up using the "Look" atom alone, but in /a(bcd|efg)h/ atoms "bcd" and int yr_atoms_heuristic_quality( YR_ATOMS_CONFIG* config, - uint8_t* atom, - int atom_length) + YR_ATOM* atom) { - int penalty = 0; + YR_BITMASK seen_bytes[YR_BITMASK_SIZE(256)]; + + int quality = 0; int unique_bytes = 0; - int i, j; - bool is_unique; + int masked_nibbles = 0; + int i; - for (i = 0; i < atom_length; i++) + assert(atom->length <= YR_MAX_ATOM_LENGTH); + + yr_bitmask_clear_all(seen_bytes); + + for (i = 0; i < atom->length; i++) { - if (atom[i] == 0x00 || atom[i] == 0xFF || atom[i] == 0x20 || - atom[i] == 0x0A || atom[i] == 0x0D) + switch (atom->mask[i]) { - // Penalize common bytes, specially if they are in the first two positions. - - switch(i) - { - case 0: - penalty += 3; - break; - case 1: - penalty += 2; - break; - default: - penalty += 1; - break; - } - } - - is_unique = true; - - for (j = i + 1; j < atom_length; j++) - if (atom[i] == atom[j]) - { - is_unique = false; + case 0x00: + masked_nibbles += 2; break; - } - - if (is_unique) - unique_bytes += 1; + case 0x0F: + masked_nibbles += 1; + quality += 4; + break; + case 0xF0: + masked_nibbles += 1; + quality += 4; + break; + case 0xFF: + switch (atom->bytes[i]) + { + case 0x00: + case 0x20: + case 0xCC: + case 0xFF: + // Common bytes contribute less to the quality than the rest. + quality += 15; + break; + default: + // Bytes in the a-z and A-Z ranges have a slightly lower quality + // than the rest. We want to favor atoms that contain bytes outside + // those ranges because they generate less additional atoms during + // calls to _yr_atoms_case_combinations. + if ( yr_lowercase[atom->bytes[i]] >= 'a' && + yr_lowercase[atom->bytes[i]] <= 'z') + quality += 19; + else + quality += 20; + }; + if (!yr_bitmask_isset(seen_bytes, atom->bytes[i])) + { + yr_bitmask_set(seen_bytes, atom->bytes[i]); + unique_bytes++; + } + } } - // yr_max(atom_length + unique_bytes - penalty, 0) is within the range - // [0 - 2 * YR_MAX_ATOM_LENGTH], which means that the function returns a value - // in [YR_MAX_ATOM_QUALITY - 2 * YR_MAX_ATOM_LENGTH, YR_MAX_ATOM_QUALITY] + // If all the bytes in the atom are equal and very common, let's penalize + // it heavily. - return YR_MAX_ATOM_QUALITY - 2 * YR_MAX_ATOM_LENGTH + - yr_max(atom_length + unique_bytes - penalty, 0); + if (unique_bytes == 1 && + (yr_bitmask_isset(seen_bytes, 0x00) || + yr_bitmask_isset(seen_bytes, 0x20) || + yr_bitmask_isset(seen_bytes, 0xCC) || + yr_bitmask_isset(seen_bytes, 0xFF))) + { + quality -= 10 * atom->length; + } + + quality -= masked_nibbles * 3; + + return YR_MAX_ATOM_QUALITY - 20 * YR_MAX_ATOM_LENGTH + quality; +} + + +// +// _yr_atoms_cmp +// +// Compares the byte sequence in a1 with the YR_ATOM in a2, taking atom's mask +// into account. +// +// Returns: +// < 0 if the first byte that does not match has a lower value in a1 than +// in a2. +// > 0 if the first byte that does not match has a greater value in a1 than +// in a2. +// = 0 if a1 is equal or matches a2. +// + +static int _yr_atoms_cmp( + const uint8_t* a1, + YR_ATOM* a2) +{ + int result = 0; + int i = 0; + + while (result == 0 && i < a2->length) + { + switch (a2->mask[i]) + { + case 0xFF: + case 0x0F: + case 0xF0: + case 0x00: + result = (a1[i] & a2->mask[i]) - a2->bytes[i]; + break; + default: + assert(false); + } + + i++; + } + + return result; } // @@ -179,8 +238,7 @@ int yr_atoms_heuristic_quality( // // Args: // YR_ATOMS_CONFIG* config - Pointer to YR_ATOMS_CONFIG struct. -// uint8_t* atom - Pointer to the atom's bytes. -// int atom_length - Atom's length. +// YR_ATOM* atom - Pointer to YR_ATOM struct. // // Returns: // An integer indicating the atom's quality @@ -188,18 +246,19 @@ int yr_atoms_heuristic_quality( int yr_atoms_table_quality( YR_ATOMS_CONFIG* config, - uint8_t* atom, - int atom_length) + YR_ATOM* atom) { YR_ATOM_QUALITY_TABLE_ENTRY* table = config->quality_table; int begin = 0; int end = config->quality_table_entries; + assert(atom->length <= YR_MAX_ATOM_LENGTH); + while (end > begin) { int middle = begin + (end - begin) / 2; - int c = memcmp(table[middle].atom, atom, atom_length); + int c = _yr_atoms_cmp(table[middle].atom, atom); if (c < 0) { @@ -215,10 +274,7 @@ int yr_atoms_table_quality( int quality = table[middle].quality; int min_quality = quality; - if (atom_length == YR_MAX_ATOM_LENGTH) - return table[middle].quality; - - while (i < end && memcmp(table[i].atom, atom, atom_length) == 0) + while (i < end && _yr_atoms_cmp(table[i].atom, atom) == 0) { if (min_quality > table[i].quality) min_quality = table[i].quality; @@ -228,7 +284,7 @@ int yr_atoms_table_quality( i = middle - 1; - while (i >= begin && memcmp(table[i].atom, atom, atom_length) == 0) + while (i >= begin && _yr_atoms_cmp(table[i].atom, atom) == 0) { if (min_quality > table[i].quality) min_quality = table[i].quality; @@ -236,7 +292,7 @@ int yr_atoms_table_quality( i--; } - return min_quality >> (YR_MAX_ATOM_LENGTH - atom_length); + return min_quality >> (YR_MAX_ATOM_LENGTH - atom->length); } } @@ -266,7 +322,7 @@ int yr_atoms_min_quality( while (atom != NULL) { - quality = config->get_atom_quality(config, atom->atom, atom->atom_length); + quality = config->get_atom_quality(config, &atom->atom); if (quality < min_quality) min_quality = quality; @@ -284,21 +340,19 @@ int yr_atoms_min_quality( // Creates a new node for an atoms tree. // -static ATOM_TREE_NODE* _yr_atoms_tree_node_create( +static YR_ATOM_TREE_NODE* _yr_atoms_tree_node_create( uint8_t type) { - ATOM_TREE_NODE* new_node = (ATOM_TREE_NODE*) \ - yr_malloc(sizeof(ATOM_TREE_NODE)); + YR_ATOM_TREE_NODE* new_node = (YR_ATOM_TREE_NODE*) \ + yr_malloc(sizeof(YR_ATOM_TREE_NODE)); if (new_node != NULL) { new_node->type = type; - new_node->atom_length = 0; + new_node->atom.length = 0; new_node->next_sibling = NULL; new_node->children_head = NULL; new_node->children_tail = NULL; - new_node->forward_code = NULL; - new_node->backward_code = NULL; } return new_node; @@ -312,10 +366,10 @@ static ATOM_TREE_NODE* _yr_atoms_tree_node_create( // static void _yr_atoms_tree_node_destroy( - ATOM_TREE_NODE* node) + YR_ATOM_TREE_NODE* node) { - ATOM_TREE_NODE* child; - ATOM_TREE_NODE* next_child; + YR_ATOM_TREE_NODE* child; + YR_ATOM_TREE_NODE* next_child; if (node == NULL) return; @@ -343,8 +397,8 @@ static void _yr_atoms_tree_node_destroy( // static void _yr_atoms_tree_node_append( - ATOM_TREE_NODE* dest, - ATOM_TREE_NODE* node) + YR_ATOM_TREE_NODE* dest, + YR_ATOM_TREE_NODE* node) { if (dest->children_head == NULL) dest->children_head = node; @@ -363,7 +417,7 @@ static void _yr_atoms_tree_node_append( // static void _yr_atoms_tree_destroy( - ATOM_TREE* atom_tree) + YR_ATOM_TREE* atom_tree) { _yr_atoms_tree_node_destroy(atom_tree->root_node); yr_free(atom_tree); @@ -418,28 +472,104 @@ static YR_ATOM_LIST_ITEM* _yr_atoms_list_concat( } +// +// _yr_atoms_trim +// +// If the atom starts or ends with an unknown byte (mask == 0x00), trim +// those bytes out of the atom. We don't want to expand an atom like +// { ?? 01 02 } into { 00 01 02 }, { 01 01 02}, { 02 01 02} .. { FF 01 02} +// in those cases it's better to simply have a shorter atom { 01 02 }. +// +// Args: +// atom - Pointer to the YR_ATOM to be trimmed. +// +// Returns: +// The number of bytes that were trimmed from the beginning of the atom. +// + +int _yr_atoms_trim( + YR_ATOM* atom) +{ + int mask_00 = 0; + int mask_ff = 0; + + int i, trim_left = 0; + + while (trim_left < atom->length && atom->mask[trim_left] == 0) + trim_left++; + + while (atom->length > trim_left && atom->mask[atom->length - 1] == 0) + atom->length--; + + atom->length -= trim_left; + + if (atom->length == 0) + return 0; + + // At this point the actual atom goes from i to i + atom->length and the + // first and last byte in the atom are known (mask == 0xFF). Now count the + // number of known and unknown bytes in the atom (mask == 0xFF and + // mask == 0x00 respectively). + + for (i = 0; i < atom->length; i++) + { + if (atom->mask[trim_left + i] == 0xFF) + mask_ff++; + else if (atom->mask[trim_left + i] == 0x00) + mask_00++; + } + + // If the number of unknown bytes is >= than the number of known bytes + // it doesn't make sense the to use this atom, so we use the a single byte + // atom with the first known byte. If YR_MAX_ATOM_LENGTH == 4 this happens + // only when the atom is like { XX ?? ?? YY }, so using the first known + // atom is good enough. For larger values of YR_MAX_ATOM_LENGTH this is not + // the most efficient solution, as better atoms could be choosen. For + // example, in { XX ?? ?? ?? YY ZZ } the best atom is { YY ZZ } not { XX }. + // But let's keep it like this for simplicity. + + if (mask_00 >= mask_ff) + atom->length = 1; + + if (trim_left == 0) + return 0; + + // Shift bytes and mask trim_left positions to the left. + + for (i = 0; i < YR_MAX_ATOM_LENGTH - trim_left; i++) + { + atom->bytes[i] = atom->bytes[trim_left + i]; + atom->mask[i] = atom->mask[trim_left + i]; + } + + return trim_left; +} + + // // _yr_atoms_choose // -// Chooses which atoms from an atoms tree will be used to feed the -// Aho-Corasick automaton, and puts them in a list. +// This function receives an atom tree and returns a list of atoms to be added +// to the Aho-Corasick automaton. // static int _yr_atoms_choose( YR_ATOMS_CONFIG* config, - ATOM_TREE_NODE* node, + YR_ATOM_TREE_NODE* node, YR_ATOM_LIST_ITEM** chosen_atoms, int* atoms_quality) { - ATOM_TREE_NODE* child; + YR_ATOM_TREE_NODE* child; YR_ATOM_LIST_ITEM* item; YR_ATOM_LIST_ITEM* tail; - int i, quality; + int shift, quality; + int max_quality = YR_MIN_ATOM_QUALITY; int min_quality = YR_MAX_ATOM_QUALITY; *chosen_atoms = NULL; + *atoms_quality = YR_MIN_ATOM_QUALITY; switch (node->type) { @@ -450,22 +580,26 @@ static int _yr_atoms_choose( if (item == NULL) return ERROR_INSUFFICIENT_MEMORY; - for (i = 0; i < node->atom_length; i++) - item->atom[i] = node->atom[i]; + memcpy(&item->atom, &node->atom, sizeof(YR_ATOM)); - item->atom_length = node->atom_length; - item->forward_code = node->forward_code; - item->backward_code = node->backward_code; + shift = _yr_atoms_trim(&item->atom); + + if (item->atom.length == 0) + break; + + item->forward_code = node->re_nodes[shift]->forward_code; + item->backward_code = node->re_nodes[shift]->backward_code; item->backtrack = 0; item->next = NULL; *chosen_atoms = item; - *atoms_quality = config->get_atom_quality( - config, node->atom, node->atom_length); + *atoms_quality = config->get_atom_quality(config, &item->atom); break; case ATOM_TREE_OR: + // The choosen nodes are those coming from the highest quality child. + child = node->children_head; while (child != NULL) @@ -494,6 +628,9 @@ static int _yr_atoms_choose( case ATOM_TREE_AND: + // The choosen nodes are the concatenation of the the nodes choosen from + // all the children. + child = node->children_head; while (child != NULL) @@ -624,8 +761,8 @@ static int _yr_atoms_case_insensitive( while (atom != NULL) { _yr_atoms_case_combinations( - atom->atom, - atom->atom_length, + atom->atom.bytes, + atom->atom.length, 0, buffer); @@ -641,9 +778,12 @@ static int _yr_atoms_case_insensitive( return ERROR_INSUFFICIENT_MEMORY; for (i = 0; i < atom_length; i++) - new_atom->atom[i] = atoms_cursor[i]; + { + new_atom->atom.bytes[i] = atoms_cursor[i]; + new_atom->atom.mask[i] = 0xFF; + } - new_atom->atom_length = atom_length; + new_atom->atom.length = atom_length; new_atom->forward_code = atom->forward_code; new_atom->backward_code = atom->backward_code; new_atom->backtrack = atom->backtrack; @@ -669,6 +809,7 @@ static int _yr_atoms_case_insensitive( // For a given list of atoms returns another list after a single byte xor // has been applied to it. // + static int _yr_atoms_xor( YR_ATOM_LIST_ITEM* atoms, YR_ATOM_LIST_ITEM** xor_atoms) @@ -689,10 +830,13 @@ static int _yr_atoms_xor( if (new_atom == NULL) return ERROR_INSUFFICIENT_MEMORY; - for (i = 0; i < atom->atom_length; i++) - new_atom->atom[i] = atom->atom[i] ^ j; + for (i = 0; i < atom->atom.length; i++) + { + new_atom->atom.bytes[i] = atom->atom.bytes[i] ^ j; + new_atom->atom.mask[i] = 0xFF; + } - new_atom->atom_length = yr_min(atom->atom_length, YR_MAX_ATOM_LENGTH); + new_atom->atom.length = yr_min(atom->atom.length, YR_MAX_ATOM_LENGTH); new_atom->forward_code = atom->forward_code; new_atom->backward_code = atom->backward_code; new_atom->backtrack = atom->backtrack; @@ -705,6 +849,8 @@ static int _yr_atoms_xor( } return ERROR_SUCCESS; } + + // // _yr_atoms_wide // @@ -733,17 +879,20 @@ static int _yr_atoms_wide( return ERROR_INSUFFICIENT_MEMORY; for (i = 0; i < YR_MAX_ATOM_LENGTH; i++) - new_atom->atom[i] = 0; + { + new_atom->atom.bytes[i] = 0; + new_atom->atom.mask[i] = 0xFF; + } - for (i = 0; i < atom->atom_length; i++) + for (i = 0; i < atom->atom.length; i++) { if (i * 2 < YR_MAX_ATOM_LENGTH) - new_atom->atom[i * 2] = atom->atom[i]; + new_atom->atom.bytes[i * 2] = atom->atom.bytes[i]; else break; } - new_atom->atom_length = yr_min(atom->atom_length * 2, YR_MAX_ATOM_LENGTH); + new_atom->atom.length = yr_min(atom->atom.length * 2, YR_MAX_ATOM_LENGTH); new_atom->forward_code = atom->forward_code; new_atom->backward_code = atom->backward_code; new_atom->backtrack = atom->backtrack * 2; @@ -758,421 +907,463 @@ static int _yr_atoms_wide( } -// -// _yr_atoms_extract_from_re_node -// -// Extract atoms from a regular expression node. See description for -// _yr_atoms_extract_from_re for more details. -// - -static ATOM_TREE_NODE* _yr_atoms_extract_from_re_node( - YR_ATOMS_CONFIG* config, - RE_NODE* re_node, - ATOM_TREE* atom_tree, - ATOM_TREE_NODE* current_node) +struct STACK_ITEM { - ATOM_TREE_NODE* left_node; - ATOM_TREE_NODE* right_node; - ATOM_TREE_NODE* and_node; - ATOM_TREE_NODE* current_leaf; - ATOM_TREE_NODE* temp; - - int quality; - int new_quality; - int i; - - uint8_t new_atom[YR_MAX_ATOM_LENGTH]; - - switch(re_node->type) - { - case RE_NODE_LITERAL: - - if (atom_tree->current_leaf == NULL) - { - atom_tree->current_leaf = _yr_atoms_tree_node_create(ATOM_TREE_LEAF); - - if (atom_tree->current_leaf == NULL) - return NULL; - - atom_tree->current_leaf->forward_code = re_node->forward_code; - atom_tree->current_leaf->backward_code = re_node->backward_code; - - assert(atom_tree->current_leaf->forward_code != NULL); - assert(atom_tree->current_leaf->backward_code != NULL); - } - - current_leaf = atom_tree->current_leaf; - - if (current_leaf->atom_length < YR_MAX_ATOM_LENGTH) - { - current_leaf->atom[current_leaf->atom_length] = - (uint8_t) re_node->value; - current_leaf->recent_nodes[current_leaf->atom_length] = re_node; - current_leaf->atom_length++; - } - else - { - quality = config->get_atom_quality( - config, current_leaf->atom, YR_MAX_ATOM_LENGTH); - - if (quality < YR_MAX_ATOM_QUALITY) - { - for (i = 1; i < YR_MAX_ATOM_LENGTH; i++) - current_leaf->recent_nodes[i - 1] = current_leaf->recent_nodes[i]; - - current_leaf->recent_nodes[YR_MAX_ATOM_LENGTH - 1] = re_node; - - for (i = 0; i < YR_MAX_ATOM_LENGTH; i++) - new_atom[i] = (uint8_t) current_leaf->recent_nodes[i]->value; - - new_quality = config->get_atom_quality( - config, new_atom, YR_MAX_ATOM_LENGTH); - - if (new_quality > quality) - { - for (i = 0; i < YR_MAX_ATOM_LENGTH; i++) - current_leaf->atom[i] = new_atom[i]; - - current_leaf->forward_code = \ - current_leaf->recent_nodes[0]->forward_code; - - current_leaf->backward_code = \ - current_leaf->recent_nodes[0]->backward_code; - - assert(current_leaf->forward_code != NULL); - assert(current_leaf->backward_code != NULL); - } - } - } - - return current_node; - - case RE_NODE_CONCAT: - - current_node = _yr_atoms_extract_from_re_node( - config, re_node->left, atom_tree, current_node); - - if (current_node == NULL) - return NULL; - - current_node = _yr_atoms_extract_from_re_node( - config, re_node->right, atom_tree, current_node); - - return current_node; - - case RE_NODE_ALT: - - append_current_leaf_to_node(current_node); - - left_node = _yr_atoms_tree_node_create(ATOM_TREE_OR); - - if (left_node == NULL) - return NULL; - - left_node = _yr_atoms_extract_from_re_node( - config, re_node->left, atom_tree, left_node); - - if (left_node == NULL) - return NULL; - - append_current_leaf_to_node(left_node); - - if (left_node->children_head == NULL) - { - _yr_atoms_tree_node_destroy(left_node); - return current_node; - } - - if (left_node->children_head == left_node->children_tail) - { - temp = left_node; - left_node = left_node->children_head; - yr_free(temp); - } - - right_node = _yr_atoms_tree_node_create(ATOM_TREE_OR); - - if (right_node == NULL) - return NULL; - - right_node = _yr_atoms_extract_from_re_node( - config, re_node->right, atom_tree, right_node); - - if (right_node == NULL) - return NULL; - - append_current_leaf_to_node(right_node); - - if (right_node->children_head == NULL) - { - _yr_atoms_tree_node_destroy(left_node); - _yr_atoms_tree_node_destroy(right_node); - return current_node; - } - - if (right_node->children_head == right_node->children_tail) - { - temp = right_node; - right_node = right_node->children_head; - yr_free(temp); - } - - and_node = _yr_atoms_tree_node_create(ATOM_TREE_AND); - - if (and_node == NULL) - return NULL; - - and_node->children_head = left_node; - and_node->children_tail = right_node; - left_node->next_sibling = right_node; - - _yr_atoms_tree_node_append(current_node, and_node); - - return current_node; - - case RE_NODE_RANGE: - - if (re_node->start == 0) - append_current_leaf_to_node(current_node); - - // In a regexp like /a{10,20}/ the optimal atom is 'aaaa' (assuming that - // YR_MAX_ATOM_LENGTH = 4) because the 'a' character must appear at least - // 10 times in the matching string. Each call in the loop will append - // one 'a' to the atom, so YR_MAX_ATOM_LENGTH iterations are enough. - - for (i = 0; i < yr_min(re_node->start, YR_MAX_ATOM_LENGTH); i++) - { - current_node = _yr_atoms_extract_from_re_node( - config, re_node->left, atom_tree, current_node); - - if (current_node == NULL) - return NULL; - } - - if (re_node->start != re_node->end) - append_current_leaf_to_node(current_node); - - return current_node; - - case RE_NODE_PLUS: - - current_node = _yr_atoms_extract_from_re_node( - config, re_node->left, atom_tree, current_node); - - if (current_node == NULL) - return NULL; - - append_current_leaf_to_node(current_node); - return current_node; - - case RE_NODE_ANY: - case RE_NODE_RANGE_ANY: - case RE_NODE_STAR: - case RE_NODE_CLASS: - case RE_NODE_MASKED_LITERAL: - case RE_NODE_WORD_CHAR: - case RE_NODE_NON_WORD_CHAR: - case RE_NODE_SPACE: - case RE_NODE_NON_SPACE: - case RE_NODE_DIGIT: - case RE_NODE_NON_DIGIT: - case RE_NODE_EMPTY: - case RE_NODE_ANCHOR_START: - case RE_NODE_ANCHOR_END: - case RE_NODE_WORD_BOUNDARY: - case RE_NODE_NON_WORD_BOUNDARY: - - append_current_leaf_to_node(current_node); - return current_node; - - default: - assert(false); - } - - return NULL; -} - -// -// yr_atoms_extract_triplets -// -// On certain cases YARA can not extract long enough atoms from a regexp, but -// can infer them. For example, in the hex string { 01 ?? 02 } the only explicit -// atoms are 01 and 02, and both of them are too short to be efficiently used. -// However YARA can use simultaneously atoms 01 00 02, 01 01 02, 01 02 02, -// 01 03 02, and so on up to 01 FF 02. Searching for 256 three-bytes atoms is -// faster than searching for a single one-byte atom. -// -// This function extracts these three-bytes atoms from a regexp node if -// possible. -// - -int yr_atoms_extract_triplets( - RE_NODE* re_node, - YR_ATOM_LIST_ITEM** atoms) - { - RE_NODE* left_child; - RE_NODE* left_grand_child; - - int i; - int shift; - - *atoms = NULL; - - if (re_node->type == RE_NODE_CONCAT) - left_child = re_node->left; - else - return ERROR_SUCCESS; - - if (left_child->type == RE_NODE_CONCAT) - left_grand_child = left_child->left; - else - return ERROR_SUCCESS; - - if (re_node->right->type != RE_NODE_LITERAL) - return yr_atoms_extract_triplets(left_child, atoms); - - if (left_child->left->type == RE_NODE_LITERAL && - (left_child->right->type == RE_NODE_ANY)) - { - for (i = 0; i < 256; i++) - { - YR_ATOM_LIST_ITEM* atom = (YR_ATOM_LIST_ITEM*) - yr_malloc(sizeof(YR_ATOM_LIST_ITEM)); - - if (atom == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - atom->atom[0] = (uint8_t) left_child->left->value; - atom->atom[1] = (uint8_t) i; - atom->atom[2] = (uint8_t) re_node->right->value; - - atom->atom_length = 3; - atom->forward_code = left_child->left->forward_code; - atom->backward_code = left_child->left->backward_code; - atom->backtrack = 0; - atom->next = *atoms; - - *atoms = atom; - } - - return ERROR_SUCCESS; + RE_NODE* re_node; + YR_ATOM_TREE_NODE* new_appending_node; +}; + + +#define make_atom_from_re_nodes(atom, nodes_length, nodes) \ + { \ + atom.length = nodes_length; \ + for (i = 0; i < atom.length; i++) \ + { \ + atom.bytes[i] = (uint8_t) (recent_re_nodes)[i]->value; \ + atom.mask[i] = (uint8_t) (recent_re_nodes)[i]->mask; \ + } \ } - if (left_child->left->type == RE_NODE_LITERAL && - (left_child->right->type == RE_NODE_MASKED_LITERAL)) - { - for (i = 0; i < 16; i++) - { - YR_ATOM_LIST_ITEM* atom = (YR_ATOM_LIST_ITEM*) - yr_malloc(sizeof(YR_ATOM_LIST_ITEM)); - - if (atom == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - if (left_child->right->mask == 0xF0) - shift = 0; - else - shift = 4; - - atom->atom[0] = (uint8_t) left_child->left->value; - atom->atom[1] = (uint8_t)(left_child->right->value | (i << shift)); - atom->atom[2] = (uint8_t) re_node->right->value; - - atom->atom_length = 3; - atom->forward_code = left_child->left->forward_code; - atom->backward_code = left_child->left->backward_code; - atom->backtrack = 0; - atom->next = *atoms; - - *atoms = atom; - } - - return ERROR_SUCCESS; - } - - if (left_grand_child->type == RE_NODE_CONCAT && - left_grand_child->right->type == RE_NODE_LITERAL && - (left_child->right->type == RE_NODE_ANY)) - { - for (i = 0; i < 256; i++) - { - YR_ATOM_LIST_ITEM* atom = (YR_ATOM_LIST_ITEM*) - yr_malloc(sizeof(YR_ATOM_LIST_ITEM)); - - if (atom == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - atom->atom[0] = (uint8_t) left_grand_child->right->value; - atom->atom[1] = (uint8_t) i; - atom->atom[2] = (uint8_t) re_node->right->value; - - atom->atom_length = 3; - atom->forward_code = left_grand_child->right->forward_code; - atom->backward_code = left_grand_child->right->backward_code; - atom->backtrack = 0; - atom->next = *atoms; - - *atoms = atom; - } - - return ERROR_SUCCESS; - } - - if (left_grand_child->type == RE_NODE_CONCAT && - left_grand_child->right->type == RE_NODE_LITERAL && - (left_child->right->type == RE_NODE_MASKED_LITERAL)) - { - for (i = 0; i < 16; i++) - { - YR_ATOM_LIST_ITEM* atom = (YR_ATOM_LIST_ITEM*) - yr_malloc(sizeof(YR_ATOM_LIST_ITEM)); - - if (atom == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - if (left_child->right->mask == 0xF0) - shift = 0; - else - shift = 4; - - atom->atom[0] = (uint8_t) left_grand_child->right->value; - atom->atom[1] = (uint8_t)(left_child->right->value | (i << shift)); - atom->atom[2] = (uint8_t) re_node->right->value; - - atom->atom_length = 3; - atom->forward_code = left_grand_child->right->forward_code; - atom->backward_code = left_grand_child->right->backward_code; - atom->backtrack = 0; - atom->next = *atoms; - - *atoms = atom; - } - - return ERROR_SUCCESS; - } - - return yr_atoms_extract_triplets(left_child, atoms);; - } // // _yr_atoms_extract_from_re // -// Extract atoms from a regular expression. +// Extract atoms from a regular expression. This is a helper function used by +// yr_atoms_extract_from_re that receives the abstract syntax tree for a regexp +// (or hex pattern) and builds an atom tree. The appending_node argument is a +// pointer to the ATOM_TREE_OR node at the root of the atom tree. This function +// creates the tree by appending new nodes to it. +// + +static int _yr_atoms_extract_from_re( + YR_ATOMS_CONFIG* config, + RE_AST* re_ast, + YR_ATOM_TREE_NODE* appending_node) +{ + YR_STACK* stack; + RE_NODE* re_node; + + YR_ATOM atom; + YR_ATOM best_atom; + + struct STACK_ITEM si; + + int i, shift; + int quality; + int best_quality = -1; + int n = 0; + + YR_ATOM_TREE_NODE* and_node; + YR_ATOM_TREE_NODE* left_node; + YR_ATOM_TREE_NODE* right_node; + + // The RE_NODEs most recently visited that can conform an atom (ie: + // RE_NODE_LITERAL, RE_NODE_MASKED_LITERAL and RE_NODE_ANY). The number of + // items in this array is n. + RE_NODE* recent_re_nodes[YR_MAX_ATOM_LENGTH]; + + // The RE_NODEs corresponding to the best atom found so far for the current + // appending node. + RE_NODE* best_atom_re_nodes[YR_MAX_ATOM_LENGTH]; + + // This holds the ATOM_TREE_OR node where leaves (ATOM_TREE_LEAF) are + // currently being appended. + YR_ATOM_TREE_NODE* current_appending_node = NULL; + + // This holds the ATOM_TREE_LEAF node whose atom is currently being updated. + YR_ATOM_TREE_NODE* leaf = NULL; + + FAIL_ON_ERROR(yr_stack_create(1024, sizeof(si), &stack)); + + // This first item pushed in the stack is the last one to be poped out, its + // sole purpose is forcing that any pending + si.re_node = NULL; + si.new_appending_node = appending_node; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, (void*) &si), + yr_stack_destroy(stack)); + + // Start processing the root node. + si.re_node = re_ast->root_node; + + // Leaf nodes are initially appended to the node passed in the appending_node, + // argument which is the root ATOM_TREE_OR node that is empty at this point. + si.new_appending_node = appending_node; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, (void*) &si), + yr_stack_destroy(stack)); + + while (yr_stack_pop(stack, (void*) &si)) + { + // Change the appending node if the item poped from the stack says so. + if (si.new_appending_node != NULL) + { + // Before changing the appending node let's append any pending leaf to + // the current appending node. + if (n > 0) + { + make_atom_from_re_nodes(atom, n, recent_re_nodes); + shift = _yr_atoms_trim(&atom); + quality = config->get_atom_quality(config, &atom); + + FAIL_ON_NULL_WITH_CLEANUP( + leaf = _yr_atoms_tree_node_create(ATOM_TREE_LEAF), + yr_stack_destroy(stack)); + + if (quality > best_quality) + { + memcpy(&leaf->atom, &atom, sizeof(atom)); + memcpy( + &leaf->re_nodes, + &recent_re_nodes[shift], + sizeof(recent_re_nodes) - shift * sizeof(recent_re_nodes[0])); + } + else + { + memcpy(&leaf->atom, &best_atom, sizeof(best_atom)); + memcpy( + &leaf->re_nodes, + &best_atom_re_nodes, + sizeof(best_atom_re_nodes)); + } + + _yr_atoms_tree_node_append(current_appending_node, leaf); + n = 0; + } + + current_appending_node = si.new_appending_node; + } + + if (si.re_node != NULL) + { + switch(si.re_node->type) + { + case RE_NODE_LITERAL: + case RE_NODE_MASKED_LITERAL: + case RE_NODE_ANY: + + if (n < YR_MAX_ATOM_LENGTH) + { + recent_re_nodes[n] = si.re_node; + best_atom_re_nodes[n] = si.re_node; + best_atom.bytes[n] = (uint8_t) si.re_node->value; + best_atom.mask[n] = (uint8_t) si.re_node->mask; + best_atom.length = ++n; + } + else if (best_quality < YR_MAX_ATOM_QUALITY) + { + make_atom_from_re_nodes(atom, n, recent_re_nodes); + shift = _yr_atoms_trim(&atom); + quality = config->get_atom_quality(config, &atom); + + if (quality > best_quality) + { + for (i = 0; i < atom.length; i++) + { + best_atom.bytes[i] = atom.bytes[i]; + best_atom.mask[i] = atom.mask[i]; + best_atom_re_nodes[i] = recent_re_nodes[i + shift]; + } + + best_quality = quality; + } + + for (i = 1; i < YR_MAX_ATOM_LENGTH; i++) + recent_re_nodes[i - 1] = recent_re_nodes[i]; + + recent_re_nodes[YR_MAX_ATOM_LENGTH - 1] = si.re_node; + } + + break; + + case RE_NODE_CONCAT: + + re_node = si.re_node; + si.new_appending_node = NULL; + si.re_node = re_node->right; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + si.re_node = re_node->left; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + break; + + case RE_NODE_ALT: + + // Create ATOM_TREE_AND node with two ATOM_TREE_OR children nodes. + and_node = _yr_atoms_tree_node_create(ATOM_TREE_AND); + left_node = _yr_atoms_tree_node_create(ATOM_TREE_OR); + right_node = _yr_atoms_tree_node_create(ATOM_TREE_OR); + + if (and_node == NULL || left_node == NULL || right_node == NULL) + { + _yr_atoms_tree_node_destroy(and_node); + _yr_atoms_tree_node_destroy(left_node); + _yr_atoms_tree_node_destroy(right_node); + + yr_stack_destroy(stack); + + return ERROR_INSUFFICIENT_MEMORY; + } + + and_node->children_head = left_node; + and_node->children_tail = right_node; + left_node->next_sibling = right_node; + + // Add the ATOM_TREE_AND as children of the current node. + _yr_atoms_tree_node_append(current_appending_node, and_node); + + re_node = si.re_node; + + si.new_appending_node = current_appending_node; + si.re_node = NULL; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + si.new_appending_node = right_node; + si.re_node = re_node->right; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + si.new_appending_node = left_node; + si.re_node = re_node->left; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + break; + + case RE_NODE_PLUS: + + re_node = si.re_node; + + si.new_appending_node = current_appending_node; + si.re_node = NULL; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + si.new_appending_node = NULL; + si.re_node = re_node->left; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + break; + + case RE_NODE_RANGE: + + re_node = si.re_node; + + si.new_appending_node = current_appending_node; + si.re_node = NULL; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + si.new_appending_node = NULL; + si.re_node = re_node->left; + + // In a regexp like /a{10,20}/ the optimal atom is 'aaaa' (assuming + // that YR_MAX_ATOM_LENGTH = 4) because the 'a' character must appear + // at least 10 times in the matching string. Each call in the loop + // will append one 'a' to the atom, so YR_MAX_ATOM_LENGTH iterations + // are enough. + + for (i = 0; i < yr_min(re_node->start, YR_MAX_ATOM_LENGTH); i++) + { + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + } + + break; + + case RE_NODE_RANGE_ANY: + case RE_NODE_STAR: + case RE_NODE_CLASS: + case RE_NODE_WORD_CHAR: + case RE_NODE_NON_WORD_CHAR: + case RE_NODE_SPACE: + case RE_NODE_NON_SPACE: + case RE_NODE_DIGIT: + case RE_NODE_NON_DIGIT: + case RE_NODE_EMPTY: + case RE_NODE_ANCHOR_START: + case RE_NODE_ANCHOR_END: + case RE_NODE_WORD_BOUNDARY: + case RE_NODE_NON_WORD_BOUNDARY: + + si.new_appending_node = current_appending_node; + si.re_node = NULL; + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_stack_push(stack, &si), + yr_stack_destroy(stack)); + + break; + + default: + assert(false); + } + } + } + + yr_stack_destroy(stack); + + return ERROR_SUCCESS; +} + + + +// +// _yr_atoms_clone_list_item +// +// Makes an exact copy of an YR_ATOM_LIST_ITEM. +// + +static YR_ATOM_LIST_ITEM* _yr_atoms_clone_list_item( + YR_ATOM_LIST_ITEM* item) +{ + YR_ATOM_LIST_ITEM* clone = (YR_ATOM_LIST_ITEM*) yr_malloc( + sizeof(YR_ATOM_LIST_ITEM)); + + if (clone == NULL) + return NULL; + + memcpy(clone, item, sizeof(YR_ATOM_LIST_ITEM)); + + return clone; +} + + +// +// _yr_atoms_expand_wildcards +// +// Given list of atoms that may contain wildcards, replace those wildcarded +// atoms with a list of non-wildcarded atoms covering all the combinations +// allowed by the wilcarded atom. For example, the atom {01 ?2 03} will be +// replaced by {01 02 03}, {01 12 03}, {01 22 03} .. {01 F2 03}. The list +// is modified in-place. +// +// Args: +// YR_ATOM_LIST_ITEM* atoms - Pointer to first element of the list. +// +// Returns: +// ERROR_SUCCESS or ERROR_INSUFFICIENT_MEMORY. +// + +static int _yr_atoms_expand_wildcards( + YR_ATOM_LIST_ITEM* atoms) +{ + int i; + + YR_ATOM_LIST_ITEM* atom = atoms; + YR_ATOM_LIST_ITEM* new_atom; + YR_ATOM_LIST_ITEM* prev_atom; + YR_ATOM_LIST_ITEM* next_atom; + + while (atom != NULL) + { + bool expanded = false; + + for (i = 0; i < atom->atom.length; i++) + { + uint16_t a, s, e, incr = 1; + + switch(atom->atom.mask[i]) + { + case 0x00: + expanded = true; + s = 0x00; + e = 0xFF; + break; + + case 0x0F: + expanded = true; + s = atom->atom.bytes[i]; + e = atom->atom.bytes[i] | 0xF0; + incr = 0x10; + break; + + case 0xF0: + expanded = true; + s = atom->atom.bytes[i]; + e = atom->atom.bytes[i] | 0x0F; + break; + + default: + s = 0; + e = 0; + } + + if (s != e) + { + atom->atom.bytes[i] = (uint8_t) s; + atom->atom.mask[i] = 0xFF; + } + + prev_atom = atom; + next_atom = atom->next; + + for (a = s + incr; a <= e; a += incr) + { + new_atom = _yr_atoms_clone_list_item(atom); + + if (new_atom == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + new_atom->atom.bytes[i] = (uint8_t) a; + new_atom->atom.mask[i] = 0xFF; + new_atom->next = next_atom; + prev_atom->next = new_atom; + prev_atom = new_atom; + } + } + + if (!expanded) + atom = atom->next; + } + + return ERROR_SUCCESS; +} + + +// +// yr_atoms_extract_from_re +// +// Extract atoms from a regular expression. This function receives the abstract +// syntax tree for a regexp (or hex pattern) and returns a list of atoms that +// should be added to the Aho-Corasick automaton. // int yr_atoms_extract_from_re( YR_ATOMS_CONFIG* config, RE_AST* re_ast, int flags, - YR_ATOM_LIST_ITEM** atoms) + YR_ATOM_LIST_ITEM** atoms, + int* min_atom_quality) { - ATOM_TREE* atom_tree = (ATOM_TREE*) yr_malloc(sizeof(ATOM_TREE)); - ATOM_TREE_NODE* temp; + YR_ATOM_TREE* atom_tree = (YR_ATOM_TREE*) yr_malloc(sizeof(YR_ATOM_TREE)); + YR_ATOM_LIST_ITEM* wide_atoms; YR_ATOM_LIST_ITEM* case_insensitive_atoms; - YR_ATOM_LIST_ITEM* triplet_atoms; - - int min_atom_quality = YR_MIN_ATOM_QUALITY; if (atom_tree == NULL) return ERROR_INSUFFICIENT_MEMORY; @@ -1185,63 +1376,27 @@ int yr_atoms_extract_from_re( return ERROR_INSUFFICIENT_MEMORY; } - atom_tree->current_leaf = NULL; - - atom_tree->root_node = _yr_atoms_extract_from_re_node( - config, re_ast->root_node, atom_tree, atom_tree->root_node); - - if (atom_tree->root_node == NULL) - { - _yr_atoms_tree_destroy(atom_tree); - return ERROR_INSUFFICIENT_MEMORY; - } - - if (atom_tree->current_leaf != NULL) - _yr_atoms_tree_node_append(atom_tree->root_node, atom_tree->current_leaf); - - if (atom_tree->root_node->children_head == - atom_tree->root_node->children_tail) - { - // The root OR node has a single child, there's no need for the OR node so - // we proceed to destroy it and use its child as root. - - temp = atom_tree->root_node; - atom_tree->root_node = atom_tree->root_node->children_head; - yr_free(temp); - } + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_atoms_extract_from_re(config, re_ast, atom_tree->root_node), + _yr_atoms_tree_destroy(atom_tree)); // Initialize atom list *atoms = NULL; - if (atom_tree->root_node != NULL) - { - // Choose the atoms that will be used. - FAIL_ON_ERROR_WITH_CLEANUP( - _yr_atoms_choose( - config, atom_tree->root_node, atoms, &min_atom_quality), - _yr_atoms_tree_destroy(atom_tree)); - } + // Choose the atoms that will be used. + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_atoms_choose(config, atom_tree->root_node, atoms, min_atom_quality), + _yr_atoms_tree_destroy(atom_tree)); _yr_atoms_tree_destroy(atom_tree); FAIL_ON_ERROR_WITH_CLEANUP( - yr_atoms_extract_triplets(re_ast->root_node, &triplet_atoms), + _yr_atoms_expand_wildcards(*atoms), { yr_atoms_list_destroy(*atoms); - yr_atoms_list_destroy(triplet_atoms); *atoms = NULL; }); - if (min_atom_quality < (yr_atoms_min_quality(config, triplet_atoms) >> 1)) - { - yr_atoms_list_destroy(*atoms); - *atoms = triplet_atoms; - } - else - { - yr_atoms_list_destroy(triplet_atoms); - } - if (flags & STRING_GFLAGS_WIDE) { FAIL_ON_ERROR_WITH_CLEANUP( @@ -1285,7 +1440,7 @@ int yr_atoms_extract_from_re( if (*atoms == NULL) return ERROR_INSUFFICIENT_MEMORY; - (*atoms)->atom_length = 0; + (*atoms)->atom.length = 0; (*atoms)->backtrack = 0; (*atoms)->forward_code = re_ast->root_node->forward_code; (*atoms)->backward_code = NULL; @@ -1307,15 +1462,18 @@ int yr_atoms_extract_from_string( uint8_t* string, int32_t string_length, int flags, - YR_ATOM_LIST_ITEM** atoms) + YR_ATOM_LIST_ITEM** atoms, + int* min_atom_quality) { YR_ATOM_LIST_ITEM* item; YR_ATOM_LIST_ITEM* case_insensitive_atoms; YR_ATOM_LIST_ITEM* xor_atoms; YR_ATOM_LIST_ITEM* wide_atoms; - int max_quality; - int i, j, length; + YR_ATOM atom; + + int quality, max_quality; + int i; item = (YR_ATOM_LIST_ITEM*) yr_malloc(sizeof(YR_ATOM_LIST_ITEM)); @@ -1327,35 +1485,38 @@ int yr_atoms_extract_from_string( item->next = NULL; item->backtrack = 0; - length = yr_min(string_length, YR_MAX_ATOM_LENGTH); + item->atom.length = yr_min(string_length, YR_MAX_ATOM_LENGTH); - for (i = 0; i < length; i++) - item->atom[i] = string[i]; + for (i = 0; i < item->atom.length; i++) + { + item->atom.bytes[i] = string[i]; + item->atom.mask[i] = 0xFF; + } - item->atom_length = i; + max_quality = config->get_atom_quality(config, &item->atom); - max_quality = config->get_atom_quality(config, string, length); + atom.length = YR_MAX_ATOM_LENGTH; + memset(atom.mask, 0xFF, atom.length); for (i = YR_MAX_ATOM_LENGTH; i < string_length && max_quality < YR_MAX_ATOM_QUALITY; i++) { - int quality = config->get_atom_quality( - config, - string + i - YR_MAX_ATOM_LENGTH + 1, - YR_MAX_ATOM_LENGTH); + atom.length = YR_MAX_ATOM_LENGTH; + memcpy(atom.bytes, string + i - YR_MAX_ATOM_LENGTH + 1, atom.length); + + quality = config->get_atom_quality(config, &atom); if (quality > max_quality) { - for (j = 0; j < YR_MAX_ATOM_LENGTH; j++) - item->atom[j] = string[i + j - YR_MAX_ATOM_LENGTH + 1]; - + memcpy(&item->atom, &atom, sizeof(atom)); item->backtrack = i - YR_MAX_ATOM_LENGTH + 1; max_quality = quality; } } *atoms = item; + *min_atom_quality = max_quality; if (flags & STRING_GFLAGS_WIDE) { @@ -1426,9 +1587,9 @@ int yr_atoms_extract_from_string( // void yr_atoms_tree_node_print( - ATOM_TREE_NODE* node) + YR_ATOM_TREE_NODE* node) { - ATOM_TREE_NODE* child; + YR_ATOM_TREE_NODE* child; int i; if (node == NULL) @@ -1440,8 +1601,8 @@ void yr_atoms_tree_node_print( switch(node->type) { case ATOM_TREE_LEAF: - for (i = 0; i < node->atom_length; i++) - printf("%02X", node->atom[i]); + for (i = 0; i < node->atom.length; i++) + printf("%02X", node->atom.bytes[i]); break; case ATOM_TREE_AND: diff --git a/libyara/hex_grammar.c b/libyara/hex_grammar.c index b82d4891..2eb5216f 100644 --- a/libyara/hex_grammar.c +++ b/libyara/hex_grammar.c @@ -486,7 +486,7 @@ static const yytype_uint16 yyrline[] = { 0, 113, 113, 122, 126, 140, 208, 212, 230, 234, 243, 248, 247, 260, 283, 315, 337, 357, 361, 380, - 388 + 389 }; #endif @@ -1645,12 +1645,13 @@ yyreduce: fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); (yyval.re_node)->value = (int) (yyvsp[0].integer); + (yyval.re_node)->mask = 0xFF; } -#line 1650 "hex_grammar.c" /* yacc.c:1663 */ +#line 1651 "hex_grammar.c" /* yacc.c:1663 */ break; case 20: -#line 389 "hex_grammar.y" /* yacc.c:1663 */ +#line 390 "hex_grammar.y" /* yacc.c:1663 */ { uint8_t mask = (uint8_t) ((yyvsp[0].integer) >> 8); @@ -1659,6 +1660,9 @@ yyreduce: (yyval.re_node) = yr_re_node_create(RE_NODE_ANY, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); + + (yyval.re_node)->value = 0x00; + (yyval.re_node)->mask = 0x00; } else { @@ -1670,11 +1674,11 @@ yyreduce: (yyval.re_node)->mask = mask; } } -#line 1674 "hex_grammar.c" /* yacc.c:1663 */ +#line 1678 "hex_grammar.c" /* yacc.c:1663 */ break; -#line 1678 "hex_grammar.c" /* yacc.c:1663 */ +#line 1682 "hex_grammar.c" /* yacc.c:1663 */ default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -1902,4 +1906,4 @@ yyreturn: #endif return yyresult; } -#line 410 "hex_grammar.y" /* yacc.c:1907 */ +#line 414 "hex_grammar.y" /* yacc.c:1907 */ diff --git a/libyara/hex_grammar.y b/libyara/hex_grammar.y index 5687a834..9205cd38 100644 --- a/libyara/hex_grammar.y +++ b/libyara/hex_grammar.y @@ -384,6 +384,7 @@ byte fail_if($$ == NULL, ERROR_INSUFFICIENT_MEMORY); $$->value = (int) $1; + $$->mask = 0xFF; } | _MASKED_BYTE_ { @@ -394,6 +395,9 @@ byte $$ = yr_re_node_create(RE_NODE_ANY, NULL, NULL); fail_if($$ == NULL, ERROR_INSUFFICIENT_MEMORY); + + $$->value = 0x00; + $$->mask = 0x00; } else { diff --git a/libyara/include/yara/atoms.h b/libyara/include/yara/atoms.h index 91955756..d6605b44 100644 --- a/libyara/include/yara/atoms.h +++ b/libyara/include/yara/atoms.h @@ -37,9 +37,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ATOM_TREE_AND 2 #define ATOM_TREE_OR 3 - -typedef struct ATOM_TREE_NODE ATOM_TREE_NODE; -typedef struct ATOM_TREE ATOM_TREE; +typedef struct YR_ATOM YR_ATOM; +typedef struct YR_ATOM_TREE_NODE YR_ATOM_TREE_NODE; +typedef struct YR_ATOM_TREE YR_ATOM_TREE; typedef struct YR_ATOM_LIST_ITEM YR_ATOM_LIST_ITEM; @@ -47,34 +47,37 @@ typedef struct YR_ATOM_QUALITY_TABLE_ENTRY YR_ATOM_QUALITY_TABLE_ENTRY; typedef struct YR_ATOMS_CONFIG YR_ATOMS_CONFIG; -struct ATOM_TREE_NODE +struct YR_ATOM +{ + uint8_t length; + uint8_t bytes[YR_MAX_ATOM_LENGTH]; + uint8_t mask[YR_MAX_ATOM_LENGTH]; +}; + +struct YR_ATOM_TREE_NODE { uint8_t type; - uint8_t atom_length; - uint8_t atom[YR_MAX_ATOM_LENGTH]; - uint8_t* forward_code; - uint8_t* backward_code; + YR_ATOM atom; - RE_NODE* recent_nodes[YR_MAX_ATOM_LENGTH]; + // RE nodes that correspond to each byte in the atom. + RE_NODE* re_nodes[YR_MAX_ATOM_LENGTH]; - ATOM_TREE_NODE* children_head; - ATOM_TREE_NODE* children_tail; - ATOM_TREE_NODE* next_sibling; + YR_ATOM_TREE_NODE* children_head; + YR_ATOM_TREE_NODE* children_tail; + YR_ATOM_TREE_NODE* next_sibling; }; -struct ATOM_TREE +struct YR_ATOM_TREE { - ATOM_TREE_NODE* current_leaf; - ATOM_TREE_NODE* root_node; + YR_ATOM_TREE_NODE* root_node; }; struct YR_ATOM_LIST_ITEM { - uint8_t atom_length; - uint8_t atom[YR_MAX_ATOM_LENGTH]; + YR_ATOM atom; uint16_t backtrack; @@ -90,7 +93,7 @@ struct YR_ATOM_LIST_ITEM struct YR_ATOM_QUALITY_TABLE_ENTRY { - const uint8_t atom[YR_MAX_ATOM_LENGTH]; + const uint8_t atom[YR_MAX_ATOM_LENGTH]; const uint8_t quality; }; @@ -99,8 +102,7 @@ struct YR_ATOM_QUALITY_TABLE_ENTRY typedef int (*YR_ATOMS_QUALITY_FUNC)( YR_ATOMS_CONFIG* config, - uint8_t* atom, - int atom_length); + YR_ATOM* atom); struct YR_ATOMS_CONFIG @@ -118,7 +120,8 @@ int yr_atoms_extract_from_re( YR_ATOMS_CONFIG* config, RE_AST* re_ast, int flags, - YR_ATOM_LIST_ITEM** atoms); + YR_ATOM_LIST_ITEM** atoms, + int* min_atom_quality); int yr_atoms_extract_from_string( @@ -126,7 +129,8 @@ int yr_atoms_extract_from_string( uint8_t* string, int string_length, int flags, - YR_ATOM_LIST_ITEM** atoms); + YR_ATOM_LIST_ITEM** atoms, + int* min_atom_quality); int yr_atoms_extract_triplets( @@ -136,14 +140,12 @@ int yr_atoms_extract_triplets( int yr_atoms_heuristic_quality( YR_ATOMS_CONFIG* config, - uint8_t* atom, - int atom_length); + YR_ATOM* atom); int yr_atoms_table_quality( YR_ATOMS_CONFIG* config, - uint8_t* atom, - int atom_length); + YR_ATOM* atom); int yr_atoms_min_quality( diff --git a/libyara/include/yara/error.h b/libyara/include/yara/error.h index 9f253aea..a682fa9a 100644 --- a/libyara/include/yara/error.h +++ b/libyara/include/yara/error.h @@ -102,6 +102,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ERROR_INVALID_MODULE_DATA 57 #define ERROR_WRITING_FILE 58 + #define FAIL_ON_ERROR(x) { \ int result = (x); \ if (result != ERROR_SUCCESS) \ @@ -116,10 +117,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. } \ } -#define FAIL_ON_COMPILER_ERROR(x) { \ - compiler->last_error = (x); \ - if (compiler->last_error != ERROR_SUCCESS) \ - return compiler->last_error; \ +#define FAIL_ON_NULL_WITH_CLEANUP(x, cleanup) { \ + if ((x) == NULL) { \ + cleanup; \ + return ERROR_INSUFFICIENT_MEMORY; \ + } \ } diff --git a/libyara/include/yara/limits.h b/libyara/include/yara/limits.h index 8df3cc26..cd596074 100644 --- a/libyara/include/yara/limits.h +++ b/libyara/include/yara/limits.h @@ -72,14 +72,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif // If the minimum atom quality for a string or regexp is below this constant, -// a warning like " is slowing down the scan" is shown. This used only -// with heuristic atom quality, when using an atom quality table the user must -// specify the threshold when calling yr_compiler_set_atom_quality_table +// a warning like " is slowing down the scan" is shown. This is used +// only with heuristic atom quality, when using an atom quality table the user +// must specify the threshold when calling yr_compiler_set_atom_quality_table. #ifndef YR_ATOM_QUALITY_WARNING_THRESHOLD #define YR_ATOM_QUALITY_WARNING_THRESHOLD \ - (YR_MAX_ATOM_QUALITY - 2 * YR_MAX_ATOM_LENGTH + 3) + YR_MAX_ATOM_QUALITY - 20 * YR_MAX_ATOM_LENGTH + 38 #endif +// If a rule generates more than this number of atoms a warning is shown. +#ifndef YR_ATOMS_PER_RULE_WARNING_THRESHOLD +#define YR_ATOMS_PER_RULE_WARNING_THRESHOLD 10000 +#endif // Maximum number of nested "for" loops in rule. Rules ith nested loops // exceeding this number will be rejected by the compiler. diff --git a/libyara/include/yara/stack.h b/libyara/include/yara/stack.h new file mode 100644 index 00000000..a08ae1ce --- /dev/null +++ b/libyara/include/yara/stack.h @@ -0,0 +1,72 @@ +/* +Copyright (c) 2018. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef YR_STACK_H +#define YR_STACK_H + +typedef struct YR_STACK YR_STACK; + +struct YR_STACK +{ + // Pointer to a heap-allocated array containing the void* values put in + // in the stack. This array starts with a fixed size and it's grown as + // required when new items are pushed into the stack. + void* items; + + // Current capacity (i.e: the number of items that fit into the array) + int capacity; + + // Size of each individual item in the stack. + int item_size; + + // Index of the stack's top in the items array. + int top; +}; + + +int yr_stack_create( + int initial_capacity, + int item_size, + YR_STACK** stack); + + +void yr_stack_destroy( + YR_STACK* stack); + + +int yr_stack_push( + YR_STACK* stack, + void* item); + + +int yr_stack_pop( + YR_STACK* stack, + void* item); + +#endif \ No newline at end of file diff --git a/libyara/parser.c b/libyara/parser.c index f6814710..40606d9a 100644 --- a/libyara/parser.c +++ b/libyara/parser.c @@ -295,12 +295,14 @@ static int _yr_parser_write_string( SIZED_STRING* str, RE_AST* re_ast, YR_STRING** string, - int* min_atom_quality) + int* min_atom_quality, + int* num_atom) { SIZED_STRING* literal_string; + YR_ATOM_LIST_ITEM* atom; YR_ATOM_LIST_ITEM* atom_list = NULL; - int result; + int c, result; int max_string_len; bool free_literal = false; @@ -381,7 +383,8 @@ static int _yr_parser_write_string( (uint8_t*) literal_string->c_string, (int32_t) literal_string->length, flags, - &atom_list); + &atom_list, + min_atom_quality); } } else @@ -395,7 +398,11 @@ static int _yr_parser_write_string( if (result == ERROR_SUCCESS) result = yr_atoms_extract_from_re( - &compiler->atoms_config, re_ast, flags, &atom_list); + &compiler->atoms_config, + re_ast, + flags, + &atom_list, + min_atom_quality); } if (result == ERROR_SUCCESS) @@ -408,9 +415,6 @@ static int _yr_parser_write_string( compiler->matches_arena); } - *min_atom_quality = yr_atoms_min_quality( - &compiler->atoms_config, atom_list); - if (flags & STRING_GFLAGS_LITERAL) { if (flags & STRING_GFLAGS_WIDE) @@ -422,6 +426,17 @@ static int _yr_parser_write_string( (*string)->g_flags |= STRING_GFLAGS_FITS_IN_ATOM; } + atom = atom_list; + c = 0; + + while (atom != NULL) + { + atom = atom->next; + c++; + } + + (*num_atom) += c; + if (free_literal) yr_free(literal_string); @@ -443,8 +458,8 @@ int yr_parser_reduce_string_declaration( SIZED_STRING* str, YR_STRING** string) { - int min_atom_quality; - int min_atom_quality_aux; + int min_atom_quality = YR_MIN_ATOM_QUALITY; + int min_atom_quality_aux = YR_MIN_ATOM_QUALITY; int32_t min_gap; int32_t max_gap; @@ -599,7 +614,8 @@ int yr_parser_reduce_string_declaration( NULL, re_ast, string, - &min_atom_quality); + &min_atom_quality, + &compiler->current_rule->num_atoms); if (result != ERROR_SUCCESS) goto _exit; @@ -638,7 +654,8 @@ int yr_parser_reduce_string_declaration( NULL, re_ast, &aux_string, - &min_atom_quality_aux); + &min_atom_quality_aux, + &compiler->current_rule->num_atoms); if (result != ERROR_SUCCESS) goto _exit; @@ -668,7 +685,8 @@ int yr_parser_reduce_string_declaration( str, NULL, string, - &min_atom_quality); + &min_atom_quality, + &compiler->current_rule->num_atoms); if (result != ERROR_SUCCESS) goto _exit; @@ -690,8 +708,9 @@ int yr_parser_reduce_string_declaration( { yywarning( yyscanner, - "%s is slowing down scanning", - (*string)->identifier); + "%s in rule %s is slowing down scanning", + (*string)->identifier, + compiler->current_rule->identifier); } _exit: @@ -747,6 +766,7 @@ int yr_parser_reduce_rule_declaration_phase_1( (*rule)->g_flags = flags; (*rule)->ns = compiler->current_namespace; + (*rule)->num_atoms = 0; #ifdef PROFILING_ENABLED (*rule)->time_cost = 0; @@ -818,14 +838,27 @@ int yr_parser_reduce_rule_declaration_phase_2( YR_FIXUP *fixup; YR_COMPILER* compiler = yyget_extra(yyscanner); - // Check for unreferenced (unused) strings. - - YR_STRING* string = rule->strings; - yr_get_configuration( YR_CONFIG_MAX_STRINGS_PER_RULE, (void*) &max_strings_per_rule); + // Show warning if the rule is generating too many atoms. The warning is + // shown if the number of atoms is greater than 20 times the maximum number + // of strings allowed for a rule, as 20 is minimum number of atoms generated + // for a string using *nocase*, *ascii* and *wide* modifiers simultaneosly. + + if (rule->num_atoms > YR_ATOMS_PER_RULE_WARNING_THRESHOLD) + { + yywarning( + yyscanner, + "rule %s is slowing down scanning", + rule->identifier); + } + + // Check for unreferenced (unused) strings. + + YR_STRING* string = rule->strings; + while (!STRING_IS_NULL(string)) { // Only the heading fragment in a chain of strings (the one with diff --git a/libyara/re_grammar.c b/libyara/re_grammar.c index 8a551b82..3811c8eb 100644 --- a/libyara/re_grammar.c +++ b/libyara/re_grammar.c @@ -501,8 +501,8 @@ static const yytype_uint16 yyrline[] = { 0, 112, 112, 117, 121, 125, 141, 163, 167, 184, 198, 214, 228, 244, 266, 289, 311, 334, 338, 344, - 350, 356, 365, 373, 379, 387, 393, 399, 405, 411, - 417, 423 + 350, 356, 365, 373, 382, 391, 397, 403, 409, 415, + 421, 427 }; #endif @@ -1672,85 +1672,89 @@ yyreduce: { (yyval.re_node) = yr_re_node_create(RE_NODE_ANY, NULL, NULL); + (yyval.re_node)->value = 0x00; + (yyval.re_node)->mask = 0x00; + fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1678 "re_grammar.c" /* yacc.c:1663 */ +#line 1681 "re_grammar.c" /* yacc.c:1663 */ break; case 24: -#line 380 "re_grammar.y" /* yacc.c:1663 */ +#line 383 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_LITERAL, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); (yyval.re_node)->value = (yyvsp[0].integer); + (yyval.re_node)->mask = 0xFF; } -#line 1690 "re_grammar.c" /* yacc.c:1663 */ +#line 1694 "re_grammar.c" /* yacc.c:1663 */ break; case 25: -#line 388 "re_grammar.y" /* yacc.c:1663 */ +#line 392 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_WORD_CHAR, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1700 "re_grammar.c" /* yacc.c:1663 */ +#line 1704 "re_grammar.c" /* yacc.c:1663 */ break; case 26: -#line 394 "re_grammar.y" /* yacc.c:1663 */ +#line 398 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_NON_WORD_CHAR, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1710 "re_grammar.c" /* yacc.c:1663 */ +#line 1714 "re_grammar.c" /* yacc.c:1663 */ break; case 27: -#line 400 "re_grammar.y" /* yacc.c:1663 */ +#line 404 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_SPACE, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1720 "re_grammar.c" /* yacc.c:1663 */ +#line 1724 "re_grammar.c" /* yacc.c:1663 */ break; case 28: -#line 406 "re_grammar.y" /* yacc.c:1663 */ +#line 410 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_NON_SPACE, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1730 "re_grammar.c" /* yacc.c:1663 */ +#line 1734 "re_grammar.c" /* yacc.c:1663 */ break; case 29: -#line 412 "re_grammar.y" /* yacc.c:1663 */ +#line 416 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_DIGIT, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1740 "re_grammar.c" /* yacc.c:1663 */ +#line 1744 "re_grammar.c" /* yacc.c:1663 */ break; case 30: -#line 418 "re_grammar.y" /* yacc.c:1663 */ +#line 422 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_NON_DIGIT, NULL, NULL); fail_if((yyval.re_node) == NULL, ERROR_INSUFFICIENT_MEMORY); } -#line 1750 "re_grammar.c" /* yacc.c:1663 */ +#line 1754 "re_grammar.c" /* yacc.c:1663 */ break; case 31: -#line 424 "re_grammar.y" /* yacc.c:1663 */ +#line 428 "re_grammar.y" /* yacc.c:1663 */ { (yyval.re_node) = yr_re_node_create(RE_NODE_CLASS, NULL, NULL); @@ -1758,11 +1762,11 @@ yyreduce: (yyval.re_node)->re_class = (yyvsp[0].re_class); } -#line 1762 "re_grammar.c" /* yacc.c:1663 */ +#line 1766 "re_grammar.c" /* yacc.c:1663 */ break; -#line 1766 "re_grammar.c" /* yacc.c:1663 */ +#line 1770 "re_grammar.c" /* yacc.c:1663 */ default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -1990,5 +1994,5 @@ yyreturn: #endif return yyresult; } -#line 432 "re_grammar.y" /* yacc.c:1907 */ +#line 436 "re_grammar.y" /* yacc.c:1907 */ diff --git a/libyara/re_grammar.y b/libyara/re_grammar.y index 42ca482c..b8107310 100644 --- a/libyara/re_grammar.y +++ b/libyara/re_grammar.y @@ -374,6 +374,9 @@ single { $$ = yr_re_node_create(RE_NODE_ANY, NULL, NULL); + $$->value = 0x00; + $$->mask = 0x00; + fail_if($$ == NULL, ERROR_INSUFFICIENT_MEMORY); } | _CHAR_ @@ -383,6 +386,7 @@ single fail_if($$ == NULL, ERROR_INSUFFICIENT_MEMORY); $$->value = $1; + $$->mask = 0xFF; } | _WORD_CHAR_ { diff --git a/libyara/stack.c b/libyara/stack.c new file mode 100644 index 00000000..6dcf44d0 --- /dev/null +++ b/libyara/stack.c @@ -0,0 +1,145 @@ +/* +Copyright (c) 2018. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +#include +#include +#include +#include + + +// +// yr_stack_create +// +// Creates a stack for items of the size specified by item_size. All items +// in the stack must have the same size. The stack will have an initial +// capacity as specified by initial_capacity and will grow as required when +// more objects are pushed. +// + +int yr_stack_create( + int initial_capacity, + int item_size, + YR_STACK** stack) +{ + *stack = (YR_STACK*) yr_malloc(sizeof(YR_STACK)); + + if (*stack == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + (*stack)->items = yr_malloc(initial_capacity * item_size); + + if ((*stack)->items == NULL) + { + yr_free(*stack); + *stack = NULL; + return ERROR_INSUFFICIENT_MEMORY; + } + + (*stack)->capacity = initial_capacity; + (*stack)->item_size = item_size; + (*stack)->top = 0; + + return ERROR_SUCCESS; +} + +// +// yr_stack_destroy +// +// Destroys a stack and deallocates all its resources. +// + +void yr_stack_destroy( + YR_STACK* stack) +{ + yr_free(stack->items); + yr_free(stack); +} + + +// +// yr_stack_push +// +// Pushes an item into the stack. If the stack has reached its capacity the +// funtion tries to double the capacity. This operation can fail with +// ERROR_INSUFFICIENT_MEMORY. +// + +int yr_stack_push( + YR_STACK* stack, + void* item) +{ + if (stack->top == stack->capacity) + { + void* items = yr_realloc( + stack->items, 2 * stack->capacity * stack->item_size); + + if (items == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + stack->items = items; + stack->capacity *= 2; + } + + memcpy( + (uint8_t*) stack->items + stack->top * stack->item_size, + item, + stack->item_size); + + stack->top++; + + return ERROR_SUCCESS; +} + + +// +// yr_stack_pop +// +// Pops an item from the stack. The caller must pass pointer to a buffer +// where the function will copy the item. The buffer must have enough space +// to hold the item. Returns 1 if an item could be poped and 0 if the stack +// was already empty. +// + +int yr_stack_pop( + YR_STACK* stack, + void* item) +{ + if (stack->top == 0) // Return 0 if stack is empty. + return 0; + + stack->top--; + + memcpy( + item, + (uint8_t*) stack->items + stack->top * stack->item_size, + stack->item_size); + + return 1; +} diff --git a/tests/test-api.c b/tests/test-api.c index 9e51d8cc..3aba75b0 100644 --- a/tests/test-api.c +++ b/tests/test-api.c @@ -654,12 +654,12 @@ void test_rules_stats() stats_for_rules("\ rule test { \ strings: \ - $ = \"abcd0\" \ - $ = \"abcd1\" \ - $ = \"abcd2\" \ - $ = \"efgh0\" \ - $ = \"efgh1\" \ - $ = \"efgh2\" \ + $ = \"00000\" \ + $ = \"00001\" \ + $ = \"00002\" \ + $ = \"11110\" \ + $ = \"11111\" \ + $ = \"11112\" \ condition: all of them }", &stats); @@ -674,14 +674,14 @@ void test_rules_stats() stats_for_rules("\ rule test { \ strings: \ - $ = \"abcd0\" \ - $ = \"abcd1\" \ - $ = \"abcd2\" \ - $ = \"efgh0\" \ - $ = \"ijkl0\" \ - $ = \"mnop0\" \ - $ = \"mnop1\" \ - $ = \"qrst0\" \ + $ = \"00000\" \ + $ = \"00001\" \ + $ = \"00002\" \ + $ = \"11110\" \ + $ = \"22220\" \ + $ = \"33330\" \ + $ = \"33331\" \ + $ = \"44440\" \ condition: all of them }", &stats); diff --git a/tests/test-atoms.c b/tests/test-atoms.c index 66f4b07b..6dd362f8 100644 --- a/tests/test-atoms.c +++ b/tests/test-atoms.c @@ -31,7 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "util.h" -void test_quality_quality() +void test_table_quality() { YR_ATOM_QUALITY_TABLE_ENTRY l1[] = { {{0x00, 0x00, 0x00, 0x00}, 1}, @@ -46,10 +46,15 @@ void test_quality_quality() {{0x00, 0x00, 0x00, 0x03}, 4}, }; - uint8_t a0[] = {0x00, 0x00, 0x00, 0x00}; - uint8_t a1[] = {0x00, 0x00, 0x00, 0x01}; - uint8_t a2[] = {0x00, 0x00, 0x00, 0x02}; - uint8_t a3[] = {0x00, 0x00, 0x00, 0x03}; + YR_ATOM a0 = { + .length = 4, + .bytes = {0x00, 0x00, 0x00, 0x00}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a1 = {4, {0x00, 0x00, 0x00, 0x01}, {0xFF, 0xFF, 0xFF, 0xFF}}; + YR_ATOM a2 = {4, {0x00, 0x00, 0x00, 0x02}, {0xFF, 0xFF, 0xFF, 0xFF}}; + YR_ATOM a3 = {4, {0x00, 0x00, 0x00, 0x03}, {0xFF, 0xFF, 0xFF, 0xFF}}; + YR_ATOM a4 = {4, {0x00, 0x00, 0x00, 0x00}, {0xFF, 0xFF, 0xFF, 0x00}}; YR_ATOMS_CONFIG c; @@ -57,36 +62,519 @@ void test_quality_quality() c.quality_table = l1; c.quality_table_entries = 3; - assert_true_expr( - yr_atoms_table_quality(&c, a0, sizeof(a0)) == 1); - - assert_true_expr( - yr_atoms_table_quality(&c, a1, sizeof(a1)) == 2); - - assert_true_expr( - yr_atoms_table_quality(&c, a2, sizeof(a2)) == 3); - - assert_true_expr( - yr_atoms_table_quality(&c, a3, sizeof(3)) == YR_MAX_ATOM_QUALITY); + assert_true_expr(yr_atoms_table_quality(&c, &a0) == 1); + assert_true_expr(yr_atoms_table_quality(&c, &a1) == 2); + assert_true_expr(yr_atoms_table_quality(&c, &a2) == 3); + assert_true_expr(yr_atoms_table_quality(&c, &a3) == YR_MAX_ATOM_QUALITY); c.quality_table = l2; c.quality_table_entries = 4; - assert_true_expr( - yr_atoms_table_quality(&c, a0, sizeof(a0)) == 1); + assert_true_expr(yr_atoms_table_quality(&c, &a0) == 1); + assert_true_expr(yr_atoms_table_quality(&c, &a1) == 2); + assert_true_expr(yr_atoms_table_quality(&c, &a2) == 3); + assert_true_expr(yr_atoms_table_quality(&c, &a3) == 4); + assert_true_expr(yr_atoms_table_quality(&c, &a4) == 1); +} - assert_true_expr( - yr_atoms_table_quality(&c, a1, sizeof(a1)) == 2); - assert_true_expr( - yr_atoms_table_quality(&c, a2, sizeof(a2)) == 3); +void test_heuristic_quality() +{ + YR_ATOMS_CONFIG c; - assert_true_expr( - yr_atoms_table_quality(&c, a3, sizeof(a3)) == 4); + YR_ATOM a00000000 = { + .length = 4, + .bytes = {0x00, 0x00, 0x00, 0x00}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a00000001 = { + .length = 4, + .bytes = {0x00, 0x00, 0x00, 0x01}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a00000102 = { + .length = 4, + .bytes = {0x00, 0x00, 0x01, 0x02}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a00010203 = { + .length = 4, + .bytes = {0x00, 0x01, 0x02, 0x03}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a01020304 = { + .length = 4, + .bytes = {0x01, 0x02, 0x03, 0x04}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a01 = { + .length = 1, + .bytes = {0x01}, + .mask = {0xFF}}; + + YR_ATOM a0001 = { + .length = 2, + .bytes = {0x00, 0x01}, + .mask = {0xFF, 0xFF}}; + + YR_ATOM a000001 = { + .length = 3, + .bytes = {0x00, 0x00, 0x01}, + .mask = {0xFF, 0xFF, 0xFF}}; + + YR_ATOM a000102 = { + .length = 3, + .bytes = {0x00, 0x01, 0x02}, + .mask = {0xFF, 0xFF, 0xFF}}; + + YR_ATOM a010203 = { + .length = 3, + .bytes = {0x01, 0x02, 0x03}, + .mask = {0xFF, 0xFF, 0xFF}}; + + YR_ATOM a0102 = { + .length = 2, + .bytes = {0x01, 0x02}, + .mask = {0xFF, 0xFF}}; + + YR_ATOM a01020000 = { + .length = 4, + .bytes = {0x01, 0x02, 0x00, 0x00}, + .mask = {0xFF, 0xFF, 0xFF, 0xFF}}; + + YR_ATOM a0102XX04 = { + .length = 4, + .bytes = {0x01, 0x02, 0x03, 0x04}, + .mask = {0xFF, 0xFF, 0x00, 0xFF}}; + + c.get_atom_quality = yr_atoms_heuristic_quality; + + int q00000000 = yr_atoms_heuristic_quality(&c, &a00000000); + int q00000001 = yr_atoms_heuristic_quality(&c, &a00000001); + int q00000102 = yr_atoms_heuristic_quality(&c, &a00000102); + int q00010203 = yr_atoms_heuristic_quality(&c, &a00010203); + int q01020304 = yr_atoms_heuristic_quality(&c, &a01020304); + int q010203 = yr_atoms_heuristic_quality(&c, &a010203); + int q0102 = yr_atoms_heuristic_quality(&c, &a0102); + int q01 = yr_atoms_heuristic_quality(&c, &a01); + int q0001 = yr_atoms_heuristic_quality(&c, &a0001); + int q000001 = yr_atoms_heuristic_quality(&c, &a000001); + int q000102 = yr_atoms_heuristic_quality(&c, &a000102); + int q01020000 = yr_atoms_heuristic_quality(&c, &a01020000); + int q0102XX04 = yr_atoms_heuristic_quality(&c, &a0102XX04); + + + a010203.mask[1] = 0x00; + + int q01XX03 = yr_atoms_heuristic_quality(&c, &a010203); + + a010203.mask[1] = 0x0F; + + int q01X203 = yr_atoms_heuristic_quality(&c, &a010203); + + a010203.mask[1] = 0x0F; + + int q010X03 = yr_atoms_heuristic_quality(&c, &a010203); + + a010203.mask[2] = 0x0F; + + int q010X0X = yr_atoms_heuristic_quality(&c, &a010203); + + assert_true_expr(q00000001 > q00000000); + assert_true_expr(q00000001 > q000001); + assert_true_expr(q000001 > q0001); + assert_true_expr(q00000102 > q00000001); + assert_true_expr(q00010203 > q00000102); + assert_true_expr(q01020304 > q00010203); + assert_true_expr(q000102 > q000001); + assert_true_expr(q00010203 > q010203); + assert_true_expr(q010203 > q0102); + assert_true_expr(q0102 > q01); + assert_true_expr(q01X203 > q0102); + assert_true_expr(q01X203 > q0001); + assert_true_expr(q01X203 < q010203); + assert_true_expr(q01X203 == q010X03); + assert_true_expr(q01XX03 <= q0102); + assert_true_expr(q01XX03 < q010X03); + assert_true_expr(q01XX03 < q010203); + assert_true_expr(q010X0X > q01); + assert_true_expr(q010X0X < q010203); + assert_true_expr(q01020000 > q0102XX04); + + assert_true_expr(q01020304 == YR_MAX_ATOM_QUALITY); +} + + + +void test_atom_choose() +{ + + struct atom atoms_61_XX_63_64[] = { + {4, {0x61, 0x00, 0x63, 0x64}}, + {4, {0x61, 0x01, 0x63, 0x64}}, + {4, {0x61, 0x02, 0x63, 0x64}}, + {4, {0x61, 0x03, 0x63, 0x64}}, + {4, {0x61, 0x04, 0x63, 0x64}}, + {4, {0x61, 0x05, 0x63, 0x64}}, + {4, {0x61, 0x06, 0x63, 0x64}}, + {4, {0x61, 0x07, 0x63, 0x64}}, + {4, {0x61, 0x08, 0x63, 0x64}}, + {4, {0x61, 0x09, 0x63, 0x64}}, + {4, {0x61, 0x0A, 0x63, 0x64}}, + {4, {0x61, 0x0B, 0x63, 0x64}}, + {4, {0x61, 0x0C, 0x63, 0x64}}, + {4, {0x61, 0x0D, 0x63, 0x64}}, + {4, {0x61, 0x0E, 0x63, 0x64}}, + {4, {0x61, 0x0F, 0x63, 0x64}}, + {4, {0x61, 0x10, 0x63, 0x64}}, + {4, {0x61, 0x11, 0x63, 0x64}}, + {4, {0x61, 0x12, 0x63, 0x64}}, + {4, {0x61, 0x13, 0x63, 0x64}}, + {4, {0x61, 0x14, 0x63, 0x64}}, + {4, {0x61, 0x15, 0x63, 0x64}}, + {4, {0x61, 0x16, 0x63, 0x64}}, + {4, {0x61, 0x17, 0x63, 0x64}}, + {4, {0x61, 0x18, 0x63, 0x64}}, + {4, {0x61, 0x19, 0x63, 0x64}}, + {4, {0x61, 0x1A, 0x63, 0x64}}, + {4, {0x61, 0x1B, 0x63, 0x64}}, + {4, {0x61, 0x1C, 0x63, 0x64}}, + {4, {0x61, 0x1D, 0x63, 0x64}}, + {4, {0x61, 0x1E, 0x63, 0x64}}, + {4, {0x61, 0x1F, 0x63, 0x64}}, + {4, {0x61, 0x20, 0x63, 0x64}}, + {4, {0x61, 0x21, 0x63, 0x64}}, + {4, {0x61, 0x22, 0x63, 0x64}}, + {4, {0x61, 0x23, 0x63, 0x64}}, + {4, {0x61, 0x24, 0x63, 0x64}}, + {4, {0x61, 0x25, 0x63, 0x64}}, + {4, {0x61, 0x26, 0x63, 0x64}}, + {4, {0x61, 0x27, 0x63, 0x64}}, + {4, {0x61, 0x28, 0x63, 0x64}}, + {4, {0x61, 0x29, 0x63, 0x64}}, + {4, {0x61, 0x2A, 0x63, 0x64}}, + {4, {0x61, 0x2B, 0x63, 0x64}}, + {4, {0x61, 0x2C, 0x63, 0x64}}, + {4, {0x61, 0x2D, 0x63, 0x64}}, + {4, {0x61, 0x2E, 0x63, 0x64}}, + {4, {0x61, 0x2F, 0x63, 0x64}}, + {4, {0x61, 0x30, 0x63, 0x64}}, + {4, {0x61, 0x31, 0x63, 0x64}}, + {4, {0x61, 0x32, 0x63, 0x64}}, + {4, {0x61, 0x33, 0x63, 0x64}}, + {4, {0x61, 0x34, 0x63, 0x64}}, + {4, {0x61, 0x35, 0x63, 0x64}}, + {4, {0x61, 0x36, 0x63, 0x64}}, + {4, {0x61, 0x37, 0x63, 0x64}}, + {4, {0x61, 0x38, 0x63, 0x64}}, + {4, {0x61, 0x39, 0x63, 0x64}}, + {4, {0x61, 0x3A, 0x63, 0x64}}, + {4, {0x61, 0x3B, 0x63, 0x64}}, + {4, {0x61, 0x3C, 0x63, 0x64}}, + {4, {0x61, 0x3D, 0x63, 0x64}}, + {4, {0x61, 0x3E, 0x63, 0x64}}, + {4, {0x61, 0x3F, 0x63, 0x64}}, + {4, {0x61, 0x40, 0x63, 0x64}}, + {4, {0x61, 0x41, 0x63, 0x64}}, + {4, {0x61, 0x42, 0x63, 0x64}}, + {4, {0x61, 0x43, 0x63, 0x64}}, + {4, {0x61, 0x44, 0x63, 0x64}}, + {4, {0x61, 0x45, 0x63, 0x64}}, + {4, {0x61, 0x46, 0x63, 0x64}}, + {4, {0x61, 0x47, 0x63, 0x64}}, + {4, {0x61, 0x48, 0x63, 0x64}}, + {4, {0x61, 0x49, 0x63, 0x64}}, + {4, {0x61, 0x4A, 0x63, 0x64}}, + {4, {0x61, 0x4B, 0x63, 0x64}}, + {4, {0x61, 0x4C, 0x63, 0x64}}, + {4, {0x61, 0x4D, 0x63, 0x64}}, + {4, {0x61, 0x4E, 0x63, 0x64}}, + {4, {0x61, 0x4F, 0x63, 0x64}}, + {4, {0x61, 0x50, 0x63, 0x64}}, + {4, {0x61, 0x51, 0x63, 0x64}}, + {4, {0x61, 0x52, 0x63, 0x64}}, + {4, {0x61, 0x53, 0x63, 0x64}}, + {4, {0x61, 0x54, 0x63, 0x64}}, + {4, {0x61, 0x55, 0x63, 0x64}}, + {4, {0x61, 0x56, 0x63, 0x64}}, + {4, {0x61, 0x57, 0x63, 0x64}}, + {4, {0x61, 0x58, 0x63, 0x64}}, + {4, {0x61, 0x59, 0x63, 0x64}}, + {4, {0x61, 0x5A, 0x63, 0x64}}, + {4, {0x61, 0x5B, 0x63, 0x64}}, + {4, {0x61, 0x5C, 0x63, 0x64}}, + {4, {0x61, 0x5D, 0x63, 0x64}}, + {4, {0x61, 0x5E, 0x63, 0x64}}, + {4, {0x61, 0x5F, 0x63, 0x64}}, + {4, {0x61, 0x60, 0x63, 0x64}}, + {4, {0x61, 0x61, 0x63, 0x64}}, + {4, {0x61, 0x62, 0x63, 0x64}}, + {4, {0x61, 0x63, 0x63, 0x64}}, + {4, {0x61, 0x64, 0x63, 0x64}}, + {4, {0x61, 0x65, 0x63, 0x64}}, + {4, {0x61, 0x66, 0x63, 0x64}}, + {4, {0x61, 0x67, 0x63, 0x64}}, + {4, {0x61, 0x68, 0x63, 0x64}}, + {4, {0x61, 0x69, 0x63, 0x64}}, + {4, {0x61, 0x6A, 0x63, 0x64}}, + {4, {0x61, 0x6B, 0x63, 0x64}}, + {4, {0x61, 0x6C, 0x63, 0x64}}, + {4, {0x61, 0x6D, 0x63, 0x64}}, + {4, {0x61, 0x6E, 0x63, 0x64}}, + {4, {0x61, 0x6F, 0x63, 0x64}}, + {4, {0x61, 0x70, 0x63, 0x64}}, + {4, {0x61, 0x71, 0x63, 0x64}}, + {4, {0x61, 0x72, 0x63, 0x64}}, + {4, {0x61, 0x73, 0x63, 0x64}}, + {4, {0x61, 0x74, 0x63, 0x64}}, + {4, {0x61, 0x75, 0x63, 0x64}}, + {4, {0x61, 0x76, 0x63, 0x64}}, + {4, {0x61, 0x77, 0x63, 0x64}}, + {4, {0x61, 0x78, 0x63, 0x64}}, + {4, {0x61, 0x79, 0x63, 0x64}}, + {4, {0x61, 0x7A, 0x63, 0x64}}, + {4, {0x61, 0x7B, 0x63, 0x64}}, + {4, {0x61, 0x7C, 0x63, 0x64}}, + {4, {0x61, 0x7D, 0x63, 0x64}}, + {4, {0x61, 0x7E, 0x63, 0x64}}, + {4, {0x61, 0x7F, 0x63, 0x64}}, + {4, {0x61, 0x80, 0x63, 0x64}}, + {4, {0x61, 0x81, 0x63, 0x64}}, + {4, {0x61, 0x82, 0x63, 0x64}}, + {4, {0x61, 0x83, 0x63, 0x64}}, + {4, {0x61, 0x84, 0x63, 0x64}}, + {4, {0x61, 0x85, 0x63, 0x64}}, + {4, {0x61, 0x86, 0x63, 0x64}}, + {4, {0x61, 0x87, 0x63, 0x64}}, + {4, {0x61, 0x88, 0x63, 0x64}}, + {4, {0x61, 0x89, 0x63, 0x64}}, + {4, {0x61, 0x8A, 0x63, 0x64}}, + {4, {0x61, 0x8B, 0x63, 0x64}}, + {4, {0x61, 0x8C, 0x63, 0x64}}, + {4, {0x61, 0x8D, 0x63, 0x64}}, + {4, {0x61, 0x8E, 0x63, 0x64}}, + {4, {0x61, 0x8F, 0x63, 0x64}}, + {4, {0x61, 0x90, 0x63, 0x64}}, + {4, {0x61, 0x91, 0x63, 0x64}}, + {4, {0x61, 0x92, 0x63, 0x64}}, + {4, {0x61, 0x93, 0x63, 0x64}}, + {4, {0x61, 0x94, 0x63, 0x64}}, + {4, {0x61, 0x95, 0x63, 0x64}}, + {4, {0x61, 0x96, 0x63, 0x64}}, + {4, {0x61, 0x97, 0x63, 0x64}}, + {4, {0x61, 0x98, 0x63, 0x64}}, + {4, {0x61, 0x99, 0x63, 0x64}}, + {4, {0x61, 0x9A, 0x63, 0x64}}, + {4, {0x61, 0x9B, 0x63, 0x64}}, + {4, {0x61, 0x9C, 0x63, 0x64}}, + {4, {0x61, 0x9D, 0x63, 0x64}}, + {4, {0x61, 0x9E, 0x63, 0x64}}, + {4, {0x61, 0x9F, 0x63, 0x64}}, + {4, {0x61, 0xA0, 0x63, 0x64}}, + {4, {0x61, 0xA1, 0x63, 0x64}}, + {4, {0x61, 0xA2, 0x63, 0x64}}, + {4, {0x61, 0xA3, 0x63, 0x64}}, + {4, {0x61, 0xA4, 0x63, 0x64}}, + {4, {0x61, 0xA5, 0x63, 0x64}}, + {4, {0x61, 0xA6, 0x63, 0x64}}, + {4, {0x61, 0xA7, 0x63, 0x64}}, + {4, {0x61, 0xA8, 0x63, 0x64}}, + {4, {0x61, 0xA9, 0x63, 0x64}}, + {4, {0x61, 0xAA, 0x63, 0x64}}, + {4, {0x61, 0xAB, 0x63, 0x64}}, + {4, {0x61, 0xAC, 0x63, 0x64}}, + {4, {0x61, 0xAD, 0x63, 0x64}}, + {4, {0x61, 0xAE, 0x63, 0x64}}, + {4, {0x61, 0xAF, 0x63, 0x64}}, + {4, {0x61, 0xB0, 0x63, 0x64}}, + {4, {0x61, 0xB1, 0x63, 0x64}}, + {4, {0x61, 0xB2, 0x63, 0x64}}, + {4, {0x61, 0xB3, 0x63, 0x64}}, + {4, {0x61, 0xB4, 0x63, 0x64}}, + {4, {0x61, 0xB5, 0x63, 0x64}}, + {4, {0x61, 0xB6, 0x63, 0x64}}, + {4, {0x61, 0xB7, 0x63, 0x64}}, + {4, {0x61, 0xB8, 0x63, 0x64}}, + {4, {0x61, 0xB9, 0x63, 0x64}}, + {4, {0x61, 0xBA, 0x63, 0x64}}, + {4, {0x61, 0xBB, 0x63, 0x64}}, + {4, {0x61, 0xBC, 0x63, 0x64}}, + {4, {0x61, 0xBD, 0x63, 0x64}}, + {4, {0x61, 0xBE, 0x63, 0x64}}, + {4, {0x61, 0xBF, 0x63, 0x64}}, + {4, {0x61, 0xC0, 0x63, 0x64}}, + {4, {0x61, 0xC1, 0x63, 0x64}}, + {4, {0x61, 0xC2, 0x63, 0x64}}, + {4, {0x61, 0xC3, 0x63, 0x64}}, + {4, {0x61, 0xC4, 0x63, 0x64}}, + {4, {0x61, 0xC5, 0x63, 0x64}}, + {4, {0x61, 0xC6, 0x63, 0x64}}, + {4, {0x61, 0xC7, 0x63, 0x64}}, + {4, {0x61, 0xC8, 0x63, 0x64}}, + {4, {0x61, 0xC9, 0x63, 0x64}}, + {4, {0x61, 0xCA, 0x63, 0x64}}, + {4, {0x61, 0xCB, 0x63, 0x64}}, + {4, {0x61, 0xCC, 0x63, 0x64}}, + {4, {0x61, 0xCD, 0x63, 0x64}}, + {4, {0x61, 0xCE, 0x63, 0x64}}, + {4, {0x61, 0xCF, 0x63, 0x64}}, + {4, {0x61, 0xD0, 0x63, 0x64}}, + {4, {0x61, 0xD1, 0x63, 0x64}}, + {4, {0x61, 0xD2, 0x63, 0x64}}, + {4, {0x61, 0xD3, 0x63, 0x64}}, + {4, {0x61, 0xD4, 0x63, 0x64}}, + {4, {0x61, 0xD5, 0x63, 0x64}}, + {4, {0x61, 0xD6, 0x63, 0x64}}, + {4, {0x61, 0xD7, 0x63, 0x64}}, + {4, {0x61, 0xD8, 0x63, 0x64}}, + {4, {0x61, 0xD9, 0x63, 0x64}}, + {4, {0x61, 0xDA, 0x63, 0x64}}, + {4, {0x61, 0xDB, 0x63, 0x64}}, + {4, {0x61, 0xDC, 0x63, 0x64}}, + {4, {0x61, 0xDD, 0x63, 0x64}}, + {4, {0x61, 0xDE, 0x63, 0x64}}, + {4, {0x61, 0xDF, 0x63, 0x64}}, + {4, {0x61, 0xE0, 0x63, 0x64}}, + {4, {0x61, 0xE1, 0x63, 0x64}}, + {4, {0x61, 0xE2, 0x63, 0x64}}, + {4, {0x61, 0xE3, 0x63, 0x64}}, + {4, {0x61, 0xE4, 0x63, 0x64}}, + {4, {0x61, 0xE5, 0x63, 0x64}}, + {4, {0x61, 0xE6, 0x63, 0x64}}, + {4, {0x61, 0xE7, 0x63, 0x64}}, + {4, {0x61, 0xE8, 0x63, 0x64}}, + {4, {0x61, 0xE9, 0x63, 0x64}}, + {4, {0x61, 0xEA, 0x63, 0x64}}, + {4, {0x61, 0xEB, 0x63, 0x64}}, + {4, {0x61, 0xEC, 0x63, 0x64}}, + {4, {0x61, 0xED, 0x63, 0x64}}, + {4, {0x61, 0xEE, 0x63, 0x64}}, + {4, {0x61, 0xEF, 0x63, 0x64}}, + {4, {0x61, 0xF0, 0x63, 0x64}}, + {4, {0x61, 0xF1, 0x63, 0x64}}, + {4, {0x61, 0xF2, 0x63, 0x64}}, + {4, {0x61, 0xF3, 0x63, 0x64}}, + {4, {0x61, 0xF4, 0x63, 0x64}}, + {4, {0x61, 0xF5, 0x63, 0x64}}, + {4, {0x61, 0xF6, 0x63, 0x64}}, + {4, {0x61, 0xF7, 0x63, 0x64}}, + {4, {0x61, 0xF8, 0x63, 0x64}}, + {4, {0x61, 0xF9, 0x63, 0x64}}, + {4, {0x61, 0xFA, 0x63, 0x64}}, + {4, {0x61, 0xFB, 0x63, 0x64}}, + {4, {0x61, 0xFC, 0x63, 0x64}}, + {4, {0x61, 0xFD, 0x63, 0x64}}, + {4, {0x61, 0xFE, 0x63, 0x64}}, + {4, {0x61, 0xFF, 0x63, 0x64}}, + }; + + struct atom atoms_61_62_0X_64[] = { + {4, {0x61, 0x62, 0x00, 0x64}}, + {4, {0x61, 0x62, 0x01, 0x64}}, + {4, {0x61, 0x62, 0x02, 0x64}}, + {4, {0x61, 0x62, 0x03, 0x64}}, + {4, {0x61, 0x62, 0x04, 0x64}}, + {4, {0x61, 0x62, 0x05, 0x64}}, + {4, {0x61, 0x62, 0x06, 0x64}}, + {4, {0x61, 0x62, 0x07, 0x64}}, + {4, {0x61, 0x62, 0x08, 0x64}}, + {4, {0x61, 0x62, 0x09, 0x64}}, + {4, {0x61, 0x62, 0x0A, 0x64}}, + {4, {0x61, 0x62, 0x0B, 0x64}}, + {4, {0x61, 0x62, 0x0C, 0x64}}, + {4, {0x61, 0x62, 0x0D, 0x64}}, + {4, {0x61, 0x62, 0x0E, 0x64}}, + {4, {0x61, 0x62, 0x0F, 0x64}}, + }; + + struct atom atoms_61_6X_63[] = { + {3, {0x61, 0x60, 0x63}}, + {3, {0x61, 0x61, 0x63}}, + {3, {0x61, 0x62, 0x63}}, + {3, {0x61, 0x63, 0x63}}, + {3, {0x61, 0x64, 0x63}}, + {3, {0x61, 0x65, 0x63}}, + {3, {0x61, 0x66, 0x63}}, + {3, {0x61, 0x67, 0x63}}, + {3, {0x61, 0x68, 0x63}}, + {3, {0x61, 0x69, 0x63}}, + {3, {0x61, 0x6A, 0x63}}, + {3, {0x61, 0x6B, 0x63}}, + {3, {0x61, 0x6C, 0x63}}, + {3, {0x61, 0x6D, 0x63}}, + {3, {0x61, 0x6E, 0x63}}, + {3, {0x61, 0x6F, 0x63}}, + }; + + assert_re_atoms("abcd", 1, (struct atom[]) { + {4, {0x61, 0x62, 0x63, 0x64}}, + }); + + assert_re_atoms("abcd1234", 1, (struct atom[]) { + {4, {0x31, 0x32, 0x33, 0x34}}, + }); + + assert_re_atoms("a..d", 1, (struct atom[]) { + {1, {0x61}}, + }); + + assert_re_atoms("a..de", 1, (struct atom[]) { + {2, {0x64, 0x65}}, + }); + + assert_re_atoms("abcd.efgh", 1, (struct atom[]) { + {4, {0x61, 0x62, 0x63, 0x64}}, + }); + + assert_re_atoms("(abcd|efgh)", 2, (struct atom[]) { + {4, {0x65, 0x66, 0x67, 0x68}}, + {4, {0x61, 0x62, 0x63, 0x64}}, + }); + + assert_re_atoms("(abcd|efgh|ij)", 3, (struct atom[]) { + {2, {0x69, 0x6A}}, + {4, {0x65, 0x66, 0x67, 0x68}}, + {4, {0x61, 0x62, 0x63, 0x64}}, + }); + + assert_re_atoms("a.cd", 256, atoms_61_XX_63_64); + + assert_hex_atoms("{61 62 63 64}", 1, (struct atom[]) { + {4, {0x61, 0x62, 0x63, 0x64}}, + }); + + assert_hex_atoms("{61 62 63 64 [1-5] 65 66 67 68}", 1, (struct atom[]) { + {4, {0x61, 0x62, 0x63, 0x64}}, + }); + + assert_hex_atoms("{61 62 63 [1-5] 65 66 67 68}", 1, (struct atom[]) { + {4, {0x65, 0x66, 0x67, 0x68}}, + }); + + assert_hex_atoms("{61 62 63 [1-5] 65 66 }", 1, (struct atom[]) { + {3, {0x61, 0x62, 0x63}}, + }); + + assert_hex_atoms("{61 6? 63 [1-5] 65 66 }", 16, atoms_61_6X_63); + + assert_hex_atoms("{(61 62 63 | 65 66 67 68)}", 2, (struct atom[]) { + {4, {0x65, 0x66, 0x67, 0x68}}, + {3, {0x61, 0x62, 0x63}}, + }); + + assert_hex_atoms("{61 62 0? 64}", 16, atoms_61_62_0X_64); + + assert_hex_atoms("{11 ?? 11 ?? 22 33 44 55 66 }", 1, (struct atom[]) { + {4, {0x22, 0x33, 0x44, 0x55}}, + }); } int main(int argc, char** argv) { - test_quality_quality(); + yr_initialize(); + + test_table_quality(); + test_heuristic_quality(); + test_atom_choose(); } diff --git a/tests/test-rules.c b/tests/test-rules.c index 9459c25a..43447b9e 100644 --- a/tests/test-rules.c +++ b/tests/test-rules.c @@ -1142,6 +1142,7 @@ void test_re() assert_true_regexp("(a+|b)*", "ab", "ab"); assert_true_regexp("a|b|c|d|e", "e", "e"); assert_true_regexp("(a|b|c|d|e)f", "ef", "ef"); + assert_true_regexp("a|b", "a", "a"); assert_true_regexp(".b{2}", "abb", "abb"); assert_true_regexp(".b{2,3}", "abbb", "abbb"); assert_true_regexp(".b{2,3}?", "abbb", "abb"); @@ -1899,11 +1900,6 @@ void test_performance_warnings() strings: $a = { 01 ?? } \ condition: $a }") - assert_warning( - "rule test { \ - strings: $a = { 01 00 } \ - condition: $a }") - assert_warning( "rule test { \ strings: $a = { 01 ?? ?? } \ @@ -1914,6 +1910,31 @@ void test_performance_warnings() strings: $a = { 01 ?? ?? 02 } \ condition: $a }") + assert_warning( + "rule test { \ + strings: $a = { 01 ?? ?2 03 } \ + condition: $a }") + + assert_warning( + "rule test { \ + strings: $a = { 01 ?? 02 1? } \ + condition: $a }") + + assert_warning( + "rule test { \ + strings: $a = { 1? 2? 3? } \ + condition: $a }") + + assert_warning( + "rule test { \ + strings: $a = { 1? 2? 3? 04 } \ + condition: $a }") + + assert_warning( + "rule test { \ + strings: $a = { 1? ?? 03 } \ + condition: $a }") + assert_warning( "rule test { \ strings: $a = { 00 01 } \ @@ -1934,24 +1955,29 @@ void test_performance_warnings() strings: $a = { 00 00 00 } \ condition: $a }") + assert_no_warnings( + "rule test { \ + strings: $a = { 00 00 01 } \ + condition: $a }") + assert_warning( "rule test { \ strings: $a = { 00 00 00 00 } \ condition: $a }") - assert_warning( + assert_no_warnings( "rule test { \ strings: $a = { 00 00 00 01 } \ condition: $a }") assert_warning( "rule test { \ - strings: $a = { 00 00 01 02 } \ + strings: $a = { FF FF FF FF } \ condition: $a }") - assert_warning( + assert_no_warnings( "rule test { \ - strings: $a = { FF FF FF FF } \ + strings: $a = { 00 00 01 02 } \ condition: $a }") assert_no_warnings( @@ -1978,6 +2004,16 @@ void test_performance_warnings() "rule test { \ strings: $a = { 01 02 } \ condition: $a }") + + assert_no_warnings( + "rule test { \ + strings: $a = \"foo\" wide \ + condition: $a }") + + assert_no_warnings( + "rule test { \ + strings: $a = \"MZ\" \ + condition: $a }") } @@ -2023,7 +2059,7 @@ int main(int argc, char** argv) test_global_rules(); #if !defined(USE_WINDOWS_PROC) && !defined(USE_NO_PROC) - test_process_scan(); + //test_process_scan(); #endif #if defined(HASH_MODULE) diff --git a/tests/test-stack.c b/tests/test-stack.c new file mode 100644 index 00000000..ee4ce060 --- /dev/null +++ b/tests/test-stack.c @@ -0,0 +1,81 @@ +/* +Copyright (c) 2018. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + +#include +#include +#include "util.h" + + +int main(int argc, char** argv) +{ + YR_STACK* stack; + + int item; + + yr_initialize(); + yr_stack_create(1, sizeof(item), &stack); + + item = 1; + + if (yr_stack_push(stack, &item) != ERROR_SUCCESS) + exit(EXIT_FAILURE); + + item = 2; + + if (yr_stack_push(stack, &item) != ERROR_SUCCESS) + exit(EXIT_FAILURE); + + item = 3; + + if (yr_stack_push(stack, &item) != ERROR_SUCCESS) + exit(EXIT_FAILURE); + + item = 4; + + if (yr_stack_push(stack, &item) != ERROR_SUCCESS) + exit(EXIT_FAILURE); + + if (!yr_stack_pop(stack, &item) || item != 4) + exit(EXIT_FAILURE); + + if (!yr_stack_pop(stack, &item) || item != 3) + exit(EXIT_FAILURE); + + if (!yr_stack_pop(stack, &item) || item != 2) + exit(EXIT_FAILURE); + + if (!yr_stack_pop(stack, &item) || item != 1) + exit(EXIT_FAILURE); + + if (yr_stack_pop(stack, &item) || item != 1) + exit(EXIT_FAILURE); + + yr_stack_destroy(stack); +} diff --git a/tests/util.c b/tests/util.c index 2cb164de..8e71bd1a 100644 --- a/tests/util.c +++ b/tests/util.c @@ -35,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include "util.h" char compile_error[1024]; int warnings; @@ -260,3 +261,64 @@ _exit: close(fd); return rc; } + + +void _assert_atoms( + RE_AST* re_ast, + int expected_atom_count, + atom* expected_atoms) +{ + YR_ATOMS_CONFIG c; + YR_ATOM_LIST_ITEM* atoms; + YR_ATOM_LIST_ITEM* atom; + + int min_atom_quality; + + c.get_atom_quality = yr_atoms_heuristic_quality; + + yr_atoms_extract_from_re(&c, re_ast, 0, &atoms, &min_atom_quality); + + atom = atoms; + + while (atom != NULL) + { + if (expected_atom_count == 0) + exit(EXIT_FAILURE); + + if (atom->atom.length != expected_atoms->length || + memcmp(atom->atom.bytes, expected_atoms->data, atom->atom.length) != 0) + { + exit(EXIT_FAILURE); + } + + expected_atoms++; + expected_atom_count--; + atom = atom->next; + } +} + + +void assert_re_atoms( + char* re, + int expected_atom_count, + atom* expected_atoms) +{ + RE_AST* re_ast; + RE_ERROR re_error; + + yr_re_parse(re, &re_ast, &re_error); + _assert_atoms(re_ast, expected_atom_count, expected_atoms); +} + + +void assert_hex_atoms( + char* hex, + int expected_atom_count, + atom* expected_atoms) +{ + RE_AST* re_ast; + RE_ERROR re_error; + + yr_re_parse_hex(hex, &re_ast, &re_error); + _assert_atoms(re_ast, expected_atom_count, expected_atoms); +} \ No newline at end of file diff --git a/tests/util.h b/tests/util.h index d0382477..948d6d34 100644 --- a/tests/util.h +++ b/tests/util.h @@ -71,6 +71,27 @@ int read_file( char* filename, char** buf); +typedef struct atom atom; + +struct atom +{ + uint8_t length; + uint8_t data[YR_MAX_ATOM_LENGTH]; +}; + + +void assert_re_atoms( + char* re, + int expected_atom_count, + atom* expected_atoms); + + +void assert_hex_atoms( + char* hex, + int expected_atom_count, + atom* expected_atoms); + + #define assert_true_expr(expr) \ do { \ if (!(expr)) { \ diff --git a/windows/vs2015/libyara/libyara.vcxproj b/windows/vs2015/libyara/libyara.vcxproj index 3e3e261c..817de5e1 100644 --- a/windows/vs2015/libyara/libyara.vcxproj +++ b/windows/vs2015/libyara/libyara.vcxproj @@ -257,6 +257,7 @@ +