From 2886e2aa1a554c1cac4e1ad2f143427a803cb978 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Tue, 25 Feb 2020 15:43:10 +0100 Subject: [PATCH] Move code to new arena. --- libyara/ahocorasick.c | 22 +- libyara/arena2.c | 10 + libyara/compiler.c | 28 +-- libyara/exec.c | 37 ++- libyara/grammar.c | 392 +++++++++++++++-------------- libyara/grammar.y | 142 ++++++----- libyara/include/yara/ahocorasick.h | 1 + libyara/include/yara/arena2.h | 4 + libyara/include/yara/compiler.h | 12 +- libyara/include/yara/parser.h | 22 +- libyara/include/yara/types.h | 12 +- libyara/parser.c | 181 ++++++------- 12 files changed, 459 insertions(+), 404 deletions(-) diff --git a/libyara/ahocorasick.c b/libyara/ahocorasick.c index d2dfc7d1..287d6fb3 100644 --- a/libyara/ahocorasick.c +++ b/libyara/ahocorasick.c @@ -32,13 +32,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include +#include #include #include #include - typedef struct _QUEUE_NODE { YR_AC_STATE* value; @@ -681,7 +682,8 @@ static int _yr_ac_build_transition_table( // Prints automaton state for debug purposes. This function is invoked by // yr_ac_print_automaton, is not intended to be used stand-alone. // - +//TODO(vmalvarez): Remove or re-implement +/* static void _yr_ac_print_automaton_state( YR_AC_STATE* state) { @@ -757,7 +759,7 @@ static void _yr_ac_print_automaton_state( _yr_ac_print_automaton_state(child_state); child_state = child_state->siblings; } -} +}*/ // @@ -832,6 +834,7 @@ int yr_ac_automaton_destroy( int yr_ac_add_string( YR_AC_AUTOMATON* automaton, YR_STRING* string, + uint32_t string_idx, YR_ATOM_LIST_ITEM* atom, YR_ARENA* matches_arena) { @@ -873,10 +876,12 @@ int yr_ac_add_string( offsetof(YR_AC_MATCH, next), EOL); + if (result == ERROR_SUCCESS) { new_match->backtrack = state->depth + atom->backtrack; new_match->string = string; + new_match->string_idx = string_idx; new_match->forward_code = atom->forward_code; new_match->backward_code = atom->backward_code; new_match->next = state->matches; @@ -909,6 +914,13 @@ int yr_ac_compile( FAIL_ON_ERROR(_yr_ac_optimize_failure_links(automaton)); FAIL_ON_ERROR(_yr_ac_build_transition_table(automaton)); + /*FAIL_ON_ERROR(yr_arena2_write_data( + arena, + YR_AC_TRANSITION_TABLE, + automaton->t_table, + automaton->tables_size * sizeof(YR_AC_TRANSITION), + NULL));*/ + FAIL_ON_ERROR(yr_arena_reserve_memory( arena, automaton->tables_size * sizeof(tables->transitions[0]) + @@ -967,10 +979,12 @@ int yr_ac_compile( // // Prints automaton for debug purposes. // - +//TODO(vmalvarez): Remove or re-implement +/* void yr_ac_print_automaton(YR_AC_AUTOMATON* automaton) { printf("-------------------------------------------------------\n"); _yr_ac_print_automaton_state(automaton->root); printf("-------------------------------------------------------\n"); } +*/ \ No newline at end of file diff --git a/libyara/arena2.c b/libyara/arena2.c index e9c95242..678c57cb 100644 --- a/libyara/arena2.c +++ b/libyara/arena2.c @@ -345,6 +345,16 @@ void* yr_arena2_get_ptr( } +yr_arena_off_t yr_arena2_get_current_offset( + YR_ARENA2* arena, + int buffer_id) +{ + assert(buffer_id < arena->num_buffers); + + return arena->buffers[buffer_id].used; +} + + int yr_arena2_ptr_to_ref( YR_ARENA2* arena, const void* address, diff --git a/libyara/compiler.c b/libyara/compiler.c index addbb8cb..8ca14434 100644 --- a/libyara/compiler.c +++ b/libyara/compiler.c @@ -214,10 +214,6 @@ YR_API int yr_compiler_create( result = yr_arena2_create( YR_NUM_SECTIONS, 50*1048576, &new_compiler->arena); - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->code_arena); - if (result == ERROR_SUCCESS) result = yr_arena_create( 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->re_code_arena); @@ -255,7 +251,6 @@ YR_API void yr_compiler_destroy( yr_arena2_destroy(compiler->arena); yr_arena_destroy(compiler->compiled_rules_arena); - yr_arena_destroy(compiler->code_arena); yr_arena_destroy(compiler->re_code_arena); yr_arena_destroy(compiler->automaton_arena); yr_arena_destroy(compiler->matches_arena); @@ -663,8 +658,9 @@ static int _yr_compiler_compile_rules( int result; // Write halt instruction at the end of code. - yr_arena_write_data( - compiler->code_arena, + yr_arena2_write_data( + compiler->arena, + YR_CODE_SECTION, &halt, sizeof(uint8_t), NULL); @@ -720,8 +716,8 @@ static int _yr_compiler_compile_rules( rules_file_header->externals_list_head = (YR_EXTERNAL_VARIABLE*) \ yr_arena2_get_ptr(compiler->arena, YR_EXTERNAL_VARIABLES_TABLE, 0); - rules_file_header->code_start = (uint8_t*) yr_arena_base_address( - compiler->code_arena); + rules_file_header->code_start = (uint8_t*) yr_arena2_get_ptr( + compiler->arena, YR_CODE_SECTION, 0); rules_file_header->ac_match_table = tables.matches; rules_file_header->ac_transition_table = tables.transitions; @@ -730,14 +726,6 @@ static int _yr_compiler_compile_rules( if (result == ERROR_SUCCESS) { - result = yr_arena_append( - arena, - compiler->code_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->code_arena = NULL; result = yr_arena_append( arena, compiler->re_code_arena); @@ -796,6 +784,12 @@ static int _yr_compiler_compile_rules( arena, compiler->arena, YR_SZ_POOL); } + if (result == ERROR_SUCCESS) + { + result = yr_arena_append_arena2_buffer( + arena, compiler->arena, YR_CODE_SECTION); + } + if (result == ERROR_SUCCESS) { compiler->compiled_rules_arena = arena; diff --git a/libyara/exec.c b/libyara/exec.c index 3f234e1a..f770e68a 100644 --- a/libyara/exec.c +++ b/libyara/exec.c @@ -179,18 +179,24 @@ static const uint8_t* jmp_if( int condition, const uint8_t* ip) { - const uint8_t* result; + size_t off; if (condition) { - result = *(const uint8_t**)(ip); + // The condition is true, the instruction pointer is incremented in the + // amount specified by the jump's offset. The offset is relative to the + // jump opcode, but now the instruction pointer is pointing past the opcode + // that's why we decrement the offset by 1. + off = *(int32_t*)(ip) - 1; } else { - result = ip + sizeof(uint64_t); + // The condition is false, the execution flow proceeds with the instruction + // right after the jump. + off = sizeof(int32_t); } - return result; + return ip + off; } @@ -356,7 +362,7 @@ int yr_execute_code( uint64_t start_time; #endif - YR_INIT_RULE_ARGS init_rule_args; + uint32_t rule_idx; YR_RULE* current_rule = NULL; YR_RULE* rule; @@ -407,7 +413,10 @@ int yr_execute_code( while(!stop) { + // Read the opcode from the address indicated by the instruction pointer. opcode = *ip; + + // Advance the instruction pointer, which now points past the opcode. ip++; switch(opcode) @@ -776,14 +785,16 @@ int yr_execute_code( break; case OP_INIT_RULE: - memcpy(&init_rule_args, ip, sizeof(init_rule_args)); - - current_rule = &context->rules->rules_list_head[init_rule_args.rule_idx]; - - if (RULE_IS_DISABLED(current_rule)) - ip = init_rule_args.jmp_addr; - else - ip += sizeof(init_rule_args); + // After the opcode there's an int32_t corresponding to the jump's + // offset and an uint32_t corresponding to the rule's index. + rule_idx = *(uint32_t*)(ip + sizeof(int32_t)); + current_rule = &context->rules->rules_list_head[rule_idx]; + // If the rule is disabled let's skip its code. + ip = jmp_if(RULE_IS_DISABLED(current_rule), ip); + // Skip the bytes corresponding to the rule's index, but only if not + // taking the jump. + if (!RULE_IS_DISABLED(current_rule)) + ip += sizeof(uint32_t); break; case OP_MATCH_RULE: diff --git a/libyara/grammar.c b/libyara/grammar.c index 9e91fd9a..77cf1d89 100644 --- a/libyara/grammar.c +++ b/libyara/grammar.c @@ -680,13 +680,13 @@ static const yytype_uint16 yyrline[] = 741, 772, 777, 794, 799, 819, 820, 824, 825, 826, 827, 828, 832, 833, 837, 841, 936, 984, 1045, 1092, 1093, 1097, 1132, 1185, 1229, 1249, 1256, 1263, 1275, 1285, - 1299, 1314, 1325, 1396, 1433, 1344, 1546, 1545, 1629, 1635, - 1642, 1641, 1687, 1686, 1730, 1737, 1744, 1751, 1758, 1765, - 1772, 1776, 1784, 1804, 1832, 1906, 1934, 1943, 1952, 1976, - 1991, 2011, 2010, 2016, 2028, 2029, 2034, 2041, 2052, 2056, - 2061, 2070, 2074, 2082, 2094, 2108, 2116, 2123, 2150, 2162, - 2174, 2190, 2202, 2218, 2261, 2282, 2317, 2352, 2386, 2411, - 2428, 2438, 2448, 2458, 2468, 2488, 2508 + 1299, 1314, 1325, 1396, 1433, 1344, 1565, 1564, 1653, 1659, + 1666, 1665, 1711, 1710, 1754, 1761, 1768, 1775, 1782, 1789, + 1796, 1800, 1808, 1828, 1856, 1930, 1958, 1967, 1976, 2000, + 2015, 2035, 2034, 2040, 2052, 2053, 2058, 2065, 2076, 2080, + 2085, 2094, 2098, 2106, 2118, 2132, 2140, 2147, 2174, 2186, + 2198, 2214, 2226, 2242, 2285, 2306, 2341, 2376, 2410, 2435, + 2452, 2462, 2472, 2482, 2492, 2512, 2532 }; #endif @@ -3045,14 +3045,14 @@ yyreduce: YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[compiler->loop_index]; YR_FIXUP* fixup; - uint8_t* loop_start_addr; - void* jmp_arg_addr; + YR_ARENA2_REFERENCE loop_start_ref; + YR_ARENA2_REFERENCE jmp_offset_ref; int var_frame = _yr_compiler_get_var_frame(compiler); int i; fail_if_error(yr_parser_emit( - yyscanner, OP_ITER_NEXT, &loop_start_addr)); + yyscanner, OP_ITER_NEXT, &loop_start_ref)); // For each variable generate an instruction that pops the value from // the stack and store it into one memory slot starting at var_frame + 3 @@ -3064,26 +3064,26 @@ yyreduce: yyscanner, OP_POP_M, var_frame + 3 + i, NULL, NULL)); } - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE_P, - 0, + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_arg_addr)); + &jmp_offset_ref)); - // Push a new fixup entry in the fixup stack so that the jump - // destination is set once we know it. + // We still don't know the jump's target, so we push a fixup entry + // in the stack, so that the jump's offset can be set once we know it. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) fail_if_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_arg_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; - loop_ctx->addr = loop_start_addr; + loop_ctx->start_ref = loop_start_ref; } #line 3089 "grammar.c" /* yacc.c:1663 */ break; @@ -3091,8 +3091,9 @@ yyreduce: case 85: #line 1478 "grammar.y" /* yacc.c:1663 */ { + int32_t jmp_offset; YR_FIXUP* fixup; - uint8_t* pop_addr; + YR_ARENA2_REFERENCE pop_ref; int var_frame = _yr_compiler_get_var_frame(compiler); @@ -3105,10 +3106,14 @@ yyreduce: fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JUNDEF_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); @@ -3118,24 +3123,38 @@ yyreduce: fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JL_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); fail_if_error(yr_parser_emit( - yyscanner, OP_POP, &pop_addr)); + yyscanner, OP_POP, &pop_ref)); - // Pop from the stack the fixup entry containing the jump's address - // that needs to be fixed. + // Pop from the stack the fixup entry containing the reference to + // the jump offset that needs to be fixed. fixup = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup->next; - // Fix the jump's target address. - *(void**)(fixup->address) = (void*)(pop_addr); + // The fixup entry has a reference to the jump offset that need + // to be fixed, convert the address into a pointer. + int32_t* jmp_offset_addr = (int32_t*) yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + // The reference in the fixup entry points to the jump's offset + // but the jump instruction is one byte before, that's why we add + // one to the offset. + jmp_offset = pop_ref.offset - fixup->ref.offset + 1; + + // Fix the jump's offset. + *jmp_offset_addr = jmp_offset; yr_free(fixup); @@ -3157,15 +3176,16 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3161 "grammar.c" /* yacc.c:1663 */ +#line 3180 "grammar.c" /* yacc.c:1663 */ break; case 86: -#line 1546 "grammar.y" /* yacc.c:1663 */ +#line 1565 "grammar.y" /* yacc.c:1663 */ { + YR_ARENA2_REFERENCE ref; + int result = ERROR_SUCCESS; int var_frame; - uint8_t* addr; if (compiler->loop_index + 1 == YR_MAX_LOOP_NESTING) result = ERROR_LOOP_NESTING_LIMIT_EXCEEDED; @@ -3187,18 +3207,18 @@ yyreduce: // Pop the first string. yr_parser_emit_with_arg( - yyscanner, OP_POP_M, var_frame, &addr, NULL); + yyscanner, OP_POP_M, var_frame, &ref, NULL); compiler->loop_for_of_var_index = var_frame; compiler->loop[compiler->loop_index].vars_internal_count = 3; compiler->loop[compiler->loop_index].vars_count = 0; - compiler->loop[compiler->loop_index].addr = addr; + compiler->loop[compiler->loop_index].start_ref = ref; } -#line 3198 "grammar.c" /* yacc.c:1663 */ +#line 3218 "grammar.c" /* yacc.c:1663 */ break; case 87: -#line 1579 "grammar.y" /* yacc.c:1663 */ +#line 1599 "grammar.y" /* yacc.c:1663 */ { int var_frame = 0; @@ -3217,12 +3237,16 @@ yyreduce: yr_parser_emit_with_arg( yyscanner, OP_INCR_M, var_frame + 2, NULL, NULL); + int32_t jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION); + // If next string is not undefined, go back to the // beginning of the loop. - yr_parser_emit_with_arg_reloc( + yr_parser_emit_with_arg_int32( yyscanner, OP_JNUNDEF, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL); @@ -3249,218 +3273,218 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3253 "grammar.c" /* yacc.c:1663 */ +#line 3277 "grammar.c" /* yacc.c:1663 */ break; case 88: -#line 1630 "grammar.y" /* yacc.c:1663 */ +#line 1654 "grammar.y" /* yacc.c:1663 */ { yr_parser_emit(yyscanner, OP_OF, NULL); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3263 "grammar.c" /* yacc.c:1663 */ +#line 3287 "grammar.c" /* yacc.c:1663 */ break; case 89: -#line 1636 "grammar.y" /* yacc.c:1663 */ +#line 1660 "grammar.y" /* yacc.c:1663 */ { yr_parser_emit(yyscanner, OP_NOT, NULL); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3273 "grammar.c" /* yacc.c:1663 */ +#line 3297 "grammar.c" /* yacc.c:1663 */ break; case 90: -#line 1642 "grammar.y" /* yacc.c:1663 */ +#line 1666 "grammar.y" /* yacc.c:1663 */ { YR_FIXUP* fixup; - void* jmp_destination_addr; + YR_ARENA2_REFERENCE jmp_offset_ref; - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JFALSE, - 0, // still don't know the jump destination + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_destination_addr)); + &jmp_offset_ref)); - // create a fixup entry for the jump and push it in the stack + // Create a fixup entry for the jump and push it in the stack. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) fail_if_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_destination_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; } -#line 3299 "grammar.c" /* yacc.c:1663 */ +#line 3323 "grammar.c" /* yacc.c:1663 */ break; case 91: -#line 1664 "grammar.y" /* yacc.c:1663 */ +#line 1688 "grammar.y" /* yacc.c:1663 */ { YR_FIXUP* fixup; - uint8_t* nop_addr; fail_if_error(yr_parser_emit(yyscanner, OP_AND, NULL)); - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_AND instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*) nop_addr; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + int32_t jmp_offset = \ + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3326 "grammar.c" /* yacc.c:1663 */ +#line 3350 "grammar.c" /* yacc.c:1663 */ break; case 92: -#line 1687 "grammar.y" /* yacc.c:1663 */ +#line 1711 "grammar.y" /* yacc.c:1663 */ { YR_FIXUP* fixup; - void* jmp_destination_addr; + YR_ARENA2_REFERENCE jmp_offset_ref; - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE, - 0, // still don't know the jump destination + 0, // still don't know the jump destination, use 0 for now. NULL, - &jmp_destination_addr)); + &jmp_offset_ref)); fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) fail_if_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_destination_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; } -#line 3351 "grammar.c" /* yacc.c:1663 */ +#line 3375 "grammar.c" /* yacc.c:1663 */ break; case 93: -#line 1708 "grammar.y" /* yacc.c:1663 */ +#line 1732 "grammar.y" /* yacc.c:1663 */ { YR_FIXUP* fixup; - uint8_t* nop_addr; fail_if_error(yr_parser_emit(yyscanner, OP_OR, NULL)); - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_OR instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*)(nop_addr); + + int32_t jmp_offset = \ + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3378 "grammar.c" /* yacc.c:1663 */ +#line 3402 "grammar.c" /* yacc.c:1663 */ break; case 94: -#line 1731 "grammar.y" /* yacc.c:1663 */ +#line 1755 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "<", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3389 "grammar.c" /* yacc.c:1663 */ +#line 3413 "grammar.c" /* yacc.c:1663 */ break; case 95: -#line 1738 "grammar.y" /* yacc.c:1663 */ +#line 1762 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, ">", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3400 "grammar.c" /* yacc.c:1663 */ +#line 3424 "grammar.c" /* yacc.c:1663 */ break; case 96: -#line 1745 "grammar.y" /* yacc.c:1663 */ +#line 1769 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "<=", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3411 "grammar.c" /* yacc.c:1663 */ +#line 3435 "grammar.c" /* yacc.c:1663 */ break; case 97: -#line 1752 "grammar.y" /* yacc.c:1663 */ +#line 1776 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, ">=", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3422 "grammar.c" /* yacc.c:1663 */ +#line 3446 "grammar.c" /* yacc.c:1663 */ break; case 98: -#line 1759 "grammar.y" /* yacc.c:1663 */ +#line 1783 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "==", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3433 "grammar.c" /* yacc.c:1663 */ +#line 3457 "grammar.c" /* yacc.c:1663 */ break; case 99: -#line 1766 "grammar.y" /* yacc.c:1663 */ +#line 1790 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "!=", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3444 "grammar.c" /* yacc.c:1663 */ +#line 3468 "grammar.c" /* yacc.c:1663 */ break; case 100: -#line 1773 "grammar.y" /* yacc.c:1663 */ +#line 1797 "grammar.y" /* yacc.c:1663 */ { (yyval.expression) = (yyvsp[0].expression); } -#line 3452 "grammar.c" /* yacc.c:1663 */ +#line 3476 "grammar.c" /* yacc.c:1663 */ break; case 101: -#line 1777 "grammar.y" /* yacc.c:1663 */ +#line 1801 "grammar.y" /* yacc.c:1663 */ { (yyval.expression) = (yyvsp[-1].expression); } -#line 3460 "grammar.c" /* yacc.c:1663 */ +#line 3484 "grammar.c" /* yacc.c:1663 */ break; case 102: -#line 1785 "grammar.y" /* yacc.c:1663 */ +#line 1809 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -3480,11 +3504,11 @@ yyreduce: assert(loop_ctx->vars_count <= YR_MAX_LOOP_VARS); } -#line 3484 "grammar.c" /* yacc.c:1663 */ +#line 3508 "grammar.c" /* yacc.c:1663 */ break; case 103: -#line 1805 "grammar.y" /* yacc.c:1663 */ +#line 1829 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -3509,11 +3533,11 @@ yyreduce: loop_ctx->vars[loop_ctx->vars_count++].identifier.ptr = (yyvsp[0].c_string); } -#line 3513 "grammar.c" /* yacc.c:1663 */ +#line 3537 "grammar.c" /* yacc.c:1663 */ break; case 104: -#line 1833 "grammar.y" /* yacc.c:1663 */ +#line 1857 "grammar.y" /* yacc.c:1663 */ { YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[compiler->loop_index]; @@ -3587,11 +3611,11 @@ yyreduce: fail_if_error(result); } -#line 3591 "grammar.c" /* yacc.c:1663 */ +#line 3615 "grammar.c" /* yacc.c:1663 */ break; case 105: -#line 1907 "grammar.y" /* yacc.c:1663 */ +#line 1931 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -3615,11 +3639,11 @@ yyreduce: fail_if_error(result); } -#line 3619 "grammar.c" /* yacc.c:1663 */ +#line 3643 "grammar.c" /* yacc.c:1663 */ break; case 106: -#line 1935 "grammar.y" /* yacc.c:1663 */ +#line 1959 "grammar.y" /* yacc.c:1663 */ { // $2 contains the number of integers in the enumeration fail_if_error(yr_parser_emit_with_arg( @@ -3628,20 +3652,20 @@ yyreduce: fail_if_error(yr_parser_emit( yyscanner, OP_ITER_START_INT_ENUM, NULL)); } -#line 3632 "grammar.c" /* yacc.c:1663 */ +#line 3656 "grammar.c" /* yacc.c:1663 */ break; case 107: -#line 1944 "grammar.y" /* yacc.c:1663 */ +#line 1968 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_emit( yyscanner, OP_ITER_START_INT_RANGE, NULL)); } -#line 3641 "grammar.c" /* yacc.c:1663 */ +#line 3665 "grammar.c" /* yacc.c:1663 */ break; case 108: -#line 1953 "grammar.y" /* yacc.c:1663 */ +#line 1977 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -3661,11 +3685,11 @@ yyreduce: fail_if_error(result); } -#line 3665 "grammar.c" /* yacc.c:1663 */ +#line 3689 "grammar.c" /* yacc.c:1663 */ break; case 109: -#line 1977 "grammar.y" /* yacc.c:1663 */ +#line 2001 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -3680,11 +3704,11 @@ yyreduce: (yyval.integer) = 1; } -#line 3684 "grammar.c" /* yacc.c:1663 */ +#line 3708 "grammar.c" /* yacc.c:1663 */ break; case 110: -#line 1992 "grammar.y" /* yacc.c:1663 */ +#line 2016 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -3699,20 +3723,20 @@ yyreduce: (yyval.integer) = (yyvsp[-2].integer) + 1; } -#line 3703 "grammar.c" /* yacc.c:1663 */ +#line 3727 "grammar.c" /* yacc.c:1663 */ break; case 111: -#line 2011 "grammar.y" /* yacc.c:1663 */ +#line 2035 "grammar.y" /* yacc.c:1663 */ { // Push end-of-list marker yr_parser_emit_with_arg(yyscanner, OP_PUSH, UNDEFINED, NULL, NULL); } -#line 3712 "grammar.c" /* yacc.c:1663 */ +#line 3736 "grammar.c" /* yacc.c:1663 */ break; case 113: -#line 2017 "grammar.y" /* yacc.c:1663 */ +#line 2041 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH, UNDEFINED, NULL, NULL)); @@ -3720,67 +3744,67 @@ yyreduce: fail_if_error(yr_parser_emit_pushes_for_strings( yyscanner, "$*")); } -#line 3724 "grammar.c" /* yacc.c:1663 */ +#line 3748 "grammar.c" /* yacc.c:1663 */ break; case 116: -#line 2035 "grammar.y" /* yacc.c:1663 */ +#line 2059 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_emit_pushes_for_strings(yyscanner, (yyvsp[0].c_string)); yr_free((yyvsp[0].c_string)); fail_if_error(result); } -#line 3735 "grammar.c" /* yacc.c:1663 */ +#line 3759 "grammar.c" /* yacc.c:1663 */ break; case 117: -#line 2042 "grammar.y" /* yacc.c:1663 */ +#line 2066 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_emit_pushes_for_strings(yyscanner, (yyvsp[0].c_string)); yr_free((yyvsp[0].c_string)); fail_if_error(result); } -#line 3746 "grammar.c" /* yacc.c:1663 */ +#line 3770 "grammar.c" /* yacc.c:1663 */ break; case 118: -#line 2053 "grammar.y" /* yacc.c:1663 */ +#line 2077 "grammar.y" /* yacc.c:1663 */ { (yyval.integer) = FOR_EXPRESSION_ANY; } -#line 3754 "grammar.c" /* yacc.c:1663 */ +#line 3778 "grammar.c" /* yacc.c:1663 */ break; case 119: -#line 2057 "grammar.y" /* yacc.c:1663 */ +#line 2081 "grammar.y" /* yacc.c:1663 */ { yr_parser_emit_with_arg(yyscanner, OP_PUSH, UNDEFINED, NULL, NULL); (yyval.integer) = FOR_EXPRESSION_ALL; } -#line 3763 "grammar.c" /* yacc.c:1663 */ +#line 3787 "grammar.c" /* yacc.c:1663 */ break; case 120: -#line 2062 "grammar.y" /* yacc.c:1663 */ +#line 2086 "grammar.y" /* yacc.c:1663 */ { yr_parser_emit_with_arg(yyscanner, OP_PUSH, 1, NULL, NULL); (yyval.integer) = FOR_EXPRESSION_ANY; } -#line 3772 "grammar.c" /* yacc.c:1663 */ +#line 3796 "grammar.c" /* yacc.c:1663 */ break; case 121: -#line 2071 "grammar.y" /* yacc.c:1663 */ +#line 2095 "grammar.y" /* yacc.c:1663 */ { (yyval.expression) = (yyvsp[-1].expression); } -#line 3780 "grammar.c" /* yacc.c:1663 */ +#line 3804 "grammar.c" /* yacc.c:1663 */ break; case 122: -#line 2075 "grammar.y" /* yacc.c:1663 */ +#line 2099 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_emit( yyscanner, OP_FILESIZE, NULL)); @@ -3788,11 +3812,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3792 "grammar.c" /* yacc.c:1663 */ +#line 3816 "grammar.c" /* yacc.c:1663 */ break; case 123: -#line 2083 "grammar.y" /* yacc.c:1663 */ +#line 2107 "grammar.y" /* yacc.c:1663 */ { yywarning(yyscanner, "Using deprecated \"entrypoint\" keyword. Use the \"entry_point\" " @@ -3804,11 +3828,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3808 "grammar.c" /* yacc.c:1663 */ +#line 3832 "grammar.c" /* yacc.c:1663 */ break; case 124: -#line 2095 "grammar.y" /* yacc.c:1663 */ +#line 2119 "grammar.y" /* yacc.c:1663 */ { check_type((yyvsp[-1].expression), EXPRESSION_TYPE_INTEGER, "intXXXX or uintXXXX"); @@ -3822,11 +3846,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3826 "grammar.c" /* yacc.c:1663 */ +#line 3850 "grammar.c" /* yacc.c:1663 */ break; case 125: -#line 2109 "grammar.y" /* yacc.c:1663 */ +#line 2133 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH, (yyvsp[0].integer), NULL, NULL)); @@ -3834,22 +3858,22 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = (yyvsp[0].integer); } -#line 3838 "grammar.c" /* yacc.c:1663 */ +#line 3862 "grammar.c" /* yacc.c:1663 */ break; case 126: -#line 2117 "grammar.y" /* yacc.c:1663 */ +#line 2141 "grammar.y" /* yacc.c:1663 */ { fail_if_error(yr_parser_emit_with_arg_double( yyscanner, OP_PUSH, (yyvsp[0].double_), NULL, NULL)); (yyval.expression).type = EXPRESSION_TYPE_FLOAT; } -#line 3849 "grammar.c" /* yacc.c:1663 */ +#line 3873 "grammar.c" /* yacc.c:1663 */ break; case 127: -#line 2124 "grammar.y" /* yacc.c:1663 */ +#line 2148 "grammar.y" /* yacc.c:1663 */ { YR_ARENA2_REFERENCE ref; @@ -3876,11 +3900,11 @@ yyreduce: //TODO // $$.value.sized_string = sized_string; } -#line 3880 "grammar.c" /* yacc.c:1663 */ +#line 3904 "grammar.c" /* yacc.c:1663 */ break; case 128: -#line 2151 "grammar.y" /* yacc.c:1663 */ +#line 2175 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_string_identifier( yyscanner, (yyvsp[0].c_string), OP_COUNT, UNDEFINED); @@ -3892,11 +3916,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3896 "grammar.c" /* yacc.c:1663 */ +#line 3920 "grammar.c" /* yacc.c:1663 */ break; case 129: -#line 2163 "grammar.y" /* yacc.c:1663 */ +#line 2187 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_string_identifier( yyscanner, (yyvsp[-3].c_string), OP_OFFSET, UNDEFINED); @@ -3908,11 +3932,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3912 "grammar.c" /* yacc.c:1663 */ +#line 3936 "grammar.c" /* yacc.c:1663 */ break; case 130: -#line 2175 "grammar.y" /* yacc.c:1663 */ +#line 2199 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_emit_with_arg( yyscanner, OP_PUSH, 1, NULL, NULL); @@ -3928,11 +3952,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3932 "grammar.c" /* yacc.c:1663 */ +#line 3956 "grammar.c" /* yacc.c:1663 */ break; case 131: -#line 2191 "grammar.y" /* yacc.c:1663 */ +#line 2215 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_string_identifier( yyscanner, (yyvsp[-3].c_string), OP_LENGTH, UNDEFINED); @@ -3944,11 +3968,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3948 "grammar.c" /* yacc.c:1663 */ +#line 3972 "grammar.c" /* yacc.c:1663 */ break; case 132: -#line 2203 "grammar.y" /* yacc.c:1663 */ +#line 2227 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_emit_with_arg( yyscanner, OP_PUSH, 1, NULL, NULL); @@ -3964,11 +3988,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = UNDEFINED; } -#line 3968 "grammar.c" /* yacc.c:1663 */ +#line 3992 "grammar.c" /* yacc.c:1663 */ break; case 133: -#line 2219 "grammar.y" /* yacc.c:1663 */ +#line 2243 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -4011,11 +4035,11 @@ yyreduce: fail_if_error(result); } -#line 4015 "grammar.c" /* yacc.c:1663 */ +#line 4039 "grammar.c" /* yacc.c:1663 */ break; case 134: -#line 2262 "grammar.y" /* yacc.c:1663 */ +#line 2286 "grammar.y" /* yacc.c:1663 */ { int result = ERROR_SUCCESS; @@ -4036,11 +4060,11 @@ yyreduce: fail_if_error(result); } -#line 4040 "grammar.c" /* yacc.c:1663 */ +#line 4064 "grammar.c" /* yacc.c:1663 */ break; case 135: -#line 2283 "grammar.y" /* yacc.c:1663 */ +#line 2307 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_operation( yyscanner, "+", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -4075,11 +4099,11 @@ yyreduce: fail_if_error(result); } -#line 4079 "grammar.c" /* yacc.c:1663 */ +#line 4103 "grammar.c" /* yacc.c:1663 */ break; case 136: -#line 2318 "grammar.y" /* yacc.c:1663 */ +#line 2342 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_operation( yyscanner, "-", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -4114,11 +4138,11 @@ yyreduce: fail_if_error(result); } -#line 4118 "grammar.c" /* yacc.c:1663 */ +#line 4142 "grammar.c" /* yacc.c:1663 */ break; case 137: -#line 2353 "grammar.y" /* yacc.c:1663 */ +#line 2377 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_operation( yyscanner, "*", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -4152,11 +4176,11 @@ yyreduce: fail_if_error(result); } -#line 4156 "grammar.c" /* yacc.c:1663 */ +#line 4180 "grammar.c" /* yacc.c:1663 */ break; case 138: -#line 2387 "grammar.y" /* yacc.c:1663 */ +#line 2411 "grammar.y" /* yacc.c:1663 */ { int result = yr_parser_reduce_operation( yyscanner, "\\", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -4181,11 +4205,11 @@ yyreduce: fail_if_error(result); } -#line 4185 "grammar.c" /* yacc.c:1663 */ +#line 4209 "grammar.c" /* yacc.c:1663 */ break; case 139: -#line 2412 "grammar.y" /* yacc.c:1663 */ +#line 2436 "grammar.y" /* yacc.c:1663 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "%"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "%"); @@ -4202,11 +4226,11 @@ yyreduce: fail_if_error(ERROR_DIVISION_BY_ZERO); } } -#line 4206 "grammar.c" /* yacc.c:1663 */ +#line 4230 "grammar.c" /* yacc.c:1663 */ break; case 140: -#line 2429 "grammar.y" /* yacc.c:1663 */ +#line 2453 "grammar.y" /* yacc.c:1663 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "^"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "^"); @@ -4216,11 +4240,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = OPERATION(^, (yyvsp[-2].expression).value.integer, (yyvsp[0].expression).value.integer); } -#line 4220 "grammar.c" /* yacc.c:1663 */ +#line 4244 "grammar.c" /* yacc.c:1663 */ break; case 141: -#line 2439 "grammar.y" /* yacc.c:1663 */ +#line 2463 "grammar.y" /* yacc.c:1663 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "^"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "^"); @@ -4230,11 +4254,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = OPERATION(&, (yyvsp[-2].expression).value.integer, (yyvsp[0].expression).value.integer); } -#line 4234 "grammar.c" /* yacc.c:1663 */ +#line 4258 "grammar.c" /* yacc.c:1663 */ break; case 142: -#line 2449 "grammar.y" /* yacc.c:1663 */ +#line 2473 "grammar.y" /* yacc.c:1663 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "|"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "|"); @@ -4244,11 +4268,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = OPERATION(|, (yyvsp[-2].expression).value.integer, (yyvsp[0].expression).value.integer); } -#line 4248 "grammar.c" /* yacc.c:1663 */ +#line 4272 "grammar.c" /* yacc.c:1663 */ break; case 143: -#line 2459 "grammar.y" /* yacc.c:1663 */ +#line 2483 "grammar.y" /* yacc.c:1663 */ { check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "~"); @@ -4258,11 +4282,11 @@ yyreduce: (yyval.expression).value.integer = ((yyvsp[0].expression).value.integer == UNDEFINED) ? UNDEFINED : ~((yyvsp[0].expression).value.integer); } -#line 4262 "grammar.c" /* yacc.c:1663 */ +#line 4286 "grammar.c" /* yacc.c:1663 */ break; case 144: -#line 2469 "grammar.y" /* yacc.c:1663 */ +#line 2493 "grammar.y" /* yacc.c:1663 */ { int result; @@ -4282,11 +4306,11 @@ yyreduce: fail_if_error(result); } -#line 4286 "grammar.c" /* yacc.c:1663 */ +#line 4310 "grammar.c" /* yacc.c:1663 */ break; case 145: -#line 2489 "grammar.y" /* yacc.c:1663 */ +#line 2513 "grammar.y" /* yacc.c:1663 */ { int result; @@ -4306,19 +4330,19 @@ yyreduce: fail_if_error(result); } -#line 4310 "grammar.c" /* yacc.c:1663 */ +#line 4334 "grammar.c" /* yacc.c:1663 */ break; case 146: -#line 2509 "grammar.y" /* yacc.c:1663 */ +#line 2533 "grammar.y" /* yacc.c:1663 */ { (yyval.expression) = (yyvsp[0].expression); } -#line 4318 "grammar.c" /* yacc.c:1663 */ +#line 4342 "grammar.c" /* yacc.c:1663 */ break; -#line 4322 "grammar.c" /* yacc.c:1663 */ +#line 4346 "grammar.c" /* yacc.c:1663 */ default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -4546,5 +4570,5 @@ yyreturn: #endif return yyresult; } -#line 2514 "grammar.y" /* yacc.c:1907 */ +#line 2538 "grammar.y" /* yacc.c:1907 */ diff --git a/libyara/grammar.y b/libyara/grammar.y index f7d517b6..9777c4ab 100644 --- a/libyara/grammar.y +++ b/libyara/grammar.y @@ -1379,7 +1379,7 @@ expression // JL_P repeat ; if M[1] is less M[3] repeat // // exit: - // POP ; remove the itertor object from the stack + // POP ; remove the iterator object from the stack // // PUSH_M 0 ; pushes number of true results for // PUSH_M 2 ; pushes value of @@ -1434,14 +1434,14 @@ expression YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[compiler->loop_index]; YR_FIXUP* fixup; - uint8_t* loop_start_addr; - void* jmp_arg_addr; + YR_ARENA2_REFERENCE loop_start_ref; + YR_ARENA2_REFERENCE jmp_offset_ref; int var_frame = _yr_compiler_get_var_frame(compiler); int i; fail_if_error(yr_parser_emit( - yyscanner, OP_ITER_NEXT, &loop_start_addr)); + yyscanner, OP_ITER_NEXT, &loop_start_ref)); // For each variable generate an instruction that pops the value from // the stack and store it into one memory slot starting at var_frame + 3 @@ -1453,31 +1453,32 @@ expression yyscanner, OP_POP_M, var_frame + 3 + i, NULL, NULL)); } - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE_P, - 0, + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_arg_addr)); + &jmp_offset_ref)); - // Push a new fixup entry in the fixup stack so that the jump - // destination is set once we know it. + // We still don't know the jump's target, so we push a fixup entry + // in the stack, so that the jump's offset can be set once we know it. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) fail_if_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_arg_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; - loop_ctx->addr = loop_start_addr; + loop_ctx->start_ref = loop_start_ref; } '(' boolean_expression ')' { + int32_t jmp_offset; YR_FIXUP* fixup; - uint8_t* pop_addr; + YR_ARENA2_REFERENCE pop_ref; int var_frame = _yr_compiler_get_var_frame(compiler); @@ -1490,10 +1491,14 @@ expression fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JUNDEF_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); @@ -1503,24 +1508,38 @@ expression fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JL_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); fail_if_error(yr_parser_emit( - yyscanner, OP_POP, &pop_addr)); + yyscanner, OP_POP, &pop_ref)); - // Pop from the stack the fixup entry containing the jump's address - // that needs to be fixed. + // Pop from the stack the fixup entry containing the reference to + // the jump offset that needs to be fixed. fixup = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup->next; - // Fix the jump's target address. - *(void**)(fixup->address) = (void*)(pop_addr); + // The fixup entry has a reference to the jump offset that need + // to be fixed, convert the address into a pointer. + int32_t* jmp_offset_addr = (int32_t*) yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + // The reference in the fixup entry points to the jump's offset + // but the jump instruction is one byte before, that's why we add + // one to the offset. + jmp_offset = pop_ref.offset - fixup->ref.offset + 1; + + // Fix the jump's offset. + *jmp_offset_addr = jmp_offset; yr_free(fixup); @@ -1544,9 +1563,10 @@ expression } | _FOR_ for_expression _OF_ string_set ':' { + YR_ARENA2_REFERENCE ref; + int result = ERROR_SUCCESS; int var_frame; - uint8_t* addr; if (compiler->loop_index + 1 == YR_MAX_LOOP_NESTING) result = ERROR_LOOP_NESTING_LIMIT_EXCEEDED; @@ -1568,12 +1588,12 @@ expression // Pop the first string. yr_parser_emit_with_arg( - yyscanner, OP_POP_M, var_frame, &addr, NULL); + yyscanner, OP_POP_M, var_frame, &ref, NULL); compiler->loop_for_of_var_index = var_frame; compiler->loop[compiler->loop_index].vars_internal_count = 3; compiler->loop[compiler->loop_index].vars_count = 0; - compiler->loop[compiler->loop_index].addr = addr; + compiler->loop[compiler->loop_index].start_ref = ref; } '(' boolean_expression ')' { @@ -1594,12 +1614,16 @@ expression yr_parser_emit_with_arg( yyscanner, OP_INCR_M, var_frame + 2, NULL, NULL); + int32_t jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION); + // If next string is not undefined, go back to the // beginning of the loop. - yr_parser_emit_with_arg_reloc( + yr_parser_emit_with_arg_int32( yyscanner, OP_JNUNDEF, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL); @@ -1641,43 +1665,43 @@ expression | boolean_expression _AND_ { YR_FIXUP* fixup; - void* jmp_destination_addr; + YR_ARENA2_REFERENCE jmp_offset_ref; - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JFALSE, - 0, // still don't know the jump destination + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_destination_addr)); + &jmp_offset_ref)); - // create a fixup entry for the jump and push it in the stack + // Create a fixup entry for the jump and push it in the stack. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) fail_if_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_destination_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; } boolean_expression { YR_FIXUP* fixup; - uint8_t* nop_addr; fail_if_error(yr_parser_emit(yyscanner, OP_AND, NULL)); - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_AND instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*) nop_addr; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + int32_t jmp_offset = \ + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); @@ -1686,42 +1710,42 @@ expression | boolean_expression _OR_ { YR_FIXUP* fixup; - void* jmp_destination_addr; + YR_ARENA2_REFERENCE jmp_offset_ref; - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE, - 0, // still don't know the jump destination + 0, // still don't know the jump destination, use 0 for now. NULL, - &jmp_destination_addr)); + &jmp_offset_ref)); fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) fail_if_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_destination_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; } boolean_expression { YR_FIXUP* fixup; - uint8_t* nop_addr; fail_if_error(yr_parser_emit(yyscanner, OP_OR, NULL)); - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_OR instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*)(nop_addr); + + int32_t jmp_offset = \ + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); diff --git a/libyara/include/yara/ahocorasick.h b/libyara/include/yara/ahocorasick.h index 43f6dadd..90331fa7 100644 --- a/libyara/include/yara/ahocorasick.h +++ b/libyara/include/yara/ahocorasick.h @@ -62,6 +62,7 @@ int yr_ac_automaton_destroy( int yr_ac_add_string( YR_AC_AUTOMATON* automaton, YR_STRING* string, + uint32_t string_idx, YR_ATOM_LIST_ITEM* atom, YR_ARENA* matches_arena); diff --git a/libyara/include/yara/arena2.h b/libyara/include/yara/arena2.h index 4012501d..c3188634 100644 --- a/libyara/include/yara/arena2.h +++ b/libyara/include/yara/arena2.h @@ -180,6 +180,10 @@ void* yr_arena2_get_ptr( yr_arena_off_t offset); +yr_arena_off_t yr_arena2_get_current_offset( + YR_ARENA2* arena, + int buffer_id); + int yr_arena2_allocate_memory( YR_ARENA2* arena, diff --git a/libyara/include/yara/compiler.h b/libyara/include/yara/compiler.h index b02ac388..009761ca 100644 --- a/libyara/include/yara/compiler.h +++ b/libyara/include/yara/compiler.h @@ -67,6 +67,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define YR_CODE_SECTION 6 #define YR_RE_CODE_SECTION 7 + // This is the number of buffers used by the compiler, should match the number // of items in the list above. #define YR_NUM_SECTIONS 8 @@ -125,7 +126,6 @@ typedef void (*YR_COMPILER_RE_AST_CALLBACK_FUNC)( typedef struct _YR_FIXUP { YR_ARENA2_REFERENCE ref; - void* address; struct _YR_FIXUP* next; } YR_FIXUP; @@ -138,7 +138,9 @@ typedef struct _YR_FIXUP typedef struct _YR_LOOP_CONTEXT { - uint8_t* addr; + // Reference indicating the the place in the code where the loop starts. The + // loop goes back to this address on each iteration. + YR_ARENA2_REFERENCE start_ref; // vars_count is the number of local variables defined by the loop, and vars // is an array of expressions with the identifier and type for each of those @@ -180,6 +182,10 @@ typedef struct _YR_COMPILER // Similar to YR_CODE_SECTION, but it contains the code for regular // expressions. This is the code executed by yr_re_exec and // yr_re_fast_exec. + // YR_AC_TRANSITION_TABLE: + // Contains the Aho-Corasick transition table. + // YR_AC_MATCHES_TABLE: + // An array of YR_AC_MATCH structures. // YR_ARENA2* arena; @@ -202,7 +208,7 @@ typedef struct _YR_COMPILER jmp_buf error_recovery; - YR_ARENA* code_arena; + //YR_ARENA* code_arena; YR_ARENA* re_code_arena; YR_ARENA* compiled_rules_arena; YR_ARENA* matches_arena; diff --git a/libyara/include/yara/parser.h b/libyara/include/yara/parser.h index 23e261c0..5b529d69 100644 --- a/libyara/include/yara/parser.h +++ b/libyara/include/yara/parser.h @@ -37,31 +37,39 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. int yr_parser_emit( yyscan_t yyscanner, uint8_t instruction, - uint8_t** instruction_address); + YR_ARENA2_REFERENCE* instruction_ref); int yr_parser_emit_with_arg( yyscan_t yyscanner, uint8_t instruction, int64_t argument, - uint8_t** instruction_address, - int64_t** argument_address); + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref); + + +int yr_parser_emit_with_arg_int32( + yyscan_t yyscanner, + uint8_t instruction, + int32_t argument, + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref); int yr_parser_emit_with_arg_double( yyscan_t yyscanner, uint8_t instruction, double argument, - uint8_t** instruction_address, - double** argument_address); + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref); int yr_parser_emit_with_arg_reloc( yyscan_t yyscanner, uint8_t instruction, void* argument, - uint8_t** instruction_address, - void** argument_address); + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref); int yr_parser_check_types( diff --git a/libyara/include/yara/types.h b/libyara/include/yara/types.h index 9f13fca3..21783fb1 100644 --- a/libyara/include/yara/types.h +++ b/libyara/include/yara/types.h @@ -387,6 +387,7 @@ struct YR_EXTERNAL_VARIABLE struct YR_AC_MATCH { uint16_t backtrack; + uint32_t string_idx; DECLARE_REFERENCE(YR_STRING*, string); DECLARE_REFERENCE(const uint8_t*, forward_code); @@ -427,17 +428,6 @@ typedef struct YARA_RULES_FILE_HEADER } YARA_RULES_FILE_HEADER; - -typedef struct _YR_INIT_RULE_ARGS -{ - // Index of the rule in the rules table. - uint32_t rule_idx; - // Address of the first instruction belonging to the next rule. This is - // used when a rule is disabled and we want to skip its code. - DECLARE_REFERENCE(const uint8_t*, jmp_addr); -} YR_INIT_RULE_ARGS; - - #pragma pack(pop) diff --git a/libyara/parser.c b/libyara/parser.c index d46cee6e..452dbd5d 100644 --- a/libyara/parser.c +++ b/libyara/parser.c @@ -55,20 +55,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. int yr_parser_emit( yyscan_t yyscanner, uint8_t instruction, - uint8_t** instruction_address) + YR_ARENA2_REFERENCE* instruction_ref) { - yr_arena2_write_data( + return yr_arena2_write_data( yyget_extra(yyscanner)->arena, YR_CODE_SECTION, &instruction, sizeof(uint8_t), - NULL); - - return yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, - &instruction, - sizeof(uint8_t), - (void**) instruction_address); + instruction_ref); } @@ -76,15 +70,15 @@ int yr_parser_emit_with_arg_double( yyscan_t yyscanner, uint8_t instruction, double argument, - uint8_t** instruction_address, - double** argument_address) + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref) { int result = yr_arena2_write_data( yyget_extra(yyscanner)->arena, YR_CODE_SECTION, &instruction, sizeof(uint8_t), - NULL); + instruction_ref); if (result == ERROR_SUCCESS) result = yr_arena2_write_data( @@ -92,21 +86,33 @@ int yr_parser_emit_with_arg_double( YR_CODE_SECTION, &argument, sizeof(double), - NULL); + argument_ref); + + return result; +} + + +int yr_parser_emit_with_arg_int32( + yyscan_t yyscanner, + uint8_t instruction, + int32_t argument, + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref) +{ + int result = yr_arena2_write_data( + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, + &instruction, + sizeof(uint8_t), + instruction_ref); if (result == ERROR_SUCCESS) - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, - &instruction, - sizeof(uint8_t), - (void**) instruction_address); - - if (result == ERROR_SUCCESS) - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + result = yr_arena2_write_data( + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &argument, - sizeof(double), - (void**) argument_address); + sizeof(int32_t), + argument_ref); return result; } @@ -116,15 +122,15 @@ int yr_parser_emit_with_arg( yyscan_t yyscanner, uint8_t instruction, int64_t argument, - uint8_t** instruction_address, - int64_t** argument_address) + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref) { int result = yr_arena2_write_data( yyget_extra(yyscanner)->arena, YR_CODE_SECTION, &instruction, sizeof(uint8_t), - NULL); + instruction_ref); if (result == ERROR_SUCCESS) result = yr_arena2_write_data( @@ -132,21 +138,7 @@ int yr_parser_emit_with_arg( YR_CODE_SECTION, &argument, sizeof(int64_t), - NULL); - - if (result == ERROR_SUCCESS) - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, - &instruction, - sizeof(uint8_t), - (void**) instruction_address); - - if (result == ERROR_SUCCESS) - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, - &argument, - sizeof(int64_t), - (void**) argument_address); + argument_ref); return result; } @@ -156,10 +148,10 @@ int yr_parser_emit_with_arg_reloc( yyscan_t yyscanner, uint8_t instruction, void* argument, - uint8_t** instruction_address, - void** argument_address) + YR_ARENA2_REFERENCE* instruction_ref, + YR_ARENA2_REFERENCE* argument_ref) { - int64_t* ptr = NULL; + YR_ARENA2_REFERENCE ref; DECLARE_REFERENCE(void*, ptr) arg; @@ -171,9 +163,7 @@ int yr_parser_emit_with_arg_reloc( YR_CODE_SECTION, &instruction, sizeof(uint8_t), - NULL); - - YR_ARENA2_REFERENCE ref; + instruction_ref); if (result == ERROR_SUCCESS) result = yr_arena2_write_data( @@ -190,29 +180,8 @@ int yr_parser_emit_with_arg_reloc( ref.offset, EOL2); - if (result == ERROR_SUCCESS) - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, - &instruction, - sizeof(uint8_t), - (void**) instruction_address); - - if (result == ERROR_SUCCESS) - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, - &arg, - sizeof(arg), - (void**) &ptr); - - if (result == ERROR_SUCCESS) - result = yr_arena_make_ptr_relocatable( - yyget_extra(yyscanner)->code_arena, - ptr, - 0, - EOL); - - if (argument_address != NULL) - *argument_address = (void*) ptr; + if (argument_ref != NULL) + *argument_ref = ref; return result; } @@ -521,6 +490,7 @@ static int _yr_parser_write_string( result = yr_ac_add_string( compiler->automaton, string, + compiler->current_string_idx, atom_list, compiler->matches_arena); } @@ -886,7 +856,6 @@ int yr_parser_reduce_rule_declaration_phase_1( YR_ARENA2_REFERENCE* rule_ref) { YR_FIXUP *fixup; - YR_INIT_RULE_ARGS *init_rule_args; YR_COMPILER* compiler = yyget_extra(yyscanner); YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena2_get_ptr( @@ -944,33 +913,36 @@ int yr_parser_reduce_rule_declaration_phase_1( memset(rule->time_cost_per_thread, 0, sizeof(rule->time_cost_per_thread)); #endif - FAIL_ON_ERROR(yr_parser_emit( + YR_ARENA2_REFERENCE jmp_offset_ref; + + // The OP_INIT_RULE instruction behaves like a jump. When the rule is disabled + // it skips over the rule's code and go straight to the next rule's code. The + // jmp_offset_ref variable points to the jump's offset. The offset is set to 0 + // as we don't know the jump target yet. When we finish generating the rule's + // code in yr_parser_reduce_rule_declaration_phase_2 the jump offset is set to + // its final value. + + FAIL_ON_ERROR(yr_parser_emit_with_arg_int32( yyscanner, OP_INIT_RULE, + 0, + NULL, + &jmp_offset_ref)); + + FAIL_ON_ERROR(yr_arena2_write_data( + compiler->arena, + YR_CODE_SECTION, + &compiler->current_rule_idx, + sizeof(compiler->current_rule_idx), NULL)); - FAIL_ON_ERROR(yr_arena_allocate_struct( - compiler->code_arena, - sizeof(YR_INIT_RULE_ARGS), - (void**) &init_rule_args, - offsetof(YR_INIT_RULE_ARGS, jmp_addr), - EOL)); - - init_rule_args->rule_idx = compiler->current_rule_idx; - - // jmp_addr holds the address to jump to when we want to skip the code for - // the rule. It is initialized as NULL at this point because we don't know - // the address until emitting the code for the rule's condition. The address - // is set in yr_parser_reduce_rule_declaration_phase_2. - init_rule_args->jmp_addr = NULL; - // Create a fixup entry for the jump and push it in the stack fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) return ERROR_INSUFFICIENT_MEMORY; - fixup->address = (void*) &(init_rule_args->jmp_addr); + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; @@ -992,9 +964,6 @@ int yr_parser_reduce_rule_declaration_phase_2( { uint32_t max_strings_per_rule; uint32_t strings_in_rule = 0; - uint8_t* nop_inst_addr = NULL; - - int result; YR_FIXUP *fixup; YR_STRING* string; @@ -1046,25 +1015,25 @@ int yr_parser_reduce_rule_declaration_phase_2( string++; } - result = yr_parser_emit_with_arg( + FAIL_ON_ERROR(yr_parser_emit_with_arg( yyscanner, OP_MATCH_RULE, compiler->current_rule_idx, NULL, - NULL); - - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_INIT_RULE skip jump. We can not - // simply use the address of the OP_MATCH_RULE instruction +1 because we - // can't be sure that the instruction following the OP_MATCH_RULE is going to - // be in the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - if (result == ERROR_SUCCESS) - result = yr_parser_emit(yyscanner, OP_NOP, &nop_inst_addr); + NULL)); fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*) nop_inst_addr; + + int32_t* jmp_offset_addr = yr_arena2_ref_to_ptr( + compiler->arena, &fixup->ref); + + int32_t jmp_offset = \ + yr_arena2_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); @@ -1072,7 +1041,7 @@ int yr_parser_reduce_rule_declaration_phase_2( // a new one. compiler->current_rule_idx++; - return result; + return ERROR_SUCCESS; }