diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..118a9685 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,25 @@ +name: CIFuzz + +on: [pull_request] + +jobs: + Fuzzing: + runs-on: ubuntu-latest + steps: + - name: Build Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'yara' + dry-run: true + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'yara' + fuzz-seconds: 600 + dry-run: true + - name: Upload Crash + uses: actions/upload-artifact@v1 + if: failure() + with: + name: artifacts + path: ./out/artifacts diff --git a/.travis.yml b/.travis.yml index bbe2357c..d32860e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,10 +77,11 @@ matrix: - os: linux dist: xenial install: | - wget https://github.com/bazelbuild/bazel/releases/download/0.29.0/bazel_0.29.0-linux-x86_64.deb - sudo dpkg -i bazel_0.29.0-linux-x86_64.deb + wget https://github.com/bazelbuild/bazel/releases/download/2.1.0/bazel_2.1.0-linux-x86_64.deb + sudo dpkg -i bazel_2.1.0-linux-x86_64.deb before_script: bazel info - script: bazel test //tests/... + script: | + bazel test @jansson//... //tests/... # Build for OS X - os: osx diff --git a/BUILD.bazel b/BUILD.bazel index 2eeacbf2..8fa1cc9d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -25,222 +25,82 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -YARA_CONFIG_OPTS = [ - "-DBORINGSSL", - "-DHAVE_CLOCK_GETTIME=1", - #"-DHAVE_COMMONCRYPTO_COMMONCRYPTO_H", - "-DHAVE_DLFCN_H=1", - "-DHAVE_INTTYPES_H=1", - "-DHAVE_LIBCRYPTO=1", - "-DHAVE_LIBM=1", - "-DHAVE_MEMMEM=1", - "-DHAVE_MEMORY_H=1", - #"-DHAVE__MKGMTIME", - "-DHAVE_OPENSSL_ASN1_H=1", - "-DHAVE_OPENSSL_BIO_H=1", - "-DHAVE_OPENSSL_CRYPTO_H=1", - "-DHAVE_OPENSSL_MD5_H=1", - "-DHAVE_OPENSSL_PKCS7_H=1", - "-DHAVE_OPENSSL_SAFESTACK_H=1", - "-DHAVE_OPENSSL_SHA_H=1", - "-DHAVE_OPENSSL_X509_H=1", - #"-DHAVE_PTHREAD", - "-DHAVE_STDBOOL_H=1", - "-DHAVE_STDINT_H=1", - "-DHAVE_STDLIB_H=1", - "-DHAVE_STRING_H=1", - "-DHAVE_STRINGS_H=1", - "-DHAVE_SYS_STAT_H=1", - "-DHAVE_SYS_TYPES_H=1", - "-DHAVE_TIMEGM=1", - "-DHAVE_UNISTD_H=1", -] +# Bazel (http://bazel.io/) WORKSPACE file for YARA. -YARA_MODULE_OPTS = [ - "-DDEX_MODULE", - "-DDOTNET_MODULE", - "-DHASH_MODULE", - "-DMACHO_MODULE", -] - -YARA_COPTS = YARA_MODULE_OPTS + select({ - "@bazel_tools//src/conditions:darwin": [ - "-DUSE_MACH_PROC", - "-DHAVE_SCAN_PROC_IMPL=1", - "-DHAVE_STRLCAT=1", - "-DHAVE_STRLCPY=1", - ], - "@bazel_tools//src/conditions:freebsd": [ - "-DUSE_FREEBSD_PROC", - "-DHAVE_SCAN_PROC_IMPL=1", - "-DHAVE_STRLCAT=1", - "-DHAVE_STRLCPY=1", - ], - "@bazel_tools//src/conditions:linux_aarch64": [ - "-DUSE_LINUX_PROC", - "-DHAVE_SCAN_PROC_IMPL=1", - ], - "@bazel_tools//src/conditions:linux_x86_64": [ - "-DUSE_LINUX_PROC", - "-DHAVE_SCAN_PROC_IMPL=1", - ], - "@bazel_tools//src/conditions:windows": [ - "-DUSE_WINDOWS_PROC", - "-DHAVE_SCAN_PROC_IMPL=1", - ], - "//conditions:default": ["-DUSE_NO_PROC"], -}) + select({ - ":yara_profiling_enabled": ["-DPROFILING_ENABLED"], - "//conditions:default": [], -}) + [ - "-D_GNU_SOURCE", - # Tell the compiler we want the C99 standard. - "-std=c99", - # Additional include paths. - "-Ilibyara", - "-Ilibyara/modules", -] + YARA_CONFIG_OPTS - -# Configuration setting for enabling profiling. Usage: -# bazel build ... --define yara_profiling=enabled -config_setting( - name = "yara_profiling_enabled", - values = {"define": "yara_profiling=enabled"}, -) +load("@com_github_virustotal_yara//:bazel/yara.bzl", "yara_library") # Just YARA's error codes. This is useful for sandboxing as it avoids pulling # in the whole library. cc_library( name = "yara_errors", hdrs = ["libyara/include/yara/error.h"], - copts = YARA_COPTS, visibility = ["//visibility:public"], ) -# YARA library to be linked against. -cc_library( +# Configuration setting for enabling profiling. Usage: +# bazel build ... --define yara_profiling=enabled +config_setting( + name = "profiling_enabled", + values = {"define": "yara_profiling=enabled"}, +) + +# Configuration setting for using BoringSSL instead of OpenSSL. Notice that +# some features related to PE signatures in the "pe" module will be disabled +# because of missing functionality in BoringSSL. Usage: +# bazel build ... --define yara_crypto_library=boringssl +config_setting( + name = "crypto_library_boringssl", + values = {"define": "yara_crypto_library=boringssl"}, +) + + +yara_library( name = "libyara", - srcs = [ - "libyara/ahocorasick.c", - "libyara/arena.c", - "libyara/atoms.c", - "libyara/bitmask.c", - "libyara/compiler.c", - "libyara/crypto.h", - "libyara/endian.c", - "libyara/exception.h", - "libyara/exec.c", - "libyara/exefiles.c", - "libyara/filemap.c", - "libyara/grammar.c", - "libyara/hash.c", - "libyara/hex_grammar.c", - "libyara/hex_grammar.h", - "libyara/hex_lexer.c", - "libyara/include/yara.h", - "libyara/include/yara/ahocorasick.h", - "libyara/include/yara/arena.h", - "libyara/include/yara/atoms.h", - "libyara/include/yara/bitmask.h", - "libyara/include/yara/compiler.h", - "libyara/include/yara/dex.h", - "libyara/include/yara/dotnet.h", - "libyara/include/yara/elf.h", - "libyara/include/yara/endian.h", - "libyara/include/yara/error.h", - "libyara/include/yara/exec.h", - "libyara/include/yara/exefiles.h", - "libyara/include/yara/filemap.h", - "libyara/include/yara/globals.h", - "libyara/include/yara/hash.h", - "libyara/include/yara/hex_lexer.h", - "libyara/include/yara/integers.h", - "libyara/include/yara/lexer.h", - "libyara/include/yara/libyara.h", - "libyara/include/yara/limits.h", - "libyara/include/yara/macho.h", - "libyara/include/yara/mem.h", - "libyara/include/yara/modules.h", - "libyara/include/yara/object.h", - "libyara/include/yara/parser.h", - "libyara/include/yara/pe.h", - "libyara/include/yara/pe_utils.h", - "libyara/include/yara/proc.h", - "libyara/include/yara/re.h", - "libyara/include/yara/re_lexer.h", - "libyara/include/yara/rules.h", - "libyara/include/yara/scan.h", - "libyara/include/yara/scanner.h", - "libyara/include/yara/sizedstr.h", - "libyara/include/yara/stack.h", - "libyara/include/yara/stopwatch.h", - "libyara/include/yara/stream.h", - "libyara/include/yara/strutils.h", - "libyara/include/yara/threading.h", - "libyara/include/yara/types.h", - "libyara/include/yara/utils.h", - "libyara/lexer.c", - "libyara/libyara.c", - "libyara/mem.c", - "libyara/modules.c", - #"libyara/modules/cuckoo/cuckoo.c", # TODO(cblichmann): Add Jansson JSON depenency + defines = select({ + ":profiling_enabled": ["PROFILING_ENABLED"], + "//conditions:default": [], + }) + select({ + ":crypto_library_boringssl": ["BORINGSSL"], + "//conditions:default": [], + }), + modules = [ + "cuckoo", + "dex", + "dotnet", + "elf", + "hash", + "macho", + "magic", + "math", + "pe", + "tests", + "time", + ], + modules_srcs = [ + "libyara/modules/cuckoo/cuckoo.c", "libyara/modules/dex/dex.c", - #"libyara/modules/demo/demo.c", # Disabled "libyara/modules/dotnet/dotnet.c", "libyara/modules/elf/elf.c", "libyara/modules/hash/hash.c", + "libyara/modules/magic/magic.c", "libyara/modules/macho/macho.c", - #"libyara/modules/magic/magic.c", # TODO(cblichmann): Add libmagic dependency "libyara/modules/math/math.c", "libyara/modules/pe/pe.c", "libyara/modules/pe/pe_utils.c", "libyara/modules/tests/tests.c", "libyara/modules/time/time.c", - "libyara/object.c", - "libyara/parser.c", - "libyara/proc.c", - "libyara/proc/freebsd.c", - "libyara/proc/linux.c", - "libyara/proc/mach.c", - "libyara/proc/none.c", - "libyara/proc/openbsd.c", - "libyara/proc/windows.c", - "libyara/re.c", - "libyara/re_grammar.c", - "libyara/re_grammar.h", - "libyara/re_lexer.c", - "libyara/rules.c", - "libyara/scan.c", - "libyara/scanner.c", - "libyara/sizedstr.c", - "libyara/stack.c", - "libyara/stopwatch.c", - "libyara/stream.c", - "libyara/strutils.c", - "libyara/threading.c", ], - hdrs = [ - "libyara/include/yara.h", - "libyara/include/yara/pe.h", - "libyara/include/yara/proc.h", - "libyara/include/yara/rules.h", - ], - copts = YARA_COPTS, - includes = [ - "libyara/modules", - "libyara/include", - "libyara", - ], - textual_hdrs = [ - "libyara/grammar.h", - "libyara/hex_grammar.y", - "libyara/modules/module_list", - "libyara/re_grammar.y", - ], - visibility = ["//visibility:public"], - deps = ["@boringssl//:crypto"], + crypto_libs = select({ + ":crypto_library_boringssl": ["@boringssl//:crypto"], + "//conditions:default": ["@openssl//:crypto"], + }), + deps = [ + "@jansson//:jansson", + "@magic//:magic", + ] ) + # Code shared by the command-line tools. cc_library( name = "cli_shared", @@ -253,7 +113,6 @@ cc_library( "common.h", "threading.h", ], - copts = YARA_COPTS, deps = [":libyara"], ) @@ -261,7 +120,6 @@ cc_library( cc_binary( name = "yara", srcs = ["yara.c"], - copts = YARA_COPTS, visibility = ["//visibility:public"], deps = [ ":cli_shared", @@ -269,11 +127,10 @@ cc_binary( ], ) -# The YARA compiler +# YARA compiler cc_binary( name = "yarac", srcs = ["yarac.c"], - copts = YARA_COPTS, visibility = ["//visibility:public"], deps = [ ":cli_shared", diff --git a/Makefile.am b/Makefile.am index f0d1a349..062b77f4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,6 +29,8 @@ yarac_SOURCES = args.c args.h common.h yarac.c yarac_LDADD = -Llibyara/.libs -lyara test_alignment_SOURCES = tests/test-alignment.c +test_arena_SOURCES = tests/test-arena.c +test_arena_LDADD = libyara/.libs/libyara.a test_atoms_SOURCES = tests/test-atoms.c tests/util.c libyara/atoms.c test_atoms_LDADD = libyara/.libs/libyara.a test_rules_SOURCES = tests/test-rules.c tests/util.c @@ -52,7 +54,9 @@ test_re_split_LDADD = libyara/.libs/libyara.a TESTS = $(check_PROGRAMS) TESTS_ENVIRONMENT = TOP_SRCDIR=$(top_srcdir) -check_PROGRAMS = test-alignment \ +check_PROGRAMS = \ + test-arena \ + test-alignment \ test-atoms \ test-api \ test-rules \ diff --git a/README.md b/README.md index 798102de..5887f0cc 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ awesome list of [YARA-related stuff](https://github.com/InQuest/awesome-yara). * [Blueliv](http://www.blueliv.com) * [Cofense](https://cofense.com) * [Conix](http://www.conix.fr) +* [CounterCraft](https://www.countercraft.eu) * [CrowdStrike FMS](https://github.com/CrowdStrike/CrowdFMS) * [Cuckoo Sandbox](https://github.com/cuckoosandbox/cuckoo) * [Cyber Triage](http://www.cybertriage.com) @@ -105,6 +106,7 @@ awesome list of [YARA-related stuff](https://github.com/InQuest/awesome-yara). * [RSA ECAT](http://www.emc.com/security/rsa-ecat.htm) * [Scanii](https://scanii.com) * [SecondWrite](https://www.secondwrite.com) +* [SonicWall](https://www.sonicwall.com/) * [SpamStopsHere](https://www.spamstopshere.com) * [stoQ](http://stoq.punchcyber.com) * [Symantec](http://www.symantec.com) diff --git a/WORKSPACE b/WORKSPACE.bazel similarity index 58% rename from WORKSPACE rename to WORKSPACE.bazel index 644a7fb2..47b54a43 100644 --- a/WORKSPACE +++ b/WORKSPACE.bazel @@ -25,41 +25,23 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +# Bazel (http://bazel.io/) WORKSPACE file for YARA. -# BoringSSL, see -# https://boringssl.googlesource.com/boringssl/+/master/INCORPORATING.md#bazel -git_repository( - name = "boringssl", - commit = "095d78b14f91cc9a910408eaae84a3bdafc54da9", # 2019-06-05 - remote = "https://boringssl.googlesource.com/boringssl", - shallow_since = "1559759280 +0000", -) +workspace(name = "com_github_virustotal_yara") -# Sandboxed API -git_repository( - name = "com_google_sandboxed_api", - commit = "2301e05097818734f59b881d7fbe1624c17fc840", # 2019-07-08 - remote = "https://github.com/google/sandboxed-api.git", - shallow_since = "1562590596 -0700", +load("//:bazel/yara_deps.bzl", "yara_deps") +yara_deps() + + +load( + "@rules_proto//proto:repositories.bzl", + "rules_proto_dependencies", "rules_proto_toolchains" ) +rules_proto_dependencies() +rules_proto_toolchains() load( "@com_google_sandboxed_api//sandboxed_api/bazel:sapi_deps.bzl", "sapi_deps", ) - sapi_deps() - -load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") - -protobuf_deps() - -# GoogleTest/GoogleMock for testing the sandbox -http_archive( - name = "com_google_googletest", - sha256 = "baed63b97595c32667694de0a434f8f23da59609c4a44f3360ba94b0abd5c583", - strip_prefix = "googletest-8ffb7e5c88b20a297a2e786c480556467496463b", - urls = ["https://github.com/google/googletest/archive/8ffb7e5c88b20a297a2e786c480556467496463b.zip"], # 2019-05-30 -) diff --git a/bazel/jansson.BUILD b/bazel/jansson.BUILD new file mode 100644 index 00000000..a11456e8 --- /dev/null +++ b/bazel/jansson.BUILD @@ -0,0 +1,82 @@ +# Copyright (c) 2020. The YARA Authors. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Bazel (http://bazel.io/) BUILD file for the jansson library. + +# Rule for generating header files required during the build. These header files +# are normally generated by ./configure, but we just create empty files so that +# #include statements can find them. +genrule( + name = "config_files", + outs = [ + "jansson_config.h", + "jansson_private_config.h", + ], + cmd = """ + touch $(location jansson_config.h) + touch $(location jansson_private_config.h) + """, +) + +cc_library( + name = "jansson", + hdrs = glob(["src/*.h"]) + [ + "jansson_config.h", + "jansson_private_config.h", + ], + srcs = glob(["src/*.c"]), + includes = [".", "src"], + # These defines are usually in jansson_config.h but we define them here, + # as jansson_config.h is just an empty file. + defines = [ + "HAVE_STDINT_H=1", + "JSON_INLINE=inline", + "JSON_PARSER_MAX_DEPTH=2048", + ], + copts = ["-Wno-unused-function"], + visibility = ["//visibility:public"], +) + + +load("@com_github_virustotal_yara//:bazel/jansson.bzl", "jansson_api_test") + +jansson_api_test("test_array") +jansson_api_test("test_chaos") +jansson_api_test("test_copy") +jansson_api_test("test_dump") +jansson_api_test("test_dump_callback") +jansson_api_test("test_equal") +jansson_api_test("test_load") +jansson_api_test("test_load_callback") +jansson_api_test("test_loadb") +jansson_api_test("test_memory_funcs") +jansson_api_test("test_number") +jansson_api_test("test_object") +jansson_api_test("test_pack") +jansson_api_test("test_simple") +jansson_api_test("test_sprintf") +jansson_api_test("test_unpack") diff --git a/bazel/jansson.bzl b/bazel/jansson.bzl new file mode 100644 index 00000000..c0b4c6f4 --- /dev/null +++ b/bazel/jansson.bzl @@ -0,0 +1,36 @@ +# Copyright (c) 2020. The YARA Authors. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +def jansson_api_test(name): + native.cc_test( + name = name, + srcs = [ + "test/suites/api/util.h", + "test/suites/api/" + name + ".c", + ], + deps = [":jansson"] + ) diff --git a/bazel/magic.BUILD b/bazel/magic.BUILD new file mode 100644 index 00000000..3d30bbdd --- /dev/null +++ b/bazel/magic.BUILD @@ -0,0 +1,139 @@ + +# Copyright (c) 2020. The YARA Authors. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Bazel (http://bazel.io/) BUILD file for the libmagic library. + +# Rule for creating magic.h from magic.h.in. The two files are identical, except +# for magic.h.in not having the actual version number. Instead it has a X.YY +# placeholder that is replaced with the version number (i.e: 538) during build +# time. When this library is updated the version number in this rule must be +# updated accordingly. +genrule( + name = "magic_h", + srcs = ["src/magic.h.in"], + outs = ["src/magic.h"], + cmd = """ + sed -e 's/X.YY/538/' < $(location src/magic.h.in) > $(location src/magic.h) + """, +) + +MAGIC_COPTS = [ + # The VERSION macro usually contains the actual version (e.g: "5.38") but + # any arbitrary string will work. We simply use "BUILT_BY_YARA". + "-DVERSION=\\\"BUILT_BY_YARA\\\"", + ] + select({ + "@bazel_tools//src/conditions:darwin": [ + "-DHAVE_FORK=1", + "-DHAVE_INTTYPES_H=1", + "-DHAVE_STDINT_H=1", + "-DHAVE_STRLCAT=1", + "-DHAVE_STRLCPY=1", + "-DHAVE_UNISTD_H=1", + ], + "@bazel_tools//src/conditions:freebsd": [ + "-DHAVE_FORK=1", + "-DHAVE_INTTYPES_H=1", + "-DHAVE_STDINT_H=1", + "-DHAVE_STRLCAT=1", + "-DHAVE_STRLCPY=1", + "-DHAVE_UNISTD_H=1", + ], + "@bazel_tools//src/conditions:linux_aarch64": [ + "-DHAVE_FORK=1", + "-DHAVE_INTTYPES_H=1", + "-DHAVE_STDINT_H=1", + "-DHAVE_UNISTD_H=1", + ], + "@bazel_tools//src/conditions:linux_x86_64": [ + "-DHAVE_FORK=1", + "-DHAVE_INTTYPES_H=1", + "-DHAVE_STDINT_H=1", + "-DHAVE_UNISTD_H=1", + ], + "@bazel_tools//src/conditions:windows": [ + ] +}) + +cc_library( + name = "magic", + hdrs = ["src/magic.h"], + srcs = glob(["src/*.h"]) + [ + "src/apprentice.c", + "src/apptype.c", + "src/ascmagic.c", + "src/asctime_r.c", + "src/asprintf.c", + "src/buffer.c", + "src/cdf.c", + "src/cdf_time.c", + "src/compress.c", + "src/ctime_r.c", + "src/der.c", + "src/dprintf.c", + "src/encoding.c", + "src/fmtcheck.c", + "src/fsmagic.c", + "src/funcs.c", + "src/getline.c", + "src/getopt_long.c", + "src/gmtime_r.c", + "src/is_csv.c", + "src/is_json.c", + "src/is_tar.c", + "src/localtime_r.c", + "src/magic.c", + "src/pread.c", + "src/print.c", + "src/readcdf.c", + "src/readelf.c", + "src/seccomp.c", + "src/softmagic.c", + "src/strcasestr.c", + "src/teststrchr.c", + "src/vasprintf.c", + ] + select({ + "@bazel_tools//src/conditions:darwin": [ + ], + "@bazel_tools//src/conditions:freebsd": [ + ], + "@bazel_tools//src/conditions:linux_aarch64": [ + "src/strlcat.c", + "src/strlcpy.c", + ], + "@bazel_tools//src/conditions:linux_x86_64": [ + "src/strlcat.c", + "src/strlcpy.c", + ], + "@bazel_tools//src/conditions:windows": [ + "src/strlcat.c", + "src/strlcpy.c", + ]}), + includes = [".", "src"], + copts = MAGIC_COPTS, + visibility = ["//visibility:public"], +) diff --git a/bazel/openssl.BUILD b/bazel/openssl.BUILD new file mode 100644 index 00000000..5e6e76cb --- /dev/null +++ b/bazel/openssl.BUILD @@ -0,0 +1,77 @@ +# Copyright (c) 2020. The YARA Authors. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Bazel (http://bazel.io/) BUILD file for openssl library which is a dependency +# for YARA. + +config_setting( + name = "darwin", + values = { + "cpu": "darwin_x86_64", + }, +) + +cc_library( + name = "crypto", + hdrs = glob(["include/openssl/*.h"]) + ["include/openssl/opensslconf.h"], + srcs = ["libcrypto.a"], + includes = ["include"], + linkopts = select({ + ":darwin": [], + "//conditions:default": ["-lpthread", "-ldl"], + }), + visibility = ["//visibility:public"], +) + +cc_library( + name = "ssl", + deps = [":crypto"], + hdrs = glob(["include/openssl/*.h"]) + ["include/openssl/opensslconf.h"], + srcs = ["libssl.a"], + includes = ["include"], + visibility = ["//visibility:public"], +) + +genrule( + name = "openssl-build", + srcs = glob(["**/*"], exclude=["bazel-*"]), + outs = [ + "libcrypto.a", + "libssl.a", + "include/openssl/opensslconf.h", + ], + cmd = """ + OPENSSL_ROOT=$$(dirname $(location config)) + pushd $$OPENSSL_ROOT + ./config + make -j 4 + popd + cp $$OPENSSL_ROOT/libcrypto.a $(location libcrypto.a) + cp $$OPENSSL_ROOT/libssl.a $(location libssl.a) + cp $$OPENSSL_ROOT/include/openssl/opensslconf.h $(location include/openssl/opensslconf.h) + """, +) diff --git a/bazel/yara.bzl b/bazel/yara.bzl new file mode 100644 index 00000000..3122d0a9 --- /dev/null +++ b/bazel/yara.bzl @@ -0,0 +1,206 @@ + +YARA_CONFIG_OPTS = [ + "-DHAVE_CLOCK_GETTIME=1", + #"-DHAVE_COMMONCRYPTO_COMMONCRYPTO_H", + "-DHAVE_LIBCRYPTO=1", + "-DHAVE_MEMMEM=1", + "-DHAVE_STDBOOL_H=1", + # "-DHAVE__MKGMTIME=1", + "-DHAVE_TIMEGM=1", +] + +YARA_COPTS = YARA_CONFIG_OPTS + [ + "-D_GNU_SOURCE", + # Tell the compiler we want the C99 standard. + "-std=c99", + # Additional include paths. + "-Ilibyara", + "-Ilibyara/modules", +] + select({ + "@bazel_tools//src/conditions:darwin": [ + "-DUSE_MACH_PROC", + "-DHAVE_SCAN_PROC_IMPL=1", + "-DHAVE_STRLCAT=1", + "-DHAVE_STRLCPY=1", + ], + "@bazel_tools//src/conditions:freebsd": [ + "-DUSE_FREEBSD_PROC", + "-DHAVE_SCAN_PROC_IMPL=1", + "-DHAVE_STRLCAT=1", + "-DHAVE_STRLCPY=1", + ], + "@bazel_tools//src/conditions:linux_aarch64": [ + "-DUSE_LINUX_PROC", + "-DHAVE_SCAN_PROC_IMPL=1", + ], + "@bazel_tools//src/conditions:linux_x86_64": [ + "-DUSE_LINUX_PROC", + "-DHAVE_SCAN_PROC_IMPL=1", + ], + "@bazel_tools//src/conditions:windows": [ + "-DUSE_WINDOWS_PROC", + "-DHAVE_SCAN_PROC_IMPL=1", + ], + "//conditions:default": ["-DUSE_NO_PROC"], +}) + + +# Define rule for generating the module_list file. This rule has an attribute +# "modules" which is a list of module names that we want in the file. +def _module_list_impl(ctx): + output = ctx.outputs.out + content = '\n'.join(["MODULE(%s)" % m for m in ctx.attr.modules]) + ctx.actions.write(output = output, content = content) + +module_list = rule( + implementation = _module_list_impl, + attrs = {"modules": attr.string_list()}, + outputs = {"out": "libyara/modules/module_list"}, +) + + +def yara_library(name, defines=[], modules=[], modules_srcs=[], + deps=[], crypto_libs=["@openssl//:crypto"]): + """Macro for generating the YARA library with a specific list of modules. + + This macro allows to cherry-pick the modules that you want to build into the + library. For example, for building it with modules pe and elf only, you can + use: + + yara_library( + name = "libyara", + modules = [ + "cuckoo", + "dex", + "dotnet", + "elf", + ], + modules_srcs = [ + "libyara/modules/elf/elf.c", + "libyara/modules/pe/pe.c", + "libyara/modules/pe/pe_utils.c", + ] + ) + + The "modules_srcs" list must contain the source files implementing the modules + listed in the "modules" argument. + """ + module_list( + name = "module_list", + modules = modules, + ) + + native.cc_library( + name = name, + defines = defines + [m.upper() + "_MODULE" for m in modules], + srcs = modules_srcs + [ + "libyara/ahocorasick.c", + "libyara/arena.c", + "libyara/atoms.c", + "libyara/base64.c", + "libyara/bitmask.c", + "libyara/compiler.c", + "libyara/crypto.h", + "libyara/endian.c", + "libyara/exception.h", + "libyara/exec.c", + "libyara/exefiles.c", + "libyara/filemap.c", + "libyara/grammar.c", + "libyara/hash.c", + "libyara/hex_grammar.c", + "libyara/hex_grammar.h", + "libyara/hex_lexer.c", + "libyara/include/yara.h", + "libyara/include/yara/ahocorasick.h", + "libyara/include/yara/arena.h", + "libyara/include/yara/atoms.h", + "libyara/include/yara/base64.h", + "libyara/include/yara/bitmask.h", + "libyara/include/yara/compiler.h", + "libyara/include/yara/dex.h", + "libyara/include/yara/dotnet.h", + "libyara/include/yara/elf.h", + "libyara/include/yara/endian.h", + "libyara/include/yara/error.h", + "libyara/include/yara/exec.h", + "libyara/include/yara/exefiles.h", + "libyara/include/yara/filemap.h", + "libyara/include/yara/globals.h", + "libyara/include/yara/hash.h", + "libyara/include/yara/hex_lexer.h", + "libyara/include/yara/integers.h", + "libyara/include/yara/lexer.h", + "libyara/include/yara/libyara.h", + "libyara/include/yara/limits.h", + "libyara/include/yara/macho.h", + "libyara/include/yara/mem.h", + "libyara/include/yara/modules.h", + "libyara/include/yara/notebook.h", + "libyara/include/yara/object.h", + "libyara/include/yara/parser.h", + "libyara/include/yara/pe.h", + "libyara/include/yara/pe_utils.h", + "libyara/include/yara/proc.h", + "libyara/include/yara/re.h", + "libyara/include/yara/re_lexer.h", + "libyara/include/yara/rules.h", + "libyara/include/yara/scan.h", + "libyara/include/yara/scanner.h", + "libyara/include/yara/sizedstr.h", + "libyara/include/yara/stack.h", + "libyara/include/yara/stopwatch.h", + "libyara/include/yara/stream.h", + "libyara/include/yara/strutils.h", + "libyara/include/yara/threading.h", + "libyara/include/yara/types.h", + "libyara/include/yara/utils.h", + "libyara/lexer.c", + "libyara/libyara.c", + "libyara/mem.c", + "libyara/modules.c", + "libyara/notebook.c", + "libyara/object.c", + "libyara/parser.c", + "libyara/proc.c", + "libyara/proc/freebsd.c", + "libyara/proc/linux.c", + "libyara/proc/mach.c", + "libyara/proc/none.c", + "libyara/proc/openbsd.c", + "libyara/proc/windows.c", + "libyara/re.c", + "libyara/re_grammar.c", + "libyara/re_grammar.h", + "libyara/re_lexer.c", + "libyara/rules.c", + "libyara/scan.c", + "libyara/scanner.c", + "libyara/sizedstr.c", + "libyara/stack.c", + "libyara/stopwatch.c", + "libyara/stream.c", + "libyara/strutils.c", + "libyara/threading.c", + ], + hdrs = [ + "libyara/include/yara.h", + "libyara/include/yara/pe.h", + "libyara/include/yara/proc.h", + "libyara/include/yara/rules.h", + ], + copts = YARA_COPTS, + includes = [ + "libyara/modules", + "libyara/include", + "libyara", + ], + textual_hdrs = [ + ":module_list", + "libyara/grammar.h", + "libyara/hex_grammar.y", + "libyara/re_grammar.y", + ], + deps = deps + crypto_libs, + visibility = ["//visibility:public"], + ) diff --git a/bazel/yara_deps.bzl b/bazel/yara_deps.bzl new file mode 100644 index 00000000..a3c4702a --- /dev/null +++ b/bazel/yara_deps.bzl @@ -0,0 +1,92 @@ + +# Copyright (c) 2020. The YARA Authors. All Rights Reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation and/or +# other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Load dependencies needed to compile YARA as a 3rd-party consumer.""" +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") + +def yara_deps(): + """Loads common dependencies needed to compile YARA.""" + if not native.existing_rule("openssl"): + http_archive( + name = "openssl", + url = "https://github.com/openssl/openssl/archive/OpenSSL_1_1_0h.tar.gz", + sha256 = "f56dd7d81ce8d3e395f83285bd700a1098ed5a4cb0a81ce9522e41e6db7e0389", + strip_prefix = "openssl-OpenSSL_1_1_0h", + build_file = "@com_github_virustotal_yara//:bazel/openssl.BUILD", + ) + if not native.existing_rule("borinssl"): + git_repository( + name = "boringssl", + commit = "095d78b14f91cc9a910408eaae84a3bdafc54da9", # 2019-06-05 + remote = "https://boringssl.googlesource.com/boringssl", + shallow_since = "1559759280 +0000", + ) + if not native.existing_rule("jansson"): + http_archive( + name = "jansson", + url = "https://github.com/akheron/jansson/archive/v2.12.tar.gz", + sha256 = "76260d30e9bbd0ef392798525e8cd7fe59a6450c54ca6135672e3cd6a1642941", + strip_prefix = "jansson-2.12", + build_file = "@com_github_virustotal_yara//:bazel/jansson.BUILD", + ) + if not native.existing_rule("magic"): + # When updating this dependency to a more recent version, the version + # in the bazel/magic.BUILD must be updated acordingly. + http_archive( + name = "magic", + url = "https://github.com/file/file/archive/FILE5_38.tar.gz", + sha256 = "338ebe8cb536a3f86750b4df62be2d382f6da66afdb0087b36a1a3e14ea4baf8", + strip_prefix = "file-FILE5_38", + build_file = "@com_github_virustotal_yara//:bazel/magic.BUILD", + ) + if not native.existing_rule("com_google_sandboxed_api"): + git_repository( + name = "com_google_sandboxed_api", + commit = "2301e05097818734f59b881d7fbe1624c17fc840", # 2019-07-08 + remote = "https://github.com/google/sandboxed-api.git", + shallow_since = "1562590596 -0700", + ) + if not native.existing_rule("rules_proto"): + http_archive( + name = "rules_proto", + sha256 = "602e7161d9195e50246177e7c55b2f39950a9cf7366f74ed5f22fd45750cd208", + strip_prefix = "rules_proto-97d8af4dc474595af3900dd85cb3a29ad28cc313", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz", + ], + ) + if not native.existing_rule("com_google_googletest"): + # GoogleTest/GoogleMock for testing the sandbox + http_archive( + name = "com_google_googletest", + sha256 = "baed63b97595c32667694de0a434f8f23da59609c4a44f3360ba94b0abd5c583", + strip_prefix = "googletest-8ffb7e5c88b20a297a2e786c480556467496463b", + urls = ["https://github.com/google/googletest/archive/8ffb7e5c88b20a297a2e786c480556467496463b.zip"], # 2019-05-30 + ) diff --git a/cmake-build-debug/CMakeFiles/yara.dir/link.txt b/cmake-build-debug/CMakeFiles/yara.dir/link.txt new file mode 100644 index 00000000..40dfd44f --- /dev/null +++ b/cmake-build-debug/CMakeFiles/yara.dir/link.txt @@ -0,0 +1 @@ +/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++ -g -Wl,-search_paths_first -Wl,-headerpad_max_install_names CMakeFiles/yara.dir/libyara/modules/cuckoo/cuckoo.c.o CMakeFiles/yara.dir/libyara/modules/demo/demo.c.o CMakeFiles/yara.dir/libyara/modules/dex/dex.c.o CMakeFiles/yara.dir/libyara/modules/dotnet/dotnet.c.o CMakeFiles/yara.dir/libyara/modules/elf/elf.c.o CMakeFiles/yara.dir/libyara/modules/hash/hash.c.o CMakeFiles/yara.dir/libyara/modules/macho/macho.c.o CMakeFiles/yara.dir/libyara/modules/magic/magic.c.o CMakeFiles/yara.dir/libyara/modules/math/math.c.o CMakeFiles/yara.dir/libyara/modules/pb_tests/pb_tests.c.o CMakeFiles/yara.dir/libyara/modules/pb_tests/pb_tests.pb-c.c.o CMakeFiles/yara.dir/libyara/modules/pe/pe.c.o CMakeFiles/yara.dir/libyara/modules/pe/pe_utils.c.o CMakeFiles/yara.dir/libyara/modules/tests/tests.c.o CMakeFiles/yara.dir/libyara/modules/time/time.c.o CMakeFiles/yara.dir/libyara/proc/freebsd.c.o CMakeFiles/yara.dir/libyara/proc/linux.c.o CMakeFiles/yara.dir/libyara/proc/mach.c.o CMakeFiles/yara.dir/libyara/proc/none.c.o CMakeFiles/yara.dir/libyara/proc/openbsd.c.o CMakeFiles/yara.dir/libyara/proc/windows.c.o CMakeFiles/yara.dir/libyara/ahocorasick.c.o CMakeFiles/yara.dir/libyara/arena.c.o CMakeFiles/yara.dir/libyara/atoms.c.o CMakeFiles/yara.dir/libyara/bitmask.c.o CMakeFiles/yara.dir/libyara/compiler.c.o CMakeFiles/yara.dir/libyara/endian.c.o CMakeFiles/yara.dir/libyara/exec.c.o CMakeFiles/yara.dir/libyara/exefiles.c.o CMakeFiles/yara.dir/libyara/filemap.c.o CMakeFiles/yara.dir/libyara/grammar.c.o CMakeFiles/yara.dir/libyara/hash.c.o CMakeFiles/yara.dir/libyara/hex_grammar.c.o CMakeFiles/yara.dir/libyara/hex_lexer.c.o CMakeFiles/yara.dir/libyara/lexer.c.o CMakeFiles/yara.dir/libyara/libyara.c.o CMakeFiles/yara.dir/libyara/mem.c.o CMakeFiles/yara.dir/libyara/modules.c.o CMakeFiles/yara.dir/libyara/object.c.o CMakeFiles/yara.dir/libyara/parser.c.o CMakeFiles/yara.dir/libyara/proc.c.o CMakeFiles/yara.dir/libyara/re.c.o CMakeFiles/yara.dir/libyara/re_grammar.c.o CMakeFiles/yara.dir/libyara/re_lexer.c.o CMakeFiles/yara.dir/libyara/rules.c.o CMakeFiles/yara.dir/libyara/scan.c.o CMakeFiles/yara.dir/libyara/scanner.c.o CMakeFiles/yara.dir/libyara/sizedstr.c.o CMakeFiles/yara.dir/libyara/stack.c.o CMakeFiles/yara.dir/libyara/stopwatch.c.o CMakeFiles/yara.dir/libyara/stream.c.o CMakeFiles/yara.dir/libyara/strutils.c.o CMakeFiles/yara.dir/libyara/threading.c.o CMakeFiles/yara.dir/tests/oss-fuzz/dex_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/dotnet_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/elf_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/macho_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/pe_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/rules_fuzzer.cc.o CMakeFiles/yara.dir/tests/test-alignment.c.o CMakeFiles/yara.dir/tests/test-api.c.o CMakeFiles/yara.dir/tests/test-atoms.c.o CMakeFiles/yara.dir/tests/test-bitmask.c.o CMakeFiles/yara.dir/tests/test-dex.c.o CMakeFiles/yara.dir/tests/test-dotnet.c.o CMakeFiles/yara.dir/tests/test-elf.c.o CMakeFiles/yara.dir/tests/test-exception.c.o CMakeFiles/yara.dir/tests/test-macho.c.o CMakeFiles/yara.dir/tests/test-math.c.o CMakeFiles/yara.dir/tests/test-pb.c.o CMakeFiles/yara.dir/tests/test-pe.c.o CMakeFiles/yara.dir/tests/test-re-split.c.o CMakeFiles/yara.dir/tests/test-rules.c.o CMakeFiles/yara.dir/tests/test-stack.c.o CMakeFiles/yara.dir/tests/test-version.c.o CMakeFiles/yara.dir/tests/util.c.o CMakeFiles/yara.dir/args.c.o CMakeFiles/yara.dir/threading.c.o CMakeFiles/yara.dir/yara.c.o CMakeFiles/yara.dir/yarac.c.o CMakeFiles/yara.dir/libyara/arena2.c.o CMakeFiles/yara.dir/tests/test-arena.c.o -o yara diff --git a/cmake-build-debug/CMakeFiles/yara.dir/progress.make b/cmake-build-debug/CMakeFiles/yara.dir/progress.make new file mode 100644 index 00000000..54e93aba --- /dev/null +++ b/cmake-build-debug/CMakeFiles/yara.dir/progress.make @@ -0,0 +1,84 @@ +CMAKE_PROGRESS_1 = 1 +CMAKE_PROGRESS_2 = 2 +CMAKE_PROGRESS_3 = 3 +CMAKE_PROGRESS_4 = 4 +CMAKE_PROGRESS_5 = 5 +CMAKE_PROGRESS_6 = 6 +CMAKE_PROGRESS_7 = 7 +CMAKE_PROGRESS_8 = 8 +CMAKE_PROGRESS_9 = 9 +CMAKE_PROGRESS_10 = 10 +CMAKE_PROGRESS_11 = 11 +CMAKE_PROGRESS_12 = 12 +CMAKE_PROGRESS_13 = 13 +CMAKE_PROGRESS_14 = 14 +CMAKE_PROGRESS_15 = 15 +CMAKE_PROGRESS_16 = 16 +CMAKE_PROGRESS_17 = 17 +CMAKE_PROGRESS_18 = 18 +CMAKE_PROGRESS_19 = 19 +CMAKE_PROGRESS_20 = 20 +CMAKE_PROGRESS_21 = 21 +CMAKE_PROGRESS_22 = 22 +CMAKE_PROGRESS_23 = 23 +CMAKE_PROGRESS_24 = 24 +CMAKE_PROGRESS_25 = 25 +CMAKE_PROGRESS_26 = 26 +CMAKE_PROGRESS_27 = 27 +CMAKE_PROGRESS_28 = 28 +CMAKE_PROGRESS_29 = 29 +CMAKE_PROGRESS_30 = 30 +CMAKE_PROGRESS_31 = 31 +CMAKE_PROGRESS_32 = 32 +CMAKE_PROGRESS_33 = 33 +CMAKE_PROGRESS_34 = 34 +CMAKE_PROGRESS_35 = 35 +CMAKE_PROGRESS_36 = 36 +CMAKE_PROGRESS_37 = 37 +CMAKE_PROGRESS_38 = 38 +CMAKE_PROGRESS_39 = 39 +CMAKE_PROGRESS_40 = 40 +CMAKE_PROGRESS_41 = 41 +CMAKE_PROGRESS_42 = 42 +CMAKE_PROGRESS_43 = 43 +CMAKE_PROGRESS_44 = 44 +CMAKE_PROGRESS_45 = 45 +CMAKE_PROGRESS_46 = 46 +CMAKE_PROGRESS_47 = 47 +CMAKE_PROGRESS_48 = 48 +CMAKE_PROGRESS_49 = 49 +CMAKE_PROGRESS_50 = 50 +CMAKE_PROGRESS_51 = 51 +CMAKE_PROGRESS_52 = 52 +CMAKE_PROGRESS_53 = 53 +CMAKE_PROGRESS_54 = 54 +CMAKE_PROGRESS_55 = 55 +CMAKE_PROGRESS_56 = 56 +CMAKE_PROGRESS_57 = 57 +CMAKE_PROGRESS_58 = 58 +CMAKE_PROGRESS_59 = 59 +CMAKE_PROGRESS_60 = 60 +CMAKE_PROGRESS_61 = 61 +CMAKE_PROGRESS_62 = 62 +CMAKE_PROGRESS_63 = 63 +CMAKE_PROGRESS_64 = 64 +CMAKE_PROGRESS_65 = 65 +CMAKE_PROGRESS_66 = 66 +CMAKE_PROGRESS_67 = 67 +CMAKE_PROGRESS_68 = 68 +CMAKE_PROGRESS_69 = 69 +CMAKE_PROGRESS_70 = 70 +CMAKE_PROGRESS_71 = 71 +CMAKE_PROGRESS_72 = 72 +CMAKE_PROGRESS_73 = 73 +CMAKE_PROGRESS_74 = 74 +CMAKE_PROGRESS_75 = 75 +CMAKE_PROGRESS_76 = 76 +CMAKE_PROGRESS_77 = 77 +CMAKE_PROGRESS_78 = 78 +CMAKE_PROGRESS_79 = 79 +CMAKE_PROGRESS_80 = 80 +CMAKE_PROGRESS_81 = 81 +CMAKE_PROGRESS_82 = 82 +CMAKE_PROGRESS_83 = 83 + diff --git a/common.h b/common.h index b9d591cf..721a2762 100644 --- a/common.h +++ b/common.h @@ -56,7 +56,7 @@ bool compile_files( char* colon = NULL; int errors; - if (access(argv[1], 0) != 0) + if (access(argv[i], 0) != 0) { // A file with the name specified by the command-line argument wasn't // found, it may be because the name is prefixed with a namespace, so diff --git a/docs/capi.rst b/docs/capi.rst index 4eb6b215..a1273724 100644 --- a/docs/capi.rst +++ b/docs/capi.rst @@ -30,7 +30,7 @@ with :c:func:`yr_compiler_destroy`. You can use :c:func:`yr_compiler_add_file`, :c:func:`yr_compiler_add_fd`, or :c:func:`yr_compiler_add_string` to add one or more input sources to be -compiled. Both of these functions receive an optional namespace. Rules added +compiler. Both of these functions receive an optional namespace. Rules added under the same namespace behave as if they were contained within the same source file or string, so, rule identifiers must be unique among all the sources sharing a namespace. If the namespace argument is ``NULL`` the rules are put @@ -71,11 +71,11 @@ However, if you want to fetch the imported rules from another source (eg: from a database or remote service), a callback function can be set with :c:func:`yr_compiler_set_include_callback`. -The callback receives the following parameters: +This callback receives the following parameters: * ``include_name``: name of the requested file. * ``calling_rule_filename``: the requesting file name (NULL if not a file). * ``calling_rule_namespace``: namespace (NULL if undefined). - * ``user_data`` pointer is the same you passed to :c:func:`yr_compiler_set_include_callback`. + * ``user_data`` same pointer passed to :c:func:`yr_compiler_set_include_callback`. It should return the requested file's content as a null-terminated string. The memory for this string should be allocated by the callback function. Once it is @@ -248,6 +248,9 @@ Lastly, the callback function is also called with the ``CALLBACK_MSG_SCAN_FINISHED`` message when the scan is finished. In this case ``message_data`` is ``NULL``. +Notice that you shouldn't call any of the ``yr_rules_scan_XXXX`` functions from +within the callback as those functions are not re-entrant. + Your callback function must return one of the following values:: CALLBACK_CONTINUE @@ -260,16 +263,19 @@ If it returns ``CALLBACK_CONTINUE`` YARA will continue normally, ``CALLBACK_ERROR`` will abort the scanning too, but the result from ``yr_rules_scan_XXXX`` will be ``ERROR_CALLBACK_ERROR``. - The ``user_data`` argument passed to your callback function is the same you passed ``yr_rules_scan_XXXX``. This pointer is not touched by YARA, it's just a way for your program to pass arbitrary data to the callback function. -All ``yr_rules_scan_XXXX`` functions receive a ``flags`` argument and a -``timeout`` argument. The only flag defined at this time is -``SCAN_FLAGS_FAST_MODE``, so you must pass either this flag or a zero value. -The ``timeout`` argument forces the function to return after the specified -number of seconds approximately, with a zero meaning no timeout at all. +All ``yr_rules_scan_XXXX`` functions receive a ``flags`` argument that allows +to tweak some aspects of the scanning process. The supported flags are the following +ones: + + ``SCAN_FLAGS_FAST_MODE`` + ``SCAN_FLAGS_NO_TRYCATCH`` + ``SCAN_FLAGS_REPORT_RULES_MATCHING`` + ``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING`` + The ``SCAN_FLAGS_FAST_MODE`` flag makes the scanning a little faster by avoiding multiple matches of the same string when not necessary. Once the string was @@ -278,8 +284,22 @@ single match for the string, even if it appears multiple times in the scanned data. This flag has the same effect of the ``-f`` command-line option described in :ref:`command-line`. -Notice that you shouldn't call any of the ``yr_rules_scan_XXXX`` functions from -within the callback as those functions are not re-entrant. +``SCAN_FLAGS_REPORT_RULES_MATCHING`` and ``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING`` +control whether the callback is invoked for rules that are matching or for rules +that are not matching respectively. If ``SCAN_FLAGS_REPORT_RULES_MATCHING`` is +specified alone, the callback will be called for matching rules with the +``CALLBACK_MSG_RULE_MATCHING`` message but it won't be called for non-matching +rules. If ``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING`` is specified alone, the opposite +happens, the callback will be called with ``CALLBACK_MSG_RULE_NOT_MATCHING`` +messages but not with ``CALLBACK_MSG_RULE_MATCHING`` messages. If both flags +are combined together (the default) the callback will be called for both matching +and non-matching rules. For backward compatibility, if none of these two flags +are specified, the scanner will follow the default behavior. + +Additionally, ``yr_rules_scan_XXXX`` functions can receive a ``timeout`` argument +which forces the scan to abort after the specified number of seconds (approximately). +If ``timeout`` is 0 it means no timeout at all. + Using a scanner --------------- @@ -825,7 +845,12 @@ Functions .. versionadded:: 3.8.0 - Set the flags that will be used by any call to `yr_scanner_scan_xxx`. + Set the flags that will be used by any call to `yr_scanner_scan_xxx`. The supported flags are: + + ``SCAN_FLAGS_FAST_MODE``: Enable fast scan mode. + ``SCAN_FLAGS_NO_TRYCATCH``: Disable exception handling. + ``SCAN_FLAGS_REPORT_RULES_MATCHING``: If this + ``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING`` .. c:function:: int yr_scanner_define_integer_variable(YR_SCANNER* scanner, const char* identifier, int64_t value) diff --git a/docs/commandline.rst b/docs/commandline.rst index a4f516e1..86b044ca 100644 --- a/docs/commandline.rst +++ b/docs/commandline.rst @@ -129,7 +129,7 @@ Available options are: .. option:: -r --recursive - Recursively search for directories. + Recursively search for directories. It follows symlinks. .. option:: -f --fast-scan diff --git a/docs/docutils.conf b/docs/docutils.conf new file mode 100644 index 00000000..168f9e2b --- /dev/null +++ b/docs/docutils.conf @@ -0,0 +1,2 @@ +[parsers] +smart_quotes: false diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 7371c160..8c5ae5f2 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -88,7 +88,8 @@ You can also download and install YARA using the `vcpkg `_ on the vcpkg repository. +The YARA port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out +of date, please `create an issue or pull request `_ on the vcpkg repository. Installing on Windows @@ -98,10 +99,11 @@ Compiled binaries for Windows in both 32 and 64 bit flavors can be found in the link below. Just download the version you want, unzip the archive, and put the ``yara.exe`` and ``yarac.exe`` binaries anywhere in your disk. -To install the ``yara-python`` extension download and execute the installer -corresponding to the version of Python you're using. +`Download Windows binaries `_ -`Download Windows binaries `_ +To install YARA using `Scoop `_ or `Chocolatey `_, simply type +``scoop install yara`` or ``choco install yara``. The integration with both `Scoop` and `Chocolatey` are +not maintained their respective teams, not by the YARA authors. Installing on Mac OS X with Homebrew @@ -111,7 +113,7 @@ To install YARA using `Homebrew `_, simply type ``brew install yara``. -Installing yara-python +Installing ``yara-python`` ---------------------- If you plan to use YARA from your Python scripts you need to install the diff --git a/docs/modules/cuckoo.rst b/docs/modules/cuckoo.rst index 76a2cb74..1ae9cbc8 100644 --- a/docs/modules/cuckoo.rst +++ b/docs/modules/cuckoo.rst @@ -113,6 +113,13 @@ Reference Similar to :func:`http_request`, but only takes into account POST requests. + + .. function:: http_user_agent(regexp) + + Function returning true if the program sent a HTTP request with a + user agent matching the provided regular expression. + + *Example: cuckoo.network.http_user_agent(/MSIE 6\\.0/)* .. function:: dns_lookup(regexp) @@ -120,6 +127,29 @@ Reference request for a domain matching the provided regular expression. *Example: cuckoo.network.dns_lookup(/evil\\.com/)* + + .. function:: host(regexp) + + Function returning true if the program contacted an IP address + matching the provided regular expression. + + *Example: cuckoo.network.host(/192\\.168\\.1\\.1/)* + + .. function:: tcp(regexp, port) + + Function returning true if the program contacted an IP address + matching the provided regular expression, over TCP on the provided + port number. + + *Example: cuckoo.network.tcp(/192\\.168\\.1\\.1/, 443)* + + .. function:: udp(regexp, port) + + Function returning true if the program contacted an IP address + matching the provided regular expression, over UDP on the provided + port number. + + *Example: cuckoo.network.udp(/192\\.168\\.1\\.1/, 53)* .. type:: registry diff --git a/docs/modules/pe.rst b/docs/modules/pe.rst index 3d6d2141..02a00656 100644 --- a/docs/modules/pe.rst +++ b/docs/modules/pe.rst @@ -111,13 +111,6 @@ Reference Value of IMAGE_FILE_HEADER::PointerToSymbolTable. Used when the PE image has COFF debug info. -.. c:type:: pointer_to_symbol_table - - .. versionadded:: 3.8.0 - - Value of IMAGE_FILE_HEADER::PointerToSymbolTable. Used when the PE image has - COFF debug info. - .. c:type:: number_of_symbols .. versionadded:: 3.8.0 @@ -223,63 +216,63 @@ Reference following constants: .. c:type:: RELOCS_STRIPPED - + Relocation info stripped from file. - + .. c:type:: EXECUTABLE_IMAGE - + File is executable (i.e. no unresolved external references). - + .. c:type:: LINE_NUMS_STRIPPED - + Line numbers stripped from file. - + .. c:type:: LOCAL_SYMS_STRIPPED - + Local symbols stripped from file. - + .. c:type:: AGGRESIVE_WS_TRIM - + Aggressively trim working set - + .. c:type:: LARGE_ADDRESS_AWARE - + App can handle >2gb addresses - + .. c:type:: BYTES_REVERSED_LO - + Bytes of machine word are reversed. - + .. c:type:: MACHINE_32BIT - + 32 bit word machine. - + .. c:type:: DEBUG_STRIPPED - + Debugging info stripped from file in .DBG file - + .. c:type:: REMOVABLE_RUN_FROM_SWAP - + If Image is on removable media, copy and run from the swap file. - + .. c:type:: NET_RUN_FROM_SWAP - + If Image is on Net, copy and run from the swap file. - + .. c:type:: SYSTEM - + System File. - + .. c:type:: DLL - + File is a DLL. - + .. c:type:: UP_SYSTEM_ONLY - + File should only be run on a UP machine - + .. c:type:: BYTES_REVERSED_HI - + Bytes of machine word are reversed. *Example: pe.characteristics & pe.DLL* @@ -818,7 +811,15 @@ Reference value (0 or 1) if the given *toolid* and optional *version* is present in an entry. - *Example: pe.rich_signature.toolid(170, 40219) >= 99 and pe.rich_signature.toolid(170, 40219) <= 143* + *Example: pe.rich_signature.toolid(170, 40219) >= 99* + +.. c:type:: pdb_path + + .. versionadded:: 3.13.0 + + Path of the PDB file for this PE if present. + + * Example: pe.pdb_path == "D:\\workspace\\2018_R9_RelBld\\target\\checkout\\custprof\\Release\\custprof.pdb" .. c:function:: exports(function_name) @@ -845,12 +846,76 @@ Reference *Example: pe.exports(/^AXS@@/)* +.. c:function:: exports_index(function_name) + + .. versionadded:: 3.12.0 + + Function returning the index into the export_details array where the named + function is, undefined otherwise. + + *Example: pe.exports_index("CPlApplet")* + +.. c:function:: exports_index(ordinal) + + .. versionadded:: 3.12.0 + + Function returning the index into the export_details array where the + exported ordinal is, undefined otherwise. + + *Example: pe.exports_index(72)* + +.. c:function:: exports_index(/regular_expression/) + + .. versionadded:: 3.12.0 + + Function returning the first index into the export_details array where the + regular expression matches the exported name, undefined otherwise. + + *Example: pe.exports_index(/^ERS@@/)* + .. c:type:: number_of_exports .. versionadded:: 3.6.0 Number of exports in the PE. +.. c:type:: export_details + + .. versionadded:: 3.12.0 + + Structure containing information about the PE's exports. + + .. c:member:: offset + + Offset where the exported function starts. + + .. c:member:: name + + Name of the exported function. It will be undefined if the function has + no name. + + .. c:member:: forward_name + + The name of the function where this export forwards to. It will be + undefined if the export is not a forwarding export. + + .. c:member:: ordinal + + The ordinal of the exported function, after the ordinal base has been + applied to it. + +.. c:type:: dll_name + + .. versionadded:: 3.12.0 + + The name of the DLL, if it exists in the export directory. + +.. c:type:: export_timestamp + + .. versionadded:: 3.12.0 + + The timestamp the export data was created.. + .. c:type:: number_of_imports .. versionadded:: 3.6.0 @@ -867,11 +932,17 @@ Reference .. c:function:: imports(dll_name) .. versionadded:: 3.5.0 + .. versionchanged:: 3.12.0 - Function returning true if the PE imports anything from *dll_name*, - or false otherwise. *dll_name* is case insensitive. + Function returning the number of functions from the *dll_name*, in the PE + imports. *dll_name* is case insensitive. - *Example: pe.imports("kernel32.dll")* + Note: Prior to version 3.12.0, this function returned only a boolean value + indicating if the given DLL name was found in the PE imports. This change + is backward compatible, as any number larger than 0 also evaluates as + true. + + *Examples: pe.imports("kernel32.dll"), pe.imports("kernel32.dll") == 10* .. c:function:: imports(dll_name, ordinal) @@ -885,13 +956,19 @@ Reference .. c:function:: imports(dll_regexp, function_regexp) .. versionadded:: 3.8.0 + .. versionchanged:: 3.12.0 - Function returning true if the PE imports a function name matching - *function_regexp* from a DLL matching *dll_regexp*. *dll_regexp* is case - sensitive unless you use the "/i" modifier in the regexp, as shown in the - example below. + Function returning the number of functions from the PE imports where a + function name matches *function_regexp* and a DLL name matches + *dll_regexp*. Both *dll_regexp* and *function_regexp* are case sensitive + unless you use the "/i" modifier in the regexp, as shown in the example + below. - *Example: pe.imports(/kernel32\.dll/i, /(Read|Write)ProcessMemory/)* + Note: Prior to version 3.12.0, this function returned only a boolean value + indicating if matching import was found or not. This change is backward + compatible, as any number larger than 0 also evaluates as true. + + *Example: pe.imports(/kernel32\.dll/i, /(Read|Write)ProcessMemory/) == 2* .. c:function:: locale(locale_identifier) diff --git a/docs/writingmodules.rst b/docs/writingmodules.rst index 6285a5e0..a2ca934e 100644 --- a/docs/writingmodules.rst +++ b/docs/writingmodules.rst @@ -951,8 +951,8 @@ depending on the function's return type. In all cases *x* is a constant, variable, or expression evaluating to ``char*``, ``int64_t`` or ``double`` respectively. -You can use ``return_string(UNDEFINED)``, ``return_float(UNDEFINED)`` and -``return_integer(UNDEFINED)`` to return undefined values from the function. +You can use ``return_string(YR_UNDEFINED)``, ``return_float(YR_UNDEFINED)`` and +``return_integer(YR_UNDEFINED)`` to return undefined values from the function. This is useful in many situations, for example if the arguments passed to the functions don't make sense, or if your module expects a particular file format and the scanned file is from another format, or in any other case where your diff --git a/docs/writingrules.rst b/docs/writingrules.rst index b02cd56e..024a6f56 100644 --- a/docs/writingrules.rst +++ b/docs/writingrules.rst @@ -23,50 +23,56 @@ keywords are reserved and cannot be used as an identifier: .. list-table:: YARA keywords - :widths: 10 10 10 10 10 10 10 + :widths: 10 10 10 10 10 10 10 10 * - all - and - any - ascii - at + - base64 + - base64wide - condition - - contains - * - entrypoint + * - contains + - entrypoint - false - filesize - - fullword - for + - fullword - global - - in - * - import + - import + * - in - include - - int8 - int16 - - int32 - - int8be - int16be - * - int32be - - matches + - int32 + - int32be + - int8 + - int8be + * - matches - meta - nocase - not - - or - of - * - private + - or + - private - rule - - strings + * - strings - them - true - - uint8 - uint16 - * - uint32 - - uint8be - uint16be + - uint32 - uint32be + - uint8 + * - uint8be - wide - xor - - + - + - + - + - + - Rules are generally composed of two sections: strings definition and condition. The strings definition section can be omitted if the rule doesn't rely on any @@ -362,7 +368,7 @@ The following rule will search for every single byte xor applied to the string $xor_string = "This program cannot" xor condition: - $xor_string + $xor_string } The above rule is logically equivalent to: @@ -435,6 +441,65 @@ If you want more control over the range of bytes used with the xor modifier use: The above example will apply the bytes from 0x01 to 0xff, inclusively, to the string when searching. The general syntax is ``xor(minimum-maximum)``. +base64 strings +^^^^^^^^^^^^^^ + +The ``base64`` modifier can be used to search for strings that have been base64 +encoded. A good explanation of the technique is at: + +https://www.leeholmes.com/blog/2019/12/10/searching-for-content-in-base-64-strings-2/ + +The following rule will search for the three base64 permutations of the string +"This program cannot": + +.. code-block:: yara + + rule Base64Example1 + { + strings: + $a = "This program cannot" base64 + + condition: + $a + } + +This will cause YARA to search for these three permutations: + +VGhpcyBwcm9ncmFtIGNhbm5vd +RoaXMgcHJvZ3JhbSBjYW5ub3 +UaGlzIHByb2dyYW0gY2Fubm90 + +The ``base64wide`` modifier works just like the base64 modifier but the results +of the base64 modifier are converted to wide. + +The interaction between ``base64`` (or ``base64wide``) and ``wide`` and +``ascii`` is as you might expect. ``wide`` and ``ascii`` are applied to the +string first, and then the ``base64`` and ``base64wide`` modifiers are applied. +At no point is the plaintext of the ``ascii`` or ``wide`` versions of the +strings included in the search. If you want to also include those you can put +them in a secondary string. + +The ``base64`` and ``base64wide`` modifiers also support a custom alphabet. For +example: + +.. code-block:: yara + + rule Base64Example2 + { + strings: + $a = "This program cannot" base64("!@#$%^&*(){}[].,|ABCDEFGHIJ\x09LMNOPQRSTUVWXYZabcdefghijklmnopqrstu") + + condition: + $a + } + +The alphabet must be 64 bytes long. + +The ``base64`` and ``base64wide`` modifiers are only supported with text +strings. Using these modifiers with a hexadecimal string or a regular expression +will cause a compiler error. Also, the ``xor`` and ``nocase`` modifiers used in +combination with ``base64`` or ``base64wide`` will cause a compiler error. + Searching for full words ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -611,24 +676,10 @@ the typical Boolean operators ``and``, ``or``, and ``not``, and relational opera and bitwise operators (``&``, ``|``, ``<<``, ``>>``, ``~``, ``^``) can be used on numerical expressions. -String identifiers can be also used within a condition, acting as Boolean -variables whose value depends on the presence or not of the associated string -in the file. - -.. code-block:: yara - - rule Example - { - strings: - $a = "text1" - $b = "text2" - $c = "text3" - $d = "text4" - - condition: - ($a or $b) and ($c or $d) - } - +Integers are always 64-bits long, even the results of functions like `uint8`, +`uint16` and `uint32` are promoted to 64-bits. This is something you must take +into account, specially while using bitwise operators (for example, ~0x01 is not +0xFE but 0xFFFFFFFFFFFFFFFE). The following table lists the precedence and associativity of all operators. The table is sorted in descending precedence order, which means that operators listed @@ -687,6 +738,26 @@ Precedence Operator Description Associativity ========== ======== ========================================= ============= +String identifiers can be also used within a condition, acting as Boolean +variables whose value depends on the presence or not of the associated string +in the file. + +.. code-block:: yara + + rule Example + { + strings: + $a = "text1" + $b = "text2" + $c = "text3" + $d = "text4" + + condition: + ($a or $b) and ($c or $d) + } + + + Counting strings ---------------- @@ -1107,6 +1178,49 @@ In summary, the syntax of this operator is: for expression identifier in indexes : ( boolean_expression ) + +Iterators +--------- + +In YARA 3.12 the ``for..of`` operator was improved and now it can be used to +iterate not only over integer enumerations and ranges (e.g: 1,2,3,4 and 1..4), +but also over any kind of iterable data type, like arrays and dictionaries +defined by YARA modules. For example, the following expression is valid in +YARA 3.12: + +.. code-block:: yara + + for any section in pe.sections : ( section.name == ".text" ) + +This is equivalent to: + +.. code-block:: yara + + for any i in (0..pe.number_of_sections-1) : ( pe.sections[i].name == ".text" ) + +The new syntax is more natural and easy to understand, and is the recommended +way of expressing this type of conditions in newer versions of YARA. + +For while iterating dictionaries you must provide to variable names that will +hold the key and value of each entry in the dictionary, for example: + +.. code-block:: yara + + for any k,v in some_dict : ( k == "foo" and v == "bar" ) + +In general the ``for..of`` operator has the form: + +.. code-block:: yara + + for in : ( ) + +Where `` is either `any`, `all` or an expression that evaluates to +the number of items in the iterator that must satisfy the condition, `` +is a comma-separated list of variable names that holds the values for the +current item (the number of variables depend on the type of ``) and +`` is something that can be iterated. + + .. _referencing-rules: Referencing other rules diff --git a/docs/yarapython.rst b/docs/yarapython.rst index 02e74fff..94990da8 100644 --- a/docs/yarapython.rst +++ b/docs/yarapython.rst @@ -219,7 +219,7 @@ Here is an example: import yara def mycallback(data): - print data + print(data) return yara.CALLBACK_CONTINUE matches = rules.match('/foo/bar/my_file', callback=mycallback, which_callbacks=yara.CALLBACK_MATCHES) @@ -259,7 +259,7 @@ Here is an example: import yara def modules_callback(data): - print data + print(data) return yara.CALLBACK_CONTINUE matches = rules.match('/foo/bar/my_file', modules_callback=modules_callback) diff --git a/libyara/Makefile.am b/libyara/Makefile.am index 2712c1f7..90e171db 100644 --- a/libyara/Makefile.am +++ b/libyara/Makefile.am @@ -95,6 +95,7 @@ yarainclude_HEADERS = \ include/yara/ahocorasick.h \ include/yara/arena.h \ include/yara/atoms.h \ + include/yara/base64.h \ include/yara/bitmask.h \ include/yara/compiler.h \ include/yara/error.h \ @@ -107,6 +108,7 @@ yarainclude_HEADERS = \ include/yara/limits.h \ include/yara/mem.h \ include/yara/modules.h \ + include/yara/notebook.h \ include/yara/object.h \ include/yara/parser.h \ include/yara/proc.h \ @@ -158,6 +160,7 @@ libyara_la_SOURCES = \ ahocorasick.c \ arena.c \ atoms.c \ + base64.c \ bitmask.c \ compiler.c \ endian.c \ @@ -171,6 +174,7 @@ libyara_la_SOURCES = \ libyara.c \ mem.c \ modules.c \ + notebook.c \ object.c \ parser.c \ proc.c \ diff --git a/libyara/ahocorasick.c b/libyara/ahocorasick.c index d2dfc7d1..ceb1f0f8 100644 --- a/libyara/ahocorasick.c +++ b/libyara/ahocorasick.c @@ -33,12 +33,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include #include - typedef struct _QUEUE_NODE { YR_AC_STATE* value; @@ -208,7 +208,7 @@ static YR_AC_STATE* _yr_ac_state_create( new_state->input = input; new_state->depth = state->depth + 1; - new_state->matches = NULL; + new_state->matches_ref = YR_ARENA_NULL_REF; new_state->failure = NULL; new_state->t_table_slot = 0; new_state->first_child = NULL; @@ -249,7 +249,8 @@ static int _yr_ac_state_destroy( // static int _yr_ac_create_failure_links( - YR_AC_AUTOMATON* automaton) + YR_AC_AUTOMATON* automaton, + YR_ARENA* arena) { YR_AC_STATE* current_state; YR_AC_STATE* failure_state; @@ -285,22 +286,27 @@ static int _yr_ac_create_failure_links( while (!_yr_ac_queue_is_empty(&queue)) { current_state = _yr_ac_queue_pop(&queue); - - match = current_state->matches; + match = yr_arena_ref_to_ptr(arena, ¤t_state->matches_ref); if (match != NULL) { + // Find the last match in the list of matches. while (match->next != NULL) match = match->next; if (match->backtrack > 0) - match->next = root_state->matches; + { + match->next = yr_arena_ref_to_ptr(arena, &root_state->matches_ref); + } } else { - current_state->matches = root_state->matches; + // This state doesn't have any matches, its matches will be those + // in the root state, if any. + current_state->matches_ref = root_state->matches_ref; } + // Iterate over all the states that the current state can transition to. transition_state = current_state->first_child; while (transition_state != NULL) @@ -317,18 +323,18 @@ static int _yr_ac_create_failure_links( { transition_state->failure = temp_state; - if (transition_state->matches == NULL) + if (YR_ARENA_IS_NULL_REF(transition_state->matches_ref)) { - transition_state->matches = temp_state->matches; + transition_state->matches_ref = temp_state->matches_ref; } else { - match = transition_state->matches; + match = yr_arena_ref_to_ptr(arena, &transition_state->matches_ref); while (match != NULL && match->next != NULL) match = match->next; - match->next = temp_state->matches; + match->next = yr_arena_ref_to_ptr(arena, &temp_state->matches_ref); } break; @@ -425,7 +431,7 @@ static int _yr_ac_optimize_failure_links( current_state->failure = current_state->failure->failure; } - // Push childrens of current_state + // Push children of current_state state = current_state->first_child; while (state != NULL) @@ -450,6 +456,7 @@ static int _yr_ac_optimize_failure_links( static int _yr_ac_find_suitable_transition_table_slot( YR_AC_AUTOMATON* automaton, + YR_ARENA* arena, YR_AC_STATE* state, uint32_t* slot) { @@ -486,36 +493,31 @@ static int _yr_ac_find_suitable_transition_table_slot( if (*slot > automaton->tables_size - 257) { - size_t t_bytes_size = automaton->tables_size * - sizeof(YR_AC_TRANSITION); + FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory( + arena, + YR_AC_TRANSITION_TABLE, + 257 * sizeof(YR_AC_TRANSITION), + NULL)); - size_t m_bytes_size = automaton->tables_size * - sizeof(YR_AC_MATCH_TABLE_ENTRY); + FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory( + arena, + YR_AC_STATE_MATCHES_TABLE, + 257 * sizeof(uint8_t*), + NULL)); - size_t b_bytes_size = YR_BITMASK_SIZE(automaton->tables_size) * + size_t bm_len = YR_BITMASK_SIZE(automaton->tables_size) * sizeof(YR_BITMASK); - automaton->t_table = (YR_AC_TRANSITION_TABLE) yr_realloc( - automaton->t_table, t_bytes_size * 2); + size_t bm_len_incr = YR_BITMASK_SIZE(257) * sizeof(YR_BITMASK); - automaton->m_table = (YR_AC_MATCH_TABLE) yr_realloc( - automaton->m_table, m_bytes_size * 2); + automaton->bitmask = yr_realloc(automaton->bitmask, bm_len + bm_len_incr); - automaton->bitmask = (YR_BITMASK*) yr_realloc( - automaton->bitmask, b_bytes_size * 2); - - if (automaton->t_table == NULL || - automaton->m_table == NULL || - automaton->bitmask == NULL) - { + if (automaton->bitmask == NULL) return ERROR_INSUFFICIENT_MEMORY; - } - memset((uint8_t*) automaton->t_table + t_bytes_size, 0, t_bytes_size); - memset((uint8_t*) automaton->m_table + m_bytes_size, 0, m_bytes_size); - memset((uint8_t*) automaton->bitmask + b_bytes_size, 0, b_bytes_size); + memset((uint8_t*) automaton->bitmask + bm_len, 0, bm_len_incr); - automaton->tables_size *= 2; + automaton->tables_size += 257; } return ERROR_SUCCESS; @@ -579,43 +581,57 @@ static int _yr_ac_find_suitable_transition_table_slot( // // A more detailed description can be found in: http://goo.gl/lE6zG - static int _yr_ac_build_transition_table( - YR_AC_AUTOMATON* automaton) + YR_AC_AUTOMATON* automaton, + YR_ARENA* arena) { + YR_AC_TRANSITION* t_table; + uint32_t* m_table; YR_AC_STATE* state; YR_AC_STATE* child_state; YR_AC_STATE* root_state = automaton->root; uint32_t slot; - QUEUE queue = { NULL, NULL}; + QUEUE queue = { NULL, NULL }; - automaton->tables_size = 1024; + // Both t_table and m_table have 512 slots initially, which is enough for the + // root node's transition table. + automaton->tables_size = 512; - automaton->t_table = (YR_AC_TRANSITION_TABLE) yr_calloc( - automaton->tables_size, sizeof(YR_AC_TRANSITION)); - - automaton->m_table = (YR_AC_MATCH_TABLE) yr_calloc( - automaton->tables_size, sizeof(YR_AC_MATCH_TABLE_ENTRY)); - - automaton->bitmask = (YR_BITMASK*) yr_calloc( + automaton->bitmask = yr_calloc( YR_BITMASK_SIZE(automaton->tables_size), sizeof(YR_BITMASK)); - if (automaton->t_table == NULL || - automaton->m_table == NULL || - automaton->bitmask == NULL) - { - yr_free(automaton->t_table); - yr_free(automaton->m_table); - yr_free(automaton->bitmask); - + if (automaton->bitmask == NULL) return ERROR_INSUFFICIENT_MEMORY; - } - automaton->t_table[0] = YR_AC_MAKE_TRANSITION(0, 0); - automaton->m_table[0].match = root_state->matches; + FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory( + arena, + YR_AC_TRANSITION_TABLE, + automaton->tables_size * sizeof(YR_AC_TRANSITION), + NULL)); + FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory( + arena, + YR_AC_STATE_MATCHES_TABLE, + automaton->tables_size * sizeof(uint32_t), + NULL)); + + t_table = yr_arena_get_ptr(arena, YR_AC_TRANSITION_TABLE, 0); + m_table = yr_arena_get_ptr(arena, YR_AC_STATE_MATCHES_TABLE, 0); + + // The failure link for the root node points to itself. + t_table[0] = YR_AC_MAKE_TRANSITION(0, 0); + + // Initialize the entry corresponding to the root node in the match table. + // Entries in this table are the index within YR_AC_MATCH_POOL where resides + // the YR_AC_MATCH structure that corresponds to the head of the matches list + // for the node. The indexes start counting at 1, the zero is used for + // indicating that the node has no associated matches. + if (!YR_ARENA_IS_NULL_REF(root_state->matches_ref)) + m_table[0] = root_state->matches_ref.offset / sizeof(YR_AC_MATCH) + 1; + + // Mark the first slot in the transition table as used. yr_bitmask_set(automaton->bitmask, 0); // Index 0 is for root node. Unused indexes start at 1. @@ -625,8 +641,10 @@ static int _yr_ac_build_transition_table( while (child_state != NULL) { + // Each state stores its slot number. child_state->t_table_slot = child_state->input + 1; - automaton->t_table[child_state->input + 1] = YR_AC_MAKE_TRANSITION( + + t_table[child_state->input + 1] = YR_AC_MAKE_TRANSITION( 0, child_state->input + 1); yr_bitmask_set(automaton->bitmask, child_state->input + 1); @@ -640,27 +658,38 @@ static int _yr_ac_build_transition_table( state = _yr_ac_queue_pop(&queue); FAIL_ON_ERROR(_yr_ac_find_suitable_transition_table_slot( - automaton, state, &slot)); + automaton, arena, state, &slot)); - automaton->t_table[state->t_table_slot] |= (slot << YR_AC_SLOT_OFFSET_BITS); + // _yr_ac_find_suitable_transition_table_slot can allocate more space in + // both tables and cause the tables to be moved to a different memory + // location, we must get their up-to-date addresses. + t_table = yr_arena_get_ptr(arena, YR_AC_TRANSITION_TABLE, 0); + m_table = yr_arena_get_ptr(arena, YR_AC_STATE_MATCHES_TABLE, 0); + + t_table[state->t_table_slot] |= (slot << YR_AC_SLOT_OFFSET_BITS); + t_table[slot] = YR_AC_MAKE_TRANSITION(state->failure->t_table_slot, 0); + + // The match table is an array of indexes within YR_AC_MATCHES_POOL. The + // N-th item in the array is the index for the YR_AC_MATCH structure that + // represents the head of the matches list for state N. The indexes start + // at 1, the 0 indicates that there are no matches for the state. + if (YR_ARENA_IS_NULL_REF(state->matches_ref)) + m_table[slot] = 0; + else + m_table[slot] = state->matches_ref.offset / sizeof(YR_AC_MATCH) + 1; state->t_table_slot = slot; - automaton->t_table[slot] = YR_AC_MAKE_TRANSITION( - state->failure->t_table_slot, 0); - yr_bitmask_set(automaton->bitmask, slot); - automaton->m_table[slot].match = state->matches; - - // Push childrens of current_state - + // Push children of current_state child_state = state->first_child; while (child_state != NULL) { child_state->t_table_slot = slot + child_state->input + 1; - automaton->t_table[child_state->t_table_slot] = YR_AC_MAKE_TRANSITION( + + t_table[child_state->t_table_slot] = YR_AC_MAKE_TRANSITION( 0, child_state->input + 1); yr_bitmask_set(automaton->bitmask, child_state->t_table_slot); @@ -681,14 +710,15 @@ static int _yr_ac_build_transition_table( // Prints automaton state for debug purposes. This function is invoked by // yr_ac_print_automaton, is not intended to be used stand-alone. // - +//TODO(vmalvarez): Remove or re-implement +/* static void _yr_ac_print_automaton_state( YR_AC_STATE* state) { int i; int child_count; - YR_AC_MATCH* match; + YR_AC_MATCH_LIST_ENTRY* match; YR_AC_STATE* child_state; for (i = 0; i < state->depth; i++) @@ -757,7 +787,7 @@ static void _yr_ac_print_automaton_state( _yr_ac_print_automaton_state(child_state); child_state = child_state->siblings; } -} +}*/ // @@ -784,15 +814,13 @@ int yr_ac_automaton_create( } root_state->depth = 0; - root_state->matches = NULL; + root_state->matches_ref = YR_ARENA_NULL_REF; root_state->failure = NULL; root_state->first_child = NULL; root_state->siblings = NULL; root_state->t_table_slot = 0; new_automaton->root = root_state; - new_automaton->m_table = NULL; - new_automaton->t_table = NULL; new_automaton->bitmask = NULL; new_automaton->tables_size = 0; @@ -813,8 +841,6 @@ int yr_ac_automaton_destroy( { _yr_ac_state_destroy(automaton->root); - yr_free(automaton->t_table); - yr_free(automaton->m_table); yr_free(automaton->bitmask); yr_free(automaton); @@ -832,25 +858,17 @@ int yr_ac_automaton_destroy( int yr_ac_add_string( YR_AC_AUTOMATON* automaton, YR_STRING* string, + uint32_t string_idx, YR_ATOM_LIST_ITEM* atom, - YR_ARENA* matches_arena) + YR_ARENA* arena) { - int result = ERROR_SUCCESS; - int i; - - YR_AC_STATE* state; - YR_AC_STATE* next_state; - YR_AC_MATCH* new_match; - - // For each atom create the states in the automaton. - while (atom != NULL) { - state = automaton->root; + YR_AC_STATE* state = automaton->root; - for (i = 0; i < atom->atom.length; i++) + for (int i = 0; i < atom->atom.length; i++) { - next_state = _yr_ac_next_state(state, atom->atom.bytes[i]); + YR_AC_STATE* next_state = _yr_ac_next_state(state, atom->atom.bytes[i]); if (next_state == NULL) { @@ -863,100 +881,56 @@ int yr_ac_add_string( state = next_state; } - result = yr_arena_allocate_struct( - matches_arena, + YR_ARENA_REF new_match_ref; + + FAIL_ON_ERROR(yr_arena_allocate_struct( + arena, + YR_AC_STATE_MATCHES_POOL, sizeof(YR_AC_MATCH), - (void**) &new_match, + &new_match_ref, offsetof(YR_AC_MATCH, string), offsetof(YR_AC_MATCH, forward_code), offsetof(YR_AC_MATCH, backward_code), offsetof(YR_AC_MATCH, next), - EOL); + EOL)); - if (result == ERROR_SUCCESS) - { - new_match->backtrack = state->depth + atom->backtrack; - new_match->string = string; - new_match->forward_code = atom->forward_code; - new_match->backward_code = atom->backward_code; - new_match->next = state->matches; - state->matches = new_match; - } - else - { - break; - } + YR_AC_MATCH* new_match = yr_arena_ref_to_ptr(arena, &new_match_ref); + + new_match->backtrack = state->depth + atom->backtrack; + new_match->string = yr_arena_get_ptr( + arena, YR_STRINGS_TABLE, string_idx * sizeof(struct YR_STRING)); + + new_match->forward_code = yr_arena_ref_to_ptr( + arena, &atom->forward_code_ref); + + new_match->backward_code = yr_arena_ref_to_ptr( + arena, &atom->backward_code_ref); + + // Add newly created match to the list of matches for the state. + new_match->next = yr_arena_ref_to_ptr(arena, &state->matches_ref); + state->matches_ref = new_match_ref; atom = atom->next; } - return result; + return ERROR_SUCCESS; } // // yr_ac_compile // +// Compiles the Aho-Corasick automaton, the resulting data structures are +// are written in the provided arena. +// int yr_ac_compile( YR_AC_AUTOMATON* automaton, - YR_ARENA* arena, - YR_AC_TABLES* tables) + YR_ARENA* arena) { - uint32_t i; - - FAIL_ON_ERROR(_yr_ac_create_failure_links(automaton)); + FAIL_ON_ERROR(_yr_ac_create_failure_links(automaton, arena)); FAIL_ON_ERROR(_yr_ac_optimize_failure_links(automaton)); - FAIL_ON_ERROR(_yr_ac_build_transition_table(automaton)); - - FAIL_ON_ERROR(yr_arena_reserve_memory( - arena, - automaton->tables_size * sizeof(tables->transitions[0]) + - automaton->tables_size * sizeof(tables->matches[0]))); - - FAIL_ON_ERROR(yr_arena_write_data( - arena, - automaton->t_table, - sizeof(YR_AC_TRANSITION), - (void**) &tables->transitions)); - - for (i = 1; i < automaton->tables_size; i++) - { - FAIL_ON_ERROR(yr_arena_write_data( - arena, - automaton->t_table + i, - sizeof(YR_AC_TRANSITION), - NULL)); - } - - FAIL_ON_ERROR(yr_arena_write_data( - arena, - automaton->m_table, - sizeof(YR_AC_MATCH_TABLE_ENTRY), - (void**) &tables->matches)); - - FAIL_ON_ERROR(yr_arena_make_ptr_relocatable( - arena, - tables->matches, - offsetof(YR_AC_MATCH_TABLE_ENTRY, match), - EOL)); - - for (i = 1; i < automaton->tables_size; i++) - { - void* ptr; - - FAIL_ON_ERROR(yr_arena_write_data( - arena, - automaton->m_table + i, - sizeof(YR_AC_MATCH_TABLE_ENTRY), - (void**) &ptr)); - - FAIL_ON_ERROR(yr_arena_make_ptr_relocatable( - arena, - ptr, - offsetof(YR_AC_MATCH_TABLE_ENTRY, match), - EOL)); - } + FAIL_ON_ERROR(_yr_ac_build_transition_table(automaton, arena)); return ERROR_SUCCESS; } @@ -967,10 +941,12 @@ int yr_ac_compile( // // Prints automaton for debug purposes. // - +//TODO(vmalvarez): Remove or re-implement +/* void yr_ac_print_automaton(YR_AC_AUTOMATON* automaton) { printf("-------------------------------------------------------\n"); _yr_ac_print_automaton_state(automaton->root); printf("-------------------------------------------------------\n"); } +*/ diff --git a/libyara/arena.c b/libyara/arena.c index 805f6d70..7009052b 100644 --- a/libyara/arena.c +++ b/libyara/arena.c @@ -1,5 +1,5 @@ /* -Copyright (c) 2013. The YARA Authors. All Rights Reserved. +Copyright (c) 2020. The YARA Authors. All Rights Reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -27,142 +27,50 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* -This module implements a structure I've called "arena". An arena is a data -container composed of a set of pages. The arena grows automatically when -needed by adding new pages to hold new data. Arenas can be saved and loaded -from files. - -*/ - -#include -#include -#include #include #include - #include #include #include -#include -#include + +typedef struct YR_ARENA_FILE_HEADER YR_ARENA_FILE_HEADER; +typedef struct YR_ARENA_FILE_BUFFER YR_ARENA_FILE_BUFFER; #pragma pack(push) #pragma pack(1) -typedef struct _ARENA_FILE_HEADER +struct YR_ARENA_FILE_HEADER { - char magic[4]; - uint32_t size; - uint32_t version; + uint8_t magic[4]; + uint8_t version; + uint8_t num_buffers; +}; -} ARENA_FILE_HEADER; +struct YR_ARENA_FILE_BUFFER +{ + uint64_t offset; + uint32_t size; +}; #pragma pack(pop) -#define free_space(page) \ - ((page)->size - (page)->used) - - -// -// _yr_arena_new_page -// -// Creates a new arena page of a given size -// -// Args: -// size_t size - Size of the page -// -// Returns: -// A pointer to the newly created YR_ARENA_PAGE structure -// - -static YR_ARENA_PAGE* _yr_arena_new_page( - size_t size) -{ - YR_ARENA_PAGE* new_page; - - new_page = (YR_ARENA_PAGE*) yr_malloc(sizeof(YR_ARENA_PAGE)); - - if (new_page == NULL) - return NULL; - - new_page->address = (uint8_t*) yr_malloc(size); - - if (new_page->address == NULL) - { - yr_free(new_page); - return NULL; - } - - new_page->size = size; - new_page->used = 0; - new_page->next = NULL; - new_page->prev = NULL; - new_page->reloc_list_head = NULL; - new_page->reloc_list_tail = NULL; - - return new_page; -} - - -// -// yr_arena_page_for_address -// -// Returns the page within the arena where an address reside. -// -// Args: -// YR_ARENA* arena - Pointer to the arena -// void* address - Address to be located -// -// Returns: -// A pointer the corresponding YR_ARENA_PAGE structure where the address -// resides. -// - -YR_ARENA_PAGE* yr_arena_page_for_address( - YR_ARENA* arena, - void* address) -{ - YR_ARENA_PAGE* page; - - // Most of the times this function is called with an address within - // the current page, let's check the current page first to avoid - // looping through the page list. - - page = arena->current_page; - - if (page != NULL && - (uint8_t*) address >= page->address && - (uint8_t*) address < page->address + page->used) - return page; - - page = arena->page_list_head; - - while (page != NULL) - { - if ((uint8_t*) address >= page->address && - (uint8_t*) address < page->address + page->used) - return page; - - page = page->next; - } - - return NULL; -} - - // // _yr_arena_make_ptr_relocatable // -// Tells the arena that certain addresses contains a relocatable pointer. +// Tells the arena that certain offsets within a buffer contain relocatable +// pointers. The offsets are passed as a vararg list where the end of the +// list is indicated by the special value EOL (-1), offsets in the list are +// relative to base_offset, which in turns is relative to the beginning of the +// buffer. // // Args: -// YR_ARENA* arena - Pointer the arena -// void* address - Base address -// va_list offsets - List of offsets relative to base address +// [in] YR_ARENA* arena - Pointer the arena. +// [in] int buffer_id - Buffer number. +// [in] size_t base_offset - Base offset. +// [in] va_list offsets - List of offsets relative to base offset. // // Returns: // ERROR_SUCCESS if succeed or the corresponding error code otherwise. @@ -170,438 +78,238 @@ YR_ARENA_PAGE* yr_arena_page_for_address( static int _yr_arena_make_ptr_relocatable( YR_ARENA* arena, - void* base, + int buffer_id, + yr_arena_off_t base_offset, va_list offsets) { - YR_RELOC* reloc; - YR_ARENA_PAGE* page; - - size_t offset; - size_t base_offset; + yr_arena_off_t offset; int result = ERROR_SUCCESS; - // If the arena must be relocatable. - assert(arena->flags & ARENA_FLAGS_RELOCATABLE); + offset = va_arg(offsets, yr_arena_off_t); - page = yr_arena_page_for_address(arena, base); - - assert(page != NULL); - - base_offset = (uint8_t*) base - page->address; - offset = va_arg(offsets, size_t); - - while (offset != -1) + while (offset != EOL) { - assert(page->used >= sizeof(int64_t)); - assert(base_offset + offset <= page->used - sizeof(int64_t)); - - reloc = (YR_RELOC*) yr_malloc(sizeof(YR_RELOC)); + YR_RELOC* reloc = (YR_RELOC*) yr_malloc(sizeof(YR_RELOC)); if (reloc == NULL) return ERROR_INSUFFICIENT_MEMORY; - reloc->offset = (uint32_t) (base_offset + offset); + reloc->buffer_id = buffer_id; + reloc->offset = base_offset + offset; reloc->next = NULL; - if (page->reloc_list_head == NULL) - page->reloc_list_head = reloc; + if (arena->reloc_list_head == NULL) + arena->reloc_list_head = reloc; - if (page->reloc_list_tail != NULL) - page->reloc_list_tail->next = reloc; + if (arena->reloc_list_tail != NULL) + arena->reloc_list_tail->next = reloc; - page->reloc_list_tail = reloc; + arena->reloc_list_tail = reloc; offset = va_arg(offsets, size_t); } return result; } +// Flags for _yr_arena_allocate_memory. +#define YR_ARENA_ZERO_MEMORY 1 + + +// +// _yr_arena_allocate_memory +// +// Allocates memory in a buffer within the arena. +// +// Args: +// [in] YR_ARENA* arena - Pointer to the arena. +// [in] int flags - Flags. +// [in] int buffer_id - Buffer number. +// [in] size_t size - Size of the region to be allocated. +// [out] size_t* offset - Pointer to a variable where the function puts +// the offset within the buffer of the allocated +// region. The pointer can be NULL. +// Returns: +// ERROR_SUCCESS if succeed or the corresponding error code if otherwise. +// + +static int _yr_arena_allocate_memory( + YR_ARENA* arena, + int flags, + int buffer_id, + size_t size, + YR_ARENA_REF* ref) +{ + if (buffer_id > arena->num_buffers) + return ERROR_INVALID_ARGUMENT; + + YR_ARENA_BUFFER* b = &arena->buffers[buffer_id]; + + // If the new data doesn't fit in the remaining space the buffer must be + // re-sized. This implies moving the buffer to a different memory location + // and adjusting the pointers listed in the relocation list. + + if (b->size - b->used < size) + { + size_t new_size = (b->size == 0) ? arena->initial_buffer_size : b->size * 2; + + while (new_size < b->used + size) + new_size *= 2; + + uint8_t* new_data = yr_realloc(b->data, new_size); + + if (new_data == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + // When yr_realloc uses the Windows API (HeapAlloc, HeapReAlloc) under the + // hood this is not necessary because HeapReAlloc already sets the new + // memory to zero. + #if !defined(_WIN32) && !defined(__CYGWIN__) + if (flags & YR_ARENA_ZERO_MEMORY) + memset(new_data + b->used, 0, new_size - b->used); + #endif + + YR_RELOC* reloc = arena->reloc_list_head; + + while (reloc != NULL) + { + // If the reloc entry is for the same buffer that is being relocated, + // the base pointer that we use to access the buffer must be new_data, + // as arena->buffers[reloc->buffer_id].data which is the same than + // b->data can't be accessed anymore after the call to yr_realloc. + uint8_t* base = buffer_id == reloc->buffer_id ? + new_data : arena->buffers[reloc->buffer_id].data; + + // reloc_address holds the address inside the buffer where the pointer + // to be relocated resides. + void** reloc_address = (void**) (base + reloc->offset); + + // reloc_target is the value of the relocatable pointer. + void* reloc_target = *reloc_address; + + // reloc_target points to some data inside the buffer being moved, so + // the pointer needs to be adjusted. + if ((uint8_t*) reloc_target >= b->data && + (uint8_t*) reloc_target < b->data + b->used) + { + *reloc_address = (uint8_t*) reloc_target - b->data + new_data; + } + + reloc = reloc->next; + } + + b->size = new_size; + b->data = new_data; + } + + if (ref != NULL) + { + ref->buffer_id = buffer_id; + ref->offset = b->used; + } + + b->used += size; + + return ERROR_SUCCESS; +} + // // yr_arena_create // -// Creates a new arena. +// Creates an arena with the specified number of buffers. // // Args: -// size_t initial_size - Initial size -// int flags - Flags -// YR_ARENA** arena - Address where a pointer to the new arena will be -// written to. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. +// [in] int num_buffers - Number of buffers +// [in] size_t initial_buffer_size - Initial size of each buffer. +// [out] YR_ARENA** arena - Address of a YR_ARENA* pointer that +// will receive the address of the newly +// created arena. // int yr_arena_create( - size_t initial_size, - int flags, + int number_of_buffers, + size_t initial_buffer_size, YR_ARENA** arena) { - YR_ARENA* new_arena; - YR_ARENA_PAGE* new_page; - - *arena = NULL; - new_arena = (YR_ARENA*) yr_malloc(sizeof(YR_ARENA)); + YR_ARENA* new_arena = (YR_ARENA*) yr_calloc(1, sizeof(YR_ARENA)); if (new_arena == NULL) return ERROR_INSUFFICIENT_MEMORY; - new_page = _yr_arena_new_page(initial_size); - - if (new_page == NULL) - { - yr_free(new_arena); - return ERROR_INSUFFICIENT_MEMORY; - } - - new_arena->page_list_head = new_page; - new_arena->current_page = new_page; - new_arena->flags = flags | ARENA_FLAGS_COALESCED; + new_arena->xrefs = 1; + new_arena->num_buffers = number_of_buffers; + new_arena->initial_buffer_size = initial_buffer_size; *arena = new_arena; + return ERROR_SUCCESS; } - -// -// yr_arena_destroy -// -// Destroys an arena releasing its resource. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - -void yr_arena_destroy( +void yr_arena_acquire( YR_ARENA* arena) { - YR_RELOC* reloc; - YR_RELOC* next_reloc; - YR_ARENA_PAGE* page; - YR_ARENA_PAGE* next_page; - - if (arena == NULL) - return; - - page = arena->page_list_head; - - while(page != NULL) - { - next_page = page->next; - reloc = page->reloc_list_head; - - while (reloc != NULL) - { - next_reloc = reloc->next; - yr_free(reloc); - reloc = next_reloc; - } - - yr_free(page->address); - yr_free(page); - - page = next_page; - } - - yr_free(arena); + arena->xrefs++; } - // -// yr_arena_base_address +// yr_arena_release // -// Returns the base address for the arena. +// Releases the arena. If the number of cross-references to this arena drops +// to zero the arena is destroyed and all its resources are freed. // // Args: -// YR_ARENA* arena - Pointer to the arena. -// -// Returns: -// A pointer to the arena's data. NULL if no data has been written to -// the arena yet. +// [in] YR_ARENA* arena - Pointer to the arena. // -void* yr_arena_base_address( - YR_ARENA* arena) -{ - if (arena->page_list_head->used == 0) - return NULL; - - return arena->page_list_head->address; -} - - -// -// yr_arena_next_address -// -// Given an address and an offset, returns the address where -// address + offset resides. The arena is a collection of non-contiguous -// regions of memory (pages), if address is pointing at the end of a page, -// address + offset could cross the page boundary and point at somewhere -// within the next page, this function handles these situations. It works -// also with negative offsets. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// void* address - Base address. -// int offset - Offset. -// -// Returns: -// A pointer -// - - -void* yr_arena_next_address( - YR_ARENA* arena, - void* address, - size_t offset) -{ - YR_ARENA_PAGE* page; - - page = yr_arena_page_for_address(arena, address); - - assert(page != NULL); - - if ((uint8_t*) address + offset >= page->address && - (uint8_t*) address + offset < page->address + page->used) - { - return (uint8_t*) address + offset; - } - - if (offset > 0) - { - offset -= page->address + page->used - (uint8_t*) address; - page = page->next; - - while (page != NULL) - { - if (offset < page->used) - return page->address + offset; - - offset -= page->used; - page = page->next; - } - } - else - { - offset += page->used; - page = page->prev; - - while (page != NULL) - { - if (offset < page->used) - return page->address + page->used + offset; - - offset += page->used; - page = page->prev; - } - } - - return NULL; -} - - -// -// yr_arena_coalesce -// -// Coalesce the arena into a single page. This is a required step before -// saving the arena to a file. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - -int yr_arena_coalesce( +int yr_arena_release( YR_ARENA* arena) { - YR_ARENA_PAGE* page; - YR_ARENA_PAGE* big_page; - YR_ARENA_PAGE* next_page; - YR_RELOC* reloc; + arena->xrefs--; - uint8_t** reloc_address; - uint8_t* reloc_target; - size_t total_size = 0; + if (arena->xrefs > 0) + return ERROR_SUCCESS; - page = arena->page_list_head; - - while(page != NULL) + for (int i = 0; i < arena->num_buffers; i++) { - total_size += page->used; - page = page->next; + if (arena->buffers[i].data != NULL) + yr_free(arena->buffers[i].data); } - // Create a new page that will contain the entire arena. - big_page = _yr_arena_new_page(total_size); - - if (big_page == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - // Copy data from current pages to the big page and adjust relocs. - page = arena->page_list_head; - - while (page != NULL) - { - page->new_address = big_page->address + big_page->used; - memcpy(page->new_address, page->address, page->used); - - reloc = page->reloc_list_head; - - while (reloc != NULL) - { - reloc->offset += (uint32_t) big_page->used; - reloc = reloc->next; - } - - if (big_page->reloc_list_head == NULL) - big_page->reloc_list_head = page->reloc_list_head; - - if (big_page->reloc_list_tail != NULL) - big_page->reloc_list_tail->next = page->reloc_list_head; - - if (page->reloc_list_tail != NULL) - big_page->reloc_list_tail = page->reloc_list_tail; - - big_page->used += page->used; - page = page->next; - } - - // Relocate pointers. - reloc = big_page->reloc_list_head; + YR_RELOC* reloc = arena->reloc_list_head; while (reloc != NULL) { - reloc_address = (uint8_t**) (big_page->address + reloc->offset); - reloc_target = *reloc_address; - - if (reloc_target != NULL) - { - page = yr_arena_page_for_address(arena, reloc_target); - assert(page != NULL); - *reloc_address = page->new_address + (reloc_target - page->address); - } - - reloc = reloc->next; + YR_RELOC* next = reloc->next; + yr_free(reloc); + reloc = next; } - // Release current pages. - page = arena->page_list_head; - - while(page != NULL) - { - next_page = page->next; - yr_free(page->address); - yr_free(page); - page = next_page; - } - - arena->page_list_head = big_page; - arena->current_page = big_page; - arena->flags |= ARENA_FLAGS_COALESCED; + yr_free(arena); return ERROR_SUCCESS; } -// -// yr_arena_reserve_memory -// -// Ensures that the arena have enough contiguous memory for future allocations. -// if the available space in the current page is lower than "size", a new page -// is allocated. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// size_t size - Size of the region to be reserved. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - - -int yr_arena_reserve_memory( - YR_ARENA* arena, - size_t size) -{ - YR_ARENA_PAGE* new_page; - size_t new_page_size; - uint8_t* new_page_address; - - if (size > free_space(arena->current_page)) - { - // Requested space is bigger than current page's empty space, - // lets calculate the size for a new page. - - new_page_size = arena->current_page->size * 2; - - while (new_page_size < size) - new_page_size *= 2; - - if (arena->current_page->used == 0) - { - // Current page is not used at all, it can be reallocated. - - new_page_address = (uint8_t*) yr_realloc( - arena->current_page->address, - new_page_size); - - if (new_page_address == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - arena->current_page->address = new_page_address; - arena->current_page->size = new_page_size; - } - else - { - new_page = _yr_arena_new_page(new_page_size); - - if (new_page == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - new_page->prev = arena->current_page; - arena->current_page->next = new_page; - arena->current_page = new_page; - arena->flags &= ~ARENA_FLAGS_COALESCED; - } - } - - return ERROR_SUCCESS; -} - - -// -// yr_arena_allocate_memory -// -// Allocates memory within the arena. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// size_t size - Size of the region to be allocated. -// void** allocated_memory - Address of a pointer to newly allocated -// region. -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - int yr_arena_allocate_memory( YR_ARENA* arena, + int buffer_id, size_t size, - void** allocated_memory) + YR_ARENA_REF* ref) { - FAIL_ON_ERROR(yr_arena_reserve_memory(arena, size)); + return _yr_arena_allocate_memory( + arena, 0, buffer_id, size, ref); +} - *allocated_memory = arena->current_page->address + \ - arena->current_page->used; - arena->current_page->used += size; - - return ERROR_SUCCESS; +int yr_arena_allocate_zeroed_memory( + YR_ARENA* arena, + int buffer_id, + size_t size, + YR_ARENA_REF* ref) +{ + return _yr_arena_allocate_memory( + arena, YR_ARENA_ZERO_MEMORY, buffer_id, size, ref); } @@ -616,8 +324,9 @@ int yr_arena_allocate_memory( // // yr_arena_allocate_struct( // arena, +// 0, // sizeof(MY_STRUCTURE), -// (void**) &my_structure_ptr, +// &ref, // offsetof(MY_STRUCTURE, field_1), // offsetof(MY_STRUCTURE, field_2), // .. @@ -625,13 +334,16 @@ int yr_arena_allocate_memory( // EOL); // // Args: -// YR_ARENA* arena - Pointer to the arena. -// size_t size - Size of the region to be allocated. -// void** allocated_memory - Address of a pointer to newly allocated -// region. -// ... - Variable number of offsets relative to the -// beginning of the struct. Offsets are of type -// size_t. +// [in] YR_ARENA* arena - Pointer to the arena. +// [in] int buffer_id - Buffer number. +// [in] size_t size - Size of the region to be allocated. +// [out] YR_ARENA_REF* ref - Pointer to a reference that will point to the +// newly allocated structure when the function +// returns. The pointer can be NULL if you don't +// need the reference. +// ... - Variable number of offsets relative to the +// beginning of the struct. Offsets are of type +// size_t. // // Returns: // ERROR_SUCCESS if succeed or the corresponding error code otherwise. @@ -639,29 +351,96 @@ int yr_arena_allocate_memory( int yr_arena_allocate_struct( YR_ARENA* arena, + int buffer_id, size_t size, - void** allocated_memory, + YR_ARENA_REF* ref, ...) { + YR_ARENA_REF r; int result; - va_list offsets; - va_start(offsets, allocated_memory); + va_list field_offsets; + va_start(field_offsets, ref); - result = yr_arena_allocate_memory(arena, size, allocated_memory); + result = _yr_arena_allocate_memory( + arena, YR_ARENA_ZERO_MEMORY, buffer_id, size, &r); - if (result == ERROR_SUCCESS && arena->flags & ARENA_FLAGS_RELOCATABLE) - result = _yr_arena_make_ptr_relocatable(arena, *allocated_memory, offsets); + va_end(field_offsets); - va_end(offsets); + if (result != ERROR_SUCCESS) + return result; - if (result == ERROR_SUCCESS) - memset(*allocated_memory, 0, size); + result = _yr_arena_make_ptr_relocatable( + arena, buffer_id, r.offset, field_offsets); + + if (result == ERROR_SUCCESS && ref != NULL) + { + ref->buffer_id = r.buffer_id; + ref->offset = r.offset; + } return result; } +void* yr_arena_get_ptr( + YR_ARENA* arena, + int buffer_id, + yr_arena_off_t offset) +{ + assert(buffer_id < arena->num_buffers); + assert(offset <= arena->buffers[buffer_id].used); + + return arena->buffers[buffer_id].data + offset; +} + + +yr_arena_off_t yr_arena_get_current_offset( + YR_ARENA* arena, + int buffer_id) +{ + assert(buffer_id < arena->num_buffers); + + return arena->buffers[buffer_id].used; +} + + +int yr_arena_ptr_to_ref( + YR_ARENA* arena, + const void* address, + YR_ARENA_REF* ref) +{ + *ref = YR_ARENA_NULL_REF; + + if (address == NULL) + return 1; + + for (int i = 0; i < arena->num_buffers; ++i) + { + if ((uint8_t*) address >= arena->buffers[i].data && + (uint8_t*) address < arena->buffers[i].data + arena->buffers[i].used) + { + ref->buffer_id = i; + ref->offset = (uint8_t*) address - arena->buffers[i].data; + return 1; + } + } + + return 0; +} + +void* yr_arena_ref_to_ptr( + YR_ARENA* arena, + YR_ARENA_REF* ref) +{ + if (YR_ARENA_IS_NULL_REF(*ref)) + return NULL; + + return yr_arena_get_ptr(arena, ref->buffer_id, ref->offset); +} + + + // // yr_arena_make_ptr_relocatable // @@ -669,9 +448,9 @@ int yr_arena_allocate_struct( // // Args: // YR_ARENA* arena - Pointer to the arena. -// void* base - Address within the arena. +// int buffer_id - Buffer number. // ... - Variable number of size_t arguments with offsets -// relative to base. +// within the buffer. // // Returns: // ERROR_SUCCESS if succeed or the corresponding error code otherwise. @@ -679,15 +458,16 @@ int yr_arena_allocate_struct( int yr_arena_make_ptr_relocatable( YR_ARENA* arena, - void* base, + int buffer_id, ...) { int result; va_list offsets; - va_start(offsets, base); + va_start(offsets, buffer_id); - result = _yr_arena_make_ptr_relocatable(arena, base, offsets); + result = _yr_arena_make_ptr_relocatable( + arena, buffer_id, 0, offsets); va_end(offsets); @@ -695,331 +475,135 @@ int yr_arena_make_ptr_relocatable( } -// -// yr_arena_write_data -// -// Writes data to the arena. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// void* data - Pointer to data to be written. -// size_t size - Size of data. -// void** written_data - Address where a pointer to the written data will -// be returned. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - int yr_arena_write_data( YR_ARENA* arena, + int buffer_id, const void* data, size_t size, - void** written_data) + YR_ARENA_REF* ref) { - void* output; - int result; + YR_ARENA_REF r; - if (size > free_space(arena->current_page)) + // Allocate space in the buffer. + FAIL_ON_ERROR(yr_arena_allocate_memory(arena, buffer_id, size, &r)); + + // Copy the data into the allocated space. + memcpy(arena->buffers[buffer_id].data + r.offset, data, size); + + if (ref != NULL) { - result = yr_arena_allocate_memory(arena, size, &output); - - if (result != ERROR_SUCCESS) - return result; + ref->buffer_id = r.buffer_id; + ref->offset = r.offset; } - else - { - output = arena->current_page->address + arena->current_page->used; - arena->current_page->used += size; - } - - memcpy(output, data, size); - - if (written_data != NULL) - *written_data = output; return ERROR_SUCCESS; } -// -// yr_arena_write_string -// -// Writes string to the arena. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// const char* string - Pointer to string to be written. -// char** written_string - Address where a pointer to the written data will -// be returned. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - int yr_arena_write_string( YR_ARENA* arena, + int buffer_id, const char* string, - char** written_string) + YR_ARENA_REF* ref) { return yr_arena_write_data( - arena, - (void*) string, - strlen(string) + 1, - (void**) written_string); + arena, buffer_id, string,strlen(string) + 1, ref); } -// -// yr_arena_append -// -// Appends source_arena to target_arena. This operation destroys source_arena, -// after returning any pointer to source_arena is no longer valid. The data -// from source_arena is guaranteed to be aligned to a 16 bytes boundary when -// written to the source_arena -// -// Args: -// YR_ARENA* target_arena - Pointer to target the arena. -// YR_ARENA* source_arena - Pointer to source arena. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - -int yr_arena_append( - YR_ARENA* target_arena, - YR_ARENA* source_arena) -{ - uint8_t padding_data[15]; - size_t padding_size = 16 - target_arena->current_page->used % 16; - - if (padding_size < 16) - { - memset(&padding_data, 0xCC, padding_size); - - FAIL_ON_ERROR(yr_arena_write_data( - target_arena, - padding_data, - padding_size, - NULL)); - } - - target_arena->current_page->next = source_arena->page_list_head; - source_arena->page_list_head->prev = target_arena->current_page; - target_arena->current_page = source_arena->current_page; - - yr_free(source_arena); - - return ERROR_SUCCESS; -} - - -// -// yr_arena_duplicate -// -// Duplicates the arena, making an exact copy. This function requires the -// arena to be coalesced. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// YR_ARENA** duplicated - Address where a pointer to the new arena arena -// will be returned. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - -int yr_arena_duplicate( +int yr_arena_write_uint32( YR_ARENA* arena, - YR_ARENA** duplicated) + int buffer_id, + uint32_t integer, + YR_ARENA_REF* ref) { - YR_RELOC* reloc; - YR_RELOC* new_reloc; - YR_ARENA_PAGE* page; - YR_ARENA_PAGE* new_page; - YR_ARENA* new_arena; - uint8_t** reloc_address; - uint8_t* reloc_target; - - // Arena must be coalesced and relocatable in order to be duplicated. - assert(arena->flags & ARENA_FLAGS_COALESCED); - assert(arena->flags & ARENA_FLAGS_RELOCATABLE); - - page = arena->page_list_head; - - FAIL_ON_ERROR(yr_arena_create(page->size, arena->flags, &new_arena)); - - new_page = new_arena->current_page; - new_page->used = page->used; - - memcpy(new_page->address, page->address, page->size); - - reloc = page->reloc_list_head; - - while (reloc != NULL) - { - new_reloc = (YR_RELOC*) yr_malloc(sizeof(YR_RELOC)); - - if (new_reloc == NULL) - { - yr_arena_destroy(new_arena); - return ERROR_INSUFFICIENT_MEMORY; - } - - new_reloc->offset = reloc->offset; - new_reloc->next = NULL; - - if (new_page->reloc_list_head == NULL) - new_page->reloc_list_head = new_reloc; - - if (new_page->reloc_list_tail != NULL) - new_page->reloc_list_tail->next = new_reloc; - - new_page->reloc_list_tail = new_reloc; - - reloc_address = (uint8_t**) (new_page->address + new_reloc->offset); - reloc_target = *reloc_address; - - if (reloc_target != NULL) - { - assert(reloc_target >= page->address); - assert(reloc_target < page->address + page->used); - - *reloc_address = reloc_target - \ - page->address + \ - new_page->address; - } - - reloc = reloc->next; - } - - *duplicated = new_arena; - - return ERROR_SUCCESS; + return yr_arena_write_data( + arena, buffer_id, &integer, sizeof(integer), ref); } -// -// yr_arena_load_stream -// -// Loads an arena from a stream. The resulting arena is not relocatable, which -// implies that the arena can't be duplicated with yr_arena_duplicate nor -// saved with yr_arena_save_stream. -// -// Args: -// YR_STREAM* stream - Pointer to stream object -// YR_ARENA** - Address where a pointer to the loaded arena -// will be returned -// -// Returns: -// ERROR_SUCCESS if successful, appropriate error code otherwise. -// int yr_arena_load_stream( YR_STREAM* stream, YR_ARENA** arena) { - YR_ARENA_PAGE* page; - YR_ARENA* new_arena; - ARENA_FILE_HEADER header; + YR_ARENA_FILE_HEADER hdr; - uint32_t real_hash; - uint32_t file_hash; - uint32_t reloc_offset; - uint8_t** reloc_address; - uint8_t* reloc_target; - uint32_t max_reloc_offset; - - int result; - - if (yr_stream_read(&header, sizeof(header), 1, stream) != 1) + if (yr_stream_read(&hdr, sizeof(hdr), 1, stream) != 1) return ERROR_INVALID_FILE; - if (header.magic[0] != 'Y' || - header.magic[1] != 'A' || - header.magic[2] != 'R' || - header.magic[3] != 'A') + if (hdr.magic[0] != 'Y' || + hdr.magic[1] != 'A' || + hdr.magic[2] != 'R' || + hdr.magic[3] != 'A') { return ERROR_INVALID_FILE; } - if (header.size < 2048) // compiled rules are always larger than 2KB - return ERROR_CORRUPT_FILE; - - if (header.version != ARENA_FILE_VERSION) + if (hdr.version != YR_ARENA_FILE_VERSION) return ERROR_UNSUPPORTED_FILE_VERSION; - real_hash = yr_hash(0, &header, sizeof(header)); + if (hdr.num_buffers > YR_MAX_ARENA_BUFFERS) + return ERROR_INVALID_FILE; - result = yr_arena_create(header.size, ARENA_FLAGS_COALESCED, &new_arena); + YR_ARENA_FILE_BUFFER buffers[YR_MAX_ARENA_BUFFERS]; - if (result != ERROR_SUCCESS) - return result; + int read = yr_stream_read( + buffers, sizeof(buffers[0]), hdr.num_buffers, stream); - page = new_arena->current_page; - - if (yr_stream_read(page->address, header.size, 1, stream) != 1) - { - yr_arena_destroy(new_arena); + if (read != hdr.num_buffers) return ERROR_CORRUPT_FILE; - } - page->used = header.size; + YR_ARENA* new_arena; - real_hash = yr_hash(real_hash, page->address, page->used); + FAIL_ON_ERROR(yr_arena_create(hdr.num_buffers, 1048576, &new_arena)) - if (yr_stream_read(&reloc_offset, sizeof(reloc_offset), 1, stream) != 1) + for (int i = 0; i < hdr.num_buffers; ++i) { - yr_arena_destroy(new_arena); - return ERROR_CORRUPT_FILE; - } + if (buffers[i].size == 0) + continue; - max_reloc_offset = header.size - sizeof(uint8_t*); + YR_ARENA_REF ref; - while (reloc_offset != 0xFFFFFFFF) - { - if (reloc_offset > max_reloc_offset) + FAIL_ON_ERROR_WITH_CLEANUP( + yr_arena_allocate_memory( + new_arena, i, buffers[i].size, &ref), + yr_arena_release(new_arena)) + + void* ptr = yr_arena_get_ptr(new_arena, i, ref.offset); + + if (yr_stream_read(ptr, buffers[i].size, 1, stream) != 1) { - yr_arena_destroy(new_arena); - return ERROR_CORRUPT_FILE; - } - - //yr_arena_make_ptr_relocatable(new_arena, page->address, reloc_offset, EOL); - - reloc_address = (uint8_t**) (page->address + reloc_offset); - reloc_target = *reloc_address; - - if (reloc_target == (uint8_t*) (size_t) 0xFFFABADA) - { - *reloc_address = 0; - } - else if (reloc_target < (uint8_t*) (size_t) max_reloc_offset) - { - *reloc_address += (size_t) page->address; - } - else - { - yr_arena_destroy(new_arena); - return ERROR_CORRUPT_FILE; - } - - if (yr_stream_read(&reloc_offset, sizeof(reloc_offset), 1, stream) != 1) - { - yr_arena_destroy(new_arena); + yr_arena_release(new_arena); return ERROR_CORRUPT_FILE; } } - if (yr_stream_read(&file_hash, sizeof(file_hash), 1, stream) != 1) - { - yr_arena_destroy(new_arena); - return ERROR_CORRUPT_FILE; - } + YR_ARENA_REF ref; - if (file_hash != real_hash) + while (yr_stream_read(&ref, sizeof(ref), 1, stream) == 1) { - yr_arena_destroy(new_arena); - return ERROR_CORRUPT_FILE; + YR_ARENA_BUFFER* b = &new_arena->buffers[ref.buffer_id]; + + if (ref.buffer_id >= new_arena->num_buffers || + ref.offset > b->used - sizeof(void*)) + { + yr_arena_release(new_arena); + return ERROR_CORRUPT_FILE; + } + + void** reloc_ptr = (void**) (b->data + ref.offset); + + // Let's convert the reference into a pointer. + *reloc_ptr = yr_arena_ref_to_ptr(new_arena, (YR_ARENA_REF*) reloc_ptr); + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_arena_make_ptr_relocatable( + new_arena, + ref.buffer_id, + ref.offset, + EOL), + yr_arena_release(new_arena)) } *arena = new_arena; @@ -1028,102 +612,109 @@ int yr_arena_load_stream( } -// -// yr_arena_save_stream -// -// Saves the arena into a stream. If the file exists its overwritten. This -// function requires the arena to be coalesced. -// -// Args: -// YR_ARENA* arena - Pointer to the arena. -// YR_STREAM* stream - Pointer to stream object. -// -// Returns: -// ERROR_SUCCESS if succeed or the corresponding error code otherwise. -// - int yr_arena_save_stream( YR_ARENA* arena, YR_STREAM* stream) { - YR_ARENA_PAGE* page; - YR_RELOC* reloc; - ARENA_FILE_HEADER header; + YR_ARENA_FILE_HEADER hdr; - uint32_t end_marker = 0xFFFFFFFF; - uint32_t file_hash; - uint8_t** reloc_address; - uint8_t* reloc_target; + hdr.magic[0] = 'Y'; + hdr.magic[1] = 'A'; + hdr.magic[2] = 'R'; + hdr.magic[3] = 'A'; - // Only coalesced and relocatable arenas can be saved. - assert(arena->flags & ARENA_FLAGS_COALESCED); - assert(arena->flags & ARENA_FLAGS_RELOCATABLE); + hdr.version = YR_ARENA_FILE_VERSION; + hdr.num_buffers = arena->num_buffers; - page = arena->page_list_head; - reloc = page->reloc_list_head; - - // Convert pointers to offsets before saving. - while (reloc != NULL) - { - reloc_address = (uint8_t**) (page->address + reloc->offset); - reloc_target = *reloc_address; - - if (reloc_target != NULL) - { - assert(reloc_target >= page->address); - assert(reloc_target < page->address + page->used); - *reloc_address = (uint8_t*) (*reloc_address - page->address); - } - else - { - *reloc_address = (uint8_t*) (size_t) 0xFFFABADA; - } - - reloc = reloc->next; - } - - assert(page->size < 0x80000000); // 2GB - - header.magic[0] = 'Y'; - header.magic[1] = 'A'; - header.magic[2] = 'R'; - header.magic[3] = 'A'; - header.size = (int32_t) page->size; - header.version = ARENA_FILE_VERSION; - - if (yr_stream_write(&header, sizeof(header), 1, stream) != 1) + if (yr_stream_write(&hdr, sizeof(hdr), 1, stream) != 1) return ERROR_WRITING_FILE; - if (yr_stream_write(page->address, header.size, 1, stream) != 1) - return ERROR_WRITING_FILE; + // The first buffer in the file is after the header and the buffer table, + // calculate its offset accordingly. + uint64_t offset = sizeof(YR_ARENA_FILE_HEADER) + + sizeof(YR_ARENA_FILE_BUFFER) * arena->num_buffers; - file_hash = yr_hash(0, &header, sizeof(header)); - file_hash = yr_hash(file_hash, page->address, page->used); - - reloc = page->reloc_list_head; - - // Convert offsets back to pointers. - while (reloc != NULL) + for (int i = 0; i < arena->num_buffers; ++i) { - if (yr_stream_write(&reloc->offset, sizeof(reloc->offset), 1, stream) != 1) + YR_ARENA_FILE_BUFFER buffer = { + .offset = offset, + .size = arena->buffers[i].used, + }; + + if (yr_stream_write(&buffer, sizeof(buffer), 1, stream) != 1) return ERROR_WRITING_FILE; - reloc_address = (uint8_t**) (page->address + reloc->offset); - reloc_target = *reloc_address; + offset += buffer.size; + } - if (reloc_target != (void*) (size_t) 0xFFFABADA) - *reloc_address += (size_t) page->address; - else - *reloc_address = 0; + // Iterate the relocation list and replace all the relocatable pointers by + // references to the buffer and offset where they are pointing to. All + // relocatable pointers are expected to be null or point to data stored in + // some of the arena's buffers. If a relocatable pointer points outside the + // arena that's an error. + YR_RELOC* reloc = arena->reloc_list_head; + + while (reloc != NULL) + { + // reloc_ptr is a pointer to the relocatable pointer, while *reloc_ptr + // is the relocatable pointer itself. + void** reloc_ptr = (void**) ( + arena->buffers[reloc->buffer_id].data + reloc->offset); + + YR_ARENA_REF ref; + + int found = yr_arena_ptr_to_ref(arena, *reloc_ptr, &ref); + + // yr_arena_ptr_to_ref returns 0 if the relocatable pointer is pointing + // outside the arena, this should not happen. + assert(found); + + // Replace the relocatable pointer with a reference that holds information + // about the buffer and offset where the relocatable pointer is pointing to. + memcpy(reloc_ptr, &ref, sizeof(ref)); reloc = reloc->next; } - if (yr_stream_write(&end_marker, sizeof(end_marker), 1, stream) != 1) - return ERROR_WRITING_FILE; + // Now that all relocatable pointers are converted to references, write the + // buffers. + for (int i = 0; i < arena->num_buffers; ++i) + { + YR_ARENA_BUFFER* b = &arena->buffers[i]; - if (yr_stream_write(&file_hash, sizeof(file_hash), 1, stream) != 1) - return ERROR_WRITING_FILE; + if (b->used > 0) + if (yr_stream_write(b->data, b->used, 1, stream) != 1) + return ERROR_WRITING_FILE; + } + + // Write the relocation list and restore the pointers back. + reloc = arena->reloc_list_head; + + while (reloc != NULL) + { + YR_ARENA_REF ref = { + .buffer_id = reloc->buffer_id, + .offset = reloc->offset, + }; + + if (yr_stream_write(&ref, sizeof(ref), 1, stream) != 1) + return ERROR_WRITING_FILE; + + void** reloc_ptr = (void**) ( + arena->buffers[reloc->buffer_id].data + reloc->offset); + + // reloc_ptr is now pointing to a YR_ARENA_REF. + YR_ARENA_REF* ref_ptr = (YR_ARENA_REF*) reloc_ptr; + + // Let's convert the reference into a pointer again. + *reloc_ptr = yr_arena_ref_to_ptr(arena, ref_ptr); + + reloc = reloc->next; + } return ERROR_SUCCESS; } + + + + diff --git a/libyara/atoms.c b/libyara/atoms.c index e8359404..370de687 100644 --- a/libyara/atoms.c +++ b/libyara/atoms.c @@ -160,7 +160,7 @@ int yr_atoms_heuristic_quality( else quality += 20; }; - if (!yr_bitmask_isset(seen_bytes, atom->bytes[i])) + if (!yr_bitmask_is_set(seen_bytes, atom->bytes[i])) { yr_bitmask_set(seen_bytes, atom->bytes[i]); unique_bytes++; @@ -172,10 +172,10 @@ int yr_atoms_heuristic_quality( // it heavily. if (unique_bytes == 1 && - (yr_bitmask_isset(seen_bytes, 0x00) || - yr_bitmask_isset(seen_bytes, 0x20) || - yr_bitmask_isset(seen_bytes, 0xCC) || - yr_bitmask_isset(seen_bytes, 0xFF))) + (yr_bitmask_is_set(seen_bytes, 0x00) || + yr_bitmask_is_set(seen_bytes, 0x20) || + yr_bitmask_is_set(seen_bytes, 0xCC) || + yr_bitmask_is_set(seen_bytes, 0xFF))) { quality -= 10 * atom->length; } @@ -586,8 +586,8 @@ static int _yr_atoms_choose( if (item->atom.length > 0) { - item->forward_code = node->re_nodes[shift]->forward_code; - item->backward_code = node->re_nodes[shift]->backward_code; + item->forward_code_ref = node->re_nodes[shift]->forward_code_ref; + item->backward_code_ref = node->re_nodes[shift]->backward_code_ref; item->backtrack = 0; item->next = NULL; @@ -789,8 +789,8 @@ static int _yr_atoms_case_insensitive( } new_atom->atom.length = atom_length; - new_atom->forward_code = atom->forward_code; - new_atom->backward_code = atom->backward_code; + new_atom->forward_code_ref = atom->forward_code_ref; + new_atom->backward_code_ref = atom->backward_code_ref; new_atom->backtrack = atom->backtrack; new_atom->next = *case_insensitive_atoms; @@ -812,7 +812,7 @@ static int _yr_atoms_case_insensitive( // _yr_atoms_xor // // For a given list of atoms returns another list after a single byte xor -// has been applied to it (0x01 - 0xff). +// has been applied to it. // static int _yr_atoms_xor( @@ -844,8 +844,8 @@ static int _yr_atoms_xor( } new_atom->atom.length = yr_min(atom->atom.length, YR_MAX_ATOM_LENGTH); - new_atom->forward_code = atom->forward_code; - new_atom->backward_code = atom->backward_code; + new_atom->forward_code_ref = atom->forward_code_ref; + new_atom->backward_code_ref = atom->backward_code_ref; new_atom->backtrack = atom->backtrack; new_atom->next = *xor_atoms; @@ -900,8 +900,8 @@ static int _yr_atoms_wide( } new_atom->atom.length = yr_min(atom->atom.length * 2, YR_MAX_ATOM_LENGTH); - new_atom->forward_code = atom->forward_code; - new_atom->backward_code = atom->backward_code; + new_atom->forward_code_ref = atom->forward_code_ref; + new_atom->backward_code_ref = atom->backward_code_ref; new_atom->backtrack = atom->backtrack * 2; new_atom->next = *wide_atoms; @@ -1411,7 +1411,11 @@ int yr_atoms_extract_from_re( *atoms = NULL; }); - if (modifier.flags & STRING_GFLAGS_WIDE) + // Don't do convert atoms to wide here if either base64 modifier is used. + // This is to avoid the situation where we have "base64 wide" because + // the wide has already been applied BEFORE the base64 encoding. + if (modifier.flags & STRING_FLAGS_WIDE && + !(modifier.flags & STRING_FLAGS_BASE64 || modifier.flags & STRING_FLAGS_BASE64_WIDE)) { FAIL_ON_ERROR_WITH_CLEANUP( _yr_atoms_wide(*atoms, &wide_atoms), @@ -1421,7 +1425,7 @@ int yr_atoms_extract_from_re( *atoms = NULL; }); - if (modifier.flags & STRING_GFLAGS_ASCII) + if (modifier.flags & STRING_FLAGS_ASCII) { *atoms = _yr_atoms_list_concat(*atoms, wide_atoms); } @@ -1432,7 +1436,7 @@ int yr_atoms_extract_from_re( } } - if (modifier.flags & STRING_GFLAGS_NO_CASE) + if (modifier.flags & STRING_FLAGS_NO_CASE) { FAIL_ON_ERROR_WITH_CLEANUP( _yr_atoms_case_insensitive(*atoms, &case_insensitive_atoms), @@ -1456,8 +1460,8 @@ int yr_atoms_extract_from_re( (*atoms)->atom.length = 0; (*atoms)->backtrack = 0; - (*atoms)->forward_code = re_ast->root_node->forward_code; - (*atoms)->backward_code = NULL; + (*atoms)->forward_code_ref = re_ast->root_node->forward_code_ref; + (*atoms)->backward_code_ref = YR_ARENA_NULL_REF; (*atoms)->next = NULL; } @@ -1494,8 +1498,8 @@ int yr_atoms_extract_from_string( if (item == NULL) return ERROR_INSUFFICIENT_MEMORY; - item->forward_code = NULL; - item->backward_code = NULL; + item->forward_code_ref = YR_ARENA_NULL_REF; + item->backward_code_ref = YR_ARENA_NULL_REF; item->next = NULL; item->backtrack = 0; @@ -1532,7 +1536,7 @@ int yr_atoms_extract_from_string( *atoms = item; *min_atom_quality = max_quality; - if (modifier.flags & STRING_GFLAGS_WIDE) + if (modifier.flags & STRING_FLAGS_WIDE) { FAIL_ON_ERROR_WITH_CLEANUP( _yr_atoms_wide(*atoms, &wide_atoms), @@ -1542,7 +1546,7 @@ int yr_atoms_extract_from_string( *atoms = NULL; }); - if (modifier.flags & STRING_GFLAGS_ASCII) + if (modifier.flags & STRING_FLAGS_ASCII) { *atoms = _yr_atoms_list_concat(*atoms, wide_atoms); } @@ -1553,7 +1557,7 @@ int yr_atoms_extract_from_string( } } - if (modifier.flags & STRING_GFLAGS_NO_CASE) + if (modifier.flags & STRING_FLAGS_NO_CASE) { FAIL_ON_ERROR_WITH_CLEANUP( _yr_atoms_case_insensitive(*atoms, &case_insensitive_atoms), @@ -1566,7 +1570,7 @@ int yr_atoms_extract_from_string( *atoms = _yr_atoms_list_concat(*atoms, case_insensitive_atoms); } - if (modifier.flags & STRING_GFLAGS_XOR) + if (modifier.flags & STRING_FLAGS_XOR) { FAIL_ON_ERROR_WITH_CLEANUP( _yr_atoms_xor(*atoms, modifier.xor_min, modifier.xor_max, &xor_atoms), @@ -1576,8 +1580,18 @@ int yr_atoms_extract_from_string( *atoms = NULL; }); - yr_atoms_list_destroy(*atoms); - *atoms = xor_atoms; + yr_atoms_list_destroy(*atoms); + *atoms = xor_atoms; + + } + + // Recheck the atom quality, in case we have just generated some poor atoms. + // https://github.com/VirusTotal/yara/issues/1172 + for (item = *atoms; item != NULL; item = item->next) + { + quality = config->get_atom_quality(config, &item->atom); + if (quality < *min_atom_quality) + *min_atom_quality = quality; } return ERROR_SUCCESS; diff --git a/libyara/base64.c b/libyara/base64.c new file mode 100644 index 00000000..b41bc7f6 --- /dev/null +++ b/libyara/base64.c @@ -0,0 +1,506 @@ +/* +Copyright (c) 2020. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include + +#include +#include +#include +#include +#include + + +// +// _yr_modified_base64_encode +// +// Given a pointer to a SIZED_STRING append 0, 1 or 2 bytes and base64 encode +// the string. The number of padding bytes is returned in "pad" and the caller +// is expected to trim the appropriate number of leading and trailing bytes. +// +// This is based upon the ideas at: +// https://www.leeholmes.com/blog/2019/12/10/searching-for-content-in-base-64-strings-2/ +// +// The caller is responsible for freeing the returned string. +// +static SIZED_STRING* _yr_modified_base64_encode( + SIZED_STRING* in, + SIZED_STRING* alphabet, + int i, + int* pad) +{ + uint8_t* src = (uint8_t*) in->c_string; + size_t len = in->length; + SIZED_STRING* out; + uint8_t* p; + uint8_t* end; + char* alphabet_str = alphabet->c_string; + uint8_t* tmp; + int j; + + *pad = ((i + len) % 3) ? 3 - ((i + len) % 3) : 0; + + // Add "i" for the number of prepended bytes. + out = (SIZED_STRING*) yr_malloc( + sizeof(SIZED_STRING) + i + ((len * 4 + 3) / 3) + *pad); + if (out == NULL) + return NULL; + + tmp = (uint8_t*) yr_malloc(sizeof(uint8_t) * (len + i)); + if (tmp == NULL) + { + yr_free(out); + return NULL; + } + + // Prepend appropriate number of bytes and copy remaining input bytes into + // temporary buffer. + for (j = 0; j < i; j++) + tmp[j] = 'A'; + + memcpy(tmp + j, src, len); + src = tmp; + + p = (uint8_t*) out->c_string; + end = src + len + j; + while (end - src >= 3) + { + *p++ = alphabet_str[src[0] >> 2]; + *p++ = alphabet_str[((src[0] & 0x03) << 4 | src[1] >> 4)]; + *p++ = alphabet_str[((src[1] & 0x0f) << 2 | (src[2] >> 6))]; + *p++ = alphabet_str[src[2] & 0x3f]; + src += 3; + } + + // Handle remaining bytes and padding. + if (end - src) + { + *p++ = alphabet_str[src[0] >> 2]; + if (end - src == 1) + { + *p++ = alphabet_str[(src[0] & 0x03) << 4]; + *p++ = '='; + } + else + { + *p++ = alphabet_str[((src[0] & 0x03) << 4 | src[1] >> 4)]; + *p++ = alphabet_str[(src[1] & 0x0f) << 2]; + } + *p++ = '='; + } + + yr_free(tmp); + out->length = p - (uint8_t*) out->c_string; + return out; +} + + +// +// Given a base64 encoded string, return a new string with leading and trailing +// bytes stripped appropriately. The number of leading bytes to skip is always +// (i + 1) or zero when no leading bytes are added and the number of trailing +// bytes is always (pad + 1) or zero when pad is zero. Also, convert the final +// string to wide if desired. +// +// Note: This implementation assumes you only prepend 0, 1 or 2 bytes. +// +static SIZED_STRING* _yr_base64_get_base64_substring( + SIZED_STRING* encoded_str, + int wide, + int i, + int pad) +{ + SIZED_STRING* new_str; + SIZED_STRING* final_str; + char* start; + uint32_t length; + int trailing; + int leading; + + trailing = pad ? pad + 1 : 0; + leading = i ? i + 1 : 0; + + length = encoded_str->length - (leading + trailing); + + new_str = (SIZED_STRING*) yr_malloc(sizeof(SIZED_STRING) + length); + + if (new_str == NULL) + return NULL; + + start = encoded_str->c_string + leading; + + memcpy(new_str->c_string, start, length); + new_str->length = length; + + if (wide) + { + final_str = sized_string_convert_to_wide(new_str); + yr_free(new_str); + } + else + { + final_str = new_str; + } + + return final_str; +} + + +// RE metacharacters which need to be escaped when generating the final RE. +#define IS_METACHAR(x) \ + (x == '\\' || \ + x == '^' || \ + x == '$' || \ + x == '|' || \ + x == '(' || \ + x == ')' || \ + x == '[' || \ + x == ']' || \ + x == '*' || \ + x == '?' || \ + x == '{' || \ + x == ',' || \ + x == '.' || \ + x == '+' || \ + x == '}') + + +// +// Given a SIZED_STRING return the number of characters which will need to be +// escaped when generating the final string to pass to the regexp compiler. +// +static int _yr_base64_count_escaped( + SIZED_STRING* str) +{ + uint32_t i; + int c = 0; + + for (i = 0; i < str->length; i++) + { + // We must be careful to escape null bytes because they break the RE lexer. + if (IS_METACHAR(str->c_string[i])) + c++; + else if (str->c_string[i] == '\x00') + c += 4; + } + + return c; +} + +// +// Create nodes representing the different encodings of a base64 string. +// +static int _yr_base64_create_nodes( + SIZED_STRING* str, + SIZED_STRING* alphabet, + int wide, + BASE64_NODE** head, + BASE64_NODE** tail) +{ + SIZED_STRING* encoded_str; + SIZED_STRING* final_str; + BASE64_NODE* node; + int i; + int pad; + + for (i = 0; i <= 2; i++) + { + node = (BASE64_NODE*) yr_malloc(sizeof(BASE64_NODE)); + if (node == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + FAIL_ON_NULL_WITH_CLEANUP( + encoded_str = _yr_modified_base64_encode(str, alphabet, i, &pad), + yr_free(node)); + + // Now take the encoded string and strip the bytes which are affected by + // the leading and trailing bytes of the plaintext. + FAIL_ON_NULL_WITH_CLEANUP( + final_str = _yr_base64_get_base64_substring(encoded_str, wide, i, pad), + { + yr_free(encoded_str); + yr_free(node); + }); + + yr_free(encoded_str); + + node->str = final_str; + node->escaped = _yr_base64_count_escaped(node->str); + node->next = NULL; + + if (*head == NULL) + *head = node; + + if (*tail == NULL) + { + *tail = node; + } + else + { + (*tail)->next = node; + *tail = node; + } + } + + return ERROR_SUCCESS; +} + + +// +// Useful for printing the encoded strings. +// +void _yr_base64_print_nodes( + BASE64_NODE* head) +{ + size_t i; + BASE64_NODE* p = head; + + while (p != NULL) + { + for (i = 0; i < p->str->length; i++) + { + if (p->str->c_string[i] >= 32 && p->str->c_string[i] <= 126) + printf("%c", p->str->c_string[i]); + else + printf("\\x%02x", p->str->c_string[i]); + } + printf("\n"); + + p = p->next; + } + + return; +} + + +// +// Destroy a list of base64 nodes. +// +static void _yr_base64_destroy_nodes( + BASE64_NODE* head) +{ + BASE64_NODE* p = head; + BASE64_NODE* next; + + while (p != NULL) + { + yr_free(p->str); + next = p->next; + yr_free(p); + p = next; + } + + return; +} + + +// +// Create the regexp that is the alternatives of each of the strings collected +// in the BASE64_NODE list. +// +int _yr_base64_create_regexp( + BASE64_NODE* head, + RE_AST** re_ast, + RE_ERROR* re_error) +{ + char* re_str; + char* s; + uint32_t i; + uint32_t length = 0; + uint32_t c = 0; // The number of nodes in the list, used to know how many '|'. + BASE64_NODE* p = head; + + while (p != NULL) + { + length += (p->str->length + p->escaped); + c++; + p = p->next; + } + + if (c == 0) + return ERROR_INSUFFICIENT_MEMORY; + + // Make sure to include '(' and ')'. + // The number of '|' is number of nodes - 1. + re_str = (char*) yr_malloc(length + 2 + (c - 1) + 1); + if (re_str == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + s = re_str; + p = head; + *s++ = '('; + while (p != NULL) + { + for (i = 0; i < p->str->length; i++) + { + if (IS_METACHAR(p->str->c_string[i])) + *s++ = '\\'; + + if (p->str->c_string[i] == '\x00') + { + *s++ = '\\'; + *s++ = 'x'; + *s++ = '0'; + *s++ = '0'; + } + else + *s++ = p->str->c_string[i]; + } + + if (p->next != NULL) + *s++ = '|'; + + p = p->next; + } + *s++ = ')'; + *s = '\x00'; + + // Useful for debugging as long as the string has no NULL bytes in it. ;) + //printf("%s\n", re_str); + + FAIL_ON_ERROR_WITH_CLEANUP( + yr_re_parse(re_str, re_ast, re_error), + yr_free(re_str)); + + yr_free(re_str); + return ERROR_SUCCESS; +} + + +// +// Given a string and an alphabet, generate the RE_AST suitable for representing +// the different encodings of the string. This means we generate +// "(ABCD|EFGH|IJKL)" and must be careful to escape any special characters as +// a result of the base64 encoding. +// +// This uses ideas from: +// https://www.leeholmes.com/blog/2019/12/10/searching-for-content-in-base-64-strings-2/ +// +// This does not emit the code for the RE. A further call to yr_re_ast_emit_code +// is required to get the code. +// +int yr_base64_ast_from_string( + SIZED_STRING* in_str, + YR_MODIFIER modifier, + RE_AST** re_ast, + RE_ERROR* error) +{ + BASE64_NODE* head = NULL; + BASE64_NODE* tail = NULL; + SIZED_STRING* wide_str; + + if (modifier.flags & STRING_FLAGS_WIDE) + { + wide_str = sized_string_convert_to_wide(in_str); + + if (modifier.flags & STRING_FLAGS_BASE64) + { + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_nodes(wide_str, modifier.alphabet, 0, &head, &tail), + { + strcpy(error->message, "Failure encoding base64 wide string"); + yr_free(wide_str); + _yr_base64_destroy_nodes(head); + }); + } + + if (modifier.flags & STRING_FLAGS_BASE64_WIDE) + { + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_nodes(wide_str, modifier.alphabet, 1, &head, &tail), + { + strcpy(error->message, "Failure encoding base64wide wide string"); + yr_free(wide_str); + _yr_base64_destroy_nodes(head); + }); + } + + yr_free(wide_str); + } + + if (modifier.flags & STRING_FLAGS_ASCII) + { + if (modifier.flags & STRING_FLAGS_BASE64) + { + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_nodes(in_str, modifier.alphabet, 0, &head, &tail), + { + strcpy(error->message, "Failure encoding base64 ascii string"); + _yr_base64_destroy_nodes(head); + }); + } + + if (modifier.flags & STRING_FLAGS_BASE64_WIDE) + { + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_nodes(in_str, modifier.alphabet, 1, &head, &tail), + { + strcpy(error->message, "Failure encoding base64wide ascii string"); + _yr_base64_destroy_nodes(head); + }); + } + } + + if (!(modifier.flags & STRING_FLAGS_WIDE) && + !(modifier.flags & STRING_FLAGS_ASCII)) + { + if (modifier.flags & STRING_FLAGS_BASE64) + { + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_nodes(in_str, modifier.alphabet, 0, &head, &tail), + { + strcpy(error->message, "Failure encoding base64 string"); + _yr_base64_destroy_nodes(head); + }); + } + + if (modifier.flags & STRING_FLAGS_BASE64_WIDE) + { + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_nodes(in_str, modifier.alphabet, 1, &head, &tail), + { + strcpy(error->message, "Failure encoding base64wide string"); + _yr_base64_destroy_nodes(head); + }); + } + } + + // Useful for printing the contents of the nodes, to make sure they were + // encoded and stripped properly. + //_yr_base64_print_nodes(head); + + // Create the final regex string to be parsed from all the nodes. + // Error message is filled in by the caller in case of failure. + FAIL_ON_ERROR_WITH_CLEANUP( + _yr_base64_create_regexp(head, re_ast, error), + _yr_base64_destroy_nodes(head)); + + _yr_base64_destroy_nodes(head); + + return ERROR_SUCCESS; +} diff --git a/libyara/bitmask.c b/libyara/bitmask.c index e0fe662d..40b36793 100644 --- a/libyara/bitmask.c +++ b/libyara/bitmask.c @@ -36,7 +36,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // yr_bitmask_find_non_colliding_offset // // Finds the smaller offset within bitmask A where bitmask B can be accommodated -// without bit collisions. A collision occurs when bots bitmasks have a bit set +// without bit collisions. A collision occurs when both bitmasks have a bit set // to 1 at the same offset. This function assumes that the first bit in B is 1 // and do optimizations that rely on that. // @@ -44,7 +44,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // a value that is used for speeding-up subsequent searches over the same // bitmask A. When called for the first time with some bitmask A, the pointer // must point to a zero-initialized uint32_t. In the next call the function uses -// the previously stored value for skiping over a portion of the A bitmask and +// the previously stored value for skipping over a portion of the A bitmask and // updates the value. // // Args: @@ -71,7 +71,7 @@ uint32_t yr_bitmask_find_non_colliding_offset( // Ensure that the first bit of bitmask B is set, as this function does some // optimizations that rely on that. - assert(yr_bitmask_isset(b, 0)); + assert(yr_bitmask_is_set(b, 0)); // Skip all slots that are filled with 1s. It's safe to do that because the // first bit of B is 1, so we won't be able to accommodate B at any offset diff --git a/libyara/compiler.c b/libyara/compiler.c index 492a18c2..dedbbd20 100644 --- a/libyara/compiler.c +++ b/libyara/compiler.c @@ -150,6 +150,22 @@ const char* _yr_compiler_default_include_callback( } +// +// _yr_compiler_get_rule_by_idx returns a rule given its index in the rules +// table. The returned pointer is valid as long as no other rule is written +// to the table. This is because the write operation may cause the table to +// be moved to a different location in memory. Use the pointer only in a +// limited scope where you can be sure that no other rule is being written +// during the pointer's lifetime. +// +YR_RULE* _yr_compiler_get_rule_by_idx( + YR_COMPILER* compiler, uint32_t rule_idx) +{ + return (YR_RULE*) yr_arena_get_ptr( + compiler->arena, YR_RULES_TABLE, rule_idx * sizeof(YR_RULE)); +} + + YR_API int yr_compiler_create( YR_COMPILER** compiler) { @@ -161,8 +177,14 @@ YR_API int yr_compiler_create( if (new_compiler == NULL) return ERROR_INSUFFICIENT_MEMORY; + new_compiler->current_rule_idx = UINT32_MAX; + new_compiler->next_rule_idx = 0; + new_compiler->current_string_idx = 0; + new_compiler->current_namespace_idx = 0; + new_compiler->num_namespaces = 0; new_compiler->errors = 0; new_compiler->callback = NULL; + new_compiler->rules = NULL; new_compiler->include_callback = _yr_compiler_default_include_callback; new_compiler->incl_clbk_user_data = NULL; new_compiler->include_free = _yr_compiler_default_include_free; @@ -175,9 +197,7 @@ YR_API int yr_compiler_create( new_compiler->fixup_stack_head = NULL; new_compiler->loop_index = -1; new_compiler->loop_for_of_var_index = -1; - new_compiler->compiled_rules_arena = NULL; - new_compiler->namespaces_count = 0; - new_compiler->current_rule = NULL; + new_compiler->atoms_config.get_atom_quality = yr_atoms_heuristic_quality; new_compiler->atoms_config.quality_warning_threshold = \ YR_ATOM_QUALITY_WARNING_THRESHOLD; @@ -192,43 +212,7 @@ YR_API int yr_compiler_create( if (result == ERROR_SUCCESS) result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->sz_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->rules_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->strings_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->code_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->re_code_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->externals_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->namespaces_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->metas_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->automaton_arena); - - if (result == ERROR_SUCCESS) - result = yr_arena_create( - 65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->matches_arena); + YR_NUM_SECTIONS, 64, &new_compiler->arena); if (result == ERROR_SUCCESS) result = yr_ac_automaton_create(&new_compiler->automaton); @@ -249,20 +233,7 @@ YR_API int yr_compiler_create( YR_API void yr_compiler_destroy( YR_COMPILER* compiler) { - YR_FIXUP* fixup; - int i; - - yr_arena_destroy(compiler->compiled_rules_arena); - yr_arena_destroy(compiler->sz_arena); - yr_arena_destroy(compiler->rules_arena); - yr_arena_destroy(compiler->strings_arena); - yr_arena_destroy(compiler->code_arena); - yr_arena_destroy(compiler->re_code_arena); - yr_arena_destroy(compiler->externals_arena); - yr_arena_destroy(compiler->namespaces_arena); - yr_arena_destroy(compiler->metas_arena); - yr_arena_destroy(compiler->automaton_arena); - yr_arena_destroy(compiler->matches_arena); + yr_arena_release(compiler->arena); if (compiler->automaton != NULL) yr_ac_automaton_destroy(compiler->automaton); @@ -282,10 +253,10 @@ YR_API void yr_compiler_destroy( if (compiler-> atoms_config.free_quality_table) yr_free(compiler->atoms_config.quality_table); - for (i = 0; i < compiler->file_name_stack_ptr; i++) + for (int i = 0; i < compiler->file_name_stack_ptr; i++) yr_free(compiler->file_name_stack[i]); - fixup = compiler->fixup_stack_head; + YR_FIXUP* fixup = compiler->fixup_stack_head; while (fixup != NULL) { @@ -337,10 +308,10 @@ YR_API void yr_compiler_set_re_ast_callback( // compiler for choosing the best atoms from regular expressions and strings. // When a quality table is set, the compiler uses yr_atoms_table_quality // instead of yr_atoms_heuristic_quality for computing atom quality. The table -// has an arbitary number of entries, each composed of YR_MAX_ATOM_LENGTH + 1 +// has an arbitrary number of entries, each composed of YR_MAX_ATOM_LENGTH + 1 // bytes. The first YR_MAX_ATOM_LENGTH bytes from each entry are the atom's // ones, and the remaining byte is a value in the range 0-255 determining the -// atom's quality. Entries must be lexicografically sorted by atom in ascending +// atom's quality. Entries must be lexicographically sorted by atom in ascending // order. // // [ atom (YR_MAX_ATOM_LENGTH bytes) ] [ quality (1 byte) ] @@ -357,7 +328,7 @@ YR_API void yr_compiler_set_re_ast_callback( // caller is responsible for freeing the table. // // The "warning_threshold" argument must be a number between 0 and 255, if some -// atom choosen for a string have a quality below the specified threshold a +// atom chosen for a string have a quality below the specified threshold a // warning like " is slowing down scanning" is shown. YR_API void yr_compiler_set_atom_quality_table( @@ -378,7 +349,7 @@ YR_API void yr_compiler_set_atom_quality_table( // yr_compiler_set_atom_quality_table // // Load an atom quality table from a file. The file's content must have the -// format explained in the decription for yr_compiler_set_atom_quality_table. +// format explained in the description for yr_compiler_set_atom_quality_table. // YR_API int yr_compiler_load_atom_quality_table( @@ -479,8 +450,10 @@ int _yr_compiler_get_var_frame( int i, result = 0; for (i = 0; i < compiler->loop_index; i++) - result += compiler->loop[i].vars_count + - compiler->loop[i].vars_internal_count; + { + result += compiler->loop[i].vars_count + + compiler->loop[i].vars_internal_count; + } return result; } @@ -504,57 +477,48 @@ static int _yr_compiler_set_namespace( YR_COMPILER* compiler, const char* namespace_) { - YR_NAMESPACE* ns; + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, YR_NAMESPACES_TABLE, 0); - char* ns_name; - int result; - int i; - bool found; + bool found = false; - ns = (YR_NAMESPACE*) yr_arena_base_address(compiler->namespaces_arena); - found = false; - - for (i = 0; i < compiler->namespaces_count; i++) + for (int i = 0; i < compiler->num_namespaces; i++, ns++) { if (strcmp(ns->name, namespace_) == 0) { found = true; + compiler->current_namespace_idx = i; break; } - - ns = (YR_NAMESPACE*) yr_arena_next_address( - compiler->namespaces_arena, - ns, - sizeof(YR_NAMESPACE)); } if (!found) { - result = yr_arena_write_string( - compiler->sz_arena, + YR_ARENA_REF ref; + + FAIL_ON_ERROR(yr_arena_allocate_struct( + compiler->arena, + YR_NAMESPACES_TABLE, + sizeof(YR_NAMESPACE), + &ref, + offsetof(YR_NAMESPACE, name), + EOL)); + + ns = (YR_NAMESPACE*) yr_arena_ref_to_ptr(compiler->arena, &ref); + + FAIL_ON_ERROR(yr_arena_write_string( + compiler->arena, + YR_SZ_POOL, namespace_, - &ns_name); + &ref)); - if (result == ERROR_SUCCESS) - result = yr_arena_allocate_struct( - compiler->namespaces_arena, - sizeof(YR_NAMESPACE), - (void**) &ns, - offsetof(YR_NAMESPACE, name), - EOL); + ns->name = (const char*) yr_arena_ref_to_ptr(compiler->arena, &ref); + ns->idx = compiler->num_namespaces; - if (result != ERROR_SUCCESS) - return result; - - ns->name = ns_name; - - for (i = 0; i < YR_MAX_THREADS; i++) - ns->t_flags[i] = 0; - - compiler->namespaces_count++; + compiler->current_namespace_idx = compiler->num_namespaces; + compiler->num_namespaces++; } - compiler->current_namespace = ns; return ERROR_SUCCESS; } @@ -570,7 +534,7 @@ YR_API int yr_compiler_add_file( // Don't allow yr_compiler_add_file() after // yr_compiler_get_rules() has been called. - assert(compiler->compiled_rules_arena == NULL); + assert(compiler->rules == NULL); // Don't allow calls to yr_compiler_add_file() if a previous call to // yr_compiler_add_XXXX failed. @@ -608,7 +572,7 @@ YR_API int yr_compiler_add_fd( // Don't allow yr_compiler_add_fd() after // yr_compiler_get_rules() has been called. - assert(compiler->compiled_rules_arena == NULL); + assert(compiler->rules == NULL); // Don't allow calls to yr_compiler_add_fd() if a previous call to // yr_compiler_add_XXXX failed. @@ -643,7 +607,7 @@ YR_API int yr_compiler_add_string( // Don't allow calls to yr_compiler_add_string() after // yr_compiler_get_rules() has been called. - assert(compiler->compiled_rules_arena == NULL); + assert(compiler->rules == NULL); // Don't allow calls to yr_compiler_add_string() if a previous call to // yr_compiler_add_XXXX failed. @@ -665,170 +629,63 @@ YR_API int yr_compiler_add_string( static int _yr_compiler_compile_rules( YR_COMPILER* compiler) { - YARA_RULES_FILE_HEADER* rules_file_header = NULL; - YR_ARENA* arena = NULL; YR_RULE null_rule; YR_EXTERNAL_VARIABLE null_external; - YR_AC_TABLES tables; uint8_t halt = OP_HALT; - int result; // Write halt instruction at the end of code. - yr_arena_write_data( - compiler->code_arena, + FAIL_ON_ERROR(yr_arena_write_data( + compiler->arena, + YR_CODE_SECTION, &halt, sizeof(uint8_t), - NULL); + NULL)); // Write a null rule indicating the end. memset(&null_rule, 0xFA, sizeof(YR_RULE)); - null_rule.g_flags = RULE_GFLAGS_NULL; + null_rule.flags = RULE_FLAGS_NULL; - yr_arena_write_data( - compiler->rules_arena, - &null_rule, - sizeof(YR_RULE), - NULL); + FAIL_ON_ERROR(yr_arena_write_data( + compiler->arena, + YR_RULES_TABLE, + &null_rule, + sizeof(YR_RULE), + NULL)); - // Write a null external the end. + // Write a null external indicating the end. memset(&null_external, 0xFA, sizeof(YR_EXTERNAL_VARIABLE)); null_external.type = EXTERNAL_VARIABLE_TYPE_NULL; - yr_arena_write_data( - compiler->externals_arena, + FAIL_ON_ERROR(yr_arena_write_data( + compiler->arena, + YR_EXTERNAL_VARIABLES_TABLE, &null_external, sizeof(YR_EXTERNAL_VARIABLE), - NULL); + NULL)); // Write Aho-Corasick automaton to arena. - result = yr_ac_compile( + FAIL_ON_ERROR(yr_ac_compile( compiler->automaton, - compiler->automaton_arena, - &tables); + compiler->arena)); - if (result == ERROR_SUCCESS) - result = yr_arena_create(1024, ARENA_FLAGS_RELOCATABLE, &arena); + YR_ARENA_REF ref; - if (result == ERROR_SUCCESS) - result = yr_arena_allocate_struct( - arena, - sizeof(YARA_RULES_FILE_HEADER), - (void**) &rules_file_header, - offsetof(YARA_RULES_FILE_HEADER, rules_list_head), - offsetof(YARA_RULES_FILE_HEADER, externals_list_head), - offsetof(YARA_RULES_FILE_HEADER, code_start), - offsetof(YARA_RULES_FILE_HEADER, ac_match_table), - offsetof(YARA_RULES_FILE_HEADER, ac_transition_table), - EOL); + FAIL_ON_ERROR(yr_arena_allocate_struct( + compiler->arena, + YR_SUMMARY_SECTION, + sizeof(YR_SUMMARY), + &ref, + EOL)); - if (result == ERROR_SUCCESS) - { - rules_file_header->rules_list_head = (YR_RULE*) yr_arena_base_address( - compiler->rules_arena); + YR_SUMMARY* summary = (YR_SUMMARY*) yr_arena_ref_to_ptr( + compiler->arena, &ref); - rules_file_header->externals_list_head = (YR_EXTERNAL_VARIABLE*) - yr_arena_base_address(compiler->externals_arena); + summary->num_namespaces = compiler->num_namespaces; + summary->num_rules = compiler->next_rule_idx; + summary->num_strings = compiler->current_string_idx; - rules_file_header->code_start = (uint8_t*) yr_arena_base_address( - compiler->code_arena); - - rules_file_header->ac_match_table = tables.matches; - rules_file_header->ac_transition_table = tables.transitions; - rules_file_header->ac_tables_size = compiler->automaton->tables_size; - } - - if (result == ERROR_SUCCESS) - { - result = yr_arena_append( - arena, - compiler->code_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->code_arena = NULL; - result = yr_arena_append( - arena, - compiler->re_code_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->re_code_arena = NULL; - result = yr_arena_append( - arena, - compiler->rules_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->rules_arena = NULL; - result = yr_arena_append( - arena, - compiler->strings_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->strings_arena = NULL; - result = yr_arena_append( - arena, - compiler->externals_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->externals_arena = NULL; - result = yr_arena_append( - arena, - compiler->namespaces_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->namespaces_arena = NULL; - result = yr_arena_append( - arena, - compiler->metas_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->metas_arena = NULL; - result = yr_arena_append( - arena, - compiler->sz_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->sz_arena = NULL; - result = yr_arena_append( - arena, - compiler->automaton_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->automaton_arena = NULL; - result = yr_arena_append( - arena, - compiler->matches_arena); - } - - if (result == ERROR_SUCCESS) - { - compiler->matches_arena = NULL; - compiler->compiled_rules_arena = arena; - result = yr_arena_coalesce(arena); - } - else - { - yr_arena_destroy(arena); - } - - return result; + return yr_rules_from_arena(compiler->arena, &compiler->rules); } @@ -836,48 +693,16 @@ YR_API int yr_compiler_get_rules( YR_COMPILER* compiler, YR_RULES** rules) { - YR_RULES* yara_rules; - YARA_RULES_FILE_HEADER* rules_file_header; - // Don't allow calls to yr_compiler_get_rules() if a previous call to // yr_compiler_add_XXXX failed. - assert(compiler->errors == 0); *rules = NULL; - if (compiler->compiled_rules_arena == NULL) + if (compiler->rules == NULL) FAIL_ON_ERROR(_yr_compiler_compile_rules(compiler)); - yara_rules = (YR_RULES*) yr_malloc(sizeof(YR_RULES)); - - if (yara_rules == NULL) - return ERROR_INSUFFICIENT_MEMORY; - - FAIL_ON_ERROR_WITH_CLEANUP( - yr_arena_duplicate(compiler->compiled_rules_arena, &yara_rules->arena), - yr_free(yara_rules)); - - rules_file_header = (YARA_RULES_FILE_HEADER*) yr_arena_base_address( - yara_rules->arena); - - yara_rules->externals_list_head = rules_file_header->externals_list_head; - yara_rules->rules_list_head = rules_file_header->rules_list_head; - yara_rules->ac_match_table = rules_file_header->ac_match_table; - yara_rules->ac_transition_table = rules_file_header->ac_transition_table; - yara_rules->ac_tables_size = rules_file_header->ac_tables_size; - yara_rules->code_start = rules_file_header->code_start; - yara_rules->time_cost = 0; - - memset(yara_rules->tidx_mask, 0, sizeof(yara_rules->tidx_mask)); - - FAIL_ON_ERROR_WITH_CLEANUP( - yr_mutex_create(&yara_rules->mutex), - // cleanup - yr_arena_destroy(yara_rules->arena); - yr_free(yara_rules)); - - *rules = yara_rules; + *rules = compiler->rules; return ERROR_SUCCESS; } @@ -889,8 +714,6 @@ static int _yr_compiler_define_variable( YR_EXTERNAL_VARIABLE* ext; YR_OBJECT* object; - char* id; - if (external->identifier == NULL) return ERROR_INVALID_ARGUMENT; @@ -902,52 +725,64 @@ static int _yr_compiler_define_variable( if (object != NULL) return ERROR_DUPLICATED_EXTERNAL_VARIABLE; - FAIL_ON_ERROR(yr_arena_write_string( - compiler->sz_arena, - external->identifier, - &id)); + YR_ARENA_REF ext_ref; + YR_ARENA_REF ref; FAIL_ON_ERROR(yr_arena_allocate_struct( - compiler->externals_arena, + compiler->arena, + YR_EXTERNAL_VARIABLES_TABLE, sizeof(YR_EXTERNAL_VARIABLE), - (void**) &ext, + &ext_ref, offsetof(YR_EXTERNAL_VARIABLE, identifier), EOL)); - ext->identifier = id; + ext = (YR_EXTERNAL_VARIABLE*) yr_arena_ref_to_ptr( + compiler->arena, &ext_ref); + + FAIL_ON_ERROR(yr_arena_write_string( + compiler->arena, + YR_SZ_POOL, + external->identifier, + &ref)); + + ext->identifier = (const char*) yr_arena_ref_to_ptr( + compiler->arena, &ref); + ext->type = external->type; ext->value = external->value; if (external->type == EXTERNAL_VARIABLE_TYPE_STRING) { - char* val; - if (external->value.s == NULL) return ERROR_INVALID_ARGUMENT; FAIL_ON_ERROR(yr_arena_write_string( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, external->value.s, - &val)); - - ext->value.s = val; + &ref)); FAIL_ON_ERROR(yr_arena_make_ptr_relocatable( - compiler->externals_arena, - ext, - offsetof(YR_EXTERNAL_VARIABLE, value.s), + compiler->arena, + YR_EXTERNAL_VARIABLES_TABLE, + ext_ref.offset + offsetof(YR_EXTERNAL_VARIABLE, value.s), EOL)); + + ext->value.s = (char*) yr_arena_ref_to_ptr( + compiler->arena, &ref); } FAIL_ON_ERROR(yr_object_from_external_variable( external, &object)); - FAIL_ON_ERROR(yr_hash_table_add( + FAIL_ON_ERROR_WITH_CLEANUP(yr_hash_table_add( compiler->objects_table, external->identifier, NULL, - (void*) object)); + (void*) object), + // cleanup + yr_object_destroy(object)); return ERROR_SUCCESS; } diff --git a/libyara/exception.h b/libyara/exception.h index 66cbb97c..218cdeab 100644 --- a/libyara/exception.h +++ b/libyara/exception.h @@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define YR_EXCEPTION_H #include +#include #if _WIN32 || __CYGWIN__ @@ -75,19 +76,20 @@ static LONG CALLBACK exception_handler( #include -jmp_buf *exc_jmp_buf[YR_MAX_THREADS]; - static LONG CALLBACK exception_handler( PEXCEPTION_POINTERS ExceptionInfo) { - int tidx = yr_get_tidx(); + jmp_buf* jb_ptr; switch(ExceptionInfo->ExceptionRecord->ExceptionCode) { case EXCEPTION_IN_PAGE_ERROR: case EXCEPTION_ACCESS_VIOLATION: - if (tidx != -1 && exc_jmp_buf[tidx] != NULL) - longjmp(*exc_jmp_buf[tidx], 1); + jb_ptr = (jmp_buf*) yr_thread_storage_get_value( + &yr_trycatch_trampoline_tls); + + if (jb_ptr != NULL) + longjmp(*jb_ptr, 1); } return EXCEPTION_CONTINUE_SEARCH; @@ -99,16 +101,17 @@ static LONG CALLBACK exception_handler( if (_do_) \ { \ jmp_buf jb; \ + /* Store pointer to sigjmp_buf in TLS */ \ + yr_thread_storage_set_value( \ + &yr_trycatch_trampoline_tls, &jb); \ HANDLE exh = AddVectoredExceptionHandler(1, exception_handler); \ - int tidx = yr_get_tidx(); \ - assert(tidx != -1); \ - exc_jmp_buf[tidx] = &jb; \ if (setjmp(jb) == 0) \ { _try_clause_ } \ else \ { _catch_clause_ } \ - exc_jmp_buf[tidx] = NULL; \ RemoveVectoredExceptionHandler(exh); \ + yr_thread_storage_set_value( \ + &yr_trycatch_trampoline_tls, NULL); \ } \ else \ { \ @@ -122,16 +125,16 @@ static LONG CALLBACK exception_handler( #include #include - -sigjmp_buf *exc_jmp_buf[YR_MAX_THREADS]; +#include static void exception_handler(int sig) { if (sig == SIGBUS || sig == SIGSEGV) { - int tidx = yr_get_tidx(); + jmp_buf* jb_ptr = (jmp_buf*) yr_thread_storage_get_value( + &yr_trycatch_trampoline_tls); - if (tidx != -1 && exc_jmp_buf[tidx] != NULL) - siglongjmp(*exc_jmp_buf[tidx], 1); + if (jb_ptr != NULL) + siglongjmp(*jb_ptr, 1); } } @@ -145,22 +148,25 @@ typedef struct sigaction sa; struct sigaction old_sigbus_act; \ struct sigaction old_sigsegv_act; \ struct sigaction act; \ + sigjmp_buf jb; \ + /* Store pointer to sigjmp_buf in TLS */ \ + yr_thread_storage_set_value( \ + &yr_trycatch_trampoline_tls, &jb); \ + /* Set exception handler for SIGBUS and SIGSEGV*/ \ act.sa_handler = exception_handler; \ act.sa_flags = 0; /* SA_ONSTACK? */ \ sigfillset(&act.sa_mask); \ sigaction(SIGBUS, &act, &old_sigbus_act); \ sigaction(SIGSEGV, &act, &old_sigsegv_act); \ - int tidx = yr_get_tidx(); \ - assert(tidx != -1); \ - sigjmp_buf jb; \ - exc_jmp_buf[tidx] = &jb; \ if (sigsetjmp(jb, 1) == 0) \ { _try_clause_ } \ else \ { _catch_clause_ } \ - exc_jmp_buf[tidx] = NULL; \ + /* Stop capturing SIGBUS and SIGSEGV */ \ sigaction(SIGBUS, &old_sigbus_act, NULL); \ sigaction(SIGSEGV, &old_sigsegv_act, NULL); \ + yr_thread_storage_set_value( \ + &yr_trycatch_trampoline_tls, NULL); \ } \ else \ { \ diff --git a/libyara/exec.c b/libyara/exec.c index a73d8d50..c26bff70 100644 --- a/libyara/exec.c +++ b/libyara/exec.c @@ -60,7 +60,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Paranoid mode does not guarantee that it's safe to load compiled rules from // third parties, it only prevents severe security issues. Maliciously crafted // compiled rules can still crash YARA. Loading third-party compiled rules is -// *highly* undiscouraged. If you need to distribute YARA rules in compiled +// *highly* discouraged. If you need to distribute YARA rules in compiled // form you should encapsulate them in some digitally-signed package that // ensure that they haven't been modified by someone else. @@ -69,7 +69,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif -#define MEM_SIZE YR_MAX_LOOP_NESTING * YR_MAX_LOOP_VARS +#define MEM_SIZE \ + YR_MAX_LOOP_NESTING * (YR_MAX_LOOP_VARS + YR_INTERNAL_LOOP_VARS) #define push(x) \ @@ -92,7 +93,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define ensure_defined(x) \ if (is_undef(x)) \ { \ - r1.i = UNDEFINED; \ + r1.i = YR_UNDEFINED; \ push(r1); \ break; \ } @@ -107,11 +108,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Make sure that the string pointer is within the rules arena. #define ensure_within_rules_arena(x) \ - if (yr_arena_page_for_address(context->rules->arena, x) == NULL) \ { \ - stop = true; \ - result = ERROR_INTERNAL_FATAL_ERROR; \ - break; \ + YR_ARENA_REF ref; \ + if (yr_arena_ptr_to_ref(context->rules->arena, x, &ref) == 0) \ + { \ + stop = true; \ + result = ERROR_INTERNAL_FATAL_ERROR; \ + break; \ + } \ } #define check_object_canary(o) \ @@ -150,14 +154,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. type result; \ const uint8_t* data = block->fetch_data(block); \ if (data == NULL) \ - return UNDEFINED; \ + return YR_UNDEFINED; \ result = *(type *)(data + offset - block->base); \ result = endianess##_##type(result); \ return result; \ } \ block = iterator->next(iterator); \ } \ - return UNDEFINED; \ + return YR_UNDEFINED; \ }; @@ -179,18 +183,24 @@ static const uint8_t* jmp_if( int condition, const uint8_t* ip) { - const uint8_t* result; + size_t off; if (condition) { - result = *(const uint8_t**)(ip); + // The condition is true, the instruction pointer is incremented in the + // amount specified by the jump's offset. The offset is relative to the + // jump opcode, but now the instruction pointer is pointing past the opcode + // that's why we decrement the offset by 1. + off = *(int32_t*)(ip) - 1; } else { - result = ip + sizeof(uint64_t); + // The condition is false, the execution flow proceeds with the instruction + // right after the jump. + off = sizeof(int32_t); } - return result; + return ip + off; } @@ -217,7 +227,7 @@ static int iter_array_next( if (obj != NULL) stack->items[stack->sp++].o = obj; else - stack->items[stack->sp++].i = UNDEFINED; + stack->items[stack->sp++].i = YR_UNDEFINED; self->array_it.index++; } @@ -225,8 +235,8 @@ static int iter_array_next( { // Push true for indicating the iterator has been exhausted. stack->items[stack->sp++].i = 1; - // Push UNDEFINED as a placeholder for the next item. - stack->items[stack->sp++].i = UNDEFINED; + // Push YR_UNDEFINED as a placeholder for the next item. + stack->items[stack->sp++].i = YR_UNDEFINED; } return ERROR_SUCCESS; @@ -245,7 +255,17 @@ static int iter_dict_next( if (stack->sp + 2 >= stack->capacity) return ERROR_EXEC_STACK_OVERFLOW; - if (self->dict_it.index < items->used) + // If the dictionary has no items or the iterator reached the last item, abort + // the iteration, if not push the next key and value. + if (items == NULL || self->dict_it.index == items->used) + { + // Push true for indicating the iterator has been exhausted. + stack->items[stack->sp++].i = 1; + // Push YR_UNDEFINED as a placeholder for the next key and value. + stack->items[stack->sp++].i = YR_UNDEFINED; + stack->items[stack->sp++].i = YR_UNDEFINED; + } + else { // Push the false value that indicates that the iterator is not exhausted. stack->items[stack->sp++].i = 0; @@ -257,20 +277,12 @@ static int iter_dict_next( } else { - stack->items[stack->sp++].i = UNDEFINED; - stack->items[stack->sp++].i = UNDEFINED; + stack->items[stack->sp++].i = YR_UNDEFINED; + stack->items[stack->sp++].i = YR_UNDEFINED; } self->dict_it.index++; } - else - { - // Push true for indicating the iterator has been exhausted. - stack->items[stack->sp++].i = 1; - // Push UNDEFINED as a placeholder for the next key and value. - stack->items[stack->sp++].i = UNDEFINED; - stack->items[stack->sp++].i = UNDEFINED; - } return ERROR_SUCCESS; } @@ -297,8 +309,8 @@ static int iter_int_range_next( { // Push true for indicating the iterator has been exhausted. stack->items[stack->sp++].i = 1; - // Push UNDEFINED as a placeholder for the next item. - stack->items[stack->sp++].i = UNDEFINED; + // Push YR_UNDEFINED as a placeholder for the next item. + stack->items[stack->sp++].i = YR_UNDEFINED; } return ERROR_SUCCESS; @@ -326,8 +338,8 @@ static int iter_int_enum_next( { // Push true for indicating the iterator has been exhausted. stack->items[stack->sp++].i = 1; - // Push UNDEFINED as a placeholder for the next item. - stack->items[stack->sp++].i = UNDEFINED; + // Push YR_UNDEFINED as a placeholder for the next item. + stack->items[stack->sp++].i = YR_UNDEFINED; } return ERROR_SUCCESS; @@ -352,17 +364,16 @@ int yr_execute_code( #ifdef PROFILING_ENABLED uint64_t start_time; - YR_RULE* current_rule = NULL; #endif - YR_INIT_RULE_ARGS init_rule_args; - + uint32_t current_rule_idx; + YR_RULE* current_rule = NULL; YR_RULE* rule; YR_MATCH* match; YR_OBJECT_FUNCTION* function; YR_OBJECT** obj_ptr; YR_ARENA* obj_arena; - YR_ARENA* it_arena; + YR_NOTEBOOK* it_notebook; char* identifier; char* args_fmt; @@ -372,7 +383,7 @@ int yr_execute_code( int count; int result = ERROR_SUCCESS; int cycle = 0; - int tidx = context->tidx; + int obj_count = 0; bool stop = false; @@ -387,16 +398,16 @@ int yr_execute_code( return ERROR_INSUFFICIENT_MEMORY; FAIL_ON_ERROR_WITH_CLEANUP( - yr_arena_create(1024, 0, &obj_arena), + yr_arena_create(1, 512 * sizeof(YR_OBJECT*), &obj_arena), yr_free(stack.items)); FAIL_ON_ERROR_WITH_CLEANUP( - yr_arena_create(64 * sizeof(YR_ITERATOR), 0, &it_arena), - yr_arena_destroy(obj_arena); + yr_notebook_create(512 * sizeof(YR_ITERATOR), &it_notebook), + yr_arena_release(obj_arena); yr_free(stack.items)); #ifdef PROFILING_ENABLED - start_time = yr_stopwatch_elapsed_us(&context->stopwatch); + start_time = yr_stopwatch_elapsed_ns(&context->stopwatch); #endif #if PARANOID_EXEC @@ -405,7 +416,10 @@ int yr_execute_code( while(!stop) { + // Read the opcode from the address indicated by the instruction pointer. opcode = *ip; + + // Advance the instruction pointer, which now points past the opcode. ip++; switch(opcode) @@ -419,10 +433,13 @@ int yr_execute_code( break; case OP_ITER_START_ARRAY: - result = yr_arena_allocate_struct( - it_arena, sizeof(YR_ITERATOR), &r2.p); + r2.p = yr_notebook_alloc(it_notebook, sizeof(YR_ITERATOR)); - if (result == ERROR_SUCCESS) + if (r2.p == NULL) + { + result = ERROR_INSUFFICIENT_MEMORY; + } + else { pop(r1); r2.it->array_it.array = r1.o; @@ -435,10 +452,13 @@ int yr_execute_code( break; case OP_ITER_START_DICT: - result = yr_arena_allocate_struct( - it_arena, sizeof(YR_ITERATOR), &r2.p); + r2.p = yr_notebook_alloc(it_notebook, sizeof(YR_ITERATOR)); - if (result == ERROR_SUCCESS) + if (r2.p == NULL) + { + result = ERROR_INSUFFICIENT_MEMORY; + } + else { pop(r1); r2.it->dict_it.dict = r1.o; @@ -453,10 +473,13 @@ int yr_execute_code( case OP_ITER_START_INT_RANGE: // Creates an iterator for an integer range. The higher bound of the // range is at the top of the stack followed by the lower bound. - result = yr_arena_allocate_struct( - it_arena, sizeof(YR_ITERATOR), &r3.p); + r3.p = yr_notebook_alloc(it_notebook, sizeof(YR_ITERATOR)); - if (result == ERROR_SUCCESS) + if (r3.p == NULL) + { + result = ERROR_INSUFFICIENT_MEMORY; + } + else { pop(r2); pop(r1); @@ -475,10 +498,14 @@ int yr_execute_code( // items in reverse order. pop(r1); - result = yr_arena_allocate_struct( - it_arena, sizeof(YR_ITERATOR) + sizeof(uint64_t) * r1.i, &r3.p); + r3.p = yr_notebook_alloc( + it_notebook, sizeof(YR_ITERATOR) + sizeof(uint64_t) * r1.i); - if (result == ERROR_SUCCESS) + if (r3.p == NULL) + { + result = ERROR_INSUFFICIENT_MEMORY; + } + else { r3.it->int_enum_it.count = r1.i; r3.it->int_enum_it.next = 0; @@ -678,7 +705,7 @@ int yr_execute_code( pop(r1); if (is_undef(r1)) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; else r1.i = !r1.i; @@ -693,7 +720,7 @@ int yr_execute_code( if (r2.i != 0) r1.i = r1.i % r2.i; else - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; push(r1); break; @@ -703,7 +730,7 @@ int yr_execute_code( ensure_defined(r2); ensure_defined(r1); if (r2.i < 0) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; else if (r2.i < 64) r1.i = r1.i >> r2.i; else @@ -717,7 +744,7 @@ int yr_execute_code( ensure_defined(r2); ensure_defined(r1); if (r2.i < 0) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; else if (r2.i < 64) r1.i = r1.i << r2.i; else @@ -760,44 +787,62 @@ int yr_execute_code( break; case OP_PUSH_RULE: - rule = *(YR_RULE**)(ip); + r1.i = *(uint64_t*)(ip); ip += sizeof(uint64_t); + + rule = &context->rules->rules_list_head[r1.i]; + if (RULE_IS_DISABLED(rule)) - r1.i = UNDEFINED; + { + r2.i = YR_UNDEFINED; + } else - r1.i = rule->t_flags[tidx] & RULE_TFLAGS_MATCH ? 1 : 0; - push(r1); + { + if yr_bitmask_is_set(context->rule_matches_flags, r1.i) + r2.i = 1; + else + r2.i = 0; + } + + push(r2); break; case OP_INIT_RULE: - memcpy(&init_rule_args, ip, sizeof(init_rule_args)); - #ifdef PROFILING_ENABLED - current_rule = init_rule_args.rule; - #endif - if (RULE_IS_DISABLED(init_rule_args.rule)) - ip = init_rule_args.jmp_addr; - else - ip += sizeof(init_rule_args); + // After the opcode there's an int32_t corresponding to the jump's + // offset and an uint32_t corresponding to the rule's index. + current_rule_idx = *(uint32_t*)(ip + sizeof(int32_t)); + current_rule = &context->rules->rules_list_head[current_rule_idx]; + + // If the rule is disabled let's skip its code. + ip = jmp_if(RULE_IS_DISABLED(current_rule), ip); + + // Skip the bytes corresponding to the rule's index, but only if not + // taking the jump. + if (!RULE_IS_DISABLED(current_rule)) + ip += sizeof(uint32_t); + break; case OP_MATCH_RULE: pop(r1); - rule = *(YR_RULE**)(ip); + + r2.i = *(uint64_t*)(ip); + ip += sizeof(uint64_t); + + rule = &context->rules->rules_list_head[r2.i]; #if PARANOID_EXEC ensure_within_rules_arena(rule); #endif - ip += sizeof(uint64_t); - if (!is_undef(r1) && r1.i) - rule->t_flags[tidx] |= RULE_TFLAGS_MATCH; + yr_bitmask_set(context->rule_matches_flags, r2.i); else if (RULE_IS_GLOBAL(rule)) - rule->ns->t_flags[tidx] |= NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL; + yr_bitmask_set(context->ns_unsatisfied_flags, rule->ns->idx); #ifdef PROFILING_ENABLED - elapsed_time = yr_stopwatch_elapsed_us(&context->stopwatch); - rule->time_cost_per_thread[tidx] += (elapsed_time - start_time); + elapsed_time = yr_stopwatch_elapsed_ns(&context->stopwatch); + context->time_cost[r2.i] += (elapsed_time - start_time); start_time = elapsed_time; #endif @@ -846,14 +891,14 @@ int yr_execute_code( case OBJECT_TYPE_FLOAT: if (isnan(r1.o->value.d)) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; else r1.d = r1.o->value.d; break; case OBJECT_TYPE_STRING: if (r1.o->value.ss == NULL) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; else r1.ss = r1.o->value.ss; break; @@ -881,7 +926,7 @@ int yr_execute_code( r1.o = yr_object_array_get_item(r2.o, 0, (int) r1.i); if (r1.o == NULL) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; push(r1); break; @@ -903,7 +948,7 @@ int yr_execute_code( r2.o, 0, r1.ss->c_string); if (r1.o == NULL) - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; push(r1); break; @@ -946,10 +991,10 @@ int yr_execute_code( if (count > 0) { - // if there are undefined args, result for function call + // If there are undefined args, result for function call // is undefined as well. - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; push(r1); break; } @@ -969,23 +1014,25 @@ int yr_execute_code( } } - // if i == YR_MAX_OVERLOADED_FUNCTIONS at this point no matching + // If i == YR_MAX_OVERLOADED_FUNCTIONS at this point no matching // prototype was found, but this shouldn't happen. - assert(i < YR_MAX_OVERLOADED_FUNCTIONS); - // make a copy of the returned object and push the copy into the stack, + // Make a copy of the returned object and push the copy into the stack, // function->return_obj can't be pushed because it can change in // subsequent calls to the same function. - if (result == ERROR_SUCCESS) result = yr_object_copy(function->return_obj, &r1.o); - // a pointer to the copied object is stored in a arena in order to - // free the object before exiting yr_execute_code - + // A pointer to the copied object is stored in a arena in order to + // free the object before exiting yr_execute_code, obj_count tracks + // the number of objects written. if (result == ERROR_SUCCESS) - result = yr_arena_write_data(obj_arena, &r1.o, sizeof(r1.o), NULL); + { + result = yr_arena_write_data( + obj_arena, 0, &r1.o, sizeof(r1.o), NULL); + obj_count++; + } stop = (result != ERROR_SUCCESS); push(r1); @@ -993,12 +1040,7 @@ int yr_execute_code( case OP_FOUND: pop(r1); - - if (STRING_IS_PRIVATE(r1.s)) - r2.i = r1.s->private_matches[tidx].tail != NULL ? 1 : 0; - else - r2.i = r1.s->matches[tidx].tail != NULL ? 1 : 0; - + r2.i = context->matches[r1.s->idx].tail != NULL ? 1 : 0; push(r2); break; @@ -1017,11 +1059,7 @@ int yr_execute_code( ensure_within_rules_arena(r2.p); #endif - if (STRING_IS_PRIVATE(r2.s)) - match = r2.s->private_matches[tidx].head; - else - match = r2.s->matches[tidx].head; - + match = context->matches[r2.s->idx].head; r3.i = false; while (match != NULL) @@ -1053,11 +1091,7 @@ int yr_execute_code( ensure_within_rules_arena(r3.p); #endif - if (STRING_IS_PRIVATE(r3.s)) - match = r3.s->private_matches[tidx].head; - else - match = r3.s->matches[tidx].head; - + match = context->matches[r3.s->idx].head; r4.i = false; while (match != NULL && !r4.i) @@ -1084,11 +1118,7 @@ int yr_execute_code( ensure_within_rules_arena(r1.p); #endif - if (STRING_IS_PRIVATE(r1.s)) - r2.i = r1.s->private_matches[tidx].count; - else - r2.i = r1.s->matches[tidx].count; - + r2.i = context->matches[r1.s->idx].count; push(r2); break; @@ -1102,15 +1132,12 @@ int yr_execute_code( ensure_within_rules_arena(r2.p); #endif - if (STRING_IS_PRIVATE(r2.s)) - match = r2.s->private_matches[tidx].head; - else - match = r2.s->matches[tidx].head; + match = context->matches[r2.s->idx].head; i = 1; - r3.i = UNDEFINED; + r3.i = YR_UNDEFINED; - while (match != NULL && r3.i == UNDEFINED) + while (match != NULL && r3.i == YR_UNDEFINED) { if (r1.i == i) r3.i = match->base + match->offset; @@ -1132,15 +1159,12 @@ int yr_execute_code( ensure_within_rules_arena(r2.p); #endif - if (STRING_IS_PRIVATE(r2.s)) - match = r2.s->private_matches[tidx].head; - else - match = r2.s->matches[tidx].head; + match = context->matches[r2.s->idx].head; i = 1; - r3.i = UNDEFINED; + r3.i = YR_UNDEFINED; - while (match != NULL && r3.i == UNDEFINED) + while (match != NULL && r3.i == YR_UNDEFINED) { if (r1.i == i) r3.i = match->match_length; @@ -1159,8 +1183,10 @@ int yr_execute_code( while (!is_undef(r1)) { - if (r1.s->matches[tidx].tail != NULL || r1.s->private_matches[tidx].tail != NULL) + if (context->matches[r1.s->idx].tail != NULL) + { found++; + } count++; pop(r1); } @@ -1330,7 +1356,7 @@ int yr_execute_code( r2 = stack.items[stack.sp - r1.i]; if (is_undef(r2)) - stack.items[stack.sp - r1.i].i = UNDEFINED; + stack.items[stack.sp - r1.i].i = YR_UNDEFINED; else stack.items[stack.sp - r1.i].d = (double) r2.i; break; @@ -1431,7 +1457,7 @@ int yr_execute_code( if (r2.i != 0) r1.i = r1.i / r2.i; else - r1.i = UNDEFINED; + r1.i = YR_UNDEFINED; push(r1); break; @@ -1582,18 +1608,17 @@ int yr_execute_code( assert(false); } - // Check for timeout every 10 instruction cycles. If timeout == 0 it means + // Check for timeout every 100 instruction cycles. If timeout == 0 it means // no timeout at all. - if (context->timeout > 0L && ++cycle == 10) + if (context->timeout > 0L && ++cycle == 100) { - elapsed_time = yr_stopwatch_elapsed_us(&context->stopwatch); + elapsed_time = yr_stopwatch_elapsed_ns(&context->stopwatch); if (elapsed_time > context->timeout) { #ifdef PROFILING_ENABLED - assert(current_rule != NULL); - current_rule->time_cost_per_thread[tidx] += elapsed_time - start_time; + context->time_cost[current_rule_idx] += (elapsed_time - start_time); #endif result = ERROR_SCAN_TIMEOUT; stop = true; @@ -1603,18 +1628,13 @@ int yr_execute_code( } } - obj_ptr = (YR_OBJECT**) yr_arena_base_address(obj_arena); + obj_ptr = yr_arena_get_ptr(obj_arena, 0, 0); - while (obj_ptr != NULL) - { - yr_object_destroy(*obj_ptr); + for (int i = 0; i < obj_count; i++) + yr_object_destroy(obj_ptr[i]); - obj_ptr = (YR_OBJECT**) yr_arena_next_address( - obj_arena, obj_ptr, sizeof(YR_OBJECT*)); - } - - yr_arena_destroy(obj_arena); - yr_arena_destroy(it_arena); + yr_arena_release(obj_arena); + yr_notebook_destroy(it_notebook); yr_modules_unload_all(context); yr_free(stack.items); diff --git a/libyara/exefiles.c b/libyara/exefiles.c index 6216ab7a..aa945125 100644 --- a/libyara/exefiles.c +++ b/libyara/exefiles.c @@ -379,7 +379,7 @@ uint64_t yr_get_entry_point_offset( buffer_length); } - return UNDEFINED; + return YR_UNDEFINED; } @@ -422,5 +422,5 @@ uint64_t yr_get_entry_point_address( break; } - return UNDEFINED; + return YR_UNDEFINED; } diff --git a/libyara/grammar.c b/libyara/grammar.c index f3d6f7d8..34d94798 100644 --- a/libyara/grammar.c +++ b/libyara/grammar.c @@ -1,8 +1,8 @@ -/* A Bison parser, made by GNU Bison 3.0.5. */ +/* A Bison parser, made by GNU Bison 3.0.4. */ /* Bison implementation for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015, 2018 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,7 +44,7 @@ #define YYBISON 1 /* Bison version. */ -#define YYBISON_VERSION "3.0.5" +#define YYBISON_VERSION "3.0.4" /* Skeleton name. */ #define YYSKELETON_NAME "yacc.c" @@ -79,6 +79,7 @@ #include #include +#include #include #include #include @@ -106,28 +107,21 @@ #define FOR_EXPRESSION_ALL 1 #define FOR_EXPRESSION_ANY 2 -#define fail_if_error(e) \ - if (e != ERROR_SUCCESS) \ +#define fail_with_error(e) \ { \ compiler->last_error = e; \ yyerror(yyscanner, compiler, NULL); \ YYERROR; \ - } \ - - -#define set_flag_or_error(flags, new_flag) \ - if (flags & new_flag) \ - { \ - compiler->last_error = ERROR_DUPLICATED_MODIFIER; \ - yyerror(yyscanner, compiler, NULL); \ - YYERROR; \ - } \ - else \ - { \ - flags |= new_flag; \ } +#define fail_if_error(e) \ + if (e != ERROR_SUCCESS) \ + { \ + fail_with_error(e); \ + } \ + + #define check_type_with_cleanup(expression, expected_type, op, cleanup) \ if (((expression.type) & (expected_type)) == 0) \ { \ @@ -162,20 +156,31 @@ check_type_with_cleanup(expression, expected_type, op, ) - #define loop_vars_cleanup(loop_index) \ { \ YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[loop_index]; \ for (int i = 0; i < loop_ctx->vars_count; i++) \ { \ - yr_free((void*) loop_ctx->vars[i].identifier); \ - loop_ctx->vars[i].identifier = NULL; \ + yr_free((void*) loop_ctx->vars[i].identifier.ptr); \ + loop_ctx->vars[i].identifier.ptr = NULL; \ + loop_ctx->vars[i].identifier.ref = YR_ARENA_NULL_REF; \ } \ loop_ctx->vars_count = 0; \ } \ -#line 179 "grammar.c" /* yacc.c:339 */ +// Given a YR_EXPRESSION returns its identifier. It returns identifier.ptr if not NULL and relies on identifier.ref if +// otherwise. +#define expression_identifier(expr) \ + ((expr).identifier.ptr != NULL ? \ + (expr).identifier.ptr : \ + (const char*) yr_arena_ref_to_ptr(compiler->arena, &(expr).identifier.ref)) + + +#define DEFAULT_BASE64_ALPHABET "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + + +#line 184 "grammar.c" /* yacc.c:339 */ # ifndef YY_NULLPTR # if defined __cplusplus && 201103L <= __cplusplus @@ -234,34 +239,36 @@ extern int yara_yydebug; _ASCII_ = 278, _WIDE_ = 279, _XOR_ = 280, - _NOCASE_ = 281, - _FULLWORD_ = 282, - _AT_ = 283, - _FILESIZE_ = 284, - _ENTRYPOINT_ = 285, - _ALL_ = 286, - _ANY_ = 287, - _IN_ = 288, - _OF_ = 289, - _FOR_ = 290, - _THEM_ = 291, - _MATCHES_ = 292, - _CONTAINS_ = 293, - _IMPORT_ = 294, - _TRUE_ = 295, - _FALSE_ = 296, - _OR_ = 297, - _AND_ = 298, - _NOT_ = 299, - _EQ_ = 300, - _NEQ_ = 301, - _LT_ = 302, - _LE_ = 303, - _GT_ = 304, - _GE_ = 305, - _SHIFT_LEFT_ = 306, - _SHIFT_RIGHT_ = 307, - UNARY_MINUS = 308 + _BASE64_ = 281, + _BASE64_WIDE_ = 282, + _NOCASE_ = 283, + _FULLWORD_ = 284, + _AT_ = 285, + _FILESIZE_ = 286, + _ENTRYPOINT_ = 287, + _ALL_ = 288, + _ANY_ = 289, + _IN_ = 290, + _OF_ = 291, + _FOR_ = 292, + _THEM_ = 293, + _MATCHES_ = 294, + _CONTAINS_ = 295, + _IMPORT_ = 296, + _TRUE_ = 297, + _FALSE_ = 298, + _OR_ = 299, + _AND_ = 300, + _NOT_ = 301, + _EQ_ = 302, + _NEQ_ = 303, + _LT_ = 304, + _LE_ = 305, + _GT_ = 306, + _GE_ = 307, + _SHIFT_LEFT_ = 308, + _SHIFT_RIGHT_ = 309, + UNARY_MINUS = 310 }; #endif /* Tokens. */ @@ -289,53 +296,57 @@ extern int yara_yydebug; #define _ASCII_ 278 #define _WIDE_ 279 #define _XOR_ 280 -#define _NOCASE_ 281 -#define _FULLWORD_ 282 -#define _AT_ 283 -#define _FILESIZE_ 284 -#define _ENTRYPOINT_ 285 -#define _ALL_ 286 -#define _ANY_ 287 -#define _IN_ 288 -#define _OF_ 289 -#define _FOR_ 290 -#define _THEM_ 291 -#define _MATCHES_ 292 -#define _CONTAINS_ 293 -#define _IMPORT_ 294 -#define _TRUE_ 295 -#define _FALSE_ 296 -#define _OR_ 297 -#define _AND_ 298 -#define _NOT_ 299 -#define _EQ_ 300 -#define _NEQ_ 301 -#define _LT_ 302 -#define _LE_ 303 -#define _GT_ 304 -#define _GE_ 305 -#define _SHIFT_LEFT_ 306 -#define _SHIFT_RIGHT_ 307 -#define UNARY_MINUS 308 +#define _BASE64_ 281 +#define _BASE64_WIDE_ 282 +#define _NOCASE_ 283 +#define _FULLWORD_ 284 +#define _AT_ 285 +#define _FILESIZE_ 286 +#define _ENTRYPOINT_ 287 +#define _ALL_ 288 +#define _ANY_ 289 +#define _IN_ 290 +#define _OF_ 291 +#define _FOR_ 292 +#define _THEM_ 293 +#define _MATCHES_ 294 +#define _CONTAINS_ 295 +#define _IMPORT_ 296 +#define _TRUE_ 297 +#define _FALSE_ 298 +#define _OR_ 299 +#define _AND_ 300 +#define _NOT_ 301 +#define _EQ_ 302 +#define _NEQ_ 303 +#define _LT_ 304 +#define _LE_ 305 +#define _GT_ 306 +#define _GE_ 307 +#define _SHIFT_LEFT_ 308 +#define _SHIFT_RIGHT_ 309 +#define UNARY_MINUS 310 /* Value type. */ #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED union YYSTYPE { -#line 276 "grammar.y" /* yacc.c:355 */ +#line 299 "grammar.y" /* yacc.c:355 */ YR_EXPRESSION expression; SIZED_STRING* sized_string; char* c_string; int64_t integer; double double_; - YR_STRING* string; - YR_META* meta; - YR_RULE* rule; YR_MODIFIER modifier; -#line 339 "grammar.c" /* yacc.c:355 */ + YR_ARENA_REF tag; + YR_ARENA_REF rule; + YR_ARENA_REF meta; + YR_ARENA_REF string; + +#line 350 "grammar.c" /* yacc.c:355 */ }; typedef union YYSTYPE YYSTYPE; @@ -351,7 +362,7 @@ int yara_yyparse (void *yyscanner, YR_COMPILER* compiler); /* Copy the second part of user declarations. */ -#line 355 "grammar.c" /* yacc.c:358 */ +#line 366 "grammar.c" /* yacc.c:358 */ #ifdef short # undef short @@ -593,21 +604,21 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 2 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 375 +#define YYLAST 387 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 74 +#define YYNTOKENS 76 /* YYNNTS -- Number of nonterminals. */ #define YYNNTS 48 /* YYNRULES -- Number of rules. */ -#define YYNRULES 142 +#define YYNRULES 146 /* YYNSTATES -- Number of states. */ -#define YYNSTATES 234 +#define YYNSTATES 242 /* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned by yylex, with out-of-bounds checking. */ #define YYUNDEFTOK 2 -#define YYMAXUTOK 309 +#define YYMAXUTOK 311 #define YYTRANSLATE(YYX) \ ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) @@ -619,16 +630,16 @@ static const yytype_uint8 yytranslate[] = 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 60, 55, 2, - 68, 69, 58, 56, 73, 57, 70, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 66, 2, - 2, 67, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 62, 57, 2, + 70, 71, 60, 58, 75, 59, 72, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 68, 2, + 2, 69, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 71, 59, 72, 54, 2, 2, 2, 2, 2, + 2, 73, 61, 74, 56, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 64, 53, 65, 61, 2, 2, 2, + 2, 2, 2, 66, 55, 67, 63, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -646,28 +657,29 @@ static const yytype_uint8 yytranslate[] = 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, - 45, 46, 47, 48, 49, 50, 51, 52, 62, 63 + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + 64, 65 }; #if YYDEBUG /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 291, 291, 293, 294, 295, 296, 297, 298, 306, - 319, 324, 318, 345, 348, 378, 381, 406, 411, 412, - 417, 418, 424, 427, 445, 454, 493, 494, 499, 516, - 530, 544, 558, 576, 577, 583, 582, 598, 597, 618, - 617, 642, 647, 667, 668, 669, 670, 671, 672, 678, - 699, 733, 734, 738, 739, 740, 741, 742, 746, 747, - 751, 755, 844, 891, 950, 996, 997, 1001, 1036, 1089, - 1132, 1152, 1159, 1166, 1178, 1188, 1202, 1217, 1228, 1297, - 1334, 1245, 1446, 1445, 1529, 1535, 1542, 1541, 1587, 1586, - 1630, 1637, 1644, 1651, 1658, 1665, 1672, 1676, 1684, 1704, - 1732, 1806, 1834, 1843, 1852, 1876, 1891, 1911, 1910, 1916, - 1928, 1929, 1934, 1941, 1952, 1956, 1961, 1970, 1974, 1982, - 1994, 2008, 2016, 2023, 2048, 2060, 2072, 2088, 2100, 2116, - 2158, 2179, 2214, 2249, 2283, 2308, 2325, 2335, 2345, 2355, - 2365, 2385, 2405 + 0, 316, 316, 318, 319, 320, 321, 322, 323, 331, + 344, 349, 343, 376, 379, 406, 409, 424, 429, 430, + 435, 436, 442, 445, 461, 470, 512, 513, 518, 535, + 549, 563, 577, 595, 596, 602, 601, 618, 617, 638, + 637, 662, 668, 728, 729, 730, 731, 732, 733, 739, + 760, 791, 796, 813, 818, 838, 839, 853, 854, 855, + 856, 857, 861, 862, 876, 880, 975, 1023, 1084, 1131, + 1132, 1136, 1171, 1224, 1267, 1287, 1294, 1301, 1313, 1323, + 1337, 1352, 1363, 1434, 1472, 1382, 1609, 1608, 1698, 1704, + 1711, 1710, 1756, 1755, 1799, 1806, 1813, 1820, 1827, 1834, + 1841, 1845, 1853, 1873, 1901, 1975, 2003, 2012, 2021, 2045, + 2060, 2080, 2079, 2085, 2097, 2098, 2103, 2110, 2121, 2125, + 2130, 2139, 2143, 2151, 2163, 2177, 2185, 2192, 2219, 2231, + 2243, 2259, 2271, 2287, 2330, 2351, 2386, 2421, 2455, 2480, + 2497, 2507, 2517, 2527, 2537, 2557, 2577 }; #endif @@ -684,18 +696,19 @@ static const char *const yytname[] = "\"integer number\"", "\"floating point number\"", "\"integer function\"", "\"text string\"", "\"hex string\"", "\"regular expression\"", "\"\"", "\"\"", "\"\"", - "\"\"", "\"\"", "\"\"", "\"\"", - "\"\"", "\"\"", "\"\"", "\"\"", "\"\"", - "\"\"", "\"\"", "\"\"", "\"\"", - "\"\"", "\"\"", "\"\"", "\"\"", - "\"\"", "\"==\"", "\"!=\"", "\"<\"", "\"<=\"", "\">\"", "\">=\"", - "\"<<\"", "\">>\"", "'|'", "'^'", "'&'", "'+'", "'-'", "'*'", "'\\\\'", - "'%'", "'~'", "UNARY_MINUS", "\"include\"", "'{'", "'}'", "':'", "'='", - "'('", "')'", "'.'", "'['", "']'", "','", "$accept", "rules", "import", - "rule", "@1", "$@2", "meta", "strings", "condition", "rule_modifiers", - "rule_modifier", "tags", "tag_list", "meta_declarations", - "meta_declaration", "string_declarations", "string_declaration", "$@3", - "$@4", "$@5", "string_modifiers", "string_modifier", "regexp_modifiers", + "\"\"", "\"\"", "\"\"", "\"\"", + "\"\"", "\"\"", "\"\"", "\"\"", + "\"\"", "\"\"", "\"\"", "\"\"", "\"\"", + "\"\"", "\"\"", "\"\"", "\"\"", + "\"\"", "\"\"", "\"\"", "\"==\"", "\"!=\"", + "\"<\"", "\"<=\"", "\">\"", "\">=\"", "\"<<\"", "\">>\"", "'|'", "'^'", + "'&'", "'+'", "'-'", "'*'", "'\\\\'", "'%'", "'~'", "UNARY_MINUS", + "\"include\"", "'{'", "'}'", "':'", "'='", "'('", "')'", "'.'", "'['", + "']'", "','", "$accept", "rules", "import", "rule", "@1", "$@2", "meta", + "strings", "condition", "rule_modifiers", "rule_modifier", "tags", + "tag_list", "meta_declarations", "meta_declaration", + "string_declarations", "string_declaration", "$@3", "$@4", "$@5", + "string_modifiers", "string_modifier", "regexp_modifiers", "regexp_modifier", "hex_modifiers", "hex_modifier", "identifier", "arguments", "arguments_list", "regexp", "boolean_expression", "expression", "$@6", "$@7", "$@8", "$@9", "$@10", "for_variables", @@ -715,9 +728,9 @@ static const yytype_uint16 yytoknum[] = 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 124, 94, 38, 43, 45, 42, 92, - 37, 126, 308, 309, 123, 125, 58, 61, 40, 41, - 46, 91, 93, 44 + 305, 306, 307, 308, 309, 124, 94, 38, 43, 45, + 42, 92, 37, 126, 310, 311, 123, 125, 58, 61, + 40, 41, 46, 91, 93, 44 }; # endif @@ -726,7 +739,7 @@ static const yytype_uint16 yytoknum[] = #define yypact_value_is_default(Yystate) \ (!!((Yystate) == (-74))) -#define YYTABLE_NINF -115 +#define YYTABLE_NINF -119 #define yytable_value_is_error(Yytable_value) \ 0 @@ -735,30 +748,31 @@ static const yytype_uint16 yytoknum[] = STATE-NUM. */ static const yytype_int16 yypact[] = { - -74, 96, -74, -35, -74, 9, -74, -74, 70, -74, - -74, -74, -74, 59, -74, -74, -74, -74, 17, 74, - 24, -74, 80, 84, -74, 32, 100, 108, 57, -74, - 65, 108, -74, 137, 124, 15, -74, 87, 137, -74, - 79, 101, -74, -74, -74, -74, 144, -6, -74, 49, - -74, -74, 147, 148, 150, -74, -19, -74, 102, 131, - -74, -74, 125, -74, -74, -74, -74, -74, -74, 111, - -74, -74, 49, 133, 133, 49, 50, -74, 117, -74, - 184, 206, -74, -74, -74, 133, 173, 133, 133, 133, - 133, 2, 315, -74, -74, -74, 117, 176, 179, 49, - 231, 133, -74, -74, -11, 224, 133, 133, 133, 133, + -74, 97, -74, -38, -74, -11, -74, -74, 153, -74, + -74, -74, -74, 0, -74, -74, -74, -74, -50, 13, + -29, -74, 21, 58, -74, -10, 68, 77, 17, -74, + 22, 77, -74, 87, 98, 14, -74, 41, 87, -74, + 47, 57, -74, -74, -74, -74, 120, 8, -74, 50, + -74, -74, 109, 121, 128, -74, -22, -74, 81, 84, + -74, -74, 91, -74, -74, -74, -74, -74, -74, 108, + -74, -74, 50, 133, 133, 50, 44, -74, 124, -74, + 127, 206, -74, -74, -74, 133, 100, 133, 133, 133, + 133, 24, 315, -74, -74, -74, 124, 115, 179, 50, + 176, 133, -74, -74, -34, 166, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, 133, - 133, 133, 133, 151, 81, 241, 315, 133, -74, 216, - 226, 258, 277, -74, -11, 238, -74, -74, 181, 214, - 122, -74, 248, 49, 49, -74, -74, -74, -74, 315, - 315, 315, 315, 315, 315, 315, 55, 55, 129, 140, - 163, 78, 78, -74, -74, -74, -74, -74, -74, 221, - -74, -74, -74, -74, -74, -74, -74, -74, -74, -74, - -74, 152, -74, -74, -74, 225, -74, -15, -74, 49, - -74, 247, -74, 18, 275, 133, -74, -3, 282, 122, - -74, -74, -42, -74, -45, 296, 227, 133, 50, 228, - -74, -74, -74, -74, 18, 279, -74, -74, 49, 13, - 152, -74, -74, 250, -32, -74, 133, 229, -74, -74, - 315, 49, 31, -74 + 133, 133, 133, 156, 83, 183, 315, 133, -74, 216, + 226, 258, 277, -74, -34, 180, -74, -74, 149, 129, + 131, -74, 248, 50, 50, -74, -74, -74, -74, 315, + 315, 315, 315, 315, 315, 315, 74, 74, 325, 140, + 163, 112, 112, -74, -74, -74, -74, -74, -74, 125, + 173, 174, -74, -74, -74, -74, -74, -74, -74, -74, + -74, -74, -74, 152, -74, -74, -74, 181, -74, -20, + -74, 50, -74, 202, -74, -2, 231, 232, 269, 133, + -74, 1, 240, 131, -74, -74, 5, -74, -43, 220, + 221, 296, 223, 133, 44, 227, -74, -74, -74, -74, + -2, 279, -74, -74, -74, -74, 50, 15, 152, -74, + -74, 228, 30, -74, 133, 224, -74, -74, 315, 50, + 34, -74 }; /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. @@ -771,35 +785,36 @@ static const yytype_uint8 yydefact[] = 0, 24, 23, 13, 25, 0, 15, 0, 0, 11, 0, 14, 26, 0, 0, 0, 27, 0, 16, 33, 0, 0, 29, 28, 31, 32, 0, 35, 34, 0, - 12, 30, 0, 0, 0, 61, 75, 124, 126, 128, - 121, 122, 0, 123, 69, 118, 119, 115, 116, 0, - 71, 72, 0, 0, 0, 0, 129, 142, 17, 70, - 0, 96, 41, 51, 58, 0, 0, 0, 0, 0, - 0, 0, 114, 85, 130, 139, 0, 70, 96, 65, - 0, 0, 88, 86, 0, 0, 0, 0, 0, 0, + 12, 30, 0, 0, 0, 65, 79, 128, 130, 132, + 125, 126, 0, 127, 73, 122, 123, 119, 120, 0, + 75, 76, 0, 0, 0, 0, 133, 146, 17, 74, + 0, 100, 41, 55, 62, 0, 0, 0, 0, 0, + 0, 0, 118, 89, 134, 143, 0, 74, 100, 69, + 0, 0, 92, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 36, 38, 40, 76, 0, 77, 0, - 0, 0, 0, 78, 0, 0, 97, 117, 0, 66, - 67, 62, 0, 0, 0, 109, 107, 84, 73, 74, - 94, 95, 90, 92, 91, 93, 140, 141, 138, 136, - 137, 131, 132, 133, 134, 135, 47, 44, 43, 48, - 45, 46, 42, 57, 54, 53, 55, 56, 52, 60, - 59, 0, 125, 127, 120, 0, 98, 0, 64, 0, - 63, 89, 87, 0, 0, 0, 82, 0, 0, 68, - 112, 113, 0, 110, 0, 0, 0, 0, 100, 0, - 101, 103, 99, 108, 0, 0, 49, 104, 0, 0, - 105, 80, 111, 0, 0, 102, 0, 0, 50, 83, - 106, 0, 0, 81 + 0, 0, 0, 36, 38, 40, 80, 0, 81, 0, + 0, 0, 0, 82, 0, 0, 101, 121, 0, 70, + 71, 66, 0, 0, 0, 113, 111, 88, 77, 78, + 98, 99, 94, 96, 95, 97, 144, 145, 142, 140, + 141, 135, 136, 137, 138, 139, 47, 44, 43, 48, + 51, 53, 45, 46, 42, 61, 58, 57, 59, 60, + 56, 64, 63, 0, 129, 131, 124, 0, 102, 0, + 68, 0, 67, 93, 91, 0, 0, 0, 0, 0, + 86, 0, 0, 72, 116, 117, 0, 114, 0, 0, + 0, 0, 0, 0, 104, 0, 105, 107, 103, 112, + 0, 0, 49, 52, 54, 108, 0, 0, 109, 84, + 115, 0, 0, 106, 0, 0, 50, 87, 110, 0, + 0, 85 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -74, -74, 318, 319, -74, -74, -74, -74, -74, -74, - -74, -74, -74, -74, 292, -74, 286, -74, -74, -74, - -74, -74, -74, -74, -74, -74, 128, -74, -74, 233, + -74, -74, 294, 295, -74, -74, -74, -74, -74, -74, + -74, -74, -74, -74, 290, -74, 285, -74, -74, -74, + -74, -74, -74, -74, -74, -74, 123, -74, -74, 222, -49, -73, -74, -74, -74, -74, -74, -74, -74, -74, - 142, -74, 192, -74, -74, 126, 272, -68 + 139, -74, 191, -74, -74, 106, 259, -68 }; /* YYDEFGOTO[NTERM-NUM]. */ @@ -807,9 +822,9 @@ static const yytype_int16 yydefgoto[] = { -1, 1, 6, 7, 18, 34, 26, 29, 41, 8, 16, 20, 22, 31, 32, 38, 39, 52, 53, 54, - 123, 172, 124, 178, 125, 180, 76, 138, 139, 77, - 96, 79, 135, 227, 206, 144, 143, 187, 209, 210, - 128, 219, 147, 193, 202, 203, 80, 81 + 123, 174, 124, 180, 125, 182, 76, 138, 139, 77, + 96, 79, 135, 235, 212, 144, 143, 189, 215, 216, + 128, 227, 147, 195, 206, 207, 80, 81 }; /* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If @@ -817,136 +832,139 @@ static const yytype_int16 yydefgoto[] = number is the opposite. If YYTABLE_NINF, syntax error. */ static const yytype_int16 yytable[] = { - 78, 92, 97, 133, 5, 94, 95, 98, 55, 85, - 102, 103, 215, -79, 86, -39, -37, 126, 197, 129, - 130, 131, 132, 93, 216, 145, 140, 213, 9, 12, - 200, 214, 42, 142, 201, 43, 134, 229, 149, 150, + 78, 92, 97, 5, 145, 94, 95, 98, 85, 12, + 204, 17, 55, 86, 205, 201, 221, 126, 19, 129, + 130, 131, 132, 93, 21, 133, 140, 9, 222, -39, + -37, 42, 24, 142, 43, -83, 146, 23, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, - 161, 162, 163, 164, 165, 44, 45, 146, 198, 181, - 55, 56, 57, 58, 59, 207, 60, 61, 62, 63, - 17, 64, 46, 102, 103, 13, 14, 15, 65, 66, - 67, 68, 225, 19, 69, 21, 226, 173, 23, 70, - 71, 24, 25, 72, 191, 192, 2, 3, 27, 4, - 233, -18, -18, -18, 174, 175, 73, 176, 177, 28, - 74, 118, 119, 120, 121, 122, 199, 75, 99, 30, - 100, 101, 55, 33, 57, 58, 59, 205, 60, 61, - 62, 63, 35, 64, 40, 5, 120, 121, 122, 220, - 65, 66, 67, 68, 55, 49, 57, 58, 59, 37, - 60, 61, 62, 63, 47, 64, 195, 166, 230, 102, - 103, 51, 65, 66, -70, -70, 50, 82, 73, 224, - 83, 84, 74, 87, 167, 168, 169, 170, 171, 90, - 113, 114, 232, 116, 117, 118, 119, 120, 121, 122, - 73, 113, 114, 89, 74, 117, 118, 119, 120, 121, - 122, 90, 88, 113, 114, 115, 116, 117, 118, 119, - 120, 121, 122, -114, 113, 114, 105, 106, 104, 118, - 119, 120, 121, 122, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, - -114, 127, 141, 105, 106, 136, 64, 179, 137, 186, - 188, 107, 108, 109, 110, 111, 112, 113, 114, 115, - 116, 117, 118, 119, 120, 121, 122, 113, 114, 115, - 116, 117, 118, 119, 120, 121, 122, 113, 114, 115, - 116, 117, 118, 119, 120, 121, 122, 189, 182, 194, - 103, 196, 204, 212, 221, 218, 223, 231, 183, 113, + 161, 162, 163, 164, 165, 202, 44, 45, 27, 183, + 134, 55, 56, 57, 58, 59, 25, 60, 61, 62, + 63, 213, 64, 46, 102, 103, 219, 28, 102, 103, + 220, 65, 66, 67, 68, 33, 233, 69, 30, 175, + 234, 35, 70, 71, 193, 194, 72, 2, 3, 37, + 4, 237, -18, -18, -18, 241, 176, 177, 40, 73, + 47, 178, 179, 74, 99, 49, 100, 101, 203, 55, + 75, 57, 58, 59, 50, 60, 61, 62, 63, 82, + 64, 211, 118, 119, 120, 121, 122, 51, 5, 65, + 66, 67, 68, 83, 55, 228, 57, 58, 59, 84, + 60, 61, 62, 63, 87, 64, 199, 88, 13, 14, + 15, 89, 166, 104, 65, 66, 238, 73, 102, 103, + 127, 74, 120, 121, 122, -74, -74, 232, 90, 167, + 168, 169, 170, 171, 172, 173, 136, 141, 64, 181, + 240, 188, 73, 113, 114, 196, 74, 117, 118, 119, + 120, 121, 122, 90, 191, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, -118, 113, 114, 105, 106, + 190, 118, 119, 120, 121, 122, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, + 121, 122, -118, 197, 198, 105, 106, 103, 208, 200, + 137, 218, 209, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 113, - 114, 115, 116, 117, 118, 119, 120, 121, 122, 228, - 190, 10, 11, 36, 48, 208, 185, 184, 113, 114, - 115, 116, 117, 118, 119, 120, 121, 122, 148, 211, - 222, 91, 0, 0, 0, 0, 137, 113, 114, 115, - 116, 117, 118, 119, 120, 121, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 217, 113, 114, 115, 116, - 117, 118, 119, 120, 121, 122 + 114, 115, 116, 117, 118, 119, 120, 121, 122, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 210, + 184, 223, 224, 226, 239, 229, 231, 10, 11, 236, + 185, 113, 114, 115, 116, 117, 118, 119, 120, 121, + 122, 113, 114, 115, 116, 117, 118, 119, 120, 121, + 122, 36, 192, 48, 214, 187, 230, 148, 91, 186, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, + 217, 0, 0, 0, 0, 0, 0, 0, 137, 113, + 114, 115, 116, 117, 118, 119, 120, 121, 122, 0, + 0, 0, 0, 0, 0, 0, 0, 225, 113, 114, + 115, 116, 117, 118, 119, 120, 121, 122, 113, 114, + 0, 116, 117, 118, 119, 120, 121, 122 }; static const yytype_int16 yycheck[] = { - 49, 69, 75, 1, 39, 73, 74, 75, 11, 28, - 42, 43, 57, 11, 33, 21, 22, 85, 33, 87, - 88, 89, 90, 72, 69, 36, 99, 69, 63, 20, - 12, 73, 17, 101, 16, 20, 34, 69, 106, 107, + 49, 69, 75, 41, 38, 73, 74, 75, 30, 20, + 12, 11, 11, 35, 16, 35, 59, 85, 68, 87, + 88, 89, 90, 72, 11, 1, 99, 65, 71, 21, + 22, 17, 11, 101, 20, 11, 70, 66, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 40, 41, 68, 73, 127, - 11, 12, 13, 14, 15, 68, 17, 18, 19, 20, - 11, 22, 57, 42, 43, 5, 6, 7, 29, 30, - 31, 32, 69, 66, 35, 11, 73, 6, 64, 40, - 41, 11, 8, 44, 143, 144, 0, 1, 66, 3, - 69, 5, 6, 7, 23, 24, 57, 26, 27, 9, - 61, 56, 57, 58, 59, 60, 189, 68, 68, 11, - 70, 71, 11, 66, 13, 14, 15, 195, 17, 18, - 19, 20, 67, 22, 10, 39, 58, 59, 60, 207, - 29, 30, 31, 32, 11, 66, 13, 14, 15, 12, - 17, 18, 19, 20, 67, 22, 4, 6, 226, 42, - 43, 17, 29, 30, 42, 43, 65, 20, 57, 218, - 22, 21, 61, 71, 23, 24, 25, 26, 27, 68, - 51, 52, 231, 54, 55, 56, 57, 58, 59, 60, - 57, 51, 52, 68, 61, 55, 56, 57, 58, 59, - 60, 68, 71, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 34, 51, 52, 37, 38, 34, 56, - 57, 58, 59, 60, 45, 46, 47, 48, 49, 50, + 118, 119, 120, 121, 122, 75, 42, 43, 68, 127, + 36, 11, 12, 13, 14, 15, 8, 17, 18, 19, + 20, 70, 22, 59, 44, 45, 71, 9, 44, 45, + 75, 31, 32, 33, 34, 68, 71, 37, 11, 6, + 75, 69, 42, 43, 143, 144, 46, 0, 1, 12, + 3, 71, 5, 6, 7, 71, 23, 24, 10, 59, + 69, 28, 29, 63, 70, 68, 72, 73, 191, 11, + 70, 13, 14, 15, 67, 17, 18, 19, 20, 20, + 22, 199, 58, 59, 60, 61, 62, 17, 41, 31, + 32, 33, 34, 22, 11, 213, 13, 14, 15, 21, + 17, 18, 19, 20, 73, 22, 4, 73, 5, 6, + 7, 70, 6, 36, 31, 32, 234, 59, 44, 45, + 70, 63, 60, 61, 62, 44, 45, 226, 70, 23, + 24, 25, 26, 27, 28, 29, 71, 11, 22, 6, + 239, 11, 59, 53, 54, 70, 63, 57, 58, 59, + 60, 61, 62, 70, 75, 53, 54, 55, 56, 57, + 58, 59, 60, 61, 62, 36, 53, 54, 39, 40, + 71, 58, 59, 60, 61, 62, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 34, 68, 11, 37, 38, 69, 22, 6, 69, 11, - 69, 45, 46, 47, 48, 49, 50, 51, 52, 53, - 54, 55, 56, 57, 58, 59, 60, 51, 52, 53, - 54, 55, 56, 57, 58, 59, 60, 51, 52, 53, - 54, 55, 56, 57, 58, 59, 60, 73, 72, 68, - 43, 66, 17, 11, 66, 68, 17, 68, 72, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 69, - 72, 3, 3, 31, 38, 197, 134, 69, 51, 52, - 53, 54, 55, 56, 57, 58, 59, 60, 105, 197, - 214, 69, -1, -1, -1, -1, 69, 51, 52, 53, - 54, 55, 56, 57, 58, 59, 60, -1, -1, -1, - -1, -1, -1, -1, -1, 69, 51, 52, 53, 54, - 55, 56, 57, 58, 59, 60 + 61, 62, 36, 70, 70, 39, 40, 45, 17, 68, + 71, 11, 20, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 20, + 74, 71, 71, 70, 70, 68, 17, 3, 3, 71, + 74, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 31, 74, 38, 201, 134, 220, 105, 69, 71, + 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 201, -1, -1, -1, -1, -1, -1, -1, 71, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, + -1, -1, -1, -1, -1, -1, -1, 71, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 53, 54, + -1, 56, 57, 58, 59, 60, 61, 62 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint8 yystos[] = { - 0, 75, 0, 1, 3, 39, 76, 77, 83, 63, - 76, 77, 20, 5, 6, 7, 84, 11, 78, 66, - 85, 11, 86, 64, 11, 8, 80, 66, 9, 81, - 11, 87, 88, 66, 79, 67, 88, 12, 89, 90, - 10, 82, 17, 20, 40, 41, 57, 67, 90, 66, - 65, 17, 91, 92, 93, 11, 12, 13, 14, 15, - 17, 18, 19, 20, 22, 29, 30, 31, 32, 35, - 40, 41, 44, 57, 61, 68, 100, 103, 104, 105, - 120, 121, 20, 22, 21, 28, 33, 71, 71, 68, - 68, 120, 121, 104, 121, 121, 104, 105, 121, 68, - 70, 71, 42, 43, 34, 37, 38, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 94, 96, 98, 121, 68, 114, 121, - 121, 121, 121, 1, 34, 106, 69, 69, 101, 102, - 105, 11, 121, 110, 109, 36, 68, 116, 103, 121, - 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, - 121, 121, 121, 121, 121, 121, 6, 23, 24, 25, - 26, 27, 95, 6, 23, 24, 26, 27, 97, 6, - 99, 121, 72, 72, 69, 116, 11, 111, 69, 73, - 72, 104, 104, 117, 68, 4, 66, 33, 73, 105, - 12, 16, 118, 119, 17, 121, 108, 68, 100, 112, - 113, 114, 11, 69, 73, 57, 69, 69, 68, 115, - 121, 66, 119, 17, 104, 69, 73, 107, 69, 69, - 121, 68, 104, 69 + 0, 77, 0, 1, 3, 41, 78, 79, 85, 65, + 78, 79, 20, 5, 6, 7, 86, 11, 80, 68, + 87, 11, 88, 66, 11, 8, 82, 68, 9, 83, + 11, 89, 90, 68, 81, 69, 90, 12, 91, 92, + 10, 84, 17, 20, 42, 43, 59, 69, 92, 68, + 67, 17, 93, 94, 95, 11, 12, 13, 14, 15, + 17, 18, 19, 20, 22, 31, 32, 33, 34, 37, + 42, 43, 46, 59, 63, 70, 102, 105, 106, 107, + 122, 123, 20, 22, 21, 30, 35, 73, 73, 70, + 70, 122, 123, 106, 123, 123, 106, 107, 123, 70, + 72, 73, 44, 45, 36, 39, 40, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 96, 98, 100, 123, 70, 116, 123, + 123, 123, 123, 1, 36, 108, 71, 71, 103, 104, + 107, 11, 123, 112, 111, 38, 70, 118, 105, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 6, 23, 24, 25, + 26, 27, 28, 29, 97, 6, 23, 24, 28, 29, + 99, 6, 101, 123, 74, 74, 71, 118, 11, 113, + 71, 75, 74, 106, 106, 119, 70, 70, 70, 4, + 68, 35, 75, 107, 12, 16, 120, 121, 17, 20, + 20, 123, 110, 70, 102, 114, 115, 116, 11, 71, + 75, 59, 71, 71, 71, 71, 70, 117, 123, 68, + 121, 17, 106, 71, 75, 109, 71, 71, 123, 70, + 106, 71 }; /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint8 yyr1[] = { - 0, 74, 75, 75, 75, 75, 75, 75, 75, 76, - 78, 79, 77, 80, 80, 81, 81, 82, 83, 83, - 84, 84, 85, 85, 86, 86, 87, 87, 88, 88, - 88, 88, 88, 89, 89, 91, 90, 92, 90, 93, - 90, 94, 94, 95, 95, 95, 95, 95, 95, 95, - 95, 96, 96, 97, 97, 97, 97, 97, 98, 98, - 99, 100, 100, 100, 100, 101, 101, 102, 102, 103, - 104, 105, 105, 105, 105, 105, 105, 105, 105, 106, - 107, 105, 108, 105, 105, 105, 109, 105, 110, 105, - 105, 105, 105, 105, 105, 105, 105, 105, 111, 111, - 112, 112, 113, 113, 114, 115, 115, 117, 116, 116, - 118, 118, 119, 119, 120, 120, 120, 121, 121, 121, - 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, - 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, - 121, 121, 121 + 0, 76, 77, 77, 77, 77, 77, 77, 77, 78, + 80, 81, 79, 82, 82, 83, 83, 84, 85, 85, + 86, 86, 87, 87, 88, 88, 89, 89, 90, 90, + 90, 90, 90, 91, 91, 93, 92, 94, 92, 95, + 92, 96, 96, 97, 97, 97, 97, 97, 97, 97, + 97, 97, 97, 97, 97, 98, 98, 99, 99, 99, + 99, 99, 100, 100, 101, 102, 102, 102, 102, 103, + 103, 104, 104, 105, 106, 107, 107, 107, 107, 107, + 107, 107, 107, 108, 109, 107, 110, 107, 107, 107, + 111, 107, 112, 107, 107, 107, 107, 107, 107, 107, + 107, 107, 113, 113, 114, 114, 115, 115, 116, 117, + 117, 119, 118, 118, 120, 120, 121, 121, 122, 122, + 122, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123, 123, 123, 123, + 123, 123, 123, 123, 123, 123, 123 }; /* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */ @@ -957,16 +975,16 @@ static const yytype_uint8 yyr2[] = 1, 1, 0, 2, 1, 2, 1, 2, 3, 3, 4, 3, 3, 1, 2, 0, 5, 0, 5, 0, 5, 0, 2, 1, 1, 1, 1, 1, 1, 4, - 6, 0, 2, 1, 1, 1, 1, 1, 0, 2, - 1, 1, 3, 4, 4, 0, 1, 1, 3, 1, - 1, 1, 1, 3, 3, 1, 3, 3, 3, 0, - 0, 11, 0, 9, 3, 2, 0, 4, 0, 4, - 3, 3, 3, 3, 3, 3, 1, 3, 1, 3, - 1, 1, 3, 1, 5, 1, 3, 0, 4, 1, - 1, 3, 1, 1, 1, 1, 1, 3, 1, 1, - 4, 1, 1, 1, 1, 4, 1, 4, 1, 1, - 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, - 3, 3, 1 + 6, 1, 4, 1, 4, 0, 2, 1, 1, 1, + 1, 1, 0, 2, 1, 1, 3, 4, 4, 0, + 1, 1, 3, 1, 1, 1, 1, 3, 3, 1, + 3, 3, 3, 0, 0, 11, 0, 9, 3, 2, + 0, 4, 0, 4, 3, 3, 3, 3, 3, 3, + 1, 3, 1, 3, 1, 1, 3, 1, 5, 1, + 3, 0, 4, 1, 1, 3, 1, 1, 1, 1, + 1, 3, 1, 1, 4, 1, 1, 1, 1, 4, + 1, 4, 1, 1, 2, 3, 3, 3, 3, 3, + 3, 3, 3, 2, 3, 3, 1 }; @@ -1329,7 +1347,6 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, case N: \ yyformat = S; \ break - default: /* Avoid compiler warnings. */ YYCASE_(0, YY_("syntax error")); YYCASE_(1, YY_("syntax error, unexpected %s")); YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); @@ -1395,69 +1412,93 @@ yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, void *yyscanner, Y switch (yytype) { case 11: /* "identifier" */ -#line 263 "grammar.y" /* yacc.c:1258 */ +#line 269 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1401 "grammar.c" /* yacc.c:1258 */ +#line 1418 "grammar.c" /* yacc.c:1257 */ break; case 12: /* "string identifier" */ -#line 267 "grammar.y" /* yacc.c:1258 */ +#line 273 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1407 "grammar.c" /* yacc.c:1258 */ +#line 1424 "grammar.c" /* yacc.c:1257 */ break; case 13: /* "string count" */ -#line 264 "grammar.y" /* yacc.c:1258 */ +#line 270 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1413 "grammar.c" /* yacc.c:1258 */ +#line 1430 "grammar.c" /* yacc.c:1257 */ break; case 14: /* "string offset" */ -#line 265 "grammar.y" /* yacc.c:1258 */ +#line 271 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1419 "grammar.c" /* yacc.c:1258 */ +#line 1436 "grammar.c" /* yacc.c:1257 */ break; case 15: /* "string length" */ -#line 266 "grammar.y" /* yacc.c:1258 */ +#line 272 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1425 "grammar.c" /* yacc.c:1258 */ +#line 1442 "grammar.c" /* yacc.c:1257 */ break; case 16: /* "string identifier with wildcard" */ -#line 268 "grammar.y" /* yacc.c:1258 */ +#line 274 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1431 "grammar.c" /* yacc.c:1258 */ +#line 1448 "grammar.c" /* yacc.c:1257 */ break; case 20: /* "text string" */ -#line 269 "grammar.y" /* yacc.c:1258 */ +#line 275 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).sized_string)); ((*yyvaluep).sized_string) = NULL; } -#line 1437 "grammar.c" /* yacc.c:1258 */ +#line 1454 "grammar.c" /* yacc.c:1257 */ break; case 21: /* "hex string" */ -#line 270 "grammar.y" /* yacc.c:1258 */ +#line 276 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).sized_string)); ((*yyvaluep).sized_string) = NULL; } -#line 1443 "grammar.c" /* yacc.c:1258 */ +#line 1460 "grammar.c" /* yacc.c:1257 */ break; case 22: /* "regular expression" */ -#line 271 "grammar.y" /* yacc.c:1258 */ +#line 277 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).sized_string)); ((*yyvaluep).sized_string) = NULL; } -#line 1449 "grammar.c" /* yacc.c:1258 */ +#line 1466 "grammar.c" /* yacc.c:1257 */ break; - case 101: /* arguments */ -#line 273 "grammar.y" /* yacc.c:1258 */ - { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1455 "grammar.c" /* yacc.c:1258 */ + case 96: /* string_modifiers */ +#line 290 "grammar.y" /* yacc.c:1257 */ + { + if (((*yyvaluep).modifier).alphabet != NULL) + { + yr_free(((*yyvaluep).modifier).alphabet); + ((*yyvaluep).modifier).alphabet = NULL; + } +} +#line 1478 "grammar.c" /* yacc.c:1257 */ break; - case 102: /* arguments_list */ -#line 274 "grammar.y" /* yacc.c:1258 */ + case 97: /* string_modifier */ +#line 282 "grammar.y" /* yacc.c:1257 */ + { + if (((*yyvaluep).modifier).alphabet != NULL) + { + yr_free(((*yyvaluep).modifier).alphabet); + ((*yyvaluep).modifier).alphabet = NULL; + } +} +#line 1490 "grammar.c" /* yacc.c:1257 */ + break; + + case 103: /* arguments */ +#line 279 "grammar.y" /* yacc.c:1257 */ { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } -#line 1461 "grammar.c" /* yacc.c:1258 */ +#line 1496 "grammar.c" /* yacc.c:1257 */ + break; + + case 104: /* arguments_list */ +#line 280 "grammar.y" /* yacc.c:1257 */ + { yr_free(((*yyvaluep).c_string)); ((*yyvaluep).c_string) = NULL; } +#line 1502 "grammar.c" /* yacc.c:1257 */ break; @@ -1723,15 +1764,15 @@ yyreduce: switch (yyn) { case 8: -#line 299 "grammar.y" /* yacc.c:1663 */ +#line 324 "grammar.y" /* yacc.c:1646 */ { _yr_compiler_pop_file_name(compiler); } -#line 1731 "grammar.c" /* yacc.c:1663 */ +#line 1772 "grammar.c" /* yacc.c:1646 */ break; case 9: -#line 307 "grammar.y" /* yacc.c:1663 */ +#line 332 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_import(yyscanner, (yyvsp[0].sized_string)); @@ -1739,56 +1780,60 @@ yyreduce: fail_if_error(result); } -#line 1743 "grammar.c" /* yacc.c:1663 */ +#line 1784 "grammar.c" /* yacc.c:1646 */ break; case 10: -#line 319 "grammar.y" /* yacc.c:1663 */ +#line 344 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_rule_declaration_phase_1( yyscanner, (int32_t) (yyvsp[-2].integer), (yyvsp[0].c_string), &(yyval.rule))); } -#line 1752 "grammar.c" /* yacc.c:1663 */ +#line 1793 "grammar.c" /* yacc.c:1646 */ break; case 11: -#line 324 "grammar.y" /* yacc.c:1663 */ +#line 349 "grammar.y" /* yacc.c:1646 */ { - YR_RULE* rule = (yyvsp[-4].rule); // rule created in phase 1 + YR_RULE* rule = (YR_RULE*) yr_arena_ref_to_ptr( + compiler->arena, &(yyvsp[-4].rule)); - rule->tags = (yyvsp[-3].c_string); - rule->metas = (yyvsp[-1].meta); - rule->strings = (yyvsp[0].string); + rule->tags = (char*) yr_arena_ref_to_ptr( + compiler->arena, &(yyvsp[-3].tag)); + + rule->metas = (YR_META*) yr_arena_ref_to_ptr( + compiler->arena, &(yyvsp[-1].meta)); + + rule->strings = (YR_STRING*) yr_arena_ref_to_ptr( + compiler->arena, &(yyvsp[0].string)); } -#line 1764 "grammar.c" /* yacc.c:1663 */ +#line 1811 "grammar.c" /* yacc.c:1646 */ break; case 12: -#line 332 "grammar.y" /* yacc.c:1663 */ +#line 363 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_rule_declaration_phase_2( - yyscanner, (yyvsp[-7].rule)); // rule created in phase 1 + yyscanner, &(yyvsp[-7].rule)); // rule created in phase 1 yr_free((yyvsp[-8].c_string)); fail_if_error(result); } -#line 1777 "grammar.c" /* yacc.c:1663 */ +#line 1824 "grammar.c" /* yacc.c:1646 */ break; case 13: -#line 345 "grammar.y" /* yacc.c:1663 */ +#line 376 "grammar.y" /* yacc.c:1646 */ { - (yyval.meta) = NULL; + (yyval.meta) = YR_ARENA_NULL_REF; } -#line 1785 "grammar.c" /* yacc.c:1663 */ +#line 1832 "grammar.c" /* yacc.c:1646 */ break; case 14: -#line 349 "grammar.y" /* yacc.c:1663 */ +#line 380 "grammar.y" /* yacc.c:1646 */ { - int result; - // Each rule have a list of meta-data info, consisting in a // sequence of YR_META structures. The last YR_META structure does // not represent a real meta-data, it's just a end-of-list marker @@ -1800,167 +1845,157 @@ yyreduce: memset(&null_meta, 0xFF, sizeof(YR_META)); null_meta.type = META_TYPE_NULL; - result = yr_arena_write_data( - compiler->metas_arena, + fail_if_error(yr_arena_write_data( + compiler->arena, + YR_METAS_TABLE, &null_meta, sizeof(YR_META), - NULL); + NULL)); (yyval.meta) = (yyvsp[0].meta); - - fail_if_error(result); } -#line 1814 "grammar.c" /* yacc.c:1663 */ +#line 1858 "grammar.c" /* yacc.c:1646 */ break; case 15: -#line 378 "grammar.y" /* yacc.c:1663 */ +#line 406 "grammar.y" /* yacc.c:1646 */ { - (yyval.string) = NULL; + (yyval.string) = YR_ARENA_NULL_REF; } -#line 1822 "grammar.c" /* yacc.c:1663 */ +#line 1866 "grammar.c" /* yacc.c:1646 */ break; case 16: -#line 382 "grammar.y" /* yacc.c:1663 */ +#line 410 "grammar.y" /* yacc.c:1646 */ { - // Each rule have a list of strings, consisting in a sequence - // of YR_STRING structures. The last YR_STRING structure does not - // represent a real string, it's just a end-of-list marker - // identified by a specific flag (STRING_FLAGS_NULL). Here we - // write the end-of-list marker. + YR_STRING* string = (YR_STRING*) yr_arena_get_ptr( + compiler->arena, + YR_STRINGS_TABLE, + (compiler->current_string_idx - 1) * sizeof(YR_STRING)); - YR_STRING null_string; - - memset(&null_string, 0xFF, sizeof(YR_STRING)); - null_string.g_flags = STRING_GFLAGS_NULL; - - fail_if_error(yr_arena_write_data( - compiler->strings_arena, - &null_string, - sizeof(YR_STRING), - NULL)); + string->flags |= STRING_FLAGS_LAST_IN_RULE; (yyval.string) = (yyvsp[0].string); } -#line 1847 "grammar.c" /* yacc.c:1663 */ +#line 1881 "grammar.c" /* yacc.c:1646 */ break; case 18: -#line 411 "grammar.y" /* yacc.c:1663 */ +#line 429 "grammar.y" /* yacc.c:1646 */ { (yyval.integer) = 0; } -#line 1853 "grammar.c" /* yacc.c:1663 */ +#line 1887 "grammar.c" /* yacc.c:1646 */ break; case 19: -#line 412 "grammar.y" /* yacc.c:1663 */ +#line 430 "grammar.y" /* yacc.c:1646 */ { (yyval.integer) = (yyvsp[-1].integer) | (yyvsp[0].integer); } -#line 1859 "grammar.c" /* yacc.c:1663 */ +#line 1893 "grammar.c" /* yacc.c:1646 */ break; case 20: -#line 417 "grammar.y" /* yacc.c:1663 */ - { (yyval.integer) = RULE_GFLAGS_PRIVATE; } -#line 1865 "grammar.c" /* yacc.c:1663 */ +#line 435 "grammar.y" /* yacc.c:1646 */ + { (yyval.integer) = RULE_FLAGS_PRIVATE; } +#line 1899 "grammar.c" /* yacc.c:1646 */ break; case 21: -#line 418 "grammar.y" /* yacc.c:1663 */ - { (yyval.integer) = RULE_GFLAGS_GLOBAL; } -#line 1871 "grammar.c" /* yacc.c:1663 */ +#line 436 "grammar.y" /* yacc.c:1646 */ + { (yyval.integer) = RULE_FLAGS_GLOBAL; } +#line 1905 "grammar.c" /* yacc.c:1646 */ break; case 22: -#line 424 "grammar.y" /* yacc.c:1663 */ +#line 442 "grammar.y" /* yacc.c:1646 */ { - (yyval.c_string) = NULL; + (yyval.tag) = YR_ARENA_NULL_REF; } -#line 1879 "grammar.c" /* yacc.c:1663 */ +#line 1913 "grammar.c" /* yacc.c:1646 */ break; case 23: -#line 428 "grammar.y" /* yacc.c:1663 */ +#line 446 "grammar.y" /* yacc.c:1646 */ { // Tags list is represented in the arena as a sequence // of null-terminated strings, the sequence ends with an // additional null character. Here we write the ending null //character. Example: tag1\0tag2\0tag3\0\0 - int result = yr_arena_write_string( - yyget_extra(yyscanner)->sz_arena, "", NULL); + fail_if_error(yr_arena_write_string( + yyget_extra(yyscanner)->arena, YR_SZ_POOL, "", NULL)); - fail_if_error(result); - - (yyval.c_string) = (yyvsp[0].c_string); + (yyval.tag) = (yyvsp[0].tag); } -#line 1897 "grammar.c" /* yacc.c:1663 */ +#line 1929 "grammar.c" /* yacc.c:1646 */ break; case 24: -#line 446 "grammar.y" /* yacc.c:1663 */ +#line 462 "grammar.y" /* yacc.c:1646 */ { int result = yr_arena_write_string( - yyget_extra(yyscanner)->sz_arena, (yyvsp[0].c_string), &(yyval.c_string)); + yyget_extra(yyscanner)->arena, YR_SZ_POOL, (yyvsp[0].c_string), &(yyval.tag)); yr_free((yyvsp[0].c_string)); fail_if_error(result); } -#line 1910 "grammar.c" /* yacc.c:1663 */ +#line 1942 "grammar.c" /* yacc.c:1646 */ break; case 25: -#line 455 "grammar.y" /* yacc.c:1663 */ +#line 471 "grammar.y" /* yacc.c:1646 */ { - int result = ERROR_SUCCESS; + YR_ARENA_REF ref; - char* tag_name = (yyvsp[-1].c_string); - size_t tag_length = tag_name != NULL ? strlen(tag_name) : 0; - - while (tag_length > 0) - { - if (strcmp(tag_name, (yyvsp[0].c_string)) == 0) - { - yr_compiler_set_error_extra_info(compiler, tag_name); - result = ERROR_DUPLICATED_TAG_IDENTIFIER; - break; - } - - tag_name = (char*) yr_arena_next_address( - yyget_extra(yyscanner)->sz_arena, - tag_name, - tag_length + 1); - - tag_length = tag_name != NULL ? strlen(tag_name) : 0; - } - - if (result == ERROR_SUCCESS) - result = yr_arena_write_string( - yyget_extra(yyscanner)->sz_arena, (yyvsp[0].c_string), NULL); + // Write the new tag identifier. + int result = yr_arena_write_string( + yyget_extra(yyscanner)->arena, YR_SZ_POOL, (yyvsp[0].c_string), &ref); yr_free((yyvsp[0].c_string)); fail_if_error(result); - (yyval.c_string) = (yyvsp[-1].c_string); + // Get the address for the tag identifier just written. + char* new_tag = (char*) yr_arena_ref_to_ptr( + compiler->arena, &ref); + + // Take the address of first tag's identifier in the list. + char* tag = (char*) yr_arena_ref_to_ptr( + compiler->arena, &(yyval.tag)); + + // Search for duplicated tags. Tags are written one after + // the other, with zeroes in between (i.e: tag1/0tag2/0tag3) + // that's why can use tag < new_tag as the condition for the + // loop. + while (tag < new_tag) + { + if (strcmp(tag, new_tag) == 0) + { + yr_compiler_set_error_extra_info(compiler, tag); + fail_with_error(ERROR_DUPLICATED_TAG_IDENTIFIER); + } + + tag += strlen(tag) + 1; + } + + (yyval.tag) = (yyvsp[-1].tag); } -#line 1948 "grammar.c" /* yacc.c:1663 */ +#line 1983 "grammar.c" /* yacc.c:1646 */ break; case 26: -#line 493 "grammar.y" /* yacc.c:1663 */ +#line 512 "grammar.y" /* yacc.c:1646 */ { (yyval.meta) = (yyvsp[0].meta); } -#line 1954 "grammar.c" /* yacc.c:1663 */ +#line 1989 "grammar.c" /* yacc.c:1646 */ break; case 27: -#line 494 "grammar.y" /* yacc.c:1663 */ +#line 513 "grammar.y" /* yacc.c:1646 */ { (yyval.meta) = (yyvsp[-1].meta); } -#line 1960 "grammar.c" /* yacc.c:1663 */ +#line 1995 "grammar.c" /* yacc.c:1646 */ break; case 28: -#line 500 "grammar.y" /* yacc.c:1663 */ +#line 519 "grammar.y" /* yacc.c:1646 */ { SIZED_STRING* sized_string = (yyvsp[0].sized_string); @@ -1977,11 +2012,11 @@ yyreduce: fail_if_error(result); } -#line 1981 "grammar.c" /* yacc.c:1663 */ +#line 2016 "grammar.c" /* yacc.c:1646 */ break; case 29: -#line 517 "grammar.y" /* yacc.c:1663 */ +#line 536 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_meta_declaration( yyscanner, @@ -1995,11 +2030,11 @@ yyreduce: fail_if_error(result); } -#line 1999 "grammar.c" /* yacc.c:1663 */ +#line 2034 "grammar.c" /* yacc.c:1646 */ break; case 30: -#line 531 "grammar.y" /* yacc.c:1663 */ +#line 550 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_meta_declaration( yyscanner, @@ -2013,11 +2048,11 @@ yyreduce: fail_if_error(result); } -#line 2017 "grammar.c" /* yacc.c:1663 */ +#line 2052 "grammar.c" /* yacc.c:1646 */ break; case 31: -#line 545 "grammar.y" /* yacc.c:1663 */ +#line 564 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_meta_declaration( yyscanner, @@ -2031,11 +2066,11 @@ yyreduce: fail_if_error(result); } -#line 2035 "grammar.c" /* yacc.c:1663 */ +#line 2070 "grammar.c" /* yacc.c:1646 */ break; case 32: -#line 559 "grammar.y" /* yacc.c:1663 */ +#line 578 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_meta_declaration( yyscanner, @@ -2049,58 +2084,59 @@ yyreduce: fail_if_error(result); } -#line 2053 "grammar.c" /* yacc.c:1663 */ +#line 2088 "grammar.c" /* yacc.c:1646 */ break; case 33: -#line 576 "grammar.y" /* yacc.c:1663 */ +#line 595 "grammar.y" /* yacc.c:1646 */ { (yyval.string) = (yyvsp[0].string); } -#line 2059 "grammar.c" /* yacc.c:1663 */ +#line 2094 "grammar.c" /* yacc.c:1646 */ break; case 34: -#line 577 "grammar.y" /* yacc.c:1663 */ +#line 596 "grammar.y" /* yacc.c:1646 */ { (yyval.string) = (yyvsp[-1].string); } -#line 2065 "grammar.c" /* yacc.c:1663 */ +#line 2100 "grammar.c" /* yacc.c:1646 */ break; case 35: -#line 583 "grammar.y" /* yacc.c:1663 */ +#line 602 "grammar.y" /* yacc.c:1646 */ { compiler->current_line = yyget_lineno(yyscanner); } -#line 2073 "grammar.c" /* yacc.c:1663 */ +#line 2108 "grammar.c" /* yacc.c:1646 */ break; case 36: -#line 587 "grammar.y" /* yacc.c:1663 */ +#line 606 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_string_declaration( yyscanner, (yyvsp[0].modifier), (yyvsp[-4].c_string), (yyvsp[-1].sized_string), &(yyval.string)); yr_free((yyvsp[-4].c_string)); yr_free((yyvsp[-1].sized_string)); + yr_free((yyvsp[0].modifier).alphabet); fail_if_error(result); compiler->current_line = 0; } -#line 2088 "grammar.c" /* yacc.c:1663 */ +#line 2124 "grammar.c" /* yacc.c:1646 */ break; case 37: -#line 598 "grammar.y" /* yacc.c:1663 */ +#line 618 "grammar.y" /* yacc.c:1646 */ { compiler->current_line = yyget_lineno(yyscanner); } -#line 2096 "grammar.c" /* yacc.c:1663 */ +#line 2132 "grammar.c" /* yacc.c:1646 */ break; case 38: -#line 602 "grammar.y" /* yacc.c:1663 */ +#line 622 "grammar.y" /* yacc.c:1646 */ { int result; - (yyvsp[0].modifier).flags |= STRING_GFLAGS_REGEXP; + (yyvsp[0].modifier).flags |= STRING_FLAGS_REGEXP; result = yr_parser_reduce_string_declaration( yyscanner, (yyvsp[0].modifier), (yyvsp[-4].c_string), (yyvsp[-1].sized_string), &(yyval.string)); @@ -2112,23 +2148,23 @@ yyreduce: compiler->current_line = 0; } -#line 2116 "grammar.c" /* yacc.c:1663 */ +#line 2152 "grammar.c" /* yacc.c:1646 */ break; case 39: -#line 618 "grammar.y" /* yacc.c:1663 */ +#line 638 "grammar.y" /* yacc.c:1646 */ { compiler->current_line = yyget_lineno(yyscanner); } -#line 2124 "grammar.c" /* yacc.c:1663 */ +#line 2160 "grammar.c" /* yacc.c:1646 */ break; case 40: -#line 622 "grammar.y" /* yacc.c:1663 */ +#line 642 "grammar.y" /* yacc.c:1646 */ { int result; - (yyvsp[0].modifier).flags |= STRING_GFLAGS_HEXADECIMAL; + (yyvsp[0].modifier).flags |= STRING_FLAGS_HEXADECIMAL; result = yr_parser_reduce_string_declaration( yyscanner, (yyvsp[0].modifier), (yyvsp[-4].c_string), (yyvsp[-1].sized_string), &(yyval.string)); @@ -2140,81 +2176,122 @@ yyreduce: compiler->current_line = 0; } -#line 2144 "grammar.c" /* yacc.c:1663 */ +#line 2180 "grammar.c" /* yacc.c:1646 */ break; case 41: -#line 642 "grammar.y" /* yacc.c:1663 */ +#line 662 "grammar.y" /* yacc.c:1646 */ { (yyval.modifier).flags = 0; (yyval.modifier).xor_min = 0; (yyval.modifier).xor_max = 0; + (yyval.modifier).alphabet = NULL; } -#line 2154 "grammar.c" /* yacc.c:1663 */ +#line 2191 "grammar.c" /* yacc.c:1646 */ break; case 42: -#line 648 "grammar.y" /* yacc.c:1663 */ +#line 669 "grammar.y" /* yacc.c:1646 */ { (yyval.modifier) = (yyvsp[-1].modifier); - set_flag_or_error((yyval.modifier).flags, (yyvsp[0].modifier).flags); - // Only set the xor minimum and maximum if we are dealing with the // xor modifier. If we don't check for this then we can end up with // "xor wide" resulting in whatever is on the stack for "wide" // overwriting the values for xor. - if ((yyvsp[0].modifier).flags & STRING_GFLAGS_XOR) + if ((yyvsp[0].modifier).flags & STRING_FLAGS_XOR) { (yyval.modifier).xor_min = (yyvsp[0].modifier).xor_min; (yyval.modifier).xor_max = (yyvsp[0].modifier).xor_max; } + + // Only set the base64 alphabet if we are dealing with the base64 + // modifier. If we don't check for this then we can end up with + // "base64 ascii" resulting in whatever is on the stack for "ascii" + // overwriting the values for base64. + if (((yyvsp[0].modifier).flags & STRING_FLAGS_BASE64) || + ((yyvsp[0].modifier).flags & STRING_FLAGS_BASE64_WIDE)) + { + if ((yyval.modifier).alphabet != NULL) + { + if (sized_string_cmp((yyval.modifier).alphabet, (yyvsp[0].modifier).alphabet) != 0) + { + yr_compiler_set_error_extra_info( + compiler, "can not specify multiple alphabets"); + + yr_free((yyvsp[0].modifier).alphabet); + yr_free((yyval.modifier).alphabet); + + fail_with_error(ERROR_INVALID_MODIFIER); + } + else + { + yr_free((yyvsp[0].modifier).alphabet); + } + } + else + { + (yyval.modifier).alphabet = (yyvsp[0].modifier).alphabet; + } + } + + if ((yyval.modifier).flags & (yyvsp[0].modifier).flags) + { + if ((yyval.modifier).alphabet != NULL) + yr_free((yyval.modifier).alphabet); + + fail_with_error(ERROR_DUPLICATED_MODIFIER); + } + else + { + (yyval.modifier).flags = (yyval.modifier).flags | (yyvsp[0].modifier).flags; + } } -#line 2174 "grammar.c" /* yacc.c:1663 */ +#line 2251 "grammar.c" /* yacc.c:1646 */ break; case 43: -#line 667 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_WIDE; } -#line 2180 "grammar.c" /* yacc.c:1663 */ +#line 728 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_WIDE; } +#line 2257 "grammar.c" /* yacc.c:1646 */ break; case 44: -#line 668 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_ASCII; } -#line 2186 "grammar.c" /* yacc.c:1663 */ +#line 729 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_ASCII; } +#line 2263 "grammar.c" /* yacc.c:1646 */ break; case 45: -#line 669 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_NO_CASE; } -#line 2192 "grammar.c" /* yacc.c:1663 */ +#line 730 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_NO_CASE; } +#line 2269 "grammar.c" /* yacc.c:1646 */ break; case 46: -#line 670 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_FULL_WORD; } -#line 2198 "grammar.c" /* yacc.c:1663 */ +#line 731 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_FULL_WORD; } +#line 2275 "grammar.c" /* yacc.c:1646 */ break; case 47: -#line 671 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_PRIVATE; } -#line 2204 "grammar.c" /* yacc.c:1663 */ +#line 732 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_PRIVATE; } +#line 2281 "grammar.c" /* yacc.c:1646 */ break; case 48: -#line 673 "grammar.y" /* yacc.c:1663 */ +#line 734 "grammar.y" /* yacc.c:1646 */ { - (yyval.modifier).flags = STRING_GFLAGS_XOR; + (yyval.modifier).flags = STRING_FLAGS_XOR; (yyval.modifier).xor_min = 0; (yyval.modifier).xor_max = 255; } -#line 2214 "grammar.c" /* yacc.c:1663 */ +#line 2291 "grammar.c" /* yacc.c:1646 */ break; case 49: -#line 679 "grammar.y" /* yacc.c:1663 */ +#line 740 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -2226,15 +2303,15 @@ yyreduce: fail_if_error(result); - (yyval.modifier).flags = STRING_GFLAGS_XOR; + (yyval.modifier).flags = STRING_FLAGS_XOR; (yyval.modifier).xor_min = (yyvsp[-1].integer); (yyval.modifier).xor_max = (yyvsp[-1].integer); } -#line 2234 "grammar.c" /* yacc.c:1663 */ +#line 2311 "grammar.c" /* yacc.c:1646 */ break; case 50: -#line 700 "grammar.y" /* yacc.c:1663 */ +#line 761 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -2261,75 +2338,153 @@ yyreduce: fail_if_error(result); - (yyval.modifier).flags = STRING_GFLAGS_XOR; + (yyval.modifier).flags = STRING_FLAGS_XOR; (yyval.modifier).xor_min = (yyvsp[-3].integer); (yyval.modifier).xor_max = (yyvsp[-1].integer); } -#line 2269 "grammar.c" /* yacc.c:1663 */ +#line 2346 "grammar.c" /* yacc.c:1646 */ break; case 51: -#line 733 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = 0; } -#line 2275 "grammar.c" /* yacc.c:1663 */ +#line 792 "grammar.y" /* yacc.c:1646 */ + { + (yyval.modifier).flags = STRING_FLAGS_BASE64; + (yyval.modifier).alphabet = sized_string_new(DEFAULT_BASE64_ALPHABET); + } +#line 2355 "grammar.c" /* yacc.c:1646 */ break; case 52: -#line 734 "grammar.y" /* yacc.c:1663 */ - { set_flag_or_error((yyval.modifier).flags, (yyvsp[0].modifier).flags); } -#line 2281 "grammar.c" /* yacc.c:1663 */ +#line 797 "grammar.y" /* yacc.c:1646 */ + { + int result = ERROR_SUCCESS; + + if ((yyvsp[-1].sized_string)->length != 64) + { + yr_free((yyvsp[-1].sized_string)); + result = yr_compiler_set_error_extra_info( + compiler, "length of base64 alphabet must be 64"); + result = ERROR_INVALID_MODIFIER; + } + + fail_if_error(result); + + (yyval.modifier).flags = STRING_FLAGS_BASE64; + (yyval.modifier).alphabet = (yyvsp[-1].sized_string); + } +#line 2376 "grammar.c" /* yacc.c:1646 */ break; case 53: -#line 738 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_WIDE; } -#line 2287 "grammar.c" /* yacc.c:1663 */ +#line 814 "grammar.y" /* yacc.c:1646 */ + { + (yyval.modifier).flags = STRING_FLAGS_BASE64_WIDE; + (yyval.modifier).alphabet = sized_string_new(DEFAULT_BASE64_ALPHABET); + } +#line 2385 "grammar.c" /* yacc.c:1646 */ break; case 54: -#line 739 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_ASCII; } -#line 2293 "grammar.c" /* yacc.c:1663 */ +#line 819 "grammar.y" /* yacc.c:1646 */ + { + int result = ERROR_SUCCESS; + + if ((yyvsp[-1].sized_string)->length != 64) + { + yr_free((yyvsp[-1].sized_string)); + result = yr_compiler_set_error_extra_info( + compiler, "length of base64 alphabet must be 64"); + result = ERROR_INVALID_MODIFIER; + } + + fail_if_error(result); + + (yyval.modifier).flags = STRING_FLAGS_BASE64_WIDE; + (yyval.modifier).alphabet = (yyvsp[-1].sized_string); + } +#line 2406 "grammar.c" /* yacc.c:1646 */ break; case 55: -#line 740 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_NO_CASE; } -#line 2299 "grammar.c" /* yacc.c:1663 */ +#line 838 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = 0; } +#line 2412 "grammar.c" /* yacc.c:1646 */ break; case 56: -#line 741 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_FULL_WORD; } -#line 2305 "grammar.c" /* yacc.c:1663 */ +#line 840 "grammar.y" /* yacc.c:1646 */ + { + if ((yyvsp[-1].modifier).flags & (yyvsp[0].modifier).flags) + { + fail_with_error(ERROR_DUPLICATED_MODIFIER); + } + else + { + (yyval.modifier).flags = (yyvsp[-1].modifier).flags | (yyvsp[0].modifier).flags; + } + } +#line 2427 "grammar.c" /* yacc.c:1646 */ break; case 57: -#line 742 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_PRIVATE; } -#line 2311 "grammar.c" /* yacc.c:1663 */ +#line 853 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_WIDE; } +#line 2433 "grammar.c" /* yacc.c:1646 */ break; case 58: -#line 746 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = 0; } -#line 2317 "grammar.c" /* yacc.c:1663 */ +#line 854 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_ASCII; } +#line 2439 "grammar.c" /* yacc.c:1646 */ break; case 59: -#line 747 "grammar.y" /* yacc.c:1663 */ - { set_flag_or_error((yyval.modifier).flags, (yyvsp[0].modifier).flags); } -#line 2323 "grammar.c" /* yacc.c:1663 */ +#line 855 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_NO_CASE; } +#line 2445 "grammar.c" /* yacc.c:1646 */ break; case 60: -#line 751 "grammar.y" /* yacc.c:1663 */ - { (yyval.modifier).flags = STRING_GFLAGS_PRIVATE; } -#line 2329 "grammar.c" /* yacc.c:1663 */ +#line 856 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_FULL_WORD; } +#line 2451 "grammar.c" /* yacc.c:1646 */ break; case 61: -#line 756 "grammar.y" /* yacc.c:1663 */ +#line 857 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_PRIVATE; } +#line 2457 "grammar.c" /* yacc.c:1646 */ + break; + + case 62: +#line 861 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = 0; } +#line 2463 "grammar.c" /* yacc.c:1646 */ + break; + + case 63: +#line 863 "grammar.y" /* yacc.c:1646 */ + { + if ((yyvsp[-1].modifier).flags & (yyvsp[0].modifier).flags) + { + fail_with_error(ERROR_DUPLICATED_MODIFIER); + } + else + { + (yyval.modifier).flags = (yyvsp[-1].modifier).flags | (yyvsp[0].modifier).flags; + } + } +#line 2478 "grammar.c" /* yacc.c:1646 */ + break; + + case 64: +#line 876 "grammar.y" /* yacc.c:1646 */ + { (yyval.modifier).flags = STRING_FLAGS_PRIVATE; } +#line 2484 "grammar.c" /* yacc.c:1646 */ + break; + + case 65: +#line 881 "grammar.y" /* yacc.c:1646 */ { YR_EXPRESSION expr; @@ -2358,53 +2513,59 @@ yyreduce: YR_OBJECT* object = (YR_OBJECT*) yr_hash_table_lookup( compiler->objects_table, (yyvsp[0].c_string), NULL); + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, + YR_NAMESPACES_TABLE, + compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE)); + if (object == NULL) { // If not found, search within the current namespace. - char* ns = compiler->current_namespace->name; - object = (YR_OBJECT*) yr_hash_table_lookup( - compiler->objects_table, (yyvsp[0].c_string), ns); + compiler->objects_table, (yyvsp[0].c_string), ns->name); } if (object != NULL) { - char* id; + YR_ARENA_REF ref; result = yr_arena_write_string( - compiler->sz_arena, (yyvsp[0].c_string), &id); + compiler->arena, YR_SZ_POOL, (yyvsp[0].c_string), &ref); if (result == ERROR_SUCCESS) result = yr_parser_emit_with_arg_reloc( yyscanner, OP_OBJ_LOAD, - id, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); (yyval.expression).type = EXPRESSION_TYPE_OBJECT; (yyval.expression).value.object = object; - (yyval.expression).identifier = object->identifier; + (yyval.expression).identifier.ptr = NULL; + (yyval.expression).identifier.ref = ref; } else { - YR_RULE* rule = (YR_RULE*) yr_hash_table_lookup( - compiler->rules_table, - (yyvsp[0].c_string), - compiler->current_namespace->name); + uint32_t rule_idx = yr_hash_table_lookup_uint32( + compiler->rules_table, (yyvsp[0].c_string), ns->name); - if (rule != NULL) + if (rule_idx != UINT32_MAX) { - result = yr_parser_emit_with_arg_reloc( + result = yr_parser_emit_with_arg( yyscanner, OP_PUSH_RULE, - rule, + rule_idx, NULL, NULL); + YR_RULE* rule = _yr_compiler_get_rule_by_idx(compiler, rule_idx); + + yr_arena_ptr_to_ref(compiler->arena, rule->identifier, &(yyval.expression).identifier.ref); + (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; - (yyval.expression).value.integer = UNDEFINED; - (yyval.expression).identifier = rule->identifier; + (yyval.expression).value.integer = YR_UNDEFINED; + (yyval.expression).identifier.ptr = NULL; } else { @@ -2418,11 +2579,11 @@ yyreduce: fail_if_error(result); } -#line 2422 "grammar.c" /* yacc.c:1663 */ +#line 2583 "grammar.c" /* yacc.c:1646 */ break; - case 62: -#line 845 "grammar.y" /* yacc.c:1663 */ + case 66: +#line 976 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; YR_OBJECT* field = NULL; @@ -2434,22 +2595,23 @@ yyreduce: if (field != NULL) { - char* ident; + YR_ARENA_REF ref; result = yr_arena_write_string( - compiler->sz_arena, (yyvsp[0].c_string), &ident); + compiler->arena, YR_SZ_POOL, (yyvsp[0].c_string), &ref); if (result == ERROR_SUCCESS) result = yr_parser_emit_with_arg_reloc( yyscanner, OP_OBJ_FIELD, - ident, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); (yyval.expression).type = EXPRESSION_TYPE_OBJECT; (yyval.expression).value.object = field; - (yyval.expression).identifier = field->identifier; + (yyval.expression).identifier.ref = ref; + (yyval.expression).identifier.ptr = NULL; } else { @@ -2460,7 +2622,7 @@ yyreduce: else { yr_compiler_set_error_extra_info( - compiler, (yyvsp[-2].expression).identifier); + compiler, expression_identifier((yyvsp[-2].expression))); result = ERROR_NOT_A_STRUCTURE; } @@ -2469,11 +2631,11 @@ yyreduce: fail_if_error(result); } -#line 2473 "grammar.c" /* yacc.c:1663 */ +#line 2635 "grammar.c" /* yacc.c:1646 */ break; - case 63: -#line 892 "grammar.y" /* yacc.c:1663 */ + case 67: +#line 1024 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; YR_OBJECT_ARRAY* array; @@ -2498,7 +2660,8 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_OBJECT; (yyval.expression).value.object = array->prototype_item; - (yyval.expression).identifier = array->identifier; + (yyval.expression).identifier.ptr = array->identifier; + (yyval.expression).identifier.ref = YR_ARENA_NULL_REF; } else if ((yyvsp[-3].expression).type == EXPRESSION_TYPE_OBJECT && (yyvsp[-3].expression).value.object->type == OBJECT_TYPE_DICTIONARY) @@ -2519,27 +2682,28 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_OBJECT; (yyval.expression).value.object = dict->prototype_item; - (yyval.expression).identifier = dict->identifier; + (yyval.expression).identifier.ptr = dict->identifier; + (yyval.expression).identifier.ref = YR_ARENA_NULL_REF; } else { yr_compiler_set_error_extra_info( - compiler, (yyvsp[-3].expression).identifier); + compiler, expression_identifier((yyvsp[-3].expression))); result = ERROR_NOT_INDEXABLE; } fail_if_error(result); } -#line 2535 "grammar.c" /* yacc.c:1663 */ +#line 2699 "grammar.c" /* yacc.c:1646 */ break; - case 64: -#line 951 "grammar.y" /* yacc.c:1663 */ + case 68: +#line 1085 "grammar.y" /* yacc.c:1646 */ { + YR_ARENA_REF ref; int result = ERROR_SUCCESS; YR_OBJECT_FUNCTION* function; - char* args_fmt; if ((yyvsp[-3].expression).type == EXPRESSION_TYPE_OBJECT && (yyvsp[-3].expression).value.object->type == OBJECT_TYPE_FUNCTION) @@ -2549,13 +2713,13 @@ yyreduce: if (result == ERROR_SUCCESS) result = yr_arena_write_string( - compiler->sz_arena, (yyvsp[-1].c_string), &args_fmt); + compiler->arena, YR_SZ_POOL, (yyvsp[-1].c_string), &ref); if (result == ERROR_SUCCESS) result = yr_parser_emit_with_arg_reloc( yyscanner, OP_CALL, - args_fmt, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); @@ -2563,12 +2727,13 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_OBJECT; (yyval.expression).value.object = function->return_obj; - (yyval.expression).identifier = function->identifier; + (yyval.expression).identifier.ref = ref; + (yyval.expression).identifier.ptr = NULL; } else { yr_compiler_set_error_extra_info( - compiler, (yyvsp[-3].expression).identifier); + compiler, expression_identifier((yyvsp[-3].expression))); result = ERROR_NOT_A_FUNCTION; } @@ -2577,28 +2742,28 @@ yyreduce: fail_if_error(result); } -#line 2581 "grammar.c" /* yacc.c:1663 */ +#line 2746 "grammar.c" /* yacc.c:1646 */ break; - case 65: -#line 996 "grammar.y" /* yacc.c:1663 */ + case 69: +#line 1131 "grammar.y" /* yacc.c:1646 */ { (yyval.c_string) = yr_strdup(""); } -#line 2587 "grammar.c" /* yacc.c:1663 */ +#line 2752 "grammar.c" /* yacc.c:1646 */ break; - case 66: -#line 997 "grammar.y" /* yacc.c:1663 */ + case 70: +#line 1132 "grammar.y" /* yacc.c:1646 */ { (yyval.c_string) = (yyvsp[0].c_string); } -#line 2593 "grammar.c" /* yacc.c:1663 */ +#line 2758 "grammar.c" /* yacc.c:1646 */ break; - case 67: -#line 1002 "grammar.y" /* yacc.c:1663 */ + case 71: +#line 1137 "grammar.y" /* yacc.c:1646 */ { (yyval.c_string) = (char*) yr_malloc(YR_MAX_FUNCTION_ARGS + 1); if ((yyval.c_string) == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); + fail_with_error(ERROR_INSUFFICIENT_MEMORY); switch((yyvsp[0].expression).type) { @@ -2621,18 +2786,18 @@ yyreduce: yr_free((yyval.c_string)); yr_compiler_set_error_extra_info( compiler, "unknown type for argument 1 in function call"); - fail_if_error(ERROR_WRONG_TYPE); + fail_with_error(ERROR_WRONG_TYPE); break; default: // An unknown expression type is OK iff an error ocurred. assert(compiler->last_error != ERROR_SUCCESS); } } -#line 2632 "grammar.c" /* yacc.c:1663 */ +#line 2797 "grammar.c" /* yacc.c:1646 */ break; - case 68: -#line 1037 "grammar.y" /* yacc.c:1663 */ + case 72: +#line 1172 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -2662,7 +2827,7 @@ yyreduce: case EXPRESSION_TYPE_UNKNOWN: result = ERROR_WRONG_TYPE; yr_compiler_set_error_extra_info_fmt( - compiler, "unknown type for argument %lu in function call", + compiler, "unknown type for argument %zu in function call", // As we add one character per argument, the length of $1 is // the number of arguments parsed so far, and the argument // represented by is length of $1 plus one. @@ -2681,14 +2846,14 @@ yyreduce: (yyval.c_string) = (yyvsp[-2].c_string); } -#line 2685 "grammar.c" /* yacc.c:1663 */ +#line 2850 "grammar.c" /* yacc.c:1646 */ break; - case 69: -#line 1090 "grammar.y" /* yacc.c:1663 */ + case 73: +#line 1225 "grammar.y" /* yacc.c:1646 */ { SIZED_STRING* sized_string = (yyvsp[0].sized_string); - RE* re; + YR_ARENA_REF re_ref; RE_ERROR error; int result = ERROR_SUCCESS; @@ -2703,8 +2868,8 @@ yyreduce: result = yr_re_compile( sized_string->c_string, re_flags, - compiler->re_code_arena, - &re, + compiler->arena, + &re_ref, &error); yr_free((yyvsp[0].sized_string)); @@ -2716,7 +2881,7 @@ yyreduce: result = yr_parser_emit_with_arg_reloc( yyscanner, OP_PUSH, - re, + yr_arena_ref_to_ptr(compiler->arena, &re_ref), NULL, NULL); @@ -2724,11 +2889,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_REGEXP; } -#line 2728 "grammar.c" /* yacc.c:1663 */ +#line 2893 "grammar.c" /* yacc.c:1646 */ break; - case 70: -#line 1133 "grammar.y" /* yacc.c:1663 */ + case 74: +#line 1268 "grammar.y" /* yacc.c:1646 */ { if ((yyvsp[0].expression).type == EXPRESSION_TYPE_STRING) { @@ -2745,33 +2910,33 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2749 "grammar.c" /* yacc.c:1663 */ +#line 2914 "grammar.c" /* yacc.c:1646 */ break; - case 71: -#line 1153 "grammar.y" /* yacc.c:1663 */ + case 75: +#line 1288 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH, 1, NULL, NULL)); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2760 "grammar.c" /* yacc.c:1663 */ +#line 2925 "grammar.c" /* yacc.c:1646 */ break; - case 72: -#line 1160 "grammar.y" /* yacc.c:1663 */ + case 76: +#line 1295 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH, 0, NULL, NULL)); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2771 "grammar.c" /* yacc.c:1663 */ +#line 2936 "grammar.c" /* yacc.c:1646 */ break; - case 73: -#line 1167 "grammar.y" /* yacc.c:1663 */ + case 77: +#line 1302 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_STRING, "matches"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_REGEXP, "matches"); @@ -2783,11 +2948,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2787 "grammar.c" /* yacc.c:1663 */ +#line 2952 "grammar.c" /* yacc.c:1646 */ break; - case 74: -#line 1179 "grammar.y" /* yacc.c:1663 */ + case 78: +#line 1314 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_STRING, "contains"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_STRING, "contains"); @@ -2797,17 +2962,17 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2801 "grammar.c" /* yacc.c:1663 */ +#line 2966 "grammar.c" /* yacc.c:1646 */ break; - case 75: -#line 1189 "grammar.y" /* yacc.c:1663 */ + case 79: +#line 1324 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_string_identifier( yyscanner, (yyvsp[0].c_string), OP_FOUND, - UNDEFINED); + YR_UNDEFINED); yr_free((yyvsp[0].c_string)); @@ -2815,11 +2980,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2819 "grammar.c" /* yacc.c:1663 */ +#line 2984 "grammar.c" /* yacc.c:1646 */ break; - case 76: -#line 1203 "grammar.y" /* yacc.c:1663 */ + case 80: +#line 1338 "grammar.y" /* yacc.c:1646 */ { int result; @@ -2834,14 +2999,14 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2838 "grammar.c" /* yacc.c:1663 */ +#line 3003 "grammar.c" /* yacc.c:1646 */ break; - case 77: -#line 1218 "grammar.y" /* yacc.c:1663 */ + case 81: +#line 1353 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_string_identifier( - yyscanner, (yyvsp[-2].c_string), OP_FOUND_IN, UNDEFINED); + yyscanner, (yyvsp[-2].c_string), OP_FOUND_IN, YR_UNDEFINED); yr_free((yyvsp[-2].c_string)); @@ -2849,12 +3014,14 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 2853 "grammar.c" /* yacc.c:1663 */ +#line 3018 "grammar.c" /* yacc.c:1646 */ break; - case 78: -#line 1229 "grammar.y" /* yacc.c:1663 */ + case 82: +#line 1364 "grammar.y" /* yacc.c:1646 */ { + int i; + // Free all the loop variable identifiers, including the variables for // the current loop (represented by loop_index), and set loop_index to // -1. This is OK even if we have nested loops. If an error occurs while @@ -2862,7 +3029,7 @@ yyreduce: // anyways, so it's safe to do this cleanup while processing the error // for the inner loop. - for (int i = 0; i <= compiler->loop_index; i++) + for (i = 0; i <= compiler->loop_index; i++) { loop_vars_cleanup(i); } @@ -2870,11 +3037,11 @@ yyreduce: compiler->loop_index = -1; YYERROR; } -#line 2874 "grammar.c" /* yacc.c:1663 */ +#line 3041 "grammar.c" /* yacc.c:1646 */ break; - case 79: -#line 1297 "grammar.y" /* yacc.c:1663 */ + case 83: +#line 1434 "grammar.y" /* yacc.c:1646 */ { // var_frame is used for accessing local variables used in this loop. // All local variables are accessed using var_frame as a reference, @@ -2892,9 +3059,10 @@ yyreduce: compiler->loop_index++; - // This loop uses 3 internal variables besides the ones explicitly + // This loop uses internal variables besides the ones explicitly // defined by the user. - compiler->loop[compiler->loop_index].vars_internal_count = 3; + compiler->loop[compiler->loop_index].vars_internal_count = \ + YR_INTERNAL_LOOP_VARS; // Initialize the number of variables, this number will be incremented // as variable declaration are processed by for_variables. @@ -2911,62 +3079,69 @@ yyreduce: fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_POP_M, var_frame + 2, NULL, NULL)); } -#line 2915 "grammar.c" /* yacc.c:1663 */ +#line 3083 "grammar.c" /* yacc.c:1646 */ break; - case 80: -#line 1334 "grammar.y" /* yacc.c:1663 */ + case 84: +#line 1472 "grammar.y" /* yacc.c:1646 */ { YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[compiler->loop_index]; YR_FIXUP* fixup; - uint8_t* loop_start_addr; - void* jmp_arg_addr; + YR_ARENA_REF loop_start_ref; + YR_ARENA_REF jmp_offset_ref; int var_frame = _yr_compiler_get_var_frame(compiler); + int i; fail_if_error(yr_parser_emit( - yyscanner, OP_ITER_NEXT, &loop_start_addr)); + yyscanner, OP_ITER_NEXT, &loop_start_ref)); // For each variable generate an instruction that pops the value from - // the stack and store it into one memory slot starting at var_frame + 3 - // because the first 3 slots in the frame are for the internal variables. + // the stack and store it into one memory slot starting at var_frame + + // YR_INTERNAL_LOOP_VARS because the first YR_INTERNAL_LOOP_VARS slots + // in the frame are for the internal variables. - for (int i = 0; i < loop_ctx->vars_count; i++) + for (i = 0; i < loop_ctx->vars_count; i++) { fail_if_error(yr_parser_emit_with_arg( - yyscanner, OP_POP_M, var_frame + 3 + i, NULL, NULL)); + yyscanner, + OP_POP_M, + var_frame + YR_INTERNAL_LOOP_VARS + i, + NULL, + NULL)); } - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE_P, - 0, + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_arg_addr)); + &jmp_offset_ref)); - // Push a new fixup entry in the fixup stack so that the jump - // destination is set once we know it. + // We still don't know the jump's target, so we push a fixup entry + // in the stack, so that the jump's offset can be set once we know it. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); + fail_with_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_arg_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; - loop_ctx->addr = loop_start_addr; + loop_ctx->start_ref = loop_start_ref; } -#line 2963 "grammar.c" /* yacc.c:1663 */ +#line 3137 "grammar.c" /* yacc.c:1646 */ break; - case 81: -#line 1378 "grammar.y" /* yacc.c:1663 */ + case 85: +#line 1522 "grammar.y" /* yacc.c:1646 */ { + int32_t jmp_offset; YR_FIXUP* fixup; - uint8_t* pop_addr; + YR_ARENA_REF pop_ref; int var_frame = _yr_compiler_get_var_frame(compiler); @@ -2979,10 +3154,14 @@ yyreduce: fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JUNDEF_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); @@ -2992,24 +3171,38 @@ yyreduce: fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JL_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); fail_if_error(yr_parser_emit( - yyscanner, OP_POP, &pop_addr)); + yyscanner, OP_POP, &pop_ref)); - // Pop from the stack the fixup entry containing the jump's address - // that needs to be fixed. + // Pop from the stack the fixup entry containing the reference to + // the jump offset that needs to be fixed. fixup = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup->next; - // Fix the jump's target address. - *(void**)(fixup->address) = (void*)(pop_addr); + // The fixup entry has a reference to the jump offset that need + // to be fixed, convert the address into a pointer. + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + // The reference in the fixup entry points to the jump's offset + // but the jump instruction is one byte before, that's why we add + // one to the offset. + jmp_offset = pop_ref.offset - fixup->ref.offset + 1; + + // Fix the jump's offset. + *jmp_offset_addr = jmp_offset; yr_free(fixup); @@ -3031,15 +3224,16 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3035 "grammar.c" /* yacc.c:1663 */ +#line 3228 "grammar.c" /* yacc.c:1646 */ break; - case 82: -#line 1446 "grammar.y" /* yacc.c:1663 */ + case 86: +#line 1609 "grammar.y" /* yacc.c:1646 */ { + YR_ARENA_REF ref; + int result = ERROR_SUCCESS; int var_frame; - uint8_t* addr; if (compiler->loop_index + 1 == YR_MAX_LOOP_NESTING) result = ERROR_LOOP_NESTING_LIMIT_EXCEEDED; @@ -3061,18 +3255,19 @@ yyreduce: // Pop the first string. yr_parser_emit_with_arg( - yyscanner, OP_POP_M, var_frame, &addr, NULL); + yyscanner, OP_POP_M, var_frame, &ref, NULL); compiler->loop_for_of_var_index = var_frame; - compiler->loop[compiler->loop_index].vars_internal_count = 3; + compiler->loop[compiler->loop_index].start_ref = ref; compiler->loop[compiler->loop_index].vars_count = 0; - compiler->loop[compiler->loop_index].addr = addr; + compiler->loop[compiler->loop_index].vars_internal_count = \ + YR_INTERNAL_LOOP_VARS; } -#line 3072 "grammar.c" /* yacc.c:1663 */ +#line 3267 "grammar.c" /* yacc.c:1646 */ break; - case 83: -#line 1479 "grammar.y" /* yacc.c:1663 */ + case 87: +#line 1644 "grammar.y" /* yacc.c:1646 */ { int var_frame = 0; @@ -3082,7 +3277,7 @@ yyreduce: // Increment counter by the value returned by the // boolean expression (0 or 1). If the boolean expression - // returned UNDEFINED the OP_ADD_M won't do anything. + // returned YR_UNDEFINED the OP_ADD_M won't do anything. yr_parser_emit_with_arg( yyscanner, OP_ADD_M, var_frame + 1, NULL, NULL); @@ -3091,12 +3286,16 @@ yyreduce: yr_parser_emit_with_arg( yyscanner, OP_INCR_M, var_frame + 2, NULL, NULL); + int32_t jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION); + // If next string is not undefined, go back to the // beginning of the loop. - yr_parser_emit_with_arg_reloc( + yr_parser_emit_with_arg_int32( yyscanner, OP_JNUNDEF, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL); @@ -3123,218 +3322,218 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3127 "grammar.c" /* yacc.c:1663 */ +#line 3326 "grammar.c" /* yacc.c:1646 */ break; - case 84: -#line 1530 "grammar.y" /* yacc.c:1663 */ + case 88: +#line 1699 "grammar.y" /* yacc.c:1646 */ { yr_parser_emit(yyscanner, OP_OF, NULL); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3137 "grammar.c" /* yacc.c:1663 */ +#line 3336 "grammar.c" /* yacc.c:1646 */ break; - case 85: -#line 1536 "grammar.y" /* yacc.c:1663 */ + case 89: +#line 1705 "grammar.y" /* yacc.c:1646 */ { yr_parser_emit(yyscanner, OP_NOT, NULL); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3147 "grammar.c" /* yacc.c:1663 */ - break; - - case 86: -#line 1542 "grammar.y" /* yacc.c:1663 */ - { - YR_FIXUP* fixup; - void* jmp_destination_addr; - - fail_if_error(yr_parser_emit_with_arg_reloc( - yyscanner, - OP_JFALSE, - 0, // still don't know the jump destination - NULL, - &jmp_destination_addr)); - - // create a fixup entry for the jump and push it in the stack - fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); - - if (fixup == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); - - fixup->address = jmp_destination_addr; - fixup->next = compiler->fixup_stack_head; - compiler->fixup_stack_head = fixup; - } -#line 3173 "grammar.c" /* yacc.c:1663 */ - break; - - case 87: -#line 1564 "grammar.y" /* yacc.c:1663 */ - { - YR_FIXUP* fixup; - uint8_t* nop_addr; - - fail_if_error(yr_parser_emit(yyscanner, OP_AND, NULL)); - - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_AND instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*) nop_addr; - compiler->fixup_stack_head = fixup->next; - yr_free(fixup); - - (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; - } -#line 3200 "grammar.c" /* yacc.c:1663 */ - break; - - case 88: -#line 1587 "grammar.y" /* yacc.c:1663 */ - { - YR_FIXUP* fixup; - void* jmp_destination_addr; - - fail_if_error(yr_parser_emit_with_arg_reloc( - yyscanner, - OP_JTRUE, - 0, // still don't know the jump destination - NULL, - &jmp_destination_addr)); - - fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); - - if (fixup == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); - - fixup->address = jmp_destination_addr; - fixup->next = compiler->fixup_stack_head; - compiler->fixup_stack_head = fixup; - } -#line 3225 "grammar.c" /* yacc.c:1663 */ - break; - - case 89: -#line 1608 "grammar.y" /* yacc.c:1663 */ - { - YR_FIXUP* fixup; - uint8_t* nop_addr; - - fail_if_error(yr_parser_emit(yyscanner, OP_OR, NULL)); - - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_OR instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*)(nop_addr); - compiler->fixup_stack_head = fixup->next; - yr_free(fixup); - - (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; - } -#line 3252 "grammar.c" /* yacc.c:1663 */ +#line 3346 "grammar.c" /* yacc.c:1646 */ break; case 90: -#line 1631 "grammar.y" /* yacc.c:1663 */ +#line 1711 "grammar.y" /* yacc.c:1646 */ + { + YR_FIXUP* fixup; + YR_ARENA_REF jmp_offset_ref; + + fail_if_error(yr_parser_emit_with_arg_int32( + yyscanner, + OP_JFALSE, + 0, // still don't know the jump offset, use 0 for now. + NULL, + &jmp_offset_ref)); + + // Create a fixup entry for the jump and push it in the stack. + fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); + + if (fixup == NULL) + fail_with_error(ERROR_INSUFFICIENT_MEMORY); + + fixup->ref = jmp_offset_ref; + fixup->next = compiler->fixup_stack_head; + compiler->fixup_stack_head = fixup; + } +#line 3372 "grammar.c" /* yacc.c:1646 */ + break; + + case 91: +#line 1733 "grammar.y" /* yacc.c:1646 */ + { + YR_FIXUP* fixup; + + fail_if_error(yr_parser_emit(yyscanner, OP_AND, NULL)); + + fixup = compiler->fixup_stack_head; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + int32_t jmp_offset = \ + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. + compiler->fixup_stack_head = fixup->next; + yr_free(fixup); + + (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; + } +#line 3399 "grammar.c" /* yacc.c:1646 */ + break; + + case 92: +#line 1756 "grammar.y" /* yacc.c:1646 */ + { + YR_FIXUP* fixup; + YR_ARENA_REF jmp_offset_ref; + + fail_if_error(yr_parser_emit_with_arg_int32( + yyscanner, + OP_JTRUE, + 0, // still don't know the jump destination, use 0 for now. + NULL, + &jmp_offset_ref)); + + fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); + + if (fixup == NULL) + fail_with_error(ERROR_INSUFFICIENT_MEMORY); + + fixup->ref = jmp_offset_ref; + fixup->next = compiler->fixup_stack_head; + compiler->fixup_stack_head = fixup; + } +#line 3424 "grammar.c" /* yacc.c:1646 */ + break; + + case 93: +#line 1777 "grammar.y" /* yacc.c:1646 */ + { + YR_FIXUP* fixup; + + fail_if_error(yr_parser_emit(yyscanner, OP_OR, NULL)); + + fixup = compiler->fixup_stack_head; + + int32_t jmp_offset = \ + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. + compiler->fixup_stack_head = fixup->next; + yr_free(fixup); + + (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; + } +#line 3451 "grammar.c" /* yacc.c:1646 */ + break; + + case 94: +#line 1800 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "<", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3263 "grammar.c" /* yacc.c:1663 */ +#line 3462 "grammar.c" /* yacc.c:1646 */ break; - case 91: -#line 1638 "grammar.y" /* yacc.c:1663 */ + case 95: +#line 1807 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, ">", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3274 "grammar.c" /* yacc.c:1663 */ +#line 3473 "grammar.c" /* yacc.c:1646 */ break; - case 92: -#line 1645 "grammar.y" /* yacc.c:1663 */ + case 96: +#line 1814 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "<=", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3285 "grammar.c" /* yacc.c:1663 */ +#line 3484 "grammar.c" /* yacc.c:1646 */ break; - case 93: -#line 1652 "grammar.y" /* yacc.c:1663 */ + case 97: +#line 1821 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, ">=", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3296 "grammar.c" /* yacc.c:1663 */ +#line 3495 "grammar.c" /* yacc.c:1646 */ break; - case 94: -#line 1659 "grammar.y" /* yacc.c:1663 */ + case 98: +#line 1828 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "==", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3307 "grammar.c" /* yacc.c:1663 */ +#line 3506 "grammar.c" /* yacc.c:1646 */ break; - case 95: -#line 1666 "grammar.y" /* yacc.c:1663 */ + case 99: +#line 1835 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_reduce_operation( yyscanner, "!=", (yyvsp[-2].expression), (yyvsp[0].expression))); (yyval.expression).type = EXPRESSION_TYPE_BOOLEAN; } -#line 3318 "grammar.c" /* yacc.c:1663 */ +#line 3517 "grammar.c" /* yacc.c:1646 */ break; - case 96: -#line 1673 "grammar.y" /* yacc.c:1663 */ + case 100: +#line 1842 "grammar.y" /* yacc.c:1646 */ { (yyval.expression) = (yyvsp[0].expression); } -#line 3326 "grammar.c" /* yacc.c:1663 */ +#line 3525 "grammar.c" /* yacc.c:1646 */ break; - case 97: -#line 1677 "grammar.y" /* yacc.c:1663 */ + case 101: +#line 1846 "grammar.y" /* yacc.c:1646 */ { (yyval.expression) = (yyvsp[-1].expression); } -#line 3334 "grammar.c" /* yacc.c:1663 */ +#line 3533 "grammar.c" /* yacc.c:1646 */ break; - case 98: -#line 1685 "grammar.y" /* yacc.c:1663 */ + case 102: +#line 1854 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3350,15 +3549,15 @@ yyreduce: fail_if_error(result); - loop_ctx->vars[loop_ctx->vars_count++].identifier = (yyvsp[0].c_string); + loop_ctx->vars[loop_ctx->vars_count++].identifier.ptr = (yyvsp[0].c_string); assert(loop_ctx->vars_count <= YR_MAX_LOOP_VARS); } -#line 3358 "grammar.c" /* yacc.c:1663 */ +#line 3557 "grammar.c" /* yacc.c:1646 */ break; - case 99: -#line 1705 "grammar.y" /* yacc.c:1663 */ + case 103: +#line 1874 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3381,13 +3580,13 @@ yyreduce: fail_if_error(result); - loop_ctx->vars[loop_ctx->vars_count++].identifier = (yyvsp[0].c_string); + loop_ctx->vars[loop_ctx->vars_count++].identifier.ptr = (yyvsp[0].c_string); } -#line 3387 "grammar.c" /* yacc.c:1663 */ +#line 3586 "grammar.c" /* yacc.c:1646 */ break; - case 100: -#line 1733 "grammar.y" /* yacc.c:1663 */ + case 104: +#line 1902 "grammar.y" /* yacc.c:1646 */ { YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[compiler->loop_index]; @@ -3417,7 +3616,7 @@ yyreduce: compiler, "iterator for \"%s\" yields a single item on each iteration" ", but the loop expects %d", - (yyvsp[0].expression).identifier, + expression_identifier((yyvsp[0].expression)), loop_ctx->vars_count); result = ERROR_SYNTAX_ERROR; @@ -3443,7 +3642,7 @@ yyreduce: yr_compiler_set_error_extra_info_fmt( compiler, "iterator for \"%s\" yields a key,value pair item on each iteration", - (yyvsp[0].expression).identifier); + expression_identifier((yyvsp[0].expression))); result = ERROR_SYNTAX_ERROR; } @@ -3456,16 +3655,16 @@ yyreduce: yr_compiler_set_error_extra_info_fmt( compiler, "identifier \"%s\" is not iterable", - (yyvsp[0].expression).identifier); + expression_identifier((yyvsp[0].expression))); } fail_if_error(result); } -#line 3465 "grammar.c" /* yacc.c:1663 */ +#line 3664 "grammar.c" /* yacc.c:1646 */ break; - case 101: -#line 1807 "grammar.y" /* yacc.c:1663 */ + case 105: +#line 1976 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3474,7 +3673,7 @@ yyreduce: if (loop_ctx->vars_count == 1) { loop_ctx->vars[0].type = EXPRESSION_TYPE_INTEGER; - loop_ctx->vars[0].value.integer = UNDEFINED; + loop_ctx->vars[0].value.integer = YR_UNDEFINED; } else { @@ -3489,11 +3688,11 @@ yyreduce: fail_if_error(result); } -#line 3493 "grammar.c" /* yacc.c:1663 */ +#line 3692 "grammar.c" /* yacc.c:1646 */ break; - case 102: -#line 1835 "grammar.y" /* yacc.c:1663 */ + case 106: +#line 2004 "grammar.y" /* yacc.c:1646 */ { // $2 contains the number of integers in the enumeration fail_if_error(yr_parser_emit_with_arg( @@ -3502,20 +3701,20 @@ yyreduce: fail_if_error(yr_parser_emit( yyscanner, OP_ITER_START_INT_ENUM, NULL)); } -#line 3506 "grammar.c" /* yacc.c:1663 */ +#line 3705 "grammar.c" /* yacc.c:1646 */ break; - case 103: -#line 1844 "grammar.y" /* yacc.c:1663 */ + case 107: +#line 2013 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit( yyscanner, OP_ITER_START_INT_RANGE, NULL)); } -#line 3515 "grammar.c" /* yacc.c:1663 */ +#line 3714 "grammar.c" /* yacc.c:1646 */ break; - case 104: -#line 1853 "grammar.y" /* yacc.c:1663 */ + case 108: +#line 2022 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3535,11 +3734,11 @@ yyreduce: fail_if_error(result); } -#line 3539 "grammar.c" /* yacc.c:1663 */ +#line 3738 "grammar.c" /* yacc.c:1646 */ break; - case 105: -#line 1877 "grammar.y" /* yacc.c:1663 */ + case 109: +#line 2046 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3554,11 +3753,11 @@ yyreduce: (yyval.integer) = 1; } -#line 3558 "grammar.c" /* yacc.c:1663 */ +#line 3757 "grammar.c" /* yacc.c:1646 */ break; - case 106: -#line 1892 "grammar.y" /* yacc.c:1663 */ + case 110: +#line 2061 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3573,100 +3772,100 @@ yyreduce: (yyval.integer) = (yyvsp[-2].integer) + 1; } -#line 3577 "grammar.c" /* yacc.c:1663 */ +#line 3776 "grammar.c" /* yacc.c:1646 */ break; - case 107: -#line 1911 "grammar.y" /* yacc.c:1663 */ + case 111: +#line 2080 "grammar.y" /* yacc.c:1646 */ { // Push end-of-list marker - yr_parser_emit_with_arg(yyscanner, OP_PUSH, UNDEFINED, NULL, NULL); + yr_parser_emit_with_arg(yyscanner, OP_PUSH, YR_UNDEFINED, NULL, NULL); } -#line 3586 "grammar.c" /* yacc.c:1663 */ +#line 3785 "grammar.c" /* yacc.c:1646 */ break; - case 109: -#line 1917 "grammar.y" /* yacc.c:1663 */ + case 113: +#line 2086 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit_with_arg( - yyscanner, OP_PUSH, UNDEFINED, NULL, NULL)); + yyscanner, OP_PUSH, YR_UNDEFINED, NULL, NULL)); fail_if_error(yr_parser_emit_pushes_for_strings( yyscanner, "$*")); } -#line 3598 "grammar.c" /* yacc.c:1663 */ - break; - - case 112: -#line 1935 "grammar.y" /* yacc.c:1663 */ - { - int result = yr_parser_emit_pushes_for_strings(yyscanner, (yyvsp[0].c_string)); - yr_free((yyvsp[0].c_string)); - - fail_if_error(result); - } -#line 3609 "grammar.c" /* yacc.c:1663 */ - break; - - case 113: -#line 1942 "grammar.y" /* yacc.c:1663 */ - { - int result = yr_parser_emit_pushes_for_strings(yyscanner, (yyvsp[0].c_string)); - yr_free((yyvsp[0].c_string)); - - fail_if_error(result); - } -#line 3620 "grammar.c" /* yacc.c:1663 */ - break; - - case 114: -#line 1953 "grammar.y" /* yacc.c:1663 */ - { - (yyval.integer) = FOR_EXPRESSION_ANY; - } -#line 3628 "grammar.c" /* yacc.c:1663 */ - break; - - case 115: -#line 1957 "grammar.y" /* yacc.c:1663 */ - { - yr_parser_emit_with_arg(yyscanner, OP_PUSH, UNDEFINED, NULL, NULL); - (yyval.integer) = FOR_EXPRESSION_ALL; - } -#line 3637 "grammar.c" /* yacc.c:1663 */ +#line 3797 "grammar.c" /* yacc.c:1646 */ break; case 116: -#line 1962 "grammar.y" /* yacc.c:1663 */ +#line 2104 "grammar.y" /* yacc.c:1646 */ + { + int result = yr_parser_emit_pushes_for_strings(yyscanner, (yyvsp[0].c_string)); + yr_free((yyvsp[0].c_string)); + + fail_if_error(result); + } +#line 3808 "grammar.c" /* yacc.c:1646 */ + break; + + case 117: +#line 2111 "grammar.y" /* yacc.c:1646 */ + { + int result = yr_parser_emit_pushes_for_strings(yyscanner, (yyvsp[0].c_string)); + yr_free((yyvsp[0].c_string)); + + fail_if_error(result); + } +#line 3819 "grammar.c" /* yacc.c:1646 */ + break; + + case 118: +#line 2122 "grammar.y" /* yacc.c:1646 */ + { + (yyval.integer) = FOR_EXPRESSION_ANY; + } +#line 3827 "grammar.c" /* yacc.c:1646 */ + break; + + case 119: +#line 2126 "grammar.y" /* yacc.c:1646 */ + { + yr_parser_emit_with_arg(yyscanner, OP_PUSH, YR_UNDEFINED, NULL, NULL); + (yyval.integer) = FOR_EXPRESSION_ALL; + } +#line 3836 "grammar.c" /* yacc.c:1646 */ + break; + + case 120: +#line 2131 "grammar.y" /* yacc.c:1646 */ { yr_parser_emit_with_arg(yyscanner, OP_PUSH, 1, NULL, NULL); (yyval.integer) = FOR_EXPRESSION_ANY; } -#line 3646 "grammar.c" /* yacc.c:1663 */ +#line 3845 "grammar.c" /* yacc.c:1646 */ break; - case 117: -#line 1971 "grammar.y" /* yacc.c:1663 */ + case 121: +#line 2140 "grammar.y" /* yacc.c:1646 */ { (yyval.expression) = (yyvsp[-1].expression); } -#line 3654 "grammar.c" /* yacc.c:1663 */ +#line 3853 "grammar.c" /* yacc.c:1646 */ break; - case 118: -#line 1975 "grammar.y" /* yacc.c:1663 */ + case 122: +#line 2144 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit( yyscanner, OP_FILESIZE, NULL)); (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; + (yyval.expression).value.integer = YR_UNDEFINED; } -#line 3666 "grammar.c" /* yacc.c:1663 */ +#line 3865 "grammar.c" /* yacc.c:1646 */ break; - case 119: -#line 1983 "grammar.y" /* yacc.c:1663 */ + case 123: +#line 2152 "grammar.y" /* yacc.c:1646 */ { yywarning(yyscanner, "Using deprecated \"entrypoint\" keyword. Use the \"entry_point\" " @@ -3676,13 +3875,13 @@ yyreduce: yyscanner, OP_ENTRYPOINT, NULL)); (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; + (yyval.expression).value.integer = YR_UNDEFINED; } -#line 3682 "grammar.c" /* yacc.c:1663 */ +#line 3881 "grammar.c" /* yacc.c:1646 */ break; - case 120: -#line 1995 "grammar.y" /* yacc.c:1663 */ + case 124: +#line 2164 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-1].expression), EXPRESSION_TYPE_INTEGER, "intXXXX or uintXXXX"); @@ -3694,13 +3893,13 @@ yyreduce: yyscanner, (uint8_t) (OP_READ_INT + (yyvsp[-3].integer)), NULL)); (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; + (yyval.expression).value.integer = YR_UNDEFINED; } -#line 3700 "grammar.c" /* yacc.c:1663 */ +#line 3899 "grammar.c" /* yacc.c:1646 */ break; - case 121: -#line 2009 "grammar.y" /* yacc.c:1663 */ + case 125: +#line 2178 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH, (yyvsp[0].integer), NULL, NULL)); @@ -3708,30 +3907,31 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = (yyvsp[0].integer); } -#line 3712 "grammar.c" /* yacc.c:1663 */ +#line 3911 "grammar.c" /* yacc.c:1646 */ break; - case 122: -#line 2017 "grammar.y" /* yacc.c:1663 */ + case 126: +#line 2186 "grammar.y" /* yacc.c:1646 */ { fail_if_error(yr_parser_emit_with_arg_double( yyscanner, OP_PUSH, (yyvsp[0].double_), NULL, NULL)); (yyval.expression).type = EXPRESSION_TYPE_FLOAT; } -#line 3723 "grammar.c" /* yacc.c:1663 */ +#line 3922 "grammar.c" /* yacc.c:1646 */ break; - case 123: -#line 2024 "grammar.y" /* yacc.c:1663 */ + case 127: +#line 2193 "grammar.y" /* yacc.c:1646 */ { - SIZED_STRING* sized_string; + YR_ARENA_REF ref; int result = yr_arena_write_data( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, (yyvsp[0].sized_string), (yyvsp[0].sized_string)->length + sizeof(SIZED_STRING), - (void**) &sized_string); + &ref); yr_free((yyvsp[0].sized_string)); @@ -3739,108 +3939,109 @@ yyreduce: result = yr_parser_emit_with_arg_reloc( yyscanner, OP_PUSH, - sized_string, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); fail_if_error(result); (yyval.expression).type = EXPRESSION_TYPE_STRING; - (yyval.expression).value.sized_string = sized_string; + //TODO + // $$.value.sized_string = sized_string; } -#line 3752 "grammar.c" /* yacc.c:1663 */ - break; - - case 124: -#line 2049 "grammar.y" /* yacc.c:1663 */ - { - int result = yr_parser_reduce_string_identifier( - yyscanner, (yyvsp[0].c_string), OP_COUNT, UNDEFINED); - - yr_free((yyvsp[0].c_string)); - - fail_if_error(result); - - (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; - } -#line 3768 "grammar.c" /* yacc.c:1663 */ - break; - - case 125: -#line 2061 "grammar.y" /* yacc.c:1663 */ - { - int result = yr_parser_reduce_string_identifier( - yyscanner, (yyvsp[-3].c_string), OP_OFFSET, UNDEFINED); - - yr_free((yyvsp[-3].c_string)); - - fail_if_error(result); - - (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; - } -#line 3784 "grammar.c" /* yacc.c:1663 */ - break; - - case 126: -#line 2073 "grammar.y" /* yacc.c:1663 */ - { - int result = yr_parser_emit_with_arg( - yyscanner, OP_PUSH, 1, NULL, NULL); - - if (result == ERROR_SUCCESS) - result = yr_parser_reduce_string_identifier( - yyscanner, (yyvsp[0].c_string), OP_OFFSET, UNDEFINED); - - yr_free((yyvsp[0].c_string)); - - fail_if_error(result); - - (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; - } -#line 3804 "grammar.c" /* yacc.c:1663 */ - break; - - case 127: -#line 2089 "grammar.y" /* yacc.c:1663 */ - { - int result = yr_parser_reduce_string_identifier( - yyscanner, (yyvsp[-3].c_string), OP_LENGTH, UNDEFINED); - - yr_free((yyvsp[-3].c_string)); - - fail_if_error(result); - - (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; - } -#line 3820 "grammar.c" /* yacc.c:1663 */ +#line 3953 "grammar.c" /* yacc.c:1646 */ break; case 128: -#line 2101 "grammar.y" /* yacc.c:1663 */ +#line 2220 "grammar.y" /* yacc.c:1646 */ { - int result = yr_parser_emit_with_arg( - yyscanner, OP_PUSH, 1, NULL, NULL); - - if (result == ERROR_SUCCESS) - result = yr_parser_reduce_string_identifier( - yyscanner, (yyvsp[0].c_string), OP_LENGTH, UNDEFINED); + int result = yr_parser_reduce_string_identifier( + yyscanner, (yyvsp[0].c_string), OP_COUNT, YR_UNDEFINED); yr_free((yyvsp[0].c_string)); fail_if_error(result); (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; + (yyval.expression).value.integer = YR_UNDEFINED; } -#line 3840 "grammar.c" /* yacc.c:1663 */ +#line 3969 "grammar.c" /* yacc.c:1646 */ break; case 129: -#line 2117 "grammar.y" /* yacc.c:1663 */ +#line 2232 "grammar.y" /* yacc.c:1646 */ + { + int result = yr_parser_reduce_string_identifier( + yyscanner, (yyvsp[-3].c_string), OP_OFFSET, YR_UNDEFINED); + + yr_free((yyvsp[-3].c_string)); + + fail_if_error(result); + + (yyval.expression).type = EXPRESSION_TYPE_INTEGER; + (yyval.expression).value.integer = YR_UNDEFINED; + } +#line 3985 "grammar.c" /* yacc.c:1646 */ + break; + + case 130: +#line 2244 "grammar.y" /* yacc.c:1646 */ + { + int result = yr_parser_emit_with_arg( + yyscanner, OP_PUSH, 1, NULL, NULL); + + if (result == ERROR_SUCCESS) + result = yr_parser_reduce_string_identifier( + yyscanner, (yyvsp[0].c_string), OP_OFFSET, YR_UNDEFINED); + + yr_free((yyvsp[0].c_string)); + + fail_if_error(result); + + (yyval.expression).type = EXPRESSION_TYPE_INTEGER; + (yyval.expression).value.integer = YR_UNDEFINED; + } +#line 4005 "grammar.c" /* yacc.c:1646 */ + break; + + case 131: +#line 2260 "grammar.y" /* yacc.c:1646 */ + { + int result = yr_parser_reduce_string_identifier( + yyscanner, (yyvsp[-3].c_string), OP_LENGTH, YR_UNDEFINED); + + yr_free((yyvsp[-3].c_string)); + + fail_if_error(result); + + (yyval.expression).type = EXPRESSION_TYPE_INTEGER; + (yyval.expression).value.integer = YR_UNDEFINED; + } +#line 4021 "grammar.c" /* yacc.c:1646 */ + break; + + case 132: +#line 2272 "grammar.y" /* yacc.c:1646 */ + { + int result = yr_parser_emit_with_arg( + yyscanner, OP_PUSH, 1, NULL, NULL); + + if (result == ERROR_SUCCESS) + result = yr_parser_reduce_string_identifier( + yyscanner, (yyvsp[0].c_string), OP_LENGTH, YR_UNDEFINED); + + yr_free((yyvsp[0].c_string)); + + fail_if_error(result); + + (yyval.expression).type = EXPRESSION_TYPE_INTEGER; + (yyval.expression).value.integer = YR_UNDEFINED; + } +#line 4041 "grammar.c" /* yacc.c:1646 */ + break; + + case 133: +#line 2288 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3853,7 +4054,7 @@ yyreduce: { case OBJECT_TYPE_INTEGER: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = UNDEFINED; + (yyval.expression).value.integer = YR_UNDEFINED; break; case OBJECT_TYPE_FLOAT: (yyval.expression).type = EXPRESSION_TYPE_FLOAT; @@ -3871,7 +4072,8 @@ yyreduce: yr_compiler_set_error_extra_info_fmt( compiler, "wrong usage of identifier \"%s\"", - (yyvsp[0].expression).identifier); + expression_identifier((yyvsp[0].expression))); + result = ERROR_WRONG_TYPE; } } @@ -3882,11 +4084,11 @@ yyreduce: fail_if_error(result); } -#line 3886 "grammar.c" /* yacc.c:1663 */ +#line 4088 "grammar.c" /* yacc.c:1646 */ break; - case 130: -#line 2159 "grammar.y" /* yacc.c:1663 */ + case 134: +#line 2331 "grammar.y" /* yacc.c:1646 */ { int result = ERROR_SUCCESS; @@ -3895,8 +4097,8 @@ yyreduce: if ((yyvsp[0].expression).type == EXPRESSION_TYPE_INTEGER) { (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = ((yyvsp[0].expression).value.integer == UNDEFINED) ? - UNDEFINED : -((yyvsp[0].expression).value.integer); + (yyval.expression).value.integer = ((yyvsp[0].expression).value.integer == YR_UNDEFINED) ? + YR_UNDEFINED : -((yyvsp[0].expression).value.integer); result = yr_parser_emit(yyscanner, OP_INT_MINUS, NULL); } else if ((yyvsp[0].expression).type == EXPRESSION_TYPE_FLOAT) @@ -3907,11 +4109,11 @@ yyreduce: fail_if_error(result); } -#line 3911 "grammar.c" /* yacc.c:1663 */ +#line 4113 "grammar.c" /* yacc.c:1646 */ break; - case 131: -#line 2180 "grammar.y" /* yacc.c:1663 */ + case 135: +#line 2352 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_operation( yyscanner, "+", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -3946,11 +4148,11 @@ yyreduce: fail_if_error(result); } -#line 3950 "grammar.c" /* yacc.c:1663 */ +#line 4152 "grammar.c" /* yacc.c:1646 */ break; - case 132: -#line 2215 "grammar.y" /* yacc.c:1663 */ + case 136: +#line 2387 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_operation( yyscanner, "-", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -3985,11 +4187,11 @@ yyreduce: fail_if_error(result); } -#line 3989 "grammar.c" /* yacc.c:1663 */ +#line 4191 "grammar.c" /* yacc.c:1646 */ break; - case 133: -#line 2250 "grammar.y" /* yacc.c:1663 */ + case 137: +#line 2422 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_operation( yyscanner, "*", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -4023,11 +4225,11 @@ yyreduce: fail_if_error(result); } -#line 4027 "grammar.c" /* yacc.c:1663 */ +#line 4229 "grammar.c" /* yacc.c:1646 */ break; - case 134: -#line 2284 "grammar.y" /* yacc.c:1663 */ + case 138: +#line 2456 "grammar.y" /* yacc.c:1646 */ { int result = yr_parser_reduce_operation( yyscanner, "\\", (yyvsp[-2].expression), (yyvsp[0].expression)); @@ -4052,11 +4254,11 @@ yyreduce: fail_if_error(result); } -#line 4056 "grammar.c" /* yacc.c:1663 */ +#line 4258 "grammar.c" /* yacc.c:1646 */ break; - case 135: -#line 2309 "grammar.y" /* yacc.c:1663 */ + case 139: +#line 2481 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "%"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "%"); @@ -4073,11 +4275,11 @@ yyreduce: fail_if_error(ERROR_DIVISION_BY_ZERO); } } -#line 4077 "grammar.c" /* yacc.c:1663 */ +#line 4279 "grammar.c" /* yacc.c:1646 */ break; - case 136: -#line 2326 "grammar.y" /* yacc.c:1663 */ + case 140: +#line 2498 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "^"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "^"); @@ -4087,11 +4289,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = OPERATION(^, (yyvsp[-2].expression).value.integer, (yyvsp[0].expression).value.integer); } -#line 4091 "grammar.c" /* yacc.c:1663 */ +#line 4293 "grammar.c" /* yacc.c:1646 */ break; - case 137: -#line 2336 "grammar.y" /* yacc.c:1663 */ + case 141: +#line 2508 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "^"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "^"); @@ -4101,11 +4303,11 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = OPERATION(&, (yyvsp[-2].expression).value.integer, (yyvsp[0].expression).value.integer); } -#line 4105 "grammar.c" /* yacc.c:1663 */ +#line 4307 "grammar.c" /* yacc.c:1646 */ break; - case 138: -#line 2346 "grammar.y" /* yacc.c:1663 */ + case 142: +#line 2518 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[-2].expression), EXPRESSION_TYPE_INTEGER, "|"); check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "|"); @@ -4115,25 +4317,25 @@ yyreduce: (yyval.expression).type = EXPRESSION_TYPE_INTEGER; (yyval.expression).value.integer = OPERATION(|, (yyvsp[-2].expression).value.integer, (yyvsp[0].expression).value.integer); } -#line 4119 "grammar.c" /* yacc.c:1663 */ +#line 4321 "grammar.c" /* yacc.c:1646 */ break; - case 139: -#line 2356 "grammar.y" /* yacc.c:1663 */ + case 143: +#line 2528 "grammar.y" /* yacc.c:1646 */ { check_type((yyvsp[0].expression), EXPRESSION_TYPE_INTEGER, "~"); fail_if_error(yr_parser_emit(yyscanner, OP_BITWISE_NOT, NULL)); (yyval.expression).type = EXPRESSION_TYPE_INTEGER; - (yyval.expression).value.integer = ((yyvsp[0].expression).value.integer == UNDEFINED) ? - UNDEFINED : ~((yyvsp[0].expression).value.integer); + (yyval.expression).value.integer = ((yyvsp[0].expression).value.integer == YR_UNDEFINED) ? + YR_UNDEFINED : ~((yyvsp[0].expression).value.integer); } -#line 4133 "grammar.c" /* yacc.c:1663 */ +#line 4335 "grammar.c" /* yacc.c:1646 */ break; - case 140: -#line 2366 "grammar.y" /* yacc.c:1663 */ + case 144: +#line 2538 "grammar.y" /* yacc.c:1646 */ { int result; @@ -4153,11 +4355,11 @@ yyreduce: fail_if_error(result); } -#line 4157 "grammar.c" /* yacc.c:1663 */ +#line 4359 "grammar.c" /* yacc.c:1646 */ break; - case 141: -#line 2386 "grammar.y" /* yacc.c:1663 */ + case 145: +#line 2558 "grammar.y" /* yacc.c:1646 */ { int result; @@ -4177,19 +4379,19 @@ yyreduce: fail_if_error(result); } -#line 4181 "grammar.c" /* yacc.c:1663 */ +#line 4383 "grammar.c" /* yacc.c:1646 */ break; - case 142: -#line 2406 "grammar.y" /* yacc.c:1663 */ + case 146: +#line 2578 "grammar.y" /* yacc.c:1646 */ { (yyval.expression) = (yyvsp[0].expression); } -#line 4189 "grammar.c" /* yacc.c:1663 */ +#line 4391 "grammar.c" /* yacc.c:1646 */ break; -#line 4193 "grammar.c" /* yacc.c:1663 */ +#line 4395 "grammar.c" /* yacc.c:1646 */ default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -4417,5 +4619,5 @@ yyreturn: #endif return yyresult; } -#line 2411 "grammar.y" /* yacc.c:1907 */ +#line 2583 "grammar.y" /* yacc.c:1906 */ diff --git a/libyara/grammar.h b/libyara/grammar.h index b32b89d9..45c60c5b 100644 --- a/libyara/grammar.h +++ b/libyara/grammar.h @@ -1,8 +1,8 @@ -/* A Bison parser, made by GNU Bison 3.0.5. */ +/* A Bison parser, made by GNU Bison 3.0.4. */ /* Bison interface for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015, 2018 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -69,34 +69,36 @@ extern int yara_yydebug; _ASCII_ = 278, _WIDE_ = 279, _XOR_ = 280, - _NOCASE_ = 281, - _FULLWORD_ = 282, - _AT_ = 283, - _FILESIZE_ = 284, - _ENTRYPOINT_ = 285, - _ALL_ = 286, - _ANY_ = 287, - _IN_ = 288, - _OF_ = 289, - _FOR_ = 290, - _THEM_ = 291, - _MATCHES_ = 292, - _CONTAINS_ = 293, - _IMPORT_ = 294, - _TRUE_ = 295, - _FALSE_ = 296, - _OR_ = 297, - _AND_ = 298, - _NOT_ = 299, - _EQ_ = 300, - _NEQ_ = 301, - _LT_ = 302, - _LE_ = 303, - _GT_ = 304, - _GE_ = 305, - _SHIFT_LEFT_ = 306, - _SHIFT_RIGHT_ = 307, - UNARY_MINUS = 308 + _BASE64_ = 281, + _BASE64_WIDE_ = 282, + _NOCASE_ = 283, + _FULLWORD_ = 284, + _AT_ = 285, + _FILESIZE_ = 286, + _ENTRYPOINT_ = 287, + _ALL_ = 288, + _ANY_ = 289, + _IN_ = 290, + _OF_ = 291, + _FOR_ = 292, + _THEM_ = 293, + _MATCHES_ = 294, + _CONTAINS_ = 295, + _IMPORT_ = 296, + _TRUE_ = 297, + _FALSE_ = 298, + _OR_ = 299, + _AND_ = 300, + _NOT_ = 301, + _EQ_ = 302, + _NEQ_ = 303, + _LT_ = 304, + _LE_ = 305, + _GT_ = 306, + _GE_ = 307, + _SHIFT_LEFT_ = 308, + _SHIFT_RIGHT_ = 309, + UNARY_MINUS = 310 }; #endif /* Tokens. */ @@ -124,53 +126,57 @@ extern int yara_yydebug; #define _ASCII_ 278 #define _WIDE_ 279 #define _XOR_ 280 -#define _NOCASE_ 281 -#define _FULLWORD_ 282 -#define _AT_ 283 -#define _FILESIZE_ 284 -#define _ENTRYPOINT_ 285 -#define _ALL_ 286 -#define _ANY_ 287 -#define _IN_ 288 -#define _OF_ 289 -#define _FOR_ 290 -#define _THEM_ 291 -#define _MATCHES_ 292 -#define _CONTAINS_ 293 -#define _IMPORT_ 294 -#define _TRUE_ 295 -#define _FALSE_ 296 -#define _OR_ 297 -#define _AND_ 298 -#define _NOT_ 299 -#define _EQ_ 300 -#define _NEQ_ 301 -#define _LT_ 302 -#define _LE_ 303 -#define _GT_ 304 -#define _GE_ 305 -#define _SHIFT_LEFT_ 306 -#define _SHIFT_RIGHT_ 307 -#define UNARY_MINUS 308 +#define _BASE64_ 281 +#define _BASE64_WIDE_ 282 +#define _NOCASE_ 283 +#define _FULLWORD_ 284 +#define _AT_ 285 +#define _FILESIZE_ 286 +#define _ENTRYPOINT_ 287 +#define _ALL_ 288 +#define _ANY_ 289 +#define _IN_ 290 +#define _OF_ 291 +#define _FOR_ 292 +#define _THEM_ 293 +#define _MATCHES_ 294 +#define _CONTAINS_ 295 +#define _IMPORT_ 296 +#define _TRUE_ 297 +#define _FALSE_ 298 +#define _OR_ 299 +#define _AND_ 300 +#define _NOT_ 301 +#define _EQ_ 302 +#define _NEQ_ 303 +#define _LT_ 304 +#define _LE_ 305 +#define _GT_ 306 +#define _GE_ 307 +#define _SHIFT_LEFT_ 308 +#define _SHIFT_RIGHT_ 309 +#define UNARY_MINUS 310 /* Value type. */ #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED union YYSTYPE { -#line 276 "grammar.y" /* yacc.c:1916 */ +#line 299 "grammar.y" /* yacc.c:1909 */ YR_EXPRESSION expression; SIZED_STRING* sized_string; char* c_string; int64_t integer; double double_; - YR_STRING* string; - YR_META* meta; - YR_RULE* rule; YR_MODIFIER modifier; -#line 174 "grammar.h" /* yacc.c:1916 */ + YR_ARENA_REF tag; + YR_ARENA_REF rule; + YR_ARENA_REF meta; + YR_ARENA_REF string; + +#line 180 "grammar.h" /* yacc.c:1909 */ }; typedef union YYSTYPE YYSTYPE; diff --git a/libyara/grammar.y b/libyara/grammar.y index 1e367a92..e620383b 100644 --- a/libyara/grammar.y +++ b/libyara/grammar.y @@ -37,6 +37,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include #include @@ -64,28 +65,21 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define FOR_EXPRESSION_ALL 1 #define FOR_EXPRESSION_ANY 2 -#define fail_if_error(e) \ - if (e != ERROR_SUCCESS) \ +#define fail_with_error(e) \ { \ compiler->last_error = e; \ yyerror(yyscanner, compiler, NULL); \ YYERROR; \ - } \ - - -#define set_flag_or_error(flags, new_flag) \ - if (flags & new_flag) \ - { \ - compiler->last_error = ERROR_DUPLICATED_MODIFIER; \ - yyerror(yyscanner, compiler, NULL); \ - YYERROR; \ - } \ - else \ - { \ - flags |= new_flag; \ } +#define fail_if_error(e) \ + if (e != ERROR_SUCCESS) \ + { \ + fail_with_error(e); \ + } \ + + #define check_type_with_cleanup(expression, expected_type, op, cleanup) \ if (((expression.type) & (expected_type)) == 0) \ { \ @@ -120,20 +114,30 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. check_type_with_cleanup(expression, expected_type, op, ) - #define loop_vars_cleanup(loop_index) \ { \ YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[loop_index]; \ for (int i = 0; i < loop_ctx->vars_count; i++) \ { \ - yr_free((void*) loop_ctx->vars[i].identifier); \ - loop_ctx->vars[i].identifier = NULL; \ + yr_free((void*) loop_ctx->vars[i].identifier.ptr); \ + loop_ctx->vars[i].identifier.ptr = NULL; \ + loop_ctx->vars[i].identifier.ref = YR_ARENA_NULL_REF; \ } \ loop_ctx->vars_count = 0; \ } \ -%} +// Given a YR_EXPRESSION returns its identifier. It returns identifier.ptr if not NULL and relies on identifier.ref if +// otherwise. +#define expression_identifier(expr) \ + ((expr).identifier.ptr != NULL ? \ + (expr).identifier.ptr : \ + (const char*) yr_arena_ref_to_ptr(compiler->arena, &(expr).identifier.ref)) + + +#define DEFAULT_BASE64_ALPHABET "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +%} %expect 1 // expect 1 shift/reduce conflicts @@ -181,6 +185,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %token _ASCII_ "" %token _WIDE_ "" %token _XOR_ "" +%token _BASE64_ "" +%token _BASE64_WIDE_ "" %token _NOCASE_ "" %token _FULLWORD_ "" %token _AT_ "" @@ -233,8 +239,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %type meta_declaration %type meta_declarations -%type tags -%type tag_list +%type tags +%type tag_list %type string_modifier %type string_modifiers @@ -273,16 +279,35 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %destructor { yr_free($$); $$ = NULL; } arguments %destructor { yr_free($$); $$ = NULL; } arguments_list +%destructor { + if ($$.alphabet != NULL) + { + yr_free($$.alphabet); + $$.alphabet = NULL; + } +} string_modifier + +%destructor { + if ($$.alphabet != NULL) + { + yr_free($$.alphabet); + $$.alphabet = NULL; + } +} string_modifiers + + %union { YR_EXPRESSION expression; SIZED_STRING* sized_string; char* c_string; int64_t integer; double double_; - YR_STRING* string; - YR_META* meta; - YR_RULE* rule; YR_MODIFIER modifier; + + YR_ARENA_REF tag; + YR_ARENA_REF rule; + YR_ARENA_REF meta; + YR_ARENA_REF string; } @@ -322,16 +347,22 @@ rule } tags '{' meta strings { - YR_RULE* rule = $4; // rule created in phase 1 + YR_RULE* rule = (YR_RULE*) yr_arena_ref_to_ptr( + compiler->arena, &$4); - rule->tags = $5; - rule->metas = $7; - rule->strings = $8; + rule->tags = (char*) yr_arena_ref_to_ptr( + compiler->arena, &$5); + + rule->metas = (YR_META*) yr_arena_ref_to_ptr( + compiler->arena, &$7); + + rule->strings = (YR_STRING*) yr_arena_ref_to_ptr( + compiler->arena, &$8); } condition '}' { int result = yr_parser_reduce_rule_declaration_phase_2( - yyscanner, $4); // rule created in phase 1 + yyscanner, &$4); // rule created in phase 1 yr_free($3); @@ -343,12 +374,10 @@ rule meta : /* empty */ { - $$ = NULL; + $$ = YR_ARENA_NULL_REF; } | _META_ ':' meta_declarations { - int result; - // Each rule have a list of meta-data info, consisting in a // sequence of YR_META structures. The last YR_META structure does // not represent a real meta-data, it's just a end-of-list marker @@ -360,15 +389,14 @@ meta memset(&null_meta, 0xFF, sizeof(YR_META)); null_meta.type = META_TYPE_NULL; - result = yr_arena_write_data( - compiler->metas_arena, + fail_if_error(yr_arena_write_data( + compiler->arena, + YR_METAS_TABLE, &null_meta, sizeof(YR_META), - NULL); + NULL)); $$ = $3; - - fail_if_error(result); } ; @@ -376,26 +404,16 @@ meta strings : /* empty */ { - $$ = NULL; + $$ = YR_ARENA_NULL_REF; } | _STRINGS_ ':' string_declarations { - // Each rule have a list of strings, consisting in a sequence - // of YR_STRING structures. The last YR_STRING structure does not - // represent a real string, it's just a end-of-list marker - // identified by a specific flag (STRING_FLAGS_NULL). Here we - // write the end-of-list marker. + YR_STRING* string = (YR_STRING*) yr_arena_get_ptr( + compiler->arena, + YR_STRINGS_TABLE, + (compiler->current_string_idx - 1) * sizeof(YR_STRING)); - YR_STRING null_string; - - memset(&null_string, 0xFF, sizeof(YR_STRING)); - null_string.g_flags = STRING_GFLAGS_NULL; - - fail_if_error(yr_arena_write_data( - compiler->strings_arena, - &null_string, - sizeof(YR_STRING), - NULL)); + string->flags |= STRING_FLAGS_LAST_IN_RULE; $$ = $3; } @@ -414,15 +432,15 @@ rule_modifiers rule_modifier - : _PRIVATE_ { $$ = RULE_GFLAGS_PRIVATE; } - | _GLOBAL_ { $$ = RULE_GFLAGS_GLOBAL; } + : _PRIVATE_ { $$ = RULE_FLAGS_PRIVATE; } + | _GLOBAL_ { $$ = RULE_FLAGS_GLOBAL; } ; tags : /* empty */ { - $$ = NULL; + $$ = YR_ARENA_NULL_REF; } | ':' tag_list { @@ -431,10 +449,8 @@ tags // additional null character. Here we write the ending null //character. Example: tag1\0tag2\0tag3\0\0 - int result = yr_arena_write_string( - yyget_extra(yyscanner)->sz_arena, "", NULL); - - fail_if_error(result); + fail_if_error(yr_arena_write_string( + yyget_extra(yyscanner)->arena, YR_SZ_POOL, "", NULL)); $$ = $2; } @@ -445,7 +461,7 @@ tag_list : _IDENTIFIER_ { int result = yr_arena_write_string( - yyget_extra(yyscanner)->sz_arena, $1, &$$); + yyget_extra(yyscanner)->arena, YR_SZ_POOL, $1, &$$); yr_free($1); @@ -453,36 +469,39 @@ tag_list } | tag_list _IDENTIFIER_ { - int result = ERROR_SUCCESS; + YR_ARENA_REF ref; - char* tag_name = $1; - size_t tag_length = tag_name != NULL ? strlen(tag_name) : 0; - - while (tag_length > 0) - { - if (strcmp(tag_name, $2) == 0) - { - yr_compiler_set_error_extra_info(compiler, tag_name); - result = ERROR_DUPLICATED_TAG_IDENTIFIER; - break; - } - - tag_name = (char*) yr_arena_next_address( - yyget_extra(yyscanner)->sz_arena, - tag_name, - tag_length + 1); - - tag_length = tag_name != NULL ? strlen(tag_name) : 0; - } - - if (result == ERROR_SUCCESS) - result = yr_arena_write_string( - yyget_extra(yyscanner)->sz_arena, $2, NULL); + // Write the new tag identifier. + int result = yr_arena_write_string( + yyget_extra(yyscanner)->arena, YR_SZ_POOL, $2, &ref); yr_free($2); fail_if_error(result); + // Get the address for the tag identifier just written. + char* new_tag = (char*) yr_arena_ref_to_ptr( + compiler->arena, &ref); + + // Take the address of first tag's identifier in the list. + char* tag = (char*) yr_arena_ref_to_ptr( + compiler->arena, &$$); + + // Search for duplicated tags. Tags are written one after + // the other, with zeroes in between (i.e: tag1/0tag2/0tag3) + // that's why can use tag < new_tag as the condition for the + // loop. + while (tag < new_tag) + { + if (strcmp(tag, new_tag) == 0) + { + yr_compiler_set_error_extra_info(compiler, tag); + fail_with_error(ERROR_DUPLICATED_TAG_IDENTIFIER); + } + + tag += strlen(tag) + 1; + } + $$ = $1; } ; @@ -506,7 +525,7 @@ meta_declaration $1, sized_string->c_string, 0, - &$$); + &$$); yr_free($1); yr_free($3); @@ -521,7 +540,7 @@ meta_declaration $1, NULL, $3, - &$$); + &$$); yr_free($1); @@ -535,7 +554,7 @@ meta_declaration $1, NULL, -$4, - &$$); + &$$); yr_free($1); @@ -549,7 +568,7 @@ meta_declaration $1, NULL, true, - &$$); + &$$); yr_free($1); @@ -563,7 +582,7 @@ meta_declaration $1, NULL, false, - &$$); + &$$); yr_free($1); @@ -586,10 +605,11 @@ string_declaration _TEXT_STRING_ string_modifiers { int result = yr_parser_reduce_string_declaration( - yyscanner, $5, $1, $4, &$$); + yyscanner, $5, $1, $4, &$$); yr_free($1); yr_free($4); + yr_free($5.alphabet); fail_if_error(result); compiler->current_line = 0; @@ -602,10 +622,10 @@ string_declaration { int result; - $5.flags |= STRING_GFLAGS_REGEXP; + $5.flags |= STRING_FLAGS_REGEXP; result = yr_parser_reduce_string_declaration( - yyscanner, $5, $1, $4, &$$); + yyscanner, $5, $1, $4, &$$); yr_free($1); yr_free($4); @@ -622,10 +642,10 @@ string_declaration { int result; - $5.flags |= STRING_GFLAGS_HEXADECIMAL; + $5.flags |= STRING_FLAGS_HEXADECIMAL; result = yr_parser_reduce_string_declaration( - yyscanner, $5, $1, $4, &$$); + yyscanner, $5, $1, $4, &$$); yr_free($1); yr_free($4); @@ -643,35 +663,76 @@ string_modifiers $$.flags = 0; $$.xor_min = 0; $$.xor_max = 0; + $$.alphabet = NULL; } | string_modifiers string_modifier { $$ = $1; - set_flag_or_error($$.flags, $2.flags); - // Only set the xor minimum and maximum if we are dealing with the // xor modifier. If we don't check for this then we can end up with // "xor wide" resulting in whatever is on the stack for "wide" // overwriting the values for xor. - if ($2.flags & STRING_GFLAGS_XOR) + if ($2.flags & STRING_FLAGS_XOR) { $$.xor_min = $2.xor_min; $$.xor_max = $2.xor_max; } + + // Only set the base64 alphabet if we are dealing with the base64 + // modifier. If we don't check for this then we can end up with + // "base64 ascii" resulting in whatever is on the stack for "ascii" + // overwriting the values for base64. + if (($2.flags & STRING_FLAGS_BASE64) || + ($2.flags & STRING_FLAGS_BASE64_WIDE)) + { + if ($$.alphabet != NULL) + { + if (sized_string_cmp($$.alphabet, $2.alphabet) != 0) + { + yr_compiler_set_error_extra_info( + compiler, "can not specify multiple alphabets"); + + yr_free($2.alphabet); + yr_free($$.alphabet); + + fail_with_error(ERROR_INVALID_MODIFIER); + } + else + { + yr_free($2.alphabet); + } + } + else + { + $$.alphabet = $2.alphabet; + } + } + + if ($$.flags & $2.flags) + { + if ($$.alphabet != NULL) + yr_free($$.alphabet); + + fail_with_error(ERROR_DUPLICATED_MODIFIER); + } + else + { + $$.flags = $$.flags | $2.flags; + } } ; string_modifier - : _WIDE_ { $$.flags = STRING_GFLAGS_WIDE; } - | _ASCII_ { $$.flags = STRING_GFLAGS_ASCII; } - | _NOCASE_ { $$.flags = STRING_GFLAGS_NO_CASE; } - | _FULLWORD_ { $$.flags = STRING_GFLAGS_FULL_WORD; } - | _PRIVATE_ { $$.flags = STRING_GFLAGS_PRIVATE; } + : _WIDE_ { $$.flags = STRING_FLAGS_WIDE; } + | _ASCII_ { $$.flags = STRING_FLAGS_ASCII; } + | _NOCASE_ { $$.flags = STRING_FLAGS_NO_CASE; } + | _FULLWORD_ { $$.flags = STRING_FLAGS_FULL_WORD; } + | _PRIVATE_ { $$.flags = STRING_FLAGS_PRIVATE; } | _XOR_ { - $$.flags = STRING_GFLAGS_XOR; + $$.flags = STRING_FLAGS_XOR; $$.xor_min = 0; $$.xor_max = 255; } @@ -687,7 +748,7 @@ string_modifier fail_if_error(result); - $$.flags = STRING_GFLAGS_XOR; + $$.flags = STRING_FLAGS_XOR; $$.xor_min = $3; $$.xor_max = $3; } @@ -723,32 +784,96 @@ string_modifier fail_if_error(result); - $$.flags = STRING_GFLAGS_XOR; + $$.flags = STRING_FLAGS_XOR; $$.xor_min = $3; $$.xor_max = $5; } + | _BASE64_ + { + $$.flags = STRING_FLAGS_BASE64; + $$.alphabet = sized_string_new(DEFAULT_BASE64_ALPHABET); + } + | _BASE64_ '(' _TEXT_STRING_ ')' + { + int result = ERROR_SUCCESS; + + if ($3->length != 64) + { + yr_free($3); + result = yr_compiler_set_error_extra_info( + compiler, "length of base64 alphabet must be 64"); + result = ERROR_INVALID_MODIFIER; + } + + fail_if_error(result); + + $$.flags = STRING_FLAGS_BASE64; + $$.alphabet = $3; + } + | _BASE64_WIDE_ + { + $$.flags = STRING_FLAGS_BASE64_WIDE; + $$.alphabet = sized_string_new(DEFAULT_BASE64_ALPHABET); + } + | _BASE64_WIDE_ '(' _TEXT_STRING_ ')' + { + int result = ERROR_SUCCESS; + + if ($3->length != 64) + { + yr_free($3); + result = yr_compiler_set_error_extra_info( + compiler, "length of base64 alphabet must be 64"); + result = ERROR_INVALID_MODIFIER; + } + + fail_if_error(result); + + $$.flags = STRING_FLAGS_BASE64_WIDE; + $$.alphabet = $3; + } ; regexp_modifiers : /* empty */ { $$.flags = 0; } - | regexp_modifiers regexp_modifier { set_flag_or_error($$.flags, $2.flags); } + | regexp_modifiers regexp_modifier + { + if ($1.flags & $2.flags) + { + fail_with_error(ERROR_DUPLICATED_MODIFIER); + } + else + { + $$.flags = $1.flags | $2.flags; + } + } ; regexp_modifier - : _WIDE_ { $$.flags = STRING_GFLAGS_WIDE; } - | _ASCII_ { $$.flags = STRING_GFLAGS_ASCII; } - | _NOCASE_ { $$.flags = STRING_GFLAGS_NO_CASE; } - | _FULLWORD_ { $$.flags = STRING_GFLAGS_FULL_WORD; } - | _PRIVATE_ { $$.flags = STRING_GFLAGS_PRIVATE; } + : _WIDE_ { $$.flags = STRING_FLAGS_WIDE; } + | _ASCII_ { $$.flags = STRING_FLAGS_ASCII; } + | _NOCASE_ { $$.flags = STRING_FLAGS_NO_CASE; } + | _FULLWORD_ { $$.flags = STRING_FLAGS_FULL_WORD; } + | _PRIVATE_ { $$.flags = STRING_FLAGS_PRIVATE; } ; hex_modifiers : /* empty */ { $$.flags = 0; } - | hex_modifiers hex_modifier { set_flag_or_error($$.flags, $2.flags); } + | hex_modifiers hex_modifier + { + if ($1.flags & $2.flags) + { + fail_with_error(ERROR_DUPLICATED_MODIFIER); + } + else + { + $$.flags = $1.flags | $2.flags; + } + } ; hex_modifier - : _PRIVATE_ { $$.flags = STRING_GFLAGS_PRIVATE; } + : _PRIVATE_ { $$.flags = STRING_FLAGS_PRIVATE; } ; identifier @@ -781,53 +906,59 @@ identifier YR_OBJECT* object = (YR_OBJECT*) yr_hash_table_lookup( compiler->objects_table, $1, NULL); + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, + YR_NAMESPACES_TABLE, + compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE)); + if (object == NULL) { // If not found, search within the current namespace. - char* ns = compiler->current_namespace->name; - object = (YR_OBJECT*) yr_hash_table_lookup( - compiler->objects_table, $1, ns); + compiler->objects_table, $1, ns->name); } if (object != NULL) { - char* id; + YR_ARENA_REF ref; result = yr_arena_write_string( - compiler->sz_arena, $1, &id); + compiler->arena, YR_SZ_POOL, $1, &ref); if (result == ERROR_SUCCESS) result = yr_parser_emit_with_arg_reloc( yyscanner, OP_OBJ_LOAD, - id, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); $$.type = EXPRESSION_TYPE_OBJECT; $$.value.object = object; - $$.identifier = object->identifier; + $$.identifier.ptr = NULL; + $$.identifier.ref = ref; } else { - YR_RULE* rule = (YR_RULE*) yr_hash_table_lookup( - compiler->rules_table, - $1, - compiler->current_namespace->name); + uint32_t rule_idx = yr_hash_table_lookup_uint32( + compiler->rules_table, $1, ns->name); - if (rule != NULL) + if (rule_idx != UINT32_MAX) { - result = yr_parser_emit_with_arg_reloc( + result = yr_parser_emit_with_arg( yyscanner, OP_PUSH_RULE, - rule, + rule_idx, NULL, NULL); + YR_RULE* rule = _yr_compiler_get_rule_by_idx(compiler, rule_idx); + + yr_arena_ptr_to_ref(compiler->arena, rule->identifier, &$$.identifier.ref); + $$.type = EXPRESSION_TYPE_BOOLEAN; - $$.value.integer = UNDEFINED; - $$.identifier = rule->identifier; + $$.value.integer = YR_UNDEFINED; + $$.identifier.ptr = NULL; } else { @@ -853,22 +984,23 @@ identifier if (field != NULL) { - char* ident; + YR_ARENA_REF ref; result = yr_arena_write_string( - compiler->sz_arena, $3, &ident); + compiler->arena, YR_SZ_POOL, $3, &ref); if (result == ERROR_SUCCESS) result = yr_parser_emit_with_arg_reloc( yyscanner, OP_OBJ_FIELD, - ident, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); $$.type = EXPRESSION_TYPE_OBJECT; $$.value.object = field; - $$.identifier = field->identifier; + $$.identifier.ref = ref; + $$.identifier.ptr = NULL; } else { @@ -879,7 +1011,7 @@ identifier else { yr_compiler_set_error_extra_info( - compiler, $1.identifier); + compiler, expression_identifier($1)); result = ERROR_NOT_A_STRUCTURE; } @@ -913,7 +1045,8 @@ identifier $$.type = EXPRESSION_TYPE_OBJECT; $$.value.object = array->prototype_item; - $$.identifier = array->identifier; + $$.identifier.ptr = array->identifier; + $$.identifier.ref = YR_ARENA_NULL_REF; } else if ($1.type == EXPRESSION_TYPE_OBJECT && $1.value.object->type == OBJECT_TYPE_DICTIONARY) @@ -934,12 +1067,13 @@ identifier $$.type = EXPRESSION_TYPE_OBJECT; $$.value.object = dict->prototype_item; - $$.identifier = dict->identifier; + $$.identifier.ptr = dict->identifier; + $$.identifier.ref = YR_ARENA_NULL_REF; } else { yr_compiler_set_error_extra_info( - compiler, $1.identifier); + compiler, expression_identifier($1)); result = ERROR_NOT_INDEXABLE; } @@ -949,9 +1083,9 @@ identifier | identifier '(' arguments ')' { + YR_ARENA_REF ref; int result = ERROR_SUCCESS; YR_OBJECT_FUNCTION* function; - char* args_fmt; if ($1.type == EXPRESSION_TYPE_OBJECT && $1.value.object->type == OBJECT_TYPE_FUNCTION) @@ -961,13 +1095,13 @@ identifier if (result == ERROR_SUCCESS) result = yr_arena_write_string( - compiler->sz_arena, $3, &args_fmt); + compiler->arena, YR_SZ_POOL, $3, &ref); if (result == ERROR_SUCCESS) result = yr_parser_emit_with_arg_reloc( yyscanner, OP_CALL, - args_fmt, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); @@ -975,12 +1109,13 @@ identifier $$.type = EXPRESSION_TYPE_OBJECT; $$.value.object = function->return_obj; - $$.identifier = function->identifier; + $$.identifier.ref = ref; + $$.identifier.ptr = NULL; } else { yr_compiler_set_error_extra_info( - compiler, $1.identifier); + compiler, expression_identifier($1)); result = ERROR_NOT_A_FUNCTION; } @@ -1003,7 +1138,7 @@ arguments_list $$ = (char*) yr_malloc(YR_MAX_FUNCTION_ARGS + 1); if ($$ == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); + fail_with_error(ERROR_INSUFFICIENT_MEMORY); switch($1.type) { @@ -1026,7 +1161,7 @@ arguments_list yr_free($$); yr_compiler_set_error_extra_info( compiler, "unknown type for argument 1 in function call"); - fail_if_error(ERROR_WRONG_TYPE); + fail_with_error(ERROR_WRONG_TYPE); break; default: // An unknown expression type is OK iff an error ocurred. @@ -1063,7 +1198,7 @@ arguments_list case EXPRESSION_TYPE_UNKNOWN: result = ERROR_WRONG_TYPE; yr_compiler_set_error_extra_info_fmt( - compiler, "unknown type for argument %lu in function call", + compiler, "unknown type for argument %zu in function call", // As we add one character per argument, the length of $1 is // the number of arguments parsed so far, and the argument // represented by is length of $1 plus one. @@ -1089,7 +1224,7 @@ regexp : _REGEXP_ { SIZED_STRING* sized_string = $1; - RE* re; + YR_ARENA_REF re_ref; RE_ERROR error; int result = ERROR_SUCCESS; @@ -1104,8 +1239,8 @@ regexp result = yr_re_compile( sized_string->c_string, re_flags, - compiler->re_code_arena, - &re, + compiler->arena, + &re_ref, &error); yr_free($1); @@ -1117,7 +1252,7 @@ regexp result = yr_parser_emit_with_arg_reloc( yyscanner, OP_PUSH, - re, + yr_arena_ref_to_ptr(compiler->arena, &re_ref), NULL, NULL); @@ -1191,7 +1326,7 @@ expression yyscanner, $1, OP_FOUND, - UNDEFINED); + YR_UNDEFINED); yr_free($1); @@ -1217,7 +1352,7 @@ expression | _STRING_IDENTIFIER_ _IN_ range { int result = yr_parser_reduce_string_identifier( - yyscanner, $1, OP_FOUND_IN, UNDEFINED); + yyscanner, $1, OP_FOUND_IN, YR_UNDEFINED); yr_free($1); @@ -1227,6 +1362,8 @@ expression } | _FOR_ for_expression error { + int i; + // Free all the loop variable identifiers, including the variables for // the current loop (represented by loop_index), and set loop_index to // -1. This is OK even if we have nested loops. If an error occurs while @@ -1234,7 +1371,7 @@ expression // anyways, so it's safe to do this cleanup while processing the error // for the inner loop. - for (int i = 0; i <= compiler->loop_index; i++) + for (i = 0; i <= compiler->loop_index; i++) { loop_vars_cleanup(i); } @@ -1280,7 +1417,7 @@ expression // JL_P repeat ; if M[1] is less M[3] repeat // // exit: - // POP ; remove the itertor object from the stack + // POP ; remove the iterator object from the stack // // PUSH_M 0 ; pushes number of true results for // PUSH_M 2 ; pushes value of @@ -1311,9 +1448,10 @@ expression compiler->loop_index++; - // This loop uses 3 internal variables besides the ones explicitly + // This loop uses internal variables besides the ones explicitly // defined by the user. - compiler->loop[compiler->loop_index].vars_internal_count = 3; + compiler->loop[compiler->loop_index].vars_internal_count = \ + YR_INTERNAL_LOOP_VARS; // Initialize the number of variables, this number will be incremented // as variable declaration are processed by for_variables. @@ -1335,49 +1473,56 @@ expression YR_LOOP_CONTEXT* loop_ctx = &compiler->loop[compiler->loop_index]; YR_FIXUP* fixup; - uint8_t* loop_start_addr; - void* jmp_arg_addr; + YR_ARENA_REF loop_start_ref; + YR_ARENA_REF jmp_offset_ref; int var_frame = _yr_compiler_get_var_frame(compiler); + int i; fail_if_error(yr_parser_emit( - yyscanner, OP_ITER_NEXT, &loop_start_addr)); + yyscanner, OP_ITER_NEXT, &loop_start_ref)); // For each variable generate an instruction that pops the value from - // the stack and store it into one memory slot starting at var_frame + 3 - // because the first 3 slots in the frame are for the internal variables. + // the stack and store it into one memory slot starting at var_frame + + // YR_INTERNAL_LOOP_VARS because the first YR_INTERNAL_LOOP_VARS slots + // in the frame are for the internal variables. - for (int i = 0; i < loop_ctx->vars_count; i++) + for (i = 0; i < loop_ctx->vars_count; i++) { fail_if_error(yr_parser_emit_with_arg( - yyscanner, OP_POP_M, var_frame + 3 + i, NULL, NULL)); + yyscanner, + OP_POP_M, + var_frame + YR_INTERNAL_LOOP_VARS + i, + NULL, + NULL)); } - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE_P, - 0, + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_arg_addr)); + &jmp_offset_ref)); - // Push a new fixup entry in the fixup stack so that the jump - // destination is set once we know it. + // We still don't know the jump's target, so we push a fixup entry + // in the stack, so that the jump's offset can be set once we know it. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); + fail_with_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_arg_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; - loop_ctx->addr = loop_start_addr; + loop_ctx->start_ref = loop_start_ref; } '(' boolean_expression ')' { + int32_t jmp_offset; YR_FIXUP* fixup; - uint8_t* pop_addr; + YR_ARENA_REF pop_ref; int var_frame = _yr_compiler_get_var_frame(compiler); @@ -1390,10 +1535,14 @@ expression fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JUNDEF_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); @@ -1403,24 +1552,38 @@ expression fail_if_error(yr_parser_emit_with_arg( yyscanner, OP_PUSH_M, var_frame + 2, NULL, NULL)); - fail_if_error(yr_parser_emit_with_arg_reloc( + jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION); + + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JL_P, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL)); fail_if_error(yr_parser_emit( - yyscanner, OP_POP, &pop_addr)); + yyscanner, OP_POP, &pop_ref)); - // Pop from the stack the fixup entry containing the jump's address - // that needs to be fixed. + // Pop from the stack the fixup entry containing the reference to + // the jump offset that needs to be fixed. fixup = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup->next; - // Fix the jump's target address. - *(void**)(fixup->address) = (void*)(pop_addr); + // The fixup entry has a reference to the jump offset that need + // to be fixed, convert the address into a pointer. + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + // The reference in the fixup entry points to the jump's offset + // but the jump instruction is one byte before, that's why we add + // one to the offset. + jmp_offset = pop_ref.offset - fixup->ref.offset + 1; + + // Fix the jump's offset. + *jmp_offset_addr = jmp_offset; yr_free(fixup); @@ -1444,9 +1607,10 @@ expression } | _FOR_ for_expression _OF_ string_set ':' { + YR_ARENA_REF ref; + int result = ERROR_SUCCESS; int var_frame; - uint8_t* addr; if (compiler->loop_index + 1 == YR_MAX_LOOP_NESTING) result = ERROR_LOOP_NESTING_LIMIT_EXCEEDED; @@ -1468,12 +1632,13 @@ expression // Pop the first string. yr_parser_emit_with_arg( - yyscanner, OP_POP_M, var_frame, &addr, NULL); + yyscanner, OP_POP_M, var_frame, &ref, NULL); compiler->loop_for_of_var_index = var_frame; - compiler->loop[compiler->loop_index].vars_internal_count = 3; + compiler->loop[compiler->loop_index].start_ref = ref; compiler->loop[compiler->loop_index].vars_count = 0; - compiler->loop[compiler->loop_index].addr = addr; + compiler->loop[compiler->loop_index].vars_internal_count = \ + YR_INTERNAL_LOOP_VARS; } '(' boolean_expression ')' { @@ -1485,7 +1650,7 @@ expression // Increment counter by the value returned by the // boolean expression (0 or 1). If the boolean expression - // returned UNDEFINED the OP_ADD_M won't do anything. + // returned YR_UNDEFINED the OP_ADD_M won't do anything. yr_parser_emit_with_arg( yyscanner, OP_ADD_M, var_frame + 1, NULL, NULL); @@ -1494,12 +1659,16 @@ expression yr_parser_emit_with_arg( yyscanner, OP_INCR_M, var_frame + 2, NULL, NULL); + int32_t jmp_offset = \ + compiler->loop[compiler->loop_index].start_ref.offset - + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION); + // If next string is not undefined, go back to the // beginning of the loop. - yr_parser_emit_with_arg_reloc( + yr_parser_emit_with_arg_int32( yyscanner, OP_JNUNDEF, - compiler->loop[compiler->loop_index].addr, + jmp_offset, NULL, NULL); @@ -1541,43 +1710,43 @@ expression | boolean_expression _AND_ { YR_FIXUP* fixup; - void* jmp_destination_addr; + YR_ARENA_REF jmp_offset_ref; - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JFALSE, - 0, // still don't know the jump destination + 0, // still don't know the jump offset, use 0 for now. NULL, - &jmp_destination_addr)); + &jmp_offset_ref)); - // create a fixup entry for the jump and push it in the stack + // Create a fixup entry for the jump and push it in the stack. fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); + fail_with_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_destination_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; } boolean_expression { YR_FIXUP* fixup; - uint8_t* nop_addr; fail_if_error(yr_parser_emit(yyscanner, OP_AND, NULL)); - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_AND instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*) nop_addr; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + int32_t jmp_offset = \ + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); @@ -1586,42 +1755,42 @@ expression | boolean_expression _OR_ { YR_FIXUP* fixup; - void* jmp_destination_addr; + YR_ARENA_REF jmp_offset_ref; - fail_if_error(yr_parser_emit_with_arg_reloc( + fail_if_error(yr_parser_emit_with_arg_int32( yyscanner, OP_JTRUE, - 0, // still don't know the jump destination + 0, // still don't know the jump destination, use 0 for now. NULL, - &jmp_destination_addr)); + &jmp_offset_ref)); fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); if (fixup == NULL) - fail_if_error(ERROR_INSUFFICIENT_MEMORY); + fail_with_error(ERROR_INSUFFICIENT_MEMORY); - fixup->address = jmp_destination_addr; + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; } boolean_expression { YR_FIXUP* fixup; - uint8_t* nop_addr; fail_if_error(yr_parser_emit(yyscanner, OP_OR, NULL)); - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_JFALSE. We can not simply - // use the address of the OP_OR instruction +1 because we can't be - // sure that the instruction following the OP_AND is going to be in - // the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - fail_if_error(yr_parser_emit(yyscanner, OP_NOP, &nop_addr)); - fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*)(nop_addr); + + int32_t jmp_offset = \ + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); @@ -1697,7 +1866,7 @@ for_variables fail_if_error(result); - loop_ctx->vars[loop_ctx->vars_count++].identifier = $1; + loop_ctx->vars[loop_ctx->vars_count++].identifier.ptr = $1; assert(loop_ctx->vars_count <= YR_MAX_LOOP_VARS); } @@ -1724,7 +1893,7 @@ for_variables fail_if_error(result); - loop_ctx->vars[loop_ctx->vars_count++].identifier = $3; + loop_ctx->vars[loop_ctx->vars_count++].identifier.ptr = $3; } ; @@ -1759,7 +1928,7 @@ iterator compiler, "iterator for \"%s\" yields a single item on each iteration" ", but the loop expects %d", - $1.identifier, + expression_identifier($1), loop_ctx->vars_count); result = ERROR_SYNTAX_ERROR; @@ -1785,7 +1954,7 @@ iterator yr_compiler_set_error_extra_info_fmt( compiler, "iterator for \"%s\" yields a key,value pair item on each iteration", - $1.identifier); + expression_identifier($1)); result = ERROR_SYNTAX_ERROR; } @@ -1798,7 +1967,7 @@ iterator yr_compiler_set_error_extra_info_fmt( compiler, "identifier \"%s\" is not iterable", - $1.identifier); + expression_identifier($1)); } fail_if_error(result); @@ -1812,7 +1981,7 @@ iterator if (loop_ctx->vars_count == 1) { loop_ctx->vars[0].type = EXPRESSION_TYPE_INTEGER; - loop_ctx->vars[0].value.integer = UNDEFINED; + loop_ctx->vars[0].value.integer = YR_UNDEFINED; } else { @@ -1910,13 +2079,13 @@ string_set : '(' { // Push end-of-list marker - yr_parser_emit_with_arg(yyscanner, OP_PUSH, UNDEFINED, NULL, NULL); + yr_parser_emit_with_arg(yyscanner, OP_PUSH, YR_UNDEFINED, NULL, NULL); } string_enumeration ')' | _THEM_ { fail_if_error(yr_parser_emit_with_arg( - yyscanner, OP_PUSH, UNDEFINED, NULL, NULL)); + yyscanner, OP_PUSH, YR_UNDEFINED, NULL, NULL)); fail_if_error(yr_parser_emit_pushes_for_strings( yyscanner, "$*")); @@ -1955,7 +2124,7 @@ for_expression } | _ALL_ { - yr_parser_emit_with_arg(yyscanner, OP_PUSH, UNDEFINED, NULL, NULL); + yr_parser_emit_with_arg(yyscanner, OP_PUSH, YR_UNDEFINED, NULL, NULL); $$ = FOR_EXPRESSION_ALL; } | _ANY_ @@ -1977,7 +2146,7 @@ primary_expression yyscanner, OP_FILESIZE, NULL)); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _ENTRYPOINT_ { @@ -1989,7 +2158,7 @@ primary_expression yyscanner, OP_ENTRYPOINT, NULL)); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _INTEGER_FUNCTION_ '(' primary_expression ')' { @@ -2003,7 +2172,7 @@ primary_expression yyscanner, (uint8_t) (OP_READ_INT + $1), NULL)); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _NUMBER_ { @@ -2022,13 +2191,14 @@ primary_expression } | _TEXT_STRING_ { - SIZED_STRING* sized_string; + YR_ARENA_REF ref; int result = yr_arena_write_data( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, $1, $1->length + sizeof(SIZED_STRING), - (void**) &sized_string); + &ref); yr_free($1); @@ -2036,38 +2206,39 @@ primary_expression result = yr_parser_emit_with_arg_reloc( yyscanner, OP_PUSH, - sized_string, + yr_arena_ref_to_ptr(compiler->arena, &ref), NULL, NULL); fail_if_error(result); $$.type = EXPRESSION_TYPE_STRING; - $$.value.sized_string = sized_string; + //TODO + // $$.value.sized_string = sized_string; } | _STRING_COUNT_ { int result = yr_parser_reduce_string_identifier( - yyscanner, $1, OP_COUNT, UNDEFINED); + yyscanner, $1, OP_COUNT, YR_UNDEFINED); yr_free($1); fail_if_error(result); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _STRING_OFFSET_ '[' primary_expression ']' { int result = yr_parser_reduce_string_identifier( - yyscanner, $1, OP_OFFSET, UNDEFINED); + yyscanner, $1, OP_OFFSET, YR_UNDEFINED); yr_free($1); fail_if_error(result); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _STRING_OFFSET_ { @@ -2076,26 +2247,26 @@ primary_expression if (result == ERROR_SUCCESS) result = yr_parser_reduce_string_identifier( - yyscanner, $1, OP_OFFSET, UNDEFINED); + yyscanner, $1, OP_OFFSET, YR_UNDEFINED); yr_free($1); fail_if_error(result); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _STRING_LENGTH_ '[' primary_expression ']' { int result = yr_parser_reduce_string_identifier( - yyscanner, $1, OP_LENGTH, UNDEFINED); + yyscanner, $1, OP_LENGTH, YR_UNDEFINED); yr_free($1); fail_if_error(result); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | _STRING_LENGTH_ { @@ -2104,14 +2275,14 @@ primary_expression if (result == ERROR_SUCCESS) result = yr_parser_reduce_string_identifier( - yyscanner, $1, OP_LENGTH, UNDEFINED); + yyscanner, $1, OP_LENGTH, YR_UNDEFINED); yr_free($1); fail_if_error(result); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; } | identifier { @@ -2126,7 +2297,7 @@ primary_expression { case OBJECT_TYPE_INTEGER: $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = UNDEFINED; + $$.value.integer = YR_UNDEFINED; break; case OBJECT_TYPE_FLOAT: $$.type = EXPRESSION_TYPE_FLOAT; @@ -2144,7 +2315,8 @@ primary_expression yr_compiler_set_error_extra_info_fmt( compiler, "wrong usage of identifier \"%s\"", - $1.identifier); + expression_identifier($1)); + result = ERROR_WRONG_TYPE; } } @@ -2164,8 +2336,8 @@ primary_expression if ($2.type == EXPRESSION_TYPE_INTEGER) { $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = ($2.value.integer == UNDEFINED) ? - UNDEFINED : -($2.value.integer); + $$.value.integer = ($2.value.integer == YR_UNDEFINED) ? + YR_UNDEFINED : -($2.value.integer); result = yr_parser_emit(yyscanner, OP_INT_MINUS, NULL); } else if ($2.type == EXPRESSION_TYPE_FLOAT) @@ -2359,8 +2531,8 @@ primary_expression fail_if_error(yr_parser_emit(yyscanner, OP_BITWISE_NOT, NULL)); $$.type = EXPRESSION_TYPE_INTEGER; - $$.value.integer = ($2.value.integer == UNDEFINED) ? - UNDEFINED : ~($2.value.integer); + $$.value.integer = ($2.value.integer == YR_UNDEFINED) ? + YR_UNDEFINED : ~($2.value.integer); } | primary_expression _SHIFT_LEFT_ primary_expression { diff --git a/libyara/hash.c b/libyara/hash.c index 3a650030..1bcd427c 100644 --- a/libyara/hash.c +++ b/libyara/hash.c @@ -365,3 +365,37 @@ YR_API int yr_hash_table_add( ns, value); } + + +YR_API uint32_t yr_hash_table_add_uint32( + YR_HASH_TABLE* table, + const char* key, + const char* ns, + uint32_t value) +{ + // Don't allow values equal to UINT32_MAX or UINT32_MAX - 1. + if (value >= UINT32_MAX - 1) + return ERROR_INVALID_ARGUMENT; + + // Add +1 to the value in order to avoid putting a NULL pointer in the + // hash table once the integer is casted to a pointer. This is undone + // by yr_hash_table_lookup_uint32. + return yr_hash_table_add( + table, key, ns, (void*) (size_t) (value + 1)); +} + + +YR_API uint32_t yr_hash_table_lookup_uint32( + YR_HASH_TABLE* table, + const char* key, + const char* ns) +{ + void* ptr = yr_hash_table_lookup(table, key, ns); + + if (ptr == NULL) + return UINT32_MAX; + + // Remove one from the pointer before converting back to integer, see + // comment in yr_hash_table_add_uint32. + return (uint32_t) (size_t) ((uint8_t*) ptr - 1); +} diff --git a/libyara/hex_lexer.c b/libyara/hex_lexer.c index 7c338679..87ad8731 100644 --- a/libyara/hex_lexer.c +++ b/libyara/hex_lexer.c @@ -1,6 +1,6 @@ -#line 1 "hex_lexer.c" +#line 2 "hex_lexer.c" -#line 3 "hex_lexer.c" +#line 4 "hex_lexer.c" #define YY_INT_ALIGNED short int @@ -9,233 +9,11 @@ #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 #define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 4 +#define YY_FLEX_SUBMINOR_VERSION 0 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#ifdef yy_create_buffer -#define hex_yy_create_buffer_ALREADY_DEFINED -#else -#define yy_create_buffer hex_yy_create_buffer -#endif - -#ifdef yy_delete_buffer -#define hex_yy_delete_buffer_ALREADY_DEFINED -#else -#define yy_delete_buffer hex_yy_delete_buffer -#endif - -#ifdef yy_scan_buffer -#define hex_yy_scan_buffer_ALREADY_DEFINED -#else -#define yy_scan_buffer hex_yy_scan_buffer -#endif - -#ifdef yy_scan_string -#define hex_yy_scan_string_ALREADY_DEFINED -#else -#define yy_scan_string hex_yy_scan_string -#endif - -#ifdef yy_scan_bytes -#define hex_yy_scan_bytes_ALREADY_DEFINED -#else -#define yy_scan_bytes hex_yy_scan_bytes -#endif - -#ifdef yy_init_buffer -#define hex_yy_init_buffer_ALREADY_DEFINED -#else -#define yy_init_buffer hex_yy_init_buffer -#endif - -#ifdef yy_flush_buffer -#define hex_yy_flush_buffer_ALREADY_DEFINED -#else -#define yy_flush_buffer hex_yy_flush_buffer -#endif - -#ifdef yy_load_buffer_state -#define hex_yy_load_buffer_state_ALREADY_DEFINED -#else -#define yy_load_buffer_state hex_yy_load_buffer_state -#endif - -#ifdef yy_switch_to_buffer -#define hex_yy_switch_to_buffer_ALREADY_DEFINED -#else -#define yy_switch_to_buffer hex_yy_switch_to_buffer -#endif - -#ifdef yypush_buffer_state -#define hex_yypush_buffer_state_ALREADY_DEFINED -#else -#define yypush_buffer_state hex_yypush_buffer_state -#endif - -#ifdef yypop_buffer_state -#define hex_yypop_buffer_state_ALREADY_DEFINED -#else -#define yypop_buffer_state hex_yypop_buffer_state -#endif - -#ifdef yyensure_buffer_stack -#define hex_yyensure_buffer_stack_ALREADY_DEFINED -#else -#define yyensure_buffer_stack hex_yyensure_buffer_stack -#endif - -#ifdef yylex -#define hex_yylex_ALREADY_DEFINED -#else -#define yylex hex_yylex -#endif - -#ifdef yyrestart -#define hex_yyrestart_ALREADY_DEFINED -#else -#define yyrestart hex_yyrestart -#endif - -#ifdef yylex_init -#define hex_yylex_init_ALREADY_DEFINED -#else -#define yylex_init hex_yylex_init -#endif - -#ifdef yylex_init_extra -#define hex_yylex_init_extra_ALREADY_DEFINED -#else -#define yylex_init_extra hex_yylex_init_extra -#endif - -#ifdef yylex_destroy -#define hex_yylex_destroy_ALREADY_DEFINED -#else -#define yylex_destroy hex_yylex_destroy -#endif - -#ifdef yyget_debug -#define hex_yyget_debug_ALREADY_DEFINED -#else -#define yyget_debug hex_yyget_debug -#endif - -#ifdef yyset_debug -#define hex_yyset_debug_ALREADY_DEFINED -#else -#define yyset_debug hex_yyset_debug -#endif - -#ifdef yyget_extra -#define hex_yyget_extra_ALREADY_DEFINED -#else -#define yyget_extra hex_yyget_extra -#endif - -#ifdef yyset_extra -#define hex_yyset_extra_ALREADY_DEFINED -#else -#define yyset_extra hex_yyset_extra -#endif - -#ifdef yyget_in -#define hex_yyget_in_ALREADY_DEFINED -#else -#define yyget_in hex_yyget_in -#endif - -#ifdef yyset_in -#define hex_yyset_in_ALREADY_DEFINED -#else -#define yyset_in hex_yyset_in -#endif - -#ifdef yyget_out -#define hex_yyget_out_ALREADY_DEFINED -#else -#define yyget_out hex_yyget_out -#endif - -#ifdef yyset_out -#define hex_yyset_out_ALREADY_DEFINED -#else -#define yyset_out hex_yyset_out -#endif - -#ifdef yyget_leng -#define hex_yyget_leng_ALREADY_DEFINED -#else -#define yyget_leng hex_yyget_leng -#endif - -#ifdef yyget_text -#define hex_yyget_text_ALREADY_DEFINED -#else -#define yyget_text hex_yyget_text -#endif - -#ifdef yyget_lineno -#define hex_yyget_lineno_ALREADY_DEFINED -#else -#define yyget_lineno hex_yyget_lineno -#endif - -#ifdef yyset_lineno -#define hex_yyset_lineno_ALREADY_DEFINED -#else -#define yyset_lineno hex_yyset_lineno -#endif - -#ifdef yyget_column -#define hex_yyget_column_ALREADY_DEFINED -#else -#define yyget_column hex_yyget_column -#endif - -#ifdef yyset_column -#define hex_yyset_column_ALREADY_DEFINED -#else -#define yyset_column hex_yyset_column -#endif - -#ifdef yywrap -#define hex_yywrap_ALREADY_DEFINED -#else -#define yywrap hex_yywrap -#endif - -#ifdef yyget_lval -#define hex_yyget_lval_ALREADY_DEFINED -#else -#define yyget_lval hex_yyget_lval -#endif - -#ifdef yyset_lval -#define hex_yyset_lval_ALREADY_DEFINED -#else -#define yyset_lval hex_yyset_lval -#endif - -#ifdef yyalloc -#define hex_yyalloc_ALREADY_DEFINED -#else -#define yyalloc hex_yyalloc -#endif - -#ifdef yyrealloc -#define hex_yyrealloc_ALREADY_DEFINED -#else -#define yyrealloc hex_yyrealloc -#endif - -#ifdef yyfree -#define hex_yyfree_ALREADY_DEFINED -#else -#define yyfree hex_yyfree -#endif - /* First, we deal with platform-specific or compiler-specific issues. */ /* begin standard C headers. */ @@ -306,32 +84,40 @@ typedef unsigned int flex_uint32_t; #define UINT32_MAX (4294967295U) #endif -#ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) -#endif - #endif /* ! C99 */ #endif /* ! FLEXINT_H */ -/* begin standard C++ headers. */ +#ifdef __cplusplus -/* TODO: this is always defined, so inline it */ +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST + +#else /* ! __cplusplus */ + +/* C99 requires __STDC__ to be defined as 1. */ +#if defined (__STDC__) + +#define YY_USE_CONST + +#endif /* defined (__STDC__) */ +#endif /* ! __cplusplus */ + +#ifdef YY_USE_CONST #define yyconst const - -#if defined(__GNUC__) && __GNUC__ >= 3 -#define yynoreturn __attribute__((__noreturn__)) #else -#define yynoreturn +#define yyconst #endif /* Returned upon end-of-file. */ #define YY_NULL 0 -/* Promotes a possibly negative, possibly signed char to an - * integer in range [0..255] for use as an array index. +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. */ -#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) /* An opaque pointer. */ #ifndef YY_TYPEDEF_YY_SCANNER_T @@ -355,16 +141,20 @@ typedef void* yyscan_t; * definition of BEGIN. */ #define BEGIN yyg->yy_start = 1 + 2 * + /* Translate the current start state into a value that can be later handed * to BEGIN to return to the state. The YYSTATE alias is for lex * compatibility. */ #define YY_START ((yyg->yy_start - 1) / 2) #define YYSTATE YY_START + /* Action number for EOF rule of a given start state. */ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin , yyscanner ) +#define YY_NEW_FILE hex_yyrestart(yyin ,yyscanner ) + #define YY_END_OF_BUFFER_CHAR 0 /* Size of default input buffer. */ @@ -397,10 +187,10 @@ typedef size_t yy_size_t; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 #define EOB_ACT_LAST_MATCH 2 - + /* Note: We specifically omit the test for yy_rule_can_match_eol because it requires * access to the local variable yy_act. Since yyless() is a macro, it would break - * existing scanners that call yyless() from OUTSIDE yylex. + * existing scanners that call yyless() from OUTSIDE hex_yylex. * One obvious solution it to make yy_act a global. I tried that, and saw * a 5% performance hit in a non-yylineno scanner, because yy_act is * normally declared as a register variable-- so it is not worth it. @@ -433,6 +223,7 @@ typedef size_t yy_size_t; YY_DO_BEFORE_ACTION; /* set up yytext again */ \ } \ while ( 0 ) + #define unput(c) yyunput( c, yyg->yytext_ptr , yyscanner ) #ifndef YY_STRUCT_YY_BUFFER_STATE @@ -447,12 +238,12 @@ struct yy_buffer_state /* Size of input buffer in bytes, not including room for EOB * characters. */ - int yy_buf_size; + yy_size_t yy_buf_size; /* Number of characters read into yy_ch_buf, not including EOB * characters. */ - int yy_n_chars; + yy_size_t yy_n_chars; /* Whether we "own" the buffer - i.e., we know we created it, * and can realloc() it to grow it, and should free() it to @@ -475,7 +266,7 @@ struct yy_buffer_state int yy_bs_lineno; /**< The line count. */ int yy_bs_column; /**< The column count. */ - + /* Whether to try to fill the input buffer when we reach the * end of it. */ @@ -492,7 +283,7 @@ struct yy_buffer_state * possible backing-up. * * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by + * (via hex_yyrestart()), so that the user can continue scanning by * just pointing yyin at a new input file. */ #define YY_BUFFER_EOF_PENDING 2 @@ -509,77 +300,87 @@ struct yy_buffer_state #define YY_CURRENT_BUFFER ( yyg->yy_buffer_stack \ ? yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] \ : NULL) + /* Same as previous macro, but useful when we know that the buffer stack is not * NULL or when we need an lvalue. For internal use only. */ #define YY_CURRENT_BUFFER_LVALUE yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] -void yyrestart ( FILE *input_file , yyscan_t yyscanner ); -void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size , yyscan_t yyscanner ); -void yy_delete_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yy_flush_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yypush_buffer_state ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -void yypop_buffer_state ( yyscan_t yyscanner ); +void hex_yyrestart (FILE *input_file ,yyscan_t yyscanner ); +void hex_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +YY_BUFFER_STATE hex_yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner ); +void hex_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void hex_yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void hex_yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +void hex_yypop_buffer_state (yyscan_t yyscanner ); -static void yyensure_buffer_stack ( yyscan_t yyscanner ); -static void yy_load_buffer_state ( yyscan_t yyscanner ); -static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file , yyscan_t yyscanner ); -#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER , yyscanner) +static void hex_yyensure_buffer_stack (yyscan_t yyscanner ); +static void hex_yy_load_buffer_state (yyscan_t yyscanner ); +static void hex_yy_init_buffer (YY_BUFFER_STATE b,FILE *file ,yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner ); +#define YY_FLUSH_BUFFER hex_yy_flush_buffer(YY_CURRENT_BUFFER ,yyscanner) -void *yyalloc ( yy_size_t , yyscan_t yyscanner ); -void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner ); -void yyfree ( void * , yyscan_t yyscanner ); +YY_BUFFER_STATE hex_yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner ); +YY_BUFFER_STATE hex_yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner ); +YY_BUFFER_STATE hex_yy_scan_bytes (yyconst char *bytes,yy_size_t len ,yyscan_t yyscanner ); + +void *hex_yyalloc (yy_size_t ,yyscan_t yyscanner ); +void *hex_yyrealloc (void *,yy_size_t ,yyscan_t yyscanner ); +void hex_yyfree (void * ,yyscan_t yyscanner ); + +#define yy_new_buffer hex_yy_create_buffer -#define yy_new_buffer yy_create_buffer #define yy_set_interactive(is_interactive) \ { \ if ( ! YY_CURRENT_BUFFER ){ \ - yyensure_buffer_stack (yyscanner); \ + hex_yyensure_buffer_stack (yyscanner); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); \ + hex_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ } + #define yy_set_bol(at_bol) \ { \ if ( ! YY_CURRENT_BUFFER ){\ - yyensure_buffer_stack (yyscanner); \ + hex_yyensure_buffer_stack (yyscanner); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); \ + hex_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ } + #define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) /* Begin user sect3 */ #define hex_yywrap(yyscanner) (/*CONSTCOND*/1) #define YY_SKIP_YYWRAP -typedef flex_uint8_t YY_CHAR; + +typedef unsigned char YY_CHAR; typedef int yy_state_type; #define yytext_ptr yytext_r -static yy_state_type yy_get_previous_state ( yyscan_t yyscanner ); -static yy_state_type yy_try_NUL_trans ( yy_state_type current_state , yyscan_t yyscanner); -static int yy_get_next_buffer ( yyscan_t yyscanner ); -static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner ); +static yy_state_type yy_get_previous_state (yyscan_t yyscanner ); +static yy_state_type yy_try_NUL_trans (yy_state_type current_state ,yyscan_t yyscanner); +static int yy_get_next_buffer (yyscan_t yyscanner ); +#if defined(__GNUC__) && __GNUC__ >= 3 +__attribute__((__noreturn__)) +#endif +static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner ); /* Done after the current pattern has been matched and before the * corresponding action - sets up yytext. */ #define YY_DO_BEFORE_ACTION \ yyg->yytext_ptr = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ + yyleng = (size_t) (yy_cp - yy_bp); \ yyg->yy_hold_char = *yy_cp; \ *yy_cp = '\0'; \ yyg->yy_c_buf_p = yy_cp; + #define YY_NUM_RULES 18 #define YY_END_OF_BUFFER 19 /* This struct is not used in this scanner, @@ -589,7 +390,7 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[34] = +static yyconst flex_int16_t yy_accept[34] = { 0, 0, 0, 0, 0, 0, 0, 19, 17, 15, 15, 16, 17, 17, 17, 5, 8, 8, 14, 13, 13, @@ -597,7 +398,7 @@ static const flex_int16_t yy_accept[34] = 11, 9, 0 } ; -static const YY_CHAR yy_ec[256] = +static yyconst YY_CHAR yy_ec[256] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, @@ -629,13 +430,13 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[13] = +static yyconst YY_CHAR yy_meta[13] = { 0, 1, 1, 2, 1, 1, 1, 1, 3, 3, 3, 1, 1 } ; -static const flex_int16_t yy_base[38] = +static yyconst flex_uint16_t yy_base[38] = { 0, 0, 0, 36, 35, 12, 0, 39, 42, 42, 42, 42, 20, 29, 28, 42, 42, 29, 42, 42, 42, @@ -643,7 +444,7 @@ static const flex_int16_t yy_base[38] = 26, 0, 42, 27, 29, 23, 30 } ; -static const flex_int16_t yy_def[38] = +static yyconst flex_int16_t yy_def[38] = { 0, 33, 1, 34, 34, 33, 5, 33, 33, 33, 33, 33, 33, 35, 36, 33, 33, 33, 33, 33, 33, @@ -651,7 +452,7 @@ static const flex_int16_t yy_def[38] = 33, 37, 0, 33, 33, 33, 33 } ; -static const flex_int16_t yy_nxt[55] = +static yyconst flex_uint16_t yy_nxt[55] = { 0, 8, 9, 10, 11, 8, 8, 12, 13, 14, 13, 15, 8, 18, 19, 20, 18, 18, 21, 18, 22, @@ -661,7 +462,7 @@ static const flex_int16_t yy_nxt[55] = 33, 33, 33, 33 } ; -static const flex_int16_t yy_chk[55] = +static yyconst flex_int16_t yy_chk[55] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 5, 5, 5, 5, 5, 5, 5, @@ -672,7 +473,7 @@ static const flex_int16_t yy_chk[55] = } ; /* Table of booleans, true if rule could match eol. */ -static const flex_int32_t yy_rule_can_match_eol[19] = +static yyconst flex_int32_t yy_rule_can_match_eol[19] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, }; @@ -748,16 +549,16 @@ with noyywrap then we can remove this pragma. #define ERROR_IF(x, error) \ if (x) \ { \ - RE_AST* re_ast = yyget_extra(yyscanner); \ + RE_AST* re_ast = hex_yyget_extra(yyscanner); \ re_ast->error_code = error; \ YYABORT; \ } \ -#line 756 "hex_lexer.c" #define YY_NO_UNISTD_H 1 #define YY_NO_INPUT 1 -#line 760 "hex_lexer.c" + +#line 562 "hex_lexer.c" #define INITIAL 0 #define comment 1 @@ -788,8 +589,8 @@ struct yyguts_t size_t yy_buffer_stack_max; /**< capacity of stack. */ YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */ char yy_hold_char; - int yy_n_chars; - int yyleng_r; + yy_size_t yy_n_chars; + yy_size_t yyleng_r; char *yy_c_buf_p; int yy_init; int yy_start; @@ -811,52 +612,52 @@ struct yyguts_t }; /* end struct yyguts_t */ -static int yy_init_globals ( yyscan_t yyscanner ); +static int yy_init_globals (yyscan_t yyscanner ); /* This must go here because YYSTYPE and YYLTYPE are included * from bison output in section 1.*/ # define yylval yyg->yylval_r -int yylex_init (yyscan_t* scanner); +int hex_yylex_init (yyscan_t* scanner); -int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner); +int hex_yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); /* Accessor methods to globals. These are made visible to non-reentrant scanners for convenience. */ -int yylex_destroy ( yyscan_t yyscanner ); +int hex_yylex_destroy (yyscan_t yyscanner ); -int yyget_debug ( yyscan_t yyscanner ); +int hex_yyget_debug (yyscan_t yyscanner ); -void yyset_debug ( int debug_flag , yyscan_t yyscanner ); +void hex_yyset_debug (int debug_flag ,yyscan_t yyscanner ); -YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner ); +YY_EXTRA_TYPE hex_yyget_extra (yyscan_t yyscanner ); -void yyset_extra ( YY_EXTRA_TYPE user_defined , yyscan_t yyscanner ); +void hex_yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner ); -FILE *yyget_in ( yyscan_t yyscanner ); +FILE *hex_yyget_in (yyscan_t yyscanner ); -void yyset_in ( FILE * _in_str , yyscan_t yyscanner ); +void hex_yyset_in (FILE * _in_str ,yyscan_t yyscanner ); -FILE *yyget_out ( yyscan_t yyscanner ); +FILE *hex_yyget_out (yyscan_t yyscanner ); -void yyset_out ( FILE * _out_str , yyscan_t yyscanner ); +void hex_yyset_out (FILE * _out_str ,yyscan_t yyscanner ); - int yyget_leng ( yyscan_t yyscanner ); +yy_size_t hex_yyget_leng (yyscan_t yyscanner ); -char *yyget_text ( yyscan_t yyscanner ); +char *hex_yyget_text (yyscan_t yyscanner ); -int yyget_lineno ( yyscan_t yyscanner ); +int hex_yyget_lineno (yyscan_t yyscanner ); -void yyset_lineno ( int _line_number , yyscan_t yyscanner ); +void hex_yyset_lineno (int _line_number ,yyscan_t yyscanner ); -int yyget_column ( yyscan_t yyscanner ); +int hex_yyget_column (yyscan_t yyscanner ); -void yyset_column ( int _column_no , yyscan_t yyscanner ); +void hex_yyset_column (int _column_no ,yyscan_t yyscanner ); -YYSTYPE * yyget_lval ( yyscan_t yyscanner ); +YYSTYPE * hex_yyget_lval (yyscan_t yyscanner ); -void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); +void hex_yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner ); /* Macros after this point can all be overridden by user definitions in * section 1. @@ -864,9 +665,9 @@ void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap ( yyscan_t yyscanner ); +extern "C" int hex_yywrap (yyscan_t yyscanner ); #else -extern int yywrap ( yyscan_t yyscanner ); +extern int hex_yywrap (yyscan_t yyscanner ); #endif #endif @@ -875,18 +676,19 @@ extern int yywrap ( yyscan_t yyscanner ); #endif #ifndef yytext_ptr -static void yy_flex_strncpy ( char *, const char *, int , yyscan_t yyscanner); +static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen ( const char * , yyscan_t yyscanner); +static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner); #endif #ifndef YY_NO_INPUT + #ifdef __cplusplus -static int yyinput ( yyscan_t yyscanner ); +static int yyinput (yyscan_t yyscanner ); #else -static int input ( yyscan_t yyscanner ); +static int input (yyscan_t yyscanner ); #endif #endif @@ -906,7 +708,7 @@ static int input ( yyscan_t yyscanner ); /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#define ECHO do { if (fwrite( yytext, yyleng, 1, yyout )) {} } while (0) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, @@ -917,7 +719,7 @@ static int input ( yyscan_t yyscanner ); if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ { \ int c = '*'; \ - int n; \ + size_t n; \ for ( n = 0; n < max_size && \ (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ buf[n] = (char) c; \ @@ -930,7 +732,7 @@ static int input ( yyscan_t yyscanner ); else \ { \ errno=0; \ - while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \ { \ if( errno != EINTR) \ { \ @@ -971,10 +773,10 @@ static int input ( yyscan_t yyscanner ); #ifndef YY_DECL #define YY_DECL_IS_OURS 1 -extern int yylex \ - (YYSTYPE * yylval_param , yyscan_t yyscanner); +extern int hex_yylex \ + (YYSTYPE * yylval_param ,yyscan_t yyscanner); -#define YY_DECL int yylex \ +#define YY_DECL int hex_yylex \ (YYSTYPE * yylval_param , yyscan_t yyscanner) #endif /* !YY_DECL */ @@ -1022,12 +824,12 @@ YY_DECL yyout = stdout; if ( ! YY_CURRENT_BUFFER ) { - yyensure_buffer_stack (yyscanner); + hex_yyensure_buffer_stack (yyscanner); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); + hex_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); } - yy_load_buffer_state( yyscanner ); + hex_yy_load_buffer_state(yyscanner ); } { @@ -1035,7 +837,7 @@ YY_DECL -#line 1038 "hex_lexer.c" +#line 841 "hex_lexer.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1063,9 +865,9 @@ yy_match: { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 34 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } while ( yy_current_state != 33 ); @@ -1079,10 +881,10 @@ yy_find_action: if ( yy_act != YY_END_OF_BUFFER && yy_rule_can_match_eol[yy_act] ) { - int yyl; + yy_size_t yyl; for ( yyl = 0; yyl < yyleng; ++yyl ) if ( yytext[yyl] == '\n' ) - + do{ yylineno++; yycolumn=0; }while(0) @@ -1243,7 +1045,7 @@ YY_RULE_SETUP #line 182 "hex_lexer.l" ECHO; YY_BREAK -#line 1246 "hex_lexer.c" +#line 1049 "hex_lexer.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(comment): case YY_STATE_EOF(range): @@ -1263,7 +1065,7 @@ case YY_STATE_EOF(range): /* We're scanning a new file or input source. It's * possible that this happened because the user * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure + * hex_yylex(). If so, then we have to assure * consistency between YY_CURRENT_BUFFER and our * globals. Here is the right place to do so, because * this is the first action (other than possibly a @@ -1324,7 +1126,7 @@ case YY_STATE_EOF(range): { yyg->yy_did_buffer_switch_on_eof = 0; - if ( yywrap( yyscanner ) ) + if ( hex_yywrap(yyscanner ) ) { /* Note: because we've taken care in * yy_get_next_buffer() to have set up @@ -1378,7 +1180,7 @@ case YY_STATE_EOF(range): } /* end of action switch */ } /* end of scanning one token */ } /* end of user's declarations */ -} /* end of yylex */ +} /* end of hex_yylex */ /* yy_get_next_buffer - try to read in a new buffer * @@ -1392,7 +1194,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; char *source = yyg->yytext_ptr; - int number_to_move, i; + yy_size_t number_to_move, i; int ret_val; if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] ) @@ -1421,7 +1223,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) /* Try to read more data. */ /* First move last chars to start of buffer. */ - number_to_move = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr - 1); + number_to_move = (yy_size_t) (yyg->yy_c_buf_p - yyg->yytext_ptr) - 1; for ( i = 0; i < number_to_move; ++i ) *(dest++) = *(source++); @@ -1434,7 +1236,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else { - int num_to_read = + yy_size_t num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; while ( num_to_read <= 0 ) @@ -1448,7 +1250,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) if ( b->yy_is_our_buffer ) { - int new_size = b->yy_buf_size * 2; + yy_size_t new_size = b->yy_buf_size * 2; if ( new_size <= 0 ) b->yy_buf_size += b->yy_buf_size / 8; @@ -1457,12 +1259,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) b->yy_ch_buf = (char *) /* Include room in for 2 EOB chars. */ - yyrealloc( (void *) b->yy_ch_buf, - (yy_size_t) (b->yy_buf_size + 2) , yyscanner ); + hex_yyrealloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ,yyscanner ); } else /* Can't grow it, we don't own it. */ - b->yy_ch_buf = NULL; + b->yy_ch_buf = 0; if ( ! b->yy_ch_buf ) YY_FATAL_ERROR( @@ -1490,7 +1291,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) if ( number_to_move == YY_MORE_ADJ ) { ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin , yyscanner); + hex_yyrestart(yyin ,yyscanner); } else @@ -1504,15 +1305,12 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else ret_val = EOB_ACT_CONTINUE_SCAN; - if ((yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + if ((yy_size_t) (yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { /* Extend the array by 50%, plus the number we really need. */ - int new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( - (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size , yyscanner ); + yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) hex_yyrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ,yyscanner ); if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); - /* "- 2" to take care of EOB's */ - YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); } yyg->yy_n_chars += number_to_move; @@ -1546,9 +1344,9 @@ static int yy_get_next_buffer (yyscan_t yyscanner) { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 34 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; } return yy_current_state; @@ -1575,9 +1373,9 @@ static int yy_get_next_buffer (yyscan_t yyscanner) { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 34 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; yy_is_jam = (yy_current_state == 33); (void)yyg; @@ -1613,7 +1411,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else { /* need more input */ - int offset = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr); + yy_size_t offset = yyg->yy_c_buf_p - yyg->yytext_ptr; ++yyg->yy_c_buf_p; switch ( yy_get_next_buffer( yyscanner ) ) @@ -1630,14 +1428,14 @@ static int yy_get_next_buffer (yyscan_t yyscanner) */ /* Reset buffer status. */ - yyrestart( yyin , yyscanner); + hex_yyrestart(yyin ,yyscanner); /*FALLTHROUGH*/ case EOB_ACT_END_OF_FILE: { - if ( yywrap( yyscanner ) ) - return 0; + if ( hex_yywrap(yyscanner ) ) + return EOF; if ( ! yyg->yy_did_buffer_switch_on_eof ) YY_NEW_FILE; @@ -1660,7 +1458,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) yyg->yy_hold_char = *++yyg->yy_c_buf_p; if ( c == '\n' ) - + do{ yylineno++; yycolumn=0; }while(0) @@ -1675,34 +1473,34 @@ static int yy_get_next_buffer (yyscan_t yyscanner) * @param yyscanner The scanner object. * @note This function does not reset the start condition to @c INITIAL . */ - void yyrestart (FILE * input_file , yyscan_t yyscanner) + void hex_yyrestart (FILE * input_file , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if ( ! YY_CURRENT_BUFFER ){ - yyensure_buffer_stack (yyscanner); + hex_yyensure_buffer_stack (yyscanner); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); + hex_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); } - yy_init_buffer( YY_CURRENT_BUFFER, input_file , yyscanner); - yy_load_buffer_state( yyscanner ); + hex_yy_init_buffer(YY_CURRENT_BUFFER,input_file ,yyscanner); + hex_yy_load_buffer_state(yyscanner ); } /** Switch to a different input buffer. * @param new_buffer The new input buffer. * @param yyscanner The scanner object. */ - void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) + void hex_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* TODO. We should be able to replace this entire function body * with - * yypop_buffer_state(); - * yypush_buffer_state(new_buffer); + * hex_yypop_buffer_state(); + * hex_yypush_buffer_state(new_buffer); */ - yyensure_buffer_stack (yyscanner); + hex_yyensure_buffer_stack (yyscanner); if ( YY_CURRENT_BUFFER == new_buffer ) return; @@ -1715,17 +1513,17 @@ static int yy_get_next_buffer (yyscan_t yyscanner) } YY_CURRENT_BUFFER_LVALUE = new_buffer; - yy_load_buffer_state( yyscanner ); + hex_yy_load_buffer_state(yyscanner ); /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe + * EOF (hex_yywrap()) processing, but the only time this flag + * is looked at is after hex_yywrap() is called, so it's safe * to go ahead and always set it. */ yyg->yy_did_buffer_switch_on_eof = 1; } -static void yy_load_buffer_state (yyscan_t yyscanner) +static void hex_yy_load_buffer_state (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; @@ -1740,35 +1538,35 @@ static void yy_load_buffer_state (yyscan_t yyscanner) * @param yyscanner The scanner object. * @return the allocated buffer state. */ - YY_BUFFER_STATE yy_create_buffer (FILE * file, int size , yyscan_t yyscanner) + YY_BUFFER_STATE hex_yy_create_buffer (FILE * file, int size , yyscan_t yyscanner) { YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); + b = (YY_BUFFER_STATE) hex_yyalloc(sizeof( struct yy_buffer_state ) ,yyscanner ); if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in hex_yy_create_buffer()" ); - b->yy_buf_size = size; + b->yy_buf_size = (yy_size_t)size; /* yy_ch_buf has to be 2 characters longer than the size given because * we need to put in 2 end-of-buffer characters. */ - b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) , yyscanner ); + b->yy_ch_buf = (char *) hex_yyalloc(b->yy_buf_size + 2 ,yyscanner ); if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in hex_yy_create_buffer()" ); b->yy_is_our_buffer = 1; - yy_init_buffer( b, file , yyscanner); + hex_yy_init_buffer(b,file ,yyscanner); return b; } /** Destroy the buffer. - * @param b a buffer created with yy_create_buffer() + * @param b a buffer created with hex_yy_create_buffer() * @param yyscanner The scanner object. */ - void yy_delete_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) + void hex_yy_delete_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; @@ -1779,28 +1577,28 @@ static void yy_load_buffer_state (yyscan_t yyscanner) YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; if ( b->yy_is_our_buffer ) - yyfree( (void *) b->yy_ch_buf , yyscanner ); + hex_yyfree((void *) b->yy_ch_buf ,yyscanner ); - yyfree( (void *) b , yyscanner ); + hex_yyfree((void *) b ,yyscanner ); } /* Initializes or reinitializes a buffer. * This function is sometimes called more than once on the same buffer, - * such as during a yyrestart() or at EOF. + * such as during a hex_yyrestart() or at EOF. */ - static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file , yyscan_t yyscanner) + static void hex_yy_init_buffer (YY_BUFFER_STATE b, FILE * file , yyscan_t yyscanner) { int oerrno = errno; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yy_flush_buffer( b , yyscanner); + hex_yy_flush_buffer(b ,yyscanner); b->yy_input_file = file; b->yy_fill_buffer = 1; - /* If b is the current buffer, then yy_init_buffer was _probably_ - * called from yyrestart() or through yy_get_next_buffer. + /* If b is the current buffer, then hex_yy_init_buffer was _probably_ + * called from hex_yyrestart() or through yy_get_next_buffer. * In that case, we don't want to reset the lineno or column. */ if (b != YY_CURRENT_BUFFER){ @@ -1817,7 +1615,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. * @param yyscanner The scanner object. */ - void yy_flush_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) + void hex_yy_flush_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if ( ! b ) @@ -1838,7 +1636,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) b->yy_buffer_status = YY_BUFFER_NEW; if ( b == YY_CURRENT_BUFFER ) - yy_load_buffer_state( yyscanner ); + hex_yy_load_buffer_state(yyscanner ); } /** Pushes the new state onto the stack. The new state becomes @@ -1847,15 +1645,15 @@ static void yy_load_buffer_state (yyscan_t yyscanner) * @param new_buffer The new state. * @param yyscanner The scanner object. */ -void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) +void hex_yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if (new_buffer == NULL) return; - yyensure_buffer_stack(yyscanner); + hex_yyensure_buffer_stack(yyscanner); - /* This block is copied from yy_switch_to_buffer. */ + /* This block is copied from hex_yy_switch_to_buffer. */ if ( YY_CURRENT_BUFFER ) { /* Flush out information for old buffer. */ @@ -1869,8 +1667,8 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) yyg->yy_buffer_stack_top++; YY_CURRENT_BUFFER_LVALUE = new_buffer; - /* copied from yy_switch_to_buffer. */ - yy_load_buffer_state( yyscanner ); + /* copied from hex_yy_switch_to_buffer. */ + hex_yy_load_buffer_state(yyscanner ); yyg->yy_did_buffer_switch_on_eof = 1; } @@ -1878,19 +1676,19 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) * The next element becomes the new top. * @param yyscanner The scanner object. */ -void yypop_buffer_state (yyscan_t yyscanner) +void hex_yypop_buffer_state (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if (!YY_CURRENT_BUFFER) return; - yy_delete_buffer(YY_CURRENT_BUFFER , yyscanner); + hex_yy_delete_buffer(YY_CURRENT_BUFFER ,yyscanner); YY_CURRENT_BUFFER_LVALUE = NULL; if (yyg->yy_buffer_stack_top > 0) --yyg->yy_buffer_stack_top; if (YY_CURRENT_BUFFER) { - yy_load_buffer_state( yyscanner ); + hex_yy_load_buffer_state(yyscanner ); yyg->yy_did_buffer_switch_on_eof = 1; } } @@ -1898,7 +1696,7 @@ void yypop_buffer_state (yyscan_t yyscanner) /* Allocates the stack if it does not exist. * Guarantees space for at least one push. */ -static void yyensure_buffer_stack (yyscan_t yyscanner) +static void hex_yyensure_buffer_stack (yyscan_t yyscanner) { yy_size_t num_to_alloc; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; @@ -1909,15 +1707,15 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) * scanner will even need a stack. We use 2 instead of 1 to avoid an * immediate realloc on the next call. */ - num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ - yyg->yy_buffer_stack = (struct yy_buffer_state**)yyalloc + num_to_alloc = 1; // After all that talk, this was set to 1 anyways... + yyg->yy_buffer_stack = (struct yy_buffer_state**)hex_yyalloc (num_to_alloc * sizeof(struct yy_buffer_state*) , yyscanner); if ( ! yyg->yy_buffer_stack ) - YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); - + YY_FATAL_ERROR( "out of dynamic memory in hex_yyensure_buffer_stack()" ); + memset(yyg->yy_buffer_stack, 0, num_to_alloc * sizeof(struct yy_buffer_state*)); - + yyg->yy_buffer_stack_max = num_to_alloc; yyg->yy_buffer_stack_top = 0; return; @@ -1929,12 +1727,12 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) yy_size_t grow_size = 8 /* arbitrary grow size */; num_to_alloc = yyg->yy_buffer_stack_max + grow_size; - yyg->yy_buffer_stack = (struct yy_buffer_state**)yyrealloc + yyg->yy_buffer_stack = (struct yy_buffer_state**)hex_yyrealloc (yyg->yy_buffer_stack, num_to_alloc * sizeof(struct yy_buffer_state*) , yyscanner); if ( ! yyg->yy_buffer_stack ) - YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + YY_FATAL_ERROR( "out of dynamic memory in hex_yyensure_buffer_stack()" ); /* zero only the new slots.*/ memset(yyg->yy_buffer_stack + yyg->yy_buffer_stack_max, 0, grow_size * sizeof(struct yy_buffer_state*)); @@ -1946,9 +1744,9 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) * @param base the character buffer * @param size the size in bytes of the character buffer * @param yyscanner The scanner object. - * @return the newly allocated buffer state object. + * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) +YY_BUFFER_STATE hex_yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) { YY_BUFFER_STATE b; @@ -1956,69 +1754,69 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann base[size-2] != YY_END_OF_BUFFER_CHAR || base[size-1] != YY_END_OF_BUFFER_CHAR ) /* They forgot to leave room for the EOB's. */ - return NULL; + return 0; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); + b = (YY_BUFFER_STATE) hex_yyalloc(sizeof( struct yy_buffer_state ) ,yyscanner ); if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in hex_yy_scan_buffer()" ); - b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ b->yy_buf_pos = b->yy_ch_buf = base; b->yy_is_our_buffer = 0; - b->yy_input_file = NULL; + b->yy_input_file = 0; b->yy_n_chars = b->yy_buf_size; b->yy_is_interactive = 0; b->yy_at_bol = 1; b->yy_fill_buffer = 0; b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b , yyscanner ); + hex_yy_switch_to_buffer(b ,yyscanner ); return b; } -/** Setup the input buffer state to scan a string. The next call to yylex() will +/** Setup the input buffer state to scan a string. The next call to hex_yylex() will * scan from a @e copy of @a str. * @param yystr a NUL-terminated string to scan * @param yyscanner The scanner object. * @return the newly allocated buffer state object. * @note If you want to scan bytes that may contain NUL values, then use - * yy_scan_bytes() instead. + * hex_yy_scan_bytes() instead. */ -YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner) +YY_BUFFER_STATE hex_yy_scan_string (yyconst char * yystr , yyscan_t yyscanner) { - return yy_scan_bytes( yystr, (int) strlen(yystr) , yyscanner); + return hex_yy_scan_bytes(yystr,strlen(yystr) ,yyscanner); } -/** Setup the input buffer state to scan the given bytes. The next call to yylex() will +/** Setup the input buffer state to scan the given bytes. The next call to hex_yylex() will * scan from a @e copy of @a bytes. * @param yybytes the byte buffer to scan * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. * @param yyscanner The scanner object. * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan_t yyscanner) +YY_BUFFER_STATE hex_yy_scan_bytes (yyconst char * yybytes, yy_size_t _yybytes_len , yyscan_t yyscanner) { YY_BUFFER_STATE b; char *buf; yy_size_t n; - int i; + yy_size_t i; /* Get memory for full buffer, including space for trailing EOB's. */ - n = (yy_size_t) (_yybytes_len + 2); - buf = (char *) yyalloc( n , yyscanner ); + n = _yybytes_len + 2; + buf = (char *) hex_yyalloc(n ,yyscanner ); if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + YY_FATAL_ERROR( "out of dynamic memory in hex_yy_scan_bytes()" ); for ( i = 0; i < _yybytes_len; ++i ) buf[i] = yybytes[i]; buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n , yyscanner); + b = hex_yy_scan_buffer(buf,n ,yyscanner); if ( ! b ) - YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + YY_FATAL_ERROR( "bad buffer in hex_yy_scan_bytes()" ); /* It's okay to grow etc. this buffer, and we should throw it * away when we're done. @@ -2032,11 +1830,11 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan #define YY_EXIT_FAILURE 2 #endif -static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner) +static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; - fprintf( stderr, "%s\n", msg ); + (void) fprintf( stderr, "%s\n", msg ); exit( YY_EXIT_FAILURE ); } @@ -2062,7 +1860,7 @@ static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner) /** Get the user-defined data for this scanner. * @param yyscanner The scanner object. */ -YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner) +YY_EXTRA_TYPE hex_yyget_extra (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyextra; @@ -2071,10 +1869,10 @@ YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner) /** Get the current line number. * @param yyscanner The scanner object. */ -int yyget_lineno (yyscan_t yyscanner) +int hex_yyget_lineno (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - + if (! YY_CURRENT_BUFFER) return 0; @@ -2084,10 +1882,10 @@ int yyget_lineno (yyscan_t yyscanner) /** Get the current column number. * @param yyscanner The scanner object. */ -int yyget_column (yyscan_t yyscanner) +int hex_yyget_column (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - + if (! YY_CURRENT_BUFFER) return 0; @@ -2097,7 +1895,7 @@ int yyget_column (yyscan_t yyscanner) /** Get the input stream. * @param yyscanner The scanner object. */ -FILE *yyget_in (yyscan_t yyscanner) +FILE *hex_yyget_in (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyin; @@ -2106,7 +1904,7 @@ FILE *yyget_in (yyscan_t yyscanner) /** Get the output stream. * @param yyscanner The scanner object. */ -FILE *yyget_out (yyscan_t yyscanner) +FILE *hex_yyget_out (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyout; @@ -2115,7 +1913,7 @@ FILE *yyget_out (yyscan_t yyscanner) /** Get the length of the current token. * @param yyscanner The scanner object. */ -int yyget_leng (yyscan_t yyscanner) +yy_size_t hex_yyget_leng (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyleng; @@ -2125,7 +1923,7 @@ int yyget_leng (yyscan_t yyscanner) * @param yyscanner The scanner object. */ -char *yyget_text (yyscan_t yyscanner) +char *hex_yyget_text (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yytext; @@ -2135,7 +1933,7 @@ char *yyget_text (yyscan_t yyscanner) * @param user_defined The data to be associated with this scanner. * @param yyscanner The scanner object. */ -void yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) +void hex_yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyextra = user_defined ; @@ -2145,13 +1943,13 @@ void yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) * @param _line_number line number * @param yyscanner The scanner object. */ -void yyset_lineno (int _line_number , yyscan_t yyscanner) +void hex_yyset_lineno (int _line_number , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* lineno is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) - YY_FATAL_ERROR( "yyset_lineno called with no buffer" ); + YY_FATAL_ERROR( "hex_yyset_lineno called with no buffer" ); yylineno = _line_number; } @@ -2160,13 +1958,13 @@ void yyset_lineno (int _line_number , yyscan_t yyscanner) * @param _column_no column number * @param yyscanner The scanner object. */ -void yyset_column (int _column_no , yyscan_t yyscanner) +void hex_yyset_column (int _column_no , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* column is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) - YY_FATAL_ERROR( "yyset_column called with no buffer" ); + YY_FATAL_ERROR( "hex_yyset_column called with no buffer" ); yycolumn = _column_no; } @@ -2175,27 +1973,27 @@ void yyset_column (int _column_no , yyscan_t yyscanner) * input buffer. * @param _in_str A readable stream. * @param yyscanner The scanner object. - * @see yy_switch_to_buffer + * @see hex_yy_switch_to_buffer */ -void yyset_in (FILE * _in_str , yyscan_t yyscanner) +void hex_yyset_in (FILE * _in_str , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyin = _in_str ; } -void yyset_out (FILE * _out_str , yyscan_t yyscanner) +void hex_yyset_out (FILE * _out_str , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyout = _out_str ; } -int yyget_debug (yyscan_t yyscanner) +int hex_yyget_debug (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yy_flex_debug; } -void yyset_debug (int _bdebug , yyscan_t yyscanner) +void hex_yyset_debug (int _bdebug , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yy_flex_debug = _bdebug ; @@ -2203,13 +2001,13 @@ void yyset_debug (int _bdebug , yyscan_t yyscanner) /* Accessor methods for yylval and yylloc */ -YYSTYPE * yyget_lval (yyscan_t yyscanner) +YYSTYPE * hex_yyget_lval (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yylval; } -void yyset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) +void hex_yyset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yylval = yylval_param; @@ -2217,18 +2015,20 @@ void yyset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) /* User-visible API */ -/* yylex_init is special because it creates the scanner itself, so it is +/* hex_yylex_init is special because it creates the scanner itself, so it is * the ONLY reentrant function that doesn't take the scanner as the last argument. * That's why we explicitly handle the declaration, instead of using our macros. */ -int yylex_init(yyscan_t* ptr_yy_globals) + +int hex_yylex_init(yyscan_t* ptr_yy_globals) + { if (ptr_yy_globals == NULL){ errno = EINVAL; return 1; } - *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), NULL ); + *ptr_yy_globals = (yyscan_t) hex_yyalloc ( sizeof( struct yyguts_t ), NULL ); if (*ptr_yy_globals == NULL){ errno = ENOMEM; @@ -2241,37 +2041,39 @@ int yylex_init(yyscan_t* ptr_yy_globals) return yy_init_globals ( *ptr_yy_globals ); } -/* yylex_init_extra has the same functionality as yylex_init, but follows the +/* hex_yylex_init_extra has the same functionality as hex_yylex_init, but follows the * convention of taking the scanner as the last argument. Note however, that * this is a *pointer* to a scanner, as it will be allocated by this call (and * is the reason, too, why this function also must handle its own declaration). - * The user defined value in the first argument will be available to yyalloc in + * The user defined value in the first argument will be available to hex_yyalloc in * the yyextra field. */ -int yylex_init_extra( YY_EXTRA_TYPE yy_user_defined, yyscan_t* ptr_yy_globals ) + +int hex_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ) + { struct yyguts_t dummy_yyguts; - yyset_extra (yy_user_defined, &dummy_yyguts); + hex_yyset_extra (yy_user_defined, &dummy_yyguts); if (ptr_yy_globals == NULL){ errno = EINVAL; return 1; } - - *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), &dummy_yyguts ); - + + *ptr_yy_globals = (yyscan_t) hex_yyalloc ( sizeof( struct yyguts_t ), &dummy_yyguts ); + if (*ptr_yy_globals == NULL){ errno = ENOMEM; return 1; } - + /* By setting to 0xAA, we expose bugs in yy_init_globals. Leave at 0x00 for releases. */ memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t)); - - yyset_extra (yy_user_defined, *ptr_yy_globals); - + + hex_yyset_extra (yy_user_defined, *ptr_yy_globals); + return yy_init_globals ( *ptr_yy_globals ); } @@ -2279,13 +2081,13 @@ static int yy_init_globals (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* Initialization is the same as for the non-reentrant scanner. - * This function is called from yylex_destroy(), so don't allocate here. + * This function is called from hex_yylex_destroy(), so don't allocate here. */ - yyg->yy_buffer_stack = NULL; + yyg->yy_buffer_stack = 0; yyg->yy_buffer_stack_top = 0; yyg->yy_buffer_stack_max = 0; - yyg->yy_c_buf_p = NULL; + yyg->yy_c_buf_p = (char *) 0; yyg->yy_init = 0; yyg->yy_start = 0; @@ -2298,42 +2100,42 @@ static int yy_init_globals (yyscan_t yyscanner) yyin = stdin; yyout = stdout; #else - yyin = NULL; - yyout = NULL; + yyin = (FILE *) 0; + yyout = (FILE *) 0; #endif /* For future reference: Set errno on error, since we are called by - * yylex_init() + * hex_yylex_init() */ return 0; } -/* yylex_destroy is for both reentrant and non-reentrant scanners. */ -int yylex_destroy (yyscan_t yyscanner) +/* hex_yylex_destroy is for both reentrant and non-reentrant scanners. */ +int hex_yylex_destroy (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* Pop the buffer stack, destroying each element. */ while(YY_CURRENT_BUFFER){ - yy_delete_buffer( YY_CURRENT_BUFFER , yyscanner ); + hex_yy_delete_buffer(YY_CURRENT_BUFFER ,yyscanner ); YY_CURRENT_BUFFER_LVALUE = NULL; - yypop_buffer_state(yyscanner); + hex_yypop_buffer_state(yyscanner); } /* Destroy the stack itself. */ - yyfree(yyg->yy_buffer_stack , yyscanner); + hex_yyfree(yyg->yy_buffer_stack ,yyscanner); yyg->yy_buffer_stack = NULL; /* Destroy the start condition stack. */ - yyfree( yyg->yy_start_stack , yyscanner ); + hex_yyfree(yyg->yy_start_stack ,yyscanner ); yyg->yy_start_stack = NULL; /* Reset the globals. This is important in a non-reentrant scanner so the next time - * yylex() is called, initialization will occur. */ + * hex_yylex() is called, initialization will occur. */ yy_init_globals( yyscanner); /* Destroy the main struct (reentrant only). */ - yyfree ( yyscanner , yyscanner ); + hex_yyfree ( yyscanner , yyscanner ); yyscanner = NULL; return 0; } @@ -2343,7 +2145,7 @@ int yylex_destroy (yyscan_t yyscanner) */ #ifndef yytext_ptr -static void yy_flex_strncpy (char* s1, const char * s2, int n , yyscan_t yyscanner) +static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; @@ -2355,7 +2157,7 @@ static void yy_flex_strncpy (char* s1, const char * s2, int n , yyscan_t yyscann #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen (const char * s , yyscan_t yyscanner) +static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner) { int n; for ( n = 0; s[n]; ++n ) @@ -2365,14 +2167,14 @@ static int yy_flex_strlen (const char * s , yyscan_t yyscanner) } #endif -void *yyalloc (yy_size_t size , yyscan_t yyscanner) +void *hex_yyalloc (yy_size_t size , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; - return malloc(size); + return (void *) malloc( size ); } -void *yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) +void *hex_yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; @@ -2384,14 +2186,14 @@ void *yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) * any pointer type to void*, and deal with argument conversions * as though doing an assignment. */ - return realloc(ptr, size); + return (void *) realloc( (char *) ptr, size ); } -void yyfree (void * ptr , yyscan_t yyscanner) +void hex_yyfree (void * ptr , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; - free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ + free( (char *) ptr ); /* see hex_yyrealloc() for (char *) cast */ } #define YYTABLES_NAME "yytables" @@ -2399,14 +2201,22 @@ void yyfree (void * ptr , yyscan_t yyscanner) #line 182 "hex_lexer.l" + +// +// yyfatal (actually named hex_yyfatal because of the '%option prefix="hex_yy"' +// directive) is called when a fatal error occurs in the parser. When this +// happens we are deep inside the parsing logic generated by flex/bison and +// the only way to exit gracefully from there is using setjmp/longjmp. +// void yyfatal( yyscan_t yyscanner, const char *error_message) { - jmp_buf* recovery_state = (jmp_buf*) yr_thread_storage_get_value( - &yr_recovery_state_key); + jmp_buf* recovery_trampoline = (jmp_buf*) yr_thread_storage_get_value( + &yr_yyfatal_trampoline_tls); - longjmp(*recovery_state, 1); + // Never returns. + longjmp(*recovery_trampoline, 1); } void yyerror( @@ -2437,15 +2247,17 @@ int yr_parse_hex_string( RE_ERROR* error) { yyscan_t yyscanner; - jmp_buf recovery_state; + jmp_buf recovery_trampoline; HEX_LEX_ENVIRONMENT lex_env; lex_env.last_error = ERROR_SUCCESS; lex_env.inside_or = 0; - yr_thread_storage_set_value(&yr_recovery_state_key, &recovery_state); + yr_thread_storage_set_value( + &yr_yyfatal_trampoline_tls, + &recovery_trampoline); - if (setjmp(recovery_state) != 0) + if (setjmp(recovery_trampoline) != 0) return ERROR_INTERNAL_FATAL_ERROR; FAIL_ON_ERROR(yr_re_ast_create(re_ast)); @@ -2465,11 +2277,11 @@ int yr_parse_hex_string( (*re_ast)->flags |= RE_FLAGS_DOT_ALL; - yylex_init(&yyscanner); - yyset_extra(*re_ast, yyscanner); - yy_scan_string(hex_string, yyscanner); + hex_yylex_init(&yyscanner); + hex_yyset_extra(*re_ast,yyscanner); + hex_yy_scan_string(hex_string,yyscanner); yyparse(yyscanner, &lex_env); - yylex_destroy(yyscanner); + hex_yylex_destroy(yyscanner); if (lex_env.last_error != ERROR_SUCCESS) { diff --git a/libyara/hex_lexer.l b/libyara/hex_lexer.l index 8f2887fa..ce2b805f 100644 --- a/libyara/hex_lexer.l +++ b/libyara/hex_lexer.l @@ -181,14 +181,21 @@ hexdigit [a-fA-F0-9] %% +// +// yyfatal (actually named hex_yyfatal because of the '%option prefix="hex_yy"' +// directive) is called when a fatal error occurs in the parser. When this +// happens we are deep inside the parsing logic generated by flex/bison and +// the only way to exit gracefully from there is using setjmp/longjmp. +// void yyfatal( yyscan_t yyscanner, const char *error_message) { - jmp_buf* recovery_state = (jmp_buf*) yr_thread_storage_get_value( - &yr_recovery_state_key); + jmp_buf* recovery_trampoline = (jmp_buf*) yr_thread_storage_get_value( + &yr_yyfatal_trampoline_tls); - longjmp(*recovery_state, 1); + // Never returns. + longjmp(*recovery_trampoline, 1); } void yyerror( @@ -219,15 +226,17 @@ int yr_parse_hex_string( RE_ERROR* error) { yyscan_t yyscanner; - jmp_buf recovery_state; + jmp_buf recovery_trampoline; HEX_LEX_ENVIRONMENT lex_env; lex_env.last_error = ERROR_SUCCESS; lex_env.inside_or = 0; - yr_thread_storage_set_value(&yr_recovery_state_key, &recovery_state); + yr_thread_storage_set_value( + &yr_yyfatal_trampoline_tls, + &recovery_trampoline); - if (setjmp(recovery_state) != 0) + if (setjmp(recovery_trampoline) != 0) return ERROR_INTERNAL_FATAL_ERROR; FAIL_ON_ERROR(yr_re_ast_create(re_ast)); diff --git a/libyara/include/yara/ahocorasick.h b/libyara/include/yara/ahocorasick.h index 43f6dadd..5ccd4c45 100644 --- a/libyara/include/yara/ahocorasick.h +++ b/libyara/include/yara/ahocorasick.h @@ -62,15 +62,14 @@ int yr_ac_automaton_destroy( int yr_ac_add_string( YR_AC_AUTOMATON* automaton, YR_STRING* string, + uint32_t string_idx, YR_ATOM_LIST_ITEM* atom, - YR_ARENA* matches_arena); + YR_ARENA* arena); int yr_ac_compile( YR_AC_AUTOMATON* automaton, - YR_ARENA* arena, - YR_AC_TABLES* tables); - + YR_ARENA* arena); void yr_ac_print_automaton( YR_AC_AUTOMATON* automaton); diff --git a/libyara/include/yara/arena.h b/libyara/include/yara/arena.h index 64f4f7a0..16019762 100644 --- a/libyara/include/yara/arena.h +++ b/libyara/include/yara/arena.h @@ -1,8 +1,8 @@ /* -Copyright (c) 2013. The YARA Authors. All Rights Reserved. +Copyright (c) 2020. The YARA Authors. All Rights Reserved. Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: + are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. @@ -12,18 +12,18 @@ this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without + may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ @@ -31,141 +31,241 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define YR_ARENA_H #include - +#include #include #include -// Indicated that the arena is self-contained and stored in a single page. A -// self-contained arenas is one that doesn't contains any pointers to outside -// data. All pointers in a self-contained arena points at some address within -// the arena. -#define ARENA_FLAGS_COALESCED 1 +#define EOL ((uint32_t) -1) -// If an arena is created with this flag is keeps a list of YR_RELOC structures -// tracking the pointers stored within the arena. When the arena is relocated -// this allows to fix those pointers that pointed to some address within the -// relocated arena. -#define ARENA_FLAGS_RELOCATABLE 2 +#define YR_ARENA_FILE_VERSION 17 -// YARA can read an arena file (which contain compiled rules) created by a -// a different version of YARA as long as the arena file version remains the -// same. When changes in the structure of arena files are introduced, this -// version is bumped up. -// -// The arena file version is defined as ((version << 16) | YR_MAX_THREADS) so -// that the lowest 16 bits contain the value for YR_MAX_THREADS. Arena files -// generated with a YARA compiled with a certain value for YR_MAX_THREADS will -// be rejected by another YARA compiled with a different value, even if the -// "version" number is the same. -#define ARENA_FILE_VERSION ((23 << 16) | YR_MAX_THREADS) +#define YR_ARENA_NULL_REF \ + (YR_ARENA_REF){ UINT32_MAX, UINT32_MAX } -#define EOL ((size_t) -1) +#define YR_ARENA_IS_NULL_REF(ref) \ + (memcmp(&(ref), &YR_ARENA_NULL_REF, sizeof(YR_ARENA_NULL_REF)) == 0) +typedef uint32_t yr_arena_off_t; -typedef struct _YR_RELOC +typedef struct YR_ARENA YR_ARENA; +typedef struct YR_ARENA_BUFFER YR_ARENA_BUFFER; +typedef struct YR_ARENA_REF YR_ARENA_REF; +typedef struct YR_RELOC YR_RELOC; + +// The YR_ARENA_REF must be 64-bits long because this structure occupies +// the place of relocatable pointers when arenas are saved to file. The pack(4) +// directive ensures that fields are aligned to 4-byte boundaries and the total +// size of the structure is 8 bytes. When compiling YARA for 32-bits, pointers +// are 4 bytes but this structure is still 8 bytes. For that reason structures +// that are saved in arenas must declare pointers using the DECLARE_REFERENCE +// macro, which ensures that 8 bytes are reserved for the pointer no matter if +// they are 32-bits pointers. + +#pragma pack(push) +#pragma pack(4) + +struct YR_ARENA_REF { + uint32_t buffer_id; uint32_t offset; - struct _YR_RELOC* next; +}; -} YR_RELOC; +#pragma pack(pop) -typedef struct _YR_ARENA_PAGE +struct YR_ARENA_BUFFER { - uint8_t* new_address; - uint8_t* address; + // Pointer the buffer's data. + uint8_t* data; + // Total buffer size, including the used and unused areas. size_t size; + + // Number of bytes that are actually used (equal to or lower than size). size_t used; - - YR_RELOC* reloc_list_head; - YR_RELOC* reloc_list_tail; - - struct _YR_ARENA_PAGE* next; - struct _YR_ARENA_PAGE* prev; - -} YR_ARENA_PAGE; +}; -typedef struct _YR_ARENA +struct YR_RELOC { - int flags; + // Buffer ID associated to this relocation entry. + uint8_t buffer_id; - YR_ARENA_PAGE* page_list_head; - YR_ARENA_PAGE* current_page; + // Offset within the buffer where the relocatable pointer resides. + yr_arena_off_t offset; -} YR_ARENA; + // Pointer to the next entry in the list. + struct YR_RELOC* next; +}; +/* +YR_ARENA is the structure that represents an arena. + +An arena is a set of buffers in which you can write arbitrary data. When the +arena is created you can specify how many buffers it will have, this number is +limited by YR_MAX_ARENA_BUFFERS. Buffers are numbered 0, 1, 2, and so on, and +they are referenced by its number. + +Initially each buffer will have a size that is determined by an argument +passed to yr_create_arena, but the size will grow automatically as you write +data into it. When the buffer is re-sized it may be moved to a different +location in memory, which means that any pointer you hold to data contained +in the buffer is valid only until the next call to a function that may +cause a buffer re-sizing. For long-lived references to data in a buffer +you should use a YR_ARENA_REF, which is simply a struct storing the buffer +number and the offset of the data relative to the beginning of the buffer. +Such YR_ARENA_REF structures are returned by all functions that allocate space +or write data in a buffer. + +The yr_arena_get_ptr function can be used for getting a pointer to data +stored in the buffer given its offset, but this pointer should be used with +only in a limited scope where you can be sure that your program is not calling +a function that may cause a buffer re-sizing, like yr_arena_allocate_xxx and +yr_arena_write_xxx. + +Arenas can also keep track pointers stored in some of its buffers that point to +some location within the arena (it can be to the same buffer or a different one +within the same arena). This is done with yr_arena_make_ptr_relocatable, this +function receives a buffer number and an offset, and it tells the arena that +the pointer stored at that particular location of the buffer must be adjusted +every time the memory it points to is moved around. + +The yr_arena_allocate_struct function is an even more powerful mechanism for +tracking relocatable pointers. This function allows to allocate space in a +buffer for a structure, while at the same time telling the arena which fields +in the structure are relocatable pointers. + +*/ + +struct YR_ARENA +{ + // Number of users of this arena. This is set to one when the arena is created, + // and can be incremented by calling yr_arena_acquire. On each call + // to yr_arena_release it gets decremented by one, if xrefs reaches zero + // the buffers and the YR_ARENA structures are freed. + int xrefs; + + // Number of buffers in this arena. + int num_buffers; + + // Status of individual buffers. + YR_ARENA_BUFFER buffers[YR_MAX_ARENA_BUFFERS]; + + // Initial size for each buffer. + size_t initial_buffer_size; + + // Head of the list containing relocation entries. + YR_RELOC* reloc_list_head; + + // Tail of the list containing relocation entries. + YR_RELOC* reloc_list_tail; +}; + + +// Creates an arena with the specified number of buffers and takes ownership of +// it. Initially each buffer is empty, the first time that some data is written +// into a buffer at least initial_buffer_size are reserved for the buffer. int yr_arena_create( - size_t initial_size, - int flags, + int num_buffers, + size_t initial_buffer_size, YR_ARENA** arena); -void yr_arena_destroy( +// Takes ownership of the arena. +void yr_arena_acquire( YR_ARENA* arena); -void* yr_arena_base_address( +// Release ownership of the arena. If the number of owners drops to zero the +// arena is destroyed and all its resources are freed. +int yr_arena_release( YR_ARENA* arena); -YR_ARENA_PAGE* yr_arena_page_for_address( +// Given a reference to some data within the arena, it returns a pointer to +// the data. This pointer is valid only until the next call to any of the +// functions that allocates space in the buffer where the data resides, like +// yr_arena_allocate_xxx and yr_arena_write_xxx. These functions can cause +// the buffer to be moved to different memory location and the pointer won't +// valid any longer. +void* yr_arena_ref_to_ptr( YR_ARENA* arena, - void* address); + YR_ARENA_REF* ref); -void* yr_arena_next_address( +// Given a pointer into the arena, it returns a reference to it. The reference +// can be used with yr_arena_ref_to_ptr to obtain a pointer again. Unlike +// pointers, references are during the arena's lifetime, even if the buffers +// are moved to a different memory location. +int yr_arena_ptr_to_ref( YR_ARENA* arena, - void* address, - size_t offset); + const void* address, + YR_ARENA_REF* ref); -int yr_arena_coalesce( - YR_ARENA* arena); - - -int yr_arena_reserve_memory( +// Given a buffer number and an offset within the buffer, returns a pointer +// to that offset. The same limitations explained for yr_arena_ref_to_ptr +// applies for the pointers returned by this function. +void* yr_arena_get_ptr( YR_ARENA* arena, - size_t size); + int buffer_id, + yr_arena_off_t offset); + + +yr_arena_off_t yr_arena_get_current_offset( + YR_ARENA* arena, + int buffer_id); int yr_arena_allocate_memory( YR_ARENA* arena, + int buffer_id, size_t size, - void** allocated_memory); + YR_ARENA_REF* ref); + + +int yr_arena_allocate_zeroed_memory( + YR_ARENA* arena, + int buffer_id, + size_t size, + YR_ARENA_REF* ref); int yr_arena_allocate_struct( YR_ARENA* arena, + int buffer_id, size_t size, - void** allocated_memory, + YR_ARENA_REF* ref, ...); int yr_arena_make_ptr_relocatable( YR_ARENA* arena, - void* base, + int buffer_id, ...); int yr_arena_write_data( YR_ARENA* arena, + int buffer_id, const void* data, size_t size, - void** written_data); + YR_ARENA_REF* ref); int yr_arena_write_string( YR_ARENA* arena, + int buffer_id, const char* string, - char** written_string); + YR_ARENA_REF* ref); -int yr_arena_append( - YR_ARENA* target_arena, - YR_ARENA* source_arena); +int yr_arena_write_uint32( + YR_ARENA* arena, + int buffer_id, + uint32_t integer, + YR_ARENA_REF* ref); int yr_arena_load_stream( @@ -177,13 +277,4 @@ int yr_arena_save_stream( YR_ARENA* arena, YR_STREAM* stream); - -int yr_arena_duplicate( - YR_ARENA* arena, - YR_ARENA** duplicated); - - -void yr_arena_print( - YR_ARENA* arena); - -#endif +#endif // YR_ARENA_H diff --git a/libyara/include/yara/atoms.h b/libyara/include/yara/atoms.h index 99c8ccbe..1cb3d8d0 100644 --- a/libyara/include/yara/atoms.h +++ b/libyara/include/yara/atoms.h @@ -81,8 +81,8 @@ struct YR_ATOM_LIST_ITEM uint16_t backtrack; - uint8_t* forward_code; - uint8_t* backward_code; + YR_ARENA_REF forward_code_ref; + YR_ARENA_REF backward_code_ref; YR_ATOM_LIST_ITEM* next; }; diff --git a/libyara/include/yara/base64.h b/libyara/include/yara/base64.h new file mode 100644 index 00000000..6c02a64a --- /dev/null +++ b/libyara/include/yara/base64.h @@ -0,0 +1,52 @@ +/* +Copyright (c) 2020. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef YR_BASE64_H +#define YR_BASE64_H + +#include +#include +#include + +typedef struct BASE64_NODE BASE64_NODE; + +struct BASE64_NODE { + + SIZED_STRING* str; + int escaped; + BASE64_NODE* next; + +}; + +int yr_base64_ast_from_string( + SIZED_STRING* in_str, + YR_MODIFIER modifier, + RE_AST** re_ast, + RE_ERROR* error); +#endif diff --git a/libyara/include/yara/bitmask.h b/libyara/include/yara/bitmask.h index 1a35b8c1..92fd3fbc 100644 --- a/libyara/include/yara/bitmask.h +++ b/libyara/include/yara/bitmask.h @@ -48,10 +48,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // yr_bitmask_clear(my_bitmask, n) // // Check if bit n is set: -// yr_bitmask_isset(my_bitmask, n) +// yr_bitmask_is_set(my_bitmask, n) // - #define YR_BITMASK unsigned long #define YR_BITMASK_SLOT_BITS (sizeof(YR_BITMASK) * 8) @@ -74,11 +73,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. memset(bm, 0, sizeof(bm)) -#define yr_bitmask_isset(bm, i) \ +#define yr_bitmask_is_set(bm, i) \ ( \ (bm)[(i) / YR_BITMASK_SLOT_BITS] & (1UL << ((i) % YR_BITMASK_SLOT_BITS)) \ ) +#define yr_bitmask_is_not_set(bm, i) (!yr_bitmask_is_set(bm, i)) + #define yr_bitmask_print(bm) \ { \ diff --git a/libyara/include/yara/compiler.h b/libyara/include/yara/compiler.h index 54f85bb5..8e48e327 100644 --- a/libyara/include/yara/compiler.h +++ b/libyara/include/yara/compiler.h @@ -45,7 +45,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Expression type constants are powers of two because they are used as flags. - #define EXPRESSION_TYPE_UNKNOWN 0 #define EXPRESSION_TYPE_BOOLEAN 1 #define EXPRESSION_TYPE_INTEGER 2 @@ -54,6 +53,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define EXPRESSION_TYPE_OBJECT 16 #define EXPRESSION_TYPE_FLOAT 32 + +// The compiler uses an arena to store the data it generates during the +// compilation. Each buffer in the arena is used for storing a different type +// of data. The following identifiers indicate the purpose of each buffer. +#define YR_NAMESPACES_TABLE 0 +#define YR_RULES_TABLE 1 +#define YR_METAS_TABLE 2 +#define YR_STRINGS_TABLE 3 +#define YR_EXTERNAL_VARIABLES_TABLE 4 +#define YR_SZ_POOL 5 +#define YR_CODE_SECTION 6 +#define YR_RE_CODE_SECTION 7 +#define YR_AC_TRANSITION_TABLE 8 +#define YR_AC_STATE_MATCHES_TABLE 9 +#define YR_AC_STATE_MATCHES_POOL 10 +#define YR_SUMMARY_SECTION 11 + + +// This is the number of buffers used by the compiler, should match the number +// of items in the list above. +#define YR_NUM_SECTIONS 12 + + +// Number of variables used by loops. This doesn't include user defined +// variables. +#define YR_INTERNAL_LOOP_VARS 3 + + typedef struct _YR_EXPRESSION { int type; @@ -64,7 +91,15 @@ typedef struct _YR_EXPRESSION SIZED_STRING* sized_string; } value; - const char* identifier; + // An expression can have an associated identifier, if "ptr" is not NULL it + // points to the identifier name, if it is NULL, then "ref" holds a reference + // to the identifier within YR_SZ_POOL. When the identifier is in YR_SZ_POOL + // a pointer can't be used as the YR_SZ_POOL can be moved to a different + // memory location. + struct { + const char* ptr; + YR_ARENA_REF ref; + } identifier ; } YR_EXPRESSION; @@ -99,7 +134,7 @@ typedef void (*YR_COMPILER_RE_AST_CALLBACK_FUNC)( typedef struct _YR_FIXUP { - void* address; + YR_ARENA_REF ref; struct _YR_FIXUP* next; } YR_FIXUP; @@ -112,7 +147,9 @@ typedef struct _YR_FIXUP typedef struct _YR_LOOP_CONTEXT { - uint8_t* addr; + // Reference indicating the the place in the code where the loop starts. The + // loop goes back to this address on each iteration. + YR_ARENA_REF start_ref; // vars_count is the number of local variables defined by the loop, and vars // is an array of expressions with the identifier and type for each of those @@ -129,6 +166,67 @@ typedef struct _YR_LOOP_CONTEXT typedef struct _YR_COMPILER { + // Arena that contains the data generated by the compiled. The arena has + // the following buffers: + // + // YR_SUMMARY_SECTION: + // A YR_SUMMARY struct. + // YR_RULES_TABLE: + // An array of YR_RULE structures, one per each rule. + // YR_STRINGS_TABLE: + // An array of YR_STRING structures, one per each string. + // YR_METAS_TABLE: + // An array of YR_META structures, one per each meta definition. + // YR_NAMESPACES_TABLE: + // An array of YR_NAMESPACE structures, one per each namespace. + // YR_EXTERNAL_VARIABLES_TABLE: + // An array of YR_EXTERNAL_VARIABLE structures, one per each external + // variable defined. + // YR_SZ_POOL: + // A collection of null-terminated strings. This buffer contains + // identifiers, literal strings, and in general any null-terminated + // string referenced by other data structures. + // YR_CODE_SECTION: + // The code for the condition section of all the rules. This is the + // code executed by yr_execute_code. + // YR_RE_CODE_SECTION: + // Similar to YR_CODE_SECTION, but it contains the code for regular + // expressions. This is the code executed by yr_re_exec and + // yr_re_fast_exec. + // YR_AC_TRANSITION_TABLE: + // An array of uint32_t containing the Aho-Corasick transition table. + // See comment in _yr_ac_build_transition_table for details. + // YR_AC_STATE_MATCHES_TABLE: + // An array of uint32_t with the same number of items than the transition + // table. If entry N in the transition table corresponds to some + // Aho-Corasick state, the N-th item in this array has the index within + // the matches pool where the list of matches for that state begins. + // YR_AC_STATE_MATCHES_POOL: + // An array of YR_AC_MATCH structures. + // + YR_ARENA* arena; + + // Index of the rule being compiled in the array of YR_RULE structures + // stored in YR_RULES_TABLE. If this is MAX_UINT32 the compiler is not + // parsing a rule. + uint32_t current_rule_idx; + + // + uint32_t next_rule_idx; + + // Index of the string being compiled in the array of YR_STRING structures + // stored in YR_STRINGS_TABLE. + uint32_t current_string_idx; + + // Index of the current namespace in the array of YR_NAMESPACE structures + // stored in YR_NAMESPACES_TABLE. + uint32_t current_namespace_idx; + + // Pointer to a YR_RULES structure that represents the compiled rules. This + // is what yr_compiler_get_rules returns. Once these rules are generated you + // can't call any of the yr_compiler_add_xxx functions. + YR_RULES* rules; + int errors; int current_line; int last_error; @@ -136,28 +234,14 @@ typedef struct _YR_COMPILER jmp_buf error_recovery; - YR_ARENA* sz_arena; - YR_ARENA* rules_arena; - YR_ARENA* strings_arena; - YR_ARENA* code_arena; - YR_ARENA* re_code_arena; - YR_ARENA* compiled_rules_arena; - YR_ARENA* externals_arena; - YR_ARENA* namespaces_arena; - YR_ARENA* metas_arena; - YR_ARENA* matches_arena; - YR_ARENA* automaton_arena; - YR_AC_AUTOMATON* automaton; YR_HASH_TABLE* rules_table; YR_HASH_TABLE* objects_table; YR_HASH_TABLE* strings_table; - YR_NAMESPACE* current_namespace; - YR_RULE* current_rule; YR_FIXUP* fixup_stack_head; - int namespaces_count; + int num_namespaces; YR_LOOP_CONTEXT loop[YR_MAX_LOOP_NESTING]; int loop_index; @@ -220,6 +304,10 @@ const char* _yr_compiler_default_include_callback( void* user_data); +YR_RULE* _yr_compiler_get_rule_by_idx( + YR_COMPILER* compiler, uint32_t rule_idx); + + YR_API int yr_compiler_create( YR_COMPILER** compiler); diff --git a/libyara/include/yara/dex.h b/libyara/include/yara/dex.h index 918051c5..bbb7b4c9 100644 --- a/libyara/include/yara/dex.h +++ b/libyara/include/yara/dex.h @@ -10,6 +10,7 @@ #define DEX_FILE_MAGIC_036 "dex\n036\x00" #define DEX_FILE_MAGIC_037 "dex\n037\x00" #define DEX_FILE_MAGIC_038 "dex\n038\x00" +#define DEX_FILE_MAGIC_039 "dex\n039\x00" #pragma pack(push,1) @@ -75,7 +76,7 @@ typedef struct { uint32_t class_idx; uint32_t access_flags; uint32_t super_class_idx; - uint32_t interfaces_off; + uint32_t interfaces_offset; uint32_t source_file_idx; uint32_t annotations_offset; uint32_t class_data_offset; diff --git a/libyara/include/yara/exec.h b/libyara/include/yara/exec.h index 0480c65e..a402d8a7 100644 --- a/libyara/include/yara/exec.h +++ b/libyara/include/yara/exec.h @@ -36,8 +36,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -#define UNDEFINED 0xFFFABADAFABADAFFLL -#define IS_UNDEFINED(x) ((size_t)(x) == (size_t) UNDEFINED) +#define YR_UNDEFINED 0xFFFABADAFABADAFFLL +#define IS_UNDEFINED(x) ((size_t)(x) == (size_t) YR_UNDEFINED) #define OP_ERROR 0 #define OP_HALT 255 @@ -172,7 +172,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define OPERATION(operator, op1, op2) \ - (IS_UNDEFINED(op1) || IS_UNDEFINED(op2)) ? (UNDEFINED) : (op1 operator op2) + (IS_UNDEFINED(op1) || IS_UNDEFINED(op2)) ? (YR_UNDEFINED) : (op1 operator op2) #define COMPARISON(operator, op1, op2) \ diff --git a/libyara/include/yara/globals.h b/libyara/include/yara/globals.h index b508b9a0..7f3e468c 100644 --- a/libyara/include/yara/globals.h +++ b/libyara/include/yara/globals.h @@ -39,7 +39,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. extern uint8_t yr_lowercase[256]; extern uint8_t yr_altercase[256]; -extern YR_THREAD_STORAGE_KEY yr_tidx_key; -extern YR_THREAD_STORAGE_KEY yr_recovery_state_key; + +// Thread-local storage (TLS) key used by the regexp and hex string parsers. +// Each thread calling yr_parse_re_string/yr_parse_hex_string stores a pointer +// to a jmp_buf struct used by setjmp/longjmp for recovering when a fatal error +// occurs in the parser. +extern YR_THREAD_STORAGE_KEY yr_yyfatal_trampoline_tls; + +// Thread-local storage (TLS) key used by YR_TRYCATCH. +extern YR_THREAD_STORAGE_KEY yr_trycatch_trampoline_tls; #endif diff --git a/libyara/include/yara/hash.h b/libyara/include/yara/hash.h index 8108333b..464eaf71 100644 --- a/libyara/include/yara/hash.h +++ b/libyara/include/yara/hash.h @@ -120,4 +120,17 @@ YR_API int yr_hash_table_add_raw_key( const char* ns, void* value); + +YR_API uint32_t yr_hash_table_add_uint32( + YR_HASH_TABLE* table, + const char* key, + const char* ns, + uint32_t value); + + +YR_API uint32_t yr_hash_table_lookup_uint32( + YR_HASH_TABLE* table, + const char* key, + const char* ns); + #endif diff --git a/libyara/include/yara/hex_lexer.h b/libyara/include/yara/hex_lexer.h index 87941477..ed337919 100644 --- a/libyara/include/yara/hex_lexer.h +++ b/libyara/include/yara/hex_lexer.h @@ -72,6 +72,11 @@ typedef struct _HEX_LEX_ENVIRONMENT } HEX_LEX_ENVIRONMENT; +// The default behavior when a fatal error occurs in the parser is calling +// exit(YY_EXIT_FAILURE) for terminating the process. This is not acceptable +// for a library, which should return gracefully to the calling program. For +// this reason we redefine the YY_FATAL_ERROR macro so that it expands to our +// own function instead of the one provided by default. #define YY_FATAL_ERROR(msg) hex_yyfatal(yyscanner, msg) #define LEX_ENV ((HEX_LEX_ENVIRONMENT*) lex_env) diff --git a/libyara/include/yara/libyara.h b/libyara/include/yara/libyara.h index 3f88eff7..8fe4490c 100644 --- a/libyara/include/yara/libyara.h +++ b/libyara/include/yara/libyara.h @@ -69,22 +69,10 @@ typedef enum _YR_CONFIG_NAME YR_API int yr_initialize(void); - YR_API int yr_finalize(void); - -YR_DEPRECATED_API void yr_finalize_thread(void); - - -YR_API int yr_get_tidx(void); - - -YR_API void yr_set_tidx(int); - - YR_API int yr_set_configuration(YR_CONFIG_NAME, void*); - YR_API int yr_get_configuration(YR_CONFIG_NAME, void*); diff --git a/libyara/include/yara/limits.h b/libyara/include/yara/limits.h index 697d0626..59baca6e 100644 --- a/libyara/include/yara/limits.h +++ b/libyara/include/yara/limits.h @@ -36,7 +36,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "utils.h" -// Maximum lenght of file paths. This is the only limit that doesn't have the +// Maximum length of file paths. This is the only limit that doesn't have the // YR_ prefix. The intention is using the default MAX_PATH if defined. #ifndef MAX_PATH #define MAX_PATH 1024 @@ -50,6 +50,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define YR_MAX_THREADS 32 #endif +// Maximum number of buffers that an arena can have. +#ifndef YR_MAX_ARENA_BUFFERS +#define YR_MAX_ARENA_BUFFERS 16 +#endif + // Capacity of the buffer used for storing compiler error messages. Messages // will be truncated at this size. #ifndef YR_MAX_COMPILER_ERROR_EXTRA_INFO @@ -98,10 +103,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define YR_MAX_LOOP_VARS 2 #endif -#ifndef YR_MAX_ARENA_PAGES -#define YR_MAX_ARENA_PAGES 32 -#endif - // Maximum number of nested included files. #ifndef YR_MAX_INCLUDE_DEPTH #define YR_MAX_INCLUDE_DEPTH 16 @@ -154,11 +155,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define RE_MAX_STACK 1024 #endif -// Maximum code size for a compiled regexp -#ifndef RE_MAX_CODE_SIZE -#define RE_MAX_CODE_SIZE 32768 -#endif - // Maximum input size scanned by yr_re_exec #ifndef RE_SCAN_LIMIT #define RE_SCAN_LIMIT 4096 diff --git a/libyara/include/yara/modules.h b/libyara/include/yara/modules.h index ea5fb75a..313c03f3 100644 --- a/libyara/include/yara/modules.h +++ b/libyara/include/yara/modules.h @@ -356,7 +356,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. __function_obj->return_obj->type == OBJECT_TYPE_FLOAT, \ "return type differs from function declaration"); \ return yr_object_set_float( \ - (d != (double) UNDEFINED) ? d : NAN, \ + (d != (double) YR_UNDEFINED) ? d : NAN, \ __function_obj->return_obj, \ NULL); \ } @@ -368,8 +368,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. __function_obj->return_obj->type == OBJECT_TYPE_STRING, \ "return type differs from function declaration"); \ return yr_object_set_string( \ - (s != (char*) UNDEFINED) ? s : NULL, \ - (s != (char*) UNDEFINED) ? strlen(s) : 0, \ + (s != (char*) YR_UNDEFINED) ? s : NULL, \ + (s != (char*) YR_UNDEFINED) ? strlen(s) : 0, \ __function_obj->return_obj, \ NULL); \ } diff --git a/libyara/include/yara/notebook.h b/libyara/include/yara/notebook.h new file mode 100644 index 00000000..90f7ec4c --- /dev/null +++ b/libyara/include/yara/notebook.h @@ -0,0 +1,27 @@ +// +// Created by Victor Manuel Alvarez on 3/4/20. +// + +#ifndef YR_NOTEBOOK_H +#define YR_NOTEBOOK_H + +#include + +typedef struct YR_NOTEBOOK YR_NOTEBOOK; + + +int yr_notebook_create( + size_t page_size, + YR_NOTEBOOK** pool); + + +int yr_notebook_destroy( + YR_NOTEBOOK* pool); + + +void* yr_notebook_alloc( + YR_NOTEBOOK* notebook, + size_t size); + + +#endif // YR_NOTEBOOK_H diff --git a/libyara/include/yara/parser.h b/libyara/include/yara/parser.h index b1418709..77f710a1 100644 --- a/libyara/include/yara/parser.h +++ b/libyara/include/yara/parser.h @@ -37,31 +37,39 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. int yr_parser_emit( yyscan_t yyscanner, uint8_t instruction, - uint8_t** instruction_address); + YR_ARENA_REF* instruction_ref); int yr_parser_emit_with_arg( yyscan_t yyscanner, uint8_t instruction, int64_t argument, - uint8_t** instruction_address, - int64_t** argument_address); + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref); + + +int yr_parser_emit_with_arg_int32( + yyscan_t yyscanner, + uint8_t instruction, + int32_t argument, + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref); int yr_parser_emit_with_arg_double( yyscan_t yyscanner, uint8_t instruction, double argument, - uint8_t** instruction_address, - double** argument_address); + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref); int yr_parser_emit_with_arg_reloc( yyscan_t yyscanner, uint8_t instruction, void* argument, - uint8_t** instruction_address, - void** argument_address); + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref); int yr_parser_check_types( @@ -86,12 +94,12 @@ int yr_parser_reduce_rule_declaration_phase_1( yyscan_t yyscanner, int32_t flags, const char* identifier, - YR_RULE** rule); + YR_ARENA_REF* rule_ref); int yr_parser_reduce_rule_declaration_phase_2( yyscan_t yyscanner, - YR_RULE* rule); + YR_ARENA_REF* rule_ref); int yr_parser_reduce_string_declaration( @@ -99,7 +107,7 @@ int yr_parser_reduce_string_declaration( YR_MODIFIER modifier, const char* identifier, SIZED_STRING* str, - YR_STRING** string); + YR_ARENA_REF* string_ref); int yr_parser_reduce_meta_declaration( @@ -108,7 +116,7 @@ int yr_parser_reduce_meta_declaration( const char* identifier, const char* string, int64_t integer, - YR_META** meta); + YR_ARENA_REF* meta_ref); int yr_parser_reduce_string_identifier( diff --git a/libyara/include/yara/pe.h b/libyara/include/yara/pe.h index 7847ba2f..8158be64 100644 --- a/libyara/include/yara/pe.h +++ b/libyara/include/yara/pe.h @@ -38,6 +38,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #if defined(_WIN32) || defined(__CYGWIN__) #include +// PKCS7_SIGNER_INFO is defined by wincrypt.h, but it conflicts with a type +// defined in openssl/pkcs7.h which is used in pe.c. Let's undefine the macro. +#undef PKCS7_SIGNER_INFO + // These definitions are not present in older Windows headers. #ifndef IMAGE_FILE_MACHINE_ARMNT @@ -48,6 +52,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define IMAGE_FILE_MACHINE_ARM64 0xaa64 #endif + + #else #include @@ -452,10 +458,52 @@ typedef struct _IMAGE_RESOURCE_DIRECTORY { WORD NumberOfIdEntries; } IMAGE_RESOURCE_DIRECTORY, *PIMAGE_RESOURCE_DIRECTORY; +#define IMAGE_DEBUG_TYPE_UNKNOWN 0 +#define IMAGE_DEBUG_TYPE_COFF 1 +#define IMAGE_DEBUG_TYPE_CODEVIEW 2 +#define IMAGE_DEBUG_TYPE_FPO 3 +#define IMAGE_DEBUG_TYPE_MISC 4 +#define IMAGE_DEBUG_TYPE_EXCEPTION 5 +#define IMAGE_DEBUG_TYPE_FIXUP 6 +#define IMAGE_DEBUG_TYPE_BORLAND 9 + +typedef struct _IMAGE_DEBUG_DIRECTORY { + DWORD Characteristics; + DWORD TimeDateStamp; + WORD MajorVersion; + WORD MinorVersion; + DWORD Type; + DWORD SizeOfData; + DWORD AddressOfRawData; + DWORD PointerToRawData; +} IMAGE_DEBUG_DIRECTORY, *PIMAGE_DEBUG_DIRECTORY; + #pragma pack(pop) #endif // _WIN32 +#define CVINFO_PDB70_CVSIGNATURE 0x53445352 // "RSDS" +#define CVINFO_PDB20_CVSIGNATURE 0x3031424e // "NB10" + +typedef struct _CV_HEADER { + DWORD dwSignature; + DWORD dwOffset; +} CV_HEADER, *PCV_HEADER; + +typedef struct _CV_INFO_PDB20 { + CV_HEADER CvHeader; + DWORD dwSignature; + DWORD dwAge; + BYTE PdbFileName[1]; +} CV_INFO_PDB20, *PCV_INFO_PDB20; + +typedef struct _CV_INFO_PDB70 { + DWORD CvSignature; + DWORD Signature[4]; + DWORD Age; + BYTE PdbFileName[1]; +} CV_INFO_PDB70, *PCV_INFO_PDB70; + typedef struct _VERSION_INFO { WORD Length; WORD ValueLength; @@ -464,6 +512,8 @@ typedef struct _VERSION_INFO { } VERSION_INFO, *PVERSION_INFO; +#define MAX_PE_CERTS 16 + #define WIN_CERT_REVISION_1_0 0x0100 #define WIN_CERT_REVISION_2_0 0x0200 @@ -481,6 +531,8 @@ typedef struct _WIN_CERTIFICATE { BYTE Certificate[0]; } WIN_CERTIFICATE, *PWIN_CERTIFICATE; +#define SPC_NESTED_SIGNATURE_OBJID "1.3.6.1.4.1.311.2.4.1" + // // Rich signature. diff --git a/libyara/include/yara/pe_utils.h b/libyara/include/yara/pe_utils.h index 0fc50fdc..c9596011 100644 --- a/libyara/include/yara/pe_utils.h +++ b/libyara/include/yara/pe_utils.h @@ -52,20 +52,6 @@ typedef struct _IMPORT_FUNCTION } IMPORT_FUNCTION, *PIMPORT_FUNCTION; -typedef struct _EXPORT_FUNCTION -{ - char *name; - uint16_t ordinal; -} EXPORT_FUNCTION, *PEXPORT_FUNCTION; - - -typedef struct _EXPORT_FUNCTIONS -{ - uint32_t number_of_exports; - EXPORT_FUNCTION* functions; -} EXPORT_FUNCTIONS, *PEXPORT_FUNCTIONS; - - typedef struct _PE { const uint8_t* data; @@ -79,7 +65,6 @@ typedef struct _PE YR_HASH_TABLE* hash_table; YR_OBJECT* object; IMPORTED_DLL* imported_dlls; - EXPORT_FUNCTIONS* exported_functions; uint32_t resources; diff --git a/libyara/include/yara/re.h b/libyara/include/yara/re.h index 047291df..050e4bd8 100644 --- a/libyara/include/yara/re.h +++ b/libyara/include/yara/re.h @@ -105,7 +105,6 @@ typedef int RE_MATCH_CALLBACK_FUNC( int flags, void* args); - int yr_re_ast_create( RE_AST** re_ast); @@ -193,8 +192,8 @@ int yr_re_parse_hex( int yr_re_compile( const char* re_string, int flags, - YR_ARENA* code_arena, - RE** re, + YR_ARENA* arena, + YR_ARENA_REF* ref, RE_ERROR* error); @@ -203,5 +202,4 @@ int yr_re_match( RE* re, const char* target); - #endif diff --git a/libyara/include/yara/re_lexer.h b/libyara/include/yara/re_lexer.h index d278454b..853226ff 100644 --- a/libyara/include/yara/re_lexer.h +++ b/libyara/include/yara/re_lexer.h @@ -73,6 +73,11 @@ typedef struct _RE_LEX_ENVIRONMENT #define LEX_ENV ((RE_LEX_ENVIRONMENT*) lex_env) +// The default behavior when a fatal error occurs in the parser is calling +// exit(YY_EXIT_FAILURE) for terminating the process. This is not acceptable +// for a library, which should return gracefully to the calling program. For +// this reason we redefine the YY_FATAL_ERROR macro so that it expands to our +// own function instead of the one provided by default. #define YY_FATAL_ERROR(msg) re_yyfatal(yyscanner, msg) #include diff --git a/libyara/include/yara/rules.h b/libyara/include/yara/rules.h index 861cebf0..646d39ca 100644 --- a/libyara/include/yara/rules.h +++ b/libyara/include/yara/rules.h @@ -57,11 +57,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define yr_rule_strings_foreach(rule, string) \ - for (string = rule->strings; !STRING_IS_NULL(string); string++) + for (string = rule->strings; \ + string != NULL; \ + string = STRING_IS_LAST_IN_RULE(string) ? NULL : string + 1) -#define yr_string_matches_foreach(string, match) \ - for (match = STRING_MATCHES(string).head; match != NULL; match = match->next) +#define yr_string_matches_foreach(context, string, match) \ + for (match = context->matches[string->idx].head; \ + match != NULL; \ + match = match->next) \ + /* private matches are skipped */ \ + if (match->private) { continue; } \ + else /* user code block goes here */ #define yr_rules_foreach(rules, rule) \ @@ -154,14 +161,6 @@ YR_API int yr_rules_define_string_variable( const char* value); -YR_API void yr_rules_reset_profiling_info( - YR_RULES* rules); - - -YR_API void yr_rules_print_profiling_info( - YR_RULES* rules); - - YR_API int yr_rules_get_stats( YR_RULES* rules, YR_RULES_STATS *stats); @@ -174,4 +173,9 @@ YR_API void yr_rule_disable( YR_API void yr_rule_enable( YR_RULE* rule); + +int yr_rules_from_arena( + YR_ARENA* arena, + YR_RULES** rules); + #endif diff --git a/libyara/include/yara/scan.h b/libyara/include/yara/scan.h index 87682601..5326efae 100644 --- a/libyara/include/yara/scan.h +++ b/libyara/include/yara/scan.h @@ -32,10 +32,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include -// Bitmasks for flags. -#define SCAN_FLAGS_FAST_MODE 1 -#define SCAN_FLAGS_PROCESS_MEMORY 2 -#define SCAN_FLAGS_NO_TRYCATCH 4 +// +// Flags used with yr_scanner_set_flags and yr_rules_scan_xxx functions. +// +#define SCAN_FLAGS_FAST_MODE 1 +#define SCAN_FLAGS_PROCESS_MEMORY 2 +#define SCAN_FLAGS_NO_TRYCATCH 4 +#define SCAN_FLAGS_REPORT_RULES_MATCHING 8 +#define SCAN_FLAGS_REPORT_RULES_NOT_MATCHING 16 int yr_scan_verify_match( diff --git a/libyara/include/yara/sizedstr.h b/libyara/include/yara/sizedstr.h index f9da17c7..fd956071 100644 --- a/libyara/include/yara/sizedstr.h +++ b/libyara/include/yara/sizedstr.h @@ -44,8 +44,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SIZED_STRING_FLAGS_DOT_ALL 2 #pragma pack(push) -#pragma pack(8) - +#pragma pack(4) typedef struct _SIZED_STRING { @@ -59,6 +58,11 @@ typedef struct _SIZED_STRING #pragma pack(pop) +int sized_string_cmp_nocase( + SIZED_STRING* s1, + SIZED_STRING* s2); + + int sized_string_cmp( SIZED_STRING* s1, SIZED_STRING* s2); @@ -71,4 +75,7 @@ SIZED_STRING* sized_string_dup( SIZED_STRING* sized_string_new( const char* s); +SIZED_STRING* sized_string_convert_to_wide( + SIZED_STRING* s); + #endif diff --git a/libyara/include/yara/stopwatch.h b/libyara/include/yara/stopwatch.h index 753ad538..a9d806f9 100644 --- a/libyara/include/yara/stopwatch.h +++ b/libyara/include/yara/stopwatch.h @@ -75,9 +75,9 @@ typedef struct _YR_STOPWATCH void yr_stopwatch_start( YR_STOPWATCH* stopwatch); -// yr_stopwatch_elapsed_us returns the number of microseconds elapsed +// yr_stopwatch_elapsed_ns returns the number of nanoseconds elapsed // since the last call to yr_stopwatch_start. -uint64_t yr_stopwatch_elapsed_us( +uint64_t yr_stopwatch_elapsed_ns( YR_STOPWATCH* stopwatch); #endif diff --git a/libyara/include/yara/strutils.h b/libyara/include/yara/strutils.h index ca1aac37..d15aba32 100644 --- a/libyara/include/yara/strutils.h +++ b/libyara/include/yara/strutils.h @@ -51,6 +51,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define PRId64 "I64d" #endif +#if !defined (PRIi32) +#define PRIi32 "I32i" +#endif + #else #include #endif diff --git a/libyara/include/yara/types.h b/libyara/include/yara/types.h index 7aef7ed0..78d7f7b0 100644 --- a/libyara/include/yara/types.h +++ b/libyara/include/yara/types.h @@ -30,6 +30,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef YR_TYPES_H #define YR_TYPES_H +#include + #include #include #include @@ -38,131 +40,128 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include "notebook.h" #define DECLARE_REFERENCE(type, name) \ - union { type name; int64_t name##_; } YR_ALIGN(8) + union { \ + type name; \ + YR_ARENA_REF name##_; \ + } YR_ALIGN(8) - -#define NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL 0x01 - - -#define STRING_GFLAGS_REFERENCED 0x01 -#define STRING_GFLAGS_HEXADECIMAL 0x02 -#define STRING_GFLAGS_NO_CASE 0x04 -#define STRING_GFLAGS_ASCII 0x08 -#define STRING_GFLAGS_WIDE 0x10 -#define STRING_GFLAGS_REGEXP 0x20 -#define STRING_GFLAGS_FAST_REGEXP 0x40 -#define STRING_GFLAGS_FULL_WORD 0x80 -#define STRING_GFLAGS_ANONYMOUS 0x100 -#define STRING_GFLAGS_SINGLE_MATCH 0x200 -#define STRING_GFLAGS_LITERAL 0x400 -#define STRING_GFLAGS_FITS_IN_ATOM 0x800 -#define STRING_GFLAGS_NULL 0x1000 -#define STRING_GFLAGS_CHAIN_PART 0x2000 -#define STRING_GFLAGS_CHAIN_TAIL 0x4000 -#define STRING_GFLAGS_FIXED_OFFSET 0x8000 -#define STRING_GFLAGS_GREEDY_REGEXP 0x10000 -#define STRING_GFLAGS_DOT_ALL 0x20000 -#define STRING_GFLAGS_DISABLED 0x40000 -#define STRING_GFLAGS_XOR 0x80000 -#define STRING_GFLAGS_PRIVATE 0x100000 - -#define STRING_IS_HEX(x) \ - (((x)->g_flags) & STRING_GFLAGS_HEXADECIMAL) - -#define STRING_IS_NO_CASE(x) \ - (((x)->g_flags) & STRING_GFLAGS_NO_CASE) - -#define STRING_IS_DOT_ALL(x) \ - (((x)->g_flags) & STRING_GFLAGS_DOT_ALL) - -#define STRING_IS_ASCII(x) \ - (((x)->g_flags) & STRING_GFLAGS_ASCII) - -#define STRING_IS_WIDE(x) \ - (((x)->g_flags) & STRING_GFLAGS_WIDE) - -#define STRING_IS_REGEXP(x) \ - (((x)->g_flags) & STRING_GFLAGS_REGEXP) - -#define STRING_IS_GREEDY_REGEXP(x) \ - (((x)->g_flags) & STRING_GFLAGS_GREEDY_REGEXP) - -#define STRING_IS_FULL_WORD(x) \ - (((x)->g_flags) & STRING_GFLAGS_FULL_WORD) - -#define STRING_IS_ANONYMOUS(x) \ - (((x)->g_flags) & STRING_GFLAGS_ANONYMOUS) - -#define STRING_IS_REFERENCED(x) \ - (((x)->g_flags) & STRING_GFLAGS_REFERENCED) - -#define STRING_IS_SINGLE_MATCH(x) \ - (((x)->g_flags) & STRING_GFLAGS_SINGLE_MATCH) - -#define STRING_IS_FIXED_OFFSET(x) \ - (((x)->g_flags) & STRING_GFLAGS_FIXED_OFFSET) - -#define STRING_IS_LITERAL(x) \ - (((x)->g_flags) & STRING_GFLAGS_LITERAL) - -#define STRING_IS_FAST_REGEXP(x) \ - (((x)->g_flags) & STRING_GFLAGS_FAST_REGEXP) - -#define STRING_IS_CHAIN_PART(x) \ - (((x)->g_flags) & STRING_GFLAGS_CHAIN_PART) - -#define STRING_IS_CHAIN_TAIL(x) \ - (((x)->g_flags) & STRING_GFLAGS_CHAIN_TAIL) - -#define STRING_IS_NULL(x) \ - ((x) == NULL || ((x)->g_flags) & STRING_GFLAGS_NULL) - -#define STRING_FITS_IN_ATOM(x) \ - (((x)->g_flags) & STRING_GFLAGS_FITS_IN_ATOM) - -#define STRING_IS_DISABLED(x) \ - (((x)->g_flags) & STRING_GFLAGS_DISABLED) - -#define STRING_IS_XOR(x) \ - (((x)->g_flags) & STRING_GFLAGS_XOR) - -#define STRING_IS_PRIVATE(x) \ - (((x)->g_flags) & STRING_GFLAGS_PRIVATE) - -#define STRING_FOUND(x) \ - ((x)->matches[yr_get_tidx()].tail != NULL) - -#define STRING_MATCHES(x) \ - ((x)->matches[yr_get_tidx()]) - - -#define RULE_TFLAGS_MATCH 0x01 - -#define RULE_GFLAGS_PRIVATE 0x01 -#define RULE_GFLAGS_GLOBAL 0x02 -#define RULE_GFLAGS_REQUIRE_EXECUTABLE 0x04 -#define RULE_GFLAGS_REQUIRE_FILE 0x08 -#define RULE_GFLAGS_NULL 0x1000 -#define RULE_GFLAGS_DISABLED 0x2000 +// Flags for YR_RULE +#define RULE_FLAGS_PRIVATE 0x01 +#define RULE_FLAGS_GLOBAL 0x02 +#define RULE_FLAGS_NULL 0x04 +#define RULE_FLAGS_DISABLED 0x08 #define RULE_IS_PRIVATE(x) \ - (((x)->g_flags) & RULE_GFLAGS_PRIVATE) + (((x)->flags) & RULE_FLAGS_PRIVATE) #define RULE_IS_GLOBAL(x) \ - (((x)->g_flags) & RULE_GFLAGS_GLOBAL) + (((x)->flags) & RULE_FLAGS_GLOBAL) #define RULE_IS_NULL(x) \ - (((x)->g_flags) & RULE_GFLAGS_NULL) + (((x)->flags) & RULE_FLAGS_NULL) #define RULE_IS_DISABLED(x) \ - (((x)->g_flags) & RULE_GFLAGS_DISABLED) + (((x)->flags) & RULE_FLAGS_DISABLED) -#define RULE_MATCHES(x) \ - ((x)->t_flags[yr_get_tidx()] & RULE_TFLAGS_MATCH) + +// Flags for YR_STRING +#define STRING_FLAGS_REFERENCED 0x01 +#define STRING_FLAGS_HEXADECIMAL 0x02 +#define STRING_FLAGS_NO_CASE 0x04 +#define STRING_FLAGS_ASCII 0x08 +#define STRING_FLAGS_WIDE 0x10 +#define STRING_FLAGS_REGEXP 0x20 +#define STRING_FLAGS_FAST_REGEXP 0x40 +#define STRING_FLAGS_FULL_WORD 0x80 +#define STRING_FLAGS_ANONYMOUS 0x100 +#define STRING_FLAGS_SINGLE_MATCH 0x200 +#define STRING_FLAGS_LITERAL 0x400 +#define STRING_FLAGS_FITS_IN_ATOM 0x800 +#define STRING_FLAGS_LAST_IN_RULE 0x1000 +#define STRING_FLAGS_CHAIN_PART 0x2000 +#define STRING_FLAGS_CHAIN_TAIL 0x4000 +#define STRING_FLAGS_FIXED_OFFSET 0x8000 +#define STRING_FLAGS_GREEDY_REGEXP 0x10000 +#define STRING_FLAGS_DOT_ALL 0x20000 +#define STRING_FLAGS_DISABLED 0x40000 +#define STRING_FLAGS_XOR 0x80000 +#define STRING_FLAGS_PRIVATE 0x100000 +#define STRING_FLAGS_BASE64 0x200000 +#define STRING_FLAGS_BASE64_WIDE 0x400000 + +#define STRING_IS_HEX(x) \ + (((x)->flags) & STRING_FLAGS_HEXADECIMAL) + +#define STRING_IS_NO_CASE(x) \ + (((x)->flags) & STRING_FLAGS_NO_CASE) + +#define STRING_IS_DOT_ALL(x) \ + (((x)->flags) & STRING_FLAGS_DOT_ALL) + +#define STRING_IS_ASCII(x) \ + (((x)->flags) & STRING_FLAGS_ASCII) + +#define STRING_IS_WIDE(x) \ + (((x)->flags) & STRING_FLAGS_WIDE) + +#define STRING_IS_REGEXP(x) \ + (((x)->g_flags) & STRING_FLAGS_REGEXP) + +#define STRING_IS_GREEDY_REGEXP(x) \ + (((x)->flags) & STRING_FLAGS_GREEDY_REGEXP) + +#define STRING_IS_FULL_WORD(x) \ + (((x)->flags) & STRING_FLAGS_FULL_WORD) + +#define STRING_IS_ANONYMOUS(x) \ + (((x)->g_flags) & STRING_FLAGS_ANONYMOUS) + +#define STRING_IS_REFERENCED(x) \ + (((x)->flags) & STRING_FLAGS_REFERENCED) + +#define STRING_IS_SINGLE_MATCH(x) \ + (((x)->flags) & STRING_FLAGS_SINGLE_MATCH) + +#define STRING_IS_FIXED_OFFSET(x) \ + (((x)->flags) & STRING_FLAGS_FIXED_OFFSET) + +#define STRING_IS_LITERAL(x) \ + (((x)->flags) & STRING_FLAGS_LITERAL) + +#define STRING_IS_FAST_REGEXP(x) \ + (((x)->flags) & STRING_FLAGS_FAST_REGEXP) + +#define STRING_IS_CHAIN_PART(x) \ + (((x)->flags) & STRING_FLAGS_CHAIN_PART) + +#define STRING_IS_CHAIN_TAIL(x) \ + (((x)->flags) & STRING_FLAGS_CHAIN_TAIL) + +#define STRING_IS_LAST_IN_RULE(x) \ + (((x)->flags) & STRING_FLAGS_LAST_IN_RULE) + +#define STRING_FITS_IN_ATOM(x) \ + (((x)->flags) & STRING_FLAGS_FITS_IN_ATOM) + +#define STRING_IS_DISABLED(x) \ + (((x)->flags) & STRING_FLAGS_DISABLED) + +#define STRING_IS_XOR(x) \ + (((x)->flags) & STRING_FLAGS_XOR) + +#define STRING_IS_BASE64(x) \ + (((x)->flags) & STRING_FLAGS_BASE64) + +#define STRING_IS_BASE64_WIDE(x) \ + (((x)->flags) & STRING_FLAGS_BASE64_WIDE) + +#define STRING_IS_PRIVATE(x) \ + (((x)->flags) & STRING_FLAGS_PRIVATE) #define META_TYPE_NULL 0 @@ -194,11 +193,11 @@ typedef struct RE_FIBER RE_FIBER; typedef struct RE_FIBER_LIST RE_FIBER_LIST; typedef struct RE_FIBER_POOL RE_FIBER_POOL; -typedef struct YR_AC_MATCH YR_AC_MATCH; typedef struct YR_AC_STATE YR_AC_STATE; typedef struct YR_AC_AUTOMATON YR_AC_AUTOMATON; -typedef struct YR_AC_MATCH_TABLE_ENTRY YR_AC_MATCH_TABLE_ENTRY; typedef struct YR_AC_TABLES YR_AC_TABLES; +typedef struct YR_AC_MATCH_LIST_ENTRY YR_AC_MATCH_LIST_ENTRY; +typedef struct YR_AC_MATCH YR_AC_MATCH; typedef struct YR_NAMESPACE YR_NAMESPACE; typedef struct YR_META YR_META; @@ -206,6 +205,7 @@ typedef struct YR_MATCHES YR_MATCHES; typedef struct YR_STRING YR_STRING; typedef struct YR_RULE YR_RULE; typedef struct YR_RULES YR_RULES; +typedef struct YR_SUMMARY YR_SUMMARY; typedef struct YR_RULES_STATS YR_RULES_STATS; typedef struct YR_EXTERNAL_VARIABLE YR_EXTERNAL_VARIABLE; typedef struct YR_MATCH YR_MATCH; @@ -234,67 +234,70 @@ typedef struct YR_MODIFIER YR_MODIFIER; typedef struct YR_ITERATOR YR_ITERATOR; +typedef uint32_t YR_AC_TRANSITION; #pragma pack(push) #pragma pack(8) - -struct YR_MODIFIER -{ - int32_t flags; - uint8_t xor_min; - uint8_t xor_max; -}; - struct YR_NAMESPACE { - int32_t t_flags[YR_MAX_THREADS]; // Thread-specific flags - DECLARE_REFERENCE(char*, name); + // Pointer to namespace's name. + DECLARE_REFERENCE(const char*, name); + + // Index of this namespace in the array of YR_NAMESPACE structures stored + // in YR_NAMESPACES_TABLE. + uint32_t idx; }; struct YR_META { - int32_t type; - YR_ALIGN(8) int64_t integer; - DECLARE_REFERENCE(const char*, identifier); - DECLARE_REFERENCE(char*, string); -}; + DECLARE_REFERENCE(const char*, string); - -struct YR_MATCHES -{ - int32_t count; - - DECLARE_REFERENCE(YR_MATCH*, head); - DECLARE_REFERENCE(YR_MATCH*, tail); + int64_t integer; + int32_t type; }; struct YR_STRING { - int32_t g_flags; + // Flags, see STRING_FLAGS_XXX macros defined above. + uint32_t flags; + + // Index of this string in the array of YR_STRING structures stored in + // YR_STRINGS_TABLE. + uint32_t idx; + + // If the string can only match at a specific offset (for example if the + // condition is "$a at 0" the string $a can only match at offset 0), the + // fixed_offset field contains the offset, it have the YR_UNDEFINED value for + // strings that can match anywhere. + int64_t fixed_offset; + + // Index of the rule containing this string in the array of YR_RULE + // structures stored in YR_RULES_TABLE. + uint32_t rule_idx; + + // String's length. int32_t length; - DECLARE_REFERENCE(char*, identifier); + // Pointer to the string itself, the length is indicated by the "length" + // field. DECLARE_REFERENCE(uint8_t*, string); - // Strings are splited in two or more parts when they contain a "gap" that + // Strings are splitted in two or more parts when they contain a "gap" that // is larger than YR_STRING_CHAINING_THRESHOLD. This happens in strings like // { 01 02 03 04 [X-Y] 05 06 07 08 } if Y >= X + YR_STRING_CHAINING_THRESHOLD // and also in { 01 02 03 04 [-] 05 06 07 08 }. In both cases the strings are // split in { 01 02 03 04 } and { 05 06 07 08 }, and the two smaller strings - // are searched for independently. If some string S is splited in S1 and S2, + // are searched for independently. If some string S is splitted in S1 and S2, // S2 is chained to S1. In the example above { 05 06 07 08 } is chained to // { 01 02 03 04 }. The same applies when the string is splitted in more than // two parts, if S is split in S1, S2, and S3. S3 is chained to S2 and S2 is // chained to S1 (it can represented as: S1 <- S2 <- S3). DECLARE_REFERENCE(YR_STRING*, chained_to); - // Pointer to the rule that contains this string. - DECLARE_REFERENCE(YR_RULE*, rule); - // When this string is chained to some other string, chain_gap_min and // chain_gap_max contain the minimum and maximum distance between the two // strings. For example in { 01 02 03 04 [X-Y] 05 06 07 08 }, the string @@ -304,58 +307,31 @@ struct YR_STRING int32_t chain_gap_min; int32_t chain_gap_max; - // If the string can only match at a specific offset (for example if the - // condition is "$a at 0" the string $a can only match at offset 0), the - // fixed_offset field contains the offset, it have the UNDEFINED value for - // strings that can match anywhere. - int64_t fixed_offset; - - // Each item in the "matches" array represents a list of matches, there's one - // list of matches for each thread currently using the compiled rules. Up to - // YR_MAX_THREADS can use the compiled rules simultaneosly, that's why the - // array has YR_MAX_THREADS slots. The lists contain the matches that have - // been found so far. - YR_MATCHES matches[YR_MAX_THREADS]; - - // "private_matches" is similar to "matches" but it holds the matches for - // strings that are flagged as private. The matches for such strings are not - // put in the "matches" arrays. - YR_MATCHES private_matches[YR_MAX_THREADS]; - - // "unconfirmed_matches" is used for strings that are part of a chain. Let's - // suppose that the string S is split in two chained strings S1 <- S2. When - // a match is found for S1, we can't be sure that S matches until a match for - // S2 is found (within the range defined by chain_gap_min and chain_gap_max), - // so the matches for S1 are put in "unconfirmed_matches" until they can be - // confirmed or discarded. - YR_MATCHES unconfirmed_matches[YR_MAX_THREADS]; + // Identifier of this string. + DECLARE_REFERENCE(const char*, identifier); }; struct YR_RULE { - int32_t g_flags; // Global flags - int32_t t_flags[YR_MAX_THREADS]; // Thread-specific flags + int32_t flags; + + // Number of atoms generated for this rule. + int32_t num_atoms; DECLARE_REFERENCE(const char*, identifier); DECLARE_REFERENCE(const char*, tags); DECLARE_REFERENCE(YR_META*, metas); DECLARE_REFERENCE(YR_STRING*, strings); DECLARE_REFERENCE(YR_NAMESPACE*, ns); +}; - // Number of atoms generated for this rule. - int32_t num_atoms; - // Used only when PROFILING_ENABLED is defined. This is the sum of all values - // in time_cost_per_thread. This is updated once on each call to - // yr_scanner_scan_xxx. - volatile int64_t time_cost; - - // Used only when PROFILING_ENABLED is defined. This array holds the time - // cost for each thread using this structure concurrenlty. This is necessary - // because a global variable causes too much contention while trying to - // increment in a synchronized way from multiple threads. - int64_t time_cost_per_thread[YR_MAX_THREADS]; +struct YR_SUMMARY +{ + uint32_t num_rules; + uint32_t num_strings; + uint32_t num_namespaces; }; @@ -375,56 +351,22 @@ struct YR_EXTERNAL_VARIABLE struct YR_AC_MATCH { - uint16_t backtrack; - DECLARE_REFERENCE(YR_STRING*, string); DECLARE_REFERENCE(const uint8_t*, forward_code); DECLARE_REFERENCE(const uint8_t*, backward_code); DECLARE_REFERENCE(YR_AC_MATCH*, next); + + // When the Aho-Corasick automaton reaches some state that has associated + // matches, the current position in the input buffer is a few bytes past + // the point where the match actually occurs, for example, when looking for + // string "bar" in "foobarbaz", when the automaton reaches the state associated + // to the ending "r" in "bar, which is the one that has a match, the current + // position in the input is 6 (the "b" after the "r"), but the match is at + // position 3. The backtrack field indicates how many bytes the scanner has + // to go back to find the point where the match actually start. + uint16_t backtrack; }; - -struct YR_AC_MATCH_TABLE_ENTRY -{ - DECLARE_REFERENCE(YR_AC_MATCH*, match); -}; - - -typedef uint32_t YR_AC_TRANSITION; -typedef YR_AC_TRANSITION* YR_AC_TRANSITION_TABLE; -typedef YR_AC_MATCH_TABLE_ENTRY* YR_AC_MATCH_TABLE; - - -struct YR_AC_TABLES -{ - YR_AC_TRANSITION_TABLE transitions; - YR_AC_MATCH_TABLE matches; -}; - - -typedef struct YARA_RULES_FILE_HEADER -{ - DECLARE_REFERENCE(YR_RULE*, rules_list_head); - DECLARE_REFERENCE(YR_EXTERNAL_VARIABLE*, externals_list_head); - DECLARE_REFERENCE(const uint8_t*, code_start); - DECLARE_REFERENCE(YR_AC_MATCH_TABLE, ac_match_table); - DECLARE_REFERENCE(YR_AC_TRANSITION_TABLE, ac_transition_table); - - // Size of ac_match_table and ac_transition_table in number of items (both - // tables have the same number of items) - uint32_t ac_tables_size; - -} YARA_RULES_FILE_HEADER; - - -typedef struct _YR_INIT_RULE_ARGS -{ - DECLARE_REFERENCE(YR_RULE*, rule); - DECLARE_REFERENCE(const uint8_t*, jmp_addr); - -} YR_INIT_RULE_ARGS; - - #pragma pack(pop) @@ -456,8 +398,8 @@ struct RE_NODE RE_NODE* prev_sibling; RE_NODE* next_sibling; - uint8_t* forward_code; - uint8_t* backward_code; + YR_ARENA_REF forward_code_ref; + YR_ARENA_REF backward_code_ref; }; @@ -505,10 +447,10 @@ struct RE_FIBER int32_t sp; // stack pointer int32_t rc; // repeat counter - uint16_t stack[RE_MAX_STACK]; - RE_FIBER* prev; RE_FIBER* next; + + uint16_t stack[RE_MAX_STACK]; }; @@ -526,6 +468,24 @@ struct RE_FIBER_POOL }; +struct YR_MODIFIER +{ + int32_t flags; + uint8_t xor_min; + uint8_t xor_max; + SIZED_STRING *alphabet; +}; + + +struct YR_MATCHES +{ + YR_MATCH* head; + YR_MATCH* tail; + + int32_t count; +}; + + struct YR_MATCH { int64_t base; // Base address for the match @@ -535,29 +495,48 @@ struct YR_MATCH // Pointer to a buffer containing a portion of the matched data. The size of // the buffer is data_length. data_length is always <= length and is limited - // to MAX_MATCH_DATA bytes. + // to YR_CONFIG_MAX_MATCH_DATA bytes. const uint8_t* data; + YR_MATCH* prev; + YR_MATCH* next; + // If the match belongs to a chained string chain_length contains the // length of the chain. This field is used only in unconfirmed matches. int32_t chain_length; - YR_MATCH* prev; - YR_MATCH* next; + // True if this is match for a private string. + bool private; }; struct YR_AC_STATE { + YR_AC_STATE* failure; + YR_AC_STATE* first_child; + YR_AC_STATE* siblings; + + // Reference to the YR_AC_MATCH structure that heads the list of matches + // for this state. + YR_ARENA_REF matches_ref; + uint8_t depth; uint8_t input; uint32_t t_table_slot; +}; - YR_AC_STATE* failure; - YR_AC_STATE* first_child; - YR_AC_STATE* siblings; - YR_AC_MATCH* matches; + +struct YR_AC_MATCH_LIST_ENTRY +{ + uint16_t backtrack; + uint32_t string_idx; + + YR_ARENA_REF ref; + YR_ARENA_REF forward_code_ref; + YR_ARENA_REF backward_code_ref; + + YR_AC_MATCH_LIST_ENTRY* next; }; @@ -567,49 +546,54 @@ struct YR_AC_AUTOMATON // stored in tables_size. uint32_t tables_size; + // The first slot in the transition table (t_table) that may be be unused. + // Used for speeding up the construction of the transition table. uint32_t t_table_unused_candidate; - // Bitmask where each bit indicates if the corresponding slot in m_table - // and t_table is already in use. + // Bitmask where each bit indicates if the corresponding slot in the + // transition table is already in use. YR_BITMASK* bitmask; - YR_AC_TRANSITION_TABLE t_table; - YR_AC_MATCH_TABLE m_table; + // Pointer to the root Aho-Corasick state. YR_AC_STATE* root; }; struct YR_RULES { - unsigned char tidx_mask[YR_BITARRAY_NCHARS(YR_MAX_THREADS)]; + YR_ARENA* arena; + + YR_RULE* rules_list_head; + YR_STRING* strings_list_head; + YR_EXTERNAL_VARIABLE* externals_list_head; + + YR_AC_TRANSITION* ac_transition_table; + YR_AC_MATCH* ac_match_pool; + uint32_t* ac_match_table; + const uint8_t* code_start; - YR_MUTEX mutex; - YR_ARENA* arena; - YR_RULE* rules_list_head; - YR_EXTERNAL_VARIABLE* externals_list_head; - YR_AC_TRANSITION_TABLE ac_transition_table; - YR_AC_MATCH_TABLE ac_match_table; + // Total number of rules. + uint32_t num_rules; - // Size of ac_match_table and ac_transition_table in number of items (both - // tables have the same numbe of items). - uint32_t ac_tables_size; + // Total number of strings. + uint32_t num_strings; - // Used only when PROFILING_ENABLED is defined. - uint64_t time_cost; + // Total number of namespaces. + uint32_t num_namespaces; }; struct YR_RULES_STATS { // Total number of rules - uint32_t rules; + uint32_t num_rules; // Total number of strings across all rules. - uint32_t strings; + uint32_t num_strings; // Total number of Aho-Corasick matches. Each node in the Aho-Corasick - // automaton has a list of YR_AC_MATCH structures (match list) pointing to + // automaton has a list of YR_AC_MATCH_LIST_ENTRY structures (match list) pointing to // strings that are potential matches. This field holds the total number of // those structures across all nodes in the automaton. uint32_t ac_matches; @@ -661,6 +645,7 @@ struct YR_MEMORY_BLOCK_ITERATOR typedef int (*YR_CALLBACK_FUNC)( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data); @@ -677,12 +662,6 @@ struct YR_SCAN_CONTEXT // Scanning flags. int flags; - // Thread index for the thread using this scan context. The number of threads - // that can use a YR_RULES object simultaneusly is limited by the YR_MAX_THREADS - // constant. Each thread using a YR_RULES get assigned a unique thread index - // in the range [0, YR_MAX_THREADS) - int tidx; - // Canary value used for preventing hand-crafted objects from being embedded // in compiled rules and used to exploit YARA. The canary value is initialized // to a random value and is subsequently set to all objects created by @@ -713,18 +692,41 @@ struct YR_SCAN_CONTEXT // contains entries for external variables and modules. YR_HASH_TABLE* objects_table; - // Arena used for storing YR_MATCH structures asociated to the matches found. - YR_ARENA* matches_arena; - - // Arena used for storing pointers to the YR_STRING struct for each matching - // string. The pointers are used by _yr_scanner_clean_matches. - YR_ARENA* matching_strings_arena; + // Notebook used for storing YR_MATCH structures associated to the matches + // found. + YR_NOTEBOOK * matches_notebook; // Stopwatch used for measuring the time elapsed during the scan. YR_STOPWATCH stopwatch; // Fiber pool used by yr_re_exec. RE_FIBER_POOL re_fiber_pool; + + // A bitmap with one bit per rule, bit N is set when the rule with index N + // has matched. + YR_BITMASK* rule_matches_flags; + + // A bitmap with one bit per namespace, bit N is set if the namespace with + // index N has some global rule that is not satisfied. + YR_BITMASK* ns_unsatisfied_flags; + + // Array with pointers to lists of matches. Item N in the array has the + // list of matches for string with index N. + YR_MATCHES* matches; + + // "unconfirmed_matches" is like "matches" but for strings that are part of + // a chain. Let's suppose that the string S is split in two chained strings + // S1 <- S2. When a match is found for S1, we can't be sure that S matches + // until a match for S2 is found (within the range defined by chain_gap_min + // and chain_gap_max), so the matches for S1 are put in "unconfirmed_matches" + // until they can be confirmed or discarded. + YR_MATCHES* unconfirmed_matches; + + // rule_cost is a pointer to an array of 64-bit integers with one entry per + // rule. Entry N has the time cost for rule with index N. + #ifdef PROFILING_ENABLED + uint64_t* time_cost; + #endif }; @@ -865,7 +867,7 @@ struct YR_DICTIONARY_ITEMS // // The boolean shouldn't be true if the next item was pushed in the stack, it // can be true only when all the items have been returned in previous calls, -// in which case the value for the next item should be UNDEFINED. The stack +// in which case the value for the next item should be YR_UNDEFINED. The stack // should look like this after the last call to "next": // // +------------+ @@ -875,8 +877,8 @@ struct YR_DICTIONARY_ITEMS // +------------+ // | . . . | // -// We can't use the UNDEFINED value in the stack as an indicator of the end -// of the iterator, because it's legitimate for an iterator to return UNDEFINED +// We can't use the YR_UNDEFINED value in the stack as an indicator of the end +// of the iterator, because it's legitimate for an iterator to return YR_UNDEFINED // items in the middle of the iteration. // // The "next" function should return ERROR_SUCCESS if everything went fine or diff --git a/libyara/lexer.c b/libyara/lexer.c index 8de34de2..cb8dee71 100644 --- a/libyara/lexer.c +++ b/libyara/lexer.c @@ -580,8 +580,8 @@ static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner ); yyg->yy_hold_char = *yy_cp; \ *yy_cp = '\0'; \ yyg->yy_c_buf_p = yy_cp; -#define YY_NUM_RULES 75 -#define YY_END_OF_BUFFER 76 +#define YY_NUM_RULES 77 +#define YY_END_OF_BUFFER 78 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -589,39 +589,36 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[286] = +static const flex_int16_t yy_accept[258] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 74, 73, 73, 49, 70, 47, 46, 74, 71, - 52, 52, 2, 74, 3, 48, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 74, 62, 63, 56, 75, 68, 69, - 65, 75, 43, 44, 40, 40, 49, 7, 47, 45, - 46, 1, 38, 41, 0, 52, 0, 0, 0, 0, - 8, 4, 6, 5, 9, 48, 51, 51, 51, 51, - 26, 51, 51, 51, 51, 51, 51, 51, 51, 27, - 51, 51, 51, 28, 25, 51, 51, 51, 51, 51, + 78, 76, 75, 75, 51, 72, 49, 48, 76, 73, + 54, 54, 2, 76, 3, 50, 53, 53, 53, 53, + 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 53, 53, 53, 53, 76, 64, 65, 58, 77, 70, + 71, 67, 77, 45, 46, 42, 42, 51, 7, 49, + 47, 48, 1, 40, 43, 0, 54, 0, 0, 0, + 0, 8, 4, 6, 5, 9, 50, 53, 53, 53, + 53, 28, 53, 53, 53, 53, 53, 53, 53, 53, + 53, 29, 53, 53, 53, 30, 27, 53, 53, 53, - 51, 51, 51, 0, 0, 62, 64, 59, 60, 58, - 57, 64, 68, 65, 65, 67, 66, 43, 39, 41, - 53, 52, 55, 54, 31, 24, 32, 51, 51, 51, - 51, 51, 30, 51, 51, 51, 51, 51, 51, 51, - 51, 23, 51, 51, 51, 51, 51, 51, 51, 17, - 72, 0, 0, 0, 51, 51, 51, 51, 51, 51, - 51, 51, 51, 51, 51, 51, 50, 51, 13, 51, - 51, 12, 51, 29, 21, 16, 0, 0, 0, 0, - 0, 72, 61, 15, 51, 51, 51, 22, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 0, 0, + 53, 53, 53, 53, 53, 0, 0, 64, 66, 61, + 62, 60, 59, 66, 70, 67, 67, 69, 68, 45, + 41, 43, 55, 54, 57, 56, 33, 26, 34, 53, + 53, 53, 53, 53, 53, 32, 53, 53, 53, 53, + 53, 53, 53, 53, 25, 53, 53, 53, 53, 53, + 53, 53, 17, 74, 0, 0, 0, 53, 53, 53, + 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 52, 53, 13, 53, 53, 12, 53, 31, 23, 16, + 0, 0, 0, 0, 0, 74, 63, 15, 53, 53, + 53, 53, 24, 53, 53, 53, 53, 53, 53, 53, - 0, 51, 51, 51, 51, 51, 11, 37, 51, 50, - 51, 19, 51, 51, 0, 0, 0, 0, 0, 0, - 0, 72, 51, 51, 51, 51, 51, 51, 35, 10, - 14, 0, 72, 0, 0, 0, 0, 72, 0, 0, - 51, 36, 51, 34, 18, 0, 0, 0, 0, 0, - 0, 0, 0, 72, 0, 72, 0, 0, 0, 0, - 72, 20, 51, 42, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 72, 0, 0, 0, 72, 33, - 0, 0, 0, 0, 0 + 53, 53, 53, 0, 0, 18, 53, 53, 53, 53, + 53, 11, 39, 53, 52, 53, 21, 53, 53, 0, + 0, 0, 0, 0, 74, 53, 53, 53, 53, 53, + 53, 53, 37, 10, 14, 0, 74, 0, 0, 0, + 53, 53, 38, 53, 36, 20, 0, 0, 0, 0, + 53, 22, 53, 44, 19, 35, 0 } ; static const YY_CHAR yy_ec[256] = @@ -631,15 +628,15 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 5, 6, 7, 8, 1, 1, 1, 9, 9, 10, 1, 1, 9, 11, 12, 13, 14, 15, - 16, 17, 17, 18, 17, 19, 20, 1, 1, 21, - 22, 23, 9, 24, 25, 26, 25, 25, 25, 25, - 27, 27, 27, 27, 28, 27, 29, 27, 27, 27, - 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, - 9, 30, 9, 1, 31, 1, 32, 33, 34, 35, + 16, 17, 18, 19, 18, 20, 21, 1, 1, 22, + 23, 24, 9, 25, 26, 27, 26, 26, 26, 26, + 28, 28, 28, 28, 29, 28, 30, 28, 28, 28, + 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 9, 31, 9, 1, 32, 1, 33, 34, 35, 36, - 36, 37, 38, 39, 40, 27, 27, 41, 42, 43, - 44, 45, 27, 46, 47, 48, 49, 50, 51, 52, - 53, 54, 55, 9, 56, 1, 1, 1, 1, 1, + 37, 38, 39, 40, 41, 28, 28, 42, 43, 44, + 45, 46, 28, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 9, 57, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -656,343 +653,243 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[57] = +static const YY_CHAR yy_meta[58] = { 0, 1, 2, 3, 2, 1, 4, 1, 1, 2, 5, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8, - 1, 9, 1, 1, 10, 10, 11, 12, 12, 13, - 11, 10, 10, 10, 10, 10, 10, 11, 11, 11, - 11, 11, 11, 12, 11, 11, 11, 11, 11, 11, - 11, 12, 11, 11, 1, 1 + 8, 1, 9, 1, 1, 10, 10, 11, 12, 12, + 13, 11, 10, 10, 10, 10, 10, 10, 11, 11, + 11, 11, 11, 11, 12, 11, 11, 11, 11, 11, + 11, 11, 12, 11, 11, 1, 1 } ; -static const flex_int16_t yy_base[311] = +static const flex_int16_t yy_base[280] = { 0, - 0, 0, 54, 55, 56, 59, 754, 753, 748, 747, - 756, 1020, 1020, 1020, 733, 1020, 0, 738, 736, 53, - 53, 59, 45, 724, 50, 0, 0, 51, 701, 701, - 51, 700, 32, 44, 696, 32, 693, 689, 680, 62, - 687, 685, 680, 711, 0, 1020, 1020, 84, 0, 1020, - 62, 710, 0, 1020, 1020, 537, 526, 1020, 0, 1020, - 537, 1020, 1020, 0, 102, 0, 520, 519, 124, 0, - 1020, 1020, 1020, 1020, 1020, 0, 0, 503, 71, 509, - 0, 499, 493, 499, 498, 492, 496, 492, 490, 45, - 486, 485, 62, 0, 0, 492, 484, 478, 487, 473, + 0, 0, 55, 56, 57, 60, 587, 586, 581, 580, + 589, 594, 594, 594, 565, 594, 0, 577, 575, 54, + 54, 60, 45, 562, 50, 0, 0, 33, 551, 538, + 538, 52, 539, 35, 43, 535, 54, 532, 528, 528, + 56, 535, 534, 529, 561, 0, 594, 594, 88, 0, + 594, 63, 560, 0, 594, 594, 559, 547, 594, 0, + 594, 559, 594, 594, 0, 0, 0, 541, 540, 107, + 0, 594, 594, 594, 594, 594, 0, 0, 524, 62, + 530, 0, 516, 519, 513, 519, 518, 512, 516, 512, + 510, 60, 506, 505, 65, 0, 0, 512, 510, 504, - 478, 483, 471, 92, 113, 0, 1020, 1020, 1020, 1020, - 1020, 0, 0, 469, 1020, 1020, 1020, 0, 1020, 0, - 136, 1020, 144, 0, 0, 0, 0, 475, 78, 459, - 455, 465, 0, 459, 466, 338, 340, 114, 346, 347, - 346, 0, 327, 340, 335, 332, 337, 324, 335, 0, - 1020, 360, 161, 0, 329, 328, 335, 313, 329, 311, - 306, 324, 309, 305, 333, 335, 316, 309, 0, 291, - 303, 0, 291, 0, 0, 0, 323, 119, 216, 0, - 133, 0, 1020, 0, 284, 231, 225, 0, 229, 224, - 226, 218, 230, 228, 227, 226, 213, 222, 249, 272, + 513, 499, 504, 511, 499, 58, 96, 0, 594, 594, + 594, 594, 594, 0, 0, 497, 594, 594, 594, 0, + 594, 0, 0, 594, 129, 0, 0, 0, 0, 503, + 506, 81, 495, 493, 503, 0, 497, 504, 492, 494, + 113, 500, 501, 500, 0, 481, 494, 489, 486, 491, + 478, 489, 0, 594, 515, 148, 0, 483, 504, 481, + 488, 466, 482, 470, 465, 483, 468, 464, 494, 496, + 461, 452, 0, 432, 435, 0, 410, 0, 0, 0, + 436, 100, 0, 0, 124, 0, 594, 0, 425, 369, + 376, 367, 0, 369, 364, 366, 353, 361, 357, 356, - 134, 218, 214, 212, 201, 208, 0, 0, 211, 0, - 199, 0, 209, 197, 233, 156, 0, 0, 327, 383, - 157, 0, 196, 191, 187, 190, 181, 199, 0, 0, - 0, 172, 182, 439, 0, 494, 173, 0, 550, 178, - 129, 0, 128, 0, 0, 197, 179, 0, 0, 0, - 0, 137, 190, 0, 194, 125, 606, 0, 662, 195, - 0, 0, 86, 1020, 717, 102, 198, 199, 0, 0, - 0, 0, 97, 202, 0, 0, 203, 211, 0, 0, - 67, 212, 227, 0, 1020, 773, 786, 799, 812, 818, - 823, 831, 838, 843, 848, 859, 869, 881, 894, 906, - - 919, 932, 938, 941, 951, 964, 970, 980, 993, 1006 + 355, 341, 346, 205, 128, 331, 341, 333, 272, 261, + 268, 0, 0, 277, 0, 265, 0, 275, 263, 0, + 0, 261, 298, 256, 0, 268, 263, 259, 265, 268, + 268, 301, 0, 0, 0, 257, 292, 273, 289, 274, + 264, 249, 0, 228, 0, 0, 129, 279, 0, 280, + 102, 0, 79, 594, 0, 0, 594, 318, 331, 344, + 357, 363, 368, 376, 383, 388, 393, 404, 414, 426, + 439, 451, 464, 477, 110, 483, 486, 496, 502 } ; -static const flex_int16_t yy_def[311] = +static const flex_int16_t yy_def[280] = { 0, - 285, 1, 286, 286, 287, 287, 288, 288, 289, 289, - 285, 285, 285, 285, 290, 285, 291, 292, 285, 285, - 293, 293, 285, 285, 285, 294, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 296, 297, 285, 285, 298, 299, 285, - 285, 300, 301, 285, 285, 285, 290, 285, 291, 285, - 292, 285, 285, 302, 285, 22, 285, 285, 285, 303, - 285, 285, 285, 285, 285, 294, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 295, 295, + 257, 1, 258, 258, 259, 259, 260, 260, 261, 261, + 257, 257, 257, 257, 262, 257, 263, 264, 257, 257, + 265, 265, 257, 257, 257, 266, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 268, 269, 257, 257, 270, 271, + 257, 257, 272, 273, 257, 257, 257, 262, 257, 263, + 257, 264, 257, 257, 274, 275, 22, 257, 257, 257, + 276, 257, 257, 257, 257, 257, 266, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 295, 295, 295, 296, 285, 297, 285, 285, 285, 285, - 285, 304, 299, 285, 285, 285, 285, 301, 285, 302, - 285, 285, 285, 303, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 295, 295, - 285, 305, 306, 307, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 305, 308, 306, 179, - 179, 179, 285, 295, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 295, 295, 295, 295, 309, 285, + 267, 267, 267, 267, 267, 268, 257, 269, 257, 257, + 257, 257, 257, 277, 271, 257, 257, 257, 257, 273, + 257, 274, 275, 257, 257, 276, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 257, 278, 257, 279, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + 278, 278, 156, 156, 156, 156, 257, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, - 179, 295, 295, 295, 295, 295, 295, 295, 295, 295, - 295, 295, 295, 295, 309, 308, 200, 200, 305, 285, - 200, 200, 295, 295, 295, 295, 295, 295, 295, 295, - 295, 305, 305, 285, 234, 309, 179, 234, 285, 200, - 295, 295, 295, 295, 295, 285, 310, 200, 234, 234, - 236, 220, 234, 234, 309, 309, 285, 257, 285, 200, - 257, 295, 295, 285, 259, 239, 234, 310, 234, 257, - 257, 259, 220, 257, 257, 265, 308, 265, 265, 295, - 239, 257, 310, 257, 0, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285 + 267, 267, 267, 257, 156, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 204, + 204, 278, 204, 204, 204, 267, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 278, 278, 156, 204, 204, + 267, 267, 267, 267, 267, 267, 257, 278, 204, 204, + 267, 267, 267, 257, 267, 267, 0, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257 } ; -static const flex_int16_t yy_nxt[1077] = +static const flex_int16_t yy_nxt[652] = { 0, 12, 13, 14, 13, 15, 16, 17, 18, 12, 12, 19, 20, 21, 22, 22, 22, 22, 22, 22, 22, - 23, 24, 25, 26, 27, 27, 27, 27, 27, 12, - 27, 28, 27, 29, 27, 30, 31, 32, 27, 33, - 27, 34, 35, 36, 37, 38, 39, 40, 41, 27, - 42, 43, 27, 27, 44, 12, 46, 46, 50, 47, - 47, 50, 63, 65, 64, 71, 72, 51, 94, 65, - 51, 74, 75, 89, 90, 91, 273, 95, 137, 92, - 67, 68, 84, 48, 48, 52, 67, 68, 52, 108, - 85, 78, 138, 79, 86, 141, 69, 80, 81, 87, + 22, 23, 24, 25, 26, 27, 27, 27, 27, 27, + 12, 27, 28, 29, 30, 27, 31, 32, 33, 27, + 34, 27, 35, 36, 37, 38, 39, 40, 41, 42, + 27, 43, 44, 27, 27, 45, 12, 47, 47, 51, + 48, 48, 51, 64, 66, 65, 72, 73, 52, 107, + 66, 52, 75, 76, 79, 93, 80, 91, 92, 94, + 81, 82, 68, 69, 86, 49, 49, 53, 68, 69, + 53, 96, 87, 110, 140, 101, 88, 128, 70, 144, - 99, 114, 285, 105, 70, 126, 273, 100, 115, 142, - 285, 273, 156, 109, 121, 121, 121, 121, 121, 121, - 121, 121, 152, 127, 153, 157, 110, 165, 178, 166, - 104, 111, 167, 280, 216, 112, 123, 123, 123, 123, - 123, 123, 200, 200, 201, 201, 252, 151, 121, 121, - 121, 121, 121, 121, 121, 121, 123, 123, 123, 123, - 123, 123, 180, 104, 180, 216, 239, 104, 240, 180, - 263, 262, 181, 180, 180, 180, 180, 180, 180, 180, - 180, 247, 200, 248, 201, 180, 180, 239, 178, 240, - 219, 178, 180, 180, 180, 180, 180, 180, 246, 266, + 97, 89, 102, 116, 257, 155, 71, 156, 141, 182, + 117, 106, 257, 145, 154, 129, 160, 123, 111, 125, + 125, 125, 125, 125, 125, 125, 169, 256, 170, 161, + 247, 112, 171, 204, 254, 205, 113, 204, 255, 205, + 114, 125, 125, 125, 125, 125, 125, 125, 183, 184, + 106, 184, 183, 183, 183, 183, 184, 183, 183, 185, + 184, 184, 184, 184, 184, 184, 184, 184, 184, 183, + 183, 183, 183, 184, 184, 183, 183, 183, 183, 183, + 184, 184, 184, 184, 184, 184, 183, 183, 183, 183, + 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, - 246, 267, 264, 268, 239, 269, 240, 266, 277, 267, - 219, 281, 277, 282, 104, 245, 182, 180, 104, 180, - 283, 281, 284, 282, 180, 244, 243, 181, 180, 180, - 180, 180, 180, 180, 180, 180, 277, 242, 219, 241, - 180, 180, 216, 231, 230, 229, 228, 180, 180, 180, - 180, 180, 180, 227, 226, 225, 224, 223, 216, 214, - 213, 212, 211, 210, 209, 208, 207, 206, 205, 204, - 203, 182, 217, 218, 219, 218, 217, 217, 217, 217, - 218, 220, 217, 221, 218, 218, 218, 218, 218, 218, - 218, 218, 217, 217, 217, 217, 218, 218, 217, 217, + 183, 183, 183, 183, 186, 220, 221, 222, 221, 220, + 220, 220, 220, 221, 223, 220, 224, 221, 221, 221, + 221, 221, 221, 221, 221, 221, 220, 220, 220, 220, + 221, 221, 220, 220, 220, 220, 220, 221, 221, 221, + 221, 221, 221, 220, 220, 220, 220, 220, 220, 220, + 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, + 220, 225, 222, 222, 222, 239, 248, 240, 249, 222, + 182, 253, 236, 222, 222, 222, 222, 222, 222, 222, + 222, 222, 204, 239, 205, 240, 222, 222, 182, 239, + 222, 240, 252, 222, 222, 222, 222, 222, 222, 251, - 217, 217, 217, 218, 218, 218, 218, 218, 218, 217, - 217, 217, 217, 217, 217, 217, 217, 217, 217, 217, - 217, 217, 217, 217, 217, 217, 217, 222, 219, 219, - 219, 202, 178, 198, 197, 219, 178, 196, 232, 219, - 219, 219, 219, 219, 219, 219, 219, 195, 194, 167, - 167, 219, 219, 193, 192, 191, 190, 189, 219, 219, - 219, 219, 219, 219, 188, 187, 186, 185, 184, 178, - 176, 138, 175, 174, 173, 172, 171, 170, 169, 168, - 164, 163, 233, 234, 235, 236, 235, 234, 234, 234, - 234, 235, 220, 234, 237, 235, 235, 235, 235, 235, + 250, 182, 247, 246, 245, 244, 243, 242, 241, 238, + 235, 234, 233, 232, 231, 230, 229, 237, 46, 46, + 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, + 46, 50, 50, 50, 50, 50, 50, 50, 50, 50, + 50, 50, 50, 50, 54, 54, 54, 54, 54, 54, + 54, 54, 54, 54, 54, 54, 54, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 58, 58, 58, 58, 58, 60, 228, 60, 60, 60, + 62, 227, 226, 62, 219, 62, 62, 62, 67, 218, + 67, 217, 216, 215, 67, 77, 214, 77, 77, 77, - 235, 235, 235, 234, 234, 234, 234, 235, 235, 234, - 234, 234, 234, 234, 235, 235, 235, 235, 235, 235, - 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, - 234, 234, 234, 234, 234, 234, 234, 234, 238, 249, - 250, 251, 250, 249, 249, 249, 249, 250, 252, 249, - 253, 250, 250, 250, 250, 250, 250, 250, 250, 249, - 249, 249, 249, 250, 250, 249, 249, 249, 249, 249, - 250, 250, 250, 250, 250, 250, 249, 249, 249, 249, - 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - 249, 249, 249, 249, 254, 251, 251, 251, 162, 161, + 78, 213, 78, 78, 78, 106, 106, 212, 211, 210, + 106, 106, 209, 106, 108, 108, 208, 207, 108, 108, + 108, 108, 108, 108, 108, 108, 109, 109, 109, 109, + 109, 109, 109, 109, 109, 109, 109, 109, 109, 115, + 115, 206, 115, 115, 115, 182, 115, 115, 115, 115, + 115, 118, 118, 203, 118, 118, 118, 118, 118, 118, + 118, 118, 118, 118, 120, 120, 120, 202, 120, 120, + 120, 120, 120, 120, 120, 120, 120, 122, 122, 201, + 122, 122, 122, 122, 122, 122, 122, 122, 122, 122, + 126, 200, 126, 157, 199, 157, 181, 181, 181, 181, + + 181, 181, 181, 181, 181, 181, 181, 181, 181, 187, + 171, 187, 171, 198, 197, 196, 195, 194, 193, 192, + 191, 190, 189, 188, 182, 180, 141, 179, 178, 177, + 176, 175, 174, 173, 172, 168, 167, 166, 165, 164, + 163, 162, 159, 158, 117, 153, 152, 151, 150, 149, + 148, 147, 146, 143, 142, 139, 138, 137, 136, 135, + 134, 133, 132, 131, 130, 127, 124, 124, 61, 257, + 121, 119, 107, 105, 104, 103, 100, 99, 98, 95, + 90, 85, 84, 83, 74, 63, 61, 59, 257, 57, + 57, 55, 55, 11, 257, 257, 257, 257, 257, 257, - 160, 159, 251, 216, 158, 255, 251, 251, 251, 251, - 251, 251, 251, 251, 155, 115, 150, 149, 251, 251, - 148, 147, 146, 145, 144, 251, 251, 251, 251, 251, - 251, 143, 140, 139, 136, 135, 134, 133, 132, 131, - 130, 129, 128, 125, 122, 122, 60, 285, 119, 256, - 257, 258, 259, 258, 257, 257, 257, 257, 258, 220, - 257, 260, 258, 258, 258, 258, 258, 258, 258, 258, - 257, 257, 257, 257, 258, 258, 257, 257, 257, 257, - 257, 258, 258, 258, 258, 258, 258, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, - - 257, 257, 257, 257, 257, 261, 270, 271, 272, 271, - 270, 270, 270, 270, 271, 273, 270, 274, 271, 271, - 271, 271, 271, 271, 271, 271, 270, 270, 270, 270, - 271, 271, 270, 270, 270, 270, 270, 271, 271, 271, - 271, 271, 271, 270, 270, 270, 270, 270, 270, 270, - 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - 270, 275, 276, 272, 272, 272, 276, 276, 276, 276, - 272, 277, 276, 278, 272, 272, 272, 272, 272, 272, - 272, 272, 276, 276, 276, 276, 272, 272, 276, 276, - 276, 276, 276, 272, 272, 272, 272, 272, 272, 276, - - 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 279, 276, 276, - 276, 117, 105, 103, 102, 276, 101, 98, 276, 276, - 276, 276, 276, 276, 276, 276, 276, 97, 96, 93, - 88, 276, 276, 83, 82, 73, 62, 60, 276, 276, - 276, 276, 276, 276, 58, 285, 56, 56, 54, 54, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 276, 45, 45, 45, 45, 45, 45, 45, - 45, 45, 45, 45, 45, 45, 49, 49, 49, 49, - 49, 49, 49, 49, 49, 49, 49, 49, 49, 53, - - 53, 53, 53, 53, 53, 53, 53, 53, 53, 53, - 53, 53, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 57, 57, 57, 57, 57, - 59, 285, 59, 59, 59, 61, 285, 285, 61, 285, - 61, 61, 61, 66, 285, 66, 285, 285, 285, 66, - 76, 285, 76, 76, 76, 77, 285, 77, 77, 77, - 104, 104, 285, 285, 285, 104, 104, 285, 104, 106, - 106, 285, 285, 106, 106, 106, 106, 106, 106, 106, - 106, 107, 107, 107, 107, 107, 107, 107, 107, 107, - 107, 107, 107, 107, 113, 113, 285, 113, 113, 113, - - 285, 113, 113, 113, 113, 113, 116, 116, 285, 116, - 116, 116, 116, 116, 116, 116, 116, 116, 116, 118, - 118, 118, 285, 118, 118, 118, 118, 118, 118, 118, - 118, 118, 120, 120, 285, 120, 120, 120, 120, 120, - 120, 120, 120, 120, 120, 124, 285, 124, 154, 285, - 154, 177, 177, 177, 177, 177, 177, 177, 177, 177, - 177, 177, 177, 177, 179, 179, 179, 179, 179, 179, - 179, 179, 179, 179, 179, 179, 179, 183, 285, 183, - 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, - 199, 199, 199, 215, 215, 215, 215, 215, 215, 215, - - 215, 215, 215, 215, 215, 215, 265, 265, 265, 265, - 265, 265, 265, 265, 265, 265, 265, 265, 265, 11, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285 + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257 } ; -static const flex_int16_t yy_chk[1077] = +static const flex_int16_t yy_chk[652] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 3, 4, 5, 3, - 4, 6, 20, 21, 20, 23, 23, 5, 36, 22, - 6, 25, 25, 33, 33, 34, 281, 36, 90, 34, - 21, 21, 31, 3, 4, 5, 22, 22, 6, 48, - 31, 28, 90, 28, 31, 93, 21, 28, 28, 31, + 1, 1, 1, 1, 1, 1, 1, 3, 4, 5, + 3, 4, 6, 20, 21, 20, 23, 23, 5, 106, + 22, 6, 25, 25, 28, 35, 28, 34, 34, 35, + 28, 28, 21, 21, 32, 3, 4, 5, 22, 22, + 6, 37, 32, 49, 92, 41, 32, 80, 21, 95, - 40, 51, 22, 104, 21, 79, 273, 40, 51, 93, - 22, 266, 129, 48, 65, 65, 65, 65, 65, 65, - 65, 65, 105, 79, 105, 129, 48, 138, 178, 138, - 178, 48, 138, 263, 256, 48, 69, 69, 69, 69, - 69, 69, 181, 201, 181, 201, 252, 104, 121, 121, - 121, 121, 121, 121, 121, 121, 123, 123, 123, 123, - 123, 123, 153, 153, 153, 216, 221, 216, 221, 153, - 243, 241, 153, 153, 153, 153, 153, 153, 153, 153, - 153, 232, 237, 232, 237, 153, 153, 240, 247, 240, - 247, 233, 153, 153, 153, 153, 153, 153, 246, 253, + 37, 32, 41, 52, 22, 107, 21, 107, 92, 182, + 52, 182, 22, 95, 106, 80, 132, 275, 49, 70, + 70, 70, 70, 70, 70, 70, 141, 253, 141, 132, + 247, 49, 141, 185, 247, 185, 49, 205, 251, 205, + 49, 125, 125, 125, 125, 125, 125, 125, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, - 228, 253, 246, 255, 260, 255, 260, 267, 268, 267, - 268, 274, 277, 274, 277, 227, 153, 179, 179, 179, - 278, 282, 278, 282, 179, 226, 225, 179, 179, 179, - 179, 179, 179, 179, 179, 179, 283, 224, 283, 223, - 179, 179, 215, 214, 213, 211, 209, 179, 179, 179, - 179, 179, 179, 206, 205, 204, 203, 202, 199, 198, - 197, 196, 195, 194, 193, 192, 191, 190, 189, 187, - 186, 179, 200, 200, 200, 200, 200, 200, 200, 200, - 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, - 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, + 156, 156, 156, 156, 156, 204, 204, 204, 204, 204, + 204, 204, 204, 204, 204, 204, 204, 204, 204, 204, + 204, 204, 204, 204, 204, 204, 204, 204, 204, 204, + 204, 204, 204, 204, 204, 204, 204, 204, 204, 204, + 204, 204, 204, 204, 204, 204, 204, 204, 204, 204, + 204, 204, 204, 204, 204, 204, 204, 204, 204, 204, + 204, 204, 222, 222, 222, 224, 236, 224, 236, 222, + 222, 244, 222, 222, 222, 222, 222, 222, 222, 222, + 222, 222, 238, 240, 238, 240, 222, 222, 248, 250, + 248, 250, 242, 222, 222, 222, 222, 222, 222, 241, - 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, - 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, - 200, 200, 200, 200, 200, 200, 200, 200, 219, 219, - 219, 185, 177, 173, 171, 219, 219, 170, 219, 219, - 219, 219, 219, 219, 219, 219, 219, 168, 167, 166, - 165, 219, 219, 164, 163, 162, 161, 160, 219, 219, - 219, 219, 219, 219, 159, 158, 157, 156, 155, 152, - 149, 148, 147, 146, 145, 144, 143, 141, 140, 139, - 137, 136, 219, 220, 220, 220, 220, 220, 220, 220, - 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, + 239, 237, 232, 231, 230, 229, 228, 227, 226, 223, + 219, 218, 216, 214, 211, 210, 209, 222, 258, 258, + 258, 258, 258, 258, 258, 258, 258, 258, 258, 258, + 258, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 259, 259, 260, 260, 260, 260, 260, 260, + 260, 260, 260, 260, 260, 260, 260, 261, 261, 261, + 261, 261, 261, 261, 261, 261, 261, 261, 261, 261, + 262, 262, 262, 262, 262, 263, 208, 263, 263, 263, + 264, 207, 206, 264, 203, 264, 264, 264, 265, 202, + 265, 201, 200, 199, 265, 266, 198, 266, 266, 266, - 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, - 220, 220, 220, 220, 220, 220, 220, 220, 220, 234, - 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, - 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, - 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, - 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, - 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, - 234, 234, 234, 234, 234, 236, 236, 236, 135, 134, + 267, 197, 267, 267, 267, 268, 268, 196, 195, 194, + 268, 268, 192, 268, 269, 269, 191, 190, 269, 269, + 269, 269, 269, 269, 269, 269, 270, 270, 270, 270, + 270, 270, 270, 270, 270, 270, 270, 270, 270, 271, + 271, 189, 271, 271, 271, 181, 271, 271, 271, 271, + 271, 272, 272, 177, 272, 272, 272, 272, 272, 272, + 272, 272, 272, 272, 273, 273, 273, 175, 273, 273, + 273, 273, 273, 273, 273, 273, 273, 274, 274, 174, + 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, + 276, 172, 276, 277, 171, 277, 278, 278, 278, 278, - 132, 131, 236, 236, 130, 236, 236, 236, 236, 236, - 236, 236, 236, 236, 128, 114, 103, 102, 236, 236, - 101, 100, 99, 98, 97, 236, 236, 236, 236, 236, - 236, 96, 92, 91, 89, 88, 87, 86, 85, 84, - 83, 82, 80, 78, 68, 67, 61, 57, 56, 236, - 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, - 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + 278, 278, 278, 278, 278, 278, 278, 278, 278, 279, + 170, 279, 169, 168, 167, 166, 165, 164, 163, 162, + 161, 160, 159, 158, 155, 152, 151, 150, 149, 148, + 147, 146, 144, 143, 142, 140, 139, 138, 137, 135, + 134, 133, 131, 130, 116, 105, 104, 103, 102, 101, + 100, 99, 98, 94, 93, 91, 90, 89, 88, 87, + 86, 85, 84, 83, 81, 79, 69, 68, 62, 58, + 57, 53, 45, 44, 43, 42, 40, 39, 38, 36, + 33, 31, 30, 29, 24, 19, 18, 15, 11, 10, + 9, 8, 7, 257, 257, 257, 257, 257, 257, 257, - 239, 239, 239, 239, 239, 239, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, - 257, 257, 259, 259, 259, 259, 259, 259, 259, 259, - 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, - 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, - 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, - - 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, - 259, 259, 259, 259, 259, 259, 259, 259, 265, 265, - 265, 52, 44, 43, 42, 265, 41, 39, 265, 265, - 265, 265, 265, 265, 265, 265, 265, 38, 37, 35, - 32, 265, 265, 30, 29, 24, 19, 18, 265, 265, - 265, 265, 265, 265, 15, 11, 10, 9, 8, 7, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 265, 286, 286, 286, 286, 286, 286, 286, - 286, 286, 286, 286, 286, 286, 287, 287, 287, 287, - 287, 287, 287, 287, 287, 287, 287, 287, 287, 288, - - 288, 288, 288, 288, 288, 288, 288, 288, 288, 288, - 288, 288, 289, 289, 289, 289, 289, 289, 289, 289, - 289, 289, 289, 289, 289, 290, 290, 290, 290, 290, - 291, 0, 291, 291, 291, 292, 0, 0, 292, 0, - 292, 292, 292, 293, 0, 293, 0, 0, 0, 293, - 294, 0, 294, 294, 294, 295, 0, 295, 295, 295, - 296, 296, 0, 0, 0, 296, 296, 0, 296, 297, - 297, 0, 0, 297, 297, 297, 297, 297, 297, 297, - 297, 298, 298, 298, 298, 298, 298, 298, 298, 298, - 298, 298, 298, 298, 299, 299, 0, 299, 299, 299, - - 0, 299, 299, 299, 299, 299, 300, 300, 0, 300, - 300, 300, 300, 300, 300, 300, 300, 300, 300, 301, - 301, 301, 0, 301, 301, 301, 301, 301, 301, 301, - 301, 301, 302, 302, 0, 302, 302, 302, 302, 302, - 302, 302, 302, 302, 302, 303, 0, 303, 304, 0, - 304, 305, 305, 305, 305, 305, 305, 305, 305, 305, - 305, 305, 305, 305, 306, 306, 306, 306, 306, 306, - 306, 306, 306, 306, 306, 306, 306, 307, 0, 307, - 308, 308, 308, 308, 308, 308, 308, 308, 308, 308, - 308, 308, 308, 309, 309, 309, 309, 309, 309, 309, - - 309, 309, 309, 309, 309, 309, 310, 310, 310, 310, - 310, 310, 310, 310, 310, 310, 310, 310, 310, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285, 285, 285, 285, 285, - 285, 285, 285, 285, 285, 285 + 257 } ; /* Table of booleans, true if rule could match eol. */ -static const flex_int32_t yy_rule_can_match_eol[76] = +static const flex_int32_t yy_rule_can_match_eol[78] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, }; + 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, }; /* The intent behind this definition is that it'll catch * any uses of REJECT which flex missed. @@ -1137,11 +1034,11 @@ static bool is_absolute_path( #endif } -#line 1141 "lexer.c" +#line 1038 "lexer.c" #define YY_NO_UNISTD_H 1 #define YY_NO_INPUT 1 -#line 1145 "lexer.c" +#line 1042 "lexer.c" #define INITIAL 0 #define str 1 @@ -1420,7 +1317,7 @@ YY_DECL #line 163 "lexer.l" -#line 1424 "lexer.c" +#line 1321 "lexer.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1447,13 +1344,13 @@ yy_match: while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 286 ) + if ( yy_current_state >= 258 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; ++yy_cp; } - while ( yy_current_state != 285 ); + while ( yy_current_state != 257 ); yy_cp = yyg->yy_last_accepting_cpos; yy_current_state = yyg->yy_last_accepting_state; @@ -1573,142 +1470,152 @@ YY_RULE_SETUP case 18: YY_RULE_SETUP #line 182 "lexer.l" -{ return _FULLWORD_; } +{ return _BASE64_; } YY_BREAK case 19: YY_RULE_SETUP #line 183 "lexer.l" -{ return _NOCASE_; } +{ return _BASE64_WIDE_; } YY_BREAK case 20: YY_RULE_SETUP #line 184 "lexer.l" -{ return _CONDITION_; } +{ return _FULLWORD_; } YY_BREAK case 21: YY_RULE_SETUP #line 185 "lexer.l" -{ return _TRUE_; } +{ return _NOCASE_; } YY_BREAK case 22: YY_RULE_SETUP #line 186 "lexer.l" -{ return _FALSE_; } +{ return _CONDITION_; } YY_BREAK case 23: YY_RULE_SETUP #line 187 "lexer.l" -{ return _NOT_; } +{ return _TRUE_; } YY_BREAK case 24: YY_RULE_SETUP #line 188 "lexer.l" -{ return _AND_; } +{ return _FALSE_; } YY_BREAK case 25: YY_RULE_SETUP #line 189 "lexer.l" -{ return _OR_; } +{ return _NOT_; } YY_BREAK case 26: YY_RULE_SETUP #line 190 "lexer.l" -{ return _AT_; } +{ return _AND_; } YY_BREAK case 27: YY_RULE_SETUP #line 191 "lexer.l" -{ return _IN_; } +{ return _OR_; } YY_BREAK case 28: YY_RULE_SETUP #line 192 "lexer.l" -{ return _OF_; } +{ return _AT_; } YY_BREAK case 29: YY_RULE_SETUP #line 193 "lexer.l" -{ return _THEM_; } +{ return _IN_; } YY_BREAK case 30: YY_RULE_SETUP #line 194 "lexer.l" -{ return _FOR_; } +{ return _OF_; } YY_BREAK case 31: YY_RULE_SETUP #line 195 "lexer.l" -{ return _ALL_; } +{ return _THEM_; } YY_BREAK case 32: YY_RULE_SETUP #line 196 "lexer.l" -{ return _ANY_; } +{ return _FOR_; } YY_BREAK case 33: YY_RULE_SETUP #line 197 "lexer.l" -{ return _ENTRYPOINT_; } +{ return _ALL_; } YY_BREAK case 34: YY_RULE_SETUP #line 198 "lexer.l" -{ return _FILESIZE_; } +{ return _ANY_; } YY_BREAK case 35: YY_RULE_SETUP #line 199 "lexer.l" -{ return _MATCHES_; } +{ return _ENTRYPOINT_; } YY_BREAK case 36: YY_RULE_SETUP #line 200 "lexer.l" -{ return _CONTAINS_; } +{ return _FILESIZE_; } YY_BREAK case 37: YY_RULE_SETUP #line 201 "lexer.l" -{ return _IMPORT_; } +{ return _MATCHES_; } YY_BREAK case 38: YY_RULE_SETUP -#line 204 "lexer.l" -{ BEGIN(comment); } +#line 202 "lexer.l" +{ return _CONTAINS_; } YY_BREAK case 39: YY_RULE_SETUP -#line 205 "lexer.l" -{ BEGIN(INITIAL); } +#line 203 "lexer.l" +{ return _IMPORT_; } YY_BREAK case 40: -/* rule 40 can match eol */ YY_RULE_SETUP #line 206 "lexer.l" -{ /* skip comments */ } +{ BEGIN(comment); } YY_BREAK case 41: YY_RULE_SETUP -#line 209 "lexer.l" -{ /* skip single-line comments */ } +#line 207 "lexer.l" +{ BEGIN(INITIAL); } YY_BREAK case 42: +/* rule 42 can match eol */ YY_RULE_SETUP -#line 212 "lexer.l" +#line 208 "lexer.l" +{ /* skip comments */ } + YY_BREAK +case 43: +YY_RULE_SETUP +#line 211 "lexer.l" +{ /* skip single-line comments */ } + YY_BREAK +case 44: +YY_RULE_SETUP +#line 214 "lexer.l" { yyextra->lex_buf_ptr = yyextra->lex_buf; yyextra->lex_buf_len = 0; BEGIN(include); } YY_BREAK -case 43: -/* rule 43 can match eol */ +case 45: +/* rule 45 can match eol */ YY_RULE_SETUP -#line 219 "lexer.l" +#line 221 "lexer.l" { yytext_to_buffer; } YY_BREAK -case 44: +case 46: YY_RULE_SETUP -#line 222 "lexer.l" +#line 224 "lexer.l" { if (compiler->include_callback != NULL) @@ -1764,10 +1671,15 @@ YY_RULE_SETUP } } + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, + YR_NAMESPACES_TABLE, + compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE)); + included_rules = compiler->include_callback( include_path, current_file_name, - compiler->current_namespace->name, + ns->name, compiler->incl_clbk_user_data); if (included_rules != NULL) @@ -1840,7 +1752,7 @@ case YY_STATE_EOF(str): case YY_STATE_EOF(regexp): case YY_STATE_EOF(include): case YY_STATE_EOF(comment): -#line 349 "lexer.l" +#line 356 "lexer.l" { yypop_buffer_state(yyscanner); @@ -1851,9 +1763,9 @@ case YY_STATE_EOF(comment): return _END_OF_INCLUDED_FILE_; } YY_BREAK -case 45: +case 47: YY_RULE_SETUP -#line 360 "lexer.l" +#line 367 "lexer.l" { yylval->c_string = yr_strdup(yytext); @@ -1864,9 +1776,9 @@ YY_RULE_SETUP return _STRING_IDENTIFIER_WITH_WILDCARD_; } YY_BREAK -case 46: +case 48: YY_RULE_SETUP -#line 371 "lexer.l" +#line 378 "lexer.l" { yylval->c_string = yr_strdup(yytext); @@ -1877,9 +1789,9 @@ YY_RULE_SETUP return _STRING_IDENTIFIER_; } YY_BREAK -case 47: +case 49: YY_RULE_SETUP -#line 382 "lexer.l" +#line 389 "lexer.l" { yylval->c_string = yr_strdup(yytext); @@ -1896,9 +1808,9 @@ YY_RULE_SETUP return _STRING_COUNT_; } YY_BREAK -case 48: +case 50: YY_RULE_SETUP -#line 399 "lexer.l" +#line 406 "lexer.l" { yylval->c_string = yr_strdup(yytext); @@ -1915,9 +1827,9 @@ YY_RULE_SETUP return _STRING_OFFSET_; } YY_BREAK -case 49: +case 51: YY_RULE_SETUP -#line 416 "lexer.l" +#line 423 "lexer.l" { yylval->c_string = yr_strdup(yytext); @@ -1934,9 +1846,9 @@ YY_RULE_SETUP return _STRING_LENGTH_; } YY_BREAK -case 50: +case 52: YY_RULE_SETUP -#line 433 "lexer.l" +#line 440 "lexer.l" { char* text = yytext; @@ -1975,9 +1887,9 @@ YY_RULE_SETUP return _INTEGER_FUNCTION_; } YY_BREAK -case 51: +case 53: YY_RULE_SETUP -#line 472 "lexer.l" +#line 479 "lexer.l" { if (strlen(yytext) > 128) @@ -1991,9 +1903,9 @@ YY_RULE_SETUP return _IDENTIFIER_; } YY_BREAK -case 52: +case 54: YY_RULE_SETUP -#line 486 "lexer.l" +#line 493 "lexer.l" { char *endptr; @@ -2034,17 +1946,17 @@ YY_RULE_SETUP return _NUMBER_; } YY_BREAK -case 53: +case 55: YY_RULE_SETUP -#line 526 "lexer.l" +#line 533 "lexer.l" { yylval->double_ = atof(yytext); return _DOUBLE_; } YY_BREAK -case 54: +case 56: YY_RULE_SETUP -#line 531 "lexer.l" +#line 538 "lexer.l" { char *endptr; @@ -2061,9 +1973,9 @@ YY_RULE_SETUP return _NUMBER_; } YY_BREAK -case 55: +case 57: YY_RULE_SETUP -#line 547 "lexer.l" +#line 554 "lexer.l" { char *endptr; @@ -2080,9 +1992,9 @@ YY_RULE_SETUP return _NUMBER_; } YY_BREAK -case 56: +case 58: YY_RULE_SETUP -#line 564 "lexer.l" +#line 571 "lexer.l" { /* saw closing quote - all done */ alloc_sized_string(s, yyextra->lex_buf_len); @@ -2096,9 +2008,9 @@ YY_RULE_SETUP return _TEXT_STRING_; } YY_BREAK -case 57: +case 59: YY_RULE_SETUP -#line 578 "lexer.l" +#line 585 "lexer.l" { lex_check_space_ok("\t", yyextra->lex_buf_len, YR_LEX_BUF_SIZE); @@ -2106,9 +2018,9 @@ YY_RULE_SETUP yyextra->lex_buf_len++; } YY_BREAK -case 58: +case 60: YY_RULE_SETUP -#line 586 "lexer.l" +#line 593 "lexer.l" { lex_check_space_ok("\n", yyextra->lex_buf_len, YR_LEX_BUF_SIZE); @@ -2116,9 +2028,9 @@ YY_RULE_SETUP yyextra->lex_buf_len++; } YY_BREAK -case 59: +case 61: YY_RULE_SETUP -#line 594 "lexer.l" +#line 601 "lexer.l" { lex_check_space_ok("\"", yyextra->lex_buf_len, YR_LEX_BUF_SIZE); @@ -2126,9 +2038,9 @@ YY_RULE_SETUP yyextra->lex_buf_len++; } YY_BREAK -case 60: +case 62: YY_RULE_SETUP -#line 602 "lexer.l" +#line 609 "lexer.l" { lex_check_space_ok("\\", yyextra->lex_buf_len, YR_LEX_BUF_SIZE); @@ -2136,9 +2048,9 @@ YY_RULE_SETUP yyextra->lex_buf_len++; } YY_BREAK -case 61: +case 63: YY_RULE_SETUP -#line 610 "lexer.l" +#line 617 "lexer.l" { int result; @@ -2149,30 +2061,30 @@ YY_RULE_SETUP yyextra->lex_buf_len++; } YY_BREAK -case 62: +case 64: YY_RULE_SETUP -#line 621 "lexer.l" +#line 628 "lexer.l" { yytext_to_buffer; } YY_BREAK -case 63: -/* rule 63 can match eol */ +case 65: +/* rule 65 can match eol */ YY_RULE_SETUP -#line 624 "lexer.l" +#line 631 "lexer.l" { syntax_error("unterminated string"); } YY_BREAK -case 64: -/* rule 64 can match eol */ +case 66: +/* rule 66 can match eol */ YY_RULE_SETUP -#line 629 "lexer.l" +#line 636 "lexer.l" { syntax_error("illegal escape sequence"); } YY_BREAK -case 65: +case 67: YY_RULE_SETUP -#line 634 "lexer.l" +#line 641 "lexer.l" { if (yyextra->lex_buf_len > 0) @@ -2198,9 +2110,9 @@ YY_RULE_SETUP return _REGEXP_; } YY_BREAK -case 66: +case 68: YY_RULE_SETUP -#line 660 "lexer.l" +#line 667 "lexer.l" { lex_check_space_ok("/", yyextra->lex_buf_len, YR_LEX_BUF_SIZE); @@ -2208,9 +2120,9 @@ YY_RULE_SETUP yyextra->lex_buf_len++ ; } YY_BREAK -case 67: +case 69: YY_RULE_SETUP -#line 668 "lexer.l" +#line 675 "lexer.l" { lex_check_space_ok("\\.", yyextra->lex_buf_len, YR_LEX_BUF_SIZE); @@ -2223,22 +2135,22 @@ YY_RULE_SETUP yyextra->lex_buf_len += 2; } YY_BREAK -case 68: +case 70: YY_RULE_SETUP -#line 681 "lexer.l" +#line 688 "lexer.l" { yytext_to_buffer; } YY_BREAK -case 69: -/* rule 69 can match eol */ +case 71: +/* rule 71 can match eol */ YY_RULE_SETUP -#line 684 "lexer.l" +#line 691 "lexer.l" { syntax_error("unterminated regular expression"); } YY_BREAK -case 70: +case 72: YY_RULE_SETUP -#line 689 "lexer.l" +#line 696 "lexer.l" { yylval->sized_string = NULL; @@ -2247,9 +2159,9 @@ YY_RULE_SETUP BEGIN(str); } YY_BREAK -case 71: +case 73: YY_RULE_SETUP -#line 698 "lexer.l" +#line 705 "lexer.l" { yylval->sized_string = NULL; @@ -2258,10 +2170,10 @@ YY_RULE_SETUP BEGIN(regexp); } YY_BREAK -case 72: -/* rule 72 can match eol */ +case 74: +/* rule 74 can match eol */ YY_RULE_SETUP -#line 707 "lexer.l" +#line 714 "lexer.l" { // Match hex-digits with whitespace or comments. The latter are stripped // out by hex_lexer.l @@ -2278,15 +2190,15 @@ YY_RULE_SETUP return _HEX_STRING_; } YY_BREAK -case 73: -/* rule 73 can match eol */ +case 75: +/* rule 75 can match eol */ YY_RULE_SETUP -#line 724 "lexer.l" +#line 731 "lexer.l" /* skip whitespace */ YY_BREAK -case 74: +case 76: YY_RULE_SETUP -#line 726 "lexer.l" +#line 733 "lexer.l" { if (yytext[0] >= 32 && yytext[0] < 127) @@ -2299,12 +2211,12 @@ YY_RULE_SETUP } } YY_BREAK -case 75: +case 77: YY_RULE_SETUP -#line 738 "lexer.l" +#line 745 "lexer.l" ECHO; YY_BREAK -#line 2308 "lexer.c" +#line 2220 "lexer.c" case YY_END_OF_BUFFER: { @@ -2602,7 +2514,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 286 ) + if ( yy_current_state >= 258 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; @@ -2631,11 +2543,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 286 ) + if ( yy_current_state >= 258 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 285); + yy_is_jam = (yy_current_state == 257); (void)yyg; return yy_is_jam ? 0 : yy_current_state; @@ -3453,7 +3365,7 @@ void yyfree (void * ptr , yyscan_t yyscanner) #define YYTABLES_NAME "yytables" -#line 738 "lexer.l" +#line 745 "lexer.l" @@ -3480,11 +3392,19 @@ void yywarning( vsnprintf(message, sizeof(message), message_fmt, message_args); + YR_RULE* current_rule = NULL; + + if (compiler->current_rule_idx != UINT32_MAX) + current_rule = yr_arena_get_ptr( + compiler->arena, + YR_RULES_TABLE, + compiler->current_rule_idx * sizeof(YR_RULE)); + compiler->callback( YARA_ERROR_LEVEL_WARNING, file_name, compiler->current_line ? compiler->current_line : yyget_lineno(yyscanner), - compiler->current_rule, + current_rule, message, compiler->user_data); @@ -3535,6 +3455,14 @@ void yyerror( file_name = NULL; } + YR_RULE* current_rule = NULL; + + if (compiler->current_rule_idx != UINT32_MAX) + current_rule = yr_arena_get_ptr( + compiler->arena, + YR_RULES_TABLE, + compiler->current_rule_idx * sizeof(YR_RULE)); + if (error_message != NULL) { yr_compiler_set_error_extra_info(compiler, error_message); @@ -3546,7 +3474,7 @@ void yyerror( YARA_ERROR_LEVEL_ERROR, file_name, compiler->last_error_line, - compiler->current_rule, + current_rule, error_message, compiler->user_data); } @@ -3559,7 +3487,7 @@ void yyerror( YARA_ERROR_LEVEL_ERROR, file_name, compiler->last_error_line, - compiler->current_rule, + current_rule, message, compiler->user_data); } diff --git a/libyara/lexer.l b/libyara/lexer.l index 7eb87449..ea7c74ac 100644 --- a/libyara/lexer.l +++ b/libyara/lexer.l @@ -179,6 +179,8 @@ octdigit [0-7] "ascii" { return _ASCII_; } "wide" { return _WIDE_; } "xor" { return _XOR_; } +"base64" { return _BASE64_; } +"base64wide" { return _BASE64_WIDE_; } "fullword" { return _FULLWORD_; } "nocase" { return _NOCASE_; } "condition" { return _CONDITION_; } @@ -274,10 +276,15 @@ include[ \t]+\" { } } + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, + YR_NAMESPACES_TABLE, + compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE)); + included_rules = compiler->include_callback( include_path, current_file_name, - compiler->current_namespace->name, + ns->name, compiler->incl_clbk_user_data); if (included_rules != NULL) @@ -704,7 +711,7 @@ u?int(8|16|32)(be)? { } -\{(({hexdigit}|[ \-|\?\[\]\(\)\n\r\t]|\/\*[^*]*\*+([^*/][^*]*\*+)*\/)+|\/\/.*)+\} { +\{(({hexdigit}|[ \-|\?\[\]\(\)\n\r\t]|\/\*(\/|\**[^*/])*\*+\/)+|\/\/.*)+\} { // Match hex-digits with whitespace or comments. The latter are stripped // out by hex_lexer.l // TODO(vmalvarez): Integrate the hex string lexer and parser into this one, @@ -761,11 +768,19 @@ void yywarning( vsnprintf(message, sizeof(message), message_fmt, message_args); + YR_RULE* current_rule = NULL; + + if (compiler->current_rule_idx != UINT32_MAX) + current_rule = yr_arena_get_ptr( + compiler->arena, + YR_RULES_TABLE, + compiler->current_rule_idx * sizeof(YR_RULE)); + compiler->callback( YARA_ERROR_LEVEL_WARNING, file_name, compiler->current_line ? compiler->current_line : yyget_lineno(yyscanner), - compiler->current_rule, + current_rule, message, compiler->user_data); @@ -816,6 +831,14 @@ void yyerror( file_name = NULL; } + YR_RULE* current_rule = NULL; + + if (compiler->current_rule_idx != UINT32_MAX) + current_rule = yr_arena_get_ptr( + compiler->arena, + YR_RULES_TABLE, + compiler->current_rule_idx * sizeof(YR_RULE)); + if (error_message != NULL) { yr_compiler_set_error_extra_info(compiler, error_message); @@ -827,7 +850,7 @@ void yyerror( YARA_ERROR_LEVEL_ERROR, file_name, compiler->last_error_line, - compiler->current_rule, + current_rule, error_message, compiler->user_data); } @@ -837,12 +860,12 @@ void yyerror( yr_compiler_get_error_message(compiler, message, sizeof(message)); compiler->callback( - YARA_ERROR_LEVEL_ERROR, - file_name, - compiler->last_error_line, - compiler->current_rule, - message, - compiler->user_data); + YARA_ERROR_LEVEL_ERROR, + file_name, + compiler->last_error_line, + current_rule, + message, + compiler->user_data); } } diff --git a/libyara/libyara.c b/libyara/libyara.c index fdc1550c..65c79677 100644 --- a/libyara/libyara.c +++ b/libyara/libyara.c @@ -50,10 +50,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #endif - -YR_THREAD_STORAGE_KEY yr_tidx_key; -YR_THREAD_STORAGE_KEY yr_recovery_state_key; - +YR_THREAD_STORAGE_KEY yr_yyfatal_trampoline_tls; +YR_THREAD_STORAGE_KEY yr_trycatch_trampoline_tls; static int init_count = 0; @@ -142,8 +140,8 @@ YR_API int yr_initialize(void) } FAIL_ON_ERROR(yr_heap_alloc()); - FAIL_ON_ERROR(yr_thread_storage_create(&yr_tidx_key)); - FAIL_ON_ERROR(yr_thread_storage_create(&yr_recovery_state_key)); + FAIL_ON_ERROR(yr_thread_storage_create(&yr_yyfatal_trampoline_tls)); + FAIL_ON_ERROR(yr_thread_storage_create(&yr_trycatch_trampoline_tls)); #if defined HAVE_LIBCRYPTO && OPENSSL_VERSION_NUMBER < 0x10100000L @@ -184,18 +182,6 @@ YR_API int yr_initialize(void) } -// -// yr_finalize_thread -// -// This function is deprecated, it's maintained only for backward compatibility -// with programs that already use it. Calling yr_finalize_thread from each -// thread using libyara is not required anymore. - -YR_DEPRECATED_API void yr_finalize_thread(void) -{ -} - - // // yr_finalize // @@ -233,8 +219,8 @@ YR_API int yr_finalize(void) #endif - FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_tidx_key)); - FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_recovery_state_key)); + FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_yyfatal_trampoline_tls)); + FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_trycatch_trampoline_tls)); FAIL_ON_ERROR(yr_modules_finalize()); FAIL_ON_ERROR(yr_heap_free()); @@ -246,38 +232,6 @@ YR_API int yr_finalize(void) return ERROR_SUCCESS; } -// -// yr_set_tidx -// -// Set the thread index (tidx) for the current thread. The tidx is the index -// that will be used by the thread to access thread-specific data stored in -// YR_RULES structure. -// -// Args: -// int tidx - The zero-based tidx that will be associated to the current -// thread. -// - -YR_API void yr_set_tidx(int tidx) -{ - yr_thread_storage_set_value(&yr_tidx_key, (void*) (size_t) (tidx + 1)); -} - - -// -// yr_get_tidx -// -// Get the thread index (tidx) for the current thread. -// -// Returns: -// The tidx for the current thread or -1 if the current thread doesn't -// have any tidx associated. -// - -YR_API int yr_get_tidx(void) -{ - return (int) (size_t) yr_thread_storage_get_value(&yr_tidx_key) - 1; -} // @@ -289,7 +243,7 @@ YR_API int yr_get_tidx(void) // // Args: // YR_CONFIG_NAME name - Any of the values defined by the YR_CONFIG_NAME -// enum. Posible values are: +// enum. Possible values are: // // YR_CONFIG_STACK_SIZE data type: uint32_t // YR_CONFIG_MAX_STRINGS_PER_RULE data type: uint32_t diff --git a/libyara/mem.c b/libyara/mem.c index 62ee5c4f..1a87c3a7 100644 --- a/libyara/mem.c +++ b/libyara/mem.c @@ -71,6 +71,9 @@ void* yr_malloc(size_t size) void* yr_realloc(void* ptr, size_t size) { + if (ptr == NULL) + return (void*) HeapAlloc(hHeap, HEAP_ZERO_MEMORY, size); + return (void*) HeapReAlloc(hHeap, HEAP_ZERO_MEMORY, ptr, size); } diff --git a/libyara/modules.c b/libyara/modules.c index 456006a2..76812f3d 100644 --- a/libyara/modules.c +++ b/libyara/modules.c @@ -150,6 +150,7 @@ int yr_modules_load( mi.module_data_size = 0; result = context->callback( + context, CALLBACK_MSG_IMPORT_MODULE, &mi, context->user_data); @@ -188,6 +189,7 @@ int yr_modules_load( } result = context->callback( + context, CALLBACK_MSG_MODULE_IMPORTED, module_structure, context->user_data); diff --git a/libyara/modules/cuckoo/cuckoo.c b/libyara/modules/cuckoo/cuckoo.c index b4cdc2a6..1793cf4c 100644 --- a/libyara/modules/cuckoo/cuckoo.c +++ b/libyara/modules/cuckoo/cuckoo.c @@ -27,6 +27,7 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #include #include @@ -34,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include + #if defined(_WIN32) || defined(__CYGWIN__) #define strcasecmp _stricmp #endif @@ -174,6 +176,138 @@ define_function(network_http_post) } +// checks the host array in network JSON +// loops through array and checks value for argument +define_function(network_host) +{ + YR_SCAN_CONTEXT* context = scan_context(); + YR_OBJECT* network_obj = parent(); + + json_t* network_json = (json_t*) network_obj->data; + json_t* value; + + uint64_t result = 0; + size_t index; + + json_t* hosts_info_json = json_object_get(network_json, "hosts"); + + json_array_foreach(hosts_info_json, index, value) + { + if (yr_re_match(context, regexp_argument(1), json_string_value(value)) > 0) + { + result = 1; + break; + } + } + + return_integer(result); +} + +// loops through TCP array in network JSON +// checks for dest IP regex and dest port integer match +define_function(network_tcp) +{ + YR_SCAN_CONTEXT* context = scan_context(); + YR_OBJECT* network_obj = parent(); + + json_t* network_json = (json_t*) network_obj->data; + json_t* value; + + uint64_t result = 0; + size_t index; + + int dport; + char* dst; + + json_t* tcp_info_json = json_object_get(network_json, "tcp"); + + json_array_foreach(tcp_info_json, index, value) + { + if (json_unpack(value, "{s:i, s:s}", "dport", &dport, "dst", &dst) == 0) + { + if (yr_re_match(context, regexp_argument(1), dst) > 0) + { + if ((int64_t)dport == integer_argument(2)) { + result = 1; + break; + } + } + } + } + + return_integer(result); +} + + +// loops through UDP array in network JSON +// checks for dest IP regex and dest port integer match +define_function(network_udp) +{ + YR_SCAN_CONTEXT* context = scan_context(); + YR_OBJECT* network_obj = parent(); + + json_t* network_json = (json_t*) network_obj->data; + json_t* value; + + uint64_t result = 0; + size_t index; + + int dport; + char* dst; + + json_t* udp_info_json = json_object_get(network_json, "udp"); + + json_array_foreach(udp_info_json, index, value) + { + if (json_unpack(value, "{s:i, s:s}", "dport", &dport, "dst", &dst) == 0) + { + if (yr_re_match(context, regexp_argument(1), dst) > 0) + { + if ((int64_t)dport == integer_argument(2)) { + result = 1; + break; + } + } + } + } + + return_integer(result); +} + + +// loops through http array in network JSON +// checks for regex match on user-agent +define_function(network_http_user_agent) +{ + + YR_SCAN_CONTEXT* context = scan_context(); + YR_OBJECT* network_obj = parent(); + + json_t* network_json = (json_t*) network_obj->data; + json_t* http_json = json_object_get(network_json, "http"); + json_t* value; + + uint64_t result = 0; + size_t index; + + char* user_agent; + + json_array_foreach(http_json, index, value) + { + if (json_unpack(value, "{s:s}", "user-agent", &user_agent) == 0) + { + if (yr_re_match(context, regexp_argument(1), user_agent) > 0) + { + result = 1; + break; + } + } + } + + return_integer(result); +} + + define_function(registry_key_access) { YR_SCAN_CONTEXT* context = scan_context(); @@ -254,6 +388,10 @@ begin_declarations; declare_function("http_get", "r", "i", network_http_get); declare_function("http_post", "r", "i", network_http_post); declare_function("http_request", "r", "i", network_http_request); + declare_function("http_user_agent", "r", "i", network_http_user_agent); + declare_function("host", "r", "i", network_host); + declare_function("tcp", "ri", "i", network_tcp); + declare_function("udp", "ri", "i", network_udp); end_struct("network"); begin_struct("registry"); @@ -344,3 +482,4 @@ int module_unload(YR_OBJECT* module) return ERROR_SUCCESS; } + diff --git a/libyara/modules/dex/dex.c b/libyara/modules/dex/dex.c index 6ca85a71..1135ed92 100644 --- a/libyara/modules/dex/dex.c +++ b/libyara/modules/dex/dex.c @@ -45,6 +45,7 @@ begin_declarations; declare_string("DEX_FILE_MAGIC_036"); declare_string("DEX_FILE_MAGIC_037"); declare_string("DEX_FILE_MAGIC_038"); + declare_string("DEX_FILE_MAGIC_039"); declare_integer("ENDIAN_CONSTANT"); declare_integer("REVERSE_ENDIAN_CONSTANT"); @@ -147,7 +148,7 @@ begin_declarations; begin_struct_array("class_defs"); declare_integer("class_idx"); declare_integer("access_flags"); - declare_integer("superclass_idx"); + declare_integer("super_class_idx"); declare_integer("interfaces_offset"); declare_integer("source_file_idx"); declare_integer("annotations_offset"); @@ -264,12 +265,12 @@ static int64_t dex_get_integer( const char* pattern, int64_t index) { - if (index == UNDEFINED) - return UNDEFINED; + if (index == YR_UNDEFINED) + return YR_UNDEFINED; // Impose a reasonably large limit to table indexes. if (index > 0x80000) - return UNDEFINED; + return YR_UNDEFINED; return get_integer(object, pattern, (int) index); } @@ -280,7 +281,7 @@ static SIZED_STRING* dex_get_string( const char* pattern, int64_t index) { - if (index == UNDEFINED) + if (index == YR_UNDEFINED) return NULL; // Impose a reasonably large limit to table indexes. @@ -306,7 +307,8 @@ dex_header_t* dex_get_header( if (memcmp(dex_header->magic, DEX_FILE_MAGIC_035, 8) != 0 && memcmp(dex_header->magic, DEX_FILE_MAGIC_036, 8) != 0 && memcmp(dex_header->magic, DEX_FILE_MAGIC_037, 8) != 0 && - memcmp(dex_header->magic, DEX_FILE_MAGIC_038, 8) != 0) + memcmp(dex_header->magic, DEX_FILE_MAGIC_038, 8) != 0 && + memcmp(dex_header->magic, DEX_FILE_MAGIC_039, 8) != 0) { return NULL; } @@ -440,7 +442,7 @@ uint32_t load_encoded_field( int64_t name_idx = dex_get_integer( dex->object, "field_ids[%i].name_idx", *previous_field_idx); - if (name_idx == UNDEFINED) + if (name_idx == YR_UNDEFINED) return 0; SIZED_STRING* field_name = dex_get_string( @@ -580,7 +582,7 @@ uint32_t load_encoded_method( int64_t name_idx = dex_get_integer( dex->object, "method_ids[%i].name_idx", *previous_method_idx); - if (name_idx == UNDEFINED) + if (name_idx == YR_UNDEFINED) return 0; #ifdef DEBUG_DEX_MODULE @@ -939,13 +941,13 @@ void dex_parse( #ifdef DEBUG_DEX_MODULE printf("[DEX] CLASS ID item class_idx:0x%x access_flags:0x%x " \ - "super_class_idx:0x%x interfaces_off:0x%x source_file_idx:0x%x "\ + "super_class_idx:0x%x interfaces_offset:0x%x source_file_idx:0x%x "\ "annotations_offset:0x%x class_data_offset:0x%x "\ "static_values_offset:0x%x\n", yr_le32toh(class_id_item->class_idx), yr_le32toh(class_id_item->access_flags), yr_le32toh(class_id_item->super_class_idx), - yr_le32toh(class_id_item->interfaces_off), + yr_le32toh(class_id_item->interfaces_offset), yr_le32toh(class_id_item->source_file_idx), yr_le32toh(class_id_item->annotations_offset), yr_le32toh(class_id_item->class_data_offset), @@ -958,14 +960,14 @@ void dex_parse( "class_defs[%i].access_flags", i); set_integer(yr_le32toh(class_id_item->super_class_idx), dex->object, "class_defs[%i].super_class_idx", i); - set_integer(yr_le32toh(class_id_item->interfaces_off), dex->object, - "class_defs[%i].interfaces_off", i); + set_integer(yr_le32toh(class_id_item->interfaces_offset), dex->object, + "class_defs[%i].interfaces_offset", i); set_integer(yr_le32toh(class_id_item->source_file_idx), dex->object, "class_defs[%i].source_file_idx", i); set_integer(yr_le32toh(class_id_item->annotations_offset), dex->object, "class_defs[%i].annotations_offset", i); set_integer(yr_le32toh(class_id_item->class_data_offset), dex->object, - "class_defs[%i].class_data_off", i); + "class_defs[%i].class_data_offset", i); set_integer(yr_le32toh(class_id_item->static_values_offset), dex->object, "class_defs[%i].static_values_offset", i); @@ -1143,6 +1145,7 @@ int module_load( set_string(DEX_FILE_MAGIC_036, module_object, "DEX_FILE_MAGIC_036"); set_string(DEX_FILE_MAGIC_037, module_object, "DEX_FILE_MAGIC_037"); set_string(DEX_FILE_MAGIC_038, module_object, "DEX_FILE_MAGIC_038"); + set_string(DEX_FILE_MAGIC_039, module_object, "DEX_FILE_MAGIC_039"); set_integer(0x12345678, module_object, "ENDIAN_CONSTANT"); set_integer(0x78563412, module_object, "REVERSE_ENDIAN_CONSTANT"); diff --git a/libyara/modules/dotnet/dotnet.c b/libyara/modules/dotnet/dotnet.c index 2cfc82da..816316cc 100644 --- a/libyara/modules/dotnet/dotnet.c +++ b/libyara/modules/dotnet/dotnet.c @@ -196,6 +196,12 @@ BLOB_PARSE_RESULT dotnet_parse_blob_entry( result.size = 0; } + // There is an additional terminal byte which is 0x01 under certain + // conditions. The exact conditions are not relevant to our parsing but are + // documented in ECMA-335 II.24.2.4. + if (result.length > 0) + result.length--; + return result; } @@ -796,6 +802,9 @@ void dotnet_parse_tilde_2( // Now follow the Class index into the TypeRef table. typeref_row = typeref_ptr + (typeref_row_size * class_index); + if (!fits_in_pe(pe, typeref_row, typeref_row_size)) + break; + // Skip over the ResolutionScope and check the Name field, // which is an index into the Strings heap. row_count = max_rows(4, @@ -1439,6 +1448,7 @@ void dotnet_parse_tilde( // number of rows being indexed into. ROWS rows; INDEX_SIZES index_sizes; + uint32_t heap_sizes; // Default all rows to 0. They will be set to actual values later on, if // they exist in the file. @@ -1455,13 +1465,15 @@ void dotnet_parse_tilde( if (!struct_fits_in_pe(pe, tilde_header, TILDE_HEADER)) return; - uint32_t heap_sizes = yr_le32toh(tilde_header->HeapSizes); + heap_sizes = yr_le32toh(tilde_header->HeapSizes); // Set index sizes for various heaps. if (heap_sizes & 0x01) index_sizes.string = 4; + if (heap_sizes & 0x02) index_sizes.guid = 4; + if (heap_sizes & 0x04) index_sizes.blob = 4; diff --git a/libyara/modules/elf/elf.c b/libyara/modules/elf/elf.c index 81a353de..bc23d418 100644 --- a/libyara/modules/elf/elf.c +++ b/libyara/modules/elf/elf.c @@ -135,7 +135,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \ if(ULONG_MAX - yr_##bo##bits##toh(elf_header->ph_offset) < \ ELF_SIZE_OF_PROGRAM_TABLE(bits,bo,elf_header)) \ { \ - return UNDEFINED; \ + return YR_UNDEFINED; \ } \ \ if (yr_##bo##bits##toh(elf_header->ph_offset) == 0 || \ @@ -144,7 +144,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \ ELF_SIZE_OF_PROGRAM_TABLE(bits,bo,elf_header) > elf_size || \ yr_##bo##16toh(elf_header->ph_entry_count) == 0) \ { \ - return UNDEFINED; \ + return YR_UNDEFINED; \ } \ \ program = (elf##bits##_program_header_t*) \ @@ -174,7 +174,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \ if(ULONG_MAX - yr_##bo##bits##toh(elf_header->sh_offset) < \ ELF_SIZE_OF_SECTION_TABLE(bits,bo,elf_header)) \ { \ - return UNDEFINED; \ + return YR_UNDEFINED; \ } \ \ if (yr_##bo##bits##toh(elf_header->sh_offset) == 0 || \ @@ -183,7 +183,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \ ELF_SIZE_OF_SECTION_TABLE(bits,bo,elf_header) > elf_size || \ yr_##bo##16toh(elf_header->sh_entry_count) == 0) \ { \ - return UNDEFINED; \ + return YR_UNDEFINED; \ } \ \ section = (elf##bits##_section_header_t*) \ @@ -204,7 +204,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \ section++; \ } \ } \ - return UNDEFINED; \ + return YR_UNDEFINED; \ } #define PARSE_ELF_HEADER(bits,bo) \ diff --git a/libyara/modules/hash/hash.c b/libyara/modules/hash/hash.c index f6cb30d9..8b33dc56 100644 --- a/libyara/modules/hash/hash.c +++ b/libyara/modules/hash/hash.c @@ -242,7 +242,7 @@ define_function(data_md5) yr_md5_init(&md5_context); if (offset < 0 || length < 0 || offset < block->base) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); cached_ascii_digest = get_from_cache( module(), "md5", arg_offset, arg_length); @@ -281,7 +281,7 @@ define_function(data_md5) // range contains gaps of undefined data the checksum is // undefined. - return_string(UNDEFINED); + return_string(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -289,7 +289,7 @@ define_function(data_md5) } if (!past_first_block) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); yr_md5_final(digest, &md5_context); @@ -325,7 +325,7 @@ define_function(data_sha1) yr_sha1_init(&sha_context); if (offset < 0 || length < 0 || offset < block->base) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); cached_ascii_digest = get_from_cache( module(), "sha1", arg_offset, arg_length); @@ -363,7 +363,7 @@ define_function(data_sha1) // range contains gaps of undefined data the checksum is // undefined. - return_string(UNDEFINED); + return_string(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -371,7 +371,7 @@ define_function(data_sha1) } if (!past_first_block) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); yr_sha1_final(digest, &sha_context); @@ -407,7 +407,7 @@ define_function(data_sha256) yr_sha256_init(&sha256_context); if (offset < 0 || length < 0 || offset < block->base) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); cached_ascii_digest = get_from_cache( module(), "sha256", arg_offset, arg_length); @@ -444,7 +444,7 @@ define_function(data_sha256) // range contains gaps of undefined data the checksum is // undefined. - return_string(UNDEFINED); + return_string(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -452,7 +452,7 @@ define_function(data_sha256) } if (!past_first_block) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); yr_sha256_final(digest, &sha256_context); @@ -478,7 +478,7 @@ define_function(data_checksum32) int past_first_block = false; if (offset < 0 || length < 0 || offset < block->base) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -511,7 +511,7 @@ define_function(data_checksum32) // range contains gaps of undefined data the checksum is // undefined. - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -519,7 +519,7 @@ define_function(data_checksum32) } if (!past_first_block) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); return_integer(checksum); } @@ -551,7 +551,7 @@ define_function(data_crc32) int past_first_block = false; if (offset < 0 || length < 0 || offset < block->base) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -584,7 +584,7 @@ define_function(data_crc32) // range contains gaps of undefined data the checksum is // undefined. - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -592,7 +592,7 @@ define_function(data_crc32) } if (!past_first_block) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); return_integer(checksum ^ 0xFFFFFFFF); } diff --git a/libyara/modules/macho/macho.c b/libyara/modules/macho/macho.c index b0c80b99..5f25d896 100644 --- a/libyara/modules/macho/macho.c +++ b/libyara/modules/macho/macho.c @@ -481,7 +481,7 @@ void macho_parse_fat_file_##bits( \ continue; \ \ /* Force 'file' array entry creation. */ \ - set_integer(UNDEFINED, object, "file[%i].magic", i); \ + set_integer(YR_UNDEFINED, object, "file[%i].magic", i); \ \ /* Get specific Mach-O file data. */ \ macho_parse_file(data + offset, file_size, \ @@ -720,7 +720,7 @@ define_function(file_index_type) uint64_t nfat = get_integer(module, "nfat_arch"); if (is_undefined(module, "nfat_arch")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); for (int i = 0; i < nfat; i++) { @@ -730,7 +730,7 @@ define_function(file_index_type) return_integer(i); } } - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } @@ -744,7 +744,7 @@ define_function(file_index_subtype) uint64_t nfat = get_integer(module, "nfat_arch"); if (is_undefined(module, "nfat_arch")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); for (int i = 0; i < nfat; i++) { @@ -756,7 +756,7 @@ define_function(file_index_subtype) return_integer(i); } } - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } @@ -769,7 +769,7 @@ define_function(ep_for_arch_type) uint64_t nfat = get_integer(module, "nfat_arch"); if (is_undefined(module, "nfat_arch")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); for (int i = 0; i < nfat; i++) { @@ -781,7 +781,7 @@ define_function(ep_for_arch_type) return_integer(file_offset + entry_point); } } - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } @@ -795,7 +795,7 @@ define_function(ep_for_arch_subtype) uint64_t nfat = get_integer(module, "nfat_arch"); if (is_undefined(module, "nfat_arch")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); for (int i = 0; i < nfat; i++) { @@ -808,7 +808,7 @@ define_function(ep_for_arch_subtype) return_integer(file_offset + entry_point); } } - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } diff --git a/libyara/modules/magic/magic.c b/libyara/modules/magic/magic.c index 4e7e6284..5d9ba978 100644 --- a/libyara/modules/magic/magic.c +++ b/libyara/modules/magic/magic.c @@ -33,80 +33,127 @@ The original idea and inspiration for this module comes from Armin Buescher. */ +#include #include #include #define MODULE_NAME magic -magic_t magic_cookie[YR_MAX_THREADS]; -const char* cached_types[YR_MAX_THREADS]; -const char* cached_mime_types[YR_MAX_THREADS]; +// Thread-local storage key used to store a pointer to a MAGIC_CACHE struct. +YR_THREAD_STORAGE_KEY magic_tls; + + +typedef struct +{ + magic_t magic_cookie; + const char* cached_type; + const char* cached_mime_type; + +} MAGIC_CACHE ; + + +static int get_cache(MAGIC_CACHE** cache) +{ + *cache = (MAGIC_CACHE*) yr_thread_storage_get_value(&magic_tls); + + if (*cache == NULL) + { + *cache = (MAGIC_CACHE*) yr_malloc(sizeof(MAGIC_CACHE)); + + if (*cache == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + (*cache)->magic_cookie = magic_open(0); + + if ((*cache)->magic_cookie == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + if (magic_load((*cache)->magic_cookie, NULL) != 0) + { + magic_close((*cache)->magic_cookie); + return ERROR_INTERNAL_FATAL_ERROR; + } + + (*cache)->cached_type = NULL; + (*cache)->cached_mime_type = NULL; + + return yr_thread_storage_set_value(&magic_tls, *cache); + } + + return ERROR_SUCCESS; +} define_function(magic_mime_type) { - YR_MEMORY_BLOCK* block; YR_SCAN_CONTEXT* context = scan_context(); + YR_MEMORY_BLOCK* block; + MAGIC_CACHE* cache; const uint8_t* block_data; if (context->flags & SCAN_FLAGS_PROCESS_MEMORY) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); - if (cached_mime_types[context->tidx] == NULL) + FAIL_ON_ERROR(get_cache(&cache)); + + if (cache->cached_mime_type == NULL) { block = first_memory_block(context); block_data = block->fetch_data(block); if (block_data != NULL) { - magic_setflags(magic_cookie[context->tidx], MAGIC_MIME_TYPE); + magic_setflags(cache->magic_cookie, MAGIC_MIME_TYPE); - cached_mime_types[context->tidx] = magic_buffer( - magic_cookie[context->tidx], + cache->cached_mime_type = magic_buffer( + cache->magic_cookie, block_data, block->size); } } - if (cached_mime_types[context->tidx] == NULL) - return_string(UNDEFINED); + if (cache->cached_mime_type == NULL) + return_string(YR_UNDEFINED); - return_string((char*) cached_mime_types[context->tidx]); + return_string((char*) cache->cached_mime_type); } define_function(magic_type) { + MAGIC_CACHE* cache; YR_MEMORY_BLOCK* block; YR_SCAN_CONTEXT* context = scan_context(); const uint8_t* block_data; if (context->flags & SCAN_FLAGS_PROCESS_MEMORY) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); - if (cached_types[context->tidx] == NULL) + FAIL_ON_ERROR(get_cache(&cache)); + + if (cache->cached_type == NULL) { block = first_memory_block(context); block_data = block->fetch_data(block); if (block_data != NULL) { - magic_setflags(magic_cookie[context->tidx], 0); + magic_setflags(cache->magic_cookie, 0); - cached_types[context->tidx] = magic_buffer( - magic_cookie[context->tidx], + cache->cached_type = magic_buffer( + cache->magic_cookie, block_data, block->size); } } - if (cached_types[context->tidx] == NULL) - return_string(UNDEFINED); + if (cache->cached_type == NULL) + return_string(YR_UNDEFINED); - return_string((char*) cached_types[context->tidx]); + return_string((char*) cache->cached_type); } begin_declarations; @@ -120,25 +167,22 @@ end_declarations; int module_initialize( YR_MODULE* module) { - int i; - - for (i = 0; i < YR_MAX_THREADS; i++) - magic_cookie[i] = NULL; - - return ERROR_SUCCESS; + return yr_thread_storage_create(&magic_tls); } int module_finalize( YR_MODULE* module) { - int i; + MAGIC_CACHE* cache = (MAGIC_CACHE*) yr_thread_storage_get_value(&magic_tls); - for (i = 0; i < YR_MAX_THREADS; i++) - if (magic_cookie[i] != NULL) - magic_close(magic_cookie[i]); + if (cache != NULL) + { + magic_close(cache->magic_cookie); + yr_free(cache); + } - return ERROR_SUCCESS; + return yr_thread_storage_destroy(&magic_tls); } @@ -148,27 +192,6 @@ int module_load( void* module_data, size_t module_data_size) { - cached_types[context->tidx] = NULL; - cached_mime_types[context->tidx] = NULL; - - if (magic_cookie[context->tidx] == NULL) - { - magic_cookie[context->tidx] = magic_open(0); - - if (magic_cookie[context->tidx] != NULL) - { - if (magic_load(magic_cookie[context->tidx], NULL) != 0) - { - magic_close(magic_cookie[context->tidx]); - return ERROR_INTERNAL_FATAL_ERROR; - } - } - else - { - return ERROR_INSUFFICIENT_MEMORY; - } - } - return ERROR_SUCCESS; } @@ -176,5 +199,13 @@ int module_load( int module_unload( YR_OBJECT* module) { + MAGIC_CACHE* cache = (MAGIC_CACHE*) yr_thread_storage_get_value(&magic_tls); + + if (cache != NULL) + { + cache->cached_type = NULL; + cache->cached_mime_type = NULL; + } + return ERROR_SUCCESS; } diff --git a/libyara/modules/math/math.c b/libyara/modules/math/math.c index 7043f598..3037c556 100644 --- a/libyara/modules/math/math.c +++ b/libyara/modules/math/math.c @@ -57,7 +57,7 @@ define_function(string_entropy) uint32_t* data = (uint32_t*) yr_calloc(256, sizeof(uint32_t)); if (data == NULL) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); for (i = 0; i < s->length; i++) { @@ -97,12 +97,12 @@ define_function(data_entropy) YR_MEMORY_BLOCK_ITERATOR* iterator = context->iterator; if (offset < 0 || length < 0 || offset < block->base) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); data = (uint32_t*) yr_calloc(256, sizeof(uint32_t)); if (data == NULL) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -118,7 +118,7 @@ define_function(data_entropy) if (block_data == NULL) { yr_free(data); - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } total_len += data_len; @@ -142,7 +142,7 @@ define_function(data_entropy) // undefined. yr_free(data); - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -152,7 +152,7 @@ define_function(data_entropy) if (!past_first_block) { yr_free(data); - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } for (i = 0; i < 256; i++) @@ -207,7 +207,7 @@ define_function(data_deviation) YR_MEMORY_BLOCK_ITERATOR* iterator = context->iterator; if (offset < 0 || length < 0 || offset < block->base) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -220,7 +220,7 @@ define_function(data_deviation) block_data = block->fetch_data(block); if (block_data == NULL) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); total_len += data_len; offset += data_len; @@ -238,7 +238,7 @@ define_function(data_deviation) // the checksum over a range of non contiguous blocks. As // range contains gaps of undefined data the checksum is // undefined. - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -246,7 +246,7 @@ define_function(data_deviation) } if (!past_first_block) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); return_float(sum / total_len); } @@ -282,7 +282,7 @@ define_function(data_mean) size_t i; if (offset < 0 || length < 0 || offset < block->base) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -296,7 +296,7 @@ define_function(data_mean) const uint8_t* block_data = block->fetch_data(block); if (block_data == NULL) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); total_len += data_len; offset += data_len; @@ -314,7 +314,7 @@ define_function(data_mean) // the checksum over a range of non contiguous blocks. As // range contains gaps of undefined data the checksum is // undefined. - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -322,7 +322,7 @@ define_function(data_mean) } if (!past_first_block) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); return_float(sum / total_len); } @@ -350,7 +350,7 @@ define_function(data_serial_correlation) double scc = 0; if (offset < 0 || length < 0 || offset < block->base) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -364,7 +364,7 @@ define_function(data_serial_correlation) const uint8_t* block_data = block->fetch_data(block); if (block_data == NULL) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); total_len += data_len; offset += data_len; @@ -388,7 +388,7 @@ define_function(data_serial_correlation) // the checksum over a range of non contiguous blocks. As // range contains gaps of undefined data the checksum is // undefined. - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -396,7 +396,7 @@ define_function(data_serial_correlation) } if (!past_first_block) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); scct1 += scclast * sccun; scct2 *= scct2; @@ -467,7 +467,7 @@ define_function(data_monte_carlo_pi) YR_MEMORY_BLOCK_ITERATOR* iterator = context->iterator; if (offset < 0 || length < 0 || offset < block->base) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); foreach_memory_block(iterator, block) { @@ -483,7 +483,7 @@ define_function(data_monte_carlo_pi) const uint8_t* block_data = block->fetch_data(block); if (block_data == NULL) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); offset += data_len; length -= data_len; @@ -520,7 +520,7 @@ define_function(data_monte_carlo_pi) // the checksum over a range of non contiguous blocks. As // range contains gaps of undefined data the checksum is // undefined. - return_float(UNDEFINED); + return_float(YR_UNDEFINED); } if (block->base + block->size > offset + length) @@ -528,7 +528,7 @@ define_function(data_monte_carlo_pi) } if (!past_first_block || mcount == 0) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); mpi = 4.0 * ((double) inmont / mcount); @@ -575,7 +575,7 @@ define_function(string_monte_carlo_pi) } if (mcount == 0) - return_float(UNDEFINED); + return_float(YR_UNDEFINED); mpi = 4.0 * ((double) inmont / mcount); return_float(fabs((mpi - PI) / PI)); diff --git a/libyara/modules/pe/pe.c b/libyara/modules/pe/pe.c index 23659b93..8c26d101 100644 --- a/libyara/modules/pe/pe.c +++ b/libyara/modules/pe/pe.c @@ -38,8 +38,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include + #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) #define X509_get_signature_nid(o) OBJ_obj2nid((o)->sig_alg->algorithm) +#endif + +#if OPENSSL_VERSION_NUMBER < 0x10100000L #define X509_getm_notBefore X509_get_notBefore #define X509_getm_notAfter X509_get_notAfter #endif @@ -277,6 +281,96 @@ static void pe_parse_rich_signature( } +static void pe_parse_debug_directory( + PE* pe) +{ + PIMAGE_DATA_DIRECTORY data_dir; + PIMAGE_DEBUG_DIRECTORY debug_dir; + int64_t debug_dir_offset; + int64_t pcv_hdr_offset; + int i, dcount; + size_t pdb_path_len; + char* pdb_path = NULL; + + data_dir = pe_get_directory_entry( + pe, IMAGE_DIRECTORY_ENTRY_DEBUG); + + if (data_dir == NULL) + return; + + if (yr_le32toh(data_dir->Size) == 0) + return; + + if (yr_le32toh(data_dir->Size) % sizeof(IMAGE_DEBUG_DIRECTORY) != 0) + return; + + if (yr_le32toh(data_dir->VirtualAddress) == 0) + return; + + debug_dir_offset = pe_rva_to_offset( + pe, yr_le32toh(data_dir->VirtualAddress)); + + if (debug_dir_offset < 0) + return; + + dcount = yr_le32toh(data_dir->Size) / sizeof(IMAGE_DEBUG_DIRECTORY); + + for (i = 0; i < dcount; i++) + { + debug_dir = (PIMAGE_DEBUG_DIRECTORY) \ + (pe->data + debug_dir_offset + i * sizeof(IMAGE_DEBUG_DIRECTORY)); + + if (!struct_fits_in_pe(pe, debug_dir, IMAGE_DEBUG_DIRECTORY)) + break; + + if (yr_le32toh(debug_dir->Type) != IMAGE_DEBUG_TYPE_CODEVIEW) + continue; + + if (yr_le32toh(debug_dir->AddressOfRawData) == 0) + continue; + + pcv_hdr_offset = pe_rva_to_offset( + pe, yr_le32toh(debug_dir->AddressOfRawData)); + + if (pcv_hdr_offset < 0) + continue; + + PCV_HEADER cv_hdr = (PCV_HEADER) (pe->data + pcv_hdr_offset); + + if (!struct_fits_in_pe(pe, cv_hdr, CV_HEADER)) + continue; + + if (yr_le32toh(cv_hdr->dwSignature) == CVINFO_PDB20_CVSIGNATURE) + { + PCV_INFO_PDB20 pdb20 = (PCV_INFO_PDB20) cv_hdr; + + if (struct_fits_in_pe(pe, pdb20, CV_INFO_PDB20)) + pdb_path = (char*) (pdb20->PdbFileName); + } + else if (yr_le32toh(cv_hdr->dwSignature) == CVINFO_PDB70_CVSIGNATURE) + { + PCV_INFO_PDB70 pdb70 = (PCV_INFO_PDB70) cv_hdr; + + if (struct_fits_in_pe(pe, pdb70, CV_INFO_PDB70)) + pdb_path = (char*) (pdb70->PdbFileName); + } + + if (pdb_path != NULL) + { + pdb_path_len = strnlen( + pdb_path, yr_min(available_space(pe, pdb_path), MAX_PATH)); + + if (pdb_path_len > 0 && pdb_path_len < MAX_PATH) + { + set_sized_string(pdb_path, pdb_path_len, pe->object, "pdb_path"); + break; + } + } + } + + return; +} + // Return a pointer to the resource directory string or NULL. // The callback function will parse this and call set_sized_string(). // The pointer is guaranteed to have enough space to contain the entire string. @@ -982,27 +1076,31 @@ static IMPORTED_DLL* pe_parse_imports( // "exports" function for comparison. // -static EXPORT_FUNCTIONS* pe_parse_exports( +static void pe_parse_exports( PE* pe) { PIMAGE_DATA_DIRECTORY directory; PIMAGE_EXPORT_DIRECTORY exports; - EXPORT_FUNCTIONS* exported_functions; - uint32_t i; + uint32_t i, j; uint32_t number_of_exports; uint32_t number_of_names; - uint16_t ordinal; + uint32_t ordinal_base; + uint32_t export_start; + uint32_t export_size; int64_t offset; size_t remaining; + size_t name_len; + uint32_t exp_sz = 0; DWORD* names = NULL; WORD* ordinals = NULL; + DWORD* function_addrs = NULL; - // If not a PE file, return UNDEFINED + // If not a PE file, return YR_UNDEFINED if (pe == NULL) - return NULL; + return; // Default to 0 exports until we know there are any set_integer(0, pe->object, "number_of_exports"); @@ -1011,112 +1109,165 @@ static EXPORT_FUNCTIONS* pe_parse_exports( pe, IMAGE_DIRECTORY_ENTRY_EXPORT); if (directory == NULL) - return NULL; + return; if (yr_le32toh(directory->VirtualAddress) == 0) - return NULL; + return; offset = pe_rva_to_offset(pe, yr_le32toh(directory->VirtualAddress)); if (offset < 0) - return NULL; + return; + + export_start = offset; + export_size = directory->Size; exports = (PIMAGE_EXPORT_DIRECTORY) (pe->data + offset); if (!struct_fits_in_pe(pe, exports, IMAGE_EXPORT_DIRECTORY)) - return NULL; + return; number_of_exports = yr_min( yr_le32toh(exports->NumberOfFunctions), MAX_PE_EXPORTS); + ordinal_base = exports->Base; + + set_integer(exports->TimeDateStamp, pe->object, "export_timestamp"); + + offset = pe_rva_to_offset(pe, yr_le32toh(exports->Name)); + + if (offset > 0) + { + remaining = pe->data_size - (size_t) offset; + name_len = strnlen((char*) (pe->data + offset), remaining); + set_sized_string( + (char*) (pe->data + offset), name_len, pe->object, "dll_name"); + } + if (number_of_exports * sizeof(DWORD) > pe->data_size - offset) - return NULL; + return; if (yr_le32toh(exports->NumberOfNames) > 0) { offset = pe_rva_to_offset(pe, yr_le32toh(exports->AddressOfNames)); if (offset < 0) - return NULL; + return; if (yr_le32toh(exports->NumberOfNames) * sizeof(DWORD) > pe->data_size - offset) - return NULL; + return; names = (DWORD*)(pe->data + offset); - - offset = pe_rva_to_offset(pe, yr_le32toh(exports->AddressOfNameOrdinals)); - - if (offset < 0) - return NULL; - - ordinals = (WORD*)(pe->data + offset); } - exported_functions = (EXPORT_FUNCTIONS*) yr_malloc(sizeof(EXPORT_FUNCTIONS)); + offset = pe_rva_to_offset(pe, yr_le32toh(exports->AddressOfNameOrdinals)); - if (exported_functions == NULL) - return NULL; + if (offset < 0) + return; - exported_functions->number_of_exports = number_of_exports; - exported_functions->functions = (EXPORT_FUNCTION*) yr_malloc( - number_of_exports * sizeof(EXPORT_FUNCTION)); + ordinals = (WORD*)(pe->data + offset); - if (exported_functions->functions == NULL) - { - yr_free(exported_functions); - return NULL; - } + if (available_space(pe, ordinals) < sizeof(WORD) * number_of_exports) + return; - // At first, iterate through Functions array and create representation for - // each exported function. Ordinal is just array index that starts from 1 - for (i = 0; i < exported_functions->number_of_exports; i++) - { - exported_functions->functions[i].name = NULL; - exported_functions->functions[i].ordinal = i + 1; - } + offset = pe_rva_to_offset(pe, yr_le32toh(exports->AddressOfFunctions)); + + if (offset < 0) + return; + + function_addrs = (DWORD*)(pe->data + offset); + + if (available_space(pe, function_addrs) < sizeof(DWORD) * number_of_exports) + return; - // Now, we can iterate through Names and NameOrdinals arrays to obtain - // function names. Not all functions have names. number_of_names = yr_min( - yr_le32toh(exports->NumberOfNames), - exported_functions->number_of_exports); + yr_le32toh(exports->NumberOfNames), number_of_exports); - for (i = 0; i < number_of_names; i++) + // Mapping out the exports is a bit janky. We start with the export address + // array. The index from that array plus the ordinal base is the ordinal for + // that export. To find the name we walk the ordinal array looking for a value + // that matches our index. If one exists we look up the corresponding RVA from + // the names array and follow it to get the name. If one does not exist then + // the export has no name. + // + // Ordinal base: 5 + // 0 1 2 + // Address array: [ 0x00000011 | 0x00000022 | 0x00000033 ] + // 0 1 2 + // Ordinal array: [ 0x0000 | 0x0002 | 0x0001 ] + // 0 1 + // Names array: [ 0x00000044 | 0x00000055 ] + // + // The function at RVA 0x00000011 (index 0) has ordinal 5 (base + index). The + // index can be found in position 0 in the ordinal array. Using 0 to index + // into the name array gives us an RVA (0x00000044) which we can follow to get + // the name. + // + // The function at RVA 0x00000022 (index 1) has ordinal 6 (base + index). The + // index can be found in position 2 in the ordinal array. 2 is out of bounds + // for the names array so this function is exported without a name. + // + // The function at RVA 0x00000033 (index 2) has ordinal 7 (base + index). The + // index can be found in position 1 in the ordinal array. Using 1 to index + // into the name array gives us an RVA (0x00000055) which we can follow to get + // the name. + // + // If the RVA from the address array is within the export directory it is a + // forwarder RVA and points to a NULL terminated ASCII string. + + for (i = 0; i < number_of_exports; i++) { - if (available_space(pe, names + i) < sizeof(DWORD) || - available_space(pe, ordinals + i) < sizeof(WORD)) - { - break; - } - - offset = pe_rva_to_offset(pe, names[i]); - - if (offset < 0) + offset = pe_rva_to_offset(pe, function_addrs[i]); + if (offset <= 0) continue; - // Even though it is called ordinal, it is just index to Functions array - // If it was ordinal it would start from 1 but it starts from 0 - ordinal = yr_le16toh(ordinals[i]); + set_integer( + ordinal_base + i, pe->object, "export_details[%i].ordinal", exp_sz); - if (ordinal >= exported_functions->number_of_exports) - continue; - - remaining = pe->data_size - (size_t) offset; - - if (exported_functions->functions[ordinal].name == NULL) + if (offset > export_start && offset < export_start + export_size) { - exported_functions->functions[ordinal].name = yr_strndup( + remaining = pe->data_size - (size_t) offset; + name_len = strnlen((char*) (pe->data + offset), remaining); + + set_sized_string( (char*) (pe->data + offset), - yr_min(remaining, MAX_EXPORT_NAME_LENGTH)); + yr_min(name_len, MAX_EXPORT_NAME_LENGTH), + pe->object, + "export_details[%i].forward_name", exp_sz); } + else + { + set_integer(offset, pe->object, "export_details[%i].offset", exp_sz); + } + + if (names != NULL) + { + for (j = 0; j < number_of_exports; j++) + { + if (ordinals[j] == i && j < number_of_names) + { + offset = pe_rva_to_offset(pe, names[j]); + if (offset > 0) + { + remaining = pe->data_size - (size_t) offset; + name_len = strnlen((char*) (pe->data + offset), remaining); + + set_sized_string( + (char*) (pe->data + offset), + yr_min(name_len, MAX_EXPORT_NAME_LENGTH), + pe->object, + "export_details[%i].name", exp_sz); + } + break; + } + } + } + exp_sz++; } - set_integer( - exported_functions->number_of_exports, - pe->object, "number_of_exports"); - - return exported_functions; + set_integer(exp_sz, pe->object, "number_of_exports"); + return; } // BoringSSL (https://boringssl.googlesource.com/boringssl/) doesn't support @@ -1125,12 +1276,208 @@ static EXPORT_FUNCTIONS* pe_parse_exports( // but you won't have signature-related features in the PE module. #if defined(HAVE_LIBCRYPTO) && !defined(BORINGSSL) +// +// Parse a PKCS7 blob, looking for certs and nested PKCS7 blobs. +// + +void _parse_pkcs7( + PE* pe, + PKCS7* pkcs7, + int* counter) +{ + int i, j; + time_t date_time; + const char* sig_alg; + char buffer[256]; + int bytes; + int idx; + const EVP_MD* sha1_digest = EVP_sha1(); + const unsigned char* p; + unsigned char thumbprint[YR_SHA1_LEN]; + char thumbprint_ascii[YR_SHA1_LEN * 2 + 1]; + + PKCS7_SIGNER_INFO* signer_info = NULL; + PKCS7* nested_pkcs7 = NULL; + ASN1_INTEGER* serial = NULL; + ASN1_TYPE* nested = NULL; + ASN1_STRING* value = NULL; + X509* cert = NULL; + STACK_OF(X509)* certs = NULL; + X509_ATTRIBUTE *xa = NULL; + STACK_OF(X509_ATTRIBUTE)* attrs = NULL; + + if (*counter >= MAX_PE_CERTS) + return; + + certs = PKCS7_get0_signers(pkcs7, NULL, 0); + + if (!certs) + return; + + for (i = 0; i < sk_X509_num(certs) && *counter < MAX_PE_CERTS; i++) + { + cert = sk_X509_value(certs, i); + + X509_digest(cert, sha1_digest, thumbprint, NULL); + + for (j = 0; j < YR_SHA1_LEN; j++) + sprintf(thumbprint_ascii + (j * 2), "%02x", thumbprint[j]); + + set_string( + (char*) thumbprint_ascii, + pe->object, + "signatures[%i].thumbprint", + *counter); + + X509_NAME_oneline( + X509_get_issuer_name(cert), buffer, sizeof(buffer)); + + set_string(buffer, pe->object, "signatures[%i].issuer", *counter); + + X509_NAME_oneline( + X509_get_subject_name(cert), buffer, sizeof(buffer)); + + set_string(buffer, pe->object, "signatures[%i].subject", *counter); + + set_integer( + X509_get_version(cert) + 1, // Versions are zero based, so add one. + pe->object, + "signatures[%i].version", *counter); + + sig_alg = OBJ_nid2ln(X509_get_signature_nid(cert)); + + set_string(sig_alg, pe->object, "signatures[%i].algorithm", *counter); + + serial = X509_get_serialNumber(cert); + + if (serial) + { + // ASN1_INTEGER can be negative (serial->type & V_ASN1_NEG_INTEGER), + // in which case the serial number will be stored in 2's complement. + // + // Handle negative serial numbers, which are technically not allowed + // by RFC5280, but do exist. An example binary which has a negative + // serial number is: 4bfe05f182aa273e113db6ed7dae4bb8. + // + // Negative serial numbers are handled by calling i2d_ASN1_INTEGER() + // with a NULL second parameter. This will return the size of the + // buffer necessary to store the proper serial number. + // + // Do this even for positive serial numbers because it makes the code + // cleaner and easier to read. + + bytes = i2d_ASN1_INTEGER(serial, NULL); + + // According to X.509 specification the maximum length for the + // serial number is 20 octets. Add two bytes to account for + // DER type and length information. + + if (bytes > 2 && bytes <= 22) + { + // Now that we know the size of the serial number allocate enough + // space to hold it, and use i2d_ASN1_INTEGER() one last time to + // hold it in the allocated buffer. + + unsigned char* serial_der = (unsigned char*) yr_malloc(bytes); + + if (serial_der != NULL) + { + unsigned char* serial_bytes; + char *serial_ascii; + + bytes = i2d_ASN1_INTEGER(serial, &serial_der); + + // i2d_ASN1_INTEGER() moves the pointer as it writes into + // serial_bytes. Move it back. + + serial_der -= bytes; + + // Skip over DER type, length information + serial_bytes = serial_der + 2; + bytes -= 2; + + // Also allocate space to hold the "common" string format: + // 00:01:02:03:04... + // + // For each byte in the serial to convert to hexlified format we + // need three bytes, two for the byte itself and one for colon. + // The last one doesn't have the colon, but the extra byte is used + // for the NULL terminator. + + serial_ascii = (char*) yr_malloc(bytes * 3); + + if (serial_ascii) + { + for (j = 0; j < bytes; j++) + { + // Don't put the colon on the last one. + if (j < bytes - 1) + snprintf( + serial_ascii + 3 * j, 4, "%02x:", serial_bytes[j]); + else + snprintf( + serial_ascii + 3 * j, 3, "%02x", serial_bytes[j]); + } + + set_string( + serial_ascii, + pe->object, + "signatures[%i].serial", + *counter); + + yr_free(serial_ascii); + } + + yr_free(serial_der); + } + } + } + + date_time = ASN1_get_time_t(X509_get_notBefore(cert)); + set_integer(date_time, pe->object, "signatures[%i].not_before", *counter); + + date_time = ASN1_get_time_t(X509_get_notAfter(cert)); + set_integer(date_time, pe->object, "signatures[%i].not_after", *counter); + + (*counter)++; + } + + // See if there is a nested signature, which is apparently an authenticode + // specific feature. See https://github.com/VirusTotal/yara/issues/515. + signer_info = sk_PKCS7_SIGNER_INFO_value(pkcs7->d.sign->signer_info, 0); + if (signer_info != NULL) + { + attrs = PKCS7_get_attributes(signer_info); + idx = X509at_get_attr_by_NID( + attrs, OBJ_txt2nid(SPC_NESTED_SIGNATURE_OBJID), -1); + xa = X509at_get_attr(attrs, idx); + for (j = 0; j < MAX_PE_CERTS; j++) + { + nested = X509_ATTRIBUTE_get0_type(xa, j); + if (nested == NULL) + break; + value = nested->value.sequence; + p = value->data; + nested_pkcs7 = d2i_PKCS7(NULL, &p, value->length); + if (nested_pkcs7 != NULL) + { + _parse_pkcs7(pe, nested_pkcs7, counter); + PKCS7_free(nested_pkcs7); + } + } + } + + sk_X509_free(certs); +} + + static void pe_parse_certificates( PE* pe) { - int i, counter = 0; + int counter = 0; const uint8_t* eod; + const unsigned char* cert_p; uintptr_t end; PWIN_CERTIFICATE win_cert; @@ -1178,9 +1525,7 @@ static void pe_parse_certificates( (uint8_t*) win_cert + sizeof(WIN_CERTIFICATE) < eod && (uint8_t*) win_cert + yr_le32toh(win_cert->Length) <= eod) { - BIO* cert_bio; PKCS7* pkcs7; - STACK_OF(X509)* certs; // Some sanity checks @@ -1197,176 +1542,24 @@ static void pe_parse_certificates( if (yr_le16toh(win_cert->Revision) != WIN_CERT_REVISION_2_0 || yr_le16toh(win_cert->CertificateType) != WIN_CERT_TYPE_PKCS_SIGNED_DATA) { - uintptr_t end = (uintptr_t) - ((uint8_t *) win_cert) + yr_le32toh(win_cert->Length); + end = (uintptr_t)((uint8_t*) win_cert) + yr_le32toh(win_cert->Length); win_cert = (PWIN_CERTIFICATE) (end + (end % 8)); continue; } - cert_bio = BIO_new_mem_buf( - win_cert->Certificate, - yr_le32toh(win_cert->Length) - WIN_CERTIFICATE_HEADER_SIZE); - - if (!cert_bio) - break; - - pkcs7 = d2i_PKCS7_bio(cert_bio, NULL); - certs = PKCS7_get0_signers(pkcs7, NULL, 0); - - if (!certs) + cert_p = win_cert->Certificate; + end = (uintptr_t)((uint8_t*) win_cert) + yr_le32toh(win_cert->Length); + while ((uintptr_t) cert_p < end && counter < MAX_PE_CERTS) { - BIO_free(cert_bio); + pkcs7 = d2i_PKCS7(NULL, &cert_p, (win_cert->Length)); + if (pkcs7 == NULL) + break; + _parse_pkcs7(pe, pkcs7, &counter); PKCS7_free(pkcs7); - break; } - for (i = 0; i < sk_X509_num(certs); i++) - { - time_t date_time; - const char* sig_alg; - char buffer[256]; - int bytes; - const EVP_MD* sha1_digest = EVP_sha1(); - unsigned char thumbprint[YR_SHA1_LEN]; - char thumbprint_ascii[YR_SHA1_LEN * 2 + 1]; - - ASN1_INTEGER* serial; - - X509* cert = sk_X509_value(certs, i); - - X509_digest(cert, sha1_digest, thumbprint, NULL); - - for (i = 0; i < YR_SHA1_LEN; i++) - sprintf(thumbprint_ascii + (i * 2), "%02x", thumbprint[i]); - - set_string( - (char*) thumbprint_ascii, - pe->object, - "signatures[%i].thumbprint", - counter); - - X509_NAME_oneline( - X509_get_issuer_name(cert), buffer, sizeof(buffer)); - - set_string(buffer, pe->object, "signatures[%i].issuer", counter); - - X509_NAME_oneline( - X509_get_subject_name(cert), buffer, sizeof(buffer)); - - set_string(buffer, pe->object, "signatures[%i].subject", counter); - - set_integer( - X509_get_version(cert) + 1, // Versions are zero based, so add one. - pe->object, - "signatures[%i].version", counter); - - sig_alg = OBJ_nid2ln(X509_get_signature_nid(cert)); - - set_string(sig_alg, pe->object, "signatures[%i].algorithm", counter); - - serial = X509_get_serialNumber(cert); - - if (serial) - { - // ASN1_INTEGER can be negative (serial->type & V_ASN1_NEG_INTEGER), - // in which case the serial number will be stored in 2's complement. - // - // Handle negative serial numbers, which are technically not allowed - // by RFC5280, but do exist. An example binary which has a negative - // serial number is: 4bfe05f182aa273e113db6ed7dae4bb8. - // - // Negative serial numbers are handled by calling i2d_ASN1_INTEGER() - // with a NULL second parameter. This will return the size of the - // buffer necessary to store the proper serial number. - // - // Do this even for positive serial numbers because it makes the code - // cleaner and easier to read. - - bytes = i2d_ASN1_INTEGER(serial, NULL); - - // According to X.509 specification the maximum length for the - // serial number is 20 octets. Add two bytes to account for - // DER type and length information. - - if (bytes > 2 && bytes <= 22) - { - // Now that we know the size of the serial number allocate enough - // space to hold it, and use i2d_ASN1_INTEGER() one last time to - // hold it in the allocated buffer. - - unsigned char* serial_der = (unsigned char*) yr_malloc(bytes); - - if (serial_der != NULL) - { - unsigned char* serial_bytes; - char *serial_ascii; - - bytes = i2d_ASN1_INTEGER(serial, &serial_der); - - // i2d_ASN1_INTEGER() moves the pointer as it writes into - // serial_bytes. Move it back. - - serial_der -= bytes; - - // Skip over DER type, length information - serial_bytes = serial_der + 2; - bytes -= 2; - - // Also allocate space to hold the "common" string format: - // 00:01:02:03:04... - // - // For each byte in the serial to convert to hexlified format we - // need three bytes, two for the byte itself and one for colon. - // The last one doesn't have the colon, but the extra byte is used - // for the NULL terminator. - - serial_ascii = (char*) yr_malloc(bytes * 3); - - if (serial_ascii) - { - int j; - - for (j = 0; j < bytes; j++) - { - // Don't put the colon on the last one. - if (j < bytes - 1) - snprintf( - serial_ascii + 3 * j, 4, "%02x:", serial_bytes[j]); - else - snprintf( - serial_ascii + 3 * j, 3, "%02x", serial_bytes[j]); - } - - set_string( - serial_ascii, - pe->object, - "signatures[%i].serial", - counter); - - yr_free(serial_ascii); - } - - yr_free(serial_der); - } - } - } - - date_time = ASN1_get_time_t(X509_getm_notBefore(cert)); - set_integer(date_time, pe->object, "signatures[%i].not_before", counter); - - date_time = ASN1_get_time_t(X509_getm_notAfter(cert)); - set_integer(date_time, pe->object, "signatures[%i].not_after", counter); - - counter++; - } - - end = (uintptr_t)((uint8_t *) win_cert) + yr_le32toh(win_cert->Length); - win_cert = (PWIN_CERTIFICATE)(end + (end % 8)); - - BIO_free(cert_bio); - PKCS7_free(pkcs7); - sk_X509_free(certs); + win_cert = (PWIN_CERTIFICATE) (end + (end % 8)); } set_integer(counter, pe->object, "number_of_signatures"); @@ -1661,12 +1854,12 @@ static void pe_parse_header( // RawData + RawOffset of the last section on the physical file last_section_end = highest_sec_siz + highest_sec_ofs; - // "overlay.offset" is set to UNDEFINED for files that do not have an overlay + // "overlay.offset" is set to YR_UNDEFINED for files that do not have an overlay if (last_section_end && (pe->data_size > last_section_end)) set_integer(last_section_end, pe->object, "overlay.offset"); - // "overlay.size" is zero for well formed PE files that don not have an - // overlay and UNDEFINED for malformed PE files or non-PE files. + // "overlay.size" is zero for well formed PE files that do not have an + // overlay and YR_UNDEFINED for malformed PE files or non-PE files. if (last_section_end && (pe->data_size >= last_section_end)) set_integer(pe->data_size - last_section_end, pe->object, "overlay.size"); } @@ -1684,7 +1877,7 @@ define_function(valid_on) if (is_undefined(parent(), "not_before") || is_undefined(parent(), "not_after")) { - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } timestamp = integer_argument(1); @@ -1709,7 +1902,7 @@ define_function(section_index_addr) int64_t n = get_integer(module, "number_of_sections"); if (is_undefined(module, "number_of_sections")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); for (i = 0; i < yr_min(n, MAX_PE_SECTIONS); i++) { @@ -1728,7 +1921,7 @@ define_function(section_index_addr) return_integer(i); } - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } @@ -1742,7 +1935,7 @@ define_function(section_index_name) int i; if (is_undefined(module, "number_of_sections")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); for (i = 0; i < yr_min(n, MAX_PE_SECTIONS); i++) { @@ -1752,33 +1945,36 @@ define_function(section_index_name) return_integer(i); } - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); } define_function(exports) { - SIZED_STRING* function_name = sized_string_argument(1); + SIZED_STRING* search_name = sized_string_argument(1); + SIZED_STRING* function_name = NULL; YR_OBJECT* module = module(); PE* pe = (PE*) module->data; - int i; + int i, n; - // If not a PE, return UNDEFINED. + // If not a PE, return YR_UNDEFINED. if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); - // If PE, but not exported functions, return false. - if (pe->exported_functions == NULL) + // If PE, but no exported functions, return false. + n = get_integer(module, "number_of_exports"); + if (n == 0) return_integer(0); - for (i = 0; i < pe->exported_functions->number_of_exports; i++) + for (i = 0; i < n; i++) { - if (pe->exported_functions->functions[i].name && - strcasecmp( - pe->exported_functions->functions[i].name, - function_name->c_string) == 0) + function_name = get_string(module, "export_details[%i].name", i); + if (function_name == NULL) + continue; + + if (sized_string_cmp_nocase(function_name, search_name) == 0) { return_integer(1); } @@ -1792,26 +1988,28 @@ define_function(exports_regexp) { RE* regex = regexp_argument(1); + SIZED_STRING* function_name = NULL; YR_OBJECT* module = module(); PE* pe = (PE*) module->data; - int i; + int i, n; - // If not a PE, return UNDEFINED. + // If not a PE, return YR_UNDEFINED. if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); - // If PE, but not exported functions, return false. - if (pe->exported_functions == NULL) + // If PE, but no exported functions, return false. + n = get_integer(module, "number_of_exports"); + if (n == 0) return_integer(0); - for (i = 0; i < pe->exported_functions->number_of_exports; i++) + for (i = 0; i < n; i++) { - if (pe->exported_functions->functions[i].name && - yr_re_match( - scan_context(), - regex, - pe->exported_functions->functions[i].name) != -1) + function_name = get_string(module, "export_details[%i].name", i); + if (function_name == NULL) + continue; + + if (yr_re_match(scan_context(), regex, function_name->c_string) != -1) { return_integer(1); } @@ -1827,25 +2025,134 @@ define_function(exports_ordinal) YR_OBJECT* module = module(); PE* pe = (PE*) module->data; + int i, n, exported_ordinal; - // If not a PE, return UNDEFINED. + // If not a PE, return YR_UNDEFINED. if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); - // If PE, but not exported functions, return false. - if (pe->exported_functions == NULL) + // If PE, but no exported functions, return false. + n = get_integer(module, "number_of_exports"); + if (n == 0) return_integer(0); - if (ordinal == 0 || ordinal > pe->exported_functions->number_of_exports) + if (ordinal == 0 || ordinal > n) return_integer(0); - // Just in case, this should always be true - if (pe->exported_functions->functions[ordinal - 1].ordinal == ordinal) - return_integer(1); + for (i = 0; i < n; i++) + { + exported_ordinal = yr_object_get_integer( + module, "export_details[%i].ordinal", i); + if (exported_ordinal == ordinal) + return_integer(1); + } return_integer(0); } + +define_function(exports_index_name) +{ + SIZED_STRING* search_name = sized_string_argument(1); + + SIZED_STRING* function_name = NULL; + YR_OBJECT* module = module(); + PE* pe = (PE*) module->data; + + int i, n; + + // If not a PE, return YR_UNDEFINED. + if (pe == NULL) + return_integer(YR_UNDEFINED); + + // If PE, but no exported functions, return false. + n = get_integer(module, "number_of_exports"); + if (n == 0) + return_integer(YR_UNDEFINED); + + for (i = 0; i < n; i++) + { + function_name = get_string(module, "export_details[%i].name", i); + if (function_name == NULL) + continue; + + if (sized_string_cmp_nocase(function_name, search_name) == 0) + { + return_integer(i); + } + } + + return_integer(YR_UNDEFINED); +} + + +define_function(exports_index_ordinal) +{ + uint64_t ordinal = integer_argument(1); + + YR_OBJECT* module = module(); + PE* pe = (PE*) module->data; + int i, n, exported_ordinal; + + // If not a PE, return YR_UNDEFINED. + if (pe == NULL) + return_integer(YR_UNDEFINED); + + // If PE, but no exported functions, return false. + n = get_integer(module, "number_of_exports"); + if (n == 0) + return_integer(YR_UNDEFINED); + + if (ordinal == 0 || ordinal > n) + return_integer(YR_UNDEFINED); + + for (i = 0; i < n; i++) + { + exported_ordinal = yr_object_get_integer( + module, "export_details[%i].ordinal", i); + if (exported_ordinal == ordinal) + return_integer(i); + } + + return_integer(YR_UNDEFINED); +} + + +define_function(exports_index_regex) +{ + RE* regex = regexp_argument(1); + + SIZED_STRING* function_name = NULL; + YR_OBJECT* module = module(); + PE* pe = (PE*) module->data; + + int i, n; + + // If not a PE, return YR_UNDEFINED. + if (pe == NULL) + return_integer(YR_UNDEFINED); + + // If PE, but no exported functions, return false. + n = get_integer(module, "number_of_exports"); + if (n == 0) + return_integer(YR_UNDEFINED); + + for (i = 0; i < n; i++) + { + function_name = get_string(module, "export_details[%i].name", i); + if (function_name == NULL) + continue; + + if (yr_re_match(scan_context(), regex, function_name->c_string) != -1) + { + return_integer(i); + } + } + + return_integer(YR_UNDEFINED); +} + + #if defined(HAVE_LIBCRYPTO) || \ defined(HAVE_WINCRYPT_H) || \ defined(HAVE_COMMONCRYPTO_COMMONCRYPTO_H) @@ -1872,10 +2179,10 @@ define_function(imphash) PE* pe = (PE*) module->data; - // If not a PE, return UNDEFINED. + // If not a PE, return YR_UNDEFINED. if (!pe) - return_string(UNDEFINED); + return_string(YR_UNDEFINED); // Lookup in cache first. digest_ascii = (char*) yr_hash_table_lookup( @@ -1991,7 +2298,7 @@ define_function(imports) IMPORTED_DLL* imported_dll; if (!pe) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); imported_dll = pe->imported_dlls; @@ -2028,7 +2335,7 @@ define_function(imports_ordinal) IMPORTED_DLL* imported_dll; if (!pe) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); imported_dll = pe->imported_dlls; @@ -2059,9 +2366,10 @@ define_function(imports_regex) PE* pe = (PE*)module->data; IMPORTED_DLL* imported_dll; + uint64_t imported_func_count = 0; if (!pe) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); imported_dll = pe->imported_dlls; @@ -2074,7 +2382,7 @@ define_function(imports_regex) while (imported_func != NULL) { if (yr_re_match(scan_context(), regexp_argument(2), imported_func->name) > 0) - return_integer(1); + imported_func_count++; imported_func = imported_func->next; } } @@ -2082,7 +2390,7 @@ define_function(imports_regex) imported_dll = imported_dll->next; } - return_integer(0); + return_integer(imported_func_count); } define_function(imports_dll) @@ -2093,9 +2401,10 @@ define_function(imports_dll) PE* pe = (PE*) module->data; IMPORTED_DLL* imported_dll; + uint64_t imported_func_count = 0; if (!pe) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); imported_dll = pe->imported_dlls; @@ -2103,13 +2412,19 @@ define_function(imports_dll) { if (strcasecmp(imported_dll->name, dll_name) == 0) { - return_integer(1); + IMPORT_FUNCTION* imported_func = imported_dll->functions; + + while (imported_func != NULL) + { + imported_func_count++; + imported_func = imported_func->next; + } } imported_dll = imported_dll->next; } - return_integer(0); + return_integer(imported_func_count); } define_function(locale) @@ -2121,19 +2436,19 @@ define_function(locale) int64_t n, i; if (is_undefined(module, "number_of_resources")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); - // If not a PE file, return UNDEFINED + // If not a PE file, return YR_UNDEFINED if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); n = get_integer(module, "number_of_resources"); for (i = 0; i < n; i++) { uint64_t rsrc_language = get_integer( - module, "resources[%" PRId64 "].language", i); + module, "resources[%" PRIi32 "].language", (int32_t)i); if ((rsrc_language & 0xFFFF) == locale) return_integer(1); @@ -2152,19 +2467,19 @@ define_function(language) int64_t n, i; if (is_undefined(module, "number_of_resources")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); - // If not a PE file, return UNDEFINED + // If not a PE file, return YR_UNDEFINED if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); n = get_integer(module, "number_of_resources"); for (i = 0; i < n; i++) { uint64_t rsrc_language = get_integer( - module, "resources[%" PRId64 "].language", i); + module, "resources[%" PRIi32 "].language", (int32_t)i); if ((rsrc_language & 0xFF) == language) return_integer(1); @@ -2180,7 +2495,7 @@ define_function(is_dll) YR_OBJECT* module = module(); if (is_undefined(module, "characteristics")) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); characteristics = get_integer(module, "characteristics"); return_integer(characteristics & IMAGE_FILE_DLL); @@ -2193,7 +2508,7 @@ define_function(is_32bit) PE* pe = (PE*) module->data; if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); return_integer(IS_64BITS_PE(pe) ? 0 : 1); } @@ -2205,7 +2520,7 @@ define_function(is_64bit) PE* pe = (PE*) module->data; if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); return_integer(IS_64BITS_PE(pe) ? 1 : 0); } @@ -2233,16 +2548,16 @@ static uint64_t _rich_version( // Check if the required fields are set if (is_undefined(module, "rich_signature.length")) - return UNDEFINED; + return YR_UNDEFINED; rich_length = get_integer(module, "rich_signature.length"); rich_string = get_string(module, "rich_signature.clear_data"); - // If the clear_data was not set, return UNDEFINED + // If the clear_data was not set, return YR_UNDEFINED if (rich_string == NULL) - return UNDEFINED; + return YR_UNDEFINED; - if (version == UNDEFINED && toolid == UNDEFINED) + if (version == YR_UNDEFINED && toolid == YR_UNDEFINED) return false; clear_rich_signature = (PRICH_SIGNATURE) rich_string->c_string; @@ -2259,8 +2574,8 @@ static uint64_t _rich_version( int match_version = (version == RICH_VERSION_VERSION(id_version)); int match_toolid = (toolid == RICH_VERSION_ID(id_version)); - if ((version == UNDEFINED || match_version) && - (toolid == UNDEFINED || match_toolid)) + if ((version == YR_UNDEFINED || match_version) && + (toolid == YR_UNDEFINED || match_toolid)) { result += yr_le32toh(clear_rich_signature->versions[i].times); } @@ -2273,7 +2588,7 @@ static uint64_t _rich_version( define_function(rich_version) { return_integer( - _rich_version(module(), integer_argument(1), UNDEFINED)); + _rich_version(module(), integer_argument(1), YR_UNDEFINED)); } @@ -2287,7 +2602,7 @@ define_function(rich_version_toolid) define_function(rich_toolid) { return_integer( - _rich_version(module(), UNDEFINED, integer_argument(1))); + _rich_version(module(), YR_UNDEFINED, integer_argument(1))); } @@ -2308,7 +2623,7 @@ define_function(calculate_checksum) size_t i, j; if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); csum_offset = ((uint8_t*) &(pe->header->OptionalHeader) + offsetof(IMAGE_OPTIONAL_HEADER32, CheckSum)) - pe->data; @@ -2355,12 +2670,12 @@ define_function(rva_to_offset) uint64_t rva, offset; if (pe == NULL) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); rva = integer_argument(1); offset = pe_rva_to_offset(pe, rva); if (offset == -1) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); return_integer(offset); } @@ -2588,6 +2903,9 @@ begin_declarations; declare_function("exports", "s", "i", exports); declare_function("exports", "r", "i", exports_regexp); declare_function("exports", "i", "i", exports_ordinal); + declare_function("exports_index", "s", "i", exports_index_name); + declare_function("exports_index", "i", "i", exports_index_ordinal); + declare_function("exports_index", "r", "i", exports_index_regex); declare_function("imports", "ss", "i", imports); declare_function("imports", "si", "i", imports_ordinal); declare_function("imports", "s", "i", imports_dll); @@ -2601,6 +2919,15 @@ begin_declarations; declare_integer("number_of_imports"); declare_integer("number_of_exports"); + declare_string("dll_name"); + declare_integer("export_timestamp"); + begin_struct_array("export_details"); + declare_integer("offset"); + declare_string("name"); + declare_string("forward_name"); + declare_integer("ordinal"); + end_struct_array("export_details"); + declare_integer("resource_timestamp"); begin_struct("resource_version"); @@ -2620,6 +2947,7 @@ begin_declarations; end_struct_array("resources"); declare_integer("number_of_resources"); + declare_string("pdb_path"); #if defined(HAVE_LIBCRYPTO) && !defined(BORINGSSL) begin_struct_array("signatures"); @@ -2645,6 +2973,11 @@ end_declarations; int module_initialize( YR_MODULE* module) { +#if defined(HAVE_LIBCRYPTO) + // Not checking return value here because if it fails we will not parse the + // nested signature silently. + OBJ_create(SPC_NESTED_SIGNATURE_OBJID, NULL, NULL); +#endif return ERROR_SUCCESS; } @@ -3033,13 +3366,14 @@ int module_load( pe_parse_header(pe, block->base, context->flags); pe_parse_rich_signature(pe, block->base); - + pe_parse_debug_directory(pe); + #if defined(HAVE_LIBCRYPTO) && !defined(BORINGSSL) pe_parse_certificates(pe); #endif pe->imported_dlls = pe_parse_imports(pe); - pe->exported_functions = pe_parse_exports(pe); + pe_parse_exports(pe); break; } @@ -3057,7 +3391,6 @@ int module_unload( IMPORTED_DLL* next_dll = NULL; IMPORT_FUNCTION* func = NULL; IMPORT_FUNCTION* next_func = NULL; - int i = 0; PE* pe = (PE *) module_object->data; @@ -3093,18 +3426,6 @@ int module_unload( dll = next_dll; } - if (pe->exported_functions) - { - for (i = 0; i < pe->exported_functions->number_of_exports; i++) - { - if (pe->exported_functions->functions[i].name) - yr_free(pe->exported_functions->functions[i].name); - } - - yr_free(pe->exported_functions->functions); - yr_free(pe->exported_functions); - } - yr_free(pe); return ERROR_SUCCESS; diff --git a/libyara/modules/tests/tests.c b/libyara/modules/tests/tests.c index 27cf42bf..70c816de 100644 --- a/libyara/modules/tests/tests.c +++ b/libyara/modules/tests/tests.c @@ -141,6 +141,10 @@ begin_declarations; declare_string("s"); end_struct_dictionary("struct_dict"); + begin_struct_dictionary("empty_struct_dict"); + declare_integer("unused"); + end_struct_dictionary("empty_struct_dict"); + declare_function("match", "rs", "i", match); declare_function("isum", "ii", "i", isum_2); declare_function("isum", "iii", "i", isum_3); diff --git a/libyara/modules/time/time.c b/libyara/modules/time/time.c index 20bb7ee4..94180abe 100644 --- a/libyara/modules/time/time.c +++ b/libyara/modules/time/time.c @@ -38,7 +38,7 @@ define_function(now) { time_t now = time(NULL); if (now == -1) - return_integer(UNDEFINED); + return_integer(YR_UNDEFINED); return_integer((long) now); } diff --git a/libyara/notebook.c b/libyara/notebook.c new file mode 100644 index 00000000..bcf52a5f --- /dev/null +++ b/libyara/notebook.c @@ -0,0 +1,152 @@ +#include + +/* +Copyright (c) 2020. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include + +#include +#include +#include + + +// Forward declaration of YR_NOTEBOOK_PAGE. +typedef struct YR_NOTEBOOK_PAGE YR_NOTEBOOK_PAGE; + + +// A notebook is a data structure that can be used for allocating memory +// space in the same way malloc() would do. However, the buffers returned +// by yr_notebook_alloc() are backed by a larger buffer reserved by the notebook +// beforehand called a "page". The notebook fulfills the allocations performed +// via yr_notebook_alloc() with space taken from the current page, or creates +// a new page when necessary. It's recommended that the page size is at least +// 4x the size of the buffers you plan to allocate with yr_notebook_alloc(). + +// Once the notebook is destroyed all the pages are freed, and consequently +// all the buffers allocated via yr_notebook_alloc(). +struct YR_NOTEBOOK +{ + // Size of each page in the notebook. + size_t page_size; + // Pointer to the first page in the book, this also the most recently + // created page, the one that is being filled. + YR_NOTEBOOK_PAGE* page_list_head; +}; + + +// YR_NOTEBOOK_PAGE +struct YR_NOTEBOOK_PAGE +{ + // Amount of bytes in the page that are actually used. + size_t used; + // Pointer to next page. + YR_NOTEBOOK_PAGE* next; + // Page's data. + uint8_t data[0]; +}; + + + +// Creates a new notebook. The notebook initially has a single page of the +// specified size, but more pages are created if needed. +int yr_notebook_create( + size_t page_size, + YR_NOTEBOOK** pool) +{ + YR_NOTEBOOK* new_notebook = yr_malloc(sizeof(YR_NOTEBOOK)); + + if (new_notebook == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + new_notebook->page_list_head = yr_malloc( + sizeof(YR_NOTEBOOK_PAGE) + page_size); + + if (new_notebook->page_list_head == NULL) + { + yr_free(new_notebook); + return ERROR_INSUFFICIENT_MEMORY; + } + + new_notebook->page_size = page_size; + new_notebook->page_list_head->used = 0; + new_notebook->page_list_head->next = NULL; + + *pool = new_notebook; + + return ERROR_SUCCESS; +} + + +// Destroys a notebook and frees all the notebook's pages. +int yr_notebook_destroy( + YR_NOTEBOOK* pool) +{ + YR_NOTEBOOK_PAGE* page = pool->page_list_head; + + while (page != NULL) + { + YR_NOTEBOOK_PAGE* next = page->next; + yr_free(page); + page = next; + } + + yr_free(pool); + + return ERROR_SUCCESS; +} + + +void* yr_notebook_alloc( + YR_NOTEBOOK* notebook, + size_t size) +{ + // The requested memory size can't be larger than a notebook's page. + assert(size <= notebook->page_size); + + // If the requested size doesn't fit in current page's free space, allocate + // a new page. + if (notebook->page_size - notebook->page_list_head->used < size) + { + YR_NOTEBOOK_PAGE* new_page = yr_malloc( + sizeof(YR_NOTEBOOK_PAGE) + notebook->page_size); + + if (new_page == NULL) + return NULL; + + new_page->used = 0; + new_page->next = notebook->page_list_head; + notebook->page_list_head = new_page; + } + + void *ptr = notebook->page_list_head->data + notebook->page_list_head->used; + + notebook->page_list_head->used += size; + + return ptr; +} \ No newline at end of file diff --git a/libyara/object.c b/libyara/object.c index 6a20b8d4..75fb38ca 100644 --- a/libyara/object.c +++ b/libyara/object.c @@ -96,7 +96,7 @@ int yr_object_create( switch(type) { case OBJECT_TYPE_INTEGER: - obj->value.i = UNDEFINED; + obj->value.i = YR_UNDEFINED; break; case OBJECT_TYPE_FLOAT: obj->value.d = NAN; @@ -314,7 +314,14 @@ int yr_object_from_external_variable( break; } - *object = obj; + if (result == ERROR_SUCCESS) + { + *object = obj; + } + else + { + yr_object_destroy(obj); + } } return result; @@ -594,6 +601,7 @@ int yr_object_copy( yr_object_copy( object_as_function(object)->return_obj, &object_as_function(copy)->return_obj), + // cleanup yr_object_destroy(copy)); for (i = 0; i < YR_MAX_OVERLOADED_FUNCTIONS; i++) @@ -614,6 +622,7 @@ int yr_object_copy( FAIL_ON_ERROR_WITH_CLEANUP( yr_object_structure_set_member(copy, o), + // cleanup yr_free(o); yr_object_destroy(copy)); @@ -911,7 +920,7 @@ bool yr_object_has_undefined_value( case OBJECT_TYPE_STRING: return field_obj->value.ss == NULL; case OBJECT_TYPE_INTEGER: - return field_obj->value.i == UNDEFINED; + return field_obj->value.i == YR_UNDEFINED; } return false; @@ -936,7 +945,7 @@ int64_t yr_object_get_integer( va_end(args); if (integer_obj == NULL) - return UNDEFINED; + return YR_UNDEFINED; assertf(integer_obj->type == OBJECT_TYPE_INTEGER, "type of \"%s\" is not integer\n", field); @@ -1155,19 +1164,19 @@ YR_API void yr_object_print_data( switch(object->type) { case OBJECT_TYPE_FLOAT: - if (object->value.d != UNDEFINED) + if (object->value.i != YR_UNDEFINED) printf(" = %f", object->value.d); else - printf(" = UNDEFINED"); + printf(" = YR_UNDEFINED"); break; case OBJECT_TYPE_INTEGER: - if (object->value.i != UNDEFINED) + if (object->value.i != YR_UNDEFINED) printf(" = %" PRId64, object->value.i); else - printf(" = UNDEFINED"); + printf(" = YR_UNDEFINED"); break; @@ -1192,7 +1201,7 @@ YR_API void yr_object_print_data( } else { - printf(" = UNDEFINED"); + printf(" = YR_UNDEFINED"); } break; diff --git a/libyara/parser.c b/libyara/parser.c index 5bc3d140..d4f9c20c 100644 --- a/libyara/parser.c +++ b/libyara/parser.c @@ -29,9 +29,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include +#include #include #include #include @@ -41,6 +43,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include + #define todigit(x) ((x) >='A'&& (x) <='F')? \ ((uint8_t) (x - 'A' + 10)) : \ @@ -50,13 +54,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. int yr_parser_emit( yyscan_t yyscanner, uint8_t instruction, - uint8_t** instruction_address) + YR_ARENA_REF* instruction_ref) { return yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &instruction, - sizeof(int8_t), - (void**) instruction_address); + sizeof(uint8_t), + instruction_ref); } @@ -64,21 +69,49 @@ int yr_parser_emit_with_arg_double( yyscan_t yyscanner, uint8_t instruction, double argument, - uint8_t** instruction_address, - double** argument_address) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { int result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &instruction, sizeof(uint8_t), - (void**) instruction_address); + instruction_ref); if (result == ERROR_SUCCESS) result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &argument, sizeof(double), - (void**) argument_address); + argument_ref); + + return result; +} + + +int yr_parser_emit_with_arg_int32( + yyscan_t yyscanner, + uint8_t instruction, + int32_t argument, + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) +{ + int result = yr_arena_write_data( + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, + &instruction, + sizeof(uint8_t), + instruction_ref); + + if (result == ERROR_SUCCESS) + result = yr_arena_write_data( + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, + &argument, + sizeof(int32_t), + argument_ref); return result; } @@ -88,21 +121,23 @@ int yr_parser_emit_with_arg( yyscan_t yyscanner, uint8_t instruction, int64_t argument, - uint8_t** instruction_address, - int64_t** argument_address) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { int result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &instruction, sizeof(uint8_t), - (void**) instruction_address); + instruction_ref); if (result == ERROR_SUCCESS) result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &argument, sizeof(int64_t), - (void**) argument_address); + argument_ref); return result; } @@ -112,39 +147,40 @@ int yr_parser_emit_with_arg_reloc( yyscan_t yyscanner, uint8_t instruction, void* argument, - uint8_t** instruction_address, - void** argument_address) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { - int64_t* ptr = NULL; - int result; + YR_ARENA_REF ref; DECLARE_REFERENCE(void*, ptr) arg; memset(&arg, 0, sizeof(arg)); arg.ptr = argument; - result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + int result = yr_arena_write_data( + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &instruction, sizeof(uint8_t), - (void**) instruction_address); + instruction_ref); if (result == ERROR_SUCCESS) result = yr_arena_write_data( - yyget_extra(yyscanner)->code_arena, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, &arg, sizeof(arg), - (void**) &ptr); + &ref); if (result == ERROR_SUCCESS) result = yr_arena_make_ptr_relocatable( - yyget_extra(yyscanner)->code_arena, - ptr, - 0, + yyget_extra(yyscanner)->arena, + YR_CODE_SECTION, + ref.offset, EOL); - if (argument_address != NULL) - *argument_address = (void*) ptr; + if (argument_ref != NULL) + *argument_ref = ref; return result; } @@ -155,14 +191,18 @@ int yr_parser_emit_pushes_for_strings( const char* identifier) { YR_COMPILER* compiler = yyget_extra(yyscanner); - YR_STRING* string = compiler->current_rule->strings; + + YR_RULE* current_rule = _yr_compiler_get_rule_by_idx( + compiler, compiler->current_rule_idx); + + YR_STRING* string; const char* string_identifier; const char* target_identifier; int matching = 0; - while(!STRING_IS_NULL(string)) + yr_rule_strings_foreach(current_rule, string) { // Don't generate pushes for strings chained to another one, we are // only interested in non-chained strings or the head of the chain. @@ -190,21 +230,16 @@ int yr_parser_emit_pushes_for_strings( NULL, NULL); - string->g_flags |= STRING_GFLAGS_REFERENCED; - string->g_flags &= ~STRING_GFLAGS_FIXED_OFFSET; + string->flags |= STRING_FLAGS_REFERENCED; + string->flags &= ~STRING_FLAGS_FIXED_OFFSET; matching++; } } - - string = (YR_STRING*) yr_arena_next_address( - compiler->strings_arena, - string, - sizeof(YR_STRING)); } if (matching == 0) { - yr_compiler_set_error_extra_info(compiler, identifier); + yr_compiler_set_error_extra_info(compiler, identifier) return ERROR_UNDEFINED_STRING; } @@ -228,7 +263,7 @@ int yr_parser_check_types( return ERROR_SUCCESS; } - yr_compiler_set_error_extra_info(compiler, function->identifier); + yr_compiler_set_error_extra_info(compiler, function->identifier) return ERROR_WRONG_ARGUMENTS; } @@ -241,28 +276,24 @@ int yr_parser_lookup_string( { YR_COMPILER* compiler = yyget_extra(yyscanner); - *string = compiler->current_rule->strings; + YR_RULE* current_rule = _yr_compiler_get_rule_by_idx( + compiler, compiler->current_rule_idx); - while(!STRING_IS_NULL(*string)) + yr_rule_strings_foreach(current_rule, *string) { // If some string $a gets fragmented into multiple chained // strings, all those fragments have the same $a identifier // but we are interested in the heading fragment, which is // that with chained_to == NULL - if (strcmp((*string)->identifier, identifier) == 0 && - (*string)->chained_to == NULL) + if ((*string)->chained_to == NULL && + strcmp((*string)->identifier, identifier) == 0) { return ERROR_SUCCESS; } - - *string = (YR_STRING*) yr_arena_next_address( - compiler->strings_arena, - *string, - sizeof(YR_STRING)); } - yr_compiler_set_error_extra_info(compiler, identifier); + yr_compiler_set_error_extra_info(compiler, identifier) *string = NULL; @@ -280,7 +311,7 @@ int yr_parser_lookup_string( // already defined by an outer loop. // // If the variable is found, the return value is the position that the variable -// occuppies among all the currently defined variables. If the variable doesn't +// occupies among all the currently defined variables. If the variable doesn't // exist the return value is -1. // // The function can receive a pointer to a YR_EXPRESSION that will populated @@ -302,8 +333,8 @@ int yr_parser_lookup_loop_variable( for (j = 0; j < compiler->loop[i].vars_count; j++) { - if (compiler->loop[i].vars[j].identifier != NULL && - strcmp(identifier, compiler->loop[i].vars[j].identifier ) == 0) + if (compiler->loop[i].vars[j].identifier.ptr != NULL && + strcmp(identifier, compiler->loop[i].vars[j].identifier.ptr) == 0) { if (expr != NULL) *expr = compiler->loop[i].vars[j]; @@ -325,7 +356,7 @@ static int _yr_parser_write_string( YR_COMPILER* compiler, SIZED_STRING* str, RE_AST* re_ast, - YR_STRING** string, + YR_ARENA_REF* string_ref, int* min_atom_quality, int* num_atom) { @@ -337,37 +368,40 @@ static int _yr_parser_write_string( int max_string_len; bool free_literal = false; - *string = NULL; - - result = yr_arena_allocate_struct( - compiler->strings_arena, + FAIL_ON_ERROR(yr_arena_allocate_struct( + compiler->arena, + YR_STRINGS_TABLE, sizeof(YR_STRING), - (void**) string, + string_ref, offsetof(YR_STRING, identifier), offsetof(YR_STRING, string), offsetof(YR_STRING, chained_to), - offsetof(YR_STRING, rule), - EOL); + EOL)); - if (result != ERROR_SUCCESS) - return result; + YR_STRING* string = (YR_STRING*) yr_arena_ref_to_ptr( + compiler->arena, string_ref); - result = yr_arena_write_string( - compiler->sz_arena, + YR_ARENA_REF ref; + + FAIL_ON_ERROR(yr_arena_write_string( + compiler->arena, + YR_SZ_POOL, identifier, - &(*string)->identifier); + &ref)); - if (result != ERROR_SUCCESS) - return result; + string->identifier = (const char*) yr_arena_ref_to_ptr( + compiler->arena, &ref); - if (modifier.flags & STRING_GFLAGS_HEXADECIMAL || - modifier.flags & STRING_GFLAGS_REGEXP) + if (modifier.flags & STRING_FLAGS_HEXADECIMAL || + modifier.flags & STRING_FLAGS_REGEXP || + modifier.flags & STRING_FLAGS_BASE64 || + modifier.flags & STRING_FLAGS_BASE64_WIDE) { literal_string = yr_re_ast_extract_literal(re_ast); if (literal_string != NULL) { - modifier.flags |= STRING_GFLAGS_LITERAL; + modifier.flags |= STRING_FLAGS_LITERAL; free_literal = true; } else @@ -377,38 +411,33 @@ static int _yr_parser_write_string( // the string should start, as the non-literal strings can contain // variable-length portions. - modifier.flags &= ~STRING_GFLAGS_FIXED_OFFSET; + modifier.flags &= ~STRING_FLAGS_FIXED_OFFSET; } } else { literal_string = str; - modifier.flags |= STRING_GFLAGS_LITERAL; + modifier.flags |= STRING_FLAGS_LITERAL; } - (*string)->g_flags = modifier.flags; - (*string)->chained_to = NULL; - (*string)->fixed_offset = UNDEFINED; - (*string)->rule = compiler->current_rule; + string->flags = modifier.flags; + string->rule_idx = compiler->current_rule_idx; + string->idx = compiler->current_string_idx; + string->fixed_offset = YR_UNDEFINED; + string->chained_to = NULL; + string->string = NULL; - memset((*string)->matches, 0, - sizeof((*string)->matches)); - - memset((*string)->private_matches, 0, - sizeof((*string)->private_matches)); - - memset((*string)->unconfirmed_matches, 0, - sizeof((*string)->unconfirmed_matches)); - - if (modifier.flags & STRING_GFLAGS_LITERAL) + if (modifier.flags & STRING_FLAGS_LITERAL) { - (*string)->length = (uint32_t) literal_string->length; - result = yr_arena_write_data( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, literal_string->c_string, literal_string->length + 1, // +1 to include terminating NULL - (void**) &(*string)->string); + &ref); + + string->length = (uint32_t) literal_string->length; + string->string = (uint8_t*) yr_arena_ref_to_ptr(compiler->arena, &ref); if (result == ERROR_SUCCESS) { @@ -424,11 +453,13 @@ static int _yr_parser_write_string( else { // Emit forwards code - result = yr_re_ast_emit_code(re_ast, compiler->re_code_arena, false); + result = yr_re_ast_emit_code( + re_ast, compiler->arena, false); // Emit backwards code if (result == ERROR_SUCCESS) - result = yr_re_ast_emit_code(re_ast, compiler->re_code_arena, true); + result = yr_re_ast_emit_code( + re_ast, compiler->arena, true); if (result == ERROR_SUCCESS) result = yr_atoms_extract_from_re( @@ -444,20 +475,21 @@ static int _yr_parser_write_string( // Add the string to Aho-Corasick automaton. result = yr_ac_add_string( compiler->automaton, - *string, + string, + compiler->current_string_idx, atom_list, - compiler->matches_arena); + compiler->arena); } - if (modifier.flags & STRING_GFLAGS_LITERAL) + if (modifier.flags & STRING_FLAGS_LITERAL) { - if (modifier.flags & STRING_GFLAGS_WIDE) - max_string_len = (*string)->length * 2; + if (modifier.flags & STRING_FLAGS_WIDE) + max_string_len = string->length * 2; else - max_string_len = (*string)->length; + max_string_len = string->length; if (max_string_len <= YR_MAX_ATOM_LENGTH) - (*string)->g_flags |= STRING_GFLAGS_FITS_IN_ATOM; + string->flags |= STRING_FLAGS_FITS_IN_ATOM; } atom = atom_list; @@ -471,6 +503,8 @@ static int _yr_parser_write_string( (*num_atom) += c; + compiler->current_string_idx++; + if (free_literal) yr_free(literal_string); @@ -480,112 +514,153 @@ static int _yr_parser_write_string( return result; } -#include - -#include - - int yr_parser_reduce_string_declaration( yyscan_t yyscanner, YR_MODIFIER modifier, const char* identifier, SIZED_STRING* str, - YR_STRING** string) + YR_ARENA_REF* string_ref) { + int result = ERROR_SUCCESS; int min_atom_quality = YR_MAX_ATOM_QUALITY; int atom_quality; - int32_t min_gap = 0; - int32_t max_gap = 0; - int32_t prev_min_gap; - int32_t prev_max_gap; - char message[512]; + int32_t min_gap = 0; + int32_t max_gap = 0; + YR_COMPILER* compiler = yyget_extra(yyscanner); - YR_STRING* prev_string = NULL; RE_AST* re_ast = NULL; RE_AST* remainder_re_ast = NULL; - RE_ERROR re_error; - int result = ERROR_SUCCESS; + YR_RULE* current_rule = _yr_compiler_get_rule_by_idx( + compiler, compiler->current_rule_idx); // Determine if a string with the same identifier was already defined - // by searching for the identifier in string_table. - - YR_STRING* new_string = (YR_STRING*) yr_hash_table_lookup( + // by searching for the identifier in strings_table. + uint32_t string_idx = yr_hash_table_lookup_uint32( compiler->strings_table, identifier, NULL); - if (new_string != NULL) + // The string was already defined, return an error. + if (string_idx != UINT32_MAX) { result = ERROR_DUPLICATED_STRING_IDENTIFIER; - yr_compiler_set_error_extra_info(compiler, identifier); + yr_compiler_set_error_extra_info(compiler, identifier) goto _exit; } // Empty strings are not allowed - if (str->length == 0) { result = ERROR_EMPTY_STRING; - yr_compiler_set_error_extra_info(compiler, identifier); + yr_compiler_set_error_extra_info(compiler, identifier) goto _exit; } - if (str->flags & SIZED_STRING_FLAGS_NO_CASE) - modifier.flags |= STRING_GFLAGS_NO_CASE; - - // xor and nocase together is not implemented. - - if (modifier.flags & STRING_GFLAGS_XOR && modifier.flags & STRING_GFLAGS_NO_CASE) + // If string identifier is $ this is an anonymous string, if not add the + // identifier to strings_table. + if (strcmp(identifier, "$") == 0) { - result = ERROR_INVALID_MODIFIER; - yr_compiler_set_error_extra_info(compiler, "xor nocase"); + modifier.flags |= STRING_FLAGS_ANONYMOUS; + } + else + { + result = yr_hash_table_add_uint32( + compiler->strings_table, + identifier, + NULL, + compiler->current_string_idx); + + if (result != ERROR_SUCCESS) goto _exit; } + if (str->flags & SIZED_STRING_FLAGS_NO_CASE) + modifier.flags |= STRING_FLAGS_NO_CASE; + if (str->flags & SIZED_STRING_FLAGS_DOT_ALL) - modifier.flags |= STRING_GFLAGS_DOT_ALL; - - if (strcmp(identifier,"$") == 0) - modifier.flags |= STRING_GFLAGS_ANONYMOUS; - - if (!(modifier.flags & STRING_GFLAGS_WIDE) && - !(modifier.flags & STRING_GFLAGS_XOR)) - modifier.flags |= STRING_GFLAGS_ASCII; + modifier.flags |= STRING_FLAGS_DOT_ALL; // Hex strings are always handled as DOT_ALL regexps. + if (modifier.flags & STRING_FLAGS_HEXADECIMAL) + modifier.flags |= STRING_FLAGS_DOT_ALL; - if (modifier.flags & STRING_GFLAGS_HEXADECIMAL) - modifier.flags |= STRING_GFLAGS_DOT_ALL; + // xor and nocase together is not implemented. + if (modifier.flags & STRING_FLAGS_XOR && + modifier.flags & STRING_FLAGS_NO_CASE) + { + result = ERROR_INVALID_MODIFIER; + yr_compiler_set_error_extra_info(compiler, "xor nocase") + goto _exit; + } - // The STRING_GFLAGS_SINGLE_MATCH flag indicates that finding + // base64 and nocase together is not implemented. + if (modifier.flags & STRING_FLAGS_NO_CASE && + (modifier.flags & STRING_FLAGS_BASE64 || + modifier.flags & STRING_FLAGS_BASE64_WIDE)) + { + result = ERROR_INVALID_MODIFIER; + yr_compiler_set_error_extra_info( + compiler, + modifier.flags & STRING_FLAGS_BASE64 ? + "base64 nocase" : + "base64wide nocase") + goto _exit; + } + + // base64 and xor together is not implemented. + if (modifier.flags & STRING_FLAGS_XOR && + (modifier.flags & STRING_FLAGS_BASE64 || + modifier.flags & STRING_FLAGS_BASE64_WIDE)) + { + result = ERROR_INVALID_MODIFIER; + yr_compiler_set_error_extra_info( + compiler, + modifier.flags & STRING_FLAGS_BASE64 ? + "base64 xor" : + "base64wide xor") + goto _exit; + } + + if (!(modifier.flags & STRING_FLAGS_WIDE) && + !(modifier.flags & STRING_FLAGS_XOR) && + !(modifier.flags & STRING_FLAGS_BASE64 || + modifier.flags & STRING_FLAGS_BASE64_WIDE)) + { + modifier.flags |= STRING_FLAGS_ASCII; + } + + // The STRING_FLAGS_SINGLE_MATCH flag indicates that finding // a single match for the string is enough. This is true in // most cases, except when the string count (#) and string offset (@) // operators are used. All strings are marked STRING_FLAGS_SINGLE_MATCH // initially, and unmarked later if required. + modifier.flags |= STRING_FLAGS_SINGLE_MATCH; - modifier.flags |= STRING_GFLAGS_SINGLE_MATCH; - - // The STRING_GFLAGS_FIXED_OFFSET indicates that the string doesn't + // The STRING_FLAGS_FIXED_OFFSET indicates that the string doesn't // need to be searched all over the file because the user is using the // "at" operator. The string must be searched at a fixed offset in the - // file. All strings are marked STRING_GFLAGS_FIXED_OFFSET initially, + // file. All strings are marked STRING_FLAGS_FIXED_OFFSET initially, // and unmarked later if required. + modifier.flags |= STRING_FLAGS_FIXED_OFFSET; - modifier.flags |= STRING_GFLAGS_FIXED_OFFSET; - if (modifier.flags & STRING_GFLAGS_HEXADECIMAL || - modifier.flags & STRING_GFLAGS_REGEXP) + if (modifier.flags & STRING_FLAGS_HEXADECIMAL || + modifier.flags & STRING_FLAGS_REGEXP || + modifier.flags & STRING_FLAGS_BASE64 || + modifier.flags & STRING_FLAGS_BASE64_WIDE) { - if (modifier.flags & STRING_GFLAGS_HEXADECIMAL) + if (modifier.flags & STRING_FLAGS_HEXADECIMAL) result = yr_re_parse_hex(str->c_string, &re_ast, &re_error); - else + else if (modifier.flags & STRING_FLAGS_REGEXP) result = yr_re_parse(str->c_string, &re_ast, &re_error); + else + result = yr_base64_ast_from_string(str, modifier, &re_ast, &re_error); if (result != ERROR_SUCCESS) { @@ -593,19 +668,22 @@ int yr_parser_reduce_string_declaration( message, sizeof(message), "invalid %s \"%s\": %s", - (modifier.flags & STRING_GFLAGS_HEXADECIMAL) ? + (modifier.flags & STRING_FLAGS_HEXADECIMAL) ? "hex string" : "regular expression", identifier, re_error.message); yr_compiler_set_error_extra_info( - compiler, message); + compiler, message) goto _exit; } if (re_ast->flags & RE_FLAGS_FAST_REGEXP) - modifier.flags |= STRING_GFLAGS_FAST_REGEXP; + modifier.flags |= STRING_FLAGS_FAST_REGEXP; + + if (re_ast->flags & RE_FLAGS_GREEDY) + modifier.flags |= STRING_FLAGS_GREEDY_REGEXP; // Regular expressions in the strings section can't mix greedy and ungreedy // quantifiers like .* and .*?. That's because these regular expressions can @@ -620,14 +698,11 @@ int yr_parser_reduce_string_declaration( yr_compiler_set_error_extra_info(compiler, "greedy and ungreedy quantifiers can't be mixed in a regular " - "expression"); + "expression") goto _exit; } - if (re_ast->flags & RE_FLAGS_GREEDY) - modifier.flags |= STRING_GFLAGS_GREEDY_REGEXP; - if (yr_re_ast_contains_dot_star(re_ast)) { yywarning( @@ -639,17 +714,22 @@ int yr_parser_reduce_string_declaration( if (compiler->re_ast_callback != NULL) { compiler->re_ast_callback( - compiler->current_rule, + current_rule, identifier, re_ast, compiler->re_ast_clbk_user_data); } + *string_ref = YR_ARENA_NULL_REF; + while (re_ast != NULL) { - prev_string = new_string; - prev_min_gap = min_gap; - prev_max_gap = max_gap; + YR_ARENA_REF ref; + + uint32_t prev_string_idx = compiler->current_string_idx - 1; + + int32_t prev_min_gap = min_gap; + int32_t prev_max_gap = max_gap; result = yr_re_ast_split_at_chaining_point( re_ast, &remainder_re_ast, &min_gap, &max_gap); @@ -663,9 +743,9 @@ int yr_parser_reduce_string_declaration( compiler, NULL, re_ast, - &new_string, + &ref, &atom_quality, - &compiler->current_rule->num_atoms); + ¤t_rule->num_atoms); if (result != ERROR_SUCCESS) goto _exit; @@ -673,39 +753,51 @@ int yr_parser_reduce_string_declaration( if (atom_quality < min_atom_quality) min_atom_quality = atom_quality; - if (prev_string != NULL) + if (YR_ARENA_IS_NULL_REF(*string_ref)) { + // This is the first string in the chain, the string reference returned + // by this function must point to this string. + *string_ref = ref; + } + else + { + // This is not the first string in the chain, set the appropriate flags + // and fill the chained_to, chain_gap_min and chain_gap_max fields. + YR_STRING* prev_string = (YR_STRING*) yr_arena_get_ptr( + compiler->arena, + YR_STRINGS_TABLE, + prev_string_idx * sizeof(YR_STRING)); + + YR_STRING* new_string = (YR_STRING*) yr_arena_ref_to_ptr( + compiler->arena, &ref); + new_string->chained_to = prev_string; new_string->chain_gap_min = prev_min_gap; new_string->chain_gap_max = prev_max_gap; // A string chained to another one can't have a fixed offset, only the // head of the string chain can have a fixed offset. - new_string->g_flags &= ~STRING_GFLAGS_FIXED_OFFSET; + new_string->flags &= ~STRING_FLAGS_FIXED_OFFSET; // There is a previous string, but that string wasn't marked as part of // a chain because we can't do that until knowing there will be another // string, let's flag it now the we know. - prev_string->g_flags |= STRING_GFLAGS_CHAIN_PART; + prev_string->flags |= STRING_FLAGS_CHAIN_PART; // There is a previous string, so this string is part of a chain, but // there will be no more strings because there are no more AST to split, // which means that this is the chain's tail. if (remainder_re_ast == NULL) - new_string->g_flags |= STRING_GFLAGS_CHAIN_PART | - STRING_GFLAGS_CHAIN_TAIL; + new_string->flags |= STRING_FLAGS_CHAIN_PART | + STRING_FLAGS_CHAIN_TAIL; } yr_re_ast_destroy(re_ast); re_ast = remainder_re_ast; } - - // Walk the chain of strings from the tail to the head, we want to return - // the string at the head of the chain. - while (new_string->chained_to != NULL) - new_string = new_string->chained_to; } - else // not a STRING_GFLAGS_HEXADECIMAL or STRING_GFLAGS_REGEXP + else // not a STRING_FLAGS_HEXADECIMAL or STRING_FLAGS_REGEXP or + // STRING_FLAGS_BASE64 or STRING_FLAGS_BASE64_WIDE { result = _yr_parser_write_string( identifier, @@ -713,21 +805,9 @@ int yr_parser_reduce_string_declaration( compiler, str, NULL, - &new_string, + string_ref, &min_atom_quality, - &compiler->current_rule->num_atoms); - - if (result != ERROR_SUCCESS) - goto _exit; - } - - if (!STRING_IS_ANONYMOUS(new_string)) - { - result = yr_hash_table_add( - compiler->strings_table, - identifier, - NULL, - new_string); + ¤t_rule->num_atoms); if (result != ERROR_SUCCESS) goto _exit; @@ -735,10 +815,7 @@ int yr_parser_reduce_string_declaration( if (min_atom_quality < compiler->atoms_config.quality_warning_threshold) { - yywarning( - yyscanner, - "%s is slowing down scanning", - new_string->identifier); + yywarning(yyscanner, "%s is slowing down scanning", identifier); } _exit: @@ -749,8 +826,6 @@ _exit: if (remainder_re_ast != NULL) yr_re_ast_destroy(remainder_re_ast); - *string = new_string; - return result; } @@ -759,77 +834,84 @@ int yr_parser_reduce_rule_declaration_phase_1( yyscan_t yyscanner, int32_t flags, const char* identifier, - YR_RULE** rule) + YR_ARENA_REF* rule_ref) { YR_FIXUP *fixup; - YR_INIT_RULE_ARGS *init_rule_args; YR_COMPILER* compiler = yyget_extra(yyscanner); - *rule = NULL; + // We are starting to parse a rule, set current_rule_idx accordingly. + compiler->current_rule_idx = compiler->next_rule_idx; - if (yr_hash_table_lookup( - compiler->rules_table, - identifier, - compiler->current_namespace->name) != NULL || + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, + YR_NAMESPACES_TABLE, + compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE)); + + if (yr_hash_table_lookup_uint32( + compiler->rules_table, + identifier, + ns->name) != UINT32_MAX || yr_hash_table_lookup( - compiler->objects_table, - identifier, - NULL) != NULL) + compiler->objects_table, + identifier, + NULL) != NULL) { // A rule or variable with the same identifier already exists, return the // appropriate error. - yr_compiler_set_error_extra_info(compiler, identifier); + yr_compiler_set_error_extra_info(compiler, identifier) return ERROR_DUPLICATED_IDENTIFIER; } FAIL_ON_ERROR(yr_arena_allocate_struct( - compiler->rules_arena, + compiler->arena, + YR_RULES_TABLE, sizeof(YR_RULE), - (void**) rule, + rule_ref, offsetof(YR_RULE, identifier), offsetof(YR_RULE, tags), offsetof(YR_RULE, strings), offsetof(YR_RULE, metas), offsetof(YR_RULE, ns), - EOL)) - - (*rule)->g_flags = flags; - (*rule)->ns = compiler->current_namespace; - (*rule)->num_atoms = 0; - - #ifdef PROFILING_ENABLED - (*rule)->time_cost = 0; - - memset( - (*rule)->time_cost_per_thread, 0, sizeof((*rule)->time_cost_per_thread)); - #endif - - FAIL_ON_ERROR(yr_arena_write_string( - compiler->sz_arena, - identifier, - (char**) &(*rule)->identifier)); - - FAIL_ON_ERROR(yr_parser_emit( - yyscanner, - OP_INIT_RULE, - NULL)); - - FAIL_ON_ERROR(yr_arena_allocate_struct( - compiler->code_arena, - sizeof(YR_INIT_RULE_ARGS), - (void**) &init_rule_args, - offsetof(YR_INIT_RULE_ARGS, rule), - offsetof(YR_INIT_RULE_ARGS, jmp_addr), EOL)); - init_rule_args->rule = *rule; + YR_RULE* rule = (YR_RULE*) yr_arena_ref_to_ptr(compiler->arena, rule_ref); - // jmp_addr holds the address to jump to when we want to skip the code for - // the rule. It is iniatialized as NULL at this point because we don't know - // the address until emmiting the code for the rule's condition. The address - // is set in yr_parser_reduce_rule_declaration_phase_2. - init_rule_args->jmp_addr = NULL; + YR_ARENA_REF ref; + + FAIL_ON_ERROR(yr_arena_write_string( + compiler->arena, + YR_SZ_POOL, + identifier, + &ref)); + + rule->identifier = (const char*) yr_arena_ref_to_ptr(compiler->arena, &ref); + rule->flags = flags; + rule->ns = ns; + rule->num_atoms = 0; + + YR_ARENA_REF jmp_offset_ref; + + // The OP_INIT_RULE instruction behaves like a jump. When the rule is disabled + // it skips over the rule's code and go straight to the next rule's code. The + // jmp_offset_ref variable points to the jump's offset. The offset is set to 0 + // as we don't know the jump target yet. When we finish generating the rule's + // code in yr_parser_reduce_rule_declaration_phase_2 the jump offset is set to + // its final value. + + FAIL_ON_ERROR(yr_parser_emit_with_arg_int32( + yyscanner, + OP_INIT_RULE, + 0, + NULL, + &jmp_offset_ref)); + + FAIL_ON_ERROR(yr_arena_write_data( + compiler->arena, + YR_CODE_SECTION, + &compiler->current_rule_idx, + sizeof(compiler->current_rule_idx), + NULL)); // Create a fixup entry for the jump and push it in the stack fixup = (YR_FIXUP*) yr_malloc(sizeof(YR_FIXUP)); @@ -837,33 +919,28 @@ int yr_parser_reduce_rule_declaration_phase_1( if (fixup == NULL) return ERROR_INSUFFICIENT_MEMORY; - fixup->address = (void*) &(init_rule_args->jmp_addr); + fixup->ref = jmp_offset_ref; fixup->next = compiler->fixup_stack_head; compiler->fixup_stack_head = fixup; // Clean strings_table as we are starting to parse a new rule. yr_hash_table_clean(compiler->strings_table, NULL); - FAIL_ON_ERROR(yr_hash_table_add( + FAIL_ON_ERROR(yr_hash_table_add_uint32( compiler->rules_table, identifier, - compiler->current_namespace->name, - (void*) *rule)); - - compiler->current_rule = *rule; + ns->name, + compiler->current_rule_idx)); return ERROR_SUCCESS; } int yr_parser_reduce_rule_declaration_phase_2( yyscan_t yyscanner, - YR_RULE* rule) + YR_ARENA_REF* rule_ref) { uint32_t max_strings_per_rule; uint32_t strings_in_rule = 0; - uint8_t* nop_inst_addr = NULL; - - int result; YR_FIXUP *fixup; YR_STRING* string; @@ -873,6 +950,8 @@ int yr_parser_reduce_rule_declaration_phase_2( YR_CONFIG_MAX_STRINGS_PER_RULE, (void*) &max_strings_per_rule); + YR_RULE* rule = (YR_RULE*) yr_arena_ref_to_ptr(compiler->arena, rule_ref); + // Show warning if the rule is generating too many atoms. The warning is // shown if the number of atoms is greater than 20 times the maximum number // of strings allowed for a rule, as 20 is minimum number of atoms generated @@ -883,10 +962,7 @@ int yr_parser_reduce_rule_declaration_phase_2( yywarning(yyscanner, "rule is slowing down scanning"); } - // Check for unreferenced (unused) strings. - string = rule->strings; - - while (!STRING_IS_NULL(string)) + yr_rule_strings_foreach(rule, string) { // Only the heading fragment in a chain of strings (the one with // chained_to == NULL) must be referenced. All other fragments @@ -895,7 +971,7 @@ int yr_parser_reduce_rule_declaration_phase_2( if (!STRING_IS_REFERENCED(string) && string->chained_to == NULL) { - yr_compiler_set_error_extra_info(compiler, string->identifier); + yr_compiler_set_error_extra_info(compiler, string->identifier) return ERROR_UNREFERENCED_STRING; } @@ -903,44 +979,40 @@ int yr_parser_reduce_rule_declaration_phase_2( if (strings_in_rule > max_strings_per_rule) { - yr_compiler_set_error_extra_info(compiler, rule->identifier); + yr_compiler_set_error_extra_info(compiler, rule->identifier) return ERROR_TOO_MANY_STRINGS; } - - string = (YR_STRING*) yr_arena_next_address( - compiler->strings_arena, - string, - sizeof(YR_STRING)); } - result = yr_parser_emit_with_arg_reloc( + FAIL_ON_ERROR(yr_parser_emit_with_arg( yyscanner, OP_MATCH_RULE, - rule, + compiler->current_rule_idx, NULL, - NULL); - - // Generate a do-nothing instruction (NOP) in order to get its address - // and use it as the destination for the OP_INIT_RULE skip jump. We can not - // simply use the address of the OP_MATCH_RULE instruction +1 because we - // can't be sure that the instruction following the OP_MATCH_RULE is going to - // be in the same arena page. As we don't have a reliable way of getting the - // address of the next instruction we generate the OP_NOP. - - if (result == ERROR_SUCCESS) - result = yr_parser_emit(yyscanner, OP_NOP, &nop_inst_addr); + NULL)); fixup = compiler->fixup_stack_head; - *(void**)(fixup->address) = (void*) nop_inst_addr; + + int32_t* jmp_offset_addr = (int32_t*) yr_arena_ref_to_ptr( + compiler->arena, &fixup->ref); + + int32_t jmp_offset = \ + yr_arena_get_current_offset(compiler->arena, YR_CODE_SECTION) - + fixup->ref.offset + 1; + + *jmp_offset_addr = jmp_offset; + + // Remove fixup from the stack. compiler->fixup_stack_head = fixup->next; yr_free(fixup); - // We are leaving the scope of the current rule, let's assign NULL to - // current_rule. It will remain NULL until the next call to - // yr_parser_reduce_rule_declaration_phase_1. - compiler->current_rule = NULL; + // We have finished parsing the current rule, increment next_rule_idx + // and set current_rule_idx to UINT32_MAX indicating that we are not + // currently parsing a rule. + compiler->next_rule_idx++; + compiler->current_rule_idx = UINT32_MAX; - return result; + return ERROR_SUCCESS; } @@ -966,36 +1038,30 @@ int yr_parser_reduce_string_identifier( yr_parser_emit(yyscanner, instruction, NULL); - string = compiler->current_rule->strings; + YR_RULE* current_rule = _yr_compiler_get_rule_by_idx( + compiler, compiler->current_rule_idx); - while(!STRING_IS_NULL(string)) + yr_rule_strings_foreach(current_rule, string) { if (instruction != OP_FOUND) - string->g_flags &= ~STRING_GFLAGS_SINGLE_MATCH; + string->flags &= ~STRING_FLAGS_SINGLE_MATCH; if (instruction == OP_FOUND_AT) { // Avoid overwriting any previous fixed offset - - if (string->fixed_offset == UNDEFINED) + if (string->fixed_offset == YR_UNDEFINED) string->fixed_offset = at_offset; // If a previous fixed offset was different, disable // the STRING_GFLAGS_FIXED_OFFSET flag because we only // have room to store a single fixed offset value - if (string->fixed_offset != at_offset) - string->g_flags &= ~STRING_GFLAGS_FIXED_OFFSET; + string->flags &= ~STRING_FLAGS_FIXED_OFFSET; } else { - string->g_flags &= ~STRING_GFLAGS_FIXED_OFFSET; + string->flags &= ~STRING_FLAGS_FIXED_OFFSET; } - - string = (YR_STRING*) yr_arena_next_address( - compiler->strings_arena, - string, - sizeof(YR_STRING)); } } else @@ -1017,33 +1083,33 @@ int yr_parser_reduce_string_identifier( NULL)); if (instruction != OP_FOUND) - string->g_flags &= ~STRING_GFLAGS_SINGLE_MATCH; + string->flags &= ~STRING_FLAGS_SINGLE_MATCH; if (instruction == OP_FOUND_AT) { // Avoid overwriting any previous fixed offset - if (string->fixed_offset == UNDEFINED) + if (string->fixed_offset == YR_UNDEFINED) string->fixed_offset = at_offset; // If a previous fixed offset was different, disable // the STRING_GFLAGS_FIXED_OFFSET flag because we only // have room to store a single fixed offset value - if (string->fixed_offset == UNDEFINED || + if (string->fixed_offset == YR_UNDEFINED || string->fixed_offset != at_offset) { - string->g_flags &= ~STRING_GFLAGS_FIXED_OFFSET; + string->flags &= ~STRING_FLAGS_FIXED_OFFSET; } } else { - string->g_flags &= ~STRING_GFLAGS_FIXED_OFFSET; + string->flags &= ~STRING_FLAGS_FIXED_OFFSET; } FAIL_ON_ERROR(yr_parser_emit(yyscanner, instruction, NULL)); - string->g_flags |= STRING_GFLAGS_REFERENCED; + string->flags |= STRING_FLAGS_REFERENCED; } return ERROR_SUCCESS; @@ -1056,38 +1122,48 @@ int yr_parser_reduce_meta_declaration( const char* identifier, const char* string, int64_t integer, - YR_META** meta) + YR_ARENA_REF* meta_ref) { + YR_ARENA_REF ref; YR_COMPILER* compiler = yyget_extra(yyscanner); FAIL_ON_ERROR(yr_arena_allocate_struct( - compiler->metas_arena, + compiler->arena, + YR_METAS_TABLE, sizeof(YR_META), - (void**) meta, + meta_ref, offsetof(YR_META, identifier), offsetof(YR_META, string), EOL)); + YR_META* meta = (YR_META*) yr_arena_ref_to_ptr(compiler->arena, meta_ref); + + meta->type = type; + meta->integer = integer; + FAIL_ON_ERROR(yr_arena_write_string( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, identifier, - (char**) &(*meta)->identifier)); + &ref)); + + meta->identifier = (const char*) yr_arena_ref_to_ptr(compiler->arena, &ref); if (string != NULL) { FAIL_ON_ERROR(yr_arena_write_string( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, string, - &(*meta)->string)); + &ref)); + + meta->string = (const char*) yr_arena_ref_to_ptr(compiler->arena, &ref); } else { - (*meta)->string = NULL; + meta->string = NULL; } - (*meta)->integer = integer; - (*meta)->type = type; - return ERROR_SUCCESS; } @@ -1111,21 +1187,25 @@ int yr_parser_reduce_import( { int result; + YR_ARENA_REF ref; YR_COMPILER* compiler = yyget_extra(yyscanner); YR_OBJECT* module_structure; - char* name; - if (!_yr_parser_valid_module_name(module_name)) { - yr_compiler_set_error_extra_info(compiler, module_name->c_string); + yr_compiler_set_error_extra_info(compiler, module_name->c_string) return ERROR_INVALID_MODULE_NAME; } + YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr( + compiler->arena, + YR_NAMESPACES_TABLE, + compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE)); + module_structure = (YR_OBJECT*) yr_hash_table_lookup( compiler->objects_table, module_name->c_string, - compiler->current_namespace->name); + ns->name); // if module already imported, do nothing @@ -1141,7 +1221,7 @@ int yr_parser_reduce_import( FAIL_ON_ERROR(yr_hash_table_add( compiler->objects_table, module_name->c_string, - compiler->current_namespace->name, + ns->name, module_structure)); result = yr_modules_do_declarations( @@ -1149,22 +1229,23 @@ int yr_parser_reduce_import( module_structure); if (result == ERROR_UNKNOWN_MODULE) - yr_compiler_set_error_extra_info(compiler, module_name->c_string); + yr_compiler_set_error_extra_info(compiler, module_name->c_string) if (result != ERROR_SUCCESS) return result; FAIL_ON_ERROR(yr_arena_write_string( - compiler->sz_arena, + compiler->arena, + YR_SZ_POOL, module_name->c_string, - &name)); + &ref)); FAIL_ON_ERROR(yr_parser_emit_with_arg_reloc( - yyscanner, - OP_IMPORT, - name, - NULL, - NULL)); + yyscanner, + OP_IMPORT, + yr_arena_ref_to_ptr(compiler->arena, &ref), + NULL, + NULL)); return ERROR_SUCCESS; } @@ -1294,14 +1375,14 @@ int yr_parser_reduce_operation( else { yr_compiler_set_error_extra_info_fmt( - compiler, "strings don't support \"%s\" operation", op); + compiler, "strings don't support \"%s\" operation", op) return ERROR_WRONG_TYPE; } } else { - yr_compiler_set_error_extra_info(compiler, "type mismatch"); + yr_compiler_set_error_extra_info(compiler, "type mismatch") return ERROR_WRONG_TYPE; } diff --git a/libyara/re.c b/libyara/re.c index 0015f667..a4c08e31 100644 --- a/libyara/re.c +++ b/libyara/re.c @@ -41,6 +41,7 @@ order to avoid confusion with operating system threads. #include #include +#include #include #include #include @@ -81,9 +82,9 @@ typedef struct _RE_REPEAT_ANY_ARGS } RE_REPEAT_ANY_ARGS; -typedef struct _RE_EMIT_CONTEXT { - - YR_ARENA* arena; +typedef struct _RE_EMIT_CONTEXT +{ + YR_ARENA* arena; RE_SPLIT_ID_TYPE next_split_id; } RE_EMIT_CONTEXT; @@ -136,8 +137,8 @@ RE_NODE* yr_re_node_create( result->prev_sibling = NULL; result->next_sibling = NULL; result->greedy = true; - result->forward_code = NULL; - result->backward_code = NULL; + result->forward_code_ref = YR_ARENA_NULL_REF; + result->backward_code_ref = YR_ARENA_NULL_REF; } return result; @@ -234,7 +235,7 @@ void yr_re_ast_destroy( // yr_re_parse // // Parses a regexp but don't emit its code. A further call to -// yr_re_emit_code is required to get the code. +// yr_re_ast_emit_code is required to get the code. // int yr_re_parse( @@ -250,7 +251,7 @@ int yr_re_parse( // yr_re_parse_hex // // Parses a hex string but don't emit its code. A further call to -// yr_re_emit_code is required to get the code. +// yr_re_ast_emit_code is required to get the code. // int yr_re_parse_hex( @@ -265,36 +266,35 @@ int yr_re_parse_hex( // // yr_re_compile // -// Parses the regexp and emit its code to the provided code_arena. +// Parses the regexp and emit its code to the provided to the +// YR_RE_CODE_SECTION in the specified arena. // int yr_re_compile( const char* re_string, int flags, - YR_ARENA* code_arena, - RE** re, + YR_ARENA* arena, + YR_ARENA_REF* ref, RE_ERROR* error) { RE_AST* re_ast; RE _re; - FAIL_ON_ERROR(yr_arena_reserve_memory( - code_arena, sizeof(int64_t) + RE_MAX_CODE_SIZE)); - FAIL_ON_ERROR(yr_re_parse(re_string, &re_ast, error)); _re.flags = flags; FAIL_ON_ERROR_WITH_CLEANUP( yr_arena_write_data( - code_arena, + arena, + YR_RE_CODE_SECTION, &_re, sizeof(_re), - (void**) re), + ref), yr_re_ast_destroy(re_ast)); FAIL_ON_ERROR_WITH_CLEANUP( - yr_re_ast_emit_code(re_ast, code_arena, false), + yr_re_ast_emit_code(re_ast, arena, false), yr_re_ast_destroy(re_ast)); yr_re_ast_destroy(re_ast); @@ -386,6 +386,7 @@ SIZED_STRING* yr_re_ast_extract_literal( return NULL; string->length = length; + string->flags = 0; if (re_ast->root_node->type == RE_NODE_LITERAL) { @@ -549,9 +550,6 @@ int yr_re_ast_split_at_chaining_point( *min_gap = child->start; *max_gap = child->end; - re_ast->root_node = re_ast->root_node; - re_ast->flags = re_ast->flags; - (*remainder_re_ast)->root_node = concat; (*remainder_re_ast)->flags = re_ast->flags; @@ -570,16 +568,14 @@ int yr_re_ast_split_at_chaining_point( int _yr_emit_inst( RE_EMIT_CONTEXT* emit_context, uint8_t opcode, - uint8_t** instruction_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref) { FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); - - *code_size = sizeof(uint8_t); + instruction_ref)); return ERROR_SUCCESS; } @@ -589,23 +585,22 @@ int _yr_emit_inst_arg_uint8( RE_EMIT_CONTEXT* emit_context, uint8_t opcode, uint8_t argument, - uint8_t** instruction_addr, - uint8_t** argument_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); + instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &argument, sizeof(uint8_t), - (void**) argument_addr)); - - *code_size = 2 * sizeof(uint8_t); + argument_ref)); return ERROR_SUCCESS; } @@ -615,23 +610,22 @@ int _yr_emit_inst_arg_uint16( RE_EMIT_CONTEXT* emit_context, uint8_t opcode, uint16_t argument, - uint8_t** instruction_addr, - uint16_t** argument_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); + instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &argument, sizeof(uint16_t), - (void**) argument_addr)); - - *code_size = sizeof(uint8_t) + sizeof(uint16_t); + argument_ref)); return ERROR_SUCCESS; } @@ -641,23 +635,22 @@ int _yr_emit_inst_arg_uint32( RE_EMIT_CONTEXT* emit_context, uint8_t opcode, uint32_t argument, - uint8_t** instruction_addr, - uint32_t** argument_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); + instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &argument, sizeof(uint32_t), - (void**) argument_addr)); - - *code_size = sizeof(uint8_t) + sizeof(uint32_t); + argument_ref)); return ERROR_SUCCESS; } @@ -667,23 +660,22 @@ int _yr_emit_inst_arg_int16( RE_EMIT_CONTEXT* emit_context, uint8_t opcode, int16_t argument, - uint8_t** instruction_addr, - int16_t** argument_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); + instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &argument, sizeof(int16_t), - (void**) argument_addr)); - - *code_size = sizeof(uint8_t) + sizeof(int16_t); + argument_ref)); return ERROR_SUCCESS; } @@ -694,23 +686,22 @@ int _yr_emit_inst_arg_struct( uint8_t opcode, void* structure, size_t structure_size, - uint8_t** instruction_addr, - void** argument_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); + instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, structure, structure_size, - (void**) argument_addr)); - - *code_size = sizeof(uint8_t) + structure_size; + argument_ref)); return ERROR_SUCCESS; } @@ -720,9 +711,8 @@ int _yr_emit_split( RE_EMIT_CONTEXT* emit_context, uint8_t opcode, int16_t argument, - uint8_t** instruction_addr, - int16_t** argument_addr, - size_t* code_size) + YR_ARENA_REF* instruction_ref, + YR_ARENA_REF* argument_ref) { assert(opcode == RE_OPCODE_SPLIT_A || opcode == RE_OPCODE_SPLIT_B); @@ -731,12 +721,14 @@ int _yr_emit_split( FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &opcode, sizeof(uint8_t), - (void**) instruction_addr)); + instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &emit_context->next_split_id, sizeof(RE_SPLIT_ID_TYPE), NULL)); @@ -745,27 +737,30 @@ int _yr_emit_split( FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, &argument, sizeof(int16_t), - (void**) argument_addr)); - - *code_size = sizeof(uint8_t) + sizeof(RE_SPLIT_ID_TYPE) + sizeof(int16_t); + argument_ref)); return ERROR_SUCCESS; } +#define current_re_code_offset() \ + yr_arena_get_current_offset(emit_context->arena, YR_RE_CODE_SECTION) + static int _yr_re_emit( RE_EMIT_CONTEXT* emit_context, RE_NODE* re_node, int flags, - uint8_t** code_addr, - size_t* code_size) + YR_ARENA_REF* code_ref) { - size_t branch_size; - size_t split_size; - size_t inst_size; - size_t jmp_size; + yr_arena_off_t jmp_offset; + + yr_arena_off_t bookmark_1 = 0; + yr_arena_off_t bookmark_2 = 0; + yr_arena_off_t bookmark_3 = 0; + yr_arena_off_t bookmark_4 = 0; bool emit_split; bool emit_repeat; @@ -780,162 +775,132 @@ static int _yr_re_emit( int16_t* split_offset_addr = NULL; int16_t* jmp_offset_addr = NULL; - uint8_t* instruction_addr = NULL; - *code_size = 0; + YR_ARENA_REF instruction_ref; + YR_ARENA_REF split_offset_ref; + YR_ARENA_REF jmp_instruction_ref; + YR_ARENA_REF jmp_offset_ref; + YR_ARENA_REF repeat_start_args_ref; switch(re_node->type) { case RE_NODE_LITERAL: - FAIL_ON_ERROR(_yr_emit_inst_arg_uint8( emit_context, RE_OPCODE_LITERAL, re_node->value, - &instruction_addr, - NULL, - code_size)); + &instruction_ref, + NULL)); break; case RE_NODE_MASKED_LITERAL: - FAIL_ON_ERROR(_yr_emit_inst_arg_uint16( emit_context, RE_OPCODE_MASKED_LITERAL, re_node->mask << 8 | re_node->value, - &instruction_addr, - NULL, - code_size)); + &instruction_ref, + NULL)); break; case RE_NODE_WORD_CHAR: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_WORD_CHAR, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_NON_WORD_CHAR: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_NON_WORD_CHAR, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_WORD_BOUNDARY: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_WORD_BOUNDARY, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_NON_WORD_BOUNDARY: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_NON_WORD_BOUNDARY, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_SPACE: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_SPACE, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_NON_SPACE: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_NON_SPACE, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_DIGIT: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_DIGIT, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_NON_DIGIT: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_NON_DIGIT, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_ANY: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_ANY, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_CLASS: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_CLASS, - &instruction_addr, - code_size)); + &instruction_ref)); FAIL_ON_ERROR(yr_arena_write_data( emit_context->arena, + YR_RE_CODE_SECTION, re_node->re_class, sizeof(*re_node->re_class), NULL)); - - *code_size += sizeof(*re_node->re_class); break; case RE_NODE_ANCHOR_START: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_MATCH_AT_START, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_ANCHOR_END: - FAIL_ON_ERROR(_yr_emit_inst( emit_context, RE_OPCODE_MATCH_AT_END, - &instruction_addr, - code_size)); + &instruction_ref)); break; case RE_NODE_CONCAT: - FAIL_ON_ERROR(_yr_re_emit( emit_context, (flags & EMIT_BACKWARDS)? re_node->children_tail: re_node->children_head, flags, - &instruction_addr, - &branch_size)); - - *code_size += branch_size; + &instruction_ref)); if (flags & EMIT_BACKWARDS) child = re_node->children_tail->prev_sibling; @@ -948,95 +913,88 @@ static int _yr_re_emit( emit_context, child, flags, - NULL, - &branch_size)); - - *code_size += branch_size; + NULL)); child = (flags & EMIT_BACKWARDS) ? child->prev_sibling: child->next_sibling; } - break; case RE_NODE_PLUS: - // Code for e+ looks like: // // L1: code for e // split L1, L2 // L2: - + // FAIL_ON_ERROR(_yr_re_emit( emit_context, re_node->children_head, flags, - &instruction_addr, - &branch_size)); + &instruction_ref)); - *code_size += branch_size; + jmp_offset = instruction_ref.offset - current_re_code_offset(); + + if (jmp_offset < INT16_MIN) + return ERROR_REGULAR_EXPRESSION_TOO_LARGE; FAIL_ON_ERROR(_yr_emit_split( emit_context, re_node->greedy ? RE_OPCODE_SPLIT_B : RE_OPCODE_SPLIT_A, - -((int16_t) branch_size), + (int16_t) jmp_offset, NULL, - &split_offset_addr, - &split_size)); + NULL)); - *code_size += split_size; break; case RE_NODE_STAR: - // Code for e* looks like: // // L1: split L1, L2 // code for e // jmp L1 // L2: - FAIL_ON_ERROR(_yr_emit_split( emit_context, re_node->greedy ? RE_OPCODE_SPLIT_A : RE_OPCODE_SPLIT_B, 0, - &instruction_addr, - &split_offset_addr, - &split_size)); - - *code_size += split_size; + &instruction_ref, + &split_offset_ref)); FAIL_ON_ERROR(_yr_re_emit( emit_context, re_node->children_head, flags, - NULL, - &branch_size)); + NULL)); - *code_size += branch_size; + jmp_offset = instruction_ref.offset - current_re_code_offset(); + + if (jmp_offset < INT16_MIN) + return ERROR_REGULAR_EXPRESSION_TOO_LARGE; // Emit jump with offset set to 0. FAIL_ON_ERROR(_yr_emit_inst_arg_int16( emit_context, RE_OPCODE_JUMP, - -((uint16_t)(branch_size + split_size)), + (int16_t) jmp_offset, NULL, - &jmp_offset_addr, - &jmp_size)); + NULL)); - *code_size += jmp_size; + jmp_offset = current_re_code_offset() - instruction_ref.offset; - if (split_size + branch_size + jmp_size >= INT16_MAX) + if (jmp_offset > INT16_MAX) return ERROR_REGULAR_EXPRESSION_TOO_LARGE; // Update split offset. - *split_offset_addr = (int16_t) (split_size + branch_size + jmp_size); + split_offset_addr = (int16_t*) yr_arena_ref_to_ptr( + emit_context->arena, &split_offset_ref); + + *split_offset_addr = (int16_t) jmp_offset; break; case RE_NODE_ALT: - // Code for e1|e2 looks like: // // split L1, L2 @@ -1053,20 +1011,14 @@ static int _yr_re_emit( emit_context, RE_OPCODE_SPLIT_A, 0, - &instruction_addr, - &split_offset_addr, - &split_size)); - - *code_size += split_size; + &instruction_ref, + &split_offset_ref)); FAIL_ON_ERROR(_yr_re_emit( emit_context, re_node->children_head, flags, - NULL, - &branch_size)); - - *code_size += branch_size; + NULL)); // Emit jump with offset set to 0. @@ -1074,36 +1026,39 @@ static int _yr_re_emit( emit_context, RE_OPCODE_JUMP, 0, - NULL, - &jmp_offset_addr, - &jmp_size)); + &jmp_instruction_ref, + &jmp_offset_ref)); - *code_size += jmp_size; + jmp_offset = current_re_code_offset() - instruction_ref.offset; - if (split_size + branch_size + jmp_size >= INT16_MAX) + if (jmp_offset > INT16_MAX) return ERROR_REGULAR_EXPRESSION_TOO_LARGE; // Update split offset. - *split_offset_addr = (int16_t) (split_size + branch_size + jmp_size); + split_offset_addr = (int16_t*) yr_arena_ref_to_ptr( + emit_context->arena, &split_offset_ref); + + *split_offset_addr = (int16_t) jmp_offset; FAIL_ON_ERROR(_yr_re_emit( emit_context, re_node->children_tail, flags, - NULL, - &branch_size)); + NULL)); - *code_size += branch_size; + jmp_offset = current_re_code_offset() - jmp_instruction_ref.offset; - if (branch_size + jmp_size >= INT16_MAX) + if (jmp_offset > INT16_MAX) return ERROR_REGULAR_EXPRESSION_TOO_LARGE; // Update offset for jmp instruction. - *jmp_offset_addr = (int16_t) (branch_size + jmp_size); + jmp_offset_addr = (int16_t*) yr_arena_ref_to_ptr( + emit_context->arena, &jmp_offset_ref); + + *jmp_offset_addr = (int16_t) jmp_offset; break; case RE_NODE_RANGE_ANY: - repeat_any_args.min = re_node->start; repeat_any_args.max = re_node->end; @@ -1114,15 +1069,12 @@ static int _yr_re_emit( RE_OPCODE_REPEAT_ANY_UNGREEDY, &repeat_any_args, sizeof(repeat_any_args), - &instruction_addr, - NULL, - &inst_size)); + &instruction_ref, + NULL)); - *code_size += inst_size; break; case RE_NODE_RANGE: - // Code for e{n,m} looks like: // // code for e --- prolog @@ -1182,10 +1134,7 @@ static int _yr_re_emit( emit_context, re_node->children_head, flags, - &instruction_addr, - &branch_size)); - - *code_size += branch_size; + &instruction_ref)); } if (emit_repeat) @@ -1211,6 +1160,8 @@ static int _yr_re_emit( repeat_args.offset = 0; + bookmark_1 = current_re_code_offset(); + FAIL_ON_ERROR(_yr_emit_inst_arg_struct( emit_context, re_node->greedy ? @@ -1218,23 +1169,23 @@ static int _yr_re_emit( RE_OPCODE_REPEAT_START_UNGREEDY, &repeat_args, sizeof(repeat_args), - emit_prolog ? NULL : &instruction_addr, - (void**) &repeat_start_args_addr, - &inst_size)); + emit_prolog ? NULL : &instruction_ref, + &repeat_start_args_ref)); - *code_size += inst_size; + bookmark_2 = current_re_code_offset(); FAIL_ON_ERROR(_yr_re_emit( emit_context, re_node->children_head, flags | EMIT_DONT_SET_FORWARDS_CODE | EMIT_DONT_SET_BACKWARDS_CODE, - NULL, - &branch_size)); + NULL)); - *code_size += branch_size; + bookmark_3 = current_re_code_offset(); - repeat_start_args_addr->offset = (int32_t)(2 * inst_size + branch_size); - repeat_args.offset = -((int32_t) branch_size); + if (bookmark_2 - bookmark_3 < INT32_MIN) + return ERROR_REGULAR_EXPRESSION_TOO_LARGE; + + repeat_args.offset = (int32_t)(bookmark_2 - bookmark_3); FAIL_ON_ERROR(_yr_emit_inst_arg_struct( emit_context, @@ -1244,14 +1195,23 @@ static int _yr_re_emit( &repeat_args, sizeof(repeat_args), NULL, - NULL, - &inst_size)); + NULL)); - *code_size += inst_size; + bookmark_4 = current_re_code_offset(); + + repeat_start_args_addr = (RE_REPEAT_ARGS*) yr_arena_ref_to_ptr( + emit_context->arena, &repeat_start_args_ref); + + if (bookmark_4 - bookmark_1 > INT32_MAX) + return ERROR_REGULAR_EXPRESSION_TOO_LARGE; + + repeat_start_args_addr->offset = (int32_t)(bookmark_4 - bookmark_1); } if (emit_split) { + bookmark_1 = current_re_code_offset(); + FAIL_ON_ERROR(_yr_emit_split( emit_context, re_node->greedy ? @@ -1259,10 +1219,7 @@ static int _yr_re_emit( RE_OPCODE_SPLIT_B, 0, NULL, - &split_offset_addr, - &split_size)); - - *code_size += split_size; + &split_offset_ref)); } if (emit_epilog) @@ -1271,18 +1228,20 @@ static int _yr_re_emit( emit_context, re_node->children_head, emit_prolog ? flags | EMIT_DONT_SET_FORWARDS_CODE : flags, - emit_prolog || emit_repeat ? NULL : &instruction_addr, - &branch_size)); - - *code_size += branch_size; + emit_prolog || emit_repeat ? NULL : &instruction_ref)); } if (emit_split) { - if (split_size + branch_size >= INT16_MAX) + bookmark_2 = current_re_code_offset(); + + if (bookmark_2 - bookmark_1 > INT16_MAX) return ERROR_REGULAR_EXPRESSION_TOO_LARGE; - *split_offset_addr = (int16_t) (split_size + branch_size); + split_offset_addr = (int16_t*) yr_arena_ref_to_ptr( + emit_context->arena, &split_offset_ref); + + *split_offset_addr = (int16_t) (bookmark_2 - bookmark_1); } break; @@ -1291,16 +1250,22 @@ static int _yr_re_emit( if (flags & EMIT_BACKWARDS) { if (!(flags & EMIT_DONT_SET_BACKWARDS_CODE)) - re_node->backward_code = instruction_addr + *code_size; + { + re_node->backward_code_ref.buffer_id = YR_RE_CODE_SECTION; + re_node->backward_code_ref.offset = yr_arena_get_current_offset( + emit_context->arena, YR_RE_CODE_SECTION); + } } else { if (!(flags & EMIT_DONT_SET_FORWARDS_CODE)) - re_node->forward_code = instruction_addr; + { + re_node->forward_code_ref = instruction_ref; + } } - if (code_addr != NULL) - *code_addr = instruction_addr; + if (code_ref != NULL) + *code_ref = instruction_ref; return ERROR_SUCCESS; } @@ -1313,18 +1278,7 @@ int yr_re_ast_emit_code( { RE_EMIT_CONTEXT emit_context; - size_t code_size; - size_t total_size; - - // Ensure that we have enough contiguous memory space in the arena to - // contain the regular expression code. The code can't span over multiple - // non-contiguous pages. - - FAIL_ON_ERROR(yr_arena_reserve_memory(arena, RE_MAX_CODE_SIZE)); - // Emit code for matching the regular expressions forwards. - - total_size = 0; emit_context.arena = arena; emit_context.next_split_id = 0; @@ -1332,21 +1286,12 @@ int yr_re_ast_emit_code( &emit_context, re_ast->root_node, backwards_code ? EMIT_BACKWARDS : 0, - NULL, - &code_size)); - - total_size += code_size; + NULL)); FAIL_ON_ERROR(_yr_emit_inst( &emit_context, RE_OPCODE_MATCH, - NULL, - &code_size)); - - total_size += code_size; - - if (total_size > RE_MAX_CODE_SIZE) - return ERROR_REGULAR_EXPRESSION_TOO_LARGE; + NULL)); return ERROR_SUCCESS; } @@ -1832,7 +1777,7 @@ static int _yr_re_fiber_sync( // match the data starting at the address specified by "input". The "input" // pointer can point to any address inside a memory buffer. Arguments // "input_forwards_size" and "input_backwards_size" indicate how many bytes -// can be accesible starting at "input" and going forwards and backwards +// can be accessible starting at "input" and going forwards and backwards // respectively. // // <--- input_backwards_size -->|<----------- input_forwards_size --------> @@ -2395,7 +2340,8 @@ int yr_re_fast_exec( static void _yr_re_print_node( - RE_NODE* re_node) + RE_NODE* re_node, + uint32_t indent) { RE_NODE* child; int i; @@ -2403,14 +2349,16 @@ static void _yr_re_print_node( if (re_node == NULL) return; + if (indent > 0) + printf("\n%*s", indent, " "); switch (re_node->type) { case RE_NODE_ALT: printf("Alt("); - _yr_re_print_node(re_node->children_head); - printf(", "); - _yr_re_print_node(re_node->children_tail); - printf(")"); + _yr_re_print_node(re_node->children_head, indent + 4); + printf(","); + _yr_re_print_node(re_node->children_tail, indent + 4); + printf("\n%*s%s", indent, " " , ")"); break; case RE_NODE_CONCAT: @@ -2418,27 +2366,27 @@ static void _yr_re_print_node( child = re_node->children_head; while (child != NULL) { - _yr_re_print_node(child); - printf(", "); + _yr_re_print_node(child, indent + 4); + printf(","); child = child->next_sibling; } - printf(")"); + printf("\n%*s%s", indent, " ", ")"); break; case RE_NODE_STAR: printf("Star("); - _yr_re_print_node(re_node->children_head); + _yr_re_print_node(re_node->children_head, indent + 4); printf(")"); break; case RE_NODE_PLUS: printf("Plus("); - _yr_re_print_node(re_node->children_head); + _yr_re_print_node(re_node->children_head, indent + 4); printf(")"); break; case RE_NODE_LITERAL: - printf("Lit(%02X)", re_node->value); + printf("Lit(%c)", re_node->value); break; case RE_NODE_MASKED_LITERAL: @@ -2475,8 +2423,8 @@ static void _yr_re_print_node( case RE_NODE_RANGE: printf("Range(%d-%d, ", re_node->start, re_node->end); - _yr_re_print_node(re_node->children_head); - printf(")"); + _yr_re_print_node(re_node->children_head, indent + 4); + printf("\n%*s%s", indent, " ", ")"); break; case RE_NODE_CLASS: @@ -2496,5 +2444,5 @@ static void _yr_re_print_node( void yr_re_print( RE_AST* re_ast) { - _yr_re_print_node(re_ast->root_node); + _yr_re_print_node(re_ast->root_node, 0); } diff --git a/libyara/re_lexer.c b/libyara/re_lexer.c index be9d4f7c..bc37d9a8 100644 --- a/libyara/re_lexer.c +++ b/libyara/re_lexer.c @@ -1,6 +1,6 @@ -#line 1 "re_lexer.c" +#line 2 "re_lexer.c" -#line 3 "re_lexer.c" +#line 4 "re_lexer.c" #define YY_INT_ALIGNED short int @@ -9,233 +9,11 @@ #define FLEX_SCANNER #define YY_FLEX_MAJOR_VERSION 2 #define YY_FLEX_MINOR_VERSION 6 -#define YY_FLEX_SUBMINOR_VERSION 4 +#define YY_FLEX_SUBMINOR_VERSION 0 #if YY_FLEX_SUBMINOR_VERSION > 0 #define FLEX_BETA #endif -#ifdef yy_create_buffer -#define re_yy_create_buffer_ALREADY_DEFINED -#else -#define yy_create_buffer re_yy_create_buffer -#endif - -#ifdef yy_delete_buffer -#define re_yy_delete_buffer_ALREADY_DEFINED -#else -#define yy_delete_buffer re_yy_delete_buffer -#endif - -#ifdef yy_scan_buffer -#define re_yy_scan_buffer_ALREADY_DEFINED -#else -#define yy_scan_buffer re_yy_scan_buffer -#endif - -#ifdef yy_scan_string -#define re_yy_scan_string_ALREADY_DEFINED -#else -#define yy_scan_string re_yy_scan_string -#endif - -#ifdef yy_scan_bytes -#define re_yy_scan_bytes_ALREADY_DEFINED -#else -#define yy_scan_bytes re_yy_scan_bytes -#endif - -#ifdef yy_init_buffer -#define re_yy_init_buffer_ALREADY_DEFINED -#else -#define yy_init_buffer re_yy_init_buffer -#endif - -#ifdef yy_flush_buffer -#define re_yy_flush_buffer_ALREADY_DEFINED -#else -#define yy_flush_buffer re_yy_flush_buffer -#endif - -#ifdef yy_load_buffer_state -#define re_yy_load_buffer_state_ALREADY_DEFINED -#else -#define yy_load_buffer_state re_yy_load_buffer_state -#endif - -#ifdef yy_switch_to_buffer -#define re_yy_switch_to_buffer_ALREADY_DEFINED -#else -#define yy_switch_to_buffer re_yy_switch_to_buffer -#endif - -#ifdef yypush_buffer_state -#define re_yypush_buffer_state_ALREADY_DEFINED -#else -#define yypush_buffer_state re_yypush_buffer_state -#endif - -#ifdef yypop_buffer_state -#define re_yypop_buffer_state_ALREADY_DEFINED -#else -#define yypop_buffer_state re_yypop_buffer_state -#endif - -#ifdef yyensure_buffer_stack -#define re_yyensure_buffer_stack_ALREADY_DEFINED -#else -#define yyensure_buffer_stack re_yyensure_buffer_stack -#endif - -#ifdef yylex -#define re_yylex_ALREADY_DEFINED -#else -#define yylex re_yylex -#endif - -#ifdef yyrestart -#define re_yyrestart_ALREADY_DEFINED -#else -#define yyrestart re_yyrestart -#endif - -#ifdef yylex_init -#define re_yylex_init_ALREADY_DEFINED -#else -#define yylex_init re_yylex_init -#endif - -#ifdef yylex_init_extra -#define re_yylex_init_extra_ALREADY_DEFINED -#else -#define yylex_init_extra re_yylex_init_extra -#endif - -#ifdef yylex_destroy -#define re_yylex_destroy_ALREADY_DEFINED -#else -#define yylex_destroy re_yylex_destroy -#endif - -#ifdef yyget_debug -#define re_yyget_debug_ALREADY_DEFINED -#else -#define yyget_debug re_yyget_debug -#endif - -#ifdef yyset_debug -#define re_yyset_debug_ALREADY_DEFINED -#else -#define yyset_debug re_yyset_debug -#endif - -#ifdef yyget_extra -#define re_yyget_extra_ALREADY_DEFINED -#else -#define yyget_extra re_yyget_extra -#endif - -#ifdef yyset_extra -#define re_yyset_extra_ALREADY_DEFINED -#else -#define yyset_extra re_yyset_extra -#endif - -#ifdef yyget_in -#define re_yyget_in_ALREADY_DEFINED -#else -#define yyget_in re_yyget_in -#endif - -#ifdef yyset_in -#define re_yyset_in_ALREADY_DEFINED -#else -#define yyset_in re_yyset_in -#endif - -#ifdef yyget_out -#define re_yyget_out_ALREADY_DEFINED -#else -#define yyget_out re_yyget_out -#endif - -#ifdef yyset_out -#define re_yyset_out_ALREADY_DEFINED -#else -#define yyset_out re_yyset_out -#endif - -#ifdef yyget_leng -#define re_yyget_leng_ALREADY_DEFINED -#else -#define yyget_leng re_yyget_leng -#endif - -#ifdef yyget_text -#define re_yyget_text_ALREADY_DEFINED -#else -#define yyget_text re_yyget_text -#endif - -#ifdef yyget_lineno -#define re_yyget_lineno_ALREADY_DEFINED -#else -#define yyget_lineno re_yyget_lineno -#endif - -#ifdef yyset_lineno -#define re_yyset_lineno_ALREADY_DEFINED -#else -#define yyset_lineno re_yyset_lineno -#endif - -#ifdef yyget_column -#define re_yyget_column_ALREADY_DEFINED -#else -#define yyget_column re_yyget_column -#endif - -#ifdef yyset_column -#define re_yyset_column_ALREADY_DEFINED -#else -#define yyset_column re_yyset_column -#endif - -#ifdef yywrap -#define re_yywrap_ALREADY_DEFINED -#else -#define yywrap re_yywrap -#endif - -#ifdef yyget_lval -#define re_yyget_lval_ALREADY_DEFINED -#else -#define yyget_lval re_yyget_lval -#endif - -#ifdef yyset_lval -#define re_yyset_lval_ALREADY_DEFINED -#else -#define yyset_lval re_yyset_lval -#endif - -#ifdef yyalloc -#define re_yyalloc_ALREADY_DEFINED -#else -#define yyalloc re_yyalloc -#endif - -#ifdef yyrealloc -#define re_yyrealloc_ALREADY_DEFINED -#else -#define yyrealloc re_yyrealloc -#endif - -#ifdef yyfree -#define re_yyfree_ALREADY_DEFINED -#else -#define yyfree re_yyfree -#endif - /* First, we deal with platform-specific or compiler-specific issues. */ /* begin standard C headers. */ @@ -306,32 +84,40 @@ typedef unsigned int flex_uint32_t; #define UINT32_MAX (4294967295U) #endif -#ifndef SIZE_MAX -#define SIZE_MAX (~(size_t)0) -#endif - #endif /* ! C99 */ #endif /* ! FLEXINT_H */ -/* begin standard C++ headers. */ +#ifdef __cplusplus -/* TODO: this is always defined, so inline it */ +/* The "const" storage-class-modifier is valid. */ +#define YY_USE_CONST + +#else /* ! __cplusplus */ + +/* C99 requires __STDC__ to be defined as 1. */ +#if defined (__STDC__) + +#define YY_USE_CONST + +#endif /* defined (__STDC__) */ +#endif /* ! __cplusplus */ + +#ifdef YY_USE_CONST #define yyconst const - -#if defined(__GNUC__) && __GNUC__ >= 3 -#define yynoreturn __attribute__((__noreturn__)) #else -#define yynoreturn +#define yyconst #endif /* Returned upon end-of-file. */ #define YY_NULL 0 -/* Promotes a possibly negative, possibly signed char to an - * integer in range [0..255] for use as an array index. +/* Promotes a possibly negative, possibly signed char to an unsigned + * integer for use as an array index. If the signed char is negative, + * we want to instead treat it as an 8-bit unsigned char, hence the + * double cast. */ -#define YY_SC_TO_UI(c) ((YY_CHAR) (c)) +#define YY_SC_TO_UI(c) ((unsigned int) (unsigned char) c) /* An opaque pointer. */ #ifndef YY_TYPEDEF_YY_SCANNER_T @@ -355,16 +141,20 @@ typedef void* yyscan_t; * definition of BEGIN. */ #define BEGIN yyg->yy_start = 1 + 2 * + /* Translate the current start state into a value that can be later handed * to BEGIN to return to the state. The YYSTATE alias is for lex * compatibility. */ #define YY_START ((yyg->yy_start - 1) / 2) #define YYSTATE YY_START + /* Action number for EOF rule of a given start state. */ #define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + /* Special action meaning "start processing a new file". */ -#define YY_NEW_FILE yyrestart( yyin , yyscanner ) +#define YY_NEW_FILE re_yyrestart(yyin ,yyscanner ) + #define YY_END_OF_BUFFER_CHAR 0 /* Size of default input buffer. */ @@ -397,10 +187,10 @@ typedef size_t yy_size_t; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 #define EOB_ACT_LAST_MATCH 2 - + /* Note: We specifically omit the test for yy_rule_can_match_eol because it requires * access to the local variable yy_act. Since yyless() is a macro, it would break - * existing scanners that call yyless() from OUTSIDE yylex. + * existing scanners that call yyless() from OUTSIDE re_yylex. * One obvious solution it to make yy_act a global. I tried that, and saw * a 5% performance hit in a non-yylineno scanner, because yy_act is * normally declared as a register variable-- so it is not worth it. @@ -433,6 +223,7 @@ typedef size_t yy_size_t; YY_DO_BEFORE_ACTION; /* set up yytext again */ \ } \ while ( 0 ) + #define unput(c) yyunput( c, yyg->yytext_ptr , yyscanner ) #ifndef YY_STRUCT_YY_BUFFER_STATE @@ -447,12 +238,12 @@ struct yy_buffer_state /* Size of input buffer in bytes, not including room for EOB * characters. */ - int yy_buf_size; + yy_size_t yy_buf_size; /* Number of characters read into yy_ch_buf, not including EOB * characters. */ - int yy_n_chars; + yy_size_t yy_n_chars; /* Whether we "own" the buffer - i.e., we know we created it, * and can realloc() it to grow it, and should free() it to @@ -475,7 +266,7 @@ struct yy_buffer_state int yy_bs_lineno; /**< The line count. */ int yy_bs_column; /**< The column count. */ - + /* Whether to try to fill the input buffer when we reach the * end of it. */ @@ -492,7 +283,7 @@ struct yy_buffer_state * possible backing-up. * * When we actually see the EOF, we change the status to "new" - * (via yyrestart()), so that the user can continue scanning by + * (via re_yyrestart()), so that the user can continue scanning by * just pointing yyin at a new input file. */ #define YY_BUFFER_EOF_PENDING 2 @@ -509,77 +300,87 @@ struct yy_buffer_state #define YY_CURRENT_BUFFER ( yyg->yy_buffer_stack \ ? yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] \ : NULL) + /* Same as previous macro, but useful when we know that the buffer stack is not * NULL or when we need an lvalue. For internal use only. */ #define YY_CURRENT_BUFFER_LVALUE yyg->yy_buffer_stack[yyg->yy_buffer_stack_top] -void yyrestart ( FILE *input_file , yyscan_t yyscanner ); -void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_create_buffer ( FILE *file, int size , yyscan_t yyscanner ); -void yy_delete_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yy_flush_buffer ( YY_BUFFER_STATE b , yyscan_t yyscanner ); -void yypush_buffer_state ( YY_BUFFER_STATE new_buffer , yyscan_t yyscanner ); -void yypop_buffer_state ( yyscan_t yyscanner ); +void re_yyrestart (FILE *input_file ,yyscan_t yyscanner ); +void re_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +YY_BUFFER_STATE re_yy_create_buffer (FILE *file,int size ,yyscan_t yyscanner ); +void re_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void re_yy_flush_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +void re_yypush_buffer_state (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +void re_yypop_buffer_state (yyscan_t yyscanner ); -static void yyensure_buffer_stack ( yyscan_t yyscanner ); -static void yy_load_buffer_state ( yyscan_t yyscanner ); -static void yy_init_buffer ( YY_BUFFER_STATE b, FILE *file , yyscan_t yyscanner ); -#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER , yyscanner) +static void re_yyensure_buffer_stack (yyscan_t yyscanner ); +static void re_yy_load_buffer_state (yyscan_t yyscanner ); +static void re_yy_init_buffer (YY_BUFFER_STATE b,FILE *file ,yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_buffer ( char *base, yy_size_t size , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_string ( const char *yy_str , yyscan_t yyscanner ); -YY_BUFFER_STATE yy_scan_bytes ( const char *bytes, int len , yyscan_t yyscanner ); +#define YY_FLUSH_BUFFER re_yy_flush_buffer(YY_CURRENT_BUFFER ,yyscanner) -void *yyalloc ( yy_size_t , yyscan_t yyscanner ); -void *yyrealloc ( void *, yy_size_t , yyscan_t yyscanner ); -void yyfree ( void * , yyscan_t yyscanner ); +YY_BUFFER_STATE re_yy_scan_buffer (char *base,yy_size_t size ,yyscan_t yyscanner ); +YY_BUFFER_STATE re_yy_scan_string (yyconst char *yy_str ,yyscan_t yyscanner ); +YY_BUFFER_STATE re_yy_scan_bytes (yyconst char *bytes,yy_size_t len ,yyscan_t yyscanner ); + +void *re_yyalloc (yy_size_t ,yyscan_t yyscanner ); +void *re_yyrealloc (void *,yy_size_t ,yyscan_t yyscanner ); +void re_yyfree (void * ,yyscan_t yyscanner ); + +#define yy_new_buffer re_yy_create_buffer -#define yy_new_buffer yy_create_buffer #define yy_set_interactive(is_interactive) \ { \ if ( ! YY_CURRENT_BUFFER ){ \ - yyensure_buffer_stack (yyscanner); \ + re_yyensure_buffer_stack (yyscanner); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); \ + re_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_is_interactive = is_interactive; \ } + #define yy_set_bol(at_bol) \ { \ if ( ! YY_CURRENT_BUFFER ){\ - yyensure_buffer_stack (yyscanner); \ + re_yyensure_buffer_stack (yyscanner); \ YY_CURRENT_BUFFER_LVALUE = \ - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); \ + re_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); \ } \ YY_CURRENT_BUFFER_LVALUE->yy_at_bol = at_bol; \ } + #define YY_AT_BOL() (YY_CURRENT_BUFFER_LVALUE->yy_at_bol) /* Begin user sect3 */ #define re_yywrap(yyscanner) (/*CONSTCOND*/1) #define YY_SKIP_YYWRAP -typedef flex_uint8_t YY_CHAR; + +typedef unsigned char YY_CHAR; typedef int yy_state_type; #define yytext_ptr yytext_r -static yy_state_type yy_get_previous_state ( yyscan_t yyscanner ); -static yy_state_type yy_try_NUL_trans ( yy_state_type current_state , yyscan_t yyscanner); -static int yy_get_next_buffer ( yyscan_t yyscanner ); -static void yynoreturn yy_fatal_error ( const char* msg , yyscan_t yyscanner ); +static yy_state_type yy_get_previous_state (yyscan_t yyscanner ); +static yy_state_type yy_try_NUL_trans (yy_state_type current_state ,yyscan_t yyscanner); +static int yy_get_next_buffer (yyscan_t yyscanner ); +#if defined(__GNUC__) && __GNUC__ >= 3 +__attribute__((__noreturn__)) +#endif +static void yy_fatal_error (yyconst char msg[] ,yyscan_t yyscanner ); /* Done after the current pattern has been matched and before the * corresponding action - sets up yytext. */ #define YY_DO_BEFORE_ACTION \ yyg->yytext_ptr = yy_bp; \ - yyleng = (int) (yy_cp - yy_bp); \ + yyleng = (size_t) (yy_cp - yy_bp); \ yyg->yy_hold_char = *yy_cp; \ *yy_cp = '\0'; \ yyg->yy_c_buf_p = yy_cp; + #define YY_NUM_RULES 29 #define YY_END_OF_BUFFER 30 /* This struct is not used in this scanner, @@ -589,7 +390,7 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[45] = +static yyconst flex_int16_t yy_accept[45] = { 0, 0, 0, 0, 0, 30, 7, 7, 28, 6, 17, 7, 27, 29, 26, 18, 5, 3, 16, 15, 13, @@ -598,7 +399,7 @@ static const flex_int16_t yy_accept[45] = 2, 19, 0, 0 } ; -static const YY_CHAR yy_ec[256] = +static yyconst YY_CHAR yy_ec[256] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -630,14 +431,14 @@ static const YY_CHAR yy_ec[256] = 1, 1, 1, 1, 1 } ; -static const YY_CHAR yy_meta[23] = +static yyconst YY_CHAR yy_meta[23] = { 0, 1, 2, 1, 1, 3, 4, 4, 4, 4, 1, 1, 1, 1, 5, 1, 4, 4, 1, 1, 1, 1, 1 } ; -static const flex_int16_t yy_base[51] = +static yyconst flex_uint16_t yy_base[51] = { 0, 0, 20, 3, 5, 50, 89, 89, 89, 10, 36, 0, 44, 43, 47, 38, 89, 26, 33, 89, 89, @@ -646,7 +447,7 @@ static const flex_int16_t yy_base[51] = 89, 89, 0, 89, 67, 72, 77, 82, 84, 4 } ; -static const flex_int16_t yy_def[51] = +static yyconst flex_int16_t yy_def[51] = { 0, 45, 45, 46, 46, 44, 44, 44, 44, 44, 44, 44, 44, 44, 47, 44, 44, 44, 44, 44, 44, @@ -655,7 +456,7 @@ static const flex_int16_t yy_def[51] = 44, 44, 50, 0, 44, 44, 44, 44, 44, 44 } ; -static const flex_int16_t yy_nxt[112] = +static yyconst flex_uint16_t yy_nxt[112] = { 0, 44, 7, 8, 27, 13, 28, 13, 30, 27, 39, 28, 9, 10, 39, 8, 14, 15, 14, 15, 29, @@ -672,7 +473,7 @@ static const flex_int16_t yy_nxt[112] = 44 } ; -static const flex_int16_t yy_chk[112] = +static yyconst flex_int16_t yy_chk[112] = { 0, 0, 1, 1, 11, 3, 11, 4, 50, 28, 27, 28, 1, 1, 39, 1, 3, 3, 4, 4, 37, @@ -690,7 +491,7 @@ static const flex_int16_t yy_chk[112] = } ; /* Table of booleans, true if rule could match eol. */ -static const flex_int32_t yy_rule_can_match_eol[30] = +static yyconst flex_int32_t yy_rule_can_match_eol[30] = { 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; @@ -791,10 +592,9 @@ int read_escaped_char( yyscan_t yyscanner, uint8_t* escaped_char); -#line 794 "re_lexer.c" #define YY_NO_UNISTD_H 1 -#line 797 "re_lexer.c" +#line 598 "re_lexer.c" #define INITIAL 0 #define char_class 1 @@ -824,8 +624,8 @@ struct yyguts_t size_t yy_buffer_stack_max; /**< capacity of stack. */ YY_BUFFER_STATE * yy_buffer_stack; /**< Stack as an array. */ char yy_hold_char; - int yy_n_chars; - int yyleng_r; + yy_size_t yy_n_chars; + yy_size_t yyleng_r; char *yy_c_buf_p; int yy_init; int yy_start; @@ -847,52 +647,52 @@ struct yyguts_t }; /* end struct yyguts_t */ -static int yy_init_globals ( yyscan_t yyscanner ); +static int yy_init_globals (yyscan_t yyscanner ); /* This must go here because YYSTYPE and YYLTYPE are included * from bison output in section 1.*/ # define yylval yyg->yylval_r -int yylex_init (yyscan_t* scanner); +int re_yylex_init (yyscan_t* scanner); -int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner); +int re_yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); /* Accessor methods to globals. These are made visible to non-reentrant scanners for convenience. */ -int yylex_destroy ( yyscan_t yyscanner ); +int re_yylex_destroy (yyscan_t yyscanner ); -int yyget_debug ( yyscan_t yyscanner ); +int re_yyget_debug (yyscan_t yyscanner ); -void yyset_debug ( int debug_flag , yyscan_t yyscanner ); +void re_yyset_debug (int debug_flag ,yyscan_t yyscanner ); -YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner ); +YY_EXTRA_TYPE re_yyget_extra (yyscan_t yyscanner ); -void yyset_extra ( YY_EXTRA_TYPE user_defined , yyscan_t yyscanner ); +void re_yyset_extra (YY_EXTRA_TYPE user_defined ,yyscan_t yyscanner ); -FILE *yyget_in ( yyscan_t yyscanner ); +FILE *re_yyget_in (yyscan_t yyscanner ); -void yyset_in ( FILE * _in_str , yyscan_t yyscanner ); +void re_yyset_in (FILE * _in_str ,yyscan_t yyscanner ); -FILE *yyget_out ( yyscan_t yyscanner ); +FILE *re_yyget_out (yyscan_t yyscanner ); -void yyset_out ( FILE * _out_str , yyscan_t yyscanner ); +void re_yyset_out (FILE * _out_str ,yyscan_t yyscanner ); - int yyget_leng ( yyscan_t yyscanner ); +yy_size_t re_yyget_leng (yyscan_t yyscanner ); -char *yyget_text ( yyscan_t yyscanner ); +char *re_yyget_text (yyscan_t yyscanner ); -int yyget_lineno ( yyscan_t yyscanner ); +int re_yyget_lineno (yyscan_t yyscanner ); -void yyset_lineno ( int _line_number , yyscan_t yyscanner ); +void re_yyset_lineno (int _line_number ,yyscan_t yyscanner ); -int yyget_column ( yyscan_t yyscanner ); +int re_yyget_column (yyscan_t yyscanner ); -void yyset_column ( int _column_no , yyscan_t yyscanner ); +void re_yyset_column (int _column_no ,yyscan_t yyscanner ); -YYSTYPE * yyget_lval ( yyscan_t yyscanner ); +YYSTYPE * re_yyget_lval (yyscan_t yyscanner ); -void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); +void re_yyset_lval (YYSTYPE * yylval_param ,yyscan_t yyscanner ); /* Macros after this point can all be overridden by user definitions in * section 1. @@ -900,9 +700,9 @@ void yyset_lval ( YYSTYPE * yylval_param , yyscan_t yyscanner ); #ifndef YY_SKIP_YYWRAP #ifdef __cplusplus -extern "C" int yywrap ( yyscan_t yyscanner ); +extern "C" int re_yywrap (yyscan_t yyscanner ); #else -extern int yywrap ( yyscan_t yyscanner ); +extern int re_yywrap (yyscan_t yyscanner ); #endif #endif @@ -911,18 +711,19 @@ extern int yywrap ( yyscan_t yyscanner ); #endif #ifndef yytext_ptr -static void yy_flex_strncpy ( char *, const char *, int , yyscan_t yyscanner); +static void yy_flex_strncpy (char *,yyconst char *,int ,yyscan_t yyscanner); #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen ( const char * , yyscan_t yyscanner); +static int yy_flex_strlen (yyconst char * ,yyscan_t yyscanner); #endif #ifndef YY_NO_INPUT + #ifdef __cplusplus -static int yyinput ( yyscan_t yyscanner ); +static int yyinput (yyscan_t yyscanner ); #else -static int input ( yyscan_t yyscanner ); +static int input (yyscan_t yyscanner ); #endif #endif @@ -942,7 +743,7 @@ static int input ( yyscan_t yyscanner ); /* This used to be an fputs(), but since the string might contain NUL's, * we now use fwrite(). */ -#define ECHO do { if (fwrite( yytext, (size_t) yyleng, 1, yyout )) {} } while (0) +#define ECHO do { if (fwrite( yytext, yyleng, 1, yyout )) {} } while (0) #endif /* Gets input and stuffs it into "buf". number of characters read, or YY_NULL, @@ -953,7 +754,7 @@ static int input ( yyscan_t yyscanner ); if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \ { \ int c = '*'; \ - int n; \ + size_t n; \ for ( n = 0; n < max_size && \ (c = getc( yyin )) != EOF && c != '\n'; ++n ) \ buf[n] = (char) c; \ @@ -966,7 +767,7 @@ static int input ( yyscan_t yyscanner ); else \ { \ errno=0; \ - while ( (result = (int) fread(buf, 1, (yy_size_t) max_size, yyin)) == 0 && ferror(yyin)) \ + while ( (result = fread(buf, 1, max_size, yyin))==0 && ferror(yyin)) \ { \ if( errno != EINTR) \ { \ @@ -1007,10 +808,10 @@ static int input ( yyscan_t yyscanner ); #ifndef YY_DECL #define YY_DECL_IS_OURS 1 -extern int yylex \ - (YYSTYPE * yylval_param , yyscan_t yyscanner); +extern int re_yylex \ + (YYSTYPE * yylval_param ,yyscan_t yyscanner); -#define YY_DECL int yylex \ +#define YY_DECL int re_yylex \ (YYSTYPE * yylval_param , yyscan_t yyscanner) #endif /* !YY_DECL */ @@ -1058,19 +859,19 @@ YY_DECL yyout = stdout; if ( ! YY_CURRENT_BUFFER ) { - yyensure_buffer_stack (yyscanner); + re_yyensure_buffer_stack (yyscanner); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); + re_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); } - yy_load_buffer_state( yyscanner ); + re_yy_load_buffer_state(yyscanner ); } { #line 111 "re_lexer.l" -#line 1073 "re_lexer.c" +#line 875 "re_lexer.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1098,9 +899,9 @@ yy_match: { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 45 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } while ( yy_current_state != 44 ); @@ -1114,10 +915,10 @@ yy_find_action: if ( yy_act != YY_END_OF_BUFFER && yy_rule_can_match_eol[yy_act] ) { - int yyl; + yy_size_t yyl; for ( yyl = 0; yyl < yyleng; ++yyl ) if ( yytext[yyl] == '\n' ) - + do{ yylineno++; yycolumn=0; }while(0) @@ -1572,7 +1373,7 @@ YY_RULE_SETUP #line 483 "re_lexer.l" ECHO; YY_BREAK -#line 1575 "re_lexer.c" +#line 1377 "re_lexer.c" case YY_END_OF_BUFFER: { @@ -1588,7 +1389,7 @@ ECHO; /* We're scanning a new file or input source. It's * possible that this happened because the user * just pointed yyin at a new source and called - * yylex(). If so, then we have to assure + * re_yylex(). If so, then we have to assure * consistency between YY_CURRENT_BUFFER and our * globals. Here is the right place to do so, because * this is the first action (other than possibly a @@ -1649,7 +1450,7 @@ ECHO; { yyg->yy_did_buffer_switch_on_eof = 0; - if ( yywrap( yyscanner ) ) + if ( re_yywrap(yyscanner ) ) { /* Note: because we've taken care in * yy_get_next_buffer() to have set up @@ -1703,7 +1504,7 @@ ECHO; } /* end of action switch */ } /* end of scanning one token */ } /* end of user's declarations */ -} /* end of yylex */ +} /* end of re_yylex */ /* yy_get_next_buffer - try to read in a new buffer * @@ -1717,7 +1518,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; char *dest = YY_CURRENT_BUFFER_LVALUE->yy_ch_buf; char *source = yyg->yytext_ptr; - int number_to_move, i; + yy_size_t number_to_move, i; int ret_val; if ( yyg->yy_c_buf_p > &YY_CURRENT_BUFFER_LVALUE->yy_ch_buf[yyg->yy_n_chars + 1] ) @@ -1746,7 +1547,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) /* Try to read more data. */ /* First move last chars to start of buffer. */ - number_to_move = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr - 1); + number_to_move = (yy_size_t) (yyg->yy_c_buf_p - yyg->yytext_ptr) - 1; for ( i = 0; i < number_to_move; ++i ) *(dest++) = *(source++); @@ -1759,7 +1560,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else { - int num_to_read = + yy_size_t num_to_read = YY_CURRENT_BUFFER_LVALUE->yy_buf_size - number_to_move - 1; while ( num_to_read <= 0 ) @@ -1773,7 +1574,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) if ( b->yy_is_our_buffer ) { - int new_size = b->yy_buf_size * 2; + yy_size_t new_size = b->yy_buf_size * 2; if ( new_size <= 0 ) b->yy_buf_size += b->yy_buf_size / 8; @@ -1782,12 +1583,11 @@ static int yy_get_next_buffer (yyscan_t yyscanner) b->yy_ch_buf = (char *) /* Include room in for 2 EOB chars. */ - yyrealloc( (void *) b->yy_ch_buf, - (yy_size_t) (b->yy_buf_size + 2) , yyscanner ); + re_yyrealloc((void *) b->yy_ch_buf,b->yy_buf_size + 2 ,yyscanner ); } else /* Can't grow it, we don't own it. */ - b->yy_ch_buf = NULL; + b->yy_ch_buf = 0; if ( ! b->yy_ch_buf ) YY_FATAL_ERROR( @@ -1815,7 +1615,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) if ( number_to_move == YY_MORE_ADJ ) { ret_val = EOB_ACT_END_OF_FILE; - yyrestart( yyin , yyscanner); + re_yyrestart(yyin ,yyscanner); } else @@ -1829,15 +1629,12 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else ret_val = EOB_ACT_CONTINUE_SCAN; - if ((yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { + if ((yy_size_t) (yyg->yy_n_chars + number_to_move) > YY_CURRENT_BUFFER_LVALUE->yy_buf_size) { /* Extend the array by 50%, plus the number we really need. */ - int new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); - YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) yyrealloc( - (void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf, (yy_size_t) new_size , yyscanner ); + yy_size_t new_size = yyg->yy_n_chars + number_to_move + (yyg->yy_n_chars >> 1); + YY_CURRENT_BUFFER_LVALUE->yy_ch_buf = (char *) re_yyrealloc((void *) YY_CURRENT_BUFFER_LVALUE->yy_ch_buf,new_size ,yyscanner ); if ( ! YY_CURRENT_BUFFER_LVALUE->yy_ch_buf ) YY_FATAL_ERROR( "out of dynamic memory in yy_get_next_buffer()" ); - /* "- 2" to take care of EOB's */ - YY_CURRENT_BUFFER_LVALUE->yy_buf_size = (int) (new_size - 2); } yyg->yy_n_chars += number_to_move; @@ -1871,9 +1668,9 @@ static int yy_get_next_buffer (yyscan_t yyscanner) { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 45 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; } return yy_current_state; @@ -1900,9 +1697,9 @@ static int yy_get_next_buffer (yyscan_t yyscanner) { yy_current_state = (int) yy_def[yy_current_state]; if ( yy_current_state >= 45 ) - yy_c = yy_meta[yy_c]; + yy_c = yy_meta[(unsigned int) yy_c]; } - yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; yy_is_jam = (yy_current_state == 44); (void)yyg; @@ -1938,7 +1735,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) else { /* need more input */ - int offset = (int) (yyg->yy_c_buf_p - yyg->yytext_ptr); + yy_size_t offset = yyg->yy_c_buf_p - yyg->yytext_ptr; ++yyg->yy_c_buf_p; switch ( yy_get_next_buffer( yyscanner ) ) @@ -1955,14 +1752,14 @@ static int yy_get_next_buffer (yyscan_t yyscanner) */ /* Reset buffer status. */ - yyrestart( yyin , yyscanner); + re_yyrestart(yyin ,yyscanner); /*FALLTHROUGH*/ case EOB_ACT_END_OF_FILE: { - if ( yywrap( yyscanner ) ) - return 0; + if ( re_yywrap(yyscanner ) ) + return EOF; if ( ! yyg->yy_did_buffer_switch_on_eof ) YY_NEW_FILE; @@ -1985,7 +1782,7 @@ static int yy_get_next_buffer (yyscan_t yyscanner) yyg->yy_hold_char = *++yyg->yy_c_buf_p; if ( c == '\n' ) - + do{ yylineno++; yycolumn=0; }while(0) @@ -2000,34 +1797,34 @@ static int yy_get_next_buffer (yyscan_t yyscanner) * @param yyscanner The scanner object. * @note This function does not reset the start condition to @c INITIAL . */ - void yyrestart (FILE * input_file , yyscan_t yyscanner) + void re_yyrestart (FILE * input_file , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if ( ! YY_CURRENT_BUFFER ){ - yyensure_buffer_stack (yyscanner); + re_yyensure_buffer_stack (yyscanner); YY_CURRENT_BUFFER_LVALUE = - yy_create_buffer( yyin, YY_BUF_SIZE , yyscanner); + re_yy_create_buffer(yyin,YY_BUF_SIZE ,yyscanner); } - yy_init_buffer( YY_CURRENT_BUFFER, input_file , yyscanner); - yy_load_buffer_state( yyscanner ); + re_yy_init_buffer(YY_CURRENT_BUFFER,input_file ,yyscanner); + re_yy_load_buffer_state(yyscanner ); } /** Switch to a different input buffer. * @param new_buffer The new input buffer. * @param yyscanner The scanner object. */ - void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) + void re_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* TODO. We should be able to replace this entire function body * with - * yypop_buffer_state(); - * yypush_buffer_state(new_buffer); + * re_yypop_buffer_state(); + * re_yypush_buffer_state(new_buffer); */ - yyensure_buffer_stack (yyscanner); + re_yyensure_buffer_stack (yyscanner); if ( YY_CURRENT_BUFFER == new_buffer ) return; @@ -2040,17 +1837,17 @@ static int yy_get_next_buffer (yyscan_t yyscanner) } YY_CURRENT_BUFFER_LVALUE = new_buffer; - yy_load_buffer_state( yyscanner ); + re_yy_load_buffer_state(yyscanner ); /* We don't actually know whether we did this switch during - * EOF (yywrap()) processing, but the only time this flag - * is looked at is after yywrap() is called, so it's safe + * EOF (re_yywrap()) processing, but the only time this flag + * is looked at is after re_yywrap() is called, so it's safe * to go ahead and always set it. */ yyg->yy_did_buffer_switch_on_eof = 1; } -static void yy_load_buffer_state (yyscan_t yyscanner) +static void re_yy_load_buffer_state (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyg->yy_n_chars = YY_CURRENT_BUFFER_LVALUE->yy_n_chars; @@ -2065,35 +1862,35 @@ static void yy_load_buffer_state (yyscan_t yyscanner) * @param yyscanner The scanner object. * @return the allocated buffer state. */ - YY_BUFFER_STATE yy_create_buffer (FILE * file, int size , yyscan_t yyscanner) + YY_BUFFER_STATE re_yy_create_buffer (FILE * file, int size , yyscan_t yyscanner) { YY_BUFFER_STATE b; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); + b = (YY_BUFFER_STATE) re_yyalloc(sizeof( struct yy_buffer_state ) ,yyscanner ); if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in re_yy_create_buffer()" ); - b->yy_buf_size = size; + b->yy_buf_size = (yy_size_t)size; /* yy_ch_buf has to be 2 characters longer than the size given because * we need to put in 2 end-of-buffer characters. */ - b->yy_ch_buf = (char *) yyalloc( (yy_size_t) (b->yy_buf_size + 2) , yyscanner ); + b->yy_ch_buf = (char *) re_yyalloc(b->yy_buf_size + 2 ,yyscanner ); if ( ! b->yy_ch_buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in re_yy_create_buffer()" ); b->yy_is_our_buffer = 1; - yy_init_buffer( b, file , yyscanner); + re_yy_init_buffer(b,file ,yyscanner); return b; } /** Destroy the buffer. - * @param b a buffer created with yy_create_buffer() + * @param b a buffer created with re_yy_create_buffer() * @param yyscanner The scanner object. */ - void yy_delete_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) + void re_yy_delete_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; @@ -2104,28 +1901,28 @@ static void yy_load_buffer_state (yyscan_t yyscanner) YY_CURRENT_BUFFER_LVALUE = (YY_BUFFER_STATE) 0; if ( b->yy_is_our_buffer ) - yyfree( (void *) b->yy_ch_buf , yyscanner ); + re_yyfree((void *) b->yy_ch_buf ,yyscanner ); - yyfree( (void *) b , yyscanner ); + re_yyfree((void *) b ,yyscanner ); } /* Initializes or reinitializes a buffer. * This function is sometimes called more than once on the same buffer, - * such as during a yyrestart() or at EOF. + * such as during a re_yyrestart() or at EOF. */ - static void yy_init_buffer (YY_BUFFER_STATE b, FILE * file , yyscan_t yyscanner) + static void re_yy_init_buffer (YY_BUFFER_STATE b, FILE * file , yyscan_t yyscanner) { int oerrno = errno; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - yy_flush_buffer( b , yyscanner); + re_yy_flush_buffer(b ,yyscanner); b->yy_input_file = file; b->yy_fill_buffer = 1; - /* If b is the current buffer, then yy_init_buffer was _probably_ - * called from yyrestart() or through yy_get_next_buffer. + /* If b is the current buffer, then re_yy_init_buffer was _probably_ + * called from re_yyrestart() or through yy_get_next_buffer. * In that case, we don't want to reset the lineno or column. */ if (b != YY_CURRENT_BUFFER){ @@ -2142,7 +1939,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. * @param yyscanner The scanner object. */ - void yy_flush_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) + void re_yy_flush_buffer (YY_BUFFER_STATE b , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if ( ! b ) @@ -2163,7 +1960,7 @@ static void yy_load_buffer_state (yyscan_t yyscanner) b->yy_buffer_status = YY_BUFFER_NEW; if ( b == YY_CURRENT_BUFFER ) - yy_load_buffer_state( yyscanner ); + re_yy_load_buffer_state(yyscanner ); } /** Pushes the new state onto the stack. The new state becomes @@ -2172,15 +1969,15 @@ static void yy_load_buffer_state (yyscan_t yyscanner) * @param new_buffer The new state. * @param yyscanner The scanner object. */ -void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) +void re_yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if (new_buffer == NULL) return; - yyensure_buffer_stack(yyscanner); + re_yyensure_buffer_stack(yyscanner); - /* This block is copied from yy_switch_to_buffer. */ + /* This block is copied from re_yy_switch_to_buffer. */ if ( YY_CURRENT_BUFFER ) { /* Flush out information for old buffer. */ @@ -2194,8 +1991,8 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) yyg->yy_buffer_stack_top++; YY_CURRENT_BUFFER_LVALUE = new_buffer; - /* copied from yy_switch_to_buffer. */ - yy_load_buffer_state( yyscanner ); + /* copied from re_yy_switch_to_buffer. */ + re_yy_load_buffer_state(yyscanner ); yyg->yy_did_buffer_switch_on_eof = 1; } @@ -2203,19 +2000,19 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer , yyscan_t yyscanner) * The next element becomes the new top. * @param yyscanner The scanner object. */ -void yypop_buffer_state (yyscan_t yyscanner) +void re_yypop_buffer_state (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; if (!YY_CURRENT_BUFFER) return; - yy_delete_buffer(YY_CURRENT_BUFFER , yyscanner); + re_yy_delete_buffer(YY_CURRENT_BUFFER ,yyscanner); YY_CURRENT_BUFFER_LVALUE = NULL; if (yyg->yy_buffer_stack_top > 0) --yyg->yy_buffer_stack_top; if (YY_CURRENT_BUFFER) { - yy_load_buffer_state( yyscanner ); + re_yy_load_buffer_state(yyscanner ); yyg->yy_did_buffer_switch_on_eof = 1; } } @@ -2223,7 +2020,7 @@ void yypop_buffer_state (yyscan_t yyscanner) /* Allocates the stack if it does not exist. * Guarantees space for at least one push. */ -static void yyensure_buffer_stack (yyscan_t yyscanner) +static void re_yyensure_buffer_stack (yyscan_t yyscanner) { yy_size_t num_to_alloc; struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; @@ -2234,15 +2031,15 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) * scanner will even need a stack. We use 2 instead of 1 to avoid an * immediate realloc on the next call. */ - num_to_alloc = 1; /* After all that talk, this was set to 1 anyways... */ - yyg->yy_buffer_stack = (struct yy_buffer_state**)yyalloc + num_to_alloc = 1; // After all that talk, this was set to 1 anyways... + yyg->yy_buffer_stack = (struct yy_buffer_state**)re_yyalloc (num_to_alloc * sizeof(struct yy_buffer_state*) , yyscanner); if ( ! yyg->yy_buffer_stack ) - YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); - + YY_FATAL_ERROR( "out of dynamic memory in re_yyensure_buffer_stack()" ); + memset(yyg->yy_buffer_stack, 0, num_to_alloc * sizeof(struct yy_buffer_state*)); - + yyg->yy_buffer_stack_max = num_to_alloc; yyg->yy_buffer_stack_top = 0; return; @@ -2254,12 +2051,12 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) yy_size_t grow_size = 8 /* arbitrary grow size */; num_to_alloc = yyg->yy_buffer_stack_max + grow_size; - yyg->yy_buffer_stack = (struct yy_buffer_state**)yyrealloc + yyg->yy_buffer_stack = (struct yy_buffer_state**)re_yyrealloc (yyg->yy_buffer_stack, num_to_alloc * sizeof(struct yy_buffer_state*) , yyscanner); if ( ! yyg->yy_buffer_stack ) - YY_FATAL_ERROR( "out of dynamic memory in yyensure_buffer_stack()" ); + YY_FATAL_ERROR( "out of dynamic memory in re_yyensure_buffer_stack()" ); /* zero only the new slots.*/ memset(yyg->yy_buffer_stack + yyg->yy_buffer_stack_max, 0, grow_size * sizeof(struct yy_buffer_state*)); @@ -2271,9 +2068,9 @@ static void yyensure_buffer_stack (yyscan_t yyscanner) * @param base the character buffer * @param size the size in bytes of the character buffer * @param yyscanner The scanner object. - * @return the newly allocated buffer state object. + * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) +YY_BUFFER_STATE re_yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscanner) { YY_BUFFER_STATE b; @@ -2281,69 +2078,69 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size , yyscan_t yyscann base[size-2] != YY_END_OF_BUFFER_CHAR || base[size-1] != YY_END_OF_BUFFER_CHAR ) /* They forgot to leave room for the EOB's. */ - return NULL; + return 0; - b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) , yyscanner ); + b = (YY_BUFFER_STATE) re_yyalloc(sizeof( struct yy_buffer_state ) ,yyscanner ); if ( ! b ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_buffer()" ); + YY_FATAL_ERROR( "out of dynamic memory in re_yy_scan_buffer()" ); - b->yy_buf_size = (int) (size - 2); /* "- 2" to take care of EOB's */ + b->yy_buf_size = size - 2; /* "- 2" to take care of EOB's */ b->yy_buf_pos = b->yy_ch_buf = base; b->yy_is_our_buffer = 0; - b->yy_input_file = NULL; + b->yy_input_file = 0; b->yy_n_chars = b->yy_buf_size; b->yy_is_interactive = 0; b->yy_at_bol = 1; b->yy_fill_buffer = 0; b->yy_buffer_status = YY_BUFFER_NEW; - yy_switch_to_buffer( b , yyscanner ); + re_yy_switch_to_buffer(b ,yyscanner ); return b; } -/** Setup the input buffer state to scan a string. The next call to yylex() will +/** Setup the input buffer state to scan a string. The next call to re_yylex() will * scan from a @e copy of @a str. * @param yystr a NUL-terminated string to scan * @param yyscanner The scanner object. * @return the newly allocated buffer state object. * @note If you want to scan bytes that may contain NUL values, then use - * yy_scan_bytes() instead. + * re_yy_scan_bytes() instead. */ -YY_BUFFER_STATE yy_scan_string (const char * yystr , yyscan_t yyscanner) +YY_BUFFER_STATE re_yy_scan_string (yyconst char * yystr , yyscan_t yyscanner) { - return yy_scan_bytes( yystr, (int) strlen(yystr) , yyscanner); + return re_yy_scan_bytes(yystr,strlen(yystr) ,yyscanner); } -/** Setup the input buffer state to scan the given bytes. The next call to yylex() will +/** Setup the input buffer state to scan the given bytes. The next call to re_yylex() will * scan from a @e copy of @a bytes. * @param yybytes the byte buffer to scan * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. * @param yyscanner The scanner object. * @return the newly allocated buffer state object. */ -YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan_t yyscanner) +YY_BUFFER_STATE re_yy_scan_bytes (yyconst char * yybytes, yy_size_t _yybytes_len , yyscan_t yyscanner) { YY_BUFFER_STATE b; char *buf; yy_size_t n; - int i; + yy_size_t i; /* Get memory for full buffer, including space for trailing EOB's. */ - n = (yy_size_t) (_yybytes_len + 2); - buf = (char *) yyalloc( n , yyscanner ); + n = _yybytes_len + 2; + buf = (char *) re_yyalloc(n ,yyscanner ); if ( ! buf ) - YY_FATAL_ERROR( "out of dynamic memory in yy_scan_bytes()" ); + YY_FATAL_ERROR( "out of dynamic memory in re_yy_scan_bytes()" ); for ( i = 0; i < _yybytes_len; ++i ) buf[i] = yybytes[i]; buf[_yybytes_len] = buf[_yybytes_len+1] = YY_END_OF_BUFFER_CHAR; - b = yy_scan_buffer( buf, n , yyscanner); + b = re_yy_scan_buffer(buf,n ,yyscanner); if ( ! b ) - YY_FATAL_ERROR( "bad buffer in yy_scan_bytes()" ); + YY_FATAL_ERROR( "bad buffer in re_yy_scan_bytes()" ); /* It's okay to grow etc. this buffer, and we should throw it * away when we're done. @@ -2357,11 +2154,11 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len , yyscan #define YY_EXIT_FAILURE 2 #endif -static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner) +static void yy_fatal_error (yyconst char* msg , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; - fprintf( stderr, "%s\n", msg ); + (void) fprintf( stderr, "%s\n", msg ); exit( YY_EXIT_FAILURE ); } @@ -2387,7 +2184,7 @@ static void yynoreturn yy_fatal_error (const char* msg , yyscan_t yyscanner) /** Get the user-defined data for this scanner. * @param yyscanner The scanner object. */ -YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner) +YY_EXTRA_TYPE re_yyget_extra (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyextra; @@ -2396,10 +2193,10 @@ YY_EXTRA_TYPE yyget_extra (yyscan_t yyscanner) /** Get the current line number. * @param yyscanner The scanner object. */ -int yyget_lineno (yyscan_t yyscanner) +int re_yyget_lineno (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - + if (! YY_CURRENT_BUFFER) return 0; @@ -2409,10 +2206,10 @@ int yyget_lineno (yyscan_t yyscanner) /** Get the current column number. * @param yyscanner The scanner object. */ -int yyget_column (yyscan_t yyscanner) +int re_yyget_column (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; - + if (! YY_CURRENT_BUFFER) return 0; @@ -2422,7 +2219,7 @@ int yyget_column (yyscan_t yyscanner) /** Get the input stream. * @param yyscanner The scanner object. */ -FILE *yyget_in (yyscan_t yyscanner) +FILE *re_yyget_in (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyin; @@ -2431,7 +2228,7 @@ FILE *yyget_in (yyscan_t yyscanner) /** Get the output stream. * @param yyscanner The scanner object. */ -FILE *yyget_out (yyscan_t yyscanner) +FILE *re_yyget_out (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyout; @@ -2440,7 +2237,7 @@ FILE *yyget_out (yyscan_t yyscanner) /** Get the length of the current token. * @param yyscanner The scanner object. */ -int yyget_leng (yyscan_t yyscanner) +yy_size_t re_yyget_leng (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yyleng; @@ -2450,7 +2247,7 @@ int yyget_leng (yyscan_t yyscanner) * @param yyscanner The scanner object. */ -char *yyget_text (yyscan_t yyscanner) +char *re_yyget_text (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yytext; @@ -2460,7 +2257,7 @@ char *yyget_text (yyscan_t yyscanner) * @param user_defined The data to be associated with this scanner. * @param yyscanner The scanner object. */ -void yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) +void re_yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyextra = user_defined ; @@ -2470,13 +2267,13 @@ void yyset_extra (YY_EXTRA_TYPE user_defined , yyscan_t yyscanner) * @param _line_number line number * @param yyscanner The scanner object. */ -void yyset_lineno (int _line_number , yyscan_t yyscanner) +void re_yyset_lineno (int _line_number , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* lineno is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) - YY_FATAL_ERROR( "yyset_lineno called with no buffer" ); + YY_FATAL_ERROR( "re_yyset_lineno called with no buffer" ); yylineno = _line_number; } @@ -2485,13 +2282,13 @@ void yyset_lineno (int _line_number , yyscan_t yyscanner) * @param _column_no column number * @param yyscanner The scanner object. */ -void yyset_column (int _column_no , yyscan_t yyscanner) +void re_yyset_column (int _column_no , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* column is only valid if an input buffer exists. */ if (! YY_CURRENT_BUFFER ) - YY_FATAL_ERROR( "yyset_column called with no buffer" ); + YY_FATAL_ERROR( "re_yyset_column called with no buffer" ); yycolumn = _column_no; } @@ -2500,27 +2297,27 @@ void yyset_column (int _column_no , yyscan_t yyscanner) * input buffer. * @param _in_str A readable stream. * @param yyscanner The scanner object. - * @see yy_switch_to_buffer + * @see re_yy_switch_to_buffer */ -void yyset_in (FILE * _in_str , yyscan_t yyscanner) +void re_yyset_in (FILE * _in_str , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyin = _in_str ; } -void yyset_out (FILE * _out_str , yyscan_t yyscanner) +void re_yyset_out (FILE * _out_str , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yyout = _out_str ; } -int yyget_debug (yyscan_t yyscanner) +int re_yyget_debug (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yy_flex_debug; } -void yyset_debug (int _bdebug , yyscan_t yyscanner) +void re_yyset_debug (int _bdebug , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yy_flex_debug = _bdebug ; @@ -2528,13 +2325,13 @@ void yyset_debug (int _bdebug , yyscan_t yyscanner) /* Accessor methods for yylval and yylloc */ -YYSTYPE * yyget_lval (yyscan_t yyscanner) +YYSTYPE * re_yyget_lval (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; return yylval; } -void yyset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) +void re_yyset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; yylval = yylval_param; @@ -2542,18 +2339,20 @@ void yyset_lval (YYSTYPE * yylval_param , yyscan_t yyscanner) /* User-visible API */ -/* yylex_init is special because it creates the scanner itself, so it is +/* re_yylex_init is special because it creates the scanner itself, so it is * the ONLY reentrant function that doesn't take the scanner as the last argument. * That's why we explicitly handle the declaration, instead of using our macros. */ -int yylex_init(yyscan_t* ptr_yy_globals) + +int re_yylex_init(yyscan_t* ptr_yy_globals) + { if (ptr_yy_globals == NULL){ errno = EINVAL; return 1; } - *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), NULL ); + *ptr_yy_globals = (yyscan_t) re_yyalloc ( sizeof( struct yyguts_t ), NULL ); if (*ptr_yy_globals == NULL){ errno = ENOMEM; @@ -2566,37 +2365,39 @@ int yylex_init(yyscan_t* ptr_yy_globals) return yy_init_globals ( *ptr_yy_globals ); } -/* yylex_init_extra has the same functionality as yylex_init, but follows the +/* re_yylex_init_extra has the same functionality as re_yylex_init, but follows the * convention of taking the scanner as the last argument. Note however, that * this is a *pointer* to a scanner, as it will be allocated by this call (and * is the reason, too, why this function also must handle its own declaration). - * The user defined value in the first argument will be available to yyalloc in + * The user defined value in the first argument will be available to re_yyalloc in * the yyextra field. */ -int yylex_init_extra( YY_EXTRA_TYPE yy_user_defined, yyscan_t* ptr_yy_globals ) + +int re_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ) + { struct yyguts_t dummy_yyguts; - yyset_extra (yy_user_defined, &dummy_yyguts); + re_yyset_extra (yy_user_defined, &dummy_yyguts); if (ptr_yy_globals == NULL){ errno = EINVAL; return 1; } - - *ptr_yy_globals = (yyscan_t) yyalloc ( sizeof( struct yyguts_t ), &dummy_yyguts ); - + + *ptr_yy_globals = (yyscan_t) re_yyalloc ( sizeof( struct yyguts_t ), &dummy_yyguts ); + if (*ptr_yy_globals == NULL){ errno = ENOMEM; return 1; } - + /* By setting to 0xAA, we expose bugs in yy_init_globals. Leave at 0x00 for releases. */ memset(*ptr_yy_globals,0x00,sizeof(struct yyguts_t)); - - yyset_extra (yy_user_defined, *ptr_yy_globals); - + + re_yyset_extra (yy_user_defined, *ptr_yy_globals); + return yy_init_globals ( *ptr_yy_globals ); } @@ -2604,13 +2405,13 @@ static int yy_init_globals (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* Initialization is the same as for the non-reentrant scanner. - * This function is called from yylex_destroy(), so don't allocate here. + * This function is called from re_yylex_destroy(), so don't allocate here. */ - yyg->yy_buffer_stack = NULL; + yyg->yy_buffer_stack = 0; yyg->yy_buffer_stack_top = 0; yyg->yy_buffer_stack_max = 0; - yyg->yy_c_buf_p = NULL; + yyg->yy_c_buf_p = (char *) 0; yyg->yy_init = 0; yyg->yy_start = 0; @@ -2623,42 +2424,42 @@ static int yy_init_globals (yyscan_t yyscanner) yyin = stdin; yyout = stdout; #else - yyin = NULL; - yyout = NULL; + yyin = (FILE *) 0; + yyout = (FILE *) 0; #endif /* For future reference: Set errno on error, since we are called by - * yylex_init() + * re_yylex_init() */ return 0; } -/* yylex_destroy is for both reentrant and non-reentrant scanners. */ -int yylex_destroy (yyscan_t yyscanner) +/* re_yylex_destroy is for both reentrant and non-reentrant scanners. */ +int re_yylex_destroy (yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; /* Pop the buffer stack, destroying each element. */ while(YY_CURRENT_BUFFER){ - yy_delete_buffer( YY_CURRENT_BUFFER , yyscanner ); + re_yy_delete_buffer(YY_CURRENT_BUFFER ,yyscanner ); YY_CURRENT_BUFFER_LVALUE = NULL; - yypop_buffer_state(yyscanner); + re_yypop_buffer_state(yyscanner); } /* Destroy the stack itself. */ - yyfree(yyg->yy_buffer_stack , yyscanner); + re_yyfree(yyg->yy_buffer_stack ,yyscanner); yyg->yy_buffer_stack = NULL; /* Destroy the start condition stack. */ - yyfree( yyg->yy_start_stack , yyscanner ); + re_yyfree(yyg->yy_start_stack ,yyscanner ); yyg->yy_start_stack = NULL; /* Reset the globals. This is important in a non-reentrant scanner so the next time - * yylex() is called, initialization will occur. */ + * re_yylex() is called, initialization will occur. */ yy_init_globals( yyscanner); /* Destroy the main struct (reentrant only). */ - yyfree ( yyscanner , yyscanner ); + re_yyfree ( yyscanner , yyscanner ); yyscanner = NULL; return 0; } @@ -2668,7 +2469,7 @@ int yylex_destroy (yyscan_t yyscanner) */ #ifndef yytext_ptr -static void yy_flex_strncpy (char* s1, const char * s2, int n , yyscan_t yyscanner) +static void yy_flex_strncpy (char* s1, yyconst char * s2, int n , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; @@ -2680,7 +2481,7 @@ static void yy_flex_strncpy (char* s1, const char * s2, int n , yyscan_t yyscann #endif #ifdef YY_NEED_STRLEN -static int yy_flex_strlen (const char * s , yyscan_t yyscanner) +static int yy_flex_strlen (yyconst char * s , yyscan_t yyscanner) { int n; for ( n = 0; s[n]; ++n ) @@ -2690,14 +2491,14 @@ static int yy_flex_strlen (const char * s , yyscan_t yyscanner) } #endif -void *yyalloc (yy_size_t size , yyscan_t yyscanner) +void *re_yyalloc (yy_size_t size , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; - return malloc(size); + return (void *) malloc( size ); } -void *yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) +void *re_yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; @@ -2709,14 +2510,14 @@ void *yyrealloc (void * ptr, yy_size_t size , yyscan_t yyscanner) * any pointer type to void*, and deal with argument conversions * as though doing an assignment. */ - return realloc(ptr, size); + return (void *) realloc( (char *) ptr, size ); } -void yyfree (void * ptr , yyscan_t yyscanner) +void re_yyfree (void * ptr , yyscan_t yyscanner) { struct yyguts_t * yyg = (struct yyguts_t*)yyscanner; (void)yyg; - free( (char *) ptr ); /* see yyrealloc() for (char *) cast */ + free( (char *) ptr ); /* see re_yyrealloc() for (char *) cast */ } #define YYTABLES_NAME "yytables" @@ -2724,6 +2525,7 @@ void yyfree (void * ptr , yyscan_t yyscanner) #line 483 "re_lexer.l" + int escaped_char_value( char* text, uint8_t* value) @@ -2809,14 +2611,20 @@ int read_escaped_char( } +// +// yyfatal (actually named re_yyfatal because of the '%option prefix="re_yy"' +// directive) is called when a fatal error occurs in the parser. When this +// happens we are deep inside the parsing logic generated by flex/bison and +// the only way to exit gracefully from there is using setjmp/longjmp. +// void yyfatal( yyscan_t yyscanner, const char *error_message) { - jmp_buf* recovery_state = (jmp_buf*) yr_thread_storage_get_value( - &yr_recovery_state_key); + jmp_buf* recovery_trampoline = (jmp_buf*) yr_thread_storage_get_value( + &yr_yyfatal_trampoline_tls); - longjmp(*recovery_state, 1); + longjmp(*recovery_trampoline, 1); } @@ -2848,24 +2656,28 @@ int yr_parse_re_string( RE_ERROR* error) { yyscan_t yyscanner; - jmp_buf recovery_state; + jmp_buf recovery_trampoline; RE_LEX_ENVIRONMENT lex_env; lex_env.last_error = ERROR_SUCCESS; lex_env.last_error_message[0] = '\0'; - yr_thread_storage_set_value(&yr_recovery_state_key, &recovery_state); + yr_thread_storage_set_value( + &yr_yyfatal_trampoline_tls, + &recovery_trampoline); - if (setjmp(recovery_state) != 0) + // setjmp returns a non-zero value only when we are returning to this + // point via a call to longjmp to the recovery trampoline. + if (setjmp(recovery_trampoline) != 0) return ERROR_INTERNAL_FATAL_ERROR; FAIL_ON_ERROR(yr_re_ast_create(re_ast)); - yylex_init(&yyscanner); - yyset_extra(*re_ast, yyscanner); - yy_scan_string(re_string, yyscanner); + re_yylex_init(&yyscanner); + re_yyset_extra(*re_ast,yyscanner); + re_yy_scan_string(re_string,yyscanner); yyparse(yyscanner, &lex_env); - yylex_destroy(yyscanner); + re_yylex_destroy(yyscanner); if (lex_env.last_error != ERROR_SUCCESS) { diff --git a/libyara/re_lexer.l b/libyara/re_lexer.l index 3f42fa98..5586e9a5 100644 --- a/libyara/re_lexer.l +++ b/libyara/re_lexer.l @@ -567,14 +567,20 @@ int read_escaped_char( } +// +// yyfatal (actually named re_yyfatal because of the '%option prefix="re_yy"' +// directive) is called when a fatal error occurs in the parser. When this +// happens we are deep inside the parsing logic generated by flex/bison and +// the only way to exit gracefully from there is using setjmp/longjmp. +// void yyfatal( yyscan_t yyscanner, const char *error_message) { - jmp_buf* recovery_state = (jmp_buf*) yr_thread_storage_get_value( - &yr_recovery_state_key); + jmp_buf* recovery_trampoline = (jmp_buf*) yr_thread_storage_get_value( + &yr_yyfatal_trampoline_tls); - longjmp(*recovery_state, 1); + longjmp(*recovery_trampoline, 1); } @@ -606,15 +612,19 @@ int yr_parse_re_string( RE_ERROR* error) { yyscan_t yyscanner; - jmp_buf recovery_state; + jmp_buf recovery_trampoline; RE_LEX_ENVIRONMENT lex_env; lex_env.last_error = ERROR_SUCCESS; lex_env.last_error_message[0] = '\0'; - yr_thread_storage_set_value(&yr_recovery_state_key, &recovery_state); + yr_thread_storage_set_value( + &yr_yyfatal_trampoline_tls, + &recovery_trampoline); - if (setjmp(recovery_state) != 0) + // setjmp returns a non-zero value only when we are returning to this + // point via a call to longjmp to the recovery trampoline. + if (setjmp(recovery_trampoline) != 0) return ERROR_INTERNAL_FATAL_ERROR; FAIL_ON_ERROR(yr_re_ast_create(re_ast)); diff --git a/libyara/rules.c b/libyara/rules.c index a2b9b497..63e26188 100644 --- a/libyara/rules.c +++ b/libyara/rules.c @@ -40,6 +40,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include YR_API int yr_rules_define_integer_variable( @@ -174,47 +175,6 @@ YR_API int yr_rules_define_string_variable( } -#ifdef PROFILING_ENABLED -YR_API void yr_rules_print_profiling_info( - YR_RULES* rules) -{ - YR_RULE* rule; - - printf("\n===== PROFILING INFORMATION =====\n\n"); - - yr_rules_foreach(rules, rule) - { - printf( - "%s:%s: %" PRIu64 " (%0.3f%%)\n", - rule->ns->name, - rule->identifier, - rule->time_cost, - (float) rule->time_cost / rules->time_cost * 100); - } - - printf("\n=================================\n"); -} -#endif - - -YR_API void yr_rules_reset_profiling_info( - YR_RULES* rules) -{ - #ifdef PROFILING_ENABLED - YR_RULE* rule; - - yr_rules_foreach(rules, rule) - { - #ifdef _WIN32 - InterlockedExchange64(&rule->time_cost, 0); - #else - __sync_fetch_and_and(&rule->time_cost, 0); - #endif - } - #endif -} - - YR_API int yr_rules_scan_mem_blocks( YR_RULES* rules, YR_MEMORY_BLOCK_ITERATOR* iterator, @@ -383,37 +343,48 @@ YR_API int yr_rules_scan_proc( } -YR_API int yr_rules_load_stream( - YR_STREAM* stream, +int yr_rules_from_arena( + YR_ARENA* arena, YR_RULES** rules) { - YARA_RULES_FILE_HEADER* header; YR_RULES* new_rules = (YR_RULES*) yr_malloc(sizeof(YR_RULES)); if (new_rules == NULL) return ERROR_INSUFFICIENT_MEMORY; - FAIL_ON_ERROR_WITH_CLEANUP( - yr_arena_load_stream(stream, &new_rules->arena), - // cleanup - yr_free(new_rules)); + YR_SUMMARY* summary = (YR_SUMMARY*) yr_arena_get_ptr( + arena, YR_SUMMARY_SECTION, 0); - header = (YARA_RULES_FILE_HEADER*) - yr_arena_base_address(new_rules->arena); + // Now YR_RULES relies on this arena, let's increment the arena's + // reference count so that if the original owner of the arena calls + // yr_arena_destroy the arena is not destroyed. + yr_arena_acquire(arena); - new_rules->code_start = header->code_start; - new_rules->externals_list_head = header->externals_list_head; - new_rules->rules_list_head = header->rules_list_head; - new_rules->ac_match_table = header->ac_match_table; - new_rules->ac_transition_table = header->ac_transition_table; - new_rules->ac_tables_size = header->ac_tables_size; + new_rules->arena = arena; + new_rules->num_rules = summary->num_rules; + new_rules->num_strings = summary->num_strings; + new_rules->num_namespaces = summary->num_namespaces; - memset(new_rules->tidx_mask, 0, sizeof(new_rules->tidx_mask)); + new_rules->rules_list_head = yr_arena_get_ptr( + arena, YR_RULES_TABLE, 0); - FAIL_ON_ERROR_WITH_CLEANUP( - yr_mutex_create(&new_rules->mutex), - // cleanup - yr_free(new_rules)); + new_rules->strings_list_head = yr_arena_get_ptr( + arena, YR_STRINGS_TABLE, 0); + + new_rules->externals_list_head = yr_arena_get_ptr( + arena, YR_EXTERNAL_VARIABLES_TABLE, 0); + + new_rules->ac_transition_table = yr_arena_get_ptr( + arena, YR_AC_TRANSITION_TABLE, 0); + + new_rules->ac_match_table = yr_arena_get_ptr( + arena, YR_AC_STATE_MATCHES_TABLE, 0); + + new_rules->ac_match_pool = yr_arena_get_ptr( + arena, YR_AC_STATE_MATCHES_POOL, 0); + + new_rules->code_start = yr_arena_get_ptr( + arena, YR_CODE_SECTION, 0); *rules = new_rules; @@ -421,6 +392,28 @@ YR_API int yr_rules_load_stream( } + +YR_API int yr_rules_load_stream( + YR_STREAM* stream, + YR_RULES** rules) +{ + YR_ARENA* arena; + + // Load the arena's data the stream. We are the owners of the arena. + FAIL_ON_ERROR(yr_arena_load_stream(stream, &arena)); + + // Create the YR_RULES object from the arena, this makes YR_RULES owner + // of the arena too. + FAIL_ON_ERROR(yr_rules_from_arena(arena, rules)); + + // Release our ownership so that YR_RULES is the single owner. This way the + // arena is destroyed when YR_RULES is destroyed. + yr_arena_release(arena); + + return ERROR_SUCCESS; +} + + YR_API int yr_rules_load( const char* filename, YR_RULES** rules) @@ -447,11 +440,6 @@ YR_API int yr_rules_save_stream( YR_RULES* rules, YR_STREAM* stream) { - int i; - - for (i = 0; i < YR_BITARRAY_NCHARS(YR_MAX_THREADS); ++i) - assert(rules->tidx_mask[i] == 0); - return yr_arena_save_stream(rules->arena, stream); } @@ -489,40 +477,37 @@ YR_API int yr_rules_get_stats( YR_RULES* rules, YR_RULES_STATS *stats) { - YR_RULE* rule; - YR_STRING* string; + memset(stats, 0, sizeof(YR_RULES_STATS)); + + stats->ac_tables_size = yr_arena_get_current_offset( + rules->arena, YR_AC_TRANSITION_TABLE) / sizeof(YR_AC_TRANSITION); uint32_t* match_list_lengths = (uint32_t*) yr_malloc( - sizeof(uint32_t) * rules->ac_tables_size); - - float match_list_length_sum = 0; - int i, c = 0; + sizeof(uint32_t) * stats->ac_tables_size); if (match_list_lengths == NULL) return ERROR_INSUFFICIENT_MEMORY; - memset(stats, 0, sizeof(YR_RULES_STATS)); + stats->num_rules = rules->num_rules; + stats->num_strings = rules->num_strings; - yr_rules_foreach(rules, rule) + float match_list_length_sum = 0; + int c = 0; + + for (int i = 0; i < stats->ac_tables_size; i++) { - stats->rules++; - yr_rule_strings_foreach(rule, string) - stats->strings++; - } - - stats->ac_tables_size = rules->ac_tables_size; - - for (i = 0; i < rules->ac_tables_size; i++) - { - YR_AC_MATCH* match = rules->ac_match_table[i].match; - int match_list_length = 0; - while (match != NULL) + if (rules->ac_match_table[i] != 0) { - match_list_length++; - stats->ac_matches++; - match = match->next; + YR_AC_MATCH *m = &rules->ac_match_pool[rules->ac_match_table[i]-1]; + + while (m != NULL) + { + match_list_length++; + stats->ac_matches++; + m = m->next; + } } if (i == 0) @@ -546,7 +531,7 @@ YR_API int yr_rules_get_stats( // sort match_list_lengths in increasing order for computing percentiles. qsort(match_list_lengths, c, sizeof(match_list_lengths[0]), _uint32_cmp); - for (i = 0; i < 100; i++) + for (int i = 0; i < 100; i++) { if (i < c) stats->top_ac_match_list_lengths[i] = match_list_lengths[c-i-1]; @@ -558,7 +543,7 @@ YR_API int yr_rules_get_stats( stats->ac_match_list_length_pctls[0] = match_list_lengths[0]; stats->ac_match_list_length_pctls[100] = match_list_lengths[c-1]; - for (i = 1; i < 100; i++) + for (int i = 1; i < 100; i++) stats->ac_match_list_length_pctls[i] = match_list_lengths[(c * i) / 100]; yr_free(match_list_lengths); @@ -580,8 +565,7 @@ YR_API int yr_rules_destroy( external++; } - yr_mutex_destroy(&rules->mutex); - yr_arena_destroy(rules->arena); + yr_arena_release(rules->arena); yr_free(rules); return ERROR_SUCCESS; @@ -592,11 +576,11 @@ YR_API void yr_rule_disable( { YR_STRING* string; - rule->g_flags |= RULE_GFLAGS_DISABLED; + rule->flags |= RULE_FLAGS_DISABLED; yr_rule_strings_foreach(rule, string) { - string->g_flags |= STRING_GFLAGS_DISABLED; + string->flags |= STRING_FLAGS_DISABLED; } } @@ -606,10 +590,10 @@ YR_API void yr_rule_enable( { YR_STRING* string; - rule->g_flags &= ~RULE_GFLAGS_DISABLED; + rule->flags &= ~RULE_FLAGS_DISABLED; yr_rule_strings_foreach(rule, string) { - string->g_flags &= ~STRING_GFLAGS_DISABLED; + string->flags &= ~STRING_FLAGS_DISABLED; } } diff --git a/libyara/scan.c b/libyara/scan.c index dad17fa2..aa6f857b 100644 --- a/libyara/scan.c +++ b/libyara/scan.c @@ -210,7 +210,7 @@ static int _yr_scan_wicompare( static void _yr_scan_update_match_chain_length( - int tidx, + YR_SCAN_CONTEXT* context, YR_STRING* string, YR_MATCH* match_to_update, int chain_length) @@ -225,7 +225,7 @@ static void _yr_scan_update_match_chain_length( if (string->chained_to == NULL) return; - match = string->chained_to->unconfirmed_matches[tidx].head; + match = context->unconfirmed_matches[string->chained_to->idx].head; while (match != NULL) { @@ -235,7 +235,7 @@ static void _yr_scan_update_match_chain_length( ending_offset + string->chain_gap_min <= match_to_update->offset) { _yr_scan_update_match_chain_length( - tidx, string->chained_to, match, chain_length + 1); + context, string->chained_to, match, chain_length + 1); } match = match->next; @@ -350,7 +350,6 @@ static int _yr_scan_verify_chained_string_match( uint64_t ending_offset; int32_t full_chain_length; - int tidx = context->tidx; bool add_match = false; if (matching_string->chained_to == NULL) @@ -358,7 +357,7 @@ static int _yr_scan_verify_chained_string_match( // The matching string is the head of the chain, this match should be // added to the list of unconfirmed matches. The match will remain // unconfirmed until all the strings in the chain are found with the - // correct distances bewteen them. + // correct distances between them. add_match = true; } else @@ -368,16 +367,18 @@ static int _yr_scan_verify_chained_string_match( // list of unconfirmed matches. Unconfirmed matches are sorted in ascending // offset order. If no unconfirmed match exists, the lowest possible offset // is the offset of the current match. - if (matching_string->unconfirmed_matches[tidx].head != NULL) - lowest_offset = matching_string->unconfirmed_matches[tidx].head->offset; + match = context->unconfirmed_matches[matching_string->idx].head; + + if (match != NULL) + lowest_offset = match->offset; else lowest_offset = match_offset; // Iterate over the list of unconfirmed matches for the string that - // preceeds the currently matching string. If we have a string chain like: + // precedes the currently matching string. If we have a string chain like: // S1 <- S2 <- S3, and we just found a match for S2, we are iterating the // list of unconfirmed matches of S1. - match = matching_string->chained_to->unconfirmed_matches[tidx].head; + match = context->unconfirmed_matches[matching_string->chained_to->idx].head; while (match != NULL) { @@ -397,7 +398,8 @@ static int _yr_scan_verify_chained_string_match( // negatively confirmed (i.e: we can be sure that this unconfirmed // match can't be an actual match) _yr_scan_remove_match_from_list( - match, &matching_string->chained_to->unconfirmed_matches[tidx]); + match, + &context->unconfirmed_matches[matching_string->chained_to->idx]); } else if (ending_offset + matching_string->chain_gap_max >= match_offset && ending_offset + matching_string->chain_gap_min <= match_offset) @@ -427,12 +429,12 @@ static int _yr_scan_verify_chained_string_match( // chained to some other string. assert(matching_string->chained_to != NULL); - // Iterate over the list of unconfirmed matches of the preceeding string + // Iterate over the list of unconfirmed matches of the preceding string // in the chain and update the chain_length field for each of them. This // is a recursive operation that will update the chain_length field for // every unconfirmed match in all the strings in the chain up to the head // of the chain. - match = matching_string->chained_to->unconfirmed_matches[tidx].head; + match = context->unconfirmed_matches[matching_string->chained_to->idx].head; while (match != NULL) { @@ -442,7 +444,7 @@ static int _yr_scan_verify_chained_string_match( ending_offset + matching_string->chain_gap_min <= match_offset) { _yr_scan_update_match_chain_length( - tidx, matching_string->chained_to, match, 1); + context, matching_string->chained_to, match, 1); } match = match->next; @@ -458,7 +460,7 @@ static int _yr_scan_verify_chained_string_match( } // "string" points now to the head of the strings chain. - match = string->unconfirmed_matches[tidx].head; + match = context->unconfirmed_matches[string->idx].head; // Iterate over the list of unconfirmed matches of the head of the chain, // and move to the list of confirmed matches those with a chain_length @@ -471,24 +473,28 @@ static int _yr_scan_verify_chained_string_match( if (match->chain_length == full_chain_length) { _yr_scan_remove_match_from_list( - match, &string->unconfirmed_matches[tidx]); + match, + &context->unconfirmed_matches[string->idx]); match->match_length = (int32_t) \ (match_offset - match->offset + match_length); match->data_length = yr_min(match->match_length, max_match_data); - FAIL_ON_ERROR(yr_arena_write_data( - context->matches_arena, + match->data = yr_notebook_alloc( + context->matches_notebook, match->data_length); + + if (match->data == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + memcpy( + (void*) match->data, match_data - match_offset + match->offset, - match->data_length, - (void**) &match->data)); + match->data_length); FAIL_ON_ERROR(_yr_scan_add_match_to_list( match, - !STRING_IS_PRIVATE(string) ? - &string->matches[tidx] : - &string->private_matches[tidx], + &context->matches[string->idx], false)); } @@ -497,23 +503,10 @@ static int _yr_scan_verify_chained_string_match( } else // It's a part of a chain, but not the tail. { - // If this is the first match for the string, put the string in the - // list of strings whose flags needs to be cleared after the scan. - if (matching_string->matches[tidx].count == 0 && - matching_string->private_matches[tidx].count == 0 && - matching_string->unconfirmed_matches[tidx].count == 0) - { - FAIL_ON_ERROR(yr_arena_write_data( - context->matching_strings_arena, - &matching_string, - sizeof(matching_string), - NULL)); - } + new_match = yr_notebook_alloc(context->matches_notebook, sizeof(YR_MATCH)); - FAIL_ON_ERROR(yr_arena_allocate_memory( - context->matches_arena, - sizeof(YR_MATCH), - (void**) &new_match)); + if (new_match == NULL) + return ERROR_INSUFFICIENT_MEMORY; new_match->base = match_base; new_match->offset = match_offset; @@ -521,6 +514,7 @@ static int _yr_scan_verify_chained_string_match( new_match->chain_length = 0; new_match->prev = NULL; new_match->next = NULL; + new_match->private = STRING_IS_PRIVATE(matching_string); // A copy of the matching data is written to the matches_arena, the // amount of data copies is limited by YR_CONFIG_MAX_MATCH_DATA. @@ -528,11 +522,16 @@ static int _yr_scan_verify_chained_string_match( if (new_match->data_length > 0) { - FAIL_ON_ERROR(yr_arena_write_data( - context->matches_arena, + new_match->data = yr_notebook_alloc( + context->matches_notebook, new_match->data_length); + + if (new_match->data == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + memcpy( + (void*) new_match->data, match_data, - new_match->data_length, - (void**) &new_match->data)); + new_match->data_length); } else { @@ -544,7 +543,7 @@ static int _yr_scan_verify_chained_string_match( // an actual match until finding the remaining parts of the chain. FAIL_ON_ERROR(_yr_scan_add_match_to_list( new_match, - &matching_string->unconfirmed_matches[tidx], + &context->unconfirmed_matches[matching_string->idx], false)); } } @@ -565,7 +564,6 @@ static int _yr_scan_match_callback( YR_MATCH* new_match; int result = ERROR_SUCCESS; - int tidx = callback_args->context->tidx; size_t match_offset = match_data - callback_args->data; @@ -617,34 +615,28 @@ static int _yr_scan_match_callback( FAIL_ON_ERROR(yr_get_configuration( YR_CONFIG_MAX_MATCH_DATA, - &max_match_data)) + &max_match_data)); - if (string->matches[tidx].count == 0) - { - // If this is the first match for the string, put the string in the - // list of strings whose flags needs to be cleared after the scan. + new_match = yr_notebook_alloc( + callback_args->context->matches_notebook, sizeof(YR_MATCH)); - FAIL_ON_ERROR(yr_arena_write_data( - callback_args->context->matching_strings_arena, - &string, - sizeof(string), - NULL)); - } - - FAIL_ON_ERROR(yr_arena_allocate_memory( - callback_args->context->matches_arena, - sizeof(YR_MATCH), - (void**) &new_match)); + if (new_match == NULL) + return ERROR_INSUFFICIENT_MEMORY; new_match->data_length = yr_min(match_length, max_match_data); if (new_match->data_length > 0) { - FAIL_ON_ERROR(yr_arena_write_data( - callback_args->context->matches_arena, + new_match->data = yr_notebook_alloc( + callback_args->context->matches_notebook, new_match->data_length); + + if (new_match->data == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + memcpy( + (void*) new_match->data, match_data, - new_match->data_length, - (void**) &new_match->data)); + new_match->data_length); } else { @@ -658,10 +650,11 @@ static int _yr_scan_match_callback( new_match->match_length = match_length; new_match->prev = NULL; new_match->next = NULL; + new_match->private = STRING_IS_PRIVATE(string); FAIL_ON_ERROR(_yr_scan_add_match_to_list( new_match, - !STRING_IS_PRIVATE(string) ? &string->matches[tidx] : &string->private_matches[tidx], + &callback_args->context->matches[string->idx], STRING_IS_GREEDY_REGEXP(string))); } } @@ -711,7 +704,9 @@ static int _yr_scan_verify_re_match( else exec = yr_re_exec; - if (STRING_IS_ASCII(ac_match->string)) + if (STRING_IS_ASCII(ac_match->string) || + STRING_IS_BASE64(ac_match->string) || + STRING_IS_BASE64_WIDE(ac_match->string)) { FAIL_ON_ERROR(exec( context, @@ -725,7 +720,9 @@ static int _yr_scan_verify_re_match( &forward_matches)); } - if (STRING_IS_WIDE(ac_match->string) && forward_matches == -1) + if ((forward_matches == -1) && + (STRING_IS_WIDE(ac_match->string) && + !(STRING_IS_BASE64_WIDE(ac_match->string) || STRING_IS_BASE64_WIDE(ac_match->string)))) { flags |= RE_FLAGS_WIDE; FAIL_ON_ERROR(exec( @@ -902,7 +899,7 @@ int yr_scan_verify_match( if (context->flags & SCAN_FLAGS_FAST_MODE && STRING_IS_SINGLE_MATCH(string) && - string->matches[context->tidx].head != NULL) + context->matches[string->idx].head != NULL) return ERROR_SUCCESS; if (STRING_IS_FIXED_OFFSET(string) && @@ -910,7 +907,7 @@ int yr_scan_verify_match( return ERROR_SUCCESS; #ifdef PROFILING_ENABLED - uint64_t start_time = yr_stopwatch_elapsed_us(&context->stopwatch); + uint64_t start_time = yr_stopwatch_elapsed_ns(&context->stopwatch); #endif if (STRING_IS_LITERAL(string)) @@ -924,15 +921,13 @@ int yr_scan_verify_match( context, ac_match, data, data_size, data_base, offset); } + #ifdef PROFILING_ENABLED + uint64_t finish_time = yr_stopwatch_elapsed_ns(&context->stopwatch); + context->time_cost[string->rule_idx] += (finish_time - start_time); + #endif + if (result != ERROR_SUCCESS) context->last_error_string = string; - #ifdef PROFILING_ENABLED - uint64_t finish_time = yr_stopwatch_elapsed_us(&context->stopwatch); - - string->rule->time_cost_per_thread[context->tidx] += ( - finish_time - start_time); - #endif - return result; } diff --git a/libyara/scanner.c b/libyara/scanner.c index 035ccaab..eb630086 100644 --- a/libyara/scanner.c +++ b/libyara/scanner.c @@ -49,8 +49,8 @@ static int _yr_scanner_scan_mem_block( YR_MEMORY_BLOCK* block) { YR_RULES* rules = scanner->rules; - YR_AC_TRANSITION_TABLE transition_table = rules->ac_transition_table; - YR_AC_MATCH_TABLE match_table = rules->ac_match_table; + YR_AC_TRANSITION* transition_table = rules->ac_transition_table; + uint32_t* match_table = rules->ac_match_table; YR_AC_MATCH* match; YR_AC_TRANSITION transition; @@ -61,30 +61,39 @@ static int _yr_scanner_scan_mem_block( while (i < block->size) { - match = match_table[state].match; - if (i % 4096 == 0 && scanner->timeout > 0) { - if (yr_stopwatch_elapsed_us(&scanner->stopwatch) > scanner->timeout) + if (yr_stopwatch_elapsed_ns(&scanner->stopwatch) > scanner->timeout) return ERROR_SCAN_TIMEOUT; } - while (match != NULL) + if (match_table[state] != 0) { - if (match->backtrack <= i) - { - FAIL_ON_ERROR(yr_scan_verify_match( - scanner, - match, - block_data, - block->size, - block->base, - i - match->backtrack)); - } + // If the entry corresponding to state N in the match table is zero, it + // means that there's no match associated to the state. If it's non-zero, + // its value is the 1-based index within ac_match_pool where the first + // match resides. - match = match->next; + match = &rules->ac_match_pool[match_table[state] - 1]; + + while (match != NULL) + { + if (match->backtrack <= i) + { + FAIL_ON_ERROR(yr_scan_verify_match( + scanner, + match, + block_data, + block->size, + block->base, + i - match->backtrack)); + } + + match = match->next; + } } + index = block_data[i++] + 1; transition = transition_table[state + index]; @@ -105,22 +114,25 @@ static int _yr_scanner_scan_mem_block( state = YR_AC_NEXT_STATE(transition); } - match = match_table[state].match; - - while (match != NULL) + if (match_table[state] != 0) { - if (match->backtrack <= i) - { - FAIL_ON_ERROR(yr_scan_verify_match( - scanner, - match, - block_data, - block->size, - block->base, - i - match->backtrack)); - } + match = &rules->ac_match_pool[match_table[state] - 1]; - match = match->next; + while (match != NULL) + { + if (match->backtrack <= i) + { + FAIL_ON_ERROR(yr_scan_verify_match( + scanner, + match, + block_data, + block->size, + block->base, + i - match->backtrack)); + } + + match = match->next; + } } return ERROR_SUCCESS; @@ -130,40 +142,21 @@ static int _yr_scanner_scan_mem_block( static void _yr_scanner_clean_matches( YR_SCANNER* scanner) { - YR_RULE* rule; - YR_STRING** string; + memset( + scanner->rule_matches_flags, 0, + sizeof(YR_BITMASK) * YR_BITMASK_SIZE(scanner->rules->num_rules)); - int tidx = scanner->tidx; + memset( + scanner->ns_unsatisfied_flags, 0, + sizeof(YR_BITMASK) * YR_BITMASK_SIZE(scanner->rules->num_namespaces)); - yr_rules_foreach(scanner->rules, rule) - { - rule->t_flags[tidx] &= ~RULE_TFLAGS_MATCH; - rule->ns->t_flags[tidx] &= ~NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL; - } + memset( + scanner->matches, 0, + sizeof(YR_MATCHES) * scanner->rules->num_strings); - if (scanner->matching_strings_arena != NULL) - { - string = (YR_STRING**) yr_arena_base_address( - scanner->matching_strings_arena); - - while (string != NULL) - { - (*string)->matches[tidx].count = 0; - (*string)->matches[tidx].head = NULL; - (*string)->matches[tidx].tail = NULL; - (*string)->private_matches[tidx].count = 0; - (*string)->private_matches[tidx].head = NULL; - (*string)->private_matches[tidx].tail = NULL; - (*string)->unconfirmed_matches[tidx].count = 0; - (*string)->unconfirmed_matches[tidx].head = NULL; - (*string)->unconfirmed_matches[tidx].tail = NULL; - - string = (YR_STRING**) yr_arena_next_address( - scanner->matching_strings_arena, - string, - sizeof(YR_STRING*)); - } - } + memset( + scanner->unconfirmed_matches, 0, + sizeof(YR_MATCHES) * scanner->rules->num_strings); } @@ -184,10 +177,31 @@ YR_API int yr_scanner_create( yr_scanner_destroy(new_scanner)); new_scanner->rules = rules; - new_scanner->tidx = -1; - new_scanner->entry_point = UNDEFINED; + new_scanner->entry_point = YR_UNDEFINED; new_scanner->canary = rand(); + // By default report both matching and non-matching rules. + new_scanner->flags = \ + SCAN_FLAGS_REPORT_RULES_MATCHING | + SCAN_FLAGS_REPORT_RULES_NOT_MATCHING; + + new_scanner->rule_matches_flags = (YR_BITMASK*) yr_calloc( + sizeof(YR_BITMASK), YR_BITMASK_SIZE(rules->num_rules)); + + new_scanner->ns_unsatisfied_flags = (YR_BITMASK*) yr_calloc( + sizeof(YR_BITMASK), YR_BITMASK_SIZE(rules->num_namespaces)); + + new_scanner->matches = (YR_MATCHES*) yr_calloc( + rules->num_strings, sizeof(YR_MATCHES)); + + new_scanner->unconfirmed_matches = (YR_MATCHES*) yr_calloc( + rules->num_strings, sizeof(YR_MATCHES)); + + #ifdef PROFILING_ENABLED + new_scanner->time_cost = (uint64_t*) yr_calloc( + rules->num_rules, sizeof(uint64_t)); + #endif + external = rules->externals_list_head; while (!EXTERNAL_VARIABLE_IS_NULL(external)) @@ -196,6 +210,7 @@ YR_API int yr_scanner_create( FAIL_ON_ERROR_WITH_CLEANUP( yr_object_from_external_variable(external, &object), + // cleanup yr_scanner_destroy(new_scanner)); FAIL_ON_ERROR_WITH_CLEANUP( @@ -204,6 +219,8 @@ YR_API int yr_scanner_create( external->identifier, NULL, (void*) object), + // cleanup + yr_object_destroy(object); yr_scanner_destroy(new_scanner)); yr_object_set_canary(object, new_scanner->canary); @@ -238,6 +255,14 @@ YR_API void yr_scanner_destroy( (YR_HASH_TABLE_FREE_VALUE_FUNC) yr_object_destroy); } + #ifdef PROFILING_ENABLED + yr_free(scanner->time_cost); + #endif + + yr_free(scanner->rule_matches_flags); + yr_free(scanner->ns_unsatisfied_flags); + yr_free(scanner->matches); + yr_free(scanner->unconfirmed_matches); yr_free(scanner); } @@ -264,6 +289,16 @@ YR_API void yr_scanner_set_flags( YR_SCANNER* scanner, int flags) { + // For backward compatibility, if neither SCAN_FLAGS_REPORT_RULES_MATCHING + // nor SCAN_FLAGS_REPORT_RULES_NOT_MATCHING are specified, both are assumed. + + if (!(flags & SCAN_FLAGS_REPORT_RULES_MATCHING) && + !(flags & SCAN_FLAGS_REPORT_RULES_NOT_MATCHING)) + { + flags |= SCAN_FLAGS_REPORT_RULES_MATCHING | + SCAN_FLAGS_REPORT_RULES_NOT_MATCHING; + } + scanner->flags = flags; } @@ -345,14 +380,17 @@ YR_API int yr_scanner_scan_mem_blocks( YR_RULE* rule; YR_MEMORY_BLOCK* block; - int tidx = 0; - int result = ERROR_SUCCESS; + uint32_t max_match_data; - uint64_t elapsed_time; + int i, result = ERROR_SUCCESS; if (scanner->callback == NULL) return ERROR_CALLBACK_REQUIRED; + FAIL_ON_ERROR(yr_get_configuration( + YR_CONFIG_MAX_MATCH_DATA, + &max_match_data)) + scanner->iterator = iterator; rules = scanner->rules; block = iterator->first(iterator); @@ -360,39 +398,21 @@ YR_API int yr_scanner_scan_mem_blocks( if (block == NULL) return ERROR_SUCCESS; - yr_mutex_lock(&rules->mutex); - - while (tidx < YR_MAX_THREADS && YR_BITARRAY_TEST(rules->tidx_mask, tidx)) - { - tidx++; - } - - if (tidx < YR_MAX_THREADS) - YR_BITARRAY_SET(rules->tidx_mask, tidx); - else - result = ERROR_TOO_MANY_SCAN_THREADS; - - yr_mutex_unlock(&rules->mutex); - - if (result != ERROR_SUCCESS) - return result; - - scanner->tidx = tidx; scanner->file_size = block->size; - yr_set_tidx(tidx); + // Create the notebook that will hold the YR_MATCH structures representing + // each match found. This notebook will also contain snippets of the matching + // data (the "data" field in YR_MATCH points to the snippet corresponding to + // the match). Each notebook's page can store up to 1024 matches. + result = yr_notebook_create( + 1024 * (sizeof(YR_MATCH) + max_match_data), + &scanner->matches_notebook); + + if (result != ERROR_SUCCESS) + goto _exit; + yr_stopwatch_start(&scanner->stopwatch); - result = yr_arena_create(1048576, 0, &scanner->matches_arena); - - if (result != ERROR_SUCCESS) - goto _exit; - - result = yr_arena_create(4096, 0, &scanner->matching_strings_arena); - - if (result != ERROR_SUCCESS) - goto _exit; - while (block != NULL) { const uint8_t* data = block->fetch_data(block); @@ -404,7 +424,7 @@ YR_API int yr_scanner_scan_mem_blocks( continue; } - if (scanner->entry_point == UNDEFINED) + if (scanner->entry_point == YR_UNDEFINED) { YR_TRYCATCH( !(scanner->flags & SCAN_FLAGS_NO_TRYCATCH), @@ -449,23 +469,27 @@ YR_API int yr_scanner_scan_mem_blocks( if (result != ERROR_SUCCESS) goto _exit; - yr_rules_foreach(rules, rule) + for (i = 0, rule = rules->rules_list_head; + !RULE_IS_NULL(rule); + i++, rule++) { - int message; + int message = 0; - if (rule->t_flags[tidx] & RULE_TFLAGS_MATCH && - !(rule->ns->t_flags[tidx] & NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL)) + if (yr_bitmask_is_set(scanner->rule_matches_flags, i) && + yr_bitmask_is_not_set(scanner->ns_unsatisfied_flags, rule->ns->idx)) { - message = CALLBACK_MSG_RULE_MATCHING; + if (scanner->flags & SCAN_FLAGS_REPORT_RULES_MATCHING) + message = CALLBACK_MSG_RULE_MATCHING; } else { - message = CALLBACK_MSG_RULE_NOT_MATCHING; + if (scanner->flags & SCAN_FLAGS_REPORT_RULES_NOT_MATCHING) + message = CALLBACK_MSG_RULE_NOT_MATCHING; } - if (!RULE_IS_PRIVATE(rule)) + if (message != 0 && !RULE_IS_PRIVATE(rule)) { - switch (scanner->callback(message, rule, scanner->user_data)) + switch (scanner->callback(scanner, message, rule, scanner->user_data)) { case CALLBACK_ABORT: result = ERROR_SUCCESS; @@ -478,46 +502,22 @@ YR_API int yr_scanner_scan_mem_blocks( } } - scanner->callback(CALLBACK_MSG_SCAN_FINISHED, NULL, scanner->user_data); + scanner->callback( + scanner, + CALLBACK_MSG_SCAN_FINISHED, + NULL, + scanner->user_data); _exit: - elapsed_time = yr_stopwatch_elapsed_us(&scanner->stopwatch); - - #ifdef PROFILING_ENABLED - yr_rules_foreach(rules, rule) - { - #ifdef _WIN32 - InterlockedAdd64(&rule->time_cost, rule->time_cost_per_thread[tidx]); - #else - __sync_fetch_and_add(&rule->time_cost, rule->time_cost_per_thread[tidx]); - #endif - - rule->time_cost_per_thread[tidx] = 0; - } - #endif - _yr_scanner_clean_matches(scanner); - if (scanner->matches_arena != NULL) + if (scanner->matches_notebook != NULL) { - yr_arena_destroy(scanner->matches_arena); - scanner->matches_arena = NULL; + yr_notebook_destroy(scanner->matches_notebook); + scanner->matches_notebook = NULL; } - if (scanner->matching_strings_arena != NULL) - { - yr_arena_destroy(scanner->matching_strings_arena); - scanner->matching_strings_arena = NULL; - } - - yr_mutex_lock(&rules->mutex); - YR_BITARRAY_UNSET(rules->tidx_mask, tidx); - rules->time_cost += elapsed_time; - yr_mutex_unlock(&rules->mutex); - - yr_set_tidx(-1); - return result; } @@ -634,5 +634,68 @@ YR_API YR_RULE* yr_scanner_last_error_rule( if (scanner->last_error_string == NULL) return NULL; - return scanner->last_error_string->rule; + return &scanner->rules->rules_list_head[scanner->last_error_string->rule_idx]; } + + +#ifdef PROFILING_ENABLED +struct RULE_COST +{ + uint32_t idx; + uint64_t cost; +}; + +static int sort_by_cost_desc( + const struct RULE_COST* r1, + const struct RULE_COST* r2) +{ + if (r1->cost < r2->cost) + return 1; + + if (r1->cost > r2->cost) + return -1; + + return 0; +} + +YR_API int yr_scanner_print_profiling_info( + YR_SCANNER* scanner) +{ + printf("\n===== PROFILING INFORMATION =====\n\n"); + + struct RULE_COST* rule_costs = yr_malloc( + scanner->rules->num_rules * sizeof(struct RULE_COST)); + + if (rule_costs == NULL) + return ERROR_INSUFFICIENT_MEMORY; + + for (uint32_t i = 0; i < scanner->rules->num_rules; i++) + { + rule_costs[i].idx = i; + rule_costs[i].cost = scanner->time_cost[i]; + } + + qsort( + rule_costs, + scanner->rules->num_rules, + sizeof(struct RULE_COST), + (int (*)(const void *, const void *)) sort_by_cost_desc); + + for (uint32_t i = 0; i < scanner->rules->num_rules; i++) + { + YR_RULE* rule = &scanner->rules->rules_list_head[rule_costs[i].idx]; + + printf( + "%s:%s: %" PRIu64 "\n", + rule->ns->name, + rule->identifier, + rule_costs[i].cost); + } + + printf("\n=================================\n"); + + yr_free(rule_costs); + + return ERROR_SUCCESS; +} +#endif diff --git a/libyara/sizedstr.c b/libyara/sizedstr.c index 85206bcc..fa89829f 100644 --- a/libyara/sizedstr.c +++ b/libyara/sizedstr.c @@ -27,9 +27,39 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include +#include #include #include +#include + + +int sized_string_cmp_nocase( + SIZED_STRING* s1, + SIZED_STRING* s2) +{ + size_t i = 0; + + while (s1->length > i && + s2->length > i && + yr_lowercase[(uint8_t) s1->c_string[i]] == + yr_lowercase[(uint8_t) s2->c_string[i]]) + { + i++; + } + + if (i == s1->length && i == s2->length) + return 0; + else if (i == s1->length) + return -1; + else if (i == s2->length) + return 1; + else if (s1->c_string[i] < s2->c_string[i]) + return -1; + else + return 1; +} int sized_string_cmp( @@ -85,12 +115,43 @@ SIZED_STRING* sized_string_new( result = (SIZED_STRING*) yr_malloc(sizeof(SIZED_STRING) + length); + if (result == NULL) + return NULL; + result->length = length; result->flags = 0; - strncpy(result->c_string, s, length); - - result->c_string[length] = '\0'; - + // Copy the string and the null terminator. + strcpy(result->c_string, s); + return result; } + +// +// Convert a SIZED_STRING to a wide version. It is up to the caller to free +// the returned string. +// + +SIZED_STRING* sized_string_convert_to_wide( + SIZED_STRING* s) +{ + size_t i; + size_t j = 0; + + SIZED_STRING* wide = (SIZED_STRING*) yr_malloc( + sizeof(SIZED_STRING) + s->length * 2); + + if (wide == NULL) + return NULL; + + for (i = 0; i <= s->length; i++) + { + wide->c_string[j++] = s->c_string[i]; + wide->c_string[j++] = '\x00'; + } + + wide->length = s->length * 2; + wide->flags = s->flags | STRING_FLAGS_WIDE; + + return wide; +} diff --git a/libyara/stopwatch.c b/libyara/stopwatch.c index 5fcf2dc2..db8f8995 100644 --- a/libyara/stopwatch.c +++ b/libyara/stopwatch.c @@ -39,14 +39,14 @@ void yr_stopwatch_start( } -uint64_t yr_stopwatch_elapsed_us( +uint64_t yr_stopwatch_elapsed_ns( YR_STOPWATCH* sw) { LARGE_INTEGER li; QueryPerformanceCounter(&li); - return (li.QuadPart - sw->start.QuadPart) * 1000000L / sw->frequency.QuadPart; + return (li.QuadPart - sw->start.QuadPart) * 1000000000L / sw->frequency.QuadPart; } @@ -60,14 +60,11 @@ void yr_stopwatch_start( } -uint64_t yr_stopwatch_elapsed_us( +uint64_t yr_stopwatch_elapsed_ns( YR_STOPWATCH* sw) { - uint64_t now; - - now = mach_absolute_time(); - return (now - sw->start) * sw->timebase.numer / - (sw->timebase.denom * 1000ULL); + uint64_t now = mach_absolute_time(); + return ((now - sw->start) * sw->timebase.numer) / sw->timebase.denom; } @@ -91,7 +88,7 @@ void yr_stopwatch_start( } -uint64_t yr_stopwatch_elapsed_us( +uint64_t yr_stopwatch_elapsed_ns( YR_STOPWATCH* stopwatch) { struct timespec ts_stop; @@ -99,7 +96,7 @@ uint64_t yr_stopwatch_elapsed_us( clock_gettime(CLOCK_MONOTONIC, &ts_stop); timespecsub(&ts_stop, &stopwatch->ts_start, &ts_elapsed); - return ts_elapsed.tv_sec * 1000000L + ts_elapsed.tv_nsec / 1000; + return ts_elapsed.tv_sec * 1000000000L + ts_elapsed.tv_nsec; } @@ -125,7 +122,7 @@ void yr_stopwatch_start( } -uint64_t yr_stopwatch_elapsed_us( +uint64_t yr_stopwatch_elapsed_ns( YR_STOPWATCH* stopwatch) { struct timeval tv_stop; @@ -133,7 +130,7 @@ uint64_t yr_stopwatch_elapsed_us( gettimeofday(&tv_stop, NULL); timevalsub(&tv_stop, &stopwatch->tv_start, &tv_elapsed); - return tv_elapsed.tv_sec * 1000000L + tv_elapsed.tv_usec; + return tv_elapsed.tv_sec * 1000000000L + tv_elapsed.tv_usec * 1000L; } #endif diff --git a/sample.rules b/sample.rules index 54bd3e79..48f67f8d 100644 --- a/sample.rules +++ b/sample.rules @@ -7,6 +7,10 @@ rule UPX : Packer condition: $a at pe.entry_point - } + +fksjd + +rule test {condition: false} + diff --git a/sandbox/BUILD.bazel b/sandbox/BUILD.bazel index e64a0b88..4945ae70 100644 --- a/sandbox/BUILD.bazel +++ b/sandbox/BUILD.bazel @@ -25,6 +25,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + load( "@com_google_sandboxed_api//sandboxed_api/bazel:sapi.bzl", "sapi_library", diff --git a/test-arena-stream b/test-arena-stream new file mode 100644 index 00000000..df9179c5 Binary files /dev/null and b/test-arena-stream differ diff --git a/test-rules.yarc b/test-rules.yarc new file mode 100644 index 00000000..622a6d02 Binary files /dev/null and b/test-rules.yarc differ diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 310ed757..b92bcf46 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -49,6 +49,18 @@ cc_test( srcs = ["test-alignment.c"], copts = COPTS, deps = ["@//:libyara"], + linkstatic=True, +) + +cc_test( + name = "test_arena", + srcs = ["test-arena.c"], + copts = COPTS, + deps = [ + ":util", + "@//:libyara", + ], + linkstatic=True, ) cc_test( @@ -59,6 +71,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -66,6 +79,7 @@ cc_test( srcs = ["test-rules.c"], copts = COPTS, data = [ + "data/base64", "data/baz.yar", "data/foo.yar", "data/include/bar.yar", @@ -78,6 +92,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -91,11 +106,13 @@ cc_test( "data/tiny-idata-5200", "data/tiny-overlay", "data/weird_rich", + "data/mtxex.dll", ], deps = [ ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -107,6 +124,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -118,6 +136,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -128,6 +147,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -138,6 +158,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -148,6 +169,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) # This test fails when running under ASAN @@ -159,6 +181,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -171,6 +194,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -182,6 +206,7 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) cc_test( @@ -192,4 +217,5 @@ cc_test( ":util", "@//:libyara", ], + linkstatic=True, ) diff --git a/tests/data/base64 b/tests/data/base64 new file mode 100644 index 00000000..72db57bc Binary files /dev/null and b/tests/data/base64 differ diff --git a/tests/data/mtxex.dll b/tests/data/mtxex.dll new file mode 100755 index 00000000..89eefdfd Binary files /dev/null and b/tests/data/mtxex.dll differ diff --git a/tests/oss-fuzz/dotnet_fuzzer_corpus/clusterfuzz-testcase-minimized-dotnet_fuzzer-5195285818507264 b/tests/oss-fuzz/dotnet_fuzzer_corpus/clusterfuzz-testcase-minimized-dotnet_fuzzer-5195285818507264 new file mode 100644 index 00000000..0d308e15 Binary files /dev/null and b/tests/oss-fuzz/dotnet_fuzzer_corpus/clusterfuzz-testcase-minimized-dotnet_fuzzer-5195285818507264 differ diff --git a/tests/oss-fuzz/rules_fuzzer.dict b/tests/oss-fuzz/rules_fuzzer.dict index 6c8b83d6..f53deb7b 100644 --- a/tests/oss-fuzz/rules_fuzzer.dict +++ b/tests/oss-fuzz/rules_fuzzer.dict @@ -1,5 +1,6 @@ # Lines starting with '#' and empty lines are ignored. +# Keywords "all" "and" "any" @@ -10,33 +11,186 @@ "entrypoint" "false" "filesize" -"fullword" "for" +"fullword" "global" -"in" "import" +"in" "include" -"int8" "int16" -"int32" -"int8be" "int16be" +"int32" "int32be" +"int8" +"int8be" "matches" "meta" "nocase" "not" -"or" "of" +"or" "private" "rule" "strings" "them" "true" -"uint8" "uint16" -"uint32" -"uint8be" "uint16be" +"uint32" "uint32be" -"wide" \ No newline at end of file +"uint8" +"uint8be" +"wide" +"xor" + +# pe module +"pe" +"pe.machine" +"pe.checksum" +"pe.calculate_checksum" +"pe.subsystem" +"pe.timestamp" +"pe.pointer_to_symbol_table" +"pe.number_of_sumbols" +"pe.size_of_optional_header" +"pe.pothdr_magic" +"pe.size_of_code" +"pe.size_of_initialized_data" +"pe.size_of_unnitialized_data" +"pe.entrypoint" +"pe.base_of_code" +"pe.base_of_data" +"pe.image_base" +"pe.section_alignment" +"pe.file_alignment" +"pe.win32_version_value" +"pe.size_of_image" +"pe.size_of_headers" +"pe.characteristics" +"pe.linker_version" +"pe.os_version" +"pe.image_version" +"pe.subsystem_version" +"pe.dll_characteristics" +"pe.size_of_stack_reserve" +"pe.size_of_stack_commit" +"pe.size_of_heap_reserve" +"pe.size_of_heap_commit" +"pe.loader_flags" +"pe.number_of_rva_and_sizes" +"pe.data_directories" +"pe.number_of_sections" +"pe.sections" +"pe.overlay" +"pe.number_of_resources" +"pe.resource_timestamp" +"pe.resource_version" +"pe.resources" +"pe.version_info" +"pe.number_of_signatures" +"pe.signatures" +"pe.rich_signature" +"pe.exports" +"pe.number_of_exports" +"pe.number_of_imports" +"pe.imports" +"pe.locale" +"pe.language" +"pe.imphash" +"pe.section_index" +"pe.is_dll()" +"pe.is_32bit()" +"pe.is_64bit()" +"pe.rva_to_offset" + +# elf module +"elf" +"elf.type" +"elf.machine" +"elf.entry_point" +"elf.number_of_sections" +"elf.sections" +"elf.number_of_segments" +"elf.segments" +"elf.dynamic_section_entires" +"elf.dynamic" +"elf.symtab_entries" +"elf.symtab" + +# cuckoo module +"cuckoo" +"cuckoo.network" +"cuckoo.registry" +"cuckoo.filesystem" +"cuckoo.sync" + +# magic module +"magic" +"magic.type()" +"magic.mime_type()" + + +# hash module +"hash" +"hash.md5" +"hash.sha1" +"hash.sha256" +"hash.checksum32" +"hash.crc32" + +# math module +"math" +"math.entropuy" +"math.monte_carlo_pi" +"math.serial_correlation" +"math.mean" +"math.deviation" +"math.in_range" +"math.max" +"max.min" + +# dotnet module +"dotnet" +"dotnet.version" +"dotnet.module_name" +"dotnet.number_of_streams" +"dotnet.streams" +"dotnet.number_of_guid" +"dotnet.guids" +"dotnet.number_of_resources" +"dotnet.resources" +"dotnet.assembly" +"dotnet.number_of_modulerefs" +"dotnet.modulerefs" +"dotnet.typelib" +"dotnet.assembly_refs" +"dotnet.number_of_user_strings" +"dotnet.user_strings" +"dotnet.number_of_field_offsets" +"dotnet.field_offsets" + +# time module +"time" +"time.now()" + + +# misc +"/*" +"*/" +"//" +"$a=" +"{a?}" +"[0-9]" +"{(0A|??)}" +"<<" +">>" +"#a" +"$a" +".." +"@a" + +# regex +"*?" +"+?" +"??" +"{1,2}?" diff --git a/tests/test-alignment.c b/tests/test-alignment.c index e9f74518..8ec4b18f 100644 --- a/tests/test-alignment.c +++ b/tests/test-alignment.c @@ -61,41 +61,46 @@ int err = 0; else \ { \ printf("expected %d\n", offset); \ + err = 1; \ } \ } while (0) int main (int argc, char **argv) { - CHECK_SIZE(YR_NAMESPACE, 4 * YR_MAX_THREADS + 8); - CHECK_OFFSET(YR_NAMESPACE, 4 * YR_MAX_THREADS, name); + CHECK_SIZE(YR_NAMESPACE, 16); + CHECK_OFFSET(YR_NAMESPACE, 0, name); + CHECK_OFFSET(YR_NAMESPACE, 8, idx); CHECK_SIZE(YR_META, 32); - CHECK_OFFSET(YR_META, 8, integer); - CHECK_OFFSET(YR_META, 16, identifier); - CHECK_OFFSET(YR_META, 24, string); + CHECK_OFFSET(YR_META, 0, identifier); + CHECK_OFFSET(YR_META, 8, string); + CHECK_OFFSET(YR_META, 16, integer); + CHECK_OFFSET(YR_META, 24, type); - CHECK_SIZE(YR_MATCHES, 24); - CHECK_OFFSET(YR_MATCHES, 8, head); - CHECK_OFFSET(YR_MATCHES, 16, tail); - - CHECK_SIZE(YR_STRING, 56 + 3 * 24 /* YR_MATCHES */ * YR_MAX_THREADS); - CHECK_OFFSET(YR_STRING, 4, length); - CHECK_OFFSET(YR_STRING, 8, identifier); - CHECK_OFFSET(YR_STRING, 16, string); - CHECK_OFFSET(YR_STRING, 24, chained_to); - CHECK_OFFSET(YR_STRING, 32, rule); + CHECK_SIZE(YR_STRING, 56); + CHECK_OFFSET(YR_STRING, 0, flags); + CHECK_OFFSET(YR_STRING, 4, idx); + CHECK_OFFSET(YR_STRING, 8, fixed_offset); + CHECK_OFFSET(YR_STRING, 16, rule_idx); + CHECK_OFFSET(YR_STRING, 20, length); + CHECK_OFFSET(YR_STRING, 24, string); + CHECK_OFFSET(YR_STRING, 32, chained_to); CHECK_OFFSET(YR_STRING, 40, chain_gap_min); CHECK_OFFSET(YR_STRING, 44, chain_gap_max); - CHECK_OFFSET(YR_STRING, 48, fixed_offset); + CHECK_OFFSET(YR_STRING, 48, identifier); - CHECK_SIZE(YR_RULE, 64 + 12 * YR_MAX_THREADS); - CHECK_OFFSET(YR_RULE, 4, t_flags); - CHECK_OFFSET(YR_RULE, 8 + 4 * YR_MAX_THREADS, identifier); - CHECK_OFFSET(YR_RULE, 8 + 4 * YR_MAX_THREADS + 8, tags); - CHECK_OFFSET(YR_RULE, 8 + 4 * YR_MAX_THREADS + 16, metas); - CHECK_OFFSET(YR_RULE, 8 + 4 * YR_MAX_THREADS + 24, strings); - CHECK_OFFSET(YR_RULE, 8 + 4 * YR_MAX_THREADS + 32, ns); + CHECK_SIZE(YR_RULE, 48); + CHECK_OFFSET(YR_RULE, 8, identifier); + CHECK_OFFSET(YR_RULE, 16, tags); + CHECK_OFFSET(YR_RULE, 24, metas); + CHECK_OFFSET(YR_RULE, 32, strings); + CHECK_OFFSET(YR_RULE, 40, ns); + + CHECK_SIZE(YR_SUMMARY, 12); + CHECK_OFFSET(YR_SUMMARY, 0, num_rules); + CHECK_OFFSET(YR_SUMMARY, 4, num_strings); + CHECK_OFFSET(YR_SUMMARY, 8, num_namespaces); CHECK_SIZE(YR_EXTERNAL_VARIABLE, 24); CHECK_OFFSET(YR_EXTERNAL_VARIABLE, 8, value.i); @@ -104,17 +109,11 @@ int main (int argc, char **argv) CHECK_OFFSET(YR_EXTERNAL_VARIABLE, 16, identifier); CHECK_SIZE(YR_AC_MATCH, 40); - CHECK_OFFSET(YR_AC_MATCH, 8, string); - CHECK_OFFSET(YR_AC_MATCH, 16, forward_code); - CHECK_OFFSET(YR_AC_MATCH, 24, backward_code); - CHECK_OFFSET(YR_AC_MATCH, 32, next); - - CHECK_SIZE(YARA_RULES_FILE_HEADER, 48); - CHECK_OFFSET(YARA_RULES_FILE_HEADER, 0, rules_list_head); - CHECK_OFFSET(YARA_RULES_FILE_HEADER, 8, externals_list_head); - CHECK_OFFSET(YARA_RULES_FILE_HEADER, 16, code_start); - CHECK_OFFSET(YARA_RULES_FILE_HEADER, 24, ac_match_table); - CHECK_OFFSET(YARA_RULES_FILE_HEADER, 32, ac_transition_table); + CHECK_OFFSET(YR_AC_MATCH, 0, string); + CHECK_OFFSET(YR_AC_MATCH, 8, forward_code); + CHECK_OFFSET(YR_AC_MATCH, 16, backward_code); + CHECK_OFFSET(YR_AC_MATCH, 24, next); + CHECK_OFFSET(YR_AC_MATCH, 32, backtrack); CHECK_SIZE(SIZED_STRING, 12); CHECK_OFFSET(SIZED_STRING, 4, flags); diff --git a/tests/test-api.c b/tests/test-api.c index febbe2cb..cb2b677e 100644 --- a/tests/test-api.c +++ b/tests/test-api.c @@ -15,13 +15,16 @@ void test_disabled_rules() YR_RULES* rules; YR_RULE* rule; - int matches = 0; char* buf = "foo bar"; char* rules_str = " \ rule disabled_rule {condition: true} \ rule false_rule {condition: true and disabled_rule} \ rule true_rule {condition: true or disabled_rule}"; + struct COUNTERS counters; + + counters.rules_not_matching = 0; + counters.rules_matching = 0; yr_initialize(); @@ -39,12 +42,12 @@ void test_disabled_rules() } yr_rules_scan_mem( - rules, (uint8_t *) buf, strlen(buf), 0, count_matches, &matches, 0); + rules, (uint8_t *) buf, strlen(buf), 0, count, &counters, 0); yr_rules_destroy(rules); // matches should be exactly one. - if (matches != 1) + if (counters.rules_matching != 1) { fprintf(stderr, "test_disabled_rules failed\n"); exit(EXIT_FAILURE); @@ -206,6 +209,7 @@ void test_max_string_per_rules() int test_max_match_data_callback( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data) @@ -219,7 +223,7 @@ int test_max_match_data_callback( { YR_MATCH* m; - yr_string_matches_foreach(s, m) + yr_string_matches_foreach(context, s, m) { if (m->data_length > 0) return CALLBACK_ERROR; @@ -280,7 +284,11 @@ void test_save_load_rules() YR_COMPILER* compiler = NULL; YR_RULES* rules = NULL; - int matches = 0; + struct COUNTERS counters; + + counters.rules_not_matching = 0; + counters.rules_matching = 0; + char* rules_str = "rule t {condition: bool_var and str_var == \"foobar\"}"; yr_initialize(); @@ -330,8 +338,8 @@ void test_save_load_rules() (uint8_t *) "", 0, 0, - count_matches, - &matches, + count, + &counters, 0); if (err != ERROR_SUCCESS) @@ -340,9 +348,13 @@ void test_save_load_rules() exit(EXIT_FAILURE); } - if (matches != 1) + if (counters.rules_matching != 1) { - fprintf(stderr, "test_save_load_rules: expecting 1 match, got: %d\n", matches); + fprintf( + stderr, + "test_save_load_rules: expecting 1 match, got: %d\n", + counters.rules_matching); + exit(EXIT_FAILURE); } @@ -353,11 +365,17 @@ void test_save_load_rules() void test_scanner() { - int matches = 0; const char* buf = "dummy"; const char* rules_str = "\ + rule true_rule { \ + condition: true \ + } \ + rule false_rule { \ + condition: false \ + } \ rule test { \ - condition: bool_var and int_var == 1 and str_var == \"foo\" }"; + condition: bool_var and int_var == 1 and str_var == \"foo\" \ + }"; YR_COMPILER* compiler = NULL; YR_RULES* rules = NULL; @@ -436,8 +454,13 @@ void test_scanner() exit(EXIT_FAILURE); } - // Set the callback and a some the correct values for the rule to match. - yr_scanner_set_callback(scanner1, count_matches, &matches); + struct COUNTERS counters; + + counters.rules_not_matching = 0; + counters.rules_matching = 0; + + // Set the callback and the correct variable values for the rule to match. + yr_scanner_set_callback(scanner1, count, &counters); yr_scanner_define_integer_variable(scanner1, "int_var", 1); yr_scanner_define_boolean_variable(scanner1, "bool_var", 1); yr_scanner_define_string_variable(scanner1, "str_var", "foo"); @@ -460,7 +483,40 @@ void test_scanner() exit(EXIT_FAILURE); } - if (matches != 1) + if (counters.rules_matching != 2 || + counters.rules_not_matching != 1) + { + yr_scanner_destroy(scanner1); + yr_scanner_destroy(scanner2); + yr_rules_destroy(rules); + exit(EXIT_FAILURE); + } + + counters.rules_matching = 0; + counters.rules_not_matching = 0; + + yr_scanner_set_flags(scanner1, SCAN_FLAGS_REPORT_RULES_MATCHING); + yr_scanner_set_callback(scanner1, count, &counters); + yr_scanner_scan_mem(scanner1, (uint8_t *) buf, strlen(buf)); + + if (counters.rules_matching != 2 || + counters.rules_not_matching != 0) + { + yr_scanner_destroy(scanner1); + yr_scanner_destroy(scanner2); + yr_rules_destroy(rules); + exit(EXIT_FAILURE); + } + + counters.rules_matching = 0; + counters.rules_not_matching = 0; + + yr_scanner_set_flags(scanner2, SCAN_FLAGS_REPORT_RULES_NOT_MATCHING); + yr_scanner_set_callback(scanner2, count, &counters); + yr_scanner_scan_mem(scanner2, (uint8_t *) buf, strlen(buf)); + + if (counters.rules_not_matching != 2 || + counters.rules_matching != 0) { yr_scanner_destroy(scanner1); yr_scanner_destroy(scanner2); @@ -633,8 +689,8 @@ void test_rules_stats() condition: all of them }", &stats); - assert_true_expr(stats.rules == 1); - assert_true_expr(stats.strings == 1); + assert_true_expr(stats.num_rules == 1); + assert_true_expr(stats.num_strings == 1); assert_true_expr(stats.ac_root_match_list_length == 1); stats_for_rules("\ @@ -643,8 +699,8 @@ void test_rules_stats() condition: all of them }", &stats); - assert_true_expr(stats.rules == 1); - assert_true_expr(stats.strings == 1); + assert_true_expr(stats.num_rules == 1); + assert_true_expr(stats.num_strings == 1); assert_true_expr(stats.ac_matches == 1); assert_true_expr(stats.ac_root_match_list_length == 0); assert_true_expr(stats.top_ac_match_list_lengths[0] == 1); @@ -663,8 +719,8 @@ void test_rules_stats() condition: all of them }", &stats); - assert_true_expr(stats.rules == 1); - assert_true_expr(stats.strings == 6); + assert_true_expr(stats.num_rules == 1); + assert_true_expr(stats.num_strings == 6); assert_true_expr(stats.ac_matches == 6); assert_true_expr(stats.ac_root_match_list_length == 0); assert_true_expr(stats.top_ac_match_list_lengths[0] == 3); @@ -685,8 +741,8 @@ void test_rules_stats() condition: all of them }", &stats); - assert_true_expr(stats.rules == 1); - assert_true_expr(stats.strings == 8); + assert_true_expr(stats.num_rules == 1); + assert_true_expr(stats.num_strings == 8); assert_true_expr(stats.ac_matches == 8); assert_true_expr(stats.ac_root_match_list_length == 0); assert_true_expr(stats.top_ac_match_list_lengths[0] == 3); @@ -698,8 +754,8 @@ void test_rules_stats() condition: true }", &stats); - assert_true_expr(stats.rules == 1); - assert_true_expr(stats.strings == 0); + assert_true_expr(stats.num_rules == 1); + assert_true_expr(stats.num_strings== 0); assert_true_expr(stats.ac_matches == 0); assert_true_expr(stats.ac_root_match_list_length == 0); } @@ -754,11 +810,12 @@ int main(int argc, char** argv) test_include_callback(); test_save_load_rules(); test_scanner(); - test_ast_callback(); - test_rules_stats(); - - test_issue_834(); - test_issue_920(); +// test_ast_callback(); +// //TODO(vmalvarez): Enable these tests. +// //test_rules_stats(); +// +// test_issue_834(); +// test_issue_920(); return 0; } diff --git a/tests/test-arena.c b/tests/test-arena.c new file mode 100644 index 00000000..d0a10ef5 --- /dev/null +++ b/tests/test-arena.c @@ -0,0 +1,175 @@ +/* +Copyright (c) 2020. The YARA Authors. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include + +static void basic_tests() +{ + YR_ARENA* arena; + + yr_initialize(); + + // Create arena with 1 buffers of 10 bytes of initial size + assert(yr_arena_create(2, 10, &arena) == ERROR_SUCCESS); + + YR_ARENA_REF ref; + + // Allocate 5 bytes. + assert(yr_arena_allocate_memory(arena, 0, 5, &ref) == ERROR_SUCCESS); + + // Offset should be 0 as this is the first write. + assert(ref.offset == 0); + + // Write 16 bytes, "123456789ABCDEF" + null terminator. This forces a + // reallocation. + assert(yr_arena_write_string(arena, 0, "123456789ABCDEF", &ref) == ERROR_SUCCESS); + + // Offset should be 5 as this was written after the first 5-bytes write. + assert(ref.offset == 5); + + // Write 4 bytes, "bar" + null terminator. + assert(yr_arena_write_string(arena, 0, "123456789ABCDEF", &ref) == ERROR_SUCCESS); + + // Offset should be 21. + assert(ref.offset == 21); + + yr_arena_release(arena); + yr_finalize(); +} + + +typedef struct TEST_STRUCT TEST_STRUCT; + +struct TEST_STRUCT { + DECLARE_REFERENCE(char*, str1); + DECLARE_REFERENCE(char*, str2); +}; + + +static void advanced_tests() +{ + YR_ARENA* arena; + + yr_initialize(); + + // Create arena with 3 buffers of 10 bytes of initial size. Only the first + // two are used, the third one is left empty on purpose. + int result = yr_arena_create(3, 10, &arena); + assert(result == ERROR_SUCCESS); + + YR_ARENA_REF ref; + + // Allocate a struct in buffer 0 indicating that the field "str" is a + // relocatable pointer. + result = yr_arena_allocate_struct( + arena, + 0, + sizeof(TEST_STRUCT), + &ref, + offsetof(TEST_STRUCT, str1), + offsetof(TEST_STRUCT, str2), + EOL); + + assert(result == ERROR_SUCCESS); + + // Get the struct address, this pointer is valid as longs as we don't call + // any other function that allocates memory in buffer 0. + TEST_STRUCT* s = (TEST_STRUCT*) yr_arena_ref_to_ptr(arena, &ref); + + // Write a string in buffer 1. + yr_arena_write_string(arena, 1, "foo", &ref); + + // Get the string's address and store it in the struct's "str" field. + s->str1 = (char *) yr_arena_ref_to_ptr(arena, &ref); + + // Write another string in buffer 1. + yr_arena_write_string(arena, 1, "bar", &ref); + + // Get the string's address and store it in the struct's "str" field. + s->str2 = (char *) yr_arena_ref_to_ptr(arena, &ref); + + // The arena should have two reloc entries for the "str1" and "str2" fields. + assert(arena->reloc_list_head != NULL); + assert(arena->reloc_list_tail != NULL); + assert(arena->reloc_list_head->buffer_id == 0); + assert(arena->reloc_list_tail->buffer_id == 0); + assert(arena->reloc_list_head->offset == offsetof(TEST_STRUCT, str1)); + assert(arena->reloc_list_tail->offset == offsetof(TEST_STRUCT, str2)); + + // Write another string in buffer 1 that causes a buffer reallocation. + yr_arena_write_string(arena, 1, "aaaaaaaaaaa", NULL); + + assert(strcmp(s->str1, "foo") == 0); + assert(strcmp(s->str2, "bar") == 0); + + YR_STREAM stream; + FILE* fh = fopen("test-arena-stream", "w+"); + + assert(fh != NULL); + + stream.user_data = fh; + stream.write = (YR_STREAM_WRITE_FUNC) fwrite; + stream.read = (YR_STREAM_READ_FUNC) fread; + + if (yr_arena_save_stream(arena, &stream) != ERROR_SUCCESS) + exit(EXIT_FAILURE); + + fflush(fh); + fseek(fh, 0, SEEK_SET); + + assert(strcmp(s->str1, "foo") == 0); + assert(strcmp(s->str2, "bar") == 0); + + yr_arena_release(arena); + + result = yr_arena_load_stream(&stream, &arena); + assert(result == ERROR_SUCCESS); + + ref.buffer_id = 0; + ref.offset = 0; + + s = (TEST_STRUCT*) yr_arena_ref_to_ptr(arena, &ref); + + assert(strcmp(s->str1, "foo") == 0); + assert(strcmp(s->str2, "bar") == 0); + + fclose(fh); + yr_arena_release(arena); + yr_finalize(); +} + +int main(int argc, char** argv) +{ + basic_tests(); + advanced_tests(); + + return 0; +} diff --git a/tests/test-bitmask.c b/tests/test-bitmask.c index e967fc80..f721369a 100644 --- a/tests/test-bitmask.c +++ b/tests/test-bitmask.c @@ -39,7 +39,7 @@ void assert_clear_all(YR_BITMASK* bitmask) { for (int i = 0; i < BITMAP_SIZE; i++) { - if (yr_bitmask_isset(bitmask, i)) + if (yr_bitmask_is_set(bitmask, i)) { fprintf(stderr, "bit %d is set and should not\n", i); exit(EXIT_FAILURE); @@ -57,22 +57,22 @@ void test_set_clear() yr_bitmask_set(bitmask, 0); - if (!yr_bitmask_isset(bitmask, 0)) + if (yr_bitmask_is_not_set(bitmask, 0)) exit(EXIT_FAILURE); yr_bitmask_clear(bitmask, 0); - if (yr_bitmask_isset(bitmask, 0)) + if (yr_bitmask_is_set(bitmask, 0)) exit(EXIT_FAILURE); yr_bitmask_set(bitmask, BITMAP_SIZE-1); - if (!yr_bitmask_isset(bitmask, BITMAP_SIZE-1)) + if (yr_bitmask_is_not_set(bitmask, BITMAP_SIZE - 1)) exit(EXIT_FAILURE); yr_bitmask_clear(bitmask, BITMAP_SIZE-1); - if (yr_bitmask_isset(bitmask, BITMAP_SIZE-1)) + if (yr_bitmask_is_set(bitmask, BITMAP_SIZE - 1)) exit(EXIT_FAILURE); yr_bitmask_set(bitmask, 31); @@ -82,22 +82,22 @@ void test_set_clear() yr_bitmask_set(bitmask, 64); yr_bitmask_set(bitmask, 65); - if (!yr_bitmask_isset(bitmask, 31)) + if (yr_bitmask_is_not_set(bitmask, 31)) exit(EXIT_FAILURE); - if (!yr_bitmask_isset(bitmask, 32)) + if (yr_bitmask_is_not_set(bitmask, 32)) exit(EXIT_FAILURE); - if (!yr_bitmask_isset(bitmask, 33)) + if (yr_bitmask_is_not_set(bitmask, 33)) exit(EXIT_FAILURE); - if (!yr_bitmask_isset(bitmask, 63)) + if (yr_bitmask_is_not_set(bitmask, 63)) exit(EXIT_FAILURE); - if (!yr_bitmask_isset(bitmask, 64)) + if (yr_bitmask_is_not_set(bitmask, 64)) exit(EXIT_FAILURE); - if (!yr_bitmask_isset(bitmask, 65)) + if (yr_bitmask_is_not_set(bitmask, 65)) exit(EXIT_FAILURE); yr_bitmask_clear(bitmask, 31); @@ -111,7 +111,6 @@ void test_set_clear() } - void test_find_non_colliding_offsets_1() { uint32_t o = 0; diff --git a/tests/test-dotnet.c b/tests/test-dotnet.c index 7c096534..a5a3b602 100644 --- a/tests/test-dotnet.c +++ b/tests/test-dotnet.c @@ -72,6 +72,13 @@ int main(int argc, char** argv) }", "tests/data/33fc70f99be6d2833ae48852d611c8048d0c053ed0b2c626db4dbe902832a08b"); + assert_true_rule_file( + "import \"dotnet\" \ + rule test { \ + condition: \ + dotnet.user_strings[0] == \"F\\x00r\\x00e\\x00e\\x00D\\x00i\\x00s\\x00c\\x00B\\x00u\\x00r\\x00n\\x00e\\x00r\\x00.\\x00S\\x00t\\x00r\\x00i\\x00n\\x00g\\x00R\\x00e\\x00s\\x00o\\x00u\\x00r\\x00c\\x00e\\x00s\\x00\" \ + }", + "tests/data/33fc70f99be6d2833ae48852d611c8048d0c053ed0b2c626db4dbe902832a08b"); yr_finalize(); return 0; } diff --git a/tests/test-exception.c b/tests/test-exception.c index b33b38b4..edf96628 100644 --- a/tests/test-exception.c +++ b/tests/test-exception.c @@ -114,7 +114,9 @@ void setup_crasher() } /* Simple yr_scan_* callback function that delays execution by 2 seconds */ -int delay_callback(int message, +int delay_callback( + YR_SCAN_CONTEXT* context, + int message, void* message_data, void* user_data) { @@ -134,16 +136,20 @@ int test_crash(int handle_exceptions) setup_rules(); puts("Scanning for \"aaaa\"..."); - int matches = 0; + + struct COUNTERS counters; + + counters.rules_not_matching = 0; + counters.rules_matching = 0; int flags = (handle_exceptions ? 0 : SCAN_FLAGS_NO_TRYCATCH); int rc = yr_rules_scan_mem( - rules_a, mapped_region, COUNT * sizeof(wbuf), flags, count_matches, &matches, 0); + rules_a, mapped_region, COUNT * sizeof(wbuf), flags, count, &counters, 0); - printf("err = %d, matches = %d\n", rc, matches); + printf("err = %d, matches = %d\n", rc, counters.rules_matching); - if (rc == ERROR_SUCCESS || matches != 0) + if (rc == ERROR_SUCCESS || counters.rules_matching != 0) return 1; return 0; @@ -194,14 +200,18 @@ int test_blocked_signal() { kill(getpid(), SIGUSR1); puts("Scanning for {00 00 00 00}..."); - int matches = 0; + + struct COUNTERS counters; + + counters.rules_not_matching = 0; + counters.rules_matching = 0; int rc = yr_rules_scan_mem( - rules_0, mapped_region, COUNT * sizeof(wbuf), 0, count_matches, &matches, 0); + rules_0, mapped_region, COUNT * sizeof(wbuf), 0, count, &counters, 0); - printf("err = %d, matches = %d\n", rc, matches); + printf("err = %d, matches = %d\n", rc, counters.rules_matching); - if (rc == ERROR_SUCCESS || matches != 0) + if (rc == ERROR_SUCCESS || counters.rules_matching != 0) return 1; return 0; diff --git a/tests/test-pe.c b/tests/test-pe.c index 59fde730..d9b6295e 100644 --- a/tests/test-pe.c +++ b/tests/test-pe.c @@ -40,7 +40,7 @@ int main(int argc, char** argv) "import \"pe\" \ rule test { \ condition: \ - pe.imports(/.*/, /.*CriticalSection/) \ + pe.imports(/.*/, /.*CriticalSection/) == 4 \ }", "tests/data/tiny"); @@ -48,7 +48,7 @@ int main(int argc, char** argv) "import \"pe\" \ rule test { \ condition: \ - pe.imports(/kernel32\\.dll/i, /.*/) \ + pe.imports(/kernel32\\.dll/i, /.*/) == 21 \ }", "tests/data/tiny"); @@ -199,6 +199,14 @@ int main(int argc, char** argv) }", "tests/data/tiny"); + assert_true_rule_file( + "import \"pe\" \ + rule test { \ + condition: \ + pe.pdb_path == \"D:\\\\workspace\\\\2018_R9_RelBld\\\\target\\\\checkout\\\\custprof\\\\Release\\\\custprof.pdb\" \ + }", + "tests/data/079a472d22290a94ebb212aa8015cdc8dd28a968c6b4d3b88acdd58ce2d3b885"); + assert_false_rule_file( "import \"pe\" \ rule test { \ @@ -207,6 +215,70 @@ int main(int argc, char** argv) }", "tests/data/tiny-idata-51ff"); + /* + * mtxex.dll is 23e72ce7e9cdbc80c0095484ebeb02f56b21e48fd67044e69e7a2ae76db631e5, + * which was taken from a Windows 10 install. The details of which are: + * export_timestamp = 1827812126 + * dll_name = "mtxex.dll" + * number_of_exports = 4 + * export_details + * [0] + * offset = 1072 + * name = "DllGetClassObject" + * forward_name = YR_UNDEFINED + * ordinal = 1 + * [1] + * offset = YR_UNDEFINED + * name = "GetObjectContext" + * forward_name = "COMSVCS.GetObjectContext" + * ordinal = 2 + * [2] + * offset = YR_UNDEFINED + * name = "MTSCreateActivity" + * forward_name = "COMSVCS.MTSCreateActivity" + * ordinal = 3 + * [3] + * offset = YR_UNDEFINED + * name = "SafeRef" + * forward_name = "COMSVCS.SafeRef" + * ordinal = 4 + */ + assert_true_rule_file( + "import \"pe\" \ + rule test { \ + condition: \ + pe.number_of_exports == 4 and \ + pe.dll_name == \"mtxex.dll\" and \ + pe.export_timestamp == 1827812126 and \ + pe.export_details[0].offset == 1072 and \ + pe.export_details[0].name == \"DllGetClassObject\" and \ + pe.export_details[0].ordinal == 1 and \ + pe.export_details[1].forward_name == \"COMSVCS.GetObjectContext\" \ + }", + "tests/data/mtxex.dll"); + + // Make sure exports function is case insensitive (historically this has been + // the case) and supports ordinals... + assert_true_rule_file( + "import \"pe\" \ + rule test { \ + condition: \ + pe.exports(\"saferef\") and \ + pe.exports(4) and \ + pe.exports(/mtscreateactivity/i) \ + }", + "tests/data/mtxex.dll"); + + assert_true_rule_file( + "import \"pe\" \ + rule test { \ + condition: \ + pe.exports_index(\"MTSCreateActivity\") == 2 and \ + pe.exports_index(3) == 2 and \ + pe.exports_index(/mtscreateactivity/i) == 2 \ + }", + "tests/data/mtxex.dll"); + assert_true_rule_file( "import \"pe\" \ rule test { \ @@ -230,6 +302,14 @@ int main(int argc, char** argv) }", "tests/data/weird_rich"); + assert_true_rule_file( + "import \"pe\" \ + rule test { \ + condition: \ + pe.language(0x09) and pe.locale(0x0409) \ + }", + "tests/data/mtxex.dll"); + yr_finalize(); return 0; } diff --git a/tests/test-rules.c b/tests/test-rules.c index 13d27325..3274c7f3 100644 --- a/tests/test-rules.c +++ b/tests/test-rules.c @@ -695,6 +695,251 @@ static void test_strings() }", "AXS1111ERS2222"); + assert_error( + "rule test {\n\ + strings:\n\ + $a = \"ab\" base64 nocase\n\ + condition:\n\ + true\n\ + }", ERROR_INVALID_MODIFIER); + + assert_error( + "rule test {\n\ + strings:\n\ + $a = \"ab\" base64 xor\n\ + condition:\n\ + true\n\ + }", ERROR_INVALID_MODIFIER); + + assert_error( + "rule test {\n\ + strings:\n\ + $a = \"ab\" base64(\"AXS\")\n\ + condition:\n\ + true\n\ + }", ERROR_INVALID_MODIFIER); + + assert_error( + "rule test {\n\ + strings:\n\ + $a = \"ab\" base64wide(\"ERS\")\n\ + condition:\n\ + true\n\ + }", ERROR_INVALID_MODIFIER); + + // Specifying different alphabets is an error. + assert_error( + "rule test {\n\ + strings:\n\ + $a = \"ab\" base64 base64wide(\"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ,.\")\n\ + condition:\n\ + true\n\ + }", ERROR_INVALID_MODIFIER); + + // Be specific about the offsets in these tests to make sure we are matching + // the correct strings. Also be specific about the length because we want to + // make sure the match is not the entire base64 string, but just the + // substrings which are not dependent upon leading or trailing bytes. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64\n\ + condition:\n\ + #a == 6 and\n\ + @a[1] == 0x53 and\n\ + !a[1] == 25 and\n\ + @a[2] == 0x70 and\n\ + !a[2] == 25 and\n\ + @a[3] == 0xa2 and\n\ + !a[3] == 24 and\n\ + @a[4] == 0xbd and\n\ + !a[4] == 24 and\n\ + @a[5] == 0xef and\n\ + !a[5] == 25 and\n\ + @a[6] == 0x109 and\n\ + !a[6] == 25\n\ + }", "tests/data/base64"); + + // This is identical to "base64" alone, but test it to make sure we don't + // accidentally include the plaintext in the base64 search. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64 ascii\n\ + condition:\n\ + #a == 6 and\n\ + @a[1] == 0x53 and\n\ + !a[1] == 25 and\n\ + @a[2] == 0x70 and\n\ + !a[2] == 25 and\n\ + @a[3] == 0xa2 and\n\ + !a[3] == 24 and\n\ + @a[4] == 0xbd and\n\ + !a[4] == 24 and\n\ + @a[5] == 0xef and\n\ + !a[5] == 25 and\n\ + @a[6] == 0x109 and\n\ + !a[6] == 25\n\ + }", "tests/data/base64"); + + // Make sure the wide modifier is applied BEFORE the base64 and we do NOT + // include the wide plaintext string. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64 wide\n\ + condition:\n\ + #a == 6 and\n\ + @a[1] == 0x1b5 and\n\ + !a[1] == 50 and\n\ + @a[2] == 0x1ea and\n\ + !a[2] == 50 and\n\ + @a[3] == 0x248 and\n\ + !a[3] == 50 and\n\ + @a[4] == 0x27b and\n\ + !a[4] == 50 and\n\ + @a[5] == 0x2db and\n\ + !a[5] == 50 and\n\ + @a[6] == 0x311 and\n\ + !a[6] == 50\n\ + }", "tests/data/base64"); + + // Make sure that both wide and ascii are base64 encoded. We can skip the + // verbose length and offset checks, since the previous tests cover that. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64 wide ascii\n\ + condition:\n\ + #a == 12\n\ + }", "tests/data/base64"); + + // This checks for the ascii string in base64 form then widened. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64wide\n\ + condition:\n\ + #a == 3 and\n\ + @a[1] == 0x379 and\n\ + !a[1] == 50 and\n\ + @a[2] == 0x3b6 and\n\ + !a[2] == 48 and\n\ + @a[3] == 0x3f1 and\n\ + !a[3] == 50\n\ + }", "tests/data/base64"); + + // Logically identical to the test above but include it to make sure we don't + // accidentally include the plaintext in the future. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64wide ascii\n\ + condition:\n\ + #a == 3 and\n\ + @a[1] == 0x379 and\n\ + !a[1] == 50 and\n\ + @a[2] == 0x3b6 and\n\ + !a[2] == 48 and\n\ + @a[3] == 0x3f1 and\n\ + !a[3] == 50\n\ + }", "tests/data/base64"); + + // Make sure the wide string is base64wide encoded. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64wide wide\n\ + condition:\n\ + #a == 3 and\n\ + @a[1] == 0x458 and\n\ + !a[1] == 100 and\n\ + @a[2] == 0x4c5 and\n\ + !a[2] == 100 and\n\ + @a[3] == 0x530 and\n\ + !a[3] == 100\n\ + }", "tests/data/base64"); + + // Make sure both ascii and wide strings are base64wide encoded properly. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64wide wide ascii\n\ + condition:\n\ + #a == 6 and\n\ + @a[1] == 0x379 and\n\ + !a[1] == 50 and\n\ + @a[2] == 0x3b6 and\n\ + !a[2] == 48 and\n\ + @a[3] == 0x3f1 and\n\ + !a[3] == 50 and\n\ + @a[4] == 0x458 and\n\ + !a[4] == 100 and\n\ + @a[5] == 0x4c5 and\n\ + !a[5] == 100 and\n\ + @a[6] == 0x530 and\n\ + !a[6] == 100\n\ + }", "tests/data/base64"); + + // Make sure base64 and base64wide together work. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64 base64wide\n\ + condition:\n\ + #a == 9 and\n\ + @a[1] == 0x53 and\n\ + !a[1] == 25 and\n\ + @a[2] == 0x70 and\n\ + !a[2] == 25 and\n\ + @a[3] == 0xa2 and\n\ + !a[3] == 24 and\n\ + @a[4] == 0xbd and\n\ + !a[4] == 24 and\n\ + @a[5] == 0xef and\n\ + !a[5] == 25 and\n\ + @a[6] == 0x109 and\n\ + !a[6] == 25 and\n\ + @a[7] == 0x379 and\n\ + !a[7] == 50 and\n\ + @a[8] == 0x3b6 and\n\ + !a[8] == 48 and\n\ + @a[9] == 0x3f1 and\n\ + !a[9] == 50\n\ + }", "tests/data/base64"); + + // Identical to the test above but useful to make sure we don't accidentally + // include the ascii plaintext in the future. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64 base64wide ascii\n\ + condition:\n\ + #a == 9\n\ + }", "tests/data/base64"); + + // Making sure we don't accidentally include the wide plaintext in the future. + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64 base64wide wide\n\ + condition:\n\ + #a == 9\n\ + }", "tests/data/base64"); + + assert_true_rule_file( + "rule test {\n\ + strings:\n\ + $a = \"This program cannot\" base64(\"!@#$\%^&*(){}[].,|ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstu\")\n\ + condition:\n\ + #a == 3 and\n\ + @a[1] == 0x619 and\n\ + !a[1] == 25 and\n\ + @a[2] == 0x638 and\n\ + !a[2] == 24 and\n\ + @a[3] == 0x656 and\n\ + !a[3] == 25\n\ + }", "tests/data/base64"); } @@ -1237,6 +1482,16 @@ void test_for() }", NULL); + assert_false_rule( + "import \"tests\" \ + rule test { \ + condition: \ + for any k,v in tests.empty_struct_dict : ( \ + true \ + ) \ + }", + NULL); + assert_true_rule( "import \"tests\" \ rule test { \ @@ -1286,6 +1541,22 @@ void test_for() for any i in tests.integer_array : ( i == \"foo\" ) \ }", ERROR_WRONG_TYPE); + + + assert_false_rule( + "rule test { \ + condition: \ + for any i in (0,1): ( \ + for any j in (0,1): ( \ + for any k in (0,1): ( \ + for any l in (0,1): (\ + false \ + ) \ + ) \ + ) \ + ) \ + }", + NULL); } @@ -2197,12 +2468,28 @@ void test_integer_functions() void test_include_files() { assert_true_rule( - "include \"tests/data/baz.yar\" rule t { condition: baz }", - NULL); + "include \"tests/data/baz.yar\" rule t { condition: baz }", + NULL); assert_true_rule( - "include \"tests/data/foo.yar\" rule t { condition: foo }", - NULL); + "include \"tests/data/foo.yar\" rule t { condition: foo }", + NULL); +} + + +void test_tags() +{ + assert_true_rule( + "rule test : tag1 { condition: true}", + NULL); + + assert_true_rule( + "rule test : tag1 tag2 { condition: true}", + NULL); + + assert_error( + "rule test : tag1 tag1 { condition: true}", + ERROR_DUPLICATED_TAG_IDENTIFIER); } @@ -2210,10 +2497,14 @@ void test_process_scan() { int pid = fork(); int status = 0; - int matches = 0; YR_RULES* rules; int rc1, rc2; + struct COUNTERS counters; + + counters.rules_not_matching = 0; + counters.rules_matching = 0; + if (pid == 0) { /* The string should appear somewhere in the shell's process space. */ @@ -2232,7 +2523,7 @@ void test_process_scan() condition:\ all of them\ }", &rules) == ERROR_SUCCESS); - rc1 = yr_rules_scan_proc(rules, pid, 0, count_matches, &matches, 0); + rc1 = yr_rules_scan_proc(rules, pid, 0, count, &counters, 0); yr_rules_destroy(rules); kill(pid, SIGALRM); @@ -2250,7 +2541,7 @@ void test_process_scan() switch (rc1) { case ERROR_SUCCESS: - if (matches == 0) + if (counters.rules_matching == 0) { fputs("Found no matches\n", stderr); exit(EXIT_FAILURE); @@ -2392,6 +2683,18 @@ void test_performance_warnings() "rule test { \ strings: $a = \"MZ\" \ condition: $a }") + + assert_warning( + "rule test { \ + strings: $a = \" \" xor(0x20) \ + condition: $a }") + + // This will eventually xor with 0x41 and should cause a warning. + assert_warning( + "rule test { \ + strings: $a = \"AAAAAAAAAAAAAAAAAAAA\" xor \ + condition: $a }") + } @@ -2435,6 +2738,7 @@ int main(int argc, char** argv) // test_string_io(); test_entrypoint(); test_global_rules(); + test_tags(); #if !defined(USE_WINDOWS_PROC) && !defined(USE_NO_PROC) test_process_scan(); diff --git a/tests/util.c b/tests/util.c index ab0060cc..5818288b 100644 --- a/tests/util.c +++ b/tests/util.c @@ -49,15 +49,36 @@ int warnings; // -// A YR_CALLBACK_FUNC that counts the number of matches during the scan. The -// user_data argument must be a pointer to an int. +// A YR_CALLBACK_FUNC that counts the number of matching and non-matching rules +// during a scan. user_data must point to a COUNTERS structure. // -int count_matches( +int count( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data) { - if (message == CALLBACK_MSG_RULE_MATCHING) + switch (message) + { + case CALLBACK_MSG_RULE_MATCHING: + (*(struct COUNTERS*) user_data).rules_matching++; + break; + + case CALLBACK_MSG_RULE_NOT_MATCHING: + (*(struct COUNTERS*) user_data).rules_not_matching++; + + } + return CALLBACK_CONTINUE; +} + + +int count_non_matches( + YR_SCAN_CONTEXT* context, + int message, + void* message_data, + void* user_data) +{ + if (message == CALLBACK_MSG_RULE_NOT_MATCHING) { (*(int*) user_data)++; } @@ -65,11 +86,11 @@ int count_matches( return CALLBACK_CONTINUE; } - // // A YR_CALLBACK_FUNC that does nothing. // int do_nothing( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data) @@ -139,6 +160,7 @@ struct SCAN_CALLBACK_CTX { }; static int _scan_callback( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data) @@ -190,7 +212,7 @@ int matches_blob( }; int scan_result = yr_rules_scan_mem( - rules, blob, blob_size, 0, _scan_callback, &ctx, 0); + rules, blob, blob_size, SCAN_FLAGS_NO_TRYCATCH, _scan_callback, &ctx, 0); if (scan_result != ERROR_SUCCESS) { @@ -225,6 +247,7 @@ typedef struct static int capture_matches( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data) @@ -240,7 +263,7 @@ static int capture_matches( { YR_MATCH* match; - yr_string_matches_foreach(string, match) + yr_string_matches_foreach(context, string, match) { if (strlen(f->expected) == match->data_length && strncmp(f->expected, (char*)(match->data), match->data_length) == 0) diff --git a/tests/util.h b/tests/util.h index 693ec293..a7aefbd9 100644 --- a/tests/util.h +++ b/tests/util.h @@ -30,21 +30,31 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef _UTIL_H #define _UTIL_H +#include + extern char compile_error[1024]; extern int warnings; +struct COUNTERS +{ + int rules_matching; + int rules_not_matching; +}; + int compile_rule( char* string, YR_RULES** rules); -int count_matches( +int count( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data); int do_nothing( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data); diff --git a/windows/vs2015/libyara/libyara.vcxproj b/windows/vs2015/libyara/libyara.vcxproj index c75c131c..6d08d41b 100644 --- a/windows/vs2015/libyara/libyara.vcxproj +++ b/windows/vs2015/libyara/libyara.vcxproj @@ -90,7 +90,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB;PROFILING_ENABLED ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x86.1.1.0\include;..\packages\YARA.OpenSSL.x86.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) ProgramDatabase @@ -117,7 +117,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIBC;PROFILING_ENABLED ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x64.1.1.0\include;..\packages\YARA.OpenSSL.x64.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) ProgramDatabase false @@ -144,7 +144,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x86.1.1.0\include;..\packages\YARA.OpenSSL.x86.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) MultiThreadedDLL @@ -169,7 +169,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x64.1.1.0\include;..\packages\YARA.OpenSSL.x64.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) false @@ -196,7 +196,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB;NDEBUG=1 ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x64.1.1.0\include;..\packages\YARA.OpenSSL.x64.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) false @@ -222,6 +222,7 @@ + @@ -246,6 +247,7 @@ + diff --git a/windows/vs2015/yara/yara.vcxproj b/windows/vs2015/yara/yara.vcxproj index d138ee82..50d212a0 100644 --- a/windows/vs2015/yara/yara.vcxproj +++ b/windows/vs2015/yara/yara.vcxproj @@ -94,7 +94,7 @@ _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) false - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -113,7 +113,7 @@ _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) false - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -133,7 +133,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -158,7 +158,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL diff --git a/windows/vs2015/yarac/yarac.vcxproj b/windows/vs2015/yarac/yarac.vcxproj index 9e95924e..4b364779 100644 --- a/windows/vs2015/yarac/yarac.vcxproj +++ b/windows/vs2015/yarac/yarac.vcxproj @@ -94,7 +94,7 @@ _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) false - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -113,7 +113,7 @@ Disabled _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -133,7 +133,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -155,7 +155,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL diff --git a/windows/vs2017/libyara/libyara.vcxproj b/windows/vs2017/libyara/libyara.vcxproj index 02d7430d..bb697e3f 100644 --- a/windows/vs2017/libyara/libyara.vcxproj +++ b/windows/vs2017/libyara/libyara.vcxproj @@ -91,7 +91,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB;PROFILING_ENABLED ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x86.1.1.0\include;..\packages\YARA.OpenSSL.x86.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) ProgramDatabase @@ -118,7 +118,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIBC;PROFILING_ENABLED ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x64.1.1.0\include;..\packages\YARA.OpenSSL.x64.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) ProgramDatabase false @@ -145,7 +145,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x86.1.1.0\include;..\packages\YARA.OpenSSL.x86.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) MultiThreadedDLL @@ -170,7 +170,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x64.1.1.0\include;..\packages\YARA.OpenSSL.x64.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) false @@ -197,7 +197,7 @@ _CRT_SECURE_NO_WARNINGS;CUCKOO_MODULE;HASH_MODULE;DOTNET_MODULE;HAVE_LIBCRYPTO;USE_WINDOWS_PROC;YR_BUILDING_STATIC_LIB;NDEBUG=1 ..\..\..\libyara;..\..\..\libyara\include;..\..\..;..\packages\YARA.Jansson.x64.1.1.0\include;..\packages\YARA.OpenSSL.x64.1.1.0\include;%(AdditionalIncludeDirectories) 4005;4273;4090 - CompileAsCpp + CompileAsC $(IntDir)/%(RelativeDir) false @@ -223,6 +223,7 @@ + @@ -247,6 +248,7 @@ + diff --git a/windows/vs2017/yara/yara.vcxproj b/windows/vs2017/yara/yara.vcxproj index f3556714..f096830c 100644 --- a/windows/vs2017/yara/yara.vcxproj +++ b/windows/vs2017/yara/yara.vcxproj @@ -95,7 +95,7 @@ _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) false - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -114,7 +114,7 @@ _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) false - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -134,7 +134,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -159,7 +159,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL diff --git a/windows/vs2017/yarac/yarac.vcxproj b/windows/vs2017/yarac/yarac.vcxproj index 1dbae232..1e0b9765 100644 --- a/windows/vs2017/yarac/yarac.vcxproj +++ b/windows/vs2017/yarac/yarac.vcxproj @@ -95,7 +95,7 @@ _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) false - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -114,7 +114,7 @@ Disabled _DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -134,7 +134,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL @@ -156,7 +156,7 @@ true NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) ..\..\..\libyara\include;..\..\..;%(AdditionalIncludeDirectories) - CompileAsCpp + CompileAsC ProgramDatabase MultiThreadedDLL diff --git a/yara.c b/yara.c index cf3ebde4..9d0575ff 100644 --- a/yara.c +++ b/yara.c @@ -218,7 +218,7 @@ args_option_t options[] = "print tags"), OPT_BOOLEAN('r', "recursive", &recursive_search, - "recursively search directories"), + "recursively search directories (follows symlinks)"), OPT_INTEGER('k', "stack-size", &stack_size, "set maximum stack size (default=16384)", "SLOTS"), @@ -419,7 +419,7 @@ static void scan_dir( snprintf(full_path, sizeof(full_path), "%s/%s", dir, de->d_name); - int err = lstat(full_path, &st); + int err = stat(full_path, &st); if (err == 0) { @@ -429,7 +429,6 @@ static void scan_dir( } else if(recursive && S_ISDIR(st.st_mode) && - !S_ISLNK(st.st_mode) && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) { @@ -654,11 +653,11 @@ static void print_rules_stats( printf( "number of rules : %d\n", - stats.rules); + stats.num_rules); printf( "number of strings : %d\n", - stats.strings); + stats.num_strings); printf( "number of AC matches : %d\n", @@ -675,12 +674,13 @@ static void print_rules_stats( printf("match list length percentiles\n"); - for (int i = 0; i <= 100; i++) + for (int i = 100; i >= 0; i--) printf(" %3d: %d\n", i, stats.ac_match_list_length_pctls[i]); } static int handle_message( + YR_SCAN_CONTEXT* context, int message, YR_RULE* rule, void* data) @@ -798,7 +798,7 @@ static int handle_message( { YR_MATCH* match; - yr_string_matches_foreach(string, match) + yr_string_matches_foreach(context, string, match) { if (show_string_length) printf("0x%" PRIx64 ":%d:%s", @@ -844,6 +844,7 @@ static int handle_message( static int callback( + YR_SCAN_CONTEXT* context, int message, void* message_data, void* user_data) @@ -856,7 +857,8 @@ static int callback( { case CALLBACK_MSG_RULE_MATCHING: case CALLBACK_MSG_RULE_NOT_MATCHING: - return handle_message(message, (YR_RULE*) message_data, user_data); + return handle_message( + context, message, (YR_RULE*) message_data, user_data); case CALLBACK_MSG_IMPORT_MODULE: @@ -1328,17 +1330,16 @@ int main( if (print_count_only) printf("%d\n", user_data.current_count); + + #ifdef PROFILING_ENABLED + yr_scanner_print_profiling_info(scanner); + #endif } result = EXIT_SUCCESS; _exit: - #ifdef PROFILING_ENABLED - if (rules != NULL) - yr_rules_print_profiling_info(rules); - #endif - unload_modules_data(); if (scanner != NULL) diff --git a/yara.man b/yara.man index c0ce6119..b18be213 100644 --- a/yara.man +++ b/yara.man @@ -78,7 +78,7 @@ Print length of strings found in the file. Print the tags associated to the rule. .TP .B \-r " --recursive" -Scan files in directories recursively. +Scan files in directories recursively. It follows symlinks. .TP .BI \-k " slots" " --stack-size=" slots Set maximum stack size to the specified number of