VirusTotal/yara
1
0
Fork 0
mirror of https://github.com/VirusTotal/yara.git synced 2026-10-12 02:10:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v4'

Browse Source
This commit is contained in:
plusvic
2020-03-13 10:06:27 +01:00
parent ece08f59d1 429fb69efa
commit cc72b1aff2
120 changed files with 9365 additions and 6614 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/main.yml
+25
View File
@@ -0,0 +1,25 @@
name: CIFuzz
on: [pull_request]
jobs:
Fuzzing:
runs-on: ubuntu-latest
steps:
- name: Build Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: 'yara'
dry-run: true
- name: Run Fuzzers
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
oss-fuzz-project-name: 'yara'
fuzz-seconds: 600
dry-run: true
- name: Upload Crash
uses: actions/upload-artifact@v1
if: failure()
with:
name: artifacts
path: ./out/artifacts
.travis.yml
+4 -3
View File
@@ -77,10 +77,11 @@ matrix:
- os: linux
dist: xenial
install: |
wget https://github.com/bazelbuild/bazel/releases/download/0.29.0/bazel_0.29.0-linux-x86_64.deb
sudo dpkg -i bazel_0.29.0-linux-x86_64.deb
wget https://github.com/bazelbuild/bazel/releases/download/2.1.0/bazel_2.1.0-linux-x86_64.deb
sudo dpkg -i bazel_2.1.0-linux-x86_64.deb
before_script: bazel info
script: bazel test //tests/...
script: |
bazel test @jansson//... //tests/...
# Build for OS X
- os: osx
BUILD.bazel
+53 -196
View File
@@ -25,222 +25,82 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
YARA_CONFIG_OPTS = [
"-DBORINGSSL",
"-DHAVE_CLOCK_GETTIME=1",
#"-DHAVE_COMMONCRYPTO_COMMONCRYPTO_H",
"-DHAVE_DLFCN_H=1",
"-DHAVE_INTTYPES_H=1",
"-DHAVE_LIBCRYPTO=1",
"-DHAVE_LIBM=1",
"-DHAVE_MEMMEM=1",
"-DHAVE_MEMORY_H=1",
#"-DHAVE__MKGMTIME",
"-DHAVE_OPENSSL_ASN1_H=1",
"-DHAVE_OPENSSL_BIO_H=1",
"-DHAVE_OPENSSL_CRYPTO_H=1",
"-DHAVE_OPENSSL_MD5_H=1",
"-DHAVE_OPENSSL_PKCS7_H=1",
"-DHAVE_OPENSSL_SAFESTACK_H=1",
"-DHAVE_OPENSSL_SHA_H=1",
"-DHAVE_OPENSSL_X509_H=1",
#"-DHAVE_PTHREAD",
"-DHAVE_STDBOOL_H=1",
"-DHAVE_STDINT_H=1",
"-DHAVE_STDLIB_H=1",
"-DHAVE_STRING_H=1",
"-DHAVE_STRINGS_H=1",
"-DHAVE_SYS_STAT_H=1",
"-DHAVE_SYS_TYPES_H=1",
"-DHAVE_TIMEGM=1",
"-DHAVE_UNISTD_H=1",
]
# Bazel (http://bazel.io/) WORKSPACE file for YARA.
YARA_MODULE_OPTS = [
"-DDEX_MODULE",
"-DDOTNET_MODULE",
"-DHASH_MODULE",
"-DMACHO_MODULE",
]
YARA_COPTS = YARA_MODULE_OPTS + select({
"@bazel_tools//src/conditions:darwin": [
"-DUSE_MACH_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
"-DHAVE_STRLCAT=1",
"-DHAVE_STRLCPY=1",
],
"@bazel_tools//src/conditions:freebsd": [
"-DUSE_FREEBSD_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
"-DHAVE_STRLCAT=1",
"-DHAVE_STRLCPY=1",
],
"@bazel_tools//src/conditions:linux_aarch64": [
"-DUSE_LINUX_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
],
"@bazel_tools//src/conditions:linux_x86_64": [
"-DUSE_LINUX_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
],
"@bazel_tools//src/conditions:windows": [
"-DUSE_WINDOWS_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
],
"//conditions:default": ["-DUSE_NO_PROC"],
}) + select({
":yara_profiling_enabled": ["-DPROFILING_ENABLED"],
"//conditions:default": [],
}) + [
"-D_GNU_SOURCE",
# Tell the compiler we want the C99 standard.
"-std=c99",
# Additional include paths.
"-Ilibyara",
"-Ilibyara/modules",
] + YARA_CONFIG_OPTS
# Configuration setting for enabling profiling. Usage:
# bazel build ... --define yara_profiling=enabled
config_setting(
name = "yara_profiling_enabled",
values = {"define": "yara_profiling=enabled"},
)
load("@com_github_virustotal_yara//:bazel/yara.bzl", "yara_library")
# Just YARA's error codes. This is useful for sandboxing as it avoids pulling
# in the whole library.
cc_library(
name = "yara_errors",
hdrs = ["libyara/include/yara/error.h"],
copts = YARA_COPTS,
visibility = ["//visibility:public"],
)
# YARA library to be linked against.
cc_library(
# Configuration setting for enabling profiling. Usage:
# bazel build ... --define yara_profiling=enabled
config_setting(
name = "profiling_enabled",
values = {"define": "yara_profiling=enabled"},
)
# Configuration setting for using BoringSSL instead of OpenSSL. Notice that
# some features related to PE signatures in the "pe" module will be disabled
# because of missing functionality in BoringSSL. Usage:
# bazel build ... --define yara_crypto_library=boringssl
config_setting(
name = "crypto_library_boringssl",
values = {"define": "yara_crypto_library=boringssl"},
)
yara_library(
name = "libyara",
srcs = [
"libyara/ahocorasick.c",
"libyara/arena.c",
"libyara/atoms.c",
"libyara/bitmask.c",
"libyara/compiler.c",
"libyara/crypto.h",
"libyara/endian.c",
"libyara/exception.h",
"libyara/exec.c",
"libyara/exefiles.c",
"libyara/filemap.c",
"libyara/grammar.c",
"libyara/hash.c",
"libyara/hex_grammar.c",
"libyara/hex_grammar.h",
"libyara/hex_lexer.c",
"libyara/include/yara.h",
"libyara/include/yara/ahocorasick.h",
"libyara/include/yara/arena.h",
"libyara/include/yara/atoms.h",
"libyara/include/yara/bitmask.h",
"libyara/include/yara/compiler.h",
"libyara/include/yara/dex.h",
"libyara/include/yara/dotnet.h",
"libyara/include/yara/elf.h",
"libyara/include/yara/endian.h",
"libyara/include/yara/error.h",
"libyara/include/yara/exec.h",
"libyara/include/yara/exefiles.h",
"libyara/include/yara/filemap.h",
"libyara/include/yara/globals.h",
"libyara/include/yara/hash.h",
"libyara/include/yara/hex_lexer.h",
"libyara/include/yara/integers.h",
"libyara/include/yara/lexer.h",
"libyara/include/yara/libyara.h",
"libyara/include/yara/limits.h",
"libyara/include/yara/macho.h",
"libyara/include/yara/mem.h",
"libyara/include/yara/modules.h",
"libyara/include/yara/object.h",
"libyara/include/yara/parser.h",
"libyara/include/yara/pe.h",
"libyara/include/yara/pe_utils.h",
"libyara/include/yara/proc.h",
"libyara/include/yara/re.h",
"libyara/include/yara/re_lexer.h",
"libyara/include/yara/rules.h",
"libyara/include/yara/scan.h",
"libyara/include/yara/scanner.h",
"libyara/include/yara/sizedstr.h",
"libyara/include/yara/stack.h",
"libyara/include/yara/stopwatch.h",
"libyara/include/yara/stream.h",
"libyara/include/yara/strutils.h",
"libyara/include/yara/threading.h",
"libyara/include/yara/types.h",
"libyara/include/yara/utils.h",
"libyara/lexer.c",
"libyara/libyara.c",
"libyara/mem.c",
"libyara/modules.c",
#"libyara/modules/cuckoo/cuckoo.c", # TODO(cblichmann): Add Jansson JSON depenency
defines = select({
":profiling_enabled": ["PROFILING_ENABLED"],
"//conditions:default": [],
}) + select({
":crypto_library_boringssl": ["BORINGSSL"],
"//conditions:default": [],
}),
modules = [
"cuckoo",
"dex",
"dotnet",
"elf",
"hash",
"macho",
"magic",
"math",
"pe",
"tests",
"time",
],
modules_srcs = [
"libyara/modules/cuckoo/cuckoo.c",
"libyara/modules/dex/dex.c",
#"libyara/modules/demo/demo.c", # Disabled
"libyara/modules/dotnet/dotnet.c",
"libyara/modules/elf/elf.c",
"libyara/modules/hash/hash.c",
"libyara/modules/magic/magic.c",
"libyara/modules/macho/macho.c",
#"libyara/modules/magic/magic.c", # TODO(cblichmann): Add libmagic dependency
"libyara/modules/math/math.c",
"libyara/modules/pe/pe.c",
"libyara/modules/pe/pe_utils.c",
"libyara/modules/tests/tests.c",
"libyara/modules/time/time.c",
"libyara/object.c",
"libyara/parser.c",
"libyara/proc.c",
"libyara/proc/freebsd.c",
"libyara/proc/linux.c",
"libyara/proc/mach.c",
"libyara/proc/none.c",
"libyara/proc/openbsd.c",
"libyara/proc/windows.c",
"libyara/re.c",
"libyara/re_grammar.c",
"libyara/re_grammar.h",
"libyara/re_lexer.c",
"libyara/rules.c",
"libyara/scan.c",
"libyara/scanner.c",
"libyara/sizedstr.c",
"libyara/stack.c",
"libyara/stopwatch.c",
"libyara/stream.c",
"libyara/strutils.c",
"libyara/threading.c",
],
hdrs = [
"libyara/include/yara.h",
"libyara/include/yara/pe.h",
"libyara/include/yara/proc.h",
"libyara/include/yara/rules.h",
],
copts = YARA_COPTS,
includes = [
"libyara/modules",
"libyara/include",
"libyara",
],
textual_hdrs = [
"libyara/grammar.h",
"libyara/hex_grammar.y",
"libyara/modules/module_list",
"libyara/re_grammar.y",
],
visibility = ["//visibility:public"],
deps = ["@boringssl//:crypto"],
crypto_libs = select({
":crypto_library_boringssl": ["@boringssl//:crypto"],
"//conditions:default": ["@openssl//:crypto"],
}),
deps = [
"@jansson//:jansson",
"@magic//:magic",
]
)
# Code shared by the command-line tools.
cc_library(
name = "cli_shared",
@@ -253,7 +113,6 @@ cc_library(
"common.h",
"threading.h",
],
copts = YARA_COPTS,
deps = [":libyara"],
)
@@ -261,7 +120,6 @@ cc_library(
cc_binary(
name = "yara",
srcs = ["yara.c"],
copts = YARA_COPTS,
visibility = ["//visibility:public"],
deps = [
":cli_shared",
@@ -269,11 +127,10 @@ cc_binary(
],
)
# The YARA compiler
# YARA compiler
cc_binary(
name = "yarac",
srcs = ["yarac.c"],
copts = YARA_COPTS,
visibility = ["//visibility:public"],
deps = [
":cli_shared",
Makefile.am
+5 -1
View File
@@ -29,6 +29,8 @@ yarac_SOURCES = args.c args.h common.h yarac.c
yarac_LDADD = -Llibyara/.libs -lyara
test_alignment_SOURCES = tests/test-alignment.c
test_arena_SOURCES = tests/test-arena.c
test_arena_LDADD = libyara/.libs/libyara.a
test_atoms_SOURCES = tests/test-atoms.c tests/util.c libyara/atoms.c
test_atoms_LDADD = libyara/.libs/libyara.a
test_rules_SOURCES = tests/test-rules.c tests/util.c
@@ -52,7 +54,9 @@ test_re_split_LDADD = libyara/.libs/libyara.a
TESTS = $(check_PROGRAMS)
TESTS_ENVIRONMENT = TOP_SRCDIR=$(top_srcdir)
check_PROGRAMS = test-alignment \
check_PROGRAMS = \
test-arena \
test-alignment \
test-atoms \
test-api \
test-rules \
README.md
+2
View File
@@ -63,6 +63,7 @@ awesome list of [YARA-related stuff](https://github.com/InQuest/awesome-yara).
* [Blueliv](http://www.blueliv.com)
* [Cofense](https://cofense.com)
* [Conix](http://www.conix.fr)
* [CounterCraft](https://www.countercraft.eu)
* [CrowdStrike FMS](https://github.com/CrowdStrike/CrowdFMS)
* [Cuckoo Sandbox](https://github.com/cuckoosandbox/cuckoo)
* [Cyber Triage](http://www.cybertriage.com)
@@ -105,6 +106,7 @@ awesome list of [YARA-related stuff](https://github.com/InQuest/awesome-yara).
* [RSA ECAT](http://www.emc.com/security/rsa-ecat.htm)
* [Scanii](https://scanii.com)
* [SecondWrite](https://www.secondwrite.com)
* [SonicWall](https://www.sonicwall.com/)
* [SpamStopsHere](https://www.spamstopshere.com)
* [stoQ](http://stoq.punchcyber.com)
* [Symantec](http://www.symantec.com)
WORKSPACE → WORKSPACE.bazel
+11 -29
View File
@@ -25,41 +25,23 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
# Bazel (http://bazel.io/) WORKSPACE file for YARA.
# BoringSSL, see
# https://boringssl.googlesource.com/boringssl/+/master/INCORPORATING.md#bazel
git_repository(
name = "boringssl",
commit = "095d78b14f91cc9a910408eaae84a3bdafc54da9", # 2019-06-05
remote = "https://boringssl.googlesource.com/boringssl",
shallow_since = "1559759280 +0000",
)
workspace(name = "com_github_virustotal_yara")
# Sandboxed API
git_repository(
name = "com_google_sandboxed_api",
commit = "2301e05097818734f59b881d7fbe1624c17fc840", # 2019-07-08
remote = "https://github.com/google/sandboxed-api.git",
shallow_since = "1562590596 -0700",
load("//:bazel/yara_deps.bzl", "yara_deps")
yara_deps()
load(
"@rules_proto//proto:repositories.bzl",
"rules_proto_dependencies", "rules_proto_toolchains"
)
rules_proto_dependencies()
rules_proto_toolchains()
load(
"@com_google_sandboxed_api//sandboxed_api/bazel:sapi_deps.bzl",
"sapi_deps",
)
sapi_deps()
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
protobuf_deps()
# GoogleTest/GoogleMock for testing the sandbox
http_archive(
name = "com_google_googletest",
sha256 = "baed63b97595c32667694de0a434f8f23da59609c4a44f3360ba94b0abd5c583",
strip_prefix = "googletest-8ffb7e5c88b20a297a2e786c480556467496463b",
urls = ["https://github.com/google/googletest/archive/8ffb7e5c88b20a297a2e786c480556467496463b.zip"], # 2019-05-30
)
bazel/jansson.BUILD
+82
View File
@@ -0,0 +1,82 @@
# Copyright (c) 2020. The YARA Authors. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Bazel (http://bazel.io/) BUILD file for the jansson library.
# Rule for generating header files required during the build. These header files
# are normally generated by ./configure, but we just create empty files so that
# #include statements can find them.
genrule(
name = "config_files",
outs = [
"jansson_config.h",
"jansson_private_config.h",
],
cmd = """
touch $(location jansson_config.h)
touch $(location jansson_private_config.h)
""",
)
cc_library(
name = "jansson",
hdrs = glob(["src/*.h"]) + [
"jansson_config.h",
"jansson_private_config.h",
],
srcs = glob(["src/*.c"]),
includes = [".", "src"],
# These defines are usually in jansson_config.h but we define them here,
# as jansson_config.h is just an empty file.
defines = [
"HAVE_STDINT_H=1",
"JSON_INLINE=inline",
"JSON_PARSER_MAX_DEPTH=2048",
],
copts = ["-Wno-unused-function"],
visibility = ["//visibility:public"],
)
load("@com_github_virustotal_yara//:bazel/jansson.bzl", "jansson_api_test")
jansson_api_test("test_array")
jansson_api_test("test_chaos")
jansson_api_test("test_copy")
jansson_api_test("test_dump")
jansson_api_test("test_dump_callback")
jansson_api_test("test_equal")
jansson_api_test("test_load")
jansson_api_test("test_load_callback")
jansson_api_test("test_loadb")
jansson_api_test("test_memory_funcs")
jansson_api_test("test_number")
jansson_api_test("test_object")
jansson_api_test("test_pack")
jansson_api_test("test_simple")
jansson_api_test("test_sprintf")
jansson_api_test("test_unpack")
bazel/jansson.bzl
+36
View File
@@ -0,0 +1,36 @@
# Copyright (c) 2020. The YARA Authors. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
def jansson_api_test(name):
native.cc_test(
name = name,
srcs = [
"test/suites/api/util.h",
"test/suites/api/" + name + ".c",
],
deps = [":jansson"]
)
bazel/magic.BUILD
+139
View File
@@ -0,0 +1,139 @@
# Copyright (c) 2020. The YARA Authors. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Bazel (http://bazel.io/) BUILD file for the libmagic library.
# Rule for creating magic.h from magic.h.in. The two files are identical, except
# for magic.h.in not having the actual version number. Instead it has a X.YY
# placeholder that is replaced with the version number (i.e: 538) during build
# time. When this library is updated the version number in this rule must be
# updated accordingly.
genrule(
name = "magic_h",
srcs = ["src/magic.h.in"],
outs = ["src/magic.h"],
cmd = """
sed -e 's/X.YY/538/' < $(location src/magic.h.in) > $(location src/magic.h)
""",
)
MAGIC_COPTS = [
# The VERSION macro usually contains the actual version (e.g: "5.38") but
# any arbitrary string will work. We simply use "BUILT_BY_YARA".
"-DVERSION=\\\"BUILT_BY_YARA\\\"",
] + select({
"@bazel_tools//src/conditions:darwin": [
"-DHAVE_FORK=1",
"-DHAVE_INTTYPES_H=1",
"-DHAVE_STDINT_H=1",
"-DHAVE_STRLCAT=1",
"-DHAVE_STRLCPY=1",
"-DHAVE_UNISTD_H=1",
],
"@bazel_tools//src/conditions:freebsd": [
"-DHAVE_FORK=1",
"-DHAVE_INTTYPES_H=1",
"-DHAVE_STDINT_H=1",
"-DHAVE_STRLCAT=1",
"-DHAVE_STRLCPY=1",
"-DHAVE_UNISTD_H=1",
],
"@bazel_tools//src/conditions:linux_aarch64": [
"-DHAVE_FORK=1",
"-DHAVE_INTTYPES_H=1",
"-DHAVE_STDINT_H=1",
"-DHAVE_UNISTD_H=1",
],
"@bazel_tools//src/conditions:linux_x86_64": [
"-DHAVE_FORK=1",
"-DHAVE_INTTYPES_H=1",
"-DHAVE_STDINT_H=1",
"-DHAVE_UNISTD_H=1",
],
"@bazel_tools//src/conditions:windows": [
]
})
cc_library(
name = "magic",
hdrs = ["src/magic.h"],
srcs = glob(["src/*.h"]) + [
"src/apprentice.c",
"src/apptype.c",
"src/ascmagic.c",
"src/asctime_r.c",
"src/asprintf.c",
"src/buffer.c",
"src/cdf.c",
"src/cdf_time.c",
"src/compress.c",
"src/ctime_r.c",
"src/der.c",
"src/dprintf.c",
"src/encoding.c",
"src/fmtcheck.c",
"src/fsmagic.c",
"src/funcs.c",
"src/getline.c",
"src/getopt_long.c",
"src/gmtime_r.c",
"src/is_csv.c",
"src/is_json.c",
"src/is_tar.c",
"src/localtime_r.c",
"src/magic.c",
"src/pread.c",
"src/print.c",
"src/readcdf.c",
"src/readelf.c",
"src/seccomp.c",
"src/softmagic.c",
"src/strcasestr.c",
"src/teststrchr.c",
"src/vasprintf.c",
] + select({
"@bazel_tools//src/conditions:darwin": [
],
"@bazel_tools//src/conditions:freebsd": [
],
"@bazel_tools//src/conditions:linux_aarch64": [
"src/strlcat.c",
"src/strlcpy.c",
],
"@bazel_tools//src/conditions:linux_x86_64": [
"src/strlcat.c",
"src/strlcpy.c",
],
"@bazel_tools//src/conditions:windows": [
"src/strlcat.c",
"src/strlcpy.c",
]}),
includes = [".", "src"],
copts = MAGIC_COPTS,
visibility = ["//visibility:public"],
)
bazel/openssl.BUILD
+77
View File
@@ -0,0 +1,77 @@
# Copyright (c) 2020. The YARA Authors. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Bazel (http://bazel.io/) BUILD file for openssl library which is a dependency
# for YARA.
config_setting(
name = "darwin",
values = {
"cpu": "darwin_x86_64",
},
)
cc_library(
name = "crypto",
hdrs = glob(["include/openssl/*.h"]) + ["include/openssl/opensslconf.h"],
srcs = ["libcrypto.a"],
includes = ["include"],
linkopts = select({
":darwin": [],
"//conditions:default": ["-lpthread", "-ldl"],
}),
visibility = ["//visibility:public"],
)
cc_library(
name = "ssl",
deps = [":crypto"],
hdrs = glob(["include/openssl/*.h"]) + ["include/openssl/opensslconf.h"],
srcs = ["libssl.a"],
includes = ["include"],
visibility = ["//visibility:public"],
)
genrule(
name = "openssl-build",
srcs = glob(["**/*"], exclude=["bazel-*"]),
outs = [
"libcrypto.a",
"libssl.a",
"include/openssl/opensslconf.h",
],
cmd = """
OPENSSL_ROOT=$$(dirname $(location config))
pushd $$OPENSSL_ROOT
./config
make -j 4
popd
cp $$OPENSSL_ROOT/libcrypto.a $(location libcrypto.a)
cp $$OPENSSL_ROOT/libssl.a $(location libssl.a)
cp $$OPENSSL_ROOT/include/openssl/opensslconf.h $(location include/openssl/opensslconf.h)
""",
)
bazel/yara.bzl
+206
View File
@@ -0,0 +1,206 @@
YARA_CONFIG_OPTS = [
"-DHAVE_CLOCK_GETTIME=1",
#"-DHAVE_COMMONCRYPTO_COMMONCRYPTO_H",
"-DHAVE_LIBCRYPTO=1",
"-DHAVE_MEMMEM=1",
"-DHAVE_STDBOOL_H=1",
# "-DHAVE__MKGMTIME=1",
"-DHAVE_TIMEGM=1",
]
YARA_COPTS = YARA_CONFIG_OPTS + [
"-D_GNU_SOURCE",
# Tell the compiler we want the C99 standard.
"-std=c99",
# Additional include paths.
"-Ilibyara",
"-Ilibyara/modules",
] + select({
"@bazel_tools//src/conditions:darwin": [
"-DUSE_MACH_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
"-DHAVE_STRLCAT=1",
"-DHAVE_STRLCPY=1",
],
"@bazel_tools//src/conditions:freebsd": [
"-DUSE_FREEBSD_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
"-DHAVE_STRLCAT=1",
"-DHAVE_STRLCPY=1",
],
"@bazel_tools//src/conditions:linux_aarch64": [
"-DUSE_LINUX_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
],
"@bazel_tools//src/conditions:linux_x86_64": [
"-DUSE_LINUX_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
],
"@bazel_tools//src/conditions:windows": [
"-DUSE_WINDOWS_PROC",
"-DHAVE_SCAN_PROC_IMPL=1",
],
"//conditions:default": ["-DUSE_NO_PROC"],
})
# Define rule for generating the module_list file. This rule has an attribute
# "modules" which is a list of module names that we want in the file.
def _module_list_impl(ctx):
output = ctx.outputs.out
content = '\n'.join(["MODULE(%s)" % m for m in ctx.attr.modules])
ctx.actions.write(output = output, content = content)
module_list = rule(
implementation = _module_list_impl,
attrs = {"modules": attr.string_list()},
outputs = {"out": "libyara/modules/module_list"},
)
def yara_library(name, defines=[], modules=[], modules_srcs=[],
deps=[], crypto_libs=["@openssl//:crypto"]):
"""Macro for generating the YARA library with a specific list of modules.
This macro allows to cherry-pick the modules that you want to build into the
library. For example, for building it with modules pe and elf only, you can
use:
yara_library(
name = "libyara",
modules = [
"cuckoo",
"dex",
"dotnet",
"elf",
],
modules_srcs = [
"libyara/modules/elf/elf.c",
"libyara/modules/pe/pe.c",
"libyara/modules/pe/pe_utils.c",
]
)
The "modules_srcs" list must contain the source files implementing the modules
listed in the "modules" argument.
"""
module_list(
name = "module_list",
modules = modules,
)
native.cc_library(
name = name,
defines = defines + [m.upper() + "_MODULE" for m in modules],
srcs = modules_srcs + [
"libyara/ahocorasick.c",
"libyara/arena.c",
"libyara/atoms.c",
"libyara/base64.c",
"libyara/bitmask.c",
"libyara/compiler.c",
"libyara/crypto.h",
"libyara/endian.c",
"libyara/exception.h",
"libyara/exec.c",
"libyara/exefiles.c",
"libyara/filemap.c",
"libyara/grammar.c",
"libyara/hash.c",
"libyara/hex_grammar.c",
"libyara/hex_grammar.h",
"libyara/hex_lexer.c",
"libyara/include/yara.h",
"libyara/include/yara/ahocorasick.h",
"libyara/include/yara/arena.h",
"libyara/include/yara/atoms.h",
"libyara/include/yara/base64.h",
"libyara/include/yara/bitmask.h",
"libyara/include/yara/compiler.h",
"libyara/include/yara/dex.h",
"libyara/include/yara/dotnet.h",
"libyara/include/yara/elf.h",
"libyara/include/yara/endian.h",
"libyara/include/yara/error.h",
"libyara/include/yara/exec.h",
"libyara/include/yara/exefiles.h",
"libyara/include/yara/filemap.h",
"libyara/include/yara/globals.h",
"libyara/include/yara/hash.h",
"libyara/include/yara/hex_lexer.h",
"libyara/include/yara/integers.h",
"libyara/include/yara/lexer.h",
"libyara/include/yara/libyara.h",
"libyara/include/yara/limits.h",
"libyara/include/yara/macho.h",
"libyara/include/yara/mem.h",
"libyara/include/yara/modules.h",
"libyara/include/yara/notebook.h",
"libyara/include/yara/object.h",
"libyara/include/yara/parser.h",
"libyara/include/yara/pe.h",
"libyara/include/yara/pe_utils.h",
"libyara/include/yara/proc.h",
"libyara/include/yara/re.h",
"libyara/include/yara/re_lexer.h",
"libyara/include/yara/rules.h",
"libyara/include/yara/scan.h",
"libyara/include/yara/scanner.h",
"libyara/include/yara/sizedstr.h",
"libyara/include/yara/stack.h",
"libyara/include/yara/stopwatch.h",
"libyara/include/yara/stream.h",
"libyara/include/yara/strutils.h",
"libyara/include/yara/threading.h",
"libyara/include/yara/types.h",
"libyara/include/yara/utils.h",
"libyara/lexer.c",
"libyara/libyara.c",
"libyara/mem.c",
"libyara/modules.c",
"libyara/notebook.c",
"libyara/object.c",
"libyara/parser.c",
"libyara/proc.c",
"libyara/proc/freebsd.c",
"libyara/proc/linux.c",
"libyara/proc/mach.c",
"libyara/proc/none.c",
"libyara/proc/openbsd.c",
"libyara/proc/windows.c",
"libyara/re.c",
"libyara/re_grammar.c",
"libyara/re_grammar.h",
"libyara/re_lexer.c",
"libyara/rules.c",
"libyara/scan.c",
"libyara/scanner.c",
"libyara/sizedstr.c",
"libyara/stack.c",
"libyara/stopwatch.c",
"libyara/stream.c",
"libyara/strutils.c",
"libyara/threading.c",
],
hdrs = [
"libyara/include/yara.h",
"libyara/include/yara/pe.h",
"libyara/include/yara/proc.h",
"libyara/include/yara/rules.h",
],
copts = YARA_COPTS,
includes = [
"libyara/modules",
"libyara/include",
"libyara",
],
textual_hdrs = [
":module_list",
"libyara/grammar.h",
"libyara/hex_grammar.y",
"libyara/re_grammar.y",
],
deps = deps + crypto_libs,
visibility = ["//visibility:public"],
)
bazel/yara_deps.bzl
+92
View File
@@ -0,0 +1,92 @@
# Copyright (c) 2020. The YARA Authors. All Rights Reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Load dependencies needed to compile YARA as a 3rd-party consumer."""
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
def yara_deps():
"""Loads common dependencies needed to compile YARA."""
if not native.existing_rule("openssl"):
http_archive(
name = "openssl",
url = "https://github.com/openssl/openssl/archive/OpenSSL_1_1_0h.tar.gz",
sha256 = "f56dd7d81ce8d3e395f83285bd700a1098ed5a4cb0a81ce9522e41e6db7e0389",
strip_prefix = "openssl-OpenSSL_1_1_0h",
build_file = "@com_github_virustotal_yara//:bazel/openssl.BUILD",
)
if not native.existing_rule("borinssl"):
git_repository(
name = "boringssl",
commit = "095d78b14f91cc9a910408eaae84a3bdafc54da9", # 2019-06-05
remote = "https://boringssl.googlesource.com/boringssl",
shallow_since = "1559759280 +0000",
)
if not native.existing_rule("jansson"):
http_archive(
name = "jansson",
url = "https://github.com/akheron/jansson/archive/v2.12.tar.gz",
sha256 = "76260d30e9bbd0ef392798525e8cd7fe59a6450c54ca6135672e3cd6a1642941",
strip_prefix = "jansson-2.12",
build_file = "@com_github_virustotal_yara//:bazel/jansson.BUILD",
)
if not native.existing_rule("magic"):
# When updating this dependency to a more recent version, the version
# in the bazel/magic.BUILD must be updated acordingly.
http_archive(
name = "magic",
url = "https://github.com/file/file/archive/FILE5_38.tar.gz",
sha256 = "338ebe8cb536a3f86750b4df62be2d382f6da66afdb0087b36a1a3e14ea4baf8",
strip_prefix = "file-FILE5_38",
build_file = "@com_github_virustotal_yara//:bazel/magic.BUILD",
)
if not native.existing_rule("com_google_sandboxed_api"):
git_repository(
name = "com_google_sandboxed_api",
commit = "2301e05097818734f59b881d7fbe1624c17fc840", # 2019-07-08
remote = "https://github.com/google/sandboxed-api.git",
shallow_since = "1562590596 -0700",
)
if not native.existing_rule("rules_proto"):
http_archive(
name = "rules_proto",
sha256 = "602e7161d9195e50246177e7c55b2f39950a9cf7366f74ed5f22fd45750cd208",
strip_prefix = "rules_proto-97d8af4dc474595af3900dd85cb3a29ad28cc313",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz",
"https://github.com/bazelbuild/rules_proto/archive/97d8af4dc474595af3900dd85cb3a29ad28cc313.tar.gz",
],
)
if not native.existing_rule("com_google_googletest"):
# GoogleTest/GoogleMock for testing the sandbox
http_archive(
name = "com_google_googletest",
sha256 = "baed63b97595c32667694de0a434f8f23da59609c4a44f3360ba94b0abd5c583",
strip_prefix = "googletest-8ffb7e5c88b20a297a2e786c480556467496463b",
urls = ["https://github.com/google/googletest/archive/8ffb7e5c88b20a297a2e786c480556467496463b.zip"], # 2019-05-30
)
cmake-build-debug/CMakeFiles/yara.dir/link.txt
+1
View File
@@ -0,0 +1 @@
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++ -g -Wl,-search_paths_first -Wl,-headerpad_max_install_names CMakeFiles/yara.dir/libyara/modules/cuckoo/cuckoo.c.o CMakeFiles/yara.dir/libyara/modules/demo/demo.c.o CMakeFiles/yara.dir/libyara/modules/dex/dex.c.o CMakeFiles/yara.dir/libyara/modules/dotnet/dotnet.c.o CMakeFiles/yara.dir/libyara/modules/elf/elf.c.o CMakeFiles/yara.dir/libyara/modules/hash/hash.c.o CMakeFiles/yara.dir/libyara/modules/macho/macho.c.o CMakeFiles/yara.dir/libyara/modules/magic/magic.c.o CMakeFiles/yara.dir/libyara/modules/math/math.c.o CMakeFiles/yara.dir/libyara/modules/pb_tests/pb_tests.c.o CMakeFiles/yara.dir/libyara/modules/pb_tests/pb_tests.pb-c.c.o CMakeFiles/yara.dir/libyara/modules/pe/pe.c.o CMakeFiles/yara.dir/libyara/modules/pe/pe_utils.c.o CMakeFiles/yara.dir/libyara/modules/tests/tests.c.o CMakeFiles/yara.dir/libyara/modules/time/time.c.o CMakeFiles/yara.dir/libyara/proc/freebsd.c.o CMakeFiles/yara.dir/libyara/proc/linux.c.o CMakeFiles/yara.dir/libyara/proc/mach.c.o CMakeFiles/yara.dir/libyara/proc/none.c.o CMakeFiles/yara.dir/libyara/proc/openbsd.c.o CMakeFiles/yara.dir/libyara/proc/windows.c.o CMakeFiles/yara.dir/libyara/ahocorasick.c.o CMakeFiles/yara.dir/libyara/arena.c.o CMakeFiles/yara.dir/libyara/atoms.c.o CMakeFiles/yara.dir/libyara/bitmask.c.o CMakeFiles/yara.dir/libyara/compiler.c.o CMakeFiles/yara.dir/libyara/endian.c.o CMakeFiles/yara.dir/libyara/exec.c.o CMakeFiles/yara.dir/libyara/exefiles.c.o CMakeFiles/yara.dir/libyara/filemap.c.o CMakeFiles/yara.dir/libyara/grammar.c.o CMakeFiles/yara.dir/libyara/hash.c.o CMakeFiles/yara.dir/libyara/hex_grammar.c.o CMakeFiles/yara.dir/libyara/hex_lexer.c.o CMakeFiles/yara.dir/libyara/lexer.c.o CMakeFiles/yara.dir/libyara/libyara.c.o CMakeFiles/yara.dir/libyara/mem.c.o CMakeFiles/yara.dir/libyara/modules.c.o CMakeFiles/yara.dir/libyara/object.c.o CMakeFiles/yara.dir/libyara/parser.c.o CMakeFiles/yara.dir/libyara/proc.c.o CMakeFiles/yara.dir/libyara/re.c.o CMakeFiles/yara.dir/libyara/re_grammar.c.o CMakeFiles/yara.dir/libyara/re_lexer.c.o CMakeFiles/yara.dir/libyara/rules.c.o CMakeFiles/yara.dir/libyara/scan.c.o CMakeFiles/yara.dir/libyara/scanner.c.o CMakeFiles/yara.dir/libyara/sizedstr.c.o CMakeFiles/yara.dir/libyara/stack.c.o CMakeFiles/yara.dir/libyara/stopwatch.c.o CMakeFiles/yara.dir/libyara/stream.c.o CMakeFiles/yara.dir/libyara/strutils.c.o CMakeFiles/yara.dir/libyara/threading.c.o CMakeFiles/yara.dir/tests/oss-fuzz/dex_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/dotnet_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/elf_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/macho_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/pe_fuzzer.cc.o CMakeFiles/yara.dir/tests/oss-fuzz/rules_fuzzer.cc.o CMakeFiles/yara.dir/tests/test-alignment.c.o CMakeFiles/yara.dir/tests/test-api.c.o CMakeFiles/yara.dir/tests/test-atoms.c.o CMakeFiles/yara.dir/tests/test-bitmask.c.o CMakeFiles/yara.dir/tests/test-dex.c.o CMakeFiles/yara.dir/tests/test-dotnet.c.o CMakeFiles/yara.dir/tests/test-elf.c.o CMakeFiles/yara.dir/tests/test-exception.c.o CMakeFiles/yara.dir/tests/test-macho.c.o CMakeFiles/yara.dir/tests/test-math.c.o CMakeFiles/yara.dir/tests/test-pb.c.o CMakeFiles/yara.dir/tests/test-pe.c.o CMakeFiles/yara.dir/tests/test-re-split.c.o CMakeFiles/yara.dir/tests/test-rules.c.o CMakeFiles/yara.dir/tests/test-stack.c.o CMakeFiles/yara.dir/tests/test-version.c.o CMakeFiles/yara.dir/tests/util.c.o CMakeFiles/yara.dir/args.c.o CMakeFiles/yara.dir/threading.c.o CMakeFiles/yara.dir/yara.c.o CMakeFiles/yara.dir/yarac.c.o CMakeFiles/yara.dir/libyara/arena2.c.o CMakeFiles/yara.dir/tests/test-arena.c.o -o yara
cmake-build-debug/CMakeFiles/yara.dir/progress.make
+84
View File
@@ -0,0 +1,84 @@
CMAKE_PROGRESS_1 = 1
CMAKE_PROGRESS_2 = 2
CMAKE_PROGRESS_3 = 3
CMAKE_PROGRESS_4 = 4
CMAKE_PROGRESS_5 = 5
CMAKE_PROGRESS_6 = 6
CMAKE_PROGRESS_7 = 7
CMAKE_PROGRESS_8 = 8
CMAKE_PROGRESS_9 = 9
CMAKE_PROGRESS_10 = 10
CMAKE_PROGRESS_11 = 11
CMAKE_PROGRESS_12 = 12
CMAKE_PROGRESS_13 = 13
CMAKE_PROGRESS_14 = 14
CMAKE_PROGRESS_15 = 15
CMAKE_PROGRESS_16 = 16
CMAKE_PROGRESS_17 = 17
CMAKE_PROGRESS_18 = 18
CMAKE_PROGRESS_19 = 19
CMAKE_PROGRESS_20 = 20
CMAKE_PROGRESS_21 = 21
CMAKE_PROGRESS_22 = 22
CMAKE_PROGRESS_23 = 23
CMAKE_PROGRESS_24 = 24
CMAKE_PROGRESS_25 = 25
CMAKE_PROGRESS_26 = 26
CMAKE_PROGRESS_27 = 27
CMAKE_PROGRESS_28 = 28
CMAKE_PROGRESS_29 = 29
CMAKE_PROGRESS_30 = 30
CMAKE_PROGRESS_31 = 31
CMAKE_PROGRESS_32 = 32
CMAKE_PROGRESS_33 = 33
CMAKE_PROGRESS_34 = 34
CMAKE_PROGRESS_35 = 35
CMAKE_PROGRESS_36 = 36
CMAKE_PROGRESS_37 = 37
CMAKE_PROGRESS_38 = 38
CMAKE_PROGRESS_39 = 39
CMAKE_PROGRESS_40 = 40
CMAKE_PROGRESS_41 = 41
CMAKE_PROGRESS_42 = 42
CMAKE_PROGRESS_43 = 43
CMAKE_PROGRESS_44 = 44
CMAKE_PROGRESS_45 = 45
CMAKE_PROGRESS_46 = 46
CMAKE_PROGRESS_47 = 47
CMAKE_PROGRESS_48 = 48
CMAKE_PROGRESS_49 = 49
CMAKE_PROGRESS_50 = 50
CMAKE_PROGRESS_51 = 51
CMAKE_PROGRESS_52 = 52
CMAKE_PROGRESS_53 = 53
CMAKE_PROGRESS_54 = 54
CMAKE_PROGRESS_55 = 55
CMAKE_PROGRESS_56 = 56
CMAKE_PROGRESS_57 = 57
CMAKE_PROGRESS_58 = 58
CMAKE_PROGRESS_59 = 59
CMAKE_PROGRESS_60 = 60
CMAKE_PROGRESS_61 = 61
CMAKE_PROGRESS_62 = 62
CMAKE_PROGRESS_63 = 63
CMAKE_PROGRESS_64 = 64
CMAKE_PROGRESS_65 = 65
CMAKE_PROGRESS_66 = 66
CMAKE_PROGRESS_67 = 67
CMAKE_PROGRESS_68 = 68
CMAKE_PROGRESS_69 = 69
CMAKE_PROGRESS_70 = 70
CMAKE_PROGRESS_71 = 71
CMAKE_PROGRESS_72 = 72
CMAKE_PROGRESS_73 = 73
CMAKE_PROGRESS_74 = 74
CMAKE_PROGRESS_75 = 75
CMAKE_PROGRESS_76 = 76
CMAKE_PROGRESS_77 = 77
CMAKE_PROGRESS_78 = 78
CMAKE_PROGRESS_79 = 79
CMAKE_PROGRESS_80 = 80
CMAKE_PROGRESS_81 = 81
CMAKE_PROGRESS_82 = 82
CMAKE_PROGRESS_83 = 83
common.h
+1 -1
View File
@@ -56,7 +56,7 @@ bool compile_files(
char* colon = NULL;
int errors;
if (access(argv[1], 0) != 0)
if (access(argv[i], 0) != 0)
{
// A file with the name specified by the command-line argument wasn't
// found, it may be because the name is prefixed with a namespace, so
docs/capi.rst
+37 -12
View File
@@ -30,7 +30,7 @@ with :c:func:`yr_compiler_destroy`.
You can use :c:func:`yr_compiler_add_file`, :c:func:`yr_compiler_add_fd`, or
:c:func:`yr_compiler_add_string` to add one or more input sources to be
compiled. Both of these functions receive an optional namespace. Rules added
compiler. Both of these functions receive an optional namespace. Rules added
under the same namespace behave as if they were contained within the same
source file or string, so, rule identifiers must be unique among all the sources
sharing a namespace. If the namespace argument is ``NULL`` the rules are put
@@ -71,11 +71,11 @@ However, if you want to fetch the imported rules from another source (eg: from a
database or remote service), a callback function can be set with
:c:func:`yr_compiler_set_include_callback`.
The callback receives the following parameters:
This callback receives the following parameters:
* ``include_name``: name of the requested file.
* ``calling_rule_filename``: the requesting file name (NULL if not a file).
* ``calling_rule_namespace``: namespace (NULL if undefined).
* ``user_data`` pointer is the same you passed to :c:func:`yr_compiler_set_include_callback`.
* ``user_data`` same pointer passed to :c:func:`yr_compiler_set_include_callback`.
It should return the requested file's content as a null-terminated string. The
memory for this string should be allocated by the callback function. Once it is
@@ -248,6 +248,9 @@ Lastly, the callback function is also called with the
``CALLBACK_MSG_SCAN_FINISHED`` message when the scan is finished. In this case
``message_data`` is ``NULL``.
Notice that you shouldn't call any of the ``yr_rules_scan_XXXX`` functions from
within the callback as those functions are not re-entrant.
Your callback function must return one of the following values::
CALLBACK_CONTINUE
@@ -260,16 +263,19 @@ If it returns ``CALLBACK_CONTINUE`` YARA will continue normally,
``CALLBACK_ERROR`` will abort the scanning too, but the result from
``yr_rules_scan_XXXX`` will be ``ERROR_CALLBACK_ERROR``.
The ``user_data`` argument passed to your callback function is the same you
passed ``yr_rules_scan_XXXX``. This pointer is not touched by YARA, it's just a
way for your program to pass arbitrary data to the callback function.
All ``yr_rules_scan_XXXX`` functions receive a ``flags`` argument and a
``timeout`` argument. The only flag defined at this time is
``SCAN_FLAGS_FAST_MODE``, so you must pass either this flag or a zero value.
The ``timeout`` argument forces the function to return after the specified
number of seconds approximately, with a zero meaning no timeout at all.
All ``yr_rules_scan_XXXX`` functions receive a ``flags`` argument that allows
to tweak some aspects of the scanning process. The supported flags are the following
ones:
``SCAN_FLAGS_FAST_MODE``
``SCAN_FLAGS_NO_TRYCATCH``
``SCAN_FLAGS_REPORT_RULES_MATCHING``
``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING``
The ``SCAN_FLAGS_FAST_MODE`` flag makes the scanning a little faster by avoiding
multiple matches of the same string when not necessary. Once the string was
@@ -278,8 +284,22 @@ single match for the string, even if it appears multiple times in the scanned
data. This flag has the same effect of the ``-f`` command-line option described
in :ref:`command-line`.
Notice that you shouldn't call any of the ``yr_rules_scan_XXXX`` functions from
within the callback as those functions are not re-entrant.
``SCAN_FLAGS_REPORT_RULES_MATCHING`` and ``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING``
control whether the callback is invoked for rules that are matching or for rules
that are not matching respectively. If ``SCAN_FLAGS_REPORT_RULES_MATCHING`` is
specified alone, the callback will be called for matching rules with the
``CALLBACK_MSG_RULE_MATCHING`` message but it won't be called for non-matching
rules. If ``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING`` is specified alone, the opposite
happens, the callback will be called with ``CALLBACK_MSG_RULE_NOT_MATCHING``
messages but not with ``CALLBACK_MSG_RULE_MATCHING`` messages. If both flags
are combined together (the default) the callback will be called for both matching
and non-matching rules. For backward compatibility, if none of these two flags
are specified, the scanner will follow the default behavior.
Additionally, ``yr_rules_scan_XXXX`` functions can receive a ``timeout`` argument
which forces the scan to abort after the specified number of seconds (approximately).
If ``timeout`` is 0 it means no timeout at all.
Using a scanner
---------------
@@ -825,7 +845,12 @@ Functions
.. versionadded:: 3.8.0
Set the flags that will be used by any call to `yr_scanner_scan_xxx`.
Set the flags that will be used by any call to `yr_scanner_scan_xxx`. The supported flags are:
``SCAN_FLAGS_FAST_MODE``: Enable fast scan mode.
``SCAN_FLAGS_NO_TRYCATCH``: Disable exception handling.
``SCAN_FLAGS_REPORT_RULES_MATCHING``: If this
``SCAN_FLAGS_REPORT_RULES_NOT_MATCHING``
.. c:function:: int yr_scanner_define_integer_variable(YR_SCANNER* scanner, const char* identifier, int64_t value)
docs/commandline.rst
+1 -1
View File
@@ -129,7 +129,7 @@ Available options are:
.. option:: -r --recursive
Recursively search for directories.
Recursively search for directories. It follows symlinks.
.. option:: -f --fast-scan
docs/docutils.conf
+2
View File
@@ -0,0 +1,2 @@
[parsers]
smart_quotes: false
docs/gettingstarted.rst
+7 -5
View File
@@ -88,7 +88,8 @@ You can also download and install YARA using the `vcpkg <https://github.com/Micr
./vcpkg integrate install
vcpkg install yara
The YARA port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please `create an issue or pull request <https://github.com/Microsoft/vcpkg/>`_ on the vcpkg repository.
The YARA port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out
of date, please `create an issue or pull request <https://github.com/Microsoft/vcpkg/>`_ on the vcpkg repository.
Installing on Windows
@@ -98,10 +99,11 @@ Compiled binaries for Windows in both 32 and 64 bit flavors can be found in the
link below. Just download the version you want, unzip the archive, and put the
``yara.exe`` and ``yarac.exe`` binaries anywhere in your disk.
To install the ``yara-python`` extension download and execute the installer
corresponding to the version of Python you're using.
`Download Windows binaries <https://github.com/VirusTotal/yara/releases/latest>`_
`Download Windows binaries <https://www.dropbox.com/sh/umip8ndplytwzj1/AADdLRsrpJL1CM1vPVAxc5JZa?dl=0>`_
To install YARA using `Scoop <https://scoop.sh>`_ or `Chocolatey <https://chocolatey.org>`_, simply type
``scoop install yara`` or ``choco install yara``. The integration with both `Scoop` and `Chocolatey` are
not maintained their respective teams, not by the YARA authors.
Installing on Mac OS X with Homebrew
@@ -111,7 +113,7 @@ To install YARA using `Homebrew <https://brew.sh>`_, simply type
``brew install yara``.
Installing yara-python
Installing ``yara-python``
----------------------
If you plan to use YARA from your Python scripts you need to install the
docs/modules/cuckoo.rst
+30
View File
@@ -113,6 +113,13 @@ Reference
Similar to :func:`http_request`, but only takes into account POST
requests.
.. function:: http_user_agent(regexp)
Function returning true if the program sent a HTTP request with a
user agent matching the provided regular expression.
*Example: cuckoo.network.http_user_agent(/MSIE 6\\.0/)*
.. function:: dns_lookup(regexp)
@@ -120,6 +127,29 @@ Reference
request for a domain matching the provided regular expression.
*Example: cuckoo.network.dns_lookup(/evil\\.com/)*
.. function:: host(regexp)
Function returning true if the program contacted an IP address
matching the provided regular expression.
*Example: cuckoo.network.host(/192\\.168\\.1\\.1/)*
.. function:: tcp(regexp, port)
Function returning true if the program contacted an IP address
matching the provided regular expression, over TCP on the provided
port number.
*Example: cuckoo.network.tcp(/192\\.168\\.1\\.1/, 443)*
.. function:: udp(regexp, port)
Function returning true if the program contacted an IP address
matching the provided regular expression, over UDP on the provided
port number.
*Example: cuckoo.network.udp(/192\\.168\\.1\\.1/, 53)*
.. type:: registry
docs/modules/pe.rst
+122 -45
View File
@@ -111,13 +111,6 @@ Reference
Value of IMAGE_FILE_HEADER::PointerToSymbolTable. Used when the PE image has
COFF debug info.
.. c:type:: pointer_to_symbol_table
.. versionadded:: 3.8.0
Value of IMAGE_FILE_HEADER::PointerToSymbolTable. Used when the PE image has
COFF debug info.
.. c:type:: number_of_symbols
.. versionadded:: 3.8.0
@@ -223,63 +216,63 @@ Reference
following constants:
.. c:type:: RELOCS_STRIPPED
Relocation info stripped from file.
.. c:type:: EXECUTABLE_IMAGE
File is executable (i.e. no unresolved external references).
.. c:type:: LINE_NUMS_STRIPPED
Line numbers stripped from file.
.. c:type:: LOCAL_SYMS_STRIPPED
Local symbols stripped from file.
.. c:type:: AGGRESIVE_WS_TRIM
Aggressively trim working set
.. c:type:: LARGE_ADDRESS_AWARE
App can handle >2gb addresses
.. c:type:: BYTES_REVERSED_LO
Bytes of machine word are reversed.
.. c:type:: MACHINE_32BIT
32 bit word machine.
.. c:type:: DEBUG_STRIPPED
Debugging info stripped from file in .DBG file
.. c:type:: REMOVABLE_RUN_FROM_SWAP
If Image is on removable media, copy and run from the swap file.
.. c:type:: NET_RUN_FROM_SWAP
If Image is on Net, copy and run from the swap file.
.. c:type:: SYSTEM
System File.
.. c:type:: DLL
File is a DLL.
.. c:type:: UP_SYSTEM_ONLY
File should only be run on a UP machine
.. c:type:: BYTES_REVERSED_HI
Bytes of machine word are reversed.
*Example: pe.characteristics & pe.DLL*
@@ -818,7 +811,15 @@ Reference
value (0 or 1) if the given *toolid* and optional *version* is present
in an entry.
*Example: pe.rich_signature.toolid(170, 40219) >= 99 and pe.rich_signature.toolid(170, 40219) <= 143*
*Example: pe.rich_signature.toolid(170, 40219) >= 99*
.. c:type:: pdb_path
.. versionadded:: 3.13.0
Path of the PDB file for this PE if present.
* Example: pe.pdb_path == "D:\\workspace\\2018_R9_RelBld\\target\\checkout\\custprof\\Release\\custprof.pdb"
.. c:function:: exports(function_name)
@@ -845,12 +846,76 @@ Reference
*Example: pe.exports(/^AXS@@/)*
.. c:function:: exports_index(function_name)
.. versionadded:: 3.12.0
Function returning the index into the export_details array where the named
function is, undefined otherwise.
*Example: pe.exports_index("CPlApplet")*
.. c:function:: exports_index(ordinal)
.. versionadded:: 3.12.0
Function returning the index into the export_details array where the
exported ordinal is, undefined otherwise.
*Example: pe.exports_index(72)*
.. c:function:: exports_index(/regular_expression/)
.. versionadded:: 3.12.0
Function returning the first index into the export_details array where the
regular expression matches the exported name, undefined otherwise.
*Example: pe.exports_index(/^ERS@@/)*
.. c:type:: number_of_exports
.. versionadded:: 3.6.0
Number of exports in the PE.
.. c:type:: export_details
.. versionadded:: 3.12.0
Structure containing information about the PE's exports.
.. c:member:: offset
Offset where the exported function starts.
.. c:member:: name
Name of the exported function. It will be undefined if the function has
no name.
.. c:member:: forward_name
The name of the function where this export forwards to. It will be
undefined if the export is not a forwarding export.
.. c:member:: ordinal
The ordinal of the exported function, after the ordinal base has been
applied to it.
.. c:type:: dll_name
.. versionadded:: 3.12.0
The name of the DLL, if it exists in the export directory.
.. c:type:: export_timestamp
.. versionadded:: 3.12.0
The timestamp the export data was created..
.. c:type:: number_of_imports
.. versionadded:: 3.6.0
@@ -867,11 +932,17 @@ Reference
.. c:function:: imports(dll_name)
.. versionadded:: 3.5.0
.. versionchanged:: 3.12.0
Function returning true if the PE imports anything from *dll_name*,
or false otherwise. *dll_name* is case insensitive.
Function returning the number of functions from the *dll_name*, in the PE
imports. *dll_name* is case insensitive.
*Example: pe.imports("kernel32.dll")*
Note: Prior to version 3.12.0, this function returned only a boolean value
indicating if the given DLL name was found in the PE imports. This change
is backward compatible, as any number larger than 0 also evaluates as
true.
*Examples: pe.imports("kernel32.dll"), pe.imports("kernel32.dll") == 10*
.. c:function:: imports(dll_name, ordinal)
@@ -885,13 +956,19 @@ Reference
.. c:function:: imports(dll_regexp, function_regexp)
.. versionadded:: 3.8.0
.. versionchanged:: 3.12.0
Function returning true if the PE imports a function name matching
*function_regexp* from a DLL matching *dll_regexp*. *dll_regexp* is case
sensitive unless you use the "/i" modifier in the regexp, as shown in the
example below.
Function returning the number of functions from the PE imports where a
function name matches *function_regexp* and a DLL name matches
*dll_regexp*. Both *dll_regexp* and *function_regexp* are case sensitive
unless you use the "/i" modifier in the regexp, as shown in the example
below.
*Example: pe.imports(/kernel32\.dll/i, /(Read|Write)ProcessMemory/)*
Note: Prior to version 3.12.0, this function returned only a boolean value
indicating if matching import was found or not. This change is backward
compatible, as any number larger than 0 also evaluates as true.
*Example: pe.imports(/kernel32\.dll/i, /(Read|Write)ProcessMemory/) == 2*
.. c:function:: locale(locale_identifier)
docs/writingmodules.rst
+2 -2
View File
@@ -951,8 +951,8 @@ depending on the function's return type. In all cases *x* is a constant,
variable, or expression evaluating to ``char*``, ``int64_t`` or ``double``
respectively.
You can use ``return_string(UNDEFINED)``, ``return_float(UNDEFINED)`` and
``return_integer(UNDEFINED)`` to return undefined values from the function.
You can use ``return_string(YR_UNDEFINED)``, ``return_float(YR_UNDEFINED)`` and
``return_integer(YR_UNDEFINED)`` to return undefined values from the function.
This is useful in many situations, for example if the arguments passed to the
functions don't make sense, or if your module expects a particular file format
and the scanned file is from another format, or in any other case where your
docs/writingrules.rst
+151 -37
View File
@@ -23,50 +23,56 @@ keywords are reserved and cannot be used as an identifier:
.. list-table:: YARA keywords
:widths: 10 10 10 10 10 10 10
:widths: 10 10 10 10 10 10 10 10
* - all
- and
- any
- ascii
- at
- base64
- base64wide
- condition
- contains
* - entrypoint
* - contains
- entrypoint
- false
- filesize
- fullword
- for
- fullword
- global
- in
* - import
- import
* - in
- include
- int8
- int16
- int32
- int8be
- int16be
* - int32be
- matches
- int32
- int32be
- int8
- int8be
* - matches
- meta
- nocase
- not
- or
- of
* - private
- or
- private
- rule
- strings
* - strings
- them
- true
- uint8
- uint16
* - uint32
- uint8be
- uint16be
- uint32
- uint32be
- uint8
* - uint8be
- wide
- xor
-
-
-
-
-
-
Rules are generally composed of two sections: strings definition and condition.
The strings definition section can be omitted if the rule doesn't rely on any
@@ -362,7 +368,7 @@ The following rule will search for every single byte xor applied to the string
$xor_string = "This program cannot" xor
condition:
$xor_string
$xor_string
}
The above rule is logically equivalent to:
@@ -435,6 +441,65 @@ If you want more control over the range of bytes used with the xor modifier use:
The above example will apply the bytes from 0x01 to 0xff, inclusively, to the
string when searching. The general syntax is ``xor(minimum-maximum)``.
base64 strings
^^^^^^^^^^^^^^
The ``base64`` modifier can be used to search for strings that have been base64
encoded. A good explanation of the technique is at:
https://www.leeholmes.com/blog/2019/12/10/searching-for-content-in-base-64-strings-2/
The following rule will search for the three base64 permutations of the string
"This program cannot":
.. code-block:: yara
rule Base64Example1
{
strings:
$a = "This program cannot" base64
condition:
$a
}
This will cause YARA to search for these three permutations:
VGhpcyBwcm9ncmFtIGNhbm5vd
RoaXMgcHJvZ3JhbSBjYW5ub3
UaGlzIHByb2dyYW0gY2Fubm90
The ``base64wide`` modifier works just like the base64 modifier but the results
of the base64 modifier are converted to wide.
The interaction between ``base64`` (or ``base64wide``) and ``wide`` and
``ascii`` is as you might expect. ``wide`` and ``ascii`` are applied to the
string first, and then the ``base64`` and ``base64wide`` modifiers are applied.
At no point is the plaintext of the ``ascii`` or ``wide`` versions of the
strings included in the search. If you want to also include those you can put
them in a secondary string.
The ``base64`` and ``base64wide`` modifiers also support a custom alphabet. For
example:
.. code-block:: yara
rule Base64Example2
{
strings:
$a = "This program cannot" base64("!@#$%^&*(){}[].,|ABCDEFGHIJ\x09LMNOPQRSTUVWXYZabcdefghijklmnopqrstu")
condition:
$a
}
The alphabet must be 64 bytes long.
The ``base64`` and ``base64wide`` modifiers are only supported with text
strings. Using these modifiers with a hexadecimal string or a regular expression
will cause a compiler error. Also, the ``xor`` and ``nocase`` modifiers used in
combination with ``base64`` or ``base64wide`` will cause a compiler error.
Searching for full words
^^^^^^^^^^^^^^^^^^^^^^^^
@@ -611,24 +676,10 @@ the typical Boolean operators ``and``, ``or``, and ``not``, and relational opera
and bitwise operators (``&``, ``|``, ``<<``, ``>>``, ``~``, ``^``) can be used on numerical
expressions.
String identifiers can be also used within a condition, acting as Boolean
variables whose value depends on the presence or not of the associated string
in the file.
.. code-block:: yara
rule Example
{
strings:
$a = "text1"
$b = "text2"
$c = "text3"
$d = "text4"
condition:
($a or $b) and ($c or $d)
}
Integers are always 64-bits long, even the results of functions like `uint8`,
`uint16` and `uint32` are promoted to 64-bits. This is something you must take
into account, specially while using bitwise operators (for example, ~0x01 is not
0xFE but 0xFFFFFFFFFFFFFFFE).
The following table lists the precedence and associativity of all operators. The
table is sorted in descending precedence order, which means that operators listed
@@ -687,6 +738,26 @@ Precedence Operator Description Associativity
========== ======== ========================================= =============
String identifiers can be also used within a condition, acting as Boolean
variables whose value depends on the presence or not of the associated string
in the file.
.. code-block:: yara
rule Example
{
strings:
$a = "text1"
$b = "text2"
$c = "text3"
$d = "text4"
condition:
($a or $b) and ($c or $d)
}
Counting strings
----------------
@@ -1107,6 +1178,49 @@ In summary, the syntax of this operator is:
for expression identifier in indexes : ( boolean_expression )
Iterators
---------
In YARA 3.12 the ``for..of`` operator was improved and now it can be used to
iterate not only over integer enumerations and ranges (e.g: 1,2,3,4 and 1..4),
but also over any kind of iterable data type, like arrays and dictionaries
defined by YARA modules. For example, the following expression is valid in
YARA 3.12:
.. code-block:: yara
for any section in pe.sections : ( section.name == ".text" )
This is equivalent to:
.. code-block:: yara
for any i in (0..pe.number_of_sections-1) : ( pe.sections[i].name == ".text" )
The new syntax is more natural and easy to understand, and is the recommended
way of expressing this type of conditions in newer versions of YARA.
For while iterating dictionaries you must provide to variable names that will
hold the key and value of each entry in the dictionary, for example:
.. code-block:: yara
for any k,v in some_dict : ( k == "foo" and v == "bar" )
In general the ``for..of`` operator has the form:
.. code-block:: yara
for <quantifier> <variables> in <iterable> : ( <some condition using the loop variables> )
Where `<quantifier>` is either `any`, `all` or an expression that evaluates to
the number of items in the iterator that must satisfy the condition, `<variables>`
is a comma-separated list of variable names that holds the values for the
current item (the number of variables depend on the type of `<iterable>`) and
`<iterable>` is something that can be iterated.
.. _referencing-rules:
Referencing other rules
docs/yarapython.rst
+2 -2
View File
@@ -219,7 +219,7 @@ Here is an example:
import yara
def mycallback(data):
print data
print(data)
return yara.CALLBACK_CONTINUE
matches = rules.match('/foo/bar/my_file', callback=mycallback, which_callbacks=yara.CALLBACK_MATCHES)
@@ -259,7 +259,7 @@ Here is an example:
import yara
def modules_callback(data):
print data
print(data)
return yara.CALLBACK_CONTINUE
matches = rules.match('/foo/bar/my_file', modules_callback=modules_callback)
libyara/Makefile.am
+4
View File
@@ -95,6 +95,7 @@ yarainclude_HEADERS = \
include/yara/ahocorasick.h \
include/yara/arena.h \
include/yara/atoms.h \
include/yara/base64.h \
include/yara/bitmask.h \
include/yara/compiler.h \
include/yara/error.h \
@@ -107,6 +108,7 @@ yarainclude_HEADERS = \
include/yara/limits.h \
include/yara/mem.h \
include/yara/modules.h \
include/yara/notebook.h \
include/yara/object.h \
include/yara/parser.h \
include/yara/proc.h \
@@ -158,6 +160,7 @@ libyara_la_SOURCES = \
ahocorasick.c \
arena.c \
atoms.c \
base64.c \
bitmask.c \
compiler.c \
endian.c \
@@ -171,6 +174,7 @@ libyara_la_SOURCES = \
libyara.c \
mem.c \
modules.c \
notebook.c \
object.c \
parser.c \
proc.c \
libyara/ahocorasick.c
+137 -161
View File
@@ -33,12 +33,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <yara/arena.h>
#include <yara/ahocorasick.h>
#include <yara/compiler.h>
#include <yara/error.h>
#include <yara/utils.h>
#include <yara/mem.h>
typedef struct _QUEUE_NODE
{
YR_AC_STATE* value;
@@ -208,7 +208,7 @@ static YR_AC_STATE* _yr_ac_state_create(
new_state->input = input;
new_state->depth = state->depth + 1;
new_state->matches = NULL;
new_state->matches_ref = YR_ARENA_NULL_REF;
new_state->failure = NULL;
new_state->t_table_slot = 0;
new_state->first_child = NULL;
@@ -249,7 +249,8 @@ static int _yr_ac_state_destroy(
//
static int _yr_ac_create_failure_links(
YR_AC_AUTOMATON* automaton)
YR_AC_AUTOMATON* automaton,
YR_ARENA* arena)
{
YR_AC_STATE* current_state;
YR_AC_STATE* failure_state;
@@ -285,22 +286,27 @@ static int _yr_ac_create_failure_links(
while (!_yr_ac_queue_is_empty(&queue))
{
current_state = _yr_ac_queue_pop(&queue);
match = current_state->matches;
match = yr_arena_ref_to_ptr(arena, &current_state->matches_ref);
if (match != NULL)
{
// Find the last match in the list of matches.
while (match->next != NULL)
match = match->next;
if (match->backtrack > 0)
match->next = root_state->matches;
{
match->next = yr_arena_ref_to_ptr(arena, &root_state->matches_ref);
}
}
else
{
current_state->matches = root_state->matches;
// This state doesn't have any matches, its matches will be those
// in the root state, if any.
current_state->matches_ref = root_state->matches_ref;
}
// Iterate over all the states that the current state can transition to.
transition_state = current_state->first_child;
while (transition_state != NULL)
@@ -317,18 +323,18 @@ static int _yr_ac_create_failure_links(
{
transition_state->failure = temp_state;
if (transition_state->matches == NULL)
if (YR_ARENA_IS_NULL_REF(transition_state->matches_ref))
{
transition_state->matches = temp_state->matches;
transition_state->matches_ref = temp_state->matches_ref;
}
else
{
match = transition_state->matches;
match = yr_arena_ref_to_ptr(arena, &transition_state->matches_ref);
while (match != NULL && match->next != NULL)
match = match->next;
match->next = temp_state->matches;
match->next = yr_arena_ref_to_ptr(arena, &temp_state->matches_ref);
}
break;
@@ -425,7 +431,7 @@ static int _yr_ac_optimize_failure_links(
current_state->failure = current_state->failure->failure;
}
// Push childrens of current_state
// Push children of current_state
state = current_state->first_child;
while (state != NULL)
@@ -450,6 +456,7 @@ static int _yr_ac_optimize_failure_links(
static int _yr_ac_find_suitable_transition_table_slot(
YR_AC_AUTOMATON* automaton,
YR_ARENA* arena,
YR_AC_STATE* state,
uint32_t* slot)
{
@@ -486,36 +493,31 @@ static int _yr_ac_find_suitable_transition_table_slot(
if (*slot > automaton->tables_size - 257)
{
size_t t_bytes_size = automaton->tables_size *
sizeof(YR_AC_TRANSITION);
FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory(
arena,
YR_AC_TRANSITION_TABLE,
257 * sizeof(YR_AC_TRANSITION),
NULL));
size_t m_bytes_size = automaton->tables_size *
sizeof(YR_AC_MATCH_TABLE_ENTRY);
FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory(
arena,
YR_AC_STATE_MATCHES_TABLE,
257 * sizeof(uint8_t*),
NULL));
size_t b_bytes_size = YR_BITMASK_SIZE(automaton->tables_size) *
size_t bm_len = YR_BITMASK_SIZE(automaton->tables_size) *
sizeof(YR_BITMASK);
automaton->t_table = (YR_AC_TRANSITION_TABLE) yr_realloc(
automaton->t_table, t_bytes_size * 2);
size_t bm_len_incr = YR_BITMASK_SIZE(257) * sizeof(YR_BITMASK);
automaton->m_table = (YR_AC_MATCH_TABLE) yr_realloc(
automaton->m_table, m_bytes_size * 2);
automaton->bitmask = yr_realloc(automaton->bitmask, bm_len + bm_len_incr);
automaton->bitmask = (YR_BITMASK*) yr_realloc(
automaton->bitmask, b_bytes_size * 2);
if (automaton->t_table == NULL ||
automaton->m_table == NULL ||
automaton->bitmask == NULL)
{
if (automaton->bitmask == NULL)
return ERROR_INSUFFICIENT_MEMORY;
}
memset((uint8_t*) automaton->t_table + t_bytes_size, 0, t_bytes_size);
memset((uint8_t*) automaton->m_table + m_bytes_size, 0, m_bytes_size);
memset((uint8_t*) automaton->bitmask + b_bytes_size, 0, b_bytes_size);
memset((uint8_t*) automaton->bitmask + bm_len, 0, bm_len_incr);
automaton->tables_size *= 2;
automaton->tables_size += 257;
}
return ERROR_SUCCESS;
@@ -579,43 +581,57 @@ static int _yr_ac_find_suitable_transition_table_slot(
//
// A more detailed description can be found in: http://goo.gl/lE6zG
static int _yr_ac_build_transition_table(
YR_AC_AUTOMATON* automaton)
YR_AC_AUTOMATON* automaton,
YR_ARENA* arena)
{
YR_AC_TRANSITION* t_table;
uint32_t* m_table;
YR_AC_STATE* state;
YR_AC_STATE* child_state;
YR_AC_STATE* root_state = automaton->root;
uint32_t slot;
QUEUE queue = { NULL, NULL};
QUEUE queue = { NULL, NULL };
automaton->tables_size = 1024;
// Both t_table and m_table have 512 slots initially, which is enough for the
// root node's transition table.
automaton->tables_size = 512;
automaton->t_table = (YR_AC_TRANSITION_TABLE) yr_calloc(
automaton->tables_size, sizeof(YR_AC_TRANSITION));
automaton->m_table = (YR_AC_MATCH_TABLE) yr_calloc(
automaton->tables_size, sizeof(YR_AC_MATCH_TABLE_ENTRY));
automaton->bitmask = (YR_BITMASK*) yr_calloc(
automaton->bitmask = yr_calloc(
YR_BITMASK_SIZE(automaton->tables_size), sizeof(YR_BITMASK));
if (automaton->t_table == NULL ||
automaton->m_table == NULL ||
automaton->bitmask == NULL)
{
yr_free(automaton->t_table);
yr_free(automaton->m_table);
yr_free(automaton->bitmask);
if (automaton->bitmask == NULL)
return ERROR_INSUFFICIENT_MEMORY;
}
automaton->t_table[0] = YR_AC_MAKE_TRANSITION(0, 0);
automaton->m_table[0].match = root_state->matches;
FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory(
arena,
YR_AC_TRANSITION_TABLE,
automaton->tables_size * sizeof(YR_AC_TRANSITION),
NULL));
FAIL_ON_ERROR(yr_arena_allocate_zeroed_memory(
arena,
YR_AC_STATE_MATCHES_TABLE,
automaton->tables_size * sizeof(uint32_t),
NULL));
t_table = yr_arena_get_ptr(arena, YR_AC_TRANSITION_TABLE, 0);
m_table = yr_arena_get_ptr(arena, YR_AC_STATE_MATCHES_TABLE, 0);
// The failure link for the root node points to itself.
t_table[0] = YR_AC_MAKE_TRANSITION(0, 0);
// Initialize the entry corresponding to the root node in the match table.
// Entries in this table are the index within YR_AC_MATCH_POOL where resides
// the YR_AC_MATCH structure that corresponds to the head of the matches list
// for the node. The indexes start counting at 1, the zero is used for
// indicating that the node has no associated matches.
if (!YR_ARENA_IS_NULL_REF(root_state->matches_ref))
m_table[0] = root_state->matches_ref.offset / sizeof(YR_AC_MATCH) + 1;
// Mark the first slot in the transition table as used.
yr_bitmask_set(automaton->bitmask, 0);
// Index 0 is for root node. Unused indexes start at 1.
@@ -625,8 +641,10 @@ static int _yr_ac_build_transition_table(
while (child_state != NULL)
{
// Each state stores its slot number.
child_state->t_table_slot = child_state->input + 1;
automaton->t_table[child_state->input + 1] = YR_AC_MAKE_TRANSITION(
t_table[child_state->input + 1] = YR_AC_MAKE_TRANSITION(
0, child_state->input + 1);
yr_bitmask_set(automaton->bitmask, child_state->input + 1);
@@ -640,27 +658,38 @@ static int _yr_ac_build_transition_table(
state = _yr_ac_queue_pop(&queue);
FAIL_ON_ERROR(_yr_ac_find_suitable_transition_table_slot(
automaton, state, &slot));
automaton, arena, state, &slot));
automaton->t_table[state->t_table_slot] |= (slot << YR_AC_SLOT_OFFSET_BITS);
// _yr_ac_find_suitable_transition_table_slot can allocate more space in
// both tables and cause the tables to be moved to a different memory
// location, we must get their up-to-date addresses.
t_table = yr_arena_get_ptr(arena, YR_AC_TRANSITION_TABLE, 0);
m_table = yr_arena_get_ptr(arena, YR_AC_STATE_MATCHES_TABLE, 0);
t_table[state->t_table_slot] |= (slot << YR_AC_SLOT_OFFSET_BITS);
t_table[slot] = YR_AC_MAKE_TRANSITION(state->failure->t_table_slot, 0);
// The match table is an array of indexes within YR_AC_MATCHES_POOL. The
// N-th item in the array is the index for the YR_AC_MATCH structure that
// represents the head of the matches list for state N. The indexes start
// at 1, the 0 indicates that there are no matches for the state.
if (YR_ARENA_IS_NULL_REF(state->matches_ref))
m_table[slot] = 0;
else
m_table[slot] = state->matches_ref.offset / sizeof(YR_AC_MATCH) + 1;
state->t_table_slot = slot;
automaton->t_table[slot] = YR_AC_MAKE_TRANSITION(
state->failure->t_table_slot, 0);
yr_bitmask_set(automaton->bitmask, slot);
automaton->m_table[slot].match = state->matches;
// Push childrens of current_state
// Push children of current_state
child_state = state->first_child;
while (child_state != NULL)
{
child_state->t_table_slot = slot + child_state->input + 1;
automaton->t_table[child_state->t_table_slot] = YR_AC_MAKE_TRANSITION(
t_table[child_state->t_table_slot] = YR_AC_MAKE_TRANSITION(
0, child_state->input + 1);
yr_bitmask_set(automaton->bitmask, child_state->t_table_slot);
@@ -681,14 +710,15 @@ static int _yr_ac_build_transition_table(
// Prints automaton state for debug purposes. This function is invoked by
// yr_ac_print_automaton, is not intended to be used stand-alone.
//
//TODO(vmalvarez): Remove or re-implement
/*
static void _yr_ac_print_automaton_state(
YR_AC_STATE* state)
{
int i;
int child_count;
YR_AC_MATCH* match;
YR_AC_MATCH_LIST_ENTRY* match;
YR_AC_STATE* child_state;
for (i = 0; i < state->depth; i++)
@@ -757,7 +787,7 @@ static void _yr_ac_print_automaton_state(
_yr_ac_print_automaton_state(child_state);
child_state = child_state->siblings;
}
}
}*/
//
@@ -784,15 +814,13 @@ int yr_ac_automaton_create(
}
root_state->depth = 0;
root_state->matches = NULL;
root_state->matches_ref = YR_ARENA_NULL_REF;
root_state->failure = NULL;
root_state->first_child = NULL;
root_state->siblings = NULL;
root_state->t_table_slot = 0;
new_automaton->root = root_state;
new_automaton->m_table = NULL;
new_automaton->t_table = NULL;
new_automaton->bitmask = NULL;
new_automaton->tables_size = 0;
@@ -813,8 +841,6 @@ int yr_ac_automaton_destroy(
{
_yr_ac_state_destroy(automaton->root);
yr_free(automaton->t_table);
yr_free(automaton->m_table);
yr_free(automaton->bitmask);
yr_free(automaton);
@@ -832,25 +858,17 @@ int yr_ac_automaton_destroy(
int yr_ac_add_string(
YR_AC_AUTOMATON* automaton,
YR_STRING* string,
uint32_t string_idx,
YR_ATOM_LIST_ITEM* atom,
YR_ARENA* matches_arena)
YR_ARENA* arena)
{
int result = ERROR_SUCCESS;
int i;
YR_AC_STATE* state;
YR_AC_STATE* next_state;
YR_AC_MATCH* new_match;
// For each atom create the states in the automaton.
while (atom != NULL)
{
state = automaton->root;
YR_AC_STATE* state = automaton->root;
for (i = 0; i < atom->atom.length; i++)
for (int i = 0; i < atom->atom.length; i++)
{
next_state = _yr_ac_next_state(state, atom->atom.bytes[i]);
YR_AC_STATE* next_state = _yr_ac_next_state(state, atom->atom.bytes[i]);
if (next_state == NULL)
{
@@ -863,100 +881,56 @@ int yr_ac_add_string(
state = next_state;
}
result = yr_arena_allocate_struct(
matches_arena,
YR_ARENA_REF new_match_ref;
FAIL_ON_ERROR(yr_arena_allocate_struct(
arena,
YR_AC_STATE_MATCHES_POOL,
sizeof(YR_AC_MATCH),
(void**) &new_match,
&new_match_ref,
offsetof(YR_AC_MATCH, string),
offsetof(YR_AC_MATCH, forward_code),
offsetof(YR_AC_MATCH, backward_code),
offsetof(YR_AC_MATCH, next),
EOL);
EOL));
if (result == ERROR_SUCCESS)
{
new_match->backtrack = state->depth + atom->backtrack;
new_match->string = string;
new_match->forward_code = atom->forward_code;
new_match->backward_code = atom->backward_code;
new_match->next = state->matches;
state->matches = new_match;
}
else
{
break;
}
YR_AC_MATCH* new_match = yr_arena_ref_to_ptr(arena, &new_match_ref);
new_match->backtrack = state->depth + atom->backtrack;
new_match->string = yr_arena_get_ptr(
arena, YR_STRINGS_TABLE, string_idx * sizeof(struct YR_STRING));
new_match->forward_code = yr_arena_ref_to_ptr(
arena, &atom->forward_code_ref);
new_match->backward_code = yr_arena_ref_to_ptr(
arena, &atom->backward_code_ref);
// Add newly created match to the list of matches for the state.
new_match->next = yr_arena_ref_to_ptr(arena, &state->matches_ref);
state->matches_ref = new_match_ref;
atom = atom->next;
}
return result;
return ERROR_SUCCESS;
}
//
// yr_ac_compile
//
// Compiles the Aho-Corasick automaton, the resulting data structures are
// are written in the provided arena.
//
int yr_ac_compile(
YR_AC_AUTOMATON* automaton,
YR_ARENA* arena,
YR_AC_TABLES* tables)
YR_ARENA* arena)
{
uint32_t i;
FAIL_ON_ERROR(_yr_ac_create_failure_links(automaton));
FAIL_ON_ERROR(_yr_ac_create_failure_links(automaton, arena));
FAIL_ON_ERROR(_yr_ac_optimize_failure_links(automaton));
FAIL_ON_ERROR(_yr_ac_build_transition_table(automaton));
FAIL_ON_ERROR(yr_arena_reserve_memory(
arena,
automaton->tables_size * sizeof(tables->transitions[0]) +
automaton->tables_size * sizeof(tables->matches[0])));
FAIL_ON_ERROR(yr_arena_write_data(
arena,
automaton->t_table,
sizeof(YR_AC_TRANSITION),
(void**) &tables->transitions));
for (i = 1; i < automaton->tables_size; i++)
{
FAIL_ON_ERROR(yr_arena_write_data(
arena,
automaton->t_table + i,
sizeof(YR_AC_TRANSITION),
NULL));
}
FAIL_ON_ERROR(yr_arena_write_data(
arena,
automaton->m_table,
sizeof(YR_AC_MATCH_TABLE_ENTRY),
(void**) &tables->matches));
FAIL_ON_ERROR(yr_arena_make_ptr_relocatable(
arena,
tables->matches,
offsetof(YR_AC_MATCH_TABLE_ENTRY, match),
EOL));
for (i = 1; i < automaton->tables_size; i++)
{
void* ptr;
FAIL_ON_ERROR(yr_arena_write_data(
arena,
automaton->m_table + i,
sizeof(YR_AC_MATCH_TABLE_ENTRY),
(void**) &ptr));
FAIL_ON_ERROR(yr_arena_make_ptr_relocatable(
arena,
ptr,
offsetof(YR_AC_MATCH_TABLE_ENTRY, match),
EOL));
}
FAIL_ON_ERROR(_yr_ac_build_transition_table(automaton, arena));
return ERROR_SUCCESS;
}
@@ -967,10 +941,12 @@ int yr_ac_compile(
//
// Prints automaton for debug purposes.
//
//TODO(vmalvarez): Remove or re-implement
/*
void yr_ac_print_automaton(YR_AC_AUTOMATON* automaton)
{
printf("-------------------------------------------------------\n");
_yr_ac_print_automaton_state(automaton->root);
printf("-------------------------------------------------------\n");
}
*/
libyara/arena.c
+431 -840
View File
File diff suppressed because it is too large Load Diff
libyara/atoms.c
+41 -27
View File
@@ -160,7 +160,7 @@ int yr_atoms_heuristic_quality(
else
quality += 20;
};
if (!yr_bitmask_isset(seen_bytes, atom->bytes[i]))
if (!yr_bitmask_is_set(seen_bytes, atom->bytes[i]))
{
yr_bitmask_set(seen_bytes, atom->bytes[i]);
unique_bytes++;
@@ -172,10 +172,10 @@ int yr_atoms_heuristic_quality(
// it heavily.
if (unique_bytes == 1 &&
(yr_bitmask_isset(seen_bytes, 0x00) ||
yr_bitmask_isset(seen_bytes, 0x20) ||
yr_bitmask_isset(seen_bytes, 0xCC) ||
yr_bitmask_isset(seen_bytes, 0xFF)))
(yr_bitmask_is_set(seen_bytes, 0x00) ||
yr_bitmask_is_set(seen_bytes, 0x20) ||
yr_bitmask_is_set(seen_bytes, 0xCC) ||
yr_bitmask_is_set(seen_bytes, 0xFF)))
{
quality -= 10 * atom->length;
}
@@ -586,8 +586,8 @@ static int _yr_atoms_choose(
if (item->atom.length > 0)
{
item->forward_code = node->re_nodes[shift]->forward_code;
item->backward_code = node->re_nodes[shift]->backward_code;
item->forward_code_ref = node->re_nodes[shift]->forward_code_ref;
item->backward_code_ref = node->re_nodes[shift]->backward_code_ref;
item->backtrack = 0;
item->next = NULL;
@@ -789,8 +789,8 @@ static int _yr_atoms_case_insensitive(
}
new_atom->atom.length = atom_length;
new_atom->forward_code = atom->forward_code;
new_atom->backward_code = atom->backward_code;
new_atom->forward_code_ref = atom->forward_code_ref;
new_atom->backward_code_ref = atom->backward_code_ref;
new_atom->backtrack = atom->backtrack;
new_atom->next = *case_insensitive_atoms;
@@ -812,7 +812,7 @@ static int _yr_atoms_case_insensitive(
// _yr_atoms_xor
//
// For a given list of atoms returns another list after a single byte xor
// has been applied to it (0x01 - 0xff).
// has been applied to it.
//
static int _yr_atoms_xor(
@@ -844,8 +844,8 @@ static int _yr_atoms_xor(
}
new_atom->atom.length = yr_min(atom->atom.length, YR_MAX_ATOM_LENGTH);
new_atom->forward_code = atom->forward_code;
new_atom->backward_code = atom->backward_code;
new_atom->forward_code_ref = atom->forward_code_ref;
new_atom->backward_code_ref = atom->backward_code_ref;
new_atom->backtrack = atom->backtrack;
new_atom->next = *xor_atoms;
@@ -900,8 +900,8 @@ static int _yr_atoms_wide(
}
new_atom->atom.length = yr_min(atom->atom.length * 2, YR_MAX_ATOM_LENGTH);
new_atom->forward_code = atom->forward_code;
new_atom->backward_code = atom->backward_code;
new_atom->forward_code_ref = atom->forward_code_ref;
new_atom->backward_code_ref = atom->backward_code_ref;
new_atom->backtrack = atom->backtrack * 2;
new_atom->next = *wide_atoms;
@@ -1411,7 +1411,11 @@ int yr_atoms_extract_from_re(
*atoms = NULL;
});
if (modifier.flags & STRING_GFLAGS_WIDE)
// Don't do convert atoms to wide here if either base64 modifier is used.
// This is to avoid the situation where we have "base64 wide" because
// the wide has already been applied BEFORE the base64 encoding.
if (modifier.flags & STRING_FLAGS_WIDE &&
!(modifier.flags & STRING_FLAGS_BASE64 || modifier.flags & STRING_FLAGS_BASE64_WIDE))
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_atoms_wide(*atoms, &wide_atoms),
@@ -1421,7 +1425,7 @@ int yr_atoms_extract_from_re(
*atoms = NULL;
});
if (modifier.flags & STRING_GFLAGS_ASCII)
if (modifier.flags & STRING_FLAGS_ASCII)
{
*atoms = _yr_atoms_list_concat(*atoms, wide_atoms);
}
@@ -1432,7 +1436,7 @@ int yr_atoms_extract_from_re(
}
}
if (modifier.flags & STRING_GFLAGS_NO_CASE)
if (modifier.flags & STRING_FLAGS_NO_CASE)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_atoms_case_insensitive(*atoms, &case_insensitive_atoms),
@@ -1456,8 +1460,8 @@ int yr_atoms_extract_from_re(
(*atoms)->atom.length = 0;
(*atoms)->backtrack = 0;
(*atoms)->forward_code = re_ast->root_node->forward_code;
(*atoms)->backward_code = NULL;
(*atoms)->forward_code_ref = re_ast->root_node->forward_code_ref;
(*atoms)->backward_code_ref = YR_ARENA_NULL_REF;
(*atoms)->next = NULL;
}
@@ -1494,8 +1498,8 @@ int yr_atoms_extract_from_string(
if (item == NULL)
return ERROR_INSUFFICIENT_MEMORY;
item->forward_code = NULL;
item->backward_code = NULL;
item->forward_code_ref = YR_ARENA_NULL_REF;
item->backward_code_ref = YR_ARENA_NULL_REF;
item->next = NULL;
item->backtrack = 0;
@@ -1532,7 +1536,7 @@ int yr_atoms_extract_from_string(
*atoms = item;
*min_atom_quality = max_quality;
if (modifier.flags & STRING_GFLAGS_WIDE)
if (modifier.flags & STRING_FLAGS_WIDE)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_atoms_wide(*atoms, &wide_atoms),
@@ -1542,7 +1546,7 @@ int yr_atoms_extract_from_string(
*atoms = NULL;
});
if (modifier.flags & STRING_GFLAGS_ASCII)
if (modifier.flags & STRING_FLAGS_ASCII)
{
*atoms = _yr_atoms_list_concat(*atoms, wide_atoms);
}
@@ -1553,7 +1557,7 @@ int yr_atoms_extract_from_string(
}
}
if (modifier.flags & STRING_GFLAGS_NO_CASE)
if (modifier.flags & STRING_FLAGS_NO_CASE)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_atoms_case_insensitive(*atoms, &case_insensitive_atoms),
@@ -1566,7 +1570,7 @@ int yr_atoms_extract_from_string(
*atoms = _yr_atoms_list_concat(*atoms, case_insensitive_atoms);
}
if (modifier.flags & STRING_GFLAGS_XOR)
if (modifier.flags & STRING_FLAGS_XOR)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_atoms_xor(*atoms, modifier.xor_min, modifier.xor_max, &xor_atoms),
@@ -1576,8 +1580,18 @@ int yr_atoms_extract_from_string(
*atoms = NULL;
});
yr_atoms_list_destroy(*atoms);
*atoms = xor_atoms;
yr_atoms_list_destroy(*atoms);
*atoms = xor_atoms;
}
// Recheck the atom quality, in case we have just generated some poor atoms.
// https://github.com/VirusTotal/yara/issues/1172
for (item = *atoms; item != NULL; item = item->next)
{
quality = config->get_atom_quality(config, &item->atom);
if (quality < *min_atom_quality)
*min_atom_quality = quality;
}
return ERROR_SUCCESS;
libyara/base64.c
+506
View File
@@ -0,0 +1,506 @@
/*
Copyright (c) 2020. The YARA Authors. All Rights Reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <string.h>
#include <yara/base64.h>
#include <yara/error.h>
#include <yara/mem.h>
#include <yara/re.h>
#include <yara/sizedstr.h>
//
// _yr_modified_base64_encode
//
// Given a pointer to a SIZED_STRING append 0, 1 or 2 bytes and base64 encode
// the string. The number of padding bytes is returned in "pad" and the caller
// is expected to trim the appropriate number of leading and trailing bytes.
//
// This is based upon the ideas at:
// https://www.leeholmes.com/blog/2019/12/10/searching-for-content-in-base-64-strings-2/
//
// The caller is responsible for freeing the returned string.
//
static SIZED_STRING* _yr_modified_base64_encode(
SIZED_STRING* in,
SIZED_STRING* alphabet,
int i,
int* pad)
{
uint8_t* src = (uint8_t*) in->c_string;
size_t len = in->length;
SIZED_STRING* out;
uint8_t* p;
uint8_t* end;
char* alphabet_str = alphabet->c_string;
uint8_t* tmp;
int j;
*pad = ((i + len) % 3) ? 3 - ((i + len) % 3) : 0;
// Add "i" for the number of prepended bytes.
out = (SIZED_STRING*) yr_malloc(
sizeof(SIZED_STRING) + i + ((len * 4 + 3) / 3) + *pad);
if (out == NULL)
return NULL;
tmp = (uint8_t*) yr_malloc(sizeof(uint8_t) * (len + i));
if (tmp == NULL)
{
yr_free(out);
return NULL;
}
// Prepend appropriate number of bytes and copy remaining input bytes into
// temporary buffer.
for (j = 0; j < i; j++)
tmp[j] = 'A';
memcpy(tmp + j, src, len);
src = tmp;
p = (uint8_t*) out->c_string;
end = src + len + j;
while (end - src >= 3)
{
*p++ = alphabet_str[src[0] >> 2];
*p++ = alphabet_str[((src[0] & 0x03) << 4 | src[1] >> 4)];
*p++ = alphabet_str[((src[1] & 0x0f) << 2 | (src[2] >> 6))];
*p++ = alphabet_str[src[2] & 0x3f];
src += 3;
}
// Handle remaining bytes and padding.
if (end - src)
{
*p++ = alphabet_str[src[0] >> 2];
if (end - src == 1)
{
*p++ = alphabet_str[(src[0] & 0x03) << 4];
*p++ = '=';
}
else
{
*p++ = alphabet_str[((src[0] & 0x03) << 4 | src[1] >> 4)];
*p++ = alphabet_str[(src[1] & 0x0f) << 2];
}
*p++ = '=';
}
yr_free(tmp);
out->length = p - (uint8_t*) out->c_string;
return out;
}
//
// Given a base64 encoded string, return a new string with leading and trailing
// bytes stripped appropriately. The number of leading bytes to skip is always
// (i + 1) or zero when no leading bytes are added and the number of trailing
// bytes is always (pad + 1) or zero when pad is zero. Also, convert the final
// string to wide if desired.
//
// Note: This implementation assumes you only prepend 0, 1 or 2 bytes.
//
static SIZED_STRING* _yr_base64_get_base64_substring(
SIZED_STRING* encoded_str,
int wide,
int i,
int pad)
{
SIZED_STRING* new_str;
SIZED_STRING* final_str;
char* start;
uint32_t length;
int trailing;
int leading;
trailing = pad ? pad + 1 : 0;
leading = i ? i + 1 : 0;
length = encoded_str->length - (leading + trailing);
new_str = (SIZED_STRING*) yr_malloc(sizeof(SIZED_STRING) + length);
if (new_str == NULL)
return NULL;
start = encoded_str->c_string + leading;
memcpy(new_str->c_string, start, length);
new_str->length = length;
if (wide)
{
final_str = sized_string_convert_to_wide(new_str);
yr_free(new_str);
}
else
{
final_str = new_str;
}
return final_str;
}
// RE metacharacters which need to be escaped when generating the final RE.
#define IS_METACHAR(x) \
(x == '\\' || \
x == '^' || \
x == '$' || \
x == '|' || \
x == '(' || \
x == ')' || \
x == '[' || \
x == ']' || \
x == '*' || \
x == '?' || \
x == '{' || \
x == ',' || \
x == '.' || \
x == '+' || \
x == '}')
//
// Given a SIZED_STRING return the number of characters which will need to be
// escaped when generating the final string to pass to the regexp compiler.
//
static int _yr_base64_count_escaped(
SIZED_STRING* str)
{
uint32_t i;
int c = 0;
for (i = 0; i < str->length; i++)
{
// We must be careful to escape null bytes because they break the RE lexer.
if (IS_METACHAR(str->c_string[i]))
c++;
else if (str->c_string[i] == '\x00')
c += 4;
}
return c;
}
//
// Create nodes representing the different encodings of a base64 string.
//
static int _yr_base64_create_nodes(
SIZED_STRING* str,
SIZED_STRING* alphabet,
int wide,
BASE64_NODE** head,
BASE64_NODE** tail)
{
SIZED_STRING* encoded_str;
SIZED_STRING* final_str;
BASE64_NODE* node;
int i;
int pad;
for (i = 0; i <= 2; i++)
{
node = (BASE64_NODE*) yr_malloc(sizeof(BASE64_NODE));
if (node == NULL)
return ERROR_INSUFFICIENT_MEMORY;
FAIL_ON_NULL_WITH_CLEANUP(
encoded_str = _yr_modified_base64_encode(str, alphabet, i, &pad),
yr_free(node));
// Now take the encoded string and strip the bytes which are affected by
// the leading and trailing bytes of the plaintext.
FAIL_ON_NULL_WITH_CLEANUP(
final_str = _yr_base64_get_base64_substring(encoded_str, wide, i, pad),
{
yr_free(encoded_str);
yr_free(node);
});
yr_free(encoded_str);
node->str = final_str;
node->escaped = _yr_base64_count_escaped(node->str);
node->next = NULL;
if (*head == NULL)
*head = node;
if (*tail == NULL)
{
*tail = node;
}
else
{
(*tail)->next = node;
*tail = node;
}
}
return ERROR_SUCCESS;
}
//
// Useful for printing the encoded strings.
//
void _yr_base64_print_nodes(
BASE64_NODE* head)
{
size_t i;
BASE64_NODE* p = head;
while (p != NULL)
{
for (i = 0; i < p->str->length; i++)
{
if (p->str->c_string[i] >= 32 && p->str->c_string[i] <= 126)
printf("%c", p->str->c_string[i]);
else
printf("\\x%02x", p->str->c_string[i]);
}
printf("\n");
p = p->next;
}
return;
}
//
// Destroy a list of base64 nodes.
//
static void _yr_base64_destroy_nodes(
BASE64_NODE* head)
{
BASE64_NODE* p = head;
BASE64_NODE* next;
while (p != NULL)
{
yr_free(p->str);
next = p->next;
yr_free(p);
p = next;
}
return;
}
//
// Create the regexp that is the alternatives of each of the strings collected
// in the BASE64_NODE list.
//
int _yr_base64_create_regexp(
BASE64_NODE* head,
RE_AST** re_ast,
RE_ERROR* re_error)
{
char* re_str;
char* s;
uint32_t i;
uint32_t length = 0;
uint32_t c = 0; // The number of nodes in the list, used to know how many '|'.
BASE64_NODE* p = head;
while (p != NULL)
{
length += (p->str->length + p->escaped);
c++;
p = p->next;
}
if (c == 0)
return ERROR_INSUFFICIENT_MEMORY;
// Make sure to include '(' and ')'.
// The number of '|' is number of nodes - 1.
re_str = (char*) yr_malloc(length + 2 + (c - 1) + 1);
if (re_str == NULL)
return ERROR_INSUFFICIENT_MEMORY;
s = re_str;
p = head;
*s++ = '(';
while (p != NULL)
{
for (i = 0; i < p->str->length; i++)
{
if (IS_METACHAR(p->str->c_string[i]))
*s++ = '\\';
if (p->str->c_string[i] == '\x00')
{
*s++ = '\\';
*s++ = 'x';
*s++ = '0';
*s++ = '0';
}
else
*s++ = p->str->c_string[i];
}
if (p->next != NULL)
*s++ = '|';
p = p->next;
}
*s++ = ')';
*s = '\x00';
// Useful for debugging as long as the string has no NULL bytes in it. ;)
//printf("%s\n", re_str);
FAIL_ON_ERROR_WITH_CLEANUP(
yr_re_parse(re_str, re_ast, re_error),
yr_free(re_str));
yr_free(re_str);
return ERROR_SUCCESS;
}
//
// Given a string and an alphabet, generate the RE_AST suitable for representing
// the different encodings of the string. This means we generate
// "(ABCD|EFGH|IJKL)" and must be careful to escape any special characters as
// a result of the base64 encoding.
//
// This uses ideas from:
// https://www.leeholmes.com/blog/2019/12/10/searching-for-content-in-base-64-strings-2/
//
// This does not emit the code for the RE. A further call to yr_re_ast_emit_code
// is required to get the code.
//
int yr_base64_ast_from_string(
SIZED_STRING* in_str,
YR_MODIFIER modifier,
RE_AST** re_ast,
RE_ERROR* error)
{
BASE64_NODE* head = NULL;
BASE64_NODE* tail = NULL;
SIZED_STRING* wide_str;
if (modifier.flags & STRING_FLAGS_WIDE)
{
wide_str = sized_string_convert_to_wide(in_str);
if (modifier.flags & STRING_FLAGS_BASE64)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_nodes(wide_str, modifier.alphabet, 0, &head, &tail),
{
strcpy(error->message, "Failure encoding base64 wide string");
yr_free(wide_str);
_yr_base64_destroy_nodes(head);
});
}
if (modifier.flags & STRING_FLAGS_BASE64_WIDE)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_nodes(wide_str, modifier.alphabet, 1, &head, &tail),
{
strcpy(error->message, "Failure encoding base64wide wide string");
yr_free(wide_str);
_yr_base64_destroy_nodes(head);
});
}
yr_free(wide_str);
}
if (modifier.flags & STRING_FLAGS_ASCII)
{
if (modifier.flags & STRING_FLAGS_BASE64)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_nodes(in_str, modifier.alphabet, 0, &head, &tail),
{
strcpy(error->message, "Failure encoding base64 ascii string");
_yr_base64_destroy_nodes(head);
});
}
if (modifier.flags & STRING_FLAGS_BASE64_WIDE)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_nodes(in_str, modifier.alphabet, 1, &head, &tail),
{
strcpy(error->message, "Failure encoding base64wide ascii string");
_yr_base64_destroy_nodes(head);
});
}
}
if (!(modifier.flags & STRING_FLAGS_WIDE) &&
!(modifier.flags & STRING_FLAGS_ASCII))
{
if (modifier.flags & STRING_FLAGS_BASE64)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_nodes(in_str, modifier.alphabet, 0, &head, &tail),
{
strcpy(error->message, "Failure encoding base64 string");
_yr_base64_destroy_nodes(head);
});
}
if (modifier.flags & STRING_FLAGS_BASE64_WIDE)
{
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_nodes(in_str, modifier.alphabet, 1, &head, &tail),
{
strcpy(error->message, "Failure encoding base64wide string");
_yr_base64_destroy_nodes(head);
});
}
}
// Useful for printing the contents of the nodes, to make sure they were
// encoded and stripped properly.
//_yr_base64_print_nodes(head);
// Create the final regex string to be parsed from all the nodes.
// Error message is filled in by the caller in case of failure.
FAIL_ON_ERROR_WITH_CLEANUP(
_yr_base64_create_regexp(head, re_ast, error),
_yr_base64_destroy_nodes(head));
_yr_base64_destroy_nodes(head);
return ERROR_SUCCESS;
}
libyara/bitmask.c
+3 -3
View File
@@ -36,7 +36,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// yr_bitmask_find_non_colliding_offset
//
// Finds the smaller offset within bitmask A where bitmask B can be accommodated
// without bit collisions. A collision occurs when bots bitmasks have a bit set
// without bit collisions. A collision occurs when both bitmasks have a bit set
// to 1 at the same offset. This function assumes that the first bit in B is 1
// and do optimizations that rely on that.
//
@@ -44,7 +44,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// a value that is used for speeding-up subsequent searches over the same
// bitmask A. When called for the first time with some bitmask A, the pointer
// must point to a zero-initialized uint32_t. In the next call the function uses
// the previously stored value for skiping over a portion of the A bitmask and
// the previously stored value for skipping over a portion of the A bitmask and
// updates the value.
//
// Args:
@@ -71,7 +71,7 @@ uint32_t yr_bitmask_find_non_colliding_offset(
// Ensure that the first bit of bitmask B is set, as this function does some
// optimizations that rely on that.
assert(yr_bitmask_isset(b, 0));
assert(yr_bitmask_is_set(b, 0));
// Skip all slots that are filled with 1s. It's safe to do that because the
// first bit of B is 1, so we won't be able to accommodate B at any offset
libyara/compiler.c
+126 -291
View File
@@ -150,6 +150,22 @@ const char* _yr_compiler_default_include_callback(
}
//
// _yr_compiler_get_rule_by_idx returns a rule given its index in the rules
// table. The returned pointer is valid as long as no other rule is written
// to the table. This is because the write operation may cause the table to
// be moved to a different location in memory. Use the pointer only in a
// limited scope where you can be sure that no other rule is being written
// during the pointer's lifetime.
//
YR_RULE* _yr_compiler_get_rule_by_idx(
YR_COMPILER* compiler, uint32_t rule_idx)
{
return (YR_RULE*) yr_arena_get_ptr(
compiler->arena, YR_RULES_TABLE, rule_idx * sizeof(YR_RULE));
}
YR_API int yr_compiler_create(
YR_COMPILER** compiler)
{
@@ -161,8 +177,14 @@ YR_API int yr_compiler_create(
if (new_compiler == NULL)
return ERROR_INSUFFICIENT_MEMORY;
new_compiler->current_rule_idx = UINT32_MAX;
new_compiler->next_rule_idx = 0;
new_compiler->current_string_idx = 0;
new_compiler->current_namespace_idx = 0;
new_compiler->num_namespaces = 0;
new_compiler->errors = 0;
new_compiler->callback = NULL;
new_compiler->rules = NULL;
new_compiler->include_callback = _yr_compiler_default_include_callback;
new_compiler->incl_clbk_user_data = NULL;
new_compiler->include_free = _yr_compiler_default_include_free;
@@ -175,9 +197,7 @@ YR_API int yr_compiler_create(
new_compiler->fixup_stack_head = NULL;
new_compiler->loop_index = -1;
new_compiler->loop_for_of_var_index = -1;
new_compiler->compiled_rules_arena = NULL;
new_compiler->namespaces_count = 0;
new_compiler->current_rule = NULL;
new_compiler->atoms_config.get_atom_quality = yr_atoms_heuristic_quality;
new_compiler->atoms_config.quality_warning_threshold = \
YR_ATOM_QUALITY_WARNING_THRESHOLD;
@@ -192,43 +212,7 @@ YR_API int yr_compiler_create(
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->sz_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->rules_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->strings_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->code_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->re_code_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->externals_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->namespaces_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->metas_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->automaton_arena);
if (result == ERROR_SUCCESS)
result = yr_arena_create(
65536, ARENA_FLAGS_RELOCATABLE, &new_compiler->matches_arena);
YR_NUM_SECTIONS, 64, &new_compiler->arena);
if (result == ERROR_SUCCESS)
result = yr_ac_automaton_create(&new_compiler->automaton);
@@ -249,20 +233,7 @@ YR_API int yr_compiler_create(
YR_API void yr_compiler_destroy(
YR_COMPILER* compiler)
{
YR_FIXUP* fixup;
int i;
yr_arena_destroy(compiler->compiled_rules_arena);
yr_arena_destroy(compiler->sz_arena);
yr_arena_destroy(compiler->rules_arena);
yr_arena_destroy(compiler->strings_arena);
yr_arena_destroy(compiler->code_arena);
yr_arena_destroy(compiler->re_code_arena);
yr_arena_destroy(compiler->externals_arena);
yr_arena_destroy(compiler->namespaces_arena);
yr_arena_destroy(compiler->metas_arena);
yr_arena_destroy(compiler->automaton_arena);
yr_arena_destroy(compiler->matches_arena);
yr_arena_release(compiler->arena);
if (compiler->automaton != NULL)
yr_ac_automaton_destroy(compiler->automaton);
@@ -282,10 +253,10 @@ YR_API void yr_compiler_destroy(
if (compiler-> atoms_config.free_quality_table)
yr_free(compiler->atoms_config.quality_table);
for (i = 0; i < compiler->file_name_stack_ptr; i++)
for (int i = 0; i < compiler->file_name_stack_ptr; i++)
yr_free(compiler->file_name_stack[i]);
fixup = compiler->fixup_stack_head;
YR_FIXUP* fixup = compiler->fixup_stack_head;
while (fixup != NULL)
{
@@ -337,10 +308,10 @@ YR_API void yr_compiler_set_re_ast_callback(
// compiler for choosing the best atoms from regular expressions and strings.
// When a quality table is set, the compiler uses yr_atoms_table_quality
// instead of yr_atoms_heuristic_quality for computing atom quality. The table
// has an arbitary number of entries, each composed of YR_MAX_ATOM_LENGTH + 1
// has an arbitrary number of entries, each composed of YR_MAX_ATOM_LENGTH + 1
// bytes. The first YR_MAX_ATOM_LENGTH bytes from each entry are the atom's
// ones, and the remaining byte is a value in the range 0-255 determining the
// atom's quality. Entries must be lexicografically sorted by atom in ascending
// atom's quality. Entries must be lexicographically sorted by atom in ascending
// order.
//
// [ atom (YR_MAX_ATOM_LENGTH bytes) ] [ quality (1 byte) ]
@@ -357,7 +328,7 @@ YR_API void yr_compiler_set_re_ast_callback(
// caller is responsible for freeing the table.
//
// The "warning_threshold" argument must be a number between 0 and 255, if some
// atom choosen for a string have a quality below the specified threshold a
// atom chosen for a string have a quality below the specified threshold a
// warning like "<string> is slowing down scanning" is shown.
YR_API void yr_compiler_set_atom_quality_table(
@@ -378,7 +349,7 @@ YR_API void yr_compiler_set_atom_quality_table(
// yr_compiler_set_atom_quality_table
//
// Load an atom quality table from a file. The file's content must have the
// format explained in the decription for yr_compiler_set_atom_quality_table.
// format explained in the description for yr_compiler_set_atom_quality_table.
//
YR_API int yr_compiler_load_atom_quality_table(
@@ -479,8 +450,10 @@ int _yr_compiler_get_var_frame(
int i, result = 0;
for (i = 0; i < compiler->loop_index; i++)
result += compiler->loop[i].vars_count +
compiler->loop[i].vars_internal_count;
{
result += compiler->loop[i].vars_count +
compiler->loop[i].vars_internal_count;
}
return result;
}
@@ -504,57 +477,48 @@ static int _yr_compiler_set_namespace(
YR_COMPILER* compiler,
const char* namespace_)
{
YR_NAMESPACE* ns;
YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr(
compiler->arena, YR_NAMESPACES_TABLE, 0);
char* ns_name;
int result;
int i;
bool found;
bool found = false;
ns = (YR_NAMESPACE*) yr_arena_base_address(compiler->namespaces_arena);
found = false;
for (i = 0; i < compiler->namespaces_count; i++)
for (int i = 0; i < compiler->num_namespaces; i++, ns++)
{
if (strcmp(ns->name, namespace_) == 0)
{
found = true;
compiler->current_namespace_idx = i;
break;
}
ns = (YR_NAMESPACE*) yr_arena_next_address(
compiler->namespaces_arena,
ns,
sizeof(YR_NAMESPACE));
}
if (!found)
{
result = yr_arena_write_string(
compiler->sz_arena,
YR_ARENA_REF ref;
FAIL_ON_ERROR(yr_arena_allocate_struct(
compiler->arena,
YR_NAMESPACES_TABLE,
sizeof(YR_NAMESPACE),
&ref,
offsetof(YR_NAMESPACE, name),
EOL));
ns = (YR_NAMESPACE*) yr_arena_ref_to_ptr(compiler->arena, &ref);
FAIL_ON_ERROR(yr_arena_write_string(
compiler->arena,
YR_SZ_POOL,
namespace_,
&ns_name);
&ref));
if (result == ERROR_SUCCESS)
result = yr_arena_allocate_struct(
compiler->namespaces_arena,
sizeof(YR_NAMESPACE),
(void**) &ns,
offsetof(YR_NAMESPACE, name),
EOL);
ns->name = (const char*) yr_arena_ref_to_ptr(compiler->arena, &ref);
ns->idx = compiler->num_namespaces;
if (result != ERROR_SUCCESS)
return result;
ns->name = ns_name;
for (i = 0; i < YR_MAX_THREADS; i++)
ns->t_flags[i] = 0;
compiler->namespaces_count++;
compiler->current_namespace_idx = compiler->num_namespaces;
compiler->num_namespaces++;
}
compiler->current_namespace = ns;
return ERROR_SUCCESS;
}
@@ -570,7 +534,7 @@ YR_API int yr_compiler_add_file(
// Don't allow yr_compiler_add_file() after
// yr_compiler_get_rules() has been called.
assert(compiler->compiled_rules_arena == NULL);
assert(compiler->rules == NULL);
// Don't allow calls to yr_compiler_add_file() if a previous call to
// yr_compiler_add_XXXX failed.
@@ -608,7 +572,7 @@ YR_API int yr_compiler_add_fd(
// Don't allow yr_compiler_add_fd() after
// yr_compiler_get_rules() has been called.
assert(compiler->compiled_rules_arena == NULL);
assert(compiler->rules == NULL);
// Don't allow calls to yr_compiler_add_fd() if a previous call to
// yr_compiler_add_XXXX failed.
@@ -643,7 +607,7 @@ YR_API int yr_compiler_add_string(
// Don't allow calls to yr_compiler_add_string() after
// yr_compiler_get_rules() has been called.
assert(compiler->compiled_rules_arena == NULL);
assert(compiler->rules == NULL);
// Don't allow calls to yr_compiler_add_string() if a previous call to
// yr_compiler_add_XXXX failed.
@@ -665,170 +629,63 @@ YR_API int yr_compiler_add_string(
static int _yr_compiler_compile_rules(
YR_COMPILER* compiler)
{
YARA_RULES_FILE_HEADER* rules_file_header = NULL;
YR_ARENA* arena = NULL;
YR_RULE null_rule;
YR_EXTERNAL_VARIABLE null_external;
YR_AC_TABLES tables;
uint8_t halt = OP_HALT;
int result;
// Write halt instruction at the end of code.
yr_arena_write_data(
compiler->code_arena,
FAIL_ON_ERROR(yr_arena_write_data(
compiler->arena,
YR_CODE_SECTION,
&halt,
sizeof(uint8_t),
NULL);
NULL));
// Write a null rule indicating the end.
memset(&null_rule, 0xFA, sizeof(YR_RULE));
null_rule.g_flags = RULE_GFLAGS_NULL;
null_rule.flags = RULE_FLAGS_NULL;
yr_arena_write_data(
compiler->rules_arena,
&null_rule,
sizeof(YR_RULE),
NULL);
FAIL_ON_ERROR(yr_arena_write_data(
compiler->arena,
YR_RULES_TABLE,
&null_rule,
sizeof(YR_RULE),
NULL));
// Write a null external the end.
// Write a null external indicating the end.
memset(&null_external, 0xFA, sizeof(YR_EXTERNAL_VARIABLE));
null_external.type = EXTERNAL_VARIABLE_TYPE_NULL;
yr_arena_write_data(
compiler->externals_arena,
FAIL_ON_ERROR(yr_arena_write_data(
compiler->arena,
YR_EXTERNAL_VARIABLES_TABLE,
&null_external,
sizeof(YR_EXTERNAL_VARIABLE),
NULL);
NULL));
// Write Aho-Corasick automaton to arena.
result = yr_ac_compile(
FAIL_ON_ERROR(yr_ac_compile(
compiler->automaton,
compiler->automaton_arena,
&tables);
compiler->arena));
if (result == ERROR_SUCCESS)
result = yr_arena_create(1024, ARENA_FLAGS_RELOCATABLE, &arena);
YR_ARENA_REF ref;
if (result == ERROR_SUCCESS)
result = yr_arena_allocate_struct(
arena,
sizeof(YARA_RULES_FILE_HEADER),
(void**) &rules_file_header,
offsetof(YARA_RULES_FILE_HEADER, rules_list_head),
offsetof(YARA_RULES_FILE_HEADER, externals_list_head),
offsetof(YARA_RULES_FILE_HEADER, code_start),
offsetof(YARA_RULES_FILE_HEADER, ac_match_table),
offsetof(YARA_RULES_FILE_HEADER, ac_transition_table),
EOL);
FAIL_ON_ERROR(yr_arena_allocate_struct(
compiler->arena,
YR_SUMMARY_SECTION,
sizeof(YR_SUMMARY),
&ref,
EOL));
if (result == ERROR_SUCCESS)
{
rules_file_header->rules_list_head = (YR_RULE*) yr_arena_base_address(
compiler->rules_arena);
YR_SUMMARY* summary = (YR_SUMMARY*) yr_arena_ref_to_ptr(
compiler->arena, &ref);
rules_file_header->externals_list_head = (YR_EXTERNAL_VARIABLE*)
yr_arena_base_address(compiler->externals_arena);
summary->num_namespaces = compiler->num_namespaces;
summary->num_rules = compiler->next_rule_idx;
summary->num_strings = compiler->current_string_idx;
rules_file_header->code_start = (uint8_t*) yr_arena_base_address(
compiler->code_arena);
rules_file_header->ac_match_table = tables.matches;
rules_file_header->ac_transition_table = tables.transitions;
rules_file_header->ac_tables_size = compiler->automaton->tables_size;
}
if (result == ERROR_SUCCESS)
{
result = yr_arena_append(
arena,
compiler->code_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->code_arena = NULL;
result = yr_arena_append(
arena,
compiler->re_code_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->re_code_arena = NULL;
result = yr_arena_append(
arena,
compiler->rules_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->rules_arena = NULL;
result = yr_arena_append(
arena,
compiler->strings_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->strings_arena = NULL;
result = yr_arena_append(
arena,
compiler->externals_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->externals_arena = NULL;
result = yr_arena_append(
arena,
compiler->namespaces_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->namespaces_arena = NULL;
result = yr_arena_append(
arena,
compiler->metas_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->metas_arena = NULL;
result = yr_arena_append(
arena,
compiler->sz_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->sz_arena = NULL;
result = yr_arena_append(
arena,
compiler->automaton_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->automaton_arena = NULL;
result = yr_arena_append(
arena,
compiler->matches_arena);
}
if (result == ERROR_SUCCESS)
{
compiler->matches_arena = NULL;
compiler->compiled_rules_arena = arena;
result = yr_arena_coalesce(arena);
}
else
{
yr_arena_destroy(arena);
}
return result;
return yr_rules_from_arena(compiler->arena, &compiler->rules);
}
@@ -836,48 +693,16 @@ YR_API int yr_compiler_get_rules(
YR_COMPILER* compiler,
YR_RULES** rules)
{
YR_RULES* yara_rules;
YARA_RULES_FILE_HEADER* rules_file_header;
// Don't allow calls to yr_compiler_get_rules() if a previous call to
// yr_compiler_add_XXXX failed.
assert(compiler->errors == 0);
*rules = NULL;
if (compiler->compiled_rules_arena == NULL)
if (compiler->rules == NULL)
FAIL_ON_ERROR(_yr_compiler_compile_rules(compiler));
yara_rules = (YR_RULES*) yr_malloc(sizeof(YR_RULES));
if (yara_rules == NULL)
return ERROR_INSUFFICIENT_MEMORY;
FAIL_ON_ERROR_WITH_CLEANUP(
yr_arena_duplicate(compiler->compiled_rules_arena, &yara_rules->arena),
yr_free(yara_rules));
rules_file_header = (YARA_RULES_FILE_HEADER*) yr_arena_base_address(
yara_rules->arena);
yara_rules->externals_list_head = rules_file_header->externals_list_head;
yara_rules->rules_list_head = rules_file_header->rules_list_head;
yara_rules->ac_match_table = rules_file_header->ac_match_table;
yara_rules->ac_transition_table = rules_file_header->ac_transition_table;
yara_rules->ac_tables_size = rules_file_header->ac_tables_size;
yara_rules->code_start = rules_file_header->code_start;
yara_rules->time_cost = 0;
memset(yara_rules->tidx_mask, 0, sizeof(yara_rules->tidx_mask));
FAIL_ON_ERROR_WITH_CLEANUP(
yr_mutex_create(&yara_rules->mutex),
// cleanup
yr_arena_destroy(yara_rules->arena);
yr_free(yara_rules));
*rules = yara_rules;
*rules = compiler->rules;
return ERROR_SUCCESS;
}
@@ -889,8 +714,6 @@ static int _yr_compiler_define_variable(
YR_EXTERNAL_VARIABLE* ext;
YR_OBJECT* object;
char* id;
if (external->identifier == NULL)
return ERROR_INVALID_ARGUMENT;
@@ -902,52 +725,64 @@ static int _yr_compiler_define_variable(
if (object != NULL)
return ERROR_DUPLICATED_EXTERNAL_VARIABLE;
FAIL_ON_ERROR(yr_arena_write_string(
compiler->sz_arena,
external->identifier,
&id));
YR_ARENA_REF ext_ref;
YR_ARENA_REF ref;
FAIL_ON_ERROR(yr_arena_allocate_struct(
compiler->externals_arena,
compiler->arena,
YR_EXTERNAL_VARIABLES_TABLE,
sizeof(YR_EXTERNAL_VARIABLE),
(void**) &ext,
&ext_ref,
offsetof(YR_EXTERNAL_VARIABLE, identifier),
EOL));
ext->identifier = id;
ext = (YR_EXTERNAL_VARIABLE*) yr_arena_ref_to_ptr(
compiler->arena, &ext_ref);
FAIL_ON_ERROR(yr_arena_write_string(
compiler->arena,
YR_SZ_POOL,
external->identifier,
&ref));
ext->identifier = (const char*) yr_arena_ref_to_ptr(
compiler->arena, &ref);
ext->type = external->type;
ext->value = external->value;
if (external->type == EXTERNAL_VARIABLE_TYPE_STRING)
{
char* val;
if (external->value.s == NULL)
return ERROR_INVALID_ARGUMENT;
FAIL_ON_ERROR(yr_arena_write_string(
compiler->sz_arena,
compiler->arena,
YR_SZ_POOL,
external->value.s,
&val));
ext->value.s = val;
&ref));
FAIL_ON_ERROR(yr_arena_make_ptr_relocatable(
compiler->externals_arena,
ext,
offsetof(YR_EXTERNAL_VARIABLE, value.s),
compiler->arena,
YR_EXTERNAL_VARIABLES_TABLE,
ext_ref.offset + offsetof(YR_EXTERNAL_VARIABLE, value.s),
EOL));
ext->value.s = (char*) yr_arena_ref_to_ptr(
compiler->arena, &ref);
}
FAIL_ON_ERROR(yr_object_from_external_variable(
external,
&object));
FAIL_ON_ERROR(yr_hash_table_add(
FAIL_ON_ERROR_WITH_CLEANUP(yr_hash_table_add(
compiler->objects_table,
external->identifier,
NULL,
(void*) object));
(void*) object),
// cleanup
yr_object_destroy(object));
return ERROR_SUCCESS;
}
libyara/exception.h
+25 -19
View File
@@ -31,6 +31,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define YR_EXCEPTION_H
#include <assert.h>
#include <yara/globals.h>
#if _WIN32 || __CYGWIN__
@@ -75,19 +76,20 @@ static LONG CALLBACK exception_handler(
#include <setjmp.h>
jmp_buf *exc_jmp_buf[YR_MAX_THREADS];
static LONG CALLBACK exception_handler(
PEXCEPTION_POINTERS ExceptionInfo)
{
int tidx = yr_get_tidx();
jmp_buf* jb_ptr;
switch(ExceptionInfo->ExceptionRecord->ExceptionCode)
{
case EXCEPTION_IN_PAGE_ERROR:
case EXCEPTION_ACCESS_VIOLATION:
if (tidx != -1 && exc_jmp_buf[tidx] != NULL)
longjmp(*exc_jmp_buf[tidx], 1);
jb_ptr = (jmp_buf*) yr_thread_storage_get_value(
&yr_trycatch_trampoline_tls);
if (jb_ptr != NULL)
longjmp(*jb_ptr, 1);
}
return EXCEPTION_CONTINUE_SEARCH;
@@ -99,16 +101,17 @@ static LONG CALLBACK exception_handler(
if (_do_) \
{ \
jmp_buf jb; \
/* Store pointer to sigjmp_buf in TLS */ \
yr_thread_storage_set_value( \
&yr_trycatch_trampoline_tls, &jb); \
HANDLE exh = AddVectoredExceptionHandler(1, exception_handler); \
int tidx = yr_get_tidx(); \
assert(tidx != -1); \
exc_jmp_buf[tidx] = &jb; \
if (setjmp(jb) == 0) \
{ _try_clause_ } \
else \
{ _catch_clause_ } \
exc_jmp_buf[tidx] = NULL; \
RemoveVectoredExceptionHandler(exh); \
yr_thread_storage_set_value( \
&yr_trycatch_trampoline_tls, NULL); \
} \
else \
{ \
@@ -122,16 +125,16 @@ static LONG CALLBACK exception_handler(
#include <setjmp.h>
#include <signal.h>
sigjmp_buf *exc_jmp_buf[YR_MAX_THREADS];
#include <yara/globals.h>
static void exception_handler(int sig) {
if (sig == SIGBUS || sig == SIGSEGV)
{
int tidx = yr_get_tidx();
jmp_buf* jb_ptr = (jmp_buf*) yr_thread_storage_get_value(
&yr_trycatch_trampoline_tls);
if (tidx != -1 && exc_jmp_buf[tidx] != NULL)
siglongjmp(*exc_jmp_buf[tidx], 1);
if (jb_ptr != NULL)
siglongjmp(*jb_ptr, 1);
}
}
@@ -145,22 +148,25 @@ typedef struct sigaction sa;
struct sigaction old_sigbus_act; \
struct sigaction old_sigsegv_act; \
struct sigaction act; \
sigjmp_buf jb; \
/* Store pointer to sigjmp_buf in TLS */ \
yr_thread_storage_set_value( \
&yr_trycatch_trampoline_tls, &jb); \
/* Set exception handler for SIGBUS and SIGSEGV*/ \
act.sa_handler = exception_handler; \
act.sa_flags = 0; /* SA_ONSTACK? */ \
sigfillset(&act.sa_mask); \
sigaction(SIGBUS, &act, &old_sigbus_act); \
sigaction(SIGSEGV, &act, &old_sigsegv_act); \
int tidx = yr_get_tidx(); \
assert(tidx != -1); \
sigjmp_buf jb; \
exc_jmp_buf[tidx] = &jb; \
if (sigsetjmp(jb, 1) == 0) \
{ _try_clause_ } \
else \
{ _catch_clause_ } \
exc_jmp_buf[tidx] = NULL; \
/* Stop capturing SIGBUS and SIGSEGV */ \
sigaction(SIGBUS, &old_sigbus_act, NULL); \
sigaction(SIGSEGV, &old_sigsegv_act, NULL); \
yr_thread_storage_set_value( \
&yr_trycatch_trampoline_tls, NULL); \
} \
else \
{ \
libyara/exec.c
+160 -140
View File
@@ -60,7 +60,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Paranoid mode does not guarantee that it's safe to load compiled rules from
// third parties, it only prevents severe security issues. Maliciously crafted
// compiled rules can still crash YARA. Loading third-party compiled rules is
// *highly* undiscouraged. If you need to distribute YARA rules in compiled
// *highly* discouraged. If you need to distribute YARA rules in compiled
// form you should encapsulate them in some digitally-signed package that
// ensure that they haven't been modified by someone else.
@@ -69,7 +69,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endif
#define MEM_SIZE YR_MAX_LOOP_NESTING * YR_MAX_LOOP_VARS
#define MEM_SIZE \
YR_MAX_LOOP_NESTING * (YR_MAX_LOOP_VARS + YR_INTERNAL_LOOP_VARS)
#define push(x) \
@@ -92,7 +93,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define ensure_defined(x) \
if (is_undef(x)) \
{ \
r1.i = UNDEFINED; \
r1.i = YR_UNDEFINED; \
push(r1); \
break; \
}
@@ -107,11 +108,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Make sure that the string pointer is within the rules arena.
#define ensure_within_rules_arena(x) \
if (yr_arena_page_for_address(context->rules->arena, x) == NULL) \
{ \
stop = true; \
result = ERROR_INTERNAL_FATAL_ERROR; \
break; \
YR_ARENA_REF ref; \
if (yr_arena_ptr_to_ref(context->rules->arena, x, &ref) == 0) \
{ \
stop = true; \
result = ERROR_INTERNAL_FATAL_ERROR; \
break; \
} \
}
#define check_object_canary(o) \
@@ -150,14 +154,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
type result; \
const uint8_t* data = block->fetch_data(block); \
if (data == NULL) \
return UNDEFINED; \
return YR_UNDEFINED; \
result = *(type *)(data + offset - block->base); \
result = endianess##_##type(result); \
return result; \
} \
block = iterator->next(iterator); \
} \
return UNDEFINED; \
return YR_UNDEFINED; \
};
@@ -179,18 +183,24 @@ static const uint8_t* jmp_if(
int condition,
const uint8_t* ip)
{
const uint8_t* result;
size_t off;
if (condition)
{
result = *(const uint8_t**)(ip);
// The condition is true, the instruction pointer is incremented in the
// amount specified by the jump's offset. The offset is relative to the
// jump opcode, but now the instruction pointer is pointing past the opcode
// that's why we decrement the offset by 1.
off = *(int32_t*)(ip) - 1;
}
else
{
result = ip + sizeof(uint64_t);
// The condition is false, the execution flow proceeds with the instruction
// right after the jump.
off = sizeof(int32_t);
}
return result;
return ip + off;
}
@@ -217,7 +227,7 @@ static int iter_array_next(
if (obj != NULL)
stack->items[stack->sp++].o = obj;
else
stack->items[stack->sp++].i = UNDEFINED;
stack->items[stack->sp++].i = YR_UNDEFINED;
self->array_it.index++;
}
@@ -225,8 +235,8 @@ static int iter_array_next(
{
// Push true for indicating the iterator has been exhausted.
stack->items[stack->sp++].i = 1;
// Push UNDEFINED as a placeholder for the next item.
stack->items[stack->sp++].i = UNDEFINED;
// Push YR_UNDEFINED as a placeholder for the next item.
stack->items[stack->sp++].i = YR_UNDEFINED;
}
return ERROR_SUCCESS;
@@ -245,7 +255,17 @@ static int iter_dict_next(
if (stack->sp + 2 >= stack->capacity)
return ERROR_EXEC_STACK_OVERFLOW;
if (self->dict_it.index < items->used)
// If the dictionary has no items or the iterator reached the last item, abort
// the iteration, if not push the next key and value.
if (items == NULL || self->dict_it.index == items->used)
{
// Push true for indicating the iterator has been exhausted.
stack->items[stack->sp++].i = 1;
// Push YR_UNDEFINED as a placeholder for the next key and value.
stack->items[stack->sp++].i = YR_UNDEFINED;
stack->items[stack->sp++].i = YR_UNDEFINED;
}
else
{
// Push the false value that indicates that the iterator is not exhausted.
stack->items[stack->sp++].i = 0;
@@ -257,20 +277,12 @@ static int iter_dict_next(
}
else
{
stack->items[stack->sp++].i = UNDEFINED;
stack->items[stack->sp++].i = UNDEFINED;
stack->items[stack->sp++].i = YR_UNDEFINED;
stack->items[stack->sp++].i = YR_UNDEFINED;
}
self->dict_it.index++;
}
else
{
// Push true for indicating the iterator has been exhausted.
stack->items[stack->sp++].i = 1;
// Push UNDEFINED as a placeholder for the next key and value.
stack->items[stack->sp++].i = UNDEFINED;
stack->items[stack->sp++].i = UNDEFINED;
}
return ERROR_SUCCESS;
}
@@ -297,8 +309,8 @@ static int iter_int_range_next(
{
// Push true for indicating the iterator has been exhausted.
stack->items[stack->sp++].i = 1;
// Push UNDEFINED as a placeholder for the next item.
stack->items[stack->sp++].i = UNDEFINED;
// Push YR_UNDEFINED as a placeholder for the next item.
stack->items[stack->sp++].i = YR_UNDEFINED;
}
return ERROR_SUCCESS;
@@ -326,8 +338,8 @@ static int iter_int_enum_next(
{
// Push true for indicating the iterator has been exhausted.
stack->items[stack->sp++].i = 1;
// Push UNDEFINED as a placeholder for the next item.
stack->items[stack->sp++].i = UNDEFINED;
// Push YR_UNDEFINED as a placeholder for the next item.
stack->items[stack->sp++].i = YR_UNDEFINED;
}
return ERROR_SUCCESS;
@@ -352,17 +364,16 @@ int yr_execute_code(
#ifdef PROFILING_ENABLED
uint64_t start_time;
YR_RULE* current_rule = NULL;
#endif
YR_INIT_RULE_ARGS init_rule_args;
uint32_t current_rule_idx;
YR_RULE* current_rule = NULL;
YR_RULE* rule;
YR_MATCH* match;
YR_OBJECT_FUNCTION* function;
YR_OBJECT** obj_ptr;
YR_ARENA* obj_arena;
YR_ARENA* it_arena;
YR_NOTEBOOK* it_notebook;
char* identifier;
char* args_fmt;
@@ -372,7 +383,7 @@ int yr_execute_code(
int count;
int result = ERROR_SUCCESS;
int cycle = 0;
int tidx = context->tidx;
int obj_count = 0;
bool stop = false;
@@ -387,16 +398,16 @@ int yr_execute_code(
return ERROR_INSUFFICIENT_MEMORY;
FAIL_ON_ERROR_WITH_CLEANUP(
yr_arena_create(1024, 0, &obj_arena),
yr_arena_create(1, 512 * sizeof(YR_OBJECT*), &obj_arena),
yr_free(stack.items));
FAIL_ON_ERROR_WITH_CLEANUP(
yr_arena_create(64 * sizeof(YR_ITERATOR), 0, &it_arena),
yr_arena_destroy(obj_arena);
yr_notebook_create(512 * sizeof(YR_ITERATOR), &it_notebook),
yr_arena_release(obj_arena);
yr_free(stack.items));
#ifdef PROFILING_ENABLED
start_time = yr_stopwatch_elapsed_us(&context->stopwatch);
start_time = yr_stopwatch_elapsed_ns(&context->stopwatch);
#endif
#if PARANOID_EXEC
@@ -405,7 +416,10 @@ int yr_execute_code(
while(!stop)
{
// Read the opcode from the address indicated by the instruction pointer.
opcode = *ip;
// Advance the instruction pointer, which now points past the opcode.
ip++;
switch(opcode)
@@ -419,10 +433,13 @@ int yr_execute_code(
break;
case OP_ITER_START_ARRAY:
result = yr_arena_allocate_struct(
it_arena, sizeof(YR_ITERATOR), &r2.p);
r2.p = yr_notebook_alloc(it_notebook, sizeof(YR_ITERATOR));
if (result == ERROR_SUCCESS)
if (r2.p == NULL)
{
result = ERROR_INSUFFICIENT_MEMORY;
}
else
{
pop(r1);
r2.it->array_it.array = r1.o;
@@ -435,10 +452,13 @@ int yr_execute_code(
break;
case OP_ITER_START_DICT:
result = yr_arena_allocate_struct(
it_arena, sizeof(YR_ITERATOR), &r2.p);
r2.p = yr_notebook_alloc(it_notebook, sizeof(YR_ITERATOR));
if (result == ERROR_SUCCESS)
if (r2.p == NULL)
{
result = ERROR_INSUFFICIENT_MEMORY;
}
else
{
pop(r1);
r2.it->dict_it.dict = r1.o;
@@ -453,10 +473,13 @@ int yr_execute_code(
case OP_ITER_START_INT_RANGE:
// Creates an iterator for an integer range. The higher bound of the
// range is at the top of the stack followed by the lower bound.
result = yr_arena_allocate_struct(
it_arena, sizeof(YR_ITERATOR), &r3.p);
r3.p = yr_notebook_alloc(it_notebook, sizeof(YR_ITERATOR));
if (result == ERROR_SUCCESS)
if (r3.p == NULL)
{
result = ERROR_INSUFFICIENT_MEMORY;
}
else
{
pop(r2);
pop(r1);
@@ -475,10 +498,14 @@ int yr_execute_code(
// items in reverse order.
pop(r1);
result = yr_arena_allocate_struct(
it_arena, sizeof(YR_ITERATOR) + sizeof(uint64_t) * r1.i, &r3.p);
r3.p = yr_notebook_alloc(
it_notebook, sizeof(YR_ITERATOR) + sizeof(uint64_t) * r1.i);
if (result == ERROR_SUCCESS)
if (r3.p == NULL)
{
result = ERROR_INSUFFICIENT_MEMORY;
}
else
{
r3.it->int_enum_it.count = r1.i;
r3.it->int_enum_it.next = 0;
@@ -678,7 +705,7 @@ int yr_execute_code(
pop(r1);
if (is_undef(r1))
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
else
r1.i = !r1.i;
@@ -693,7 +720,7 @@ int yr_execute_code(
if (r2.i != 0)
r1.i = r1.i % r2.i;
else
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
push(r1);
break;
@@ -703,7 +730,7 @@ int yr_execute_code(
ensure_defined(r2);
ensure_defined(r1);
if (r2.i < 0)
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
else if (r2.i < 64)
r1.i = r1.i >> r2.i;
else
@@ -717,7 +744,7 @@ int yr_execute_code(
ensure_defined(r2);
ensure_defined(r1);
if (r2.i < 0)
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
else if (r2.i < 64)
r1.i = r1.i << r2.i;
else
@@ -760,44 +787,62 @@ int yr_execute_code(
break;
case OP_PUSH_RULE:
rule = *(YR_RULE**)(ip);
r1.i = *(uint64_t*)(ip);
ip += sizeof(uint64_t);
rule = &context->rules->rules_list_head[r1.i];
if (RULE_IS_DISABLED(rule))
r1.i = UNDEFINED;
{
r2.i = YR_UNDEFINED;
}
else
r1.i = rule->t_flags[tidx] & RULE_TFLAGS_MATCH ? 1 : 0;
push(r1);
{
if yr_bitmask_is_set(context->rule_matches_flags, r1.i)
r2.i = 1;
else
r2.i = 0;
}
push(r2);
break;
case OP_INIT_RULE:
memcpy(&init_rule_args, ip, sizeof(init_rule_args));
#ifdef PROFILING_ENABLED
current_rule = init_rule_args.rule;
#endif
if (RULE_IS_DISABLED(init_rule_args.rule))
ip = init_rule_args.jmp_addr;
else
ip += sizeof(init_rule_args);
// After the opcode there's an int32_t corresponding to the jump's
// offset and an uint32_t corresponding to the rule's index.
current_rule_idx = *(uint32_t*)(ip + sizeof(int32_t));
current_rule = &context->rules->rules_list_head[current_rule_idx];
// If the rule is disabled let's skip its code.
ip = jmp_if(RULE_IS_DISABLED(current_rule), ip);
// Skip the bytes corresponding to the rule's index, but only if not
// taking the jump.
if (!RULE_IS_DISABLED(current_rule))
ip += sizeof(uint32_t);
break;
case OP_MATCH_RULE:
pop(r1);
rule = *(YR_RULE**)(ip);
r2.i = *(uint64_t*)(ip);
ip += sizeof(uint64_t);
rule = &context->rules->rules_list_head[r2.i];
#if PARANOID_EXEC
ensure_within_rules_arena(rule);
#endif
ip += sizeof(uint64_t);
if (!is_undef(r1) && r1.i)
rule->t_flags[tidx] |= RULE_TFLAGS_MATCH;
yr_bitmask_set(context->rule_matches_flags, r2.i);
else if (RULE_IS_GLOBAL(rule))
rule->ns->t_flags[tidx] |= NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL;
yr_bitmask_set(context->ns_unsatisfied_flags, rule->ns->idx);
#ifdef PROFILING_ENABLED
elapsed_time = yr_stopwatch_elapsed_us(&context->stopwatch);
rule->time_cost_per_thread[tidx] += (elapsed_time - start_time);
elapsed_time = yr_stopwatch_elapsed_ns(&context->stopwatch);
context->time_cost[r2.i] += (elapsed_time - start_time);
start_time = elapsed_time;
#endif
@@ -846,14 +891,14 @@ int yr_execute_code(
case OBJECT_TYPE_FLOAT:
if (isnan(r1.o->value.d))
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
else
r1.d = r1.o->value.d;
break;
case OBJECT_TYPE_STRING:
if (r1.o->value.ss == NULL)
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
else
r1.ss = r1.o->value.ss;
break;
@@ -881,7 +926,7 @@ int yr_execute_code(
r1.o = yr_object_array_get_item(r2.o, 0, (int) r1.i);
if (r1.o == NULL)
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
push(r1);
break;
@@ -903,7 +948,7 @@ int yr_execute_code(
r2.o, 0, r1.ss->c_string);
if (r1.o == NULL)
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
push(r1);
break;
@@ -946,10 +991,10 @@ int yr_execute_code(
if (count > 0)
{
// if there are undefined args, result for function call
// If there are undefined args, result for function call
// is undefined as well.
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
push(r1);
break;
}
@@ -969,23 +1014,25 @@ int yr_execute_code(
}
}
// if i == YR_MAX_OVERLOADED_FUNCTIONS at this point no matching
// If i == YR_MAX_OVERLOADED_FUNCTIONS at this point no matching
// prototype was found, but this shouldn't happen.
assert(i < YR_MAX_OVERLOADED_FUNCTIONS);
// make a copy of the returned object and push the copy into the stack,
// Make a copy of the returned object and push the copy into the stack,
// function->return_obj can't be pushed because it can change in
// subsequent calls to the same function.
if (result == ERROR_SUCCESS)
result = yr_object_copy(function->return_obj, &r1.o);
// a pointer to the copied object is stored in a arena in order to
// free the object before exiting yr_execute_code
// A pointer to the copied object is stored in a arena in order to
// free the object before exiting yr_execute_code, obj_count tracks
// the number of objects written.
if (result == ERROR_SUCCESS)
result = yr_arena_write_data(obj_arena, &r1.o, sizeof(r1.o), NULL);
{
result = yr_arena_write_data(
obj_arena, 0, &r1.o, sizeof(r1.o), NULL);
obj_count++;
}
stop = (result != ERROR_SUCCESS);
push(r1);
@@ -993,12 +1040,7 @@ int yr_execute_code(
case OP_FOUND:
pop(r1);
if (STRING_IS_PRIVATE(r1.s))
r2.i = r1.s->private_matches[tidx].tail != NULL ? 1 : 0;
else
r2.i = r1.s->matches[tidx].tail != NULL ? 1 : 0;
r2.i = context->matches[r1.s->idx].tail != NULL ? 1 : 0;
push(r2);
break;
@@ -1017,11 +1059,7 @@ int yr_execute_code(
ensure_within_rules_arena(r2.p);
#endif
if (STRING_IS_PRIVATE(r2.s))
match = r2.s->private_matches[tidx].head;
else
match = r2.s->matches[tidx].head;
match = context->matches[r2.s->idx].head;
r3.i = false;
while (match != NULL)
@@ -1053,11 +1091,7 @@ int yr_execute_code(
ensure_within_rules_arena(r3.p);
#endif
if (STRING_IS_PRIVATE(r3.s))
match = r3.s->private_matches[tidx].head;
else
match = r3.s->matches[tidx].head;
match = context->matches[r3.s->idx].head;
r4.i = false;
while (match != NULL && !r4.i)
@@ -1084,11 +1118,7 @@ int yr_execute_code(
ensure_within_rules_arena(r1.p);
#endif
if (STRING_IS_PRIVATE(r1.s))
r2.i = r1.s->private_matches[tidx].count;
else
r2.i = r1.s->matches[tidx].count;
r2.i = context->matches[r1.s->idx].count;
push(r2);
break;
@@ -1102,15 +1132,12 @@ int yr_execute_code(
ensure_within_rules_arena(r2.p);
#endif
if (STRING_IS_PRIVATE(r2.s))
match = r2.s->private_matches[tidx].head;
else
match = r2.s->matches[tidx].head;
match = context->matches[r2.s->idx].head;
i = 1;
r3.i = UNDEFINED;
r3.i = YR_UNDEFINED;
while (match != NULL && r3.i == UNDEFINED)
while (match != NULL && r3.i == YR_UNDEFINED)
{
if (r1.i == i)
r3.i = match->base + match->offset;
@@ -1132,15 +1159,12 @@ int yr_execute_code(
ensure_within_rules_arena(r2.p);
#endif
if (STRING_IS_PRIVATE(r2.s))
match = r2.s->private_matches[tidx].head;
else
match = r2.s->matches[tidx].head;
match = context->matches[r2.s->idx].head;
i = 1;
r3.i = UNDEFINED;
r3.i = YR_UNDEFINED;
while (match != NULL && r3.i == UNDEFINED)
while (match != NULL && r3.i == YR_UNDEFINED)
{
if (r1.i == i)
r3.i = match->match_length;
@@ -1159,8 +1183,10 @@ int yr_execute_code(
while (!is_undef(r1))
{
if (r1.s->matches[tidx].tail != NULL || r1.s->private_matches[tidx].tail != NULL)
if (context->matches[r1.s->idx].tail != NULL)
{
found++;
}
count++;
pop(r1);
}
@@ -1330,7 +1356,7 @@ int yr_execute_code(
r2 = stack.items[stack.sp - r1.i];
if (is_undef(r2))
stack.items[stack.sp - r1.i].i = UNDEFINED;
stack.items[stack.sp - r1.i].i = YR_UNDEFINED;
else
stack.items[stack.sp - r1.i].d = (double) r2.i;
break;
@@ -1431,7 +1457,7 @@ int yr_execute_code(
if (r2.i != 0)
r1.i = r1.i / r2.i;
else
r1.i = UNDEFINED;
r1.i = YR_UNDEFINED;
push(r1);
break;
@@ -1582,18 +1608,17 @@ int yr_execute_code(
assert(false);
}
// Check for timeout every 10 instruction cycles. If timeout == 0 it means
// Check for timeout every 100 instruction cycles. If timeout == 0 it means
// no timeout at all.
if (context->timeout > 0L && ++cycle == 10)
if (context->timeout > 0L && ++cycle == 100)
{
elapsed_time = yr_stopwatch_elapsed_us(&context->stopwatch);
elapsed_time = yr_stopwatch_elapsed_ns(&context->stopwatch);
if (elapsed_time > context->timeout)
{
#ifdef PROFILING_ENABLED
assert(current_rule != NULL);
current_rule->time_cost_per_thread[tidx] += elapsed_time - start_time;
context->time_cost[current_rule_idx] += (elapsed_time - start_time);
#endif
result = ERROR_SCAN_TIMEOUT;
stop = true;
@@ -1603,18 +1628,13 @@ int yr_execute_code(
}
}
obj_ptr = (YR_OBJECT**) yr_arena_base_address(obj_arena);
obj_ptr = yr_arena_get_ptr(obj_arena, 0, 0);
while (obj_ptr != NULL)
{
yr_object_destroy(*obj_ptr);
for (int i = 0; i < obj_count; i++)
yr_object_destroy(obj_ptr[i]);
obj_ptr = (YR_OBJECT**) yr_arena_next_address(
obj_arena, obj_ptr, sizeof(YR_OBJECT*));
}
yr_arena_destroy(obj_arena);
yr_arena_destroy(it_arena);
yr_arena_release(obj_arena);
yr_notebook_destroy(it_notebook);
yr_modules_unload_all(context);
yr_free(stack.items);
libyara/exefiles.c
+2 -2
View File
@@ -379,7 +379,7 @@ uint64_t yr_get_entry_point_offset(
buffer_length);
}
return UNDEFINED;
return YR_UNDEFINED;
}
@@ -422,5 +422,5 @@ uint64_t yr_get_entry_point_address(
break;
}
return UNDEFINED;
return YR_UNDEFINED;
}
libyara/grammar.c
+1226 -1024
View File
File diff suppressed because it is too large Load Diff
libyara/grammar.h
+69 -63
View File
@@ -1,8 +1,8 @@
/* A Bison parser, made by GNU Bison 3.0.5. */
/* A Bison parser, made by GNU Bison 3.0.4. */
/* Bison interface for Yacc-like parsers in C
Copyright (C) 1984, 1989-1990, 2000-2015, 2018 Free Software Foundation, Inc.
Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -69,34 +69,36 @@ extern int yara_yydebug;
_ASCII_ = 278,
_WIDE_ = 279,
_XOR_ = 280,
_NOCASE_ = 281,
_FULLWORD_ = 282,
_AT_ = 283,
_FILESIZE_ = 284,
_ENTRYPOINT_ = 285,
_ALL_ = 286,
_ANY_ = 287,
_IN_ = 288,
_OF_ = 289,
_FOR_ = 290,
_THEM_ = 291,
_MATCHES_ = 292,
_CONTAINS_ = 293,
_IMPORT_ = 294,
_TRUE_ = 295,
_FALSE_ = 296,
_OR_ = 297,
_AND_ = 298,
_NOT_ = 299,
_EQ_ = 300,
_NEQ_ = 301,
_LT_ = 302,
_LE_ = 303,
_GT_ = 304,
_GE_ = 305,
_SHIFT_LEFT_ = 306,
_SHIFT_RIGHT_ = 307,
UNARY_MINUS = 308
_BASE64_ = 281,
_BASE64_WIDE_ = 282,
_NOCASE_ = 283,
_FULLWORD_ = 284,
_AT_ = 285,
_FILESIZE_ = 286,
_ENTRYPOINT_ = 287,
_ALL_ = 288,
_ANY_ = 289,
_IN_ = 290,
_OF_ = 291,
_FOR_ = 292,
_THEM_ = 293,
_MATCHES_ = 294,
_CONTAINS_ = 295,
_IMPORT_ = 296,
_TRUE_ = 297,
_FALSE_ = 298,
_OR_ = 299,
_AND_ = 300,
_NOT_ = 301,
_EQ_ = 302,
_NEQ_ = 303,
_LT_ = 304,
_LE_ = 305,
_GT_ = 306,
_GE_ = 307,
_SHIFT_LEFT_ = 308,
_SHIFT_RIGHT_ = 309,
UNARY_MINUS = 310
};
#endif
/* Tokens. */
@@ -124,53 +126,57 @@ extern int yara_yydebug;
#define _ASCII_ 278
#define _WIDE_ 279
#define _XOR_ 280
#define _NOCASE_ 281
#define _FULLWORD_ 282
#define _AT_ 283
#define _FILESIZE_ 284
#define _ENTRYPOINT_ 285
#define _ALL_ 286
#define _ANY_ 287
#define _IN_ 288
#define _OF_ 289
#define _FOR_ 290
#define _THEM_ 291
#define _MATCHES_ 292
#define _CONTAINS_ 293
#define _IMPORT_ 294
#define _TRUE_ 295
#define _FALSE_ 296
#define _OR_ 297
#define _AND_ 298
#define _NOT_ 299
#define _EQ_ 300
#define _NEQ_ 301
#define _LT_ 302
#define _LE_ 303
#define _GT_ 304
#define _GE_ 305
#define _SHIFT_LEFT_ 306
#define _SHIFT_RIGHT_ 307
#define UNARY_MINUS 308
#define _BASE64_ 281
#define _BASE64_WIDE_ 282
#define _NOCASE_ 283
#define _FULLWORD_ 284
#define _AT_ 285
#define _FILESIZE_ 286
#define _ENTRYPOINT_ 287
#define _ALL_ 288
#define _ANY_ 289
#define _IN_ 290
#define _OF_ 291
#define _FOR_ 292
#define _THEM_ 293
#define _MATCHES_ 294
#define _CONTAINS_ 295
#define _IMPORT_ 296
#define _TRUE_ 297
#define _FALSE_ 298
#define _OR_ 299
#define _AND_ 300
#define _NOT_ 301
#define _EQ_ 302
#define _NEQ_ 303
#define _LT_ 304
#define _LE_ 305
#define _GT_ 306
#define _GE_ 307
#define _SHIFT_LEFT_ 308
#define _SHIFT_RIGHT_ 309
#define UNARY_MINUS 310
/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
union YYSTYPE
{
#line 276 "grammar.y" /* yacc.c:1916 */
#line 299 "grammar.y" /* yacc.c:1909 */
YR_EXPRESSION expression;
SIZED_STRING* sized_string;
char* c_string;
int64_t integer;
double double_;
YR_STRING* string;
YR_META* meta;
YR_RULE* rule;
YR_MODIFIER modifier;
#line 174 "grammar.h" /* yacc.c:1916 */
YR_ARENA_REF tag;
YR_ARENA_REF rule;
YR_ARENA_REF meta;
YR_ARENA_REF string;
#line 180 "grammar.h" /* yacc.c:1909 */
};
typedef union YYSTYPE YYSTYPE;
libyara/grammar.y
+428 -256
View File
File diff suppressed because it is too large Load Diff
libyara/hash.c
+34
View File
@@ -365,3 +365,37 @@ YR_API int yr_hash_table_add(
ns,
value);
}
YR_API uint32_t yr_hash_table_add_uint32(
YR_HASH_TABLE* table,
const char* key,
const char* ns,
uint32_t value)
{
// Don't allow values equal to UINT32_MAX or UINT32_MAX - 1.
if (value >= UINT32_MAX - 1)
return ERROR_INVALID_ARGUMENT;
// Add +1 to the value in order to avoid putting a NULL pointer in the
// hash table once the integer is casted to a pointer. This is undone
// by yr_hash_table_lookup_uint32.
return yr_hash_table_add(
table, key, ns, (void*) (size_t) (value + 1));
}
YR_API uint32_t yr_hash_table_lookup_uint32(
YR_HASH_TABLE* table,
const char* key,
const char* ns)
{
void* ptr = yr_hash_table_lookup(table, key, ns);
if (ptr == NULL)
return UINT32_MAX;
// Remove one from the pointer before converting back to integer, see
// comment in yr_hash_table_add_uint32.
return (uint32_t) (size_t) ((uint8_t*) ptr - 1);
}
libyara/hex_lexer.c
+301 -489
View File
File diff suppressed because it is too large Load Diff
libyara/hex_lexer.l
+15 -6
View File
@@ -181,14 +181,21 @@ hexdigit [a-fA-F0-9]
%%
//
// yyfatal (actually named hex_yyfatal because of the '%option prefix="hex_yy"'
// directive) is called when a fatal error occurs in the parser. When this
// happens we are deep inside the parsing logic generated by flex/bison and
// the only way to exit gracefully from there is using setjmp/longjmp.
//
void yyfatal(
yyscan_t yyscanner,
const char *error_message)
{
jmp_buf* recovery_state = (jmp_buf*) yr_thread_storage_get_value(
&yr_recovery_state_key);
jmp_buf* recovery_trampoline = (jmp_buf*) yr_thread_storage_get_value(
&yr_yyfatal_trampoline_tls);
longjmp(*recovery_state, 1);
// Never returns.
longjmp(*recovery_trampoline, 1);
}
void yyerror(
@@ -219,15 +226,17 @@ int yr_parse_hex_string(
RE_ERROR* error)
{
yyscan_t yyscanner;
jmp_buf recovery_state;
jmp_buf recovery_trampoline;
HEX_LEX_ENVIRONMENT lex_env;
lex_env.last_error = ERROR_SUCCESS;
lex_env.inside_or = 0;
yr_thread_storage_set_value(&yr_recovery_state_key, &recovery_state);
yr_thread_storage_set_value(
&yr_yyfatal_trampoline_tls,
&recovery_trampoline);
if (setjmp(recovery_state) != 0)
if (setjmp(recovery_trampoline) != 0)
return ERROR_INTERNAL_FATAL_ERROR;
FAIL_ON_ERROR(yr_re_ast_create(re_ast));
libyara/include/yara/ahocorasick.h
+3 -4
View File
@@ -62,15 +62,14 @@ int yr_ac_automaton_destroy(
int yr_ac_add_string(
YR_AC_AUTOMATON* automaton,
YR_STRING* string,
uint32_t string_idx,
YR_ATOM_LIST_ITEM* atom,
YR_ARENA* matches_arena);
YR_ARENA* arena);
int yr_ac_compile(
YR_AC_AUTOMATON* automaton,
YR_ARENA* arena,
YR_AC_TABLES* tables);
YR_ARENA* arena);
void yr_ac_print_automaton(
YR_AC_AUTOMATON* automaton);
libyara/include/yara/arena.h
+172 -81
View File
@@ -1,8 +1,8 @@
/*
Copyright (c) 2013. The YARA Authors. All Rights Reserved.
Copyright (c) 2020. The YARA Authors. All Rights Reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
@@ -12,18 +12,18 @@ this list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@@ -31,141 +31,241 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define YR_ARENA_H
#include <stddef.h>
#include <yara/limits.h>
#include <yara/integers.h>
#include <yara/stream.h>
// Indicated that the arena is self-contained and stored in a single page. A
// self-contained arenas is one that doesn't contains any pointers to outside
// data. All pointers in a self-contained arena points at some address within
// the arena.
#define ARENA_FLAGS_COALESCED 1
#define EOL ((uint32_t) -1)
// If an arena is created with this flag is keeps a list of YR_RELOC structures
// tracking the pointers stored within the arena. When the arena is relocated
// this allows to fix those pointers that pointed to some address within the
// relocated arena.
#define ARENA_FLAGS_RELOCATABLE 2
#define YR_ARENA_FILE_VERSION 17
// YARA can read an arena file (which contain compiled rules) created by a
// a different version of YARA as long as the arena file version remains the
// same. When changes in the structure of arena files are introduced, this
// version is bumped up.
//
// The arena file version is defined as ((version << 16) | YR_MAX_THREADS) so
// that the lowest 16 bits contain the value for YR_MAX_THREADS. Arena files
// generated with a YARA compiled with a certain value for YR_MAX_THREADS will
// be rejected by another YARA compiled with a different value, even if the
// "version" number is the same.
#define ARENA_FILE_VERSION ((23 << 16) | YR_MAX_THREADS)
#define YR_ARENA_NULL_REF \
(YR_ARENA_REF){ UINT32_MAX, UINT32_MAX }
#define EOL ((size_t) -1)
#define YR_ARENA_IS_NULL_REF(ref) \
(memcmp(&(ref), &YR_ARENA_NULL_REF, sizeof(YR_ARENA_NULL_REF)) == 0)
typedef uint32_t yr_arena_off_t;
typedef struct _YR_RELOC
typedef struct YR_ARENA YR_ARENA;
typedef struct YR_ARENA_BUFFER YR_ARENA_BUFFER;
typedef struct YR_ARENA_REF YR_ARENA_REF;
typedef struct YR_RELOC YR_RELOC;
// The YR_ARENA_REF must be 64-bits long because this structure occupies
// the place of relocatable pointers when arenas are saved to file. The pack(4)
// directive ensures that fields are aligned to 4-byte boundaries and the total
// size of the structure is 8 bytes. When compiling YARA for 32-bits, pointers
// are 4 bytes but this structure is still 8 bytes. For that reason structures
// that are saved in arenas must declare pointers using the DECLARE_REFERENCE
// macro, which ensures that 8 bytes are reserved for the pointer no matter if
// they are 32-bits pointers.
#pragma pack(push)
#pragma pack(4)
struct YR_ARENA_REF
{
uint32_t buffer_id;
uint32_t offset;
struct _YR_RELOC* next;
};
} YR_RELOC;
#pragma pack(pop)
typedef struct _YR_ARENA_PAGE
struct YR_ARENA_BUFFER
{
uint8_t* new_address;
uint8_t* address;
// Pointer the buffer's data.
uint8_t* data;
// Total buffer size, including the used and unused areas.
size_t size;
// Number of bytes that are actually used (equal to or lower than size).
size_t used;
YR_RELOC* reloc_list_head;
YR_RELOC* reloc_list_tail;
struct _YR_ARENA_PAGE* next;
struct _YR_ARENA_PAGE* prev;
} YR_ARENA_PAGE;
};
typedef struct _YR_ARENA
struct YR_RELOC
{
int flags;
// Buffer ID associated to this relocation entry.
uint8_t buffer_id;
YR_ARENA_PAGE* page_list_head;
YR_ARENA_PAGE* current_page;
// Offset within the buffer where the relocatable pointer resides.
yr_arena_off_t offset;
} YR_ARENA;
// Pointer to the next entry in the list.
struct YR_RELOC* next;
};
/*
YR_ARENA is the structure that represents an arena.
An arena is a set of buffers in which you can write arbitrary data. When the
arena is created you can specify how many buffers it will have, this number is
limited by YR_MAX_ARENA_BUFFERS. Buffers are numbered 0, 1, 2, and so on, and
they are referenced by its number.
Initially each buffer will have a size that is determined by an argument
passed to yr_create_arena, but the size will grow automatically as you write
data into it. When the buffer is re-sized it may be moved to a different
location in memory, which means that any pointer you hold to data contained
in the buffer is valid only until the next call to a function that may
cause a buffer re-sizing. For long-lived references to data in a buffer
you should use a YR_ARENA_REF, which is simply a struct storing the buffer
number and the offset of the data relative to the beginning of the buffer.
Such YR_ARENA_REF structures are returned by all functions that allocate space
or write data in a buffer.
The yr_arena_get_ptr function can be used for getting a pointer to data
stored in the buffer given its offset, but this pointer should be used with
only in a limited scope where you can be sure that your program is not calling
a function that may cause a buffer re-sizing, like yr_arena_allocate_xxx and
yr_arena_write_xxx.
Arenas can also keep track pointers stored in some of its buffers that point to
some location within the arena (it can be to the same buffer or a different one
within the same arena). This is done with yr_arena_make_ptr_relocatable, this
function receives a buffer number and an offset, and it tells the arena that
the pointer stored at that particular location of the buffer must be adjusted
every time the memory it points to is moved around.
The yr_arena_allocate_struct function is an even more powerful mechanism for
tracking relocatable pointers. This function allows to allocate space in a
buffer for a structure, while at the same time telling the arena which fields
in the structure are relocatable pointers.
*/
struct YR_ARENA
{
// Number of users of this arena. This is set to one when the arena is created,
// and can be incremented by calling yr_arena_acquire. On each call
// to yr_arena_release it gets decremented by one, if xrefs reaches zero
// the buffers and the YR_ARENA structures are freed.
int xrefs;
// Number of buffers in this arena.
int num_buffers;
// Status of individual buffers.
YR_ARENA_BUFFER buffers[YR_MAX_ARENA_BUFFERS];
// Initial size for each buffer.
size_t initial_buffer_size;
// Head of the list containing relocation entries.
YR_RELOC* reloc_list_head;
// Tail of the list containing relocation entries.
YR_RELOC* reloc_list_tail;
};
// Creates an arena with the specified number of buffers and takes ownership of
// it. Initially each buffer is empty, the first time that some data is written
// into a buffer at least initial_buffer_size are reserved for the buffer.
int yr_arena_create(
size_t initial_size,
int flags,
int num_buffers,
size_t initial_buffer_size,
YR_ARENA** arena);
void yr_arena_destroy(
// Takes ownership of the arena.
void yr_arena_acquire(
YR_ARENA* arena);
void* yr_arena_base_address(
// Release ownership of the arena. If the number of owners drops to zero the
// arena is destroyed and all its resources are freed.
int yr_arena_release(
YR_ARENA* arena);
YR_ARENA_PAGE* yr_arena_page_for_address(
// Given a reference to some data within the arena, it returns a pointer to
// the data. This pointer is valid only until the next call to any of the
// functions that allocates space in the buffer where the data resides, like
// yr_arena_allocate_xxx and yr_arena_write_xxx. These functions can cause
// the buffer to be moved to different memory location and the pointer won't
// valid any longer.
void* yr_arena_ref_to_ptr(
YR_ARENA* arena,
void* address);
YR_ARENA_REF* ref);
void* yr_arena_next_address(
// Given a pointer into the arena, it returns a reference to it. The reference
// can be used with yr_arena_ref_to_ptr to obtain a pointer again. Unlike
// pointers, references are during the arena's lifetime, even if the buffers
// are moved to a different memory location.
int yr_arena_ptr_to_ref(
YR_ARENA* arena,
void* address,
size_t offset);
const void* address,
YR_ARENA_REF* ref);
int yr_arena_coalesce(
YR_ARENA* arena);
int yr_arena_reserve_memory(
// Given a buffer number and an offset within the buffer, returns a pointer
// to that offset. The same limitations explained for yr_arena_ref_to_ptr
// applies for the pointers returned by this function.
void* yr_arena_get_ptr(
YR_ARENA* arena,
size_t size);
int buffer_id,
yr_arena_off_t offset);
yr_arena_off_t yr_arena_get_current_offset(
YR_ARENA* arena,
int buffer_id);
int yr_arena_allocate_memory(
YR_ARENA* arena,
int buffer_id,
size_t size,
void** allocated_memory);
YR_ARENA_REF* ref);
int yr_arena_allocate_zeroed_memory(
YR_ARENA* arena,
int buffer_id,
size_t size,
YR_ARENA_REF* ref);
int yr_arena_allocate_struct(
YR_ARENA* arena,
int buffer_id,
size_t size,
void** allocated_memory,
YR_ARENA_REF* ref,
...);
int yr_arena_make_ptr_relocatable(
YR_ARENA* arena,
void* base,
int buffer_id,
...);
int yr_arena_write_data(
YR_ARENA* arena,
int buffer_id,
const void* data,
size_t size,
void** written_data);
YR_ARENA_REF* ref);
int yr_arena_write_string(
YR_ARENA* arena,
int buffer_id,
const char* string,
char** written_string);
YR_ARENA_REF* ref);
int yr_arena_append(
YR_ARENA* target_arena,
YR_ARENA* source_arena);
int yr_arena_write_uint32(
YR_ARENA* arena,
int buffer_id,
uint32_t integer,
YR_ARENA_REF* ref);
int yr_arena_load_stream(
@@ -177,13 +277,4 @@ int yr_arena_save_stream(
YR_ARENA* arena,
YR_STREAM* stream);
int yr_arena_duplicate(
YR_ARENA* arena,
YR_ARENA** duplicated);
void yr_arena_print(
YR_ARENA* arena);
#endif
#endif // YR_ARENA_H
libyara/include/yara/atoms.h
+2 -2
View File
@@ -81,8 +81,8 @@ struct YR_ATOM_LIST_ITEM
uint16_t backtrack;
uint8_t* forward_code;
uint8_t* backward_code;
YR_ARENA_REF forward_code_ref;
YR_ARENA_REF backward_code_ref;
YR_ATOM_LIST_ITEM* next;
};
libyara/include/yara/base64.h
+52
View File
@@ -0,0 +1,52 @@
/*
Copyright (c) 2020. The YARA Authors. All Rights Reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef YR_BASE64_H
#define YR_BASE64_H
#include <yara/types.h>
#include <yara/re.h>
#include <yara/sizedstr.h>
typedef struct BASE64_NODE BASE64_NODE;
struct BASE64_NODE {
SIZED_STRING* str;
int escaped;
BASE64_NODE* next;
};
int yr_base64_ast_from_string(
SIZED_STRING* in_str,
YR_MODIFIER modifier,
RE_AST** re_ast,
RE_ERROR* error);
#endif
libyara/include/yara/bitmask.h
+4 -3
View File
@@ -48,10 +48,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// yr_bitmask_clear(my_bitmask, n)
//
// Check if bit n is set:
// yr_bitmask_isset(my_bitmask, n)
// yr_bitmask_is_set(my_bitmask, n)
//
#define YR_BITMASK unsigned long
#define YR_BITMASK_SLOT_BITS (sizeof(YR_BITMASK) * 8)
@@ -74,11 +73,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
memset(bm, 0, sizeof(bm))
#define yr_bitmask_isset(bm, i) \
#define yr_bitmask_is_set(bm, i) \
( \
(bm)[(i) / YR_BITMASK_SLOT_BITS] & (1UL << ((i) % YR_BITMASK_SLOT_BITS)) \
)
#define yr_bitmask_is_not_set(bm, i) (!yr_bitmask_is_set(bm, i))
#define yr_bitmask_print(bm) \
{ \
libyara/include/yara/compiler.h
+107 -19
View File
@@ -45,7 +45,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Expression type constants are powers of two because they are used as flags.
#define EXPRESSION_TYPE_UNKNOWN 0
#define EXPRESSION_TYPE_BOOLEAN 1
#define EXPRESSION_TYPE_INTEGER 2
@@ -54,6 +53,34 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define EXPRESSION_TYPE_OBJECT 16
#define EXPRESSION_TYPE_FLOAT 32
// The compiler uses an arena to store the data it generates during the
// compilation. Each buffer in the arena is used for storing a different type
// of data. The following identifiers indicate the purpose of each buffer.
#define YR_NAMESPACES_TABLE 0
#define YR_RULES_TABLE 1
#define YR_METAS_TABLE 2
#define YR_STRINGS_TABLE 3
#define YR_EXTERNAL_VARIABLES_TABLE 4
#define YR_SZ_POOL 5
#define YR_CODE_SECTION 6
#define YR_RE_CODE_SECTION 7
#define YR_AC_TRANSITION_TABLE 8
#define YR_AC_STATE_MATCHES_TABLE 9
#define YR_AC_STATE_MATCHES_POOL 10
#define YR_SUMMARY_SECTION 11
// This is the number of buffers used by the compiler, should match the number
// of items in the list above.
#define YR_NUM_SECTIONS 12
// Number of variables used by loops. This doesn't include user defined
// variables.
#define YR_INTERNAL_LOOP_VARS 3
typedef struct _YR_EXPRESSION
{
int type;
@@ -64,7 +91,15 @@ typedef struct _YR_EXPRESSION
SIZED_STRING* sized_string;
} value;
const char* identifier;
// An expression can have an associated identifier, if "ptr" is not NULL it
// points to the identifier name, if it is NULL, then "ref" holds a reference
// to the identifier within YR_SZ_POOL. When the identifier is in YR_SZ_POOL
// a pointer can't be used as the YR_SZ_POOL can be moved to a different
// memory location.
struct {
const char* ptr;
YR_ARENA_REF ref;
} identifier ;
} YR_EXPRESSION;
@@ -99,7 +134,7 @@ typedef void (*YR_COMPILER_RE_AST_CALLBACK_FUNC)(
typedef struct _YR_FIXUP
{
void* address;
YR_ARENA_REF ref;
struct _YR_FIXUP* next;
} YR_FIXUP;
@@ -112,7 +147,9 @@ typedef struct _YR_FIXUP
typedef struct _YR_LOOP_CONTEXT
{
uint8_t* addr;
// Reference indicating the the place in the code where the loop starts. The
// loop goes back to this address on each iteration.
YR_ARENA_REF start_ref;
// vars_count is the number of local variables defined by the loop, and vars
// is an array of expressions with the identifier and type for each of those
@@ -129,6 +166,67 @@ typedef struct _YR_LOOP_CONTEXT
typedef struct _YR_COMPILER
{
// Arena that contains the data generated by the compiled. The arena has
// the following buffers:
//
// YR_SUMMARY_SECTION:
// A YR_SUMMARY struct.
// YR_RULES_TABLE:
// An array of YR_RULE structures, one per each rule.
// YR_STRINGS_TABLE:
// An array of YR_STRING structures, one per each string.
// YR_METAS_TABLE:
// An array of YR_META structures, one per each meta definition.
// YR_NAMESPACES_TABLE:
// An array of YR_NAMESPACE structures, one per each namespace.
// YR_EXTERNAL_VARIABLES_TABLE:
// An array of YR_EXTERNAL_VARIABLE structures, one per each external
// variable defined.
// YR_SZ_POOL:
// A collection of null-terminated strings. This buffer contains
// identifiers, literal strings, and in general any null-terminated
// string referenced by other data structures.
// YR_CODE_SECTION:
// The code for the condition section of all the rules. This is the
// code executed by yr_execute_code.
// YR_RE_CODE_SECTION:
// Similar to YR_CODE_SECTION, but it contains the code for regular
// expressions. This is the code executed by yr_re_exec and
// yr_re_fast_exec.
// YR_AC_TRANSITION_TABLE:
// An array of uint32_t containing the Aho-Corasick transition table.
// See comment in _yr_ac_build_transition_table for details.
// YR_AC_STATE_MATCHES_TABLE:
// An array of uint32_t with the same number of items than the transition
// table. If entry N in the transition table corresponds to some
// Aho-Corasick state, the N-th item in this array has the index within
// the matches pool where the list of matches for that state begins.
// YR_AC_STATE_MATCHES_POOL:
// An array of YR_AC_MATCH structures.
//
YR_ARENA* arena;
// Index of the rule being compiled in the array of YR_RULE structures
// stored in YR_RULES_TABLE. If this is MAX_UINT32 the compiler is not
// parsing a rule.
uint32_t current_rule_idx;
//
uint32_t next_rule_idx;
// Index of the string being compiled in the array of YR_STRING structures
// stored in YR_STRINGS_TABLE.
uint32_t current_string_idx;
// Index of the current namespace in the array of YR_NAMESPACE structures
// stored in YR_NAMESPACES_TABLE.
uint32_t current_namespace_idx;
// Pointer to a YR_RULES structure that represents the compiled rules. This
// is what yr_compiler_get_rules returns. Once these rules are generated you
// can't call any of the yr_compiler_add_xxx functions.
YR_RULES* rules;
int errors;
int current_line;
int last_error;
@@ -136,28 +234,14 @@ typedef struct _YR_COMPILER
jmp_buf error_recovery;
YR_ARENA* sz_arena;
YR_ARENA* rules_arena;
YR_ARENA* strings_arena;
YR_ARENA* code_arena;
YR_ARENA* re_code_arena;
YR_ARENA* compiled_rules_arena;
YR_ARENA* externals_arena;
YR_ARENA* namespaces_arena;
YR_ARENA* metas_arena;
YR_ARENA* matches_arena;
YR_ARENA* automaton_arena;
YR_AC_AUTOMATON* automaton;
YR_HASH_TABLE* rules_table;
YR_HASH_TABLE* objects_table;
YR_HASH_TABLE* strings_table;
YR_NAMESPACE* current_namespace;
YR_RULE* current_rule;
YR_FIXUP* fixup_stack_head;
int namespaces_count;
int num_namespaces;
YR_LOOP_CONTEXT loop[YR_MAX_LOOP_NESTING];
int loop_index;
@@ -220,6 +304,10 @@ const char* _yr_compiler_default_include_callback(
void* user_data);
YR_RULE* _yr_compiler_get_rule_by_idx(
YR_COMPILER* compiler, uint32_t rule_idx);
YR_API int yr_compiler_create(
YR_COMPILER** compiler);
libyara/include/yara/dex.h
+2 -1
View File
@@ -10,6 +10,7 @@
#define DEX_FILE_MAGIC_036 "dex\n036\x00"
#define DEX_FILE_MAGIC_037 "dex\n037\x00"
#define DEX_FILE_MAGIC_038 "dex\n038\x00"
#define DEX_FILE_MAGIC_039 "dex\n039\x00"
#pragma pack(push,1)
@@ -75,7 +76,7 @@ typedef struct {
uint32_t class_idx;
uint32_t access_flags;
uint32_t super_class_idx;
uint32_t interfaces_off;
uint32_t interfaces_offset;
uint32_t source_file_idx;
uint32_t annotations_offset;
uint32_t class_data_offset;
libyara/include/yara/exec.h
+3 -3
View File
@@ -36,8 +36,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <yara/rules.h>
#define UNDEFINED 0xFFFABADAFABADAFFLL
#define IS_UNDEFINED(x) ((size_t)(x) == (size_t) UNDEFINED)
#define YR_UNDEFINED 0xFFFABADAFABADAFFLL
#define IS_UNDEFINED(x) ((size_t)(x) == (size_t) YR_UNDEFINED)
#define OP_ERROR 0
#define OP_HALT 255
@@ -172,7 +172,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define OPERATION(operator, op1, op2) \
(IS_UNDEFINED(op1) || IS_UNDEFINED(op2)) ? (UNDEFINED) : (op1 operator op2)
(IS_UNDEFINED(op1) || IS_UNDEFINED(op2)) ? (YR_UNDEFINED) : (op1 operator op2)
#define COMPARISON(operator, op1, op2) \
libyara/include/yara/globals.h
+9 -2
View File
@@ -39,7 +39,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
extern uint8_t yr_lowercase[256];
extern uint8_t yr_altercase[256];
extern YR_THREAD_STORAGE_KEY yr_tidx_key;
extern YR_THREAD_STORAGE_KEY yr_recovery_state_key;
// Thread-local storage (TLS) key used by the regexp and hex string parsers.
// Each thread calling yr_parse_re_string/yr_parse_hex_string stores a pointer
// to a jmp_buf struct used by setjmp/longjmp for recovering when a fatal error
// occurs in the parser.
extern YR_THREAD_STORAGE_KEY yr_yyfatal_trampoline_tls;
// Thread-local storage (TLS) key used by YR_TRYCATCH.
extern YR_THREAD_STORAGE_KEY yr_trycatch_trampoline_tls;
#endif
libyara/include/yara/hash.h
+13
View File
@@ -120,4 +120,17 @@ YR_API int yr_hash_table_add_raw_key(
const char* ns,
void* value);
YR_API uint32_t yr_hash_table_add_uint32(
YR_HASH_TABLE* table,
const char* key,
const char* ns,
uint32_t value);
YR_API uint32_t yr_hash_table_lookup_uint32(
YR_HASH_TABLE* table,
const char* key,
const char* ns);
#endif
libyara/include/yara/hex_lexer.h
+5
View File
@@ -72,6 +72,11 @@ typedef struct _HEX_LEX_ENVIRONMENT
} HEX_LEX_ENVIRONMENT;
// The default behavior when a fatal error occurs in the parser is calling
// exit(YY_EXIT_FAILURE) for terminating the process. This is not acceptable
// for a library, which should return gracefully to the calling program. For
// this reason we redefine the YY_FATAL_ERROR macro so that it expands to our
// own function instead of the one provided by default.
#define YY_FATAL_ERROR(msg) hex_yyfatal(yyscanner, msg)
#define LEX_ENV ((HEX_LEX_ENVIRONMENT*) lex_env)
libyara/include/yara/libyara.h
-12
View File
@@ -69,22 +69,10 @@ typedef enum _YR_CONFIG_NAME
YR_API int yr_initialize(void);
YR_API int yr_finalize(void);
YR_DEPRECATED_API void yr_finalize_thread(void);
YR_API int yr_get_tidx(void);
YR_API void yr_set_tidx(int);
YR_API int yr_set_configuration(YR_CONFIG_NAME, void*);
YR_API int yr_get_configuration(YR_CONFIG_NAME, void*);
libyara/include/yara/limits.h
+6 -10
View File
@@ -36,7 +36,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "utils.h"
// Maximum lenght of file paths. This is the only limit that doesn't have the
// Maximum length of file paths. This is the only limit that doesn't have the
// YR_ prefix. The intention is using the default MAX_PATH if defined.
#ifndef MAX_PATH
#define MAX_PATH 1024
@@ -50,6 +50,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define YR_MAX_THREADS 32
#endif
// Maximum number of buffers that an arena can have.
#ifndef YR_MAX_ARENA_BUFFERS
#define YR_MAX_ARENA_BUFFERS 16
#endif
// Capacity of the buffer used for storing compiler error messages. Messages
// will be truncated at this size.
#ifndef YR_MAX_COMPILER_ERROR_EXTRA_INFO
@@ -98,10 +103,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define YR_MAX_LOOP_VARS 2
#endif
#ifndef YR_MAX_ARENA_PAGES
#define YR_MAX_ARENA_PAGES 32
#endif
// Maximum number of nested included files.
#ifndef YR_MAX_INCLUDE_DEPTH
#define YR_MAX_INCLUDE_DEPTH 16
@@ -154,11 +155,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define RE_MAX_STACK 1024
#endif
// Maximum code size for a compiled regexp
#ifndef RE_MAX_CODE_SIZE
#define RE_MAX_CODE_SIZE 32768
#endif
// Maximum input size scanned by yr_re_exec
#ifndef RE_SCAN_LIMIT
#define RE_SCAN_LIMIT 4096
libyara/include/yara/modules.h
+3 -3
View File
@@ -356,7 +356,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
__function_obj->return_obj->type == OBJECT_TYPE_FLOAT, \
"return type differs from function declaration"); \
return yr_object_set_float( \
(d != (double) UNDEFINED) ? d : NAN, \
(d != (double) YR_UNDEFINED) ? d : NAN, \
__function_obj->return_obj, \
NULL); \
}
@@ -368,8 +368,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
__function_obj->return_obj->type == OBJECT_TYPE_STRING, \
"return type differs from function declaration"); \
return yr_object_set_string( \
(s != (char*) UNDEFINED) ? s : NULL, \
(s != (char*) UNDEFINED) ? strlen(s) : 0, \
(s != (char*) YR_UNDEFINED) ? s : NULL, \
(s != (char*) YR_UNDEFINED) ? strlen(s) : 0, \
__function_obj->return_obj, \
NULL); \
}
libyara/include/yara/notebook.h
+27
View File
@@ -0,0 +1,27 @@
//
// Created by Victor Manuel Alvarez on 3/4/20.
//
#ifndef YR_NOTEBOOK_H
#define YR_NOTEBOOK_H
#include <stdlib.h>
typedef struct YR_NOTEBOOK YR_NOTEBOOK;
int yr_notebook_create(
size_t page_size,
YR_NOTEBOOK** pool);
int yr_notebook_destroy(
YR_NOTEBOOK* pool);
void* yr_notebook_alloc(
YR_NOTEBOOK* notebook,
size_t size);
#endif // YR_NOTEBOOK_H
libyara/include/yara/parser.h
+19 -11
View File
@@ -37,31 +37,39 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
int yr_parser_emit(
yyscan_t yyscanner,
uint8_t instruction,
uint8_t** instruction_address);
YR_ARENA_REF* instruction_ref);
int yr_parser_emit_with_arg(
yyscan_t yyscanner,
uint8_t instruction,
int64_t argument,
uint8_t** instruction_address,
int64_t** argument_address);
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref);
int yr_parser_emit_with_arg_int32(
yyscan_t yyscanner,
uint8_t instruction,
int32_t argument,
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref);
int yr_parser_emit_with_arg_double(
yyscan_t yyscanner,
uint8_t instruction,
double argument,
uint8_t** instruction_address,
double** argument_address);
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref);
int yr_parser_emit_with_arg_reloc(
yyscan_t yyscanner,
uint8_t instruction,
void* argument,
uint8_t** instruction_address,
void** argument_address);
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref);
int yr_parser_check_types(
@@ -86,12 +94,12 @@ int yr_parser_reduce_rule_declaration_phase_1(
yyscan_t yyscanner,
int32_t flags,
const char* identifier,
YR_RULE** rule);
YR_ARENA_REF* rule_ref);
int yr_parser_reduce_rule_declaration_phase_2(
yyscan_t yyscanner,
YR_RULE* rule);
YR_ARENA_REF* rule_ref);
int yr_parser_reduce_string_declaration(
@@ -99,7 +107,7 @@ int yr_parser_reduce_string_declaration(
YR_MODIFIER modifier,
const char* identifier,
SIZED_STRING* str,
YR_STRING** string);
YR_ARENA_REF* string_ref);
int yr_parser_reduce_meta_declaration(
@@ -108,7 +116,7 @@ int yr_parser_reduce_meta_declaration(
const char* identifier,
const char* string,
int64_t integer,
YR_META** meta);
YR_ARENA_REF* meta_ref);
int yr_parser_reduce_string_identifier(
libyara/include/yara/pe.h
+52
View File
@@ -38,6 +38,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#if defined(_WIN32) || defined(__CYGWIN__)
#include <windows.h>
// PKCS7_SIGNER_INFO is defined by wincrypt.h, but it conflicts with a type
// defined in openssl/pkcs7.h which is used in pe.c. Let's undefine the macro.
#undef PKCS7_SIGNER_INFO
// These definitions are not present in older Windows headers.
#ifndef IMAGE_FILE_MACHINE_ARMNT
@@ -48,6 +52,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define IMAGE_FILE_MACHINE_ARM64 0xaa64
#endif
#else
#include <stdlib.h>
@@ -452,10 +458,52 @@ typedef struct _IMAGE_RESOURCE_DIRECTORY {
WORD NumberOfIdEntries;
} IMAGE_RESOURCE_DIRECTORY, *PIMAGE_RESOURCE_DIRECTORY;
#define IMAGE_DEBUG_TYPE_UNKNOWN 0
#define IMAGE_DEBUG_TYPE_COFF 1
#define IMAGE_DEBUG_TYPE_CODEVIEW 2
#define IMAGE_DEBUG_TYPE_FPO 3
#define IMAGE_DEBUG_TYPE_MISC 4
#define IMAGE_DEBUG_TYPE_EXCEPTION 5
#define IMAGE_DEBUG_TYPE_FIXUP 6
#define IMAGE_DEBUG_TYPE_BORLAND 9
typedef struct _IMAGE_DEBUG_DIRECTORY {
DWORD Characteristics;
DWORD TimeDateStamp;
WORD MajorVersion;
WORD MinorVersion;
DWORD Type;
DWORD SizeOfData;
DWORD AddressOfRawData;
DWORD PointerToRawData;
} IMAGE_DEBUG_DIRECTORY, *PIMAGE_DEBUG_DIRECTORY;
#pragma pack(pop)
#endif // _WIN32
#define CVINFO_PDB70_CVSIGNATURE 0x53445352 // "RSDS"
#define CVINFO_PDB20_CVSIGNATURE 0x3031424e // "NB10"
typedef struct _CV_HEADER {
DWORD dwSignature;
DWORD dwOffset;
} CV_HEADER, *PCV_HEADER;
typedef struct _CV_INFO_PDB20 {
CV_HEADER CvHeader;
DWORD dwSignature;
DWORD dwAge;
BYTE PdbFileName[1];
} CV_INFO_PDB20, *PCV_INFO_PDB20;
typedef struct _CV_INFO_PDB70 {
DWORD CvSignature;
DWORD Signature[4];
DWORD Age;
BYTE PdbFileName[1];
} CV_INFO_PDB70, *PCV_INFO_PDB70;
typedef struct _VERSION_INFO {
WORD Length;
WORD ValueLength;
@@ -464,6 +512,8 @@ typedef struct _VERSION_INFO {
} VERSION_INFO, *PVERSION_INFO;
#define MAX_PE_CERTS 16
#define WIN_CERT_REVISION_1_0 0x0100
#define WIN_CERT_REVISION_2_0 0x0200
@@ -481,6 +531,8 @@ typedef struct _WIN_CERTIFICATE {
BYTE Certificate[0];
} WIN_CERTIFICATE, *PWIN_CERTIFICATE;
#define SPC_NESTED_SIGNATURE_OBJID "1.3.6.1.4.1.311.2.4.1"
//
// Rich signature.
libyara/include/yara/pe_utils.h
-15
View File
@@ -52,20 +52,6 @@ typedef struct _IMPORT_FUNCTION
} IMPORT_FUNCTION, *PIMPORT_FUNCTION;
typedef struct _EXPORT_FUNCTION
{
char *name;
uint16_t ordinal;
} EXPORT_FUNCTION, *PEXPORT_FUNCTION;
typedef struct _EXPORT_FUNCTIONS
{
uint32_t number_of_exports;
EXPORT_FUNCTION* functions;
} EXPORT_FUNCTIONS, *PEXPORT_FUNCTIONS;
typedef struct _PE
{
const uint8_t* data;
@@ -79,7 +65,6 @@ typedef struct _PE
YR_HASH_TABLE* hash_table;
YR_OBJECT* object;
IMPORTED_DLL* imported_dlls;
EXPORT_FUNCTIONS* exported_functions;
uint32_t resources;
libyara/include/yara/re.h
+2 -4
View File
@@ -105,7 +105,6 @@ typedef int RE_MATCH_CALLBACK_FUNC(
int flags,
void* args);
int yr_re_ast_create(
RE_AST** re_ast);
@@ -193,8 +192,8 @@ int yr_re_parse_hex(
int yr_re_compile(
const char* re_string,
int flags,
YR_ARENA* code_arena,
RE** re,
YR_ARENA* arena,
YR_ARENA_REF* ref,
RE_ERROR* error);
@@ -203,5 +202,4 @@ int yr_re_match(
RE* re,
const char* target);
#endif
libyara/include/yara/re_lexer.h
+5
View File
@@ -73,6 +73,11 @@ typedef struct _RE_LEX_ENVIRONMENT
#define LEX_ENV ((RE_LEX_ENVIRONMENT*) lex_env)
// The default behavior when a fatal error occurs in the parser is calling
// exit(YY_EXIT_FAILURE) for terminating the process. This is not acceptable
// for a library, which should return gracefully to the calling program. For
// this reason we redefine the YY_FATAL_ERROR macro so that it expands to our
// own function instead of the one provided by default.
#define YY_FATAL_ERROR(msg) re_yyfatal(yyscanner, msg)
#include <re_grammar.h>
libyara/include/yara/rules.h
+15 -11
View File
@@ -57,11 +57,18 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define yr_rule_strings_foreach(rule, string) \
for (string = rule->strings; !STRING_IS_NULL(string); string++)
for (string = rule->strings; \
string != NULL; \
string = STRING_IS_LAST_IN_RULE(string) ? NULL : string + 1)
#define yr_string_matches_foreach(string, match) \
for (match = STRING_MATCHES(string).head; match != NULL; match = match->next)
#define yr_string_matches_foreach(context, string, match) \
for (match = context->matches[string->idx].head; \
match != NULL; \
match = match->next) \
/* private matches are skipped */ \
if (match->private) { continue; } \
else /* user code block goes here */
#define yr_rules_foreach(rules, rule) \
@@ -154,14 +161,6 @@ YR_API int yr_rules_define_string_variable(
const char* value);
YR_API void yr_rules_reset_profiling_info(
YR_RULES* rules);
YR_API void yr_rules_print_profiling_info(
YR_RULES* rules);
YR_API int yr_rules_get_stats(
YR_RULES* rules,
YR_RULES_STATS *stats);
@@ -174,4 +173,9 @@ YR_API void yr_rule_disable(
YR_API void yr_rule_enable(
YR_RULE* rule);
int yr_rules_from_arena(
YR_ARENA* arena,
YR_RULES** rules);
#endif
libyara/include/yara/scan.h
+8 -4
View File
@@ -32,10 +32,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <yara/types.h>
// Bitmasks for flags.
#define SCAN_FLAGS_FAST_MODE 1
#define SCAN_FLAGS_PROCESS_MEMORY 2
#define SCAN_FLAGS_NO_TRYCATCH 4
//
// Flags used with yr_scanner_set_flags and yr_rules_scan_xxx functions.
//
#define SCAN_FLAGS_FAST_MODE 1
#define SCAN_FLAGS_PROCESS_MEMORY 2
#define SCAN_FLAGS_NO_TRYCATCH 4
#define SCAN_FLAGS_REPORT_RULES_MATCHING 8
#define SCAN_FLAGS_REPORT_RULES_NOT_MATCHING 16
int yr_scan_verify_match(
libyara/include/yara/sizedstr.h
+9 -2
View File
@@ -44,8 +44,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define SIZED_STRING_FLAGS_DOT_ALL 2
#pragma pack(push)
#pragma pack(8)
#pragma pack(4)
typedef struct _SIZED_STRING
{
@@ -59,6 +58,11 @@ typedef struct _SIZED_STRING
#pragma pack(pop)
int sized_string_cmp_nocase(
SIZED_STRING* s1,
SIZED_STRING* s2);
int sized_string_cmp(
SIZED_STRING* s1,
SIZED_STRING* s2);
@@ -71,4 +75,7 @@ SIZED_STRING* sized_string_dup(
SIZED_STRING* sized_string_new(
const char* s);
SIZED_STRING* sized_string_convert_to_wide(
SIZED_STRING* s);
#endif
libyara/include/yara/stopwatch.h
+2 -2
View File
@@ -75,9 +75,9 @@ typedef struct _YR_STOPWATCH
void yr_stopwatch_start(
YR_STOPWATCH* stopwatch);
// yr_stopwatch_elapsed_us returns the number of microseconds elapsed
// yr_stopwatch_elapsed_ns returns the number of nanoseconds elapsed
// since the last call to yr_stopwatch_start.
uint64_t yr_stopwatch_elapsed_us(
uint64_t yr_stopwatch_elapsed_ns(
YR_STOPWATCH* stopwatch);
#endif
libyara/include/yara/strutils.h
+4
View File
@@ -51,6 +51,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#define PRId64 "I64d"
#endif
#if !defined (PRIi32)
#define PRIi32 "I32i"
#endif
#else
#include <inttypes.h>
#endif
libyara/include/yara/types.h
+272 -270
View File
@@ -30,6 +30,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef YR_TYPES_H
#define YR_TYPES_H
#include <stdbool.h>
#include <yara/arena.h>
#include <yara/bitmask.h>
#include <yara/limits.h>
@@ -38,131 +40,128 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <yara/sizedstr.h>
#include <yara/stopwatch.h>
#include <yara/threading.h>
#include "notebook.h"
#define DECLARE_REFERENCE(type, name) \
union { type name; int64_t name##_; } YR_ALIGN(8)
union { \
type name; \
YR_ARENA_REF name##_; \
} YR_ALIGN(8)
#define NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL 0x01
#define STRING_GFLAGS_REFERENCED 0x01
#define STRING_GFLAGS_HEXADECIMAL 0x02
#define STRING_GFLAGS_NO_CASE 0x04
#define STRING_GFLAGS_ASCII 0x08
#define STRING_GFLAGS_WIDE 0x10
#define STRING_GFLAGS_REGEXP 0x20
#define STRING_GFLAGS_FAST_REGEXP 0x40
#define STRING_GFLAGS_FULL_WORD 0x80
#define STRING_GFLAGS_ANONYMOUS 0x100
#define STRING_GFLAGS_SINGLE_MATCH 0x200
#define STRING_GFLAGS_LITERAL 0x400
#define STRING_GFLAGS_FITS_IN_ATOM 0x800
#define STRING_GFLAGS_NULL 0x1000
#define STRING_GFLAGS_CHAIN_PART 0x2000
#define STRING_GFLAGS_CHAIN_TAIL 0x4000
#define STRING_GFLAGS_FIXED_OFFSET 0x8000
#define STRING_GFLAGS_GREEDY_REGEXP 0x10000
#define STRING_GFLAGS_DOT_ALL 0x20000
#define STRING_GFLAGS_DISABLED 0x40000
#define STRING_GFLAGS_XOR 0x80000
#define STRING_GFLAGS_PRIVATE 0x100000
#define STRING_IS_HEX(x) \
(((x)->g_flags) & STRING_GFLAGS_HEXADECIMAL)
#define STRING_IS_NO_CASE(x) \
(((x)->g_flags) & STRING_GFLAGS_NO_CASE)
#define STRING_IS_DOT_ALL(x) \
(((x)->g_flags) & STRING_GFLAGS_DOT_ALL)
#define STRING_IS_ASCII(x) \
(((x)->g_flags) & STRING_GFLAGS_ASCII)
#define STRING_IS_WIDE(x) \
(((x)->g_flags) & STRING_GFLAGS_WIDE)
#define STRING_IS_REGEXP(x) \
(((x)->g_flags) & STRING_GFLAGS_REGEXP)
#define STRING_IS_GREEDY_REGEXP(x) \
(((x)->g_flags) & STRING_GFLAGS_GREEDY_REGEXP)
#define STRING_IS_FULL_WORD(x) \
(((x)->g_flags) & STRING_GFLAGS_FULL_WORD)
#define STRING_IS_ANONYMOUS(x) \
(((x)->g_flags) & STRING_GFLAGS_ANONYMOUS)
#define STRING_IS_REFERENCED(x) \
(((x)->g_flags) & STRING_GFLAGS_REFERENCED)
#define STRING_IS_SINGLE_MATCH(x) \
(((x)->g_flags) & STRING_GFLAGS_SINGLE_MATCH)
#define STRING_IS_FIXED_OFFSET(x) \
(((x)->g_flags) & STRING_GFLAGS_FIXED_OFFSET)
#define STRING_IS_LITERAL(x) \
(((x)->g_flags) & STRING_GFLAGS_LITERAL)
#define STRING_IS_FAST_REGEXP(x) \
(((x)->g_flags) & STRING_GFLAGS_FAST_REGEXP)
#define STRING_IS_CHAIN_PART(x) \
(((x)->g_flags) & STRING_GFLAGS_CHAIN_PART)
#define STRING_IS_CHAIN_TAIL(x) \
(((x)->g_flags) & STRING_GFLAGS_CHAIN_TAIL)
#define STRING_IS_NULL(x) \
((x) == NULL || ((x)->g_flags) & STRING_GFLAGS_NULL)
#define STRING_FITS_IN_ATOM(x) \
(((x)->g_flags) & STRING_GFLAGS_FITS_IN_ATOM)
#define STRING_IS_DISABLED(x) \
(((x)->g_flags) & STRING_GFLAGS_DISABLED)
#define STRING_IS_XOR(x) \
(((x)->g_flags) & STRING_GFLAGS_XOR)
#define STRING_IS_PRIVATE(x) \
(((x)->g_flags) & STRING_GFLAGS_PRIVATE)
#define STRING_FOUND(x) \
((x)->matches[yr_get_tidx()].tail != NULL)
#define STRING_MATCHES(x) \
((x)->matches[yr_get_tidx()])
#define RULE_TFLAGS_MATCH 0x01
#define RULE_GFLAGS_PRIVATE 0x01
#define RULE_GFLAGS_GLOBAL 0x02
#define RULE_GFLAGS_REQUIRE_EXECUTABLE 0x04
#define RULE_GFLAGS_REQUIRE_FILE 0x08
#define RULE_GFLAGS_NULL 0x1000
#define RULE_GFLAGS_DISABLED 0x2000
// Flags for YR_RULE
#define RULE_FLAGS_PRIVATE 0x01
#define RULE_FLAGS_GLOBAL 0x02
#define RULE_FLAGS_NULL 0x04
#define RULE_FLAGS_DISABLED 0x08
#define RULE_IS_PRIVATE(x) \
(((x)->g_flags) & RULE_GFLAGS_PRIVATE)
(((x)->flags) & RULE_FLAGS_PRIVATE)
#define RULE_IS_GLOBAL(x) \
(((x)->g_flags) & RULE_GFLAGS_GLOBAL)
(((x)->flags) & RULE_FLAGS_GLOBAL)
#define RULE_IS_NULL(x) \
(((x)->g_flags) & RULE_GFLAGS_NULL)
(((x)->flags) & RULE_FLAGS_NULL)
#define RULE_IS_DISABLED(x) \
(((x)->g_flags) & RULE_GFLAGS_DISABLED)
(((x)->flags) & RULE_FLAGS_DISABLED)
#define RULE_MATCHES(x) \
((x)->t_flags[yr_get_tidx()] & RULE_TFLAGS_MATCH)
// Flags for YR_STRING
#define STRING_FLAGS_REFERENCED 0x01
#define STRING_FLAGS_HEXADECIMAL 0x02
#define STRING_FLAGS_NO_CASE 0x04
#define STRING_FLAGS_ASCII 0x08
#define STRING_FLAGS_WIDE 0x10
#define STRING_FLAGS_REGEXP 0x20
#define STRING_FLAGS_FAST_REGEXP 0x40
#define STRING_FLAGS_FULL_WORD 0x80
#define STRING_FLAGS_ANONYMOUS 0x100
#define STRING_FLAGS_SINGLE_MATCH 0x200
#define STRING_FLAGS_LITERAL 0x400
#define STRING_FLAGS_FITS_IN_ATOM 0x800
#define STRING_FLAGS_LAST_IN_RULE 0x1000
#define STRING_FLAGS_CHAIN_PART 0x2000
#define STRING_FLAGS_CHAIN_TAIL 0x4000
#define STRING_FLAGS_FIXED_OFFSET 0x8000
#define STRING_FLAGS_GREEDY_REGEXP 0x10000
#define STRING_FLAGS_DOT_ALL 0x20000
#define STRING_FLAGS_DISABLED 0x40000
#define STRING_FLAGS_XOR 0x80000
#define STRING_FLAGS_PRIVATE 0x100000
#define STRING_FLAGS_BASE64 0x200000
#define STRING_FLAGS_BASE64_WIDE 0x400000
#define STRING_IS_HEX(x) \
(((x)->flags) & STRING_FLAGS_HEXADECIMAL)
#define STRING_IS_NO_CASE(x) \
(((x)->flags) & STRING_FLAGS_NO_CASE)
#define STRING_IS_DOT_ALL(x) \
(((x)->flags) & STRING_FLAGS_DOT_ALL)
#define STRING_IS_ASCII(x) \
(((x)->flags) & STRING_FLAGS_ASCII)
#define STRING_IS_WIDE(x) \
(((x)->flags) & STRING_FLAGS_WIDE)
#define STRING_IS_REGEXP(x) \
(((x)->g_flags) & STRING_FLAGS_REGEXP)
#define STRING_IS_GREEDY_REGEXP(x) \
(((x)->flags) & STRING_FLAGS_GREEDY_REGEXP)
#define STRING_IS_FULL_WORD(x) \
(((x)->flags) & STRING_FLAGS_FULL_WORD)
#define STRING_IS_ANONYMOUS(x) \
(((x)->g_flags) & STRING_FLAGS_ANONYMOUS)
#define STRING_IS_REFERENCED(x) \
(((x)->flags) & STRING_FLAGS_REFERENCED)
#define STRING_IS_SINGLE_MATCH(x) \
(((x)->flags) & STRING_FLAGS_SINGLE_MATCH)
#define STRING_IS_FIXED_OFFSET(x) \
(((x)->flags) & STRING_FLAGS_FIXED_OFFSET)
#define STRING_IS_LITERAL(x) \
(((x)->flags) & STRING_FLAGS_LITERAL)
#define STRING_IS_FAST_REGEXP(x) \
(((x)->flags) & STRING_FLAGS_FAST_REGEXP)
#define STRING_IS_CHAIN_PART(x) \
(((x)->flags) & STRING_FLAGS_CHAIN_PART)
#define STRING_IS_CHAIN_TAIL(x) \
(((x)->flags) & STRING_FLAGS_CHAIN_TAIL)
#define STRING_IS_LAST_IN_RULE(x) \
(((x)->flags) & STRING_FLAGS_LAST_IN_RULE)
#define STRING_FITS_IN_ATOM(x) \
(((x)->flags) & STRING_FLAGS_FITS_IN_ATOM)
#define STRING_IS_DISABLED(x) \
(((x)->flags) & STRING_FLAGS_DISABLED)
#define STRING_IS_XOR(x) \
(((x)->flags) & STRING_FLAGS_XOR)
#define STRING_IS_BASE64(x) \
(((x)->flags) & STRING_FLAGS_BASE64)
#define STRING_IS_BASE64_WIDE(x) \
(((x)->flags) & STRING_FLAGS_BASE64_WIDE)
#define STRING_IS_PRIVATE(x) \
(((x)->flags) & STRING_FLAGS_PRIVATE)
#define META_TYPE_NULL 0
@@ -194,11 +193,11 @@ typedef struct RE_FIBER RE_FIBER;
typedef struct RE_FIBER_LIST RE_FIBER_LIST;
typedef struct RE_FIBER_POOL RE_FIBER_POOL;
typedef struct YR_AC_MATCH YR_AC_MATCH;
typedef struct YR_AC_STATE YR_AC_STATE;
typedef struct YR_AC_AUTOMATON YR_AC_AUTOMATON;
typedef struct YR_AC_MATCH_TABLE_ENTRY YR_AC_MATCH_TABLE_ENTRY;
typedef struct YR_AC_TABLES YR_AC_TABLES;
typedef struct YR_AC_MATCH_LIST_ENTRY YR_AC_MATCH_LIST_ENTRY;
typedef struct YR_AC_MATCH YR_AC_MATCH;
typedef struct YR_NAMESPACE YR_NAMESPACE;
typedef struct YR_META YR_META;
@@ -206,6 +205,7 @@ typedef struct YR_MATCHES YR_MATCHES;
typedef struct YR_STRING YR_STRING;
typedef struct YR_RULE YR_RULE;
typedef struct YR_RULES YR_RULES;
typedef struct YR_SUMMARY YR_SUMMARY;
typedef struct YR_RULES_STATS YR_RULES_STATS;
typedef struct YR_EXTERNAL_VARIABLE YR_EXTERNAL_VARIABLE;
typedef struct YR_MATCH YR_MATCH;
@@ -234,67 +234,70 @@ typedef struct YR_MODIFIER YR_MODIFIER;
typedef struct YR_ITERATOR YR_ITERATOR;
typedef uint32_t YR_AC_TRANSITION;
#pragma pack(push)
#pragma pack(8)
struct YR_MODIFIER
{
int32_t flags;
uint8_t xor_min;
uint8_t xor_max;
};
struct YR_NAMESPACE
{
int32_t t_flags[YR_MAX_THREADS]; // Thread-specific flags
DECLARE_REFERENCE(char*, name);
// Pointer to namespace's name.
DECLARE_REFERENCE(const char*, name);
// Index of this namespace in the array of YR_NAMESPACE structures stored
// in YR_NAMESPACES_TABLE.
uint32_t idx;
};
struct YR_META
{
int32_t type;
YR_ALIGN(8) int64_t integer;
DECLARE_REFERENCE(const char*, identifier);
DECLARE_REFERENCE(char*, string);
};
DECLARE_REFERENCE(const char*, string);
struct YR_MATCHES
{
int32_t count;
DECLARE_REFERENCE(YR_MATCH*, head);
DECLARE_REFERENCE(YR_MATCH*, tail);
int64_t integer;
int32_t type;
};
struct YR_STRING
{
int32_t g_flags;
// Flags, see STRING_FLAGS_XXX macros defined above.
uint32_t flags;
// Index of this string in the array of YR_STRING structures stored in
// YR_STRINGS_TABLE.
uint32_t idx;
// If the string can only match at a specific offset (for example if the
// condition is "$a at 0" the string $a can only match at offset 0), the
// fixed_offset field contains the offset, it have the YR_UNDEFINED value for
// strings that can match anywhere.
int64_t fixed_offset;
// Index of the rule containing this string in the array of YR_RULE
// structures stored in YR_RULES_TABLE.
uint32_t rule_idx;
// String's length.
int32_t length;
DECLARE_REFERENCE(char*, identifier);
// Pointer to the string itself, the length is indicated by the "length"
// field.
DECLARE_REFERENCE(uint8_t*, string);
// Strings are splited in two or more parts when they contain a "gap" that
// Strings are splitted in two or more parts when they contain a "gap" that
// is larger than YR_STRING_CHAINING_THRESHOLD. This happens in strings like
// { 01 02 03 04 [X-Y] 05 06 07 08 } if Y >= X + YR_STRING_CHAINING_THRESHOLD
// and also in { 01 02 03 04 [-] 05 06 07 08 }. In both cases the strings are
// split in { 01 02 03 04 } and { 05 06 07 08 }, and the two smaller strings
// are searched for independently. If some string S is splited in S1 and S2,
// are searched for independently. If some string S is splitted in S1 and S2,
// S2 is chained to S1. In the example above { 05 06 07 08 } is chained to
// { 01 02 03 04 }. The same applies when the string is splitted in more than
// two parts, if S is split in S1, S2, and S3. S3 is chained to S2 and S2 is
// chained to S1 (it can represented as: S1 <- S2 <- S3).
DECLARE_REFERENCE(YR_STRING*, chained_to);
// Pointer to the rule that contains this string.
DECLARE_REFERENCE(YR_RULE*, rule);
// When this string is chained to some other string, chain_gap_min and
// chain_gap_max contain the minimum and maximum distance between the two
// strings. For example in { 01 02 03 04 [X-Y] 05 06 07 08 }, the string
@@ -304,58 +307,31 @@ struct YR_STRING
int32_t chain_gap_min;
int32_t chain_gap_max;
// If the string can only match at a specific offset (for example if the
// condition is "$a at 0" the string $a can only match at offset 0), the
// fixed_offset field contains the offset, it have the UNDEFINED value for
// strings that can match anywhere.
int64_t fixed_offset;
// Each item in the "matches" array represents a list of matches, there's one
// list of matches for each thread currently using the compiled rules. Up to
// YR_MAX_THREADS can use the compiled rules simultaneosly, that's why the
// array has YR_MAX_THREADS slots. The lists contain the matches that have
// been found so far.
YR_MATCHES matches[YR_MAX_THREADS];
// "private_matches" is similar to "matches" but it holds the matches for
// strings that are flagged as private. The matches for such strings are not
// put in the "matches" arrays.
YR_MATCHES private_matches[YR_MAX_THREADS];
// "unconfirmed_matches" is used for strings that are part of a chain. Let's
// suppose that the string S is split in two chained strings S1 <- S2. When
// a match is found for S1, we can't be sure that S matches until a match for
// S2 is found (within the range defined by chain_gap_min and chain_gap_max),
// so the matches for S1 are put in "unconfirmed_matches" until they can be
// confirmed or discarded.
YR_MATCHES unconfirmed_matches[YR_MAX_THREADS];
// Identifier of this string.
DECLARE_REFERENCE(const char*, identifier);
};
struct YR_RULE
{
int32_t g_flags; // Global flags
int32_t t_flags[YR_MAX_THREADS]; // Thread-specific flags
int32_t flags;
// Number of atoms generated for this rule.
int32_t num_atoms;
DECLARE_REFERENCE(const char*, identifier);
DECLARE_REFERENCE(const char*, tags);
DECLARE_REFERENCE(YR_META*, metas);
DECLARE_REFERENCE(YR_STRING*, strings);
DECLARE_REFERENCE(YR_NAMESPACE*, ns);
};
// Number of atoms generated for this rule.
int32_t num_atoms;
// Used only when PROFILING_ENABLED is defined. This is the sum of all values
// in time_cost_per_thread. This is updated once on each call to
// yr_scanner_scan_xxx.
volatile int64_t time_cost;
// Used only when PROFILING_ENABLED is defined. This array holds the time
// cost for each thread using this structure concurrenlty. This is necessary
// because a global variable causes too much contention while trying to
// increment in a synchronized way from multiple threads.
int64_t time_cost_per_thread[YR_MAX_THREADS];
struct YR_SUMMARY
{
uint32_t num_rules;
uint32_t num_strings;
uint32_t num_namespaces;
};
@@ -375,56 +351,22 @@ struct YR_EXTERNAL_VARIABLE
struct YR_AC_MATCH
{
uint16_t backtrack;
DECLARE_REFERENCE(YR_STRING*, string);
DECLARE_REFERENCE(const uint8_t*, forward_code);
DECLARE_REFERENCE(const uint8_t*, backward_code);
DECLARE_REFERENCE(YR_AC_MATCH*, next);
// When the Aho-Corasick automaton reaches some state that has associated
// matches, the current position in the input buffer is a few bytes past
// the point where the match actually occurs, for example, when looking for
// string "bar" in "foobarbaz", when the automaton reaches the state associated
// to the ending "r" in "bar, which is the one that has a match, the current
// position in the input is 6 (the "b" after the "r"), but the match is at
// position 3. The backtrack field indicates how many bytes the scanner has
// to go back to find the point where the match actually start.
uint16_t backtrack;
};
struct YR_AC_MATCH_TABLE_ENTRY
{
DECLARE_REFERENCE(YR_AC_MATCH*, match);
};
typedef uint32_t YR_AC_TRANSITION;
typedef YR_AC_TRANSITION* YR_AC_TRANSITION_TABLE;
typedef YR_AC_MATCH_TABLE_ENTRY* YR_AC_MATCH_TABLE;
struct YR_AC_TABLES
{
YR_AC_TRANSITION_TABLE transitions;
YR_AC_MATCH_TABLE matches;
};
typedef struct YARA_RULES_FILE_HEADER
{
DECLARE_REFERENCE(YR_RULE*, rules_list_head);
DECLARE_REFERENCE(YR_EXTERNAL_VARIABLE*, externals_list_head);
DECLARE_REFERENCE(const uint8_t*, code_start);
DECLARE_REFERENCE(YR_AC_MATCH_TABLE, ac_match_table);
DECLARE_REFERENCE(YR_AC_TRANSITION_TABLE, ac_transition_table);
// Size of ac_match_table and ac_transition_table in number of items (both
// tables have the same number of items)
uint32_t ac_tables_size;
} YARA_RULES_FILE_HEADER;
typedef struct _YR_INIT_RULE_ARGS
{
DECLARE_REFERENCE(YR_RULE*, rule);
DECLARE_REFERENCE(const uint8_t*, jmp_addr);
} YR_INIT_RULE_ARGS;
#pragma pack(pop)
@@ -456,8 +398,8 @@ struct RE_NODE
RE_NODE* prev_sibling;
RE_NODE* next_sibling;
uint8_t* forward_code;
uint8_t* backward_code;
YR_ARENA_REF forward_code_ref;
YR_ARENA_REF backward_code_ref;
};
@@ -505,10 +447,10 @@ struct RE_FIBER
int32_t sp; // stack pointer
int32_t rc; // repeat counter
uint16_t stack[RE_MAX_STACK];
RE_FIBER* prev;
RE_FIBER* next;
uint16_t stack[RE_MAX_STACK];
};
@@ -526,6 +468,24 @@ struct RE_FIBER_POOL
};
struct YR_MODIFIER
{
int32_t flags;
uint8_t xor_min;
uint8_t xor_max;
SIZED_STRING *alphabet;
};
struct YR_MATCHES
{
YR_MATCH* head;
YR_MATCH* tail;
int32_t count;
};
struct YR_MATCH
{
int64_t base; // Base address for the match
@@ -535,29 +495,48 @@ struct YR_MATCH
// Pointer to a buffer containing a portion of the matched data. The size of
// the buffer is data_length. data_length is always <= length and is limited
// to MAX_MATCH_DATA bytes.
// to YR_CONFIG_MAX_MATCH_DATA bytes.
const uint8_t* data;
YR_MATCH* prev;
YR_MATCH* next;
// If the match belongs to a chained string chain_length contains the
// length of the chain. This field is used only in unconfirmed matches.
int32_t chain_length;
YR_MATCH* prev;
YR_MATCH* next;
// True if this is match for a private string.
bool private;
};
struct YR_AC_STATE
{
YR_AC_STATE* failure;
YR_AC_STATE* first_child;
YR_AC_STATE* siblings;
// Reference to the YR_AC_MATCH structure that heads the list of matches
// for this state.
YR_ARENA_REF matches_ref;
uint8_t depth;
uint8_t input;
uint32_t t_table_slot;
};
YR_AC_STATE* failure;
YR_AC_STATE* first_child;
YR_AC_STATE* siblings;
YR_AC_MATCH* matches;
struct YR_AC_MATCH_LIST_ENTRY
{
uint16_t backtrack;
uint32_t string_idx;
YR_ARENA_REF ref;
YR_ARENA_REF forward_code_ref;
YR_ARENA_REF backward_code_ref;
YR_AC_MATCH_LIST_ENTRY* next;
};
@@ -567,49 +546,54 @@ struct YR_AC_AUTOMATON
// stored in tables_size.
uint32_t tables_size;
// The first slot in the transition table (t_table) that may be be unused.
// Used for speeding up the construction of the transition table.
uint32_t t_table_unused_candidate;
// Bitmask where each bit indicates if the corresponding slot in m_table
// and t_table is already in use.
// Bitmask where each bit indicates if the corresponding slot in the
// transition table is already in use.
YR_BITMASK* bitmask;
YR_AC_TRANSITION_TABLE t_table;
YR_AC_MATCH_TABLE m_table;
// Pointer to the root Aho-Corasick state.
YR_AC_STATE* root;
};
struct YR_RULES
{
unsigned char tidx_mask[YR_BITARRAY_NCHARS(YR_MAX_THREADS)];
YR_ARENA* arena;
YR_RULE* rules_list_head;
YR_STRING* strings_list_head;
YR_EXTERNAL_VARIABLE* externals_list_head;
YR_AC_TRANSITION* ac_transition_table;
YR_AC_MATCH* ac_match_pool;
uint32_t* ac_match_table;
const uint8_t* code_start;
YR_MUTEX mutex;
YR_ARENA* arena;
YR_RULE* rules_list_head;
YR_EXTERNAL_VARIABLE* externals_list_head;
YR_AC_TRANSITION_TABLE ac_transition_table;
YR_AC_MATCH_TABLE ac_match_table;
// Total number of rules.
uint32_t num_rules;
// Size of ac_match_table and ac_transition_table in number of items (both
// tables have the same numbe of items).
uint32_t ac_tables_size;
// Total number of strings.
uint32_t num_strings;
// Used only when PROFILING_ENABLED is defined.
uint64_t time_cost;
// Total number of namespaces.
uint32_t num_namespaces;
};
struct YR_RULES_STATS
{
// Total number of rules
uint32_t rules;
uint32_t num_rules;
// Total number of strings across all rules.
uint32_t strings;
uint32_t num_strings;
// Total number of Aho-Corasick matches. Each node in the Aho-Corasick
// automaton has a list of YR_AC_MATCH structures (match list) pointing to
// automaton has a list of YR_AC_MATCH_LIST_ENTRY structures (match list) pointing to
// strings that are potential matches. This field holds the total number of
// those structures across all nodes in the automaton.
uint32_t ac_matches;
@@ -661,6 +645,7 @@ struct YR_MEMORY_BLOCK_ITERATOR
typedef int (*YR_CALLBACK_FUNC)(
YR_SCAN_CONTEXT* context,
int message,
void* message_data,
void* user_data);
@@ -677,12 +662,6 @@ struct YR_SCAN_CONTEXT
// Scanning flags.
int flags;
// Thread index for the thread using this scan context. The number of threads
// that can use a YR_RULES object simultaneusly is limited by the YR_MAX_THREADS
// constant. Each thread using a YR_RULES get assigned a unique thread index
// in the range [0, YR_MAX_THREADS)
int tidx;
// Canary value used for preventing hand-crafted objects from being embedded
// in compiled rules and used to exploit YARA. The canary value is initialized
// to a random value and is subsequently set to all objects created by
@@ -713,18 +692,41 @@ struct YR_SCAN_CONTEXT
// contains entries for external variables and modules.
YR_HASH_TABLE* objects_table;
// Arena used for storing YR_MATCH structures asociated to the matches found.
YR_ARENA* matches_arena;
// Arena used for storing pointers to the YR_STRING struct for each matching
// string. The pointers are used by _yr_scanner_clean_matches.
YR_ARENA* matching_strings_arena;
// Notebook used for storing YR_MATCH structures associated to the matches
// found.
YR_NOTEBOOK * matches_notebook;
// Stopwatch used for measuring the time elapsed during the scan.
YR_STOPWATCH stopwatch;
// Fiber pool used by yr_re_exec.
RE_FIBER_POOL re_fiber_pool;
// A bitmap with one bit per rule, bit N is set when the rule with index N
// has matched.
YR_BITMASK* rule_matches_flags;
// A bitmap with one bit per namespace, bit N is set if the namespace with
// index N has some global rule that is not satisfied.
YR_BITMASK* ns_unsatisfied_flags;
// Array with pointers to lists of matches. Item N in the array has the
// list of matches for string with index N.
YR_MATCHES* matches;
// "unconfirmed_matches" is like "matches" but for strings that are part of
// a chain. Let's suppose that the string S is split in two chained strings
// S1 <- S2. When a match is found for S1, we can't be sure that S matches
// until a match for S2 is found (within the range defined by chain_gap_min
// and chain_gap_max), so the matches for S1 are put in "unconfirmed_matches"
// until they can be confirmed or discarded.
YR_MATCHES* unconfirmed_matches;
// rule_cost is a pointer to an array of 64-bit integers with one entry per
// rule. Entry N has the time cost for rule with index N.
#ifdef PROFILING_ENABLED
uint64_t* time_cost;
#endif
};
@@ -865,7 +867,7 @@ struct YR_DICTIONARY_ITEMS
//
// The boolean shouldn't be true if the next item was pushed in the stack, it
// can be true only when all the items have been returned in previous calls,
// in which case the value for the next item should be UNDEFINED. The stack
// in which case the value for the next item should be YR_UNDEFINED. The stack
// should look like this after the last call to "next":
//
// +------------+
@@ -875,8 +877,8 @@ struct YR_DICTIONARY_ITEMS
// +------------+
// | . . . |
//
// We can't use the UNDEFINED value in the stack as an indicator of the end
// of the iterator, because it's legitimate for an iterator to return UNDEFINED
// We can't use the YR_UNDEFINED value in the stack as an indicator of the end
// of the iterator, because it's legitimate for an iterator to return YR_UNDEFINED
// items in the middle of the iteration.
//
// The "next" function should return ERROR_SUCCESS if everything went fine or
libyara/lexer.c
+371 -443
View File
File diff suppressed because it is too large Load Diff
libyara/lexer.l
+33 -10
View File
@@ -179,6 +179,8 @@ octdigit [0-7]
"ascii" { return _ASCII_; }
"wide" { return _WIDE_; }
"xor" { return _XOR_; }
"base64" { return _BASE64_; }
"base64wide" { return _BASE64_WIDE_; }
"fullword" { return _FULLWORD_; }
"nocase" { return _NOCASE_; }
"condition" { return _CONDITION_; }
@@ -274,10 +276,15 @@ include[ \t]+\" {
}
}
YR_NAMESPACE* ns = (YR_NAMESPACE*) yr_arena_get_ptr(
compiler->arena,
YR_NAMESPACES_TABLE,
compiler->current_namespace_idx * sizeof(struct YR_NAMESPACE));
included_rules = compiler->include_callback(
include_path,
current_file_name,
compiler->current_namespace->name,
ns->name,
compiler->incl_clbk_user_data);
if (included_rules != NULL)
@@ -704,7 +711,7 @@ u?int(8|16|32)(be)? {
}
\{(({hexdigit}|[ \-|\?\[\]\(\)\n\r\t]|\/\*[^*]*\*+([^*/][^*]*\*+)*\/)+|\/\/.*)+\} {
\{(({hexdigit}|[ \-|\?\[\]\(\)\n\r\t]|\/\*(\/|\**[^*/])*\*+\/)+|\/\/.*)+\} {
// Match hex-digits with whitespace or comments. The latter are stripped
// out by hex_lexer.l
// TODO(vmalvarez): Integrate the hex string lexer and parser into this one,
@@ -761,11 +768,19 @@ void yywarning(
vsnprintf(message, sizeof(message), message_fmt, message_args);
YR_RULE* current_rule = NULL;
if (compiler->current_rule_idx != UINT32_MAX)
current_rule = yr_arena_get_ptr(
compiler->arena,
YR_RULES_TABLE,
compiler->current_rule_idx * sizeof(YR_RULE));
compiler->callback(
YARA_ERROR_LEVEL_WARNING,
file_name,
compiler->current_line ? compiler->current_line : yyget_lineno(yyscanner),
compiler->current_rule,
current_rule,
message,
compiler->user_data);
@@ -816,6 +831,14 @@ void yyerror(
file_name = NULL;
}
YR_RULE* current_rule = NULL;
if (compiler->current_rule_idx != UINT32_MAX)
current_rule = yr_arena_get_ptr(
compiler->arena,
YR_RULES_TABLE,
compiler->current_rule_idx * sizeof(YR_RULE));
if (error_message != NULL)
{
yr_compiler_set_error_extra_info(compiler, error_message);
@@ -827,7 +850,7 @@ void yyerror(
YARA_ERROR_LEVEL_ERROR,
file_name,
compiler->last_error_line,
compiler->current_rule,
current_rule,
error_message,
compiler->user_data);
}
@@ -837,12 +860,12 @@ void yyerror(
yr_compiler_get_error_message(compiler, message, sizeof(message));
compiler->callback(
YARA_ERROR_LEVEL_ERROR,
file_name,
compiler->last_error_line,
compiler->current_rule,
message,
compiler->user_data);
YARA_ERROR_LEVEL_ERROR,
file_name,
compiler->last_error_line,
current_rule,
message,
compiler->user_data);
}
}
libyara/libyara.c
+7 -53
View File
@@ -50,10 +50,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endif
#endif
YR_THREAD_STORAGE_KEY yr_tidx_key;
YR_THREAD_STORAGE_KEY yr_recovery_state_key;
YR_THREAD_STORAGE_KEY yr_yyfatal_trampoline_tls;
YR_THREAD_STORAGE_KEY yr_trycatch_trampoline_tls;
static int init_count = 0;
@@ -142,8 +140,8 @@ YR_API int yr_initialize(void)
}
FAIL_ON_ERROR(yr_heap_alloc());
FAIL_ON_ERROR(yr_thread_storage_create(&yr_tidx_key));
FAIL_ON_ERROR(yr_thread_storage_create(&yr_recovery_state_key));
FAIL_ON_ERROR(yr_thread_storage_create(&yr_yyfatal_trampoline_tls));
FAIL_ON_ERROR(yr_thread_storage_create(&yr_trycatch_trampoline_tls));
#if defined HAVE_LIBCRYPTO && OPENSSL_VERSION_NUMBER < 0x10100000L
@@ -184,18 +182,6 @@ YR_API int yr_initialize(void)
}
//
// yr_finalize_thread
//
// This function is deprecated, it's maintained only for backward compatibility
// with programs that already use it. Calling yr_finalize_thread from each
// thread using libyara is not required anymore.
YR_DEPRECATED_API void yr_finalize_thread(void)
{
}
//
// yr_finalize
//
@@ -233,8 +219,8 @@ YR_API int yr_finalize(void)
#endif
FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_tidx_key));
FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_recovery_state_key));
FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_yyfatal_trampoline_tls));
FAIL_ON_ERROR(yr_thread_storage_destroy(&yr_trycatch_trampoline_tls));
FAIL_ON_ERROR(yr_modules_finalize());
FAIL_ON_ERROR(yr_heap_free());
@@ -246,38 +232,6 @@ YR_API int yr_finalize(void)
return ERROR_SUCCESS;
}
//
// yr_set_tidx
//
// Set the thread index (tidx) for the current thread. The tidx is the index
// that will be used by the thread to access thread-specific data stored in
// YR_RULES structure.
//
// Args:
// int tidx - The zero-based tidx that will be associated to the current
// thread.
//
YR_API void yr_set_tidx(int tidx)
{
yr_thread_storage_set_value(&yr_tidx_key, (void*) (size_t) (tidx + 1));
}
//
// yr_get_tidx
//
// Get the thread index (tidx) for the current thread.
//
// Returns:
// The tidx for the current thread or -1 if the current thread doesn't
// have any tidx associated.
//
YR_API int yr_get_tidx(void)
{
return (int) (size_t) yr_thread_storage_get_value(&yr_tidx_key) - 1;
}
//
@@ -289,7 +243,7 @@ YR_API int yr_get_tidx(void)
//
// Args:
// YR_CONFIG_NAME name - Any of the values defined by the YR_CONFIG_NAME
// enum. Posible values are:
// enum. Possible values are:
//
// YR_CONFIG_STACK_SIZE data type: uint32_t
// YR_CONFIG_MAX_STRINGS_PER_RULE data type: uint32_t
libyara/mem.c
+3
View File
@@ -71,6 +71,9 @@ void* yr_malloc(size_t size)
void* yr_realloc(void* ptr, size_t size)
{
if (ptr == NULL)
return (void*) HeapAlloc(hHeap, HEAP_ZERO_MEMORY, size);
return (void*) HeapReAlloc(hHeap, HEAP_ZERO_MEMORY, ptr, size);
}
libyara/modules.c
+2
View File
@@ -150,6 +150,7 @@ int yr_modules_load(
mi.module_data_size = 0;
result = context->callback(
context,
CALLBACK_MSG_IMPORT_MODULE,
&mi,
context->user_data);
@@ -188,6 +189,7 @@ int yr_modules_load(
}
result = context->callback(
context,
CALLBACK_MSG_MODULE_IMPORTED,
module_structure,
context->user_data);
libyara/modules/cuckoo/cuckoo.c
+139
View File
@@ -27,6 +27,7 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <string.h>
#include <jansson.h>
@@ -34,6 +35,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <yara/re.h>
#include <yara/modules.h>
#if defined(_WIN32) || defined(__CYGWIN__)
#define strcasecmp _stricmp
#endif
@@ -174,6 +176,138 @@ define_function(network_http_post)
}
// checks the host array in network JSON
// loops through array and checks value for argument
define_function(network_host)
{
YR_SCAN_CONTEXT* context = scan_context();
YR_OBJECT* network_obj = parent();
json_t* network_json = (json_t*) network_obj->data;
json_t* value;
uint64_t result = 0;
size_t index;
json_t* hosts_info_json = json_object_get(network_json, "hosts");
json_array_foreach(hosts_info_json, index, value)
{
if (yr_re_match(context, regexp_argument(1), json_string_value(value)) > 0)
{
result = 1;
break;
}
}
return_integer(result);
}
// loops through TCP array in network JSON
// checks for dest IP regex and dest port integer match
define_function(network_tcp)
{
YR_SCAN_CONTEXT* context = scan_context();
YR_OBJECT* network_obj = parent();
json_t* network_json = (json_t*) network_obj->data;
json_t* value;
uint64_t result = 0;
size_t index;
int dport;
char* dst;
json_t* tcp_info_json = json_object_get(network_json, "tcp");
json_array_foreach(tcp_info_json, index, value)
{
if (json_unpack(value, "{s:i, s:s}", "dport", &dport, "dst", &dst) == 0)
{
if (yr_re_match(context, regexp_argument(1), dst) > 0)
{
if ((int64_t)dport == integer_argument(2)) {
result = 1;
break;
}
}
}
}
return_integer(result);
}
// loops through UDP array in network JSON
// checks for dest IP regex and dest port integer match
define_function(network_udp)
{
YR_SCAN_CONTEXT* context = scan_context();
YR_OBJECT* network_obj = parent();
json_t* network_json = (json_t*) network_obj->data;
json_t* value;
uint64_t result = 0;
size_t index;
int dport;
char* dst;
json_t* udp_info_json = json_object_get(network_json, "udp");
json_array_foreach(udp_info_json, index, value)
{
if (json_unpack(value, "{s:i, s:s}", "dport", &dport, "dst", &dst) == 0)
{
if (yr_re_match(context, regexp_argument(1), dst) > 0)
{
if ((int64_t)dport == integer_argument(2)) {
result = 1;
break;
}
}
}
}
return_integer(result);
}
// loops through http array in network JSON
// checks for regex match on user-agent
define_function(network_http_user_agent)
{
YR_SCAN_CONTEXT* context = scan_context();
YR_OBJECT* network_obj = parent();
json_t* network_json = (json_t*) network_obj->data;
json_t* http_json = json_object_get(network_json, "http");
json_t* value;
uint64_t result = 0;
size_t index;
char* user_agent;
json_array_foreach(http_json, index, value)
{
if (json_unpack(value, "{s:s}", "user-agent", &user_agent) == 0)
{
if (yr_re_match(context, regexp_argument(1), user_agent) > 0)
{
result = 1;
break;
}
}
}
return_integer(result);
}
define_function(registry_key_access)
{
YR_SCAN_CONTEXT* context = scan_context();
@@ -254,6 +388,10 @@ begin_declarations;
declare_function("http_get", "r", "i", network_http_get);
declare_function("http_post", "r", "i", network_http_post);
declare_function("http_request", "r", "i", network_http_request);
declare_function("http_user_agent", "r", "i", network_http_user_agent);
declare_function("host", "r", "i", network_host);
declare_function("tcp", "ri", "i", network_tcp);
declare_function("udp", "ri", "i", network_udp);
end_struct("network");
begin_struct("registry");
@@ -344,3 +482,4 @@ int module_unload(YR_OBJECT* module)
return ERROR_SUCCESS;
}
libyara/modules/dex/dex.c
+16 -13
View File
@@ -45,6 +45,7 @@ begin_declarations;
declare_string("DEX_FILE_MAGIC_036");
declare_string("DEX_FILE_MAGIC_037");
declare_string("DEX_FILE_MAGIC_038");
declare_string("DEX_FILE_MAGIC_039");
declare_integer("ENDIAN_CONSTANT");
declare_integer("REVERSE_ENDIAN_CONSTANT");
@@ -147,7 +148,7 @@ begin_declarations;
begin_struct_array("class_defs");
declare_integer("class_idx");
declare_integer("access_flags");
declare_integer("superclass_idx");
declare_integer("super_class_idx");
declare_integer("interfaces_offset");
declare_integer("source_file_idx");
declare_integer("annotations_offset");
@@ -264,12 +265,12 @@ static int64_t dex_get_integer(
const char* pattern,
int64_t index)
{
if (index == UNDEFINED)
return UNDEFINED;
if (index == YR_UNDEFINED)
return YR_UNDEFINED;
// Impose a reasonably large limit to table indexes.
if (index > 0x80000)
return UNDEFINED;
return YR_UNDEFINED;
return get_integer(object, pattern, (int) index);
}
@@ -280,7 +281,7 @@ static SIZED_STRING* dex_get_string(
const char* pattern,
int64_t index)
{
if (index == UNDEFINED)
if (index == YR_UNDEFINED)
return NULL;
// Impose a reasonably large limit to table indexes.
@@ -306,7 +307,8 @@ dex_header_t* dex_get_header(
if (memcmp(dex_header->magic, DEX_FILE_MAGIC_035, 8) != 0 &&
memcmp(dex_header->magic, DEX_FILE_MAGIC_036, 8) != 0 &&
memcmp(dex_header->magic, DEX_FILE_MAGIC_037, 8) != 0 &&
memcmp(dex_header->magic, DEX_FILE_MAGIC_038, 8) != 0)
memcmp(dex_header->magic, DEX_FILE_MAGIC_038, 8) != 0 &&
memcmp(dex_header->magic, DEX_FILE_MAGIC_039, 8) != 0)
{
return NULL;
}
@@ -440,7 +442,7 @@ uint32_t load_encoded_field(
int64_t name_idx = dex_get_integer(
dex->object, "field_ids[%i].name_idx", *previous_field_idx);
if (name_idx == UNDEFINED)
if (name_idx == YR_UNDEFINED)
return 0;
SIZED_STRING* field_name = dex_get_string(
@@ -580,7 +582,7 @@ uint32_t load_encoded_method(
int64_t name_idx = dex_get_integer(
dex->object, "method_ids[%i].name_idx", *previous_method_idx);
if (name_idx == UNDEFINED)
if (name_idx == YR_UNDEFINED)
return 0;
#ifdef DEBUG_DEX_MODULE
@@ -939,13 +941,13 @@ void dex_parse(
#ifdef DEBUG_DEX_MODULE
printf("[DEX] CLASS ID item class_idx:0x%x access_flags:0x%x " \
"super_class_idx:0x%x interfaces_off:0x%x source_file_idx:0x%x "\
"super_class_idx:0x%x interfaces_offset:0x%x source_file_idx:0x%x "\
"annotations_offset:0x%x class_data_offset:0x%x "\
"static_values_offset:0x%x\n",
yr_le32toh(class_id_item->class_idx),
yr_le32toh(class_id_item->access_flags),
yr_le32toh(class_id_item->super_class_idx),
yr_le32toh(class_id_item->interfaces_off),
yr_le32toh(class_id_item->interfaces_offset),
yr_le32toh(class_id_item->source_file_idx),
yr_le32toh(class_id_item->annotations_offset),
yr_le32toh(class_id_item->class_data_offset),
@@ -958,14 +960,14 @@ void dex_parse(
"class_defs[%i].access_flags", i);
set_integer(yr_le32toh(class_id_item->super_class_idx), dex->object,
"class_defs[%i].super_class_idx", i);
set_integer(yr_le32toh(class_id_item->interfaces_off), dex->object,
"class_defs[%i].interfaces_off", i);
set_integer(yr_le32toh(class_id_item->interfaces_offset), dex->object,
"class_defs[%i].interfaces_offset", i);
set_integer(yr_le32toh(class_id_item->source_file_idx), dex->object,
"class_defs[%i].source_file_idx", i);
set_integer(yr_le32toh(class_id_item->annotations_offset), dex->object,
"class_defs[%i].annotations_offset", i);
set_integer(yr_le32toh(class_id_item->class_data_offset), dex->object,
"class_defs[%i].class_data_off", i);
"class_defs[%i].class_data_offset", i);
set_integer(yr_le32toh(class_id_item->static_values_offset), dex->object,
"class_defs[%i].static_values_offset", i);
@@ -1143,6 +1145,7 @@ int module_load(
set_string(DEX_FILE_MAGIC_036, module_object, "DEX_FILE_MAGIC_036");
set_string(DEX_FILE_MAGIC_037, module_object, "DEX_FILE_MAGIC_037");
set_string(DEX_FILE_MAGIC_038, module_object, "DEX_FILE_MAGIC_038");
set_string(DEX_FILE_MAGIC_039, module_object, "DEX_FILE_MAGIC_039");
set_integer(0x12345678, module_object, "ENDIAN_CONSTANT");
set_integer(0x78563412, module_object, "REVERSE_ENDIAN_CONSTANT");
libyara/modules/dotnet/dotnet.c
+13 -1
View File
@@ -196,6 +196,12 @@ BLOB_PARSE_RESULT dotnet_parse_blob_entry(
result.size = 0;
}
// There is an additional terminal byte which is 0x01 under certain
// conditions. The exact conditions are not relevant to our parsing but are
// documented in ECMA-335 II.24.2.4.
if (result.length > 0)
result.length--;
return result;
}
@@ -796,6 +802,9 @@ void dotnet_parse_tilde_2(
// Now follow the Class index into the TypeRef table.
typeref_row = typeref_ptr + (typeref_row_size * class_index);
if (!fits_in_pe(pe, typeref_row, typeref_row_size))
break;
// Skip over the ResolutionScope and check the Name field,
// which is an index into the Strings heap.
row_count = max_rows(4,
@@ -1439,6 +1448,7 @@ void dotnet_parse_tilde(
// number of rows being indexed into.
ROWS rows;
INDEX_SIZES index_sizes;
uint32_t heap_sizes;
// Default all rows to 0. They will be set to actual values later on, if
// they exist in the file.
@@ -1455,13 +1465,15 @@ void dotnet_parse_tilde(
if (!struct_fits_in_pe(pe, tilde_header, TILDE_HEADER))
return;
uint32_t heap_sizes = yr_le32toh(tilde_header->HeapSizes);
heap_sizes = yr_le32toh(tilde_header->HeapSizes);
// Set index sizes for various heaps.
if (heap_sizes & 0x01)
index_sizes.string = 4;
if (heap_sizes & 0x02)
index_sizes.guid = 4;
if (heap_sizes & 0x04)
index_sizes.blob = 4;
libyara/modules/elf/elf.c
+5 -5
View File
@@ -135,7 +135,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \
if(ULONG_MAX - yr_##bo##bits##toh(elf_header->ph_offset) < \
ELF_SIZE_OF_PROGRAM_TABLE(bits,bo,elf_header)) \
{ \
return UNDEFINED; \
return YR_UNDEFINED; \
} \
\
if (yr_##bo##bits##toh(elf_header->ph_offset) == 0 || \
@@ -144,7 +144,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \
ELF_SIZE_OF_PROGRAM_TABLE(bits,bo,elf_header) > elf_size || \
yr_##bo##16toh(elf_header->ph_entry_count) == 0) \
{ \
return UNDEFINED; \
return YR_UNDEFINED; \
} \
\
program = (elf##bits##_program_header_t*) \
@@ -174,7 +174,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \
if(ULONG_MAX - yr_##bo##bits##toh(elf_header->sh_offset) < \
ELF_SIZE_OF_SECTION_TABLE(bits,bo,elf_header)) \
{ \
return UNDEFINED; \
return YR_UNDEFINED; \
} \
\
if (yr_##bo##bits##toh(elf_header->sh_offset) == 0 || \
@@ -183,7 +183,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \
ELF_SIZE_OF_SECTION_TABLE(bits,bo,elf_header) > elf_size || \
yr_##bo##16toh(elf_header->sh_entry_count) == 0) \
{ \
return UNDEFINED; \
return YR_UNDEFINED; \
} \
\
section = (elf##bits##_section_header_t*) \
@@ -204,7 +204,7 @@ uint64_t elf_rva_to_offset_##bits##_##bo( \
section++; \
} \
} \
return UNDEFINED; \
return YR_UNDEFINED; \
}
#define PARSE_ELF_HEADER(bits,bo) \
libyara/modules/hash/hash.c
+15 -15
View File
@@ -242,7 +242,7 @@ define_function(data_md5)
yr_md5_init(&md5_context);
if (offset < 0 || length < 0 || offset < block->base)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
cached_ascii_digest = get_from_cache(
module(), "md5", arg_offset, arg_length);
@@ -281,7 +281,7 @@ define_function(data_md5)
// range contains gaps of undefined data the checksum is
// undefined.
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -289,7 +289,7 @@ define_function(data_md5)
}
if (!past_first_block)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
yr_md5_final(digest, &md5_context);
@@ -325,7 +325,7 @@ define_function(data_sha1)
yr_sha1_init(&sha_context);
if (offset < 0 || length < 0 || offset < block->base)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
cached_ascii_digest = get_from_cache(
module(), "sha1", arg_offset, arg_length);
@@ -363,7 +363,7 @@ define_function(data_sha1)
// range contains gaps of undefined data the checksum is
// undefined.
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -371,7 +371,7 @@ define_function(data_sha1)
}
if (!past_first_block)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
yr_sha1_final(digest, &sha_context);
@@ -407,7 +407,7 @@ define_function(data_sha256)
yr_sha256_init(&sha256_context);
if (offset < 0 || length < 0 || offset < block->base)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
cached_ascii_digest = get_from_cache(
module(), "sha256", arg_offset, arg_length);
@@ -444,7 +444,7 @@ define_function(data_sha256)
// range contains gaps of undefined data the checksum is
// undefined.
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -452,7 +452,7 @@ define_function(data_sha256)
}
if (!past_first_block)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
yr_sha256_final(digest, &sha256_context);
@@ -478,7 +478,7 @@ define_function(data_checksum32)
int past_first_block = false;
if (offset < 0 || length < 0 || offset < block->base)
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -511,7 +511,7 @@ define_function(data_checksum32)
// range contains gaps of undefined data the checksum is
// undefined.
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -519,7 +519,7 @@ define_function(data_checksum32)
}
if (!past_first_block)
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
return_integer(checksum);
}
@@ -551,7 +551,7 @@ define_function(data_crc32)
int past_first_block = false;
if (offset < 0 || length < 0 || offset < block->base)
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -584,7 +584,7 @@ define_function(data_crc32)
// range contains gaps of undefined data the checksum is
// undefined.
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -592,7 +592,7 @@ define_function(data_crc32)
}
if (!past_first_block)
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
return_integer(checksum ^ 0xFFFFFFFF);
}
libyara/modules/macho/macho.c
+9 -9
View File
@@ -481,7 +481,7 @@ void macho_parse_fat_file_##bits( \
continue; \
\
/* Force 'file' array entry creation. */ \
set_integer(UNDEFINED, object, "file[%i].magic", i); \
set_integer(YR_UNDEFINED, object, "file[%i].magic", i); \
\
/* Get specific Mach-O file data. */ \
macho_parse_file(data + offset, file_size, \
@@ -720,7 +720,7 @@ define_function(file_index_type)
uint64_t nfat = get_integer(module, "nfat_arch");
if (is_undefined(module, "nfat_arch"))
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
for (int i = 0; i < nfat; i++)
{
@@ -730,7 +730,7 @@ define_function(file_index_type)
return_integer(i);
}
}
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
}
@@ -744,7 +744,7 @@ define_function(file_index_subtype)
uint64_t nfat = get_integer(module, "nfat_arch");
if (is_undefined(module, "nfat_arch"))
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
for (int i = 0; i < nfat; i++)
{
@@ -756,7 +756,7 @@ define_function(file_index_subtype)
return_integer(i);
}
}
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
}
@@ -769,7 +769,7 @@ define_function(ep_for_arch_type)
uint64_t nfat = get_integer(module, "nfat_arch");
if (is_undefined(module, "nfat_arch"))
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
for (int i = 0; i < nfat; i++)
{
@@ -781,7 +781,7 @@ define_function(ep_for_arch_type)
return_integer(file_offset + entry_point);
}
}
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
}
@@ -795,7 +795,7 @@ define_function(ep_for_arch_subtype)
uint64_t nfat = get_integer(module, "nfat_arch");
if (is_undefined(module, "nfat_arch"))
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
for (int i = 0; i < nfat; i++)
{
@@ -808,7 +808,7 @@ define_function(ep_for_arch_subtype)
return_integer(file_offset + entry_point);
}
}
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
}
libyara/modules/magic/magic.c
+83 -52
View File
@@ -33,80 +33,127 @@ The original idea and inspiration for this module comes from Armin Buescher.
*/
#include <yara/mem.h>
#include <yara/modules.h>
#include <magic.h>
#define MODULE_NAME magic
magic_t magic_cookie[YR_MAX_THREADS];
const char* cached_types[YR_MAX_THREADS];
const char* cached_mime_types[YR_MAX_THREADS];
// Thread-local storage key used to store a pointer to a MAGIC_CACHE struct.
YR_THREAD_STORAGE_KEY magic_tls;
typedef struct
{
magic_t magic_cookie;
const char* cached_type;
const char* cached_mime_type;
} MAGIC_CACHE ;
static int get_cache(MAGIC_CACHE** cache)
{
*cache = (MAGIC_CACHE*) yr_thread_storage_get_value(&magic_tls);
if (*cache == NULL)
{
*cache = (MAGIC_CACHE*) yr_malloc(sizeof(MAGIC_CACHE));
if (*cache == NULL)
return ERROR_INSUFFICIENT_MEMORY;
(*cache)->magic_cookie = magic_open(0);
if ((*cache)->magic_cookie == NULL)
return ERROR_INSUFFICIENT_MEMORY;
if (magic_load((*cache)->magic_cookie, NULL) != 0)
{
magic_close((*cache)->magic_cookie);
return ERROR_INTERNAL_FATAL_ERROR;
}
(*cache)->cached_type = NULL;
(*cache)->cached_mime_type = NULL;
return yr_thread_storage_set_value(&magic_tls, *cache);
}
return ERROR_SUCCESS;
}
define_function(magic_mime_type)
{
YR_MEMORY_BLOCK* block;
YR_SCAN_CONTEXT* context = scan_context();
YR_MEMORY_BLOCK* block;
MAGIC_CACHE* cache;
const uint8_t* block_data;
if (context->flags & SCAN_FLAGS_PROCESS_MEMORY)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
if (cached_mime_types[context->tidx] == NULL)
FAIL_ON_ERROR(get_cache(&cache));
if (cache->cached_mime_type == NULL)
{
block = first_memory_block(context);
block_data = block->fetch_data(block);
if (block_data != NULL)
{
magic_setflags(magic_cookie[context->tidx], MAGIC_MIME_TYPE);
magic_setflags(cache->magic_cookie, MAGIC_MIME_TYPE);
cached_mime_types[context->tidx] = magic_buffer(
magic_cookie[context->tidx],
cache->cached_mime_type = magic_buffer(
cache->magic_cookie,
block_data,
block->size);
}
}
if (cached_mime_types[context->tidx] == NULL)
return_string(UNDEFINED);
if (cache->cached_mime_type == NULL)
return_string(YR_UNDEFINED);
return_string((char*) cached_mime_types[context->tidx]);
return_string((char*) cache->cached_mime_type);
}
define_function(magic_type)
{
MAGIC_CACHE* cache;
YR_MEMORY_BLOCK* block;
YR_SCAN_CONTEXT* context = scan_context();
const uint8_t* block_data;
if (context->flags & SCAN_FLAGS_PROCESS_MEMORY)
return_string(UNDEFINED);
return_string(YR_UNDEFINED);
if (cached_types[context->tidx] == NULL)
FAIL_ON_ERROR(get_cache(&cache));
if (cache->cached_type == NULL)
{
block = first_memory_block(context);
block_data = block->fetch_data(block);
if (block_data != NULL)
{
magic_setflags(magic_cookie[context->tidx], 0);
magic_setflags(cache->magic_cookie, 0);
cached_types[context->tidx] = magic_buffer(
magic_cookie[context->tidx],
cache->cached_type = magic_buffer(
cache->magic_cookie,
block_data,
block->size);
}
}
if (cached_types[context->tidx] == NULL)
return_string(UNDEFINED);
if (cache->cached_type == NULL)
return_string(YR_UNDEFINED);
return_string((char*) cached_types[context->tidx]);
return_string((char*) cache->cached_type);
}
begin_declarations;
@@ -120,25 +167,22 @@ end_declarations;
int module_initialize(
YR_MODULE* module)
{
int i;
for (i = 0; i < YR_MAX_THREADS; i++)
magic_cookie[i] = NULL;
return ERROR_SUCCESS;
return yr_thread_storage_create(&magic_tls);
}
int module_finalize(
YR_MODULE* module)
{
int i;
MAGIC_CACHE* cache = (MAGIC_CACHE*) yr_thread_storage_get_value(&magic_tls);
for (i = 0; i < YR_MAX_THREADS; i++)
if (magic_cookie[i] != NULL)
magic_close(magic_cookie[i]);
if (cache != NULL)
{
magic_close(cache->magic_cookie);
yr_free(cache);
}
return ERROR_SUCCESS;
return yr_thread_storage_destroy(&magic_tls);
}
@@ -148,27 +192,6 @@ int module_load(
void* module_data,
size_t module_data_size)
{
cached_types[context->tidx] = NULL;
cached_mime_types[context->tidx] = NULL;
if (magic_cookie[context->tidx] == NULL)
{
magic_cookie[context->tidx] = magic_open(0);
if (magic_cookie[context->tidx] != NULL)
{
if (magic_load(magic_cookie[context->tidx], NULL) != 0)
{
magic_close(magic_cookie[context->tidx]);
return ERROR_INTERNAL_FATAL_ERROR;
}
}
else
{
return ERROR_INSUFFICIENT_MEMORY;
}
}
return ERROR_SUCCESS;
}
@@ -176,5 +199,13 @@ int module_load(
int module_unload(
YR_OBJECT* module)
{
MAGIC_CACHE* cache = (MAGIC_CACHE*) yr_thread_storage_get_value(&magic_tls);
if (cache != NULL)
{
cache->cached_type = NULL;
cache->cached_mime_type = NULL;
}
return ERROR_SUCCESS;
}
libyara/modules/math/math.c
+23 -23
View File
@@ -57,7 +57,7 @@ define_function(string_entropy)
uint32_t* data = (uint32_t*) yr_calloc(256, sizeof(uint32_t));
if (data == NULL)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
for (i = 0; i < s->length; i++)
{
@@ -97,12 +97,12 @@ define_function(data_entropy)
YR_MEMORY_BLOCK_ITERATOR* iterator = context->iterator;
if (offset < 0 || length < 0 || offset < block->base)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
data = (uint32_t*) yr_calloc(256, sizeof(uint32_t));
if (data == NULL)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -118,7 +118,7 @@ define_function(data_entropy)
if (block_data == NULL)
{
yr_free(data);
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
total_len += data_len;
@@ -142,7 +142,7 @@ define_function(data_entropy)
// undefined.
yr_free(data);
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -152,7 +152,7 @@ define_function(data_entropy)
if (!past_first_block)
{
yr_free(data);
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
for (i = 0; i < 256; i++)
@@ -207,7 +207,7 @@ define_function(data_deviation)
YR_MEMORY_BLOCK_ITERATOR* iterator = context->iterator;
if (offset < 0 || length < 0 || offset < block->base)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -220,7 +220,7 @@ define_function(data_deviation)
block_data = block->fetch_data(block);
if (block_data == NULL)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
total_len += data_len;
offset += data_len;
@@ -238,7 +238,7 @@ define_function(data_deviation)
// the checksum over a range of non contiguous blocks. As
// range contains gaps of undefined data the checksum is
// undefined.
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -246,7 +246,7 @@ define_function(data_deviation)
}
if (!past_first_block)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
return_float(sum / total_len);
}
@@ -282,7 +282,7 @@ define_function(data_mean)
size_t i;
if (offset < 0 || length < 0 || offset < block->base)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -296,7 +296,7 @@ define_function(data_mean)
const uint8_t* block_data = block->fetch_data(block);
if (block_data == NULL)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
total_len += data_len;
offset += data_len;
@@ -314,7 +314,7 @@ define_function(data_mean)
// the checksum over a range of non contiguous blocks. As
// range contains gaps of undefined data the checksum is
// undefined.
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -322,7 +322,7 @@ define_function(data_mean)
}
if (!past_first_block)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
return_float(sum / total_len);
}
@@ -350,7 +350,7 @@ define_function(data_serial_correlation)
double scc = 0;
if (offset < 0 || length < 0 || offset < block->base)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -364,7 +364,7 @@ define_function(data_serial_correlation)
const uint8_t* block_data = block->fetch_data(block);
if (block_data == NULL)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
total_len += data_len;
offset += data_len;
@@ -388,7 +388,7 @@ define_function(data_serial_correlation)
// the checksum over a range of non contiguous blocks. As
// range contains gaps of undefined data the checksum is
// undefined.
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -396,7 +396,7 @@ define_function(data_serial_correlation)
}
if (!past_first_block)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
scct1 += scclast * sccun;
scct2 *= scct2;
@@ -467,7 +467,7 @@ define_function(data_monte_carlo_pi)
YR_MEMORY_BLOCK_ITERATOR* iterator = context->iterator;
if (offset < 0 || length < 0 || offset < block->base)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
foreach_memory_block(iterator, block)
{
@@ -483,7 +483,7 @@ define_function(data_monte_carlo_pi)
const uint8_t* block_data = block->fetch_data(block);
if (block_data == NULL)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
offset += data_len;
length -= data_len;
@@ -520,7 +520,7 @@ define_function(data_monte_carlo_pi)
// the checksum over a range of non contiguous blocks. As
// range contains gaps of undefined data the checksum is
// undefined.
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
}
if (block->base + block->size > offset + length)
@@ -528,7 +528,7 @@ define_function(data_monte_carlo_pi)
}
if (!past_first_block || mcount == 0)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
mpi = 4.0 * ((double) inmont / mcount);
@@ -575,7 +575,7 @@ define_function(string_monte_carlo_pi)
}
if (mcount == 0)
return_float(UNDEFINED);
return_float(YR_UNDEFINED);
mpi = 4.0 * ((double) inmont / mcount);
return_float(fabs((mpi - PI) / PI));
libyara/modules/pe/pe.c
+638 -317
View File
File diff suppressed because it is too large Load Diff
libyara/modules/tests/tests.c
+4
View File
@@ -141,6 +141,10 @@ begin_declarations;
declare_string("s");
end_struct_dictionary("struct_dict");
begin_struct_dictionary("empty_struct_dict");
declare_integer("unused");
end_struct_dictionary("empty_struct_dict");
declare_function("match", "rs", "i", match);
declare_function("isum", "ii", "i", isum_2);
declare_function("isum", "iii", "i", isum_3);
libyara/modules/time/time.c
+1 -1
View File
@@ -38,7 +38,7 @@ define_function(now)
{
time_t now = time(NULL);
if (now == -1)
return_integer(UNDEFINED);
return_integer(YR_UNDEFINED);
return_integer((long) now);
}
libyara/notebook.c
+152
View File
@@ -0,0 +1,152 @@
#include <stdint.h>
/*
Copyright (c) 2020. The YARA Authors. All Rights Reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <assert.h>
#include <yara/error.h>
#include <yara/mem.h>
#include <yara/notebook.h>
// Forward declaration of YR_NOTEBOOK_PAGE.
typedef struct YR_NOTEBOOK_PAGE YR_NOTEBOOK_PAGE;
// A notebook is a data structure that can be used for allocating memory
// space in the same way malloc() would do. However, the buffers returned
// by yr_notebook_alloc() are backed by a larger buffer reserved by the notebook
// beforehand called a "page". The notebook fulfills the allocations performed
// via yr_notebook_alloc() with space taken from the current page, or creates
// a new page when necessary. It's recommended that the page size is at least
// 4x the size of the buffers you plan to allocate with yr_notebook_alloc().
// Once the notebook is destroyed all the pages are freed, and consequently
// all the buffers allocated via yr_notebook_alloc().
struct YR_NOTEBOOK
{
// Size of each page in the notebook.
size_t page_size;
// Pointer to the first page in the book, this also the most recently
// created page, the one that is being filled.
YR_NOTEBOOK_PAGE* page_list_head;
};
// YR_NOTEBOOK_PAGE
struct YR_NOTEBOOK_PAGE
{
// Amount of bytes in the page that are actually used.
size_t used;
// Pointer to next page.
YR_NOTEBOOK_PAGE* next;
// Page's data.
uint8_t data[0];
};
// Creates a new notebook. The notebook initially has a single page of the
// specified size, but more pages are created if needed.
int yr_notebook_create(
size_t page_size,
YR_NOTEBOOK** pool)
{
YR_NOTEBOOK* new_notebook = yr_malloc(sizeof(YR_NOTEBOOK));
if (new_notebook == NULL)
return ERROR_INSUFFICIENT_MEMORY;
new_notebook->page_list_head = yr_malloc(
sizeof(YR_NOTEBOOK_PAGE) + page_size);
if (new_notebook->page_list_head == NULL)
{
yr_free(new_notebook);
return ERROR_INSUFFICIENT_MEMORY;
}
new_notebook->page_size = page_size;
new_notebook->page_list_head->used = 0;
new_notebook->page_list_head->next = NULL;
*pool = new_notebook;
return ERROR_SUCCESS;
}
// Destroys a notebook and frees all the notebook's pages.
int yr_notebook_destroy(
YR_NOTEBOOK* pool)
{
YR_NOTEBOOK_PAGE* page = pool->page_list_head;
while (page != NULL)
{
YR_NOTEBOOK_PAGE* next = page->next;
yr_free(page);
page = next;
}
yr_free(pool);
return ERROR_SUCCESS;
}
void* yr_notebook_alloc(
YR_NOTEBOOK* notebook,
size_t size)
{
// The requested memory size can't be larger than a notebook's page.
assert(size <= notebook->page_size);
// If the requested size doesn't fit in current page's free space, allocate
// a new page.
if (notebook->page_size - notebook->page_list_head->used < size)
{
YR_NOTEBOOK_PAGE* new_page = yr_malloc(
sizeof(YR_NOTEBOOK_PAGE) + notebook->page_size);
if (new_page == NULL)
return NULL;
new_page->used = 0;
new_page->next = notebook->page_list_head;
notebook->page_list_head = new_page;
}
void *ptr = notebook->page_list_head->data + notebook->page_list_head->used;
notebook->page_list_head->used += size;
return ptr;
}
libyara/object.c
+18 -9
View File
@@ -96,7 +96,7 @@ int yr_object_create(
switch(type)
{
case OBJECT_TYPE_INTEGER:
obj->value.i = UNDEFINED;
obj->value.i = YR_UNDEFINED;
break;
case OBJECT_TYPE_FLOAT:
obj->value.d = NAN;
@@ -314,7 +314,14 @@ int yr_object_from_external_variable(
break;
}
*object = obj;
if (result == ERROR_SUCCESS)
{
*object = obj;
}
else
{
yr_object_destroy(obj);
}
}
return result;
@@ -594,6 +601,7 @@ int yr_object_copy(
yr_object_copy(
object_as_function(object)->return_obj,
&object_as_function(copy)->return_obj),
// cleanup
yr_object_destroy(copy));
for (i = 0; i < YR_MAX_OVERLOADED_FUNCTIONS; i++)
@@ -614,6 +622,7 @@ int yr_object_copy(
FAIL_ON_ERROR_WITH_CLEANUP(
yr_object_structure_set_member(copy, o),
// cleanup
yr_free(o);
yr_object_destroy(copy));
@@ -911,7 +920,7 @@ bool yr_object_has_undefined_value(
case OBJECT_TYPE_STRING:
return field_obj->value.ss == NULL;
case OBJECT_TYPE_INTEGER:
return field_obj->value.i == UNDEFINED;
return field_obj->value.i == YR_UNDEFINED;
}
return false;
@@ -936,7 +945,7 @@ int64_t yr_object_get_integer(
va_end(args);
if (integer_obj == NULL)
return UNDEFINED;
return YR_UNDEFINED;
assertf(integer_obj->type == OBJECT_TYPE_INTEGER,
"type of \"%s\" is not integer\n", field);
@@ -1155,19 +1164,19 @@ YR_API void yr_object_print_data(
switch(object->type)
{
case OBJECT_TYPE_FLOAT:
if (object->value.d != UNDEFINED)
if (object->value.i != YR_UNDEFINED)
printf(" = %f", object->value.d);
else
printf(" = UNDEFINED");
printf(" = YR_UNDEFINED");
break;
case OBJECT_TYPE_INTEGER:
if (object->value.i != UNDEFINED)
if (object->value.i != YR_UNDEFINED)
printf(" = %" PRId64, object->value.i);
else
printf(" = UNDEFINED");
printf(" = YR_UNDEFINED");
break;
@@ -1192,7 +1201,7 @@ YR_API void yr_object_print_data(
}
else
{
printf(" = UNDEFINED");
printf(" = YR_UNDEFINED");
}
break;
libyara/parser.c
+408 -327
View File
File diff suppressed because it is too large Load Diff
libyara/re.c
+197 -249
View File
@@ -41,6 +41,7 @@ order to avoid confusion with operating system threads.
#include <assert.h>
#include <string.h>
#include <yara/compiler.h>
#include <yara/limits.h>
#include <yara/globals.h>
#include <yara/utils.h>
@@ -81,9 +82,9 @@ typedef struct _RE_REPEAT_ANY_ARGS
} RE_REPEAT_ANY_ARGS;
typedef struct _RE_EMIT_CONTEXT {
YR_ARENA* arena;
typedef struct _RE_EMIT_CONTEXT
{
YR_ARENA* arena;
RE_SPLIT_ID_TYPE next_split_id;
} RE_EMIT_CONTEXT;
@@ -136,8 +137,8 @@ RE_NODE* yr_re_node_create(
result->prev_sibling = NULL;
result->next_sibling = NULL;
result->greedy = true;
result->forward_code = NULL;
result->backward_code = NULL;
result->forward_code_ref = YR_ARENA_NULL_REF;
result->backward_code_ref = YR_ARENA_NULL_REF;
}
return result;
@@ -234,7 +235,7 @@ void yr_re_ast_destroy(
// yr_re_parse
//
// Parses a regexp but don't emit its code. A further call to
// yr_re_emit_code is required to get the code.
// yr_re_ast_emit_code is required to get the code.
//
int yr_re_parse(
@@ -250,7 +251,7 @@ int yr_re_parse(
// yr_re_parse_hex
//
// Parses a hex string but don't emit its code. A further call to
// yr_re_emit_code is required to get the code.
// yr_re_ast_emit_code is required to get the code.
//
int yr_re_parse_hex(
@@ -265,36 +266,35 @@ int yr_re_parse_hex(
//
// yr_re_compile
//
// Parses the regexp and emit its code to the provided code_arena.
// Parses the regexp and emit its code to the provided to the
// YR_RE_CODE_SECTION in the specified arena.
//
int yr_re_compile(
const char* re_string,
int flags,
YR_ARENA* code_arena,
RE** re,
YR_ARENA* arena,
YR_ARENA_REF* ref,
RE_ERROR* error)
{
RE_AST* re_ast;
RE _re;
FAIL_ON_ERROR(yr_arena_reserve_memory(
code_arena, sizeof(int64_t) + RE_MAX_CODE_SIZE));
FAIL_ON_ERROR(yr_re_parse(re_string, &re_ast, error));
_re.flags = flags;
FAIL_ON_ERROR_WITH_CLEANUP(
yr_arena_write_data(
code_arena,
arena,
YR_RE_CODE_SECTION,
&_re,
sizeof(_re),
(void**) re),
ref),
yr_re_ast_destroy(re_ast));
FAIL_ON_ERROR_WITH_CLEANUP(
yr_re_ast_emit_code(re_ast, code_arena, false),
yr_re_ast_emit_code(re_ast, arena, false),
yr_re_ast_destroy(re_ast));
yr_re_ast_destroy(re_ast);
@@ -386,6 +386,7 @@ SIZED_STRING* yr_re_ast_extract_literal(
return NULL;
string->length = length;
string->flags = 0;
if (re_ast->root_node->type == RE_NODE_LITERAL)
{
@@ -549,9 +550,6 @@ int yr_re_ast_split_at_chaining_point(
*min_gap = child->start;
*max_gap = child->end;
re_ast->root_node = re_ast->root_node;
re_ast->flags = re_ast->flags;
(*remainder_re_ast)->root_node = concat;
(*remainder_re_ast)->flags = re_ast->flags;
@@ -570,16 +568,14 @@ int yr_re_ast_split_at_chaining_point(
int _yr_emit_inst(
RE_EMIT_CONTEXT* emit_context,
uint8_t opcode,
uint8_t** instruction_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref)
{
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
*code_size = sizeof(uint8_t);
instruction_ref));
return ERROR_SUCCESS;
}
@@ -589,23 +585,22 @@ int _yr_emit_inst_arg_uint8(
RE_EMIT_CONTEXT* emit_context,
uint8_t opcode,
uint8_t argument,
uint8_t** instruction_addr,
uint8_t** argument_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref)
{
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&argument,
sizeof(uint8_t),
(void**) argument_addr));
*code_size = 2 * sizeof(uint8_t);
argument_ref));
return ERROR_SUCCESS;
}
@@ -615,23 +610,22 @@ int _yr_emit_inst_arg_uint16(
RE_EMIT_CONTEXT* emit_context,
uint8_t opcode,
uint16_t argument,
uint8_t** instruction_addr,
uint16_t** argument_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref)
{
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&argument,
sizeof(uint16_t),
(void**) argument_addr));
*code_size = sizeof(uint8_t) + sizeof(uint16_t);
argument_ref));
return ERROR_SUCCESS;
}
@@ -641,23 +635,22 @@ int _yr_emit_inst_arg_uint32(
RE_EMIT_CONTEXT* emit_context,
uint8_t opcode,
uint32_t argument,
uint8_t** instruction_addr,
uint32_t** argument_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref)
{
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&argument,
sizeof(uint32_t),
(void**) argument_addr));
*code_size = sizeof(uint8_t) + sizeof(uint32_t);
argument_ref));
return ERROR_SUCCESS;
}
@@ -667,23 +660,22 @@ int _yr_emit_inst_arg_int16(
RE_EMIT_CONTEXT* emit_context,
uint8_t opcode,
int16_t argument,
uint8_t** instruction_addr,
int16_t** argument_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref)
{
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&argument,
sizeof(int16_t),
(void**) argument_addr));
*code_size = sizeof(uint8_t) + sizeof(int16_t);
argument_ref));
return ERROR_SUCCESS;
}
@@ -694,23 +686,22 @@ int _yr_emit_inst_arg_struct(
uint8_t opcode,
void* structure,
size_t structure_size,
uint8_t** instruction_addr,
void** argument_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref)
{
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
structure,
structure_size,
(void**) argument_addr));
*code_size = sizeof(uint8_t) + structure_size;
argument_ref));
return ERROR_SUCCESS;
}
@@ -720,9 +711,8 @@ int _yr_emit_split(
RE_EMIT_CONTEXT* emit_context,
uint8_t opcode,
int16_t argument,
uint8_t** instruction_addr,
int16_t** argument_addr,
size_t* code_size)
YR_ARENA_REF* instruction_ref,
YR_ARENA_REF* argument_ref)
{
assert(opcode == RE_OPCODE_SPLIT_A || opcode == RE_OPCODE_SPLIT_B);
@@ -731,12 +721,14 @@ int _yr_emit_split(
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&opcode,
sizeof(uint8_t),
(void**) instruction_addr));
instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&emit_context->next_split_id,
sizeof(RE_SPLIT_ID_TYPE),
NULL));
@@ -745,27 +737,30 @@ int _yr_emit_split(
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
&argument,
sizeof(int16_t),
(void**) argument_addr));
*code_size = sizeof(uint8_t) + sizeof(RE_SPLIT_ID_TYPE) + sizeof(int16_t);
argument_ref));
return ERROR_SUCCESS;
}
#define current_re_code_offset() \
yr_arena_get_current_offset(emit_context->arena, YR_RE_CODE_SECTION)
static int _yr_re_emit(
RE_EMIT_CONTEXT* emit_context,
RE_NODE* re_node,
int flags,
uint8_t** code_addr,
size_t* code_size)
YR_ARENA_REF* code_ref)
{
size_t branch_size;
size_t split_size;
size_t inst_size;
size_t jmp_size;
yr_arena_off_t jmp_offset;
yr_arena_off_t bookmark_1 = 0;
yr_arena_off_t bookmark_2 = 0;
yr_arena_off_t bookmark_3 = 0;
yr_arena_off_t bookmark_4 = 0;
bool emit_split;
bool emit_repeat;
@@ -780,162 +775,132 @@ static int _yr_re_emit(
int16_t* split_offset_addr = NULL;
int16_t* jmp_offset_addr = NULL;
uint8_t* instruction_addr = NULL;
*code_size = 0;
YR_ARENA_REF instruction_ref;
YR_ARENA_REF split_offset_ref;
YR_ARENA_REF jmp_instruction_ref;
YR_ARENA_REF jmp_offset_ref;
YR_ARENA_REF repeat_start_args_ref;
switch(re_node->type)
{
case RE_NODE_LITERAL:
FAIL_ON_ERROR(_yr_emit_inst_arg_uint8(
emit_context,
RE_OPCODE_LITERAL,
re_node->value,
&instruction_addr,
NULL,
code_size));
&instruction_ref,
NULL));
break;
case RE_NODE_MASKED_LITERAL:
FAIL_ON_ERROR(_yr_emit_inst_arg_uint16(
emit_context,
RE_OPCODE_MASKED_LITERAL,
re_node->mask << 8 | re_node->value,
&instruction_addr,
NULL,
code_size));
&instruction_ref,
NULL));
break;
case RE_NODE_WORD_CHAR:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_WORD_CHAR,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_NON_WORD_CHAR:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_NON_WORD_CHAR,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_WORD_BOUNDARY:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_WORD_BOUNDARY,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_NON_WORD_BOUNDARY:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_NON_WORD_BOUNDARY,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_SPACE:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_SPACE,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_NON_SPACE:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_NON_SPACE,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_DIGIT:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_DIGIT,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_NON_DIGIT:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_NON_DIGIT,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_ANY:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_ANY,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_CLASS:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_CLASS,
&instruction_addr,
code_size));
&instruction_ref));
FAIL_ON_ERROR(yr_arena_write_data(
emit_context->arena,
YR_RE_CODE_SECTION,
re_node->re_class,
sizeof(*re_node->re_class),
NULL));
*code_size += sizeof(*re_node->re_class);
break;
case RE_NODE_ANCHOR_START:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_MATCH_AT_START,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_ANCHOR_END:
FAIL_ON_ERROR(_yr_emit_inst(
emit_context,
RE_OPCODE_MATCH_AT_END,
&instruction_addr,
code_size));
&instruction_ref));
break;
case RE_NODE_CONCAT:
FAIL_ON_ERROR(_yr_re_emit(
emit_context,
(flags & EMIT_BACKWARDS)?
re_node->children_tail:
re_node->children_head,
flags,
&instruction_addr,
&branch_size));
*code_size += branch_size;
&instruction_ref));
if (flags & EMIT_BACKWARDS)
child = re_node->children_tail->prev_sibling;
@@ -948,95 +913,88 @@ static int _yr_re_emit(
emit_context,
child,
flags,
NULL,
&branch_size));
*code_size += branch_size;
NULL));
child = (flags & EMIT_BACKWARDS) ?
child->prev_sibling:
child->next_sibling;
}
break;
case RE_NODE_PLUS:
// Code for e+ looks like:
//
// L1: code for e
// split L1, L2
// L2:
//
FAIL_ON_ERROR(_yr_re_emit(
emit_context,
re_node->children_head,
flags,
&instruction_addr,
&branch_size));
&instruction_ref));
*code_size += branch_size;
jmp_offset = instruction_ref.offset - current_re_code_offset();
if (jmp_offset < INT16_MIN)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
FAIL_ON_ERROR(_yr_emit_split(
emit_context,
re_node->greedy ? RE_OPCODE_SPLIT_B : RE_OPCODE_SPLIT_A,
-((int16_t) branch_size),
(int16_t) jmp_offset,
NULL,
&split_offset_addr,
&split_size));
NULL));
*code_size += split_size;
break;
case RE_NODE_STAR:
// Code for e* looks like:
//
// L1: split L1, L2
// code for e
// jmp L1
// L2:
FAIL_ON_ERROR(_yr_emit_split(
emit_context,
re_node->greedy ? RE_OPCODE_SPLIT_A : RE_OPCODE_SPLIT_B,
0,
&instruction_addr,
&split_offset_addr,
&split_size));
*code_size += split_size;
&instruction_ref,
&split_offset_ref));
FAIL_ON_ERROR(_yr_re_emit(
emit_context,
re_node->children_head,
flags,
NULL,
&branch_size));
NULL));
*code_size += branch_size;
jmp_offset = instruction_ref.offset - current_re_code_offset();
if (jmp_offset < INT16_MIN)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
// Emit jump with offset set to 0.
FAIL_ON_ERROR(_yr_emit_inst_arg_int16(
emit_context,
RE_OPCODE_JUMP,
-((uint16_t)(branch_size + split_size)),
(int16_t) jmp_offset,
NULL,
&jmp_offset_addr,
&jmp_size));
NULL));
*code_size += jmp_size;
jmp_offset = current_re_code_offset() - instruction_ref.offset;
if (split_size + branch_size + jmp_size >= INT16_MAX)
if (jmp_offset > INT16_MAX)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
// Update split offset.
*split_offset_addr = (int16_t) (split_size + branch_size + jmp_size);
split_offset_addr = (int16_t*) yr_arena_ref_to_ptr(
emit_context->arena, &split_offset_ref);
*split_offset_addr = (int16_t) jmp_offset;
break;
case RE_NODE_ALT:
// Code for e1|e2 looks like:
//
// split L1, L2
@@ -1053,20 +1011,14 @@ static int _yr_re_emit(
emit_context,
RE_OPCODE_SPLIT_A,
0,
&instruction_addr,
&split_offset_addr,
&split_size));
*code_size += split_size;
&instruction_ref,
&split_offset_ref));
FAIL_ON_ERROR(_yr_re_emit(
emit_context,
re_node->children_head,
flags,
NULL,
&branch_size));
*code_size += branch_size;
NULL));
// Emit jump with offset set to 0.
@@ -1074,36 +1026,39 @@ static int _yr_re_emit(
emit_context,
RE_OPCODE_JUMP,
0,
NULL,
&jmp_offset_addr,
&jmp_size));
&jmp_instruction_ref,
&jmp_offset_ref));
*code_size += jmp_size;
jmp_offset = current_re_code_offset() - instruction_ref.offset;
if (split_size + branch_size + jmp_size >= INT16_MAX)
if (jmp_offset > INT16_MAX)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
// Update split offset.
*split_offset_addr = (int16_t) (split_size + branch_size + jmp_size);
split_offset_addr = (int16_t*) yr_arena_ref_to_ptr(
emit_context->arena, &split_offset_ref);
*split_offset_addr = (int16_t) jmp_offset;
FAIL_ON_ERROR(_yr_re_emit(
emit_context,
re_node->children_tail,
flags,
NULL,
&branch_size));
NULL));
*code_size += branch_size;
jmp_offset = current_re_code_offset() - jmp_instruction_ref.offset;
if (branch_size + jmp_size >= INT16_MAX)
if (jmp_offset > INT16_MAX)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
// Update offset for jmp instruction.
*jmp_offset_addr = (int16_t) (branch_size + jmp_size);
jmp_offset_addr = (int16_t*) yr_arena_ref_to_ptr(
emit_context->arena, &jmp_offset_ref);
*jmp_offset_addr = (int16_t) jmp_offset;
break;
case RE_NODE_RANGE_ANY:
repeat_any_args.min = re_node->start;
repeat_any_args.max = re_node->end;
@@ -1114,15 +1069,12 @@ static int _yr_re_emit(
RE_OPCODE_REPEAT_ANY_UNGREEDY,
&repeat_any_args,
sizeof(repeat_any_args),
&instruction_addr,
NULL,
&inst_size));
&instruction_ref,
NULL));
*code_size += inst_size;
break;
case RE_NODE_RANGE:
// Code for e{n,m} looks like:
//
// code for e --- prolog
@@ -1182,10 +1134,7 @@ static int _yr_re_emit(
emit_context,
re_node->children_head,
flags,
&instruction_addr,
&branch_size));
*code_size += branch_size;
&instruction_ref));
}
if (emit_repeat)
@@ -1211,6 +1160,8 @@ static int _yr_re_emit(
repeat_args.offset = 0;
bookmark_1 = current_re_code_offset();
FAIL_ON_ERROR(_yr_emit_inst_arg_struct(
emit_context,
re_node->greedy ?
@@ -1218,23 +1169,23 @@ static int _yr_re_emit(
RE_OPCODE_REPEAT_START_UNGREEDY,
&repeat_args,
sizeof(repeat_args),
emit_prolog ? NULL : &instruction_addr,
(void**) &repeat_start_args_addr,
&inst_size));
emit_prolog ? NULL : &instruction_ref,
&repeat_start_args_ref));
*code_size += inst_size;
bookmark_2 = current_re_code_offset();
FAIL_ON_ERROR(_yr_re_emit(
emit_context,
re_node->children_head,
flags | EMIT_DONT_SET_FORWARDS_CODE | EMIT_DONT_SET_BACKWARDS_CODE,
NULL,
&branch_size));
NULL));
*code_size += branch_size;
bookmark_3 = current_re_code_offset();
repeat_start_args_addr->offset = (int32_t)(2 * inst_size + branch_size);
repeat_args.offset = -((int32_t) branch_size);
if (bookmark_2 - bookmark_3 < INT32_MIN)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
repeat_args.offset = (int32_t)(bookmark_2 - bookmark_3);
FAIL_ON_ERROR(_yr_emit_inst_arg_struct(
emit_context,
@@ -1244,14 +1195,23 @@ static int _yr_re_emit(
&repeat_args,
sizeof(repeat_args),
NULL,
NULL,
&inst_size));
NULL));
*code_size += inst_size;
bookmark_4 = current_re_code_offset();
repeat_start_args_addr = (RE_REPEAT_ARGS*) yr_arena_ref_to_ptr(
emit_context->arena, &repeat_start_args_ref);
if (bookmark_4 - bookmark_1 > INT32_MAX)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
repeat_start_args_addr->offset = (int32_t)(bookmark_4 - bookmark_1);
}
if (emit_split)
{
bookmark_1 = current_re_code_offset();
FAIL_ON_ERROR(_yr_emit_split(
emit_context,
re_node->greedy ?
@@ -1259,10 +1219,7 @@ static int _yr_re_emit(
RE_OPCODE_SPLIT_B,
0,
NULL,
&split_offset_addr,
&split_size));
*code_size += split_size;
&split_offset_ref));
}
if (emit_epilog)
@@ -1271,18 +1228,20 @@ static int _yr_re_emit(
emit_context,
re_node->children_head,
emit_prolog ? flags | EMIT_DONT_SET_FORWARDS_CODE : flags,
emit_prolog || emit_repeat ? NULL : &instruction_addr,
&branch_size));
*code_size += branch_size;
emit_prolog || emit_repeat ? NULL : &instruction_ref));
}
if (emit_split)
{
if (split_size + branch_size >= INT16_MAX)
bookmark_2 = current_re_code_offset();
if (bookmark_2 - bookmark_1 > INT16_MAX)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
*split_offset_addr = (int16_t) (split_size + branch_size);
split_offset_addr = (int16_t*) yr_arena_ref_to_ptr(
emit_context->arena, &split_offset_ref);
*split_offset_addr = (int16_t) (bookmark_2 - bookmark_1);
}
break;
@@ -1291,16 +1250,22 @@ static int _yr_re_emit(
if (flags & EMIT_BACKWARDS)
{
if (!(flags & EMIT_DONT_SET_BACKWARDS_CODE))
re_node->backward_code = instruction_addr + *code_size;
{
re_node->backward_code_ref.buffer_id = YR_RE_CODE_SECTION;
re_node->backward_code_ref.offset = yr_arena_get_current_offset(
emit_context->arena, YR_RE_CODE_SECTION);
}
}
else
{
if (!(flags & EMIT_DONT_SET_FORWARDS_CODE))
re_node->forward_code = instruction_addr;
{
re_node->forward_code_ref = instruction_ref;
}
}
if (code_addr != NULL)
*code_addr = instruction_addr;
if (code_ref != NULL)
*code_ref = instruction_ref;
return ERROR_SUCCESS;
}
@@ -1313,18 +1278,7 @@ int yr_re_ast_emit_code(
{
RE_EMIT_CONTEXT emit_context;
size_t code_size;
size_t total_size;
// Ensure that we have enough contiguous memory space in the arena to
// contain the regular expression code. The code can't span over multiple
// non-contiguous pages.
FAIL_ON_ERROR(yr_arena_reserve_memory(arena, RE_MAX_CODE_SIZE));
// Emit code for matching the regular expressions forwards.
total_size = 0;
emit_context.arena = arena;
emit_context.next_split_id = 0;
@@ -1332,21 +1286,12 @@ int yr_re_ast_emit_code(
&emit_context,
re_ast->root_node,
backwards_code ? EMIT_BACKWARDS : 0,
NULL,
&code_size));
total_size += code_size;
NULL));
FAIL_ON_ERROR(_yr_emit_inst(
&emit_context,
RE_OPCODE_MATCH,
NULL,
&code_size));
total_size += code_size;
if (total_size > RE_MAX_CODE_SIZE)
return ERROR_REGULAR_EXPRESSION_TOO_LARGE;
NULL));
return ERROR_SUCCESS;
}
@@ -1832,7 +1777,7 @@ static int _yr_re_fiber_sync(
// match the data starting at the address specified by "input". The "input"
// pointer can point to any address inside a memory buffer. Arguments
// "input_forwards_size" and "input_backwards_size" indicate how many bytes
// can be accesible starting at "input" and going forwards and backwards
// can be accessible starting at "input" and going forwards and backwards
// respectively.
//
// <--- input_backwards_size -->|<----------- input_forwards_size -------->
@@ -2395,7 +2340,8 @@ int yr_re_fast_exec(
static void _yr_re_print_node(
RE_NODE* re_node)
RE_NODE* re_node,
uint32_t indent)
{
RE_NODE* child;
int i;
@@ -2403,14 +2349,16 @@ static void _yr_re_print_node(
if (re_node == NULL)
return;
if (indent > 0)
printf("\n%*s", indent, " ");
switch (re_node->type)
{
case RE_NODE_ALT:
printf("Alt(");
_yr_re_print_node(re_node->children_head);
printf(", ");
_yr_re_print_node(re_node->children_tail);
printf(")");
_yr_re_print_node(re_node->children_head, indent + 4);
printf(",");
_yr_re_print_node(re_node->children_tail, indent + 4);
printf("\n%*s%s", indent, " " , ")");
break;
case RE_NODE_CONCAT:
@@ -2418,27 +2366,27 @@ static void _yr_re_print_node(
child = re_node->children_head;
while (child != NULL)
{
_yr_re_print_node(child);
printf(", ");
_yr_re_print_node(child, indent + 4);
printf(",");
child = child->next_sibling;
}
printf(")");
printf("\n%*s%s", indent, " ", ")");
break;
case RE_NODE_STAR:
printf("Star(");
_yr_re_print_node(re_node->children_head);
_yr_re_print_node(re_node->children_head, indent + 4);
printf(")");
break;
case RE_NODE_PLUS:
printf("Plus(");
_yr_re_print_node(re_node->children_head);
_yr_re_print_node(re_node->children_head, indent + 4);
printf(")");
break;
case RE_NODE_LITERAL:
printf("Lit(%02X)", re_node->value);
printf("Lit(%c)", re_node->value);
break;
case RE_NODE_MASKED_LITERAL:
@@ -2475,8 +2423,8 @@ static void _yr_re_print_node(
case RE_NODE_RANGE:
printf("Range(%d-%d, ", re_node->start, re_node->end);
_yr_re_print_node(re_node->children_head);
printf(")");
_yr_re_print_node(re_node->children_head, indent + 4);
printf("\n%*s%s", indent, " ", ")");
break;
case RE_NODE_CLASS:
@@ -2496,5 +2444,5 @@ static void _yr_re_print_node(
void yr_re_print(
RE_AST* re_ast)
{
_yr_re_print_node(re_ast->root_node);
_yr_re_print_node(re_ast->root_node, 0);
}
libyara/re_lexer.c
+300 -488
View File
File diff suppressed because it is too large Load Diff
libyara/re_lexer.l
+16 -6
View File
@@ -567,14 +567,20 @@ int read_escaped_char(
}
//
// yyfatal (actually named re_yyfatal because of the '%option prefix="re_yy"'
// directive) is called when a fatal error occurs in the parser. When this
// happens we are deep inside the parsing logic generated by flex/bison and
// the only way to exit gracefully from there is using setjmp/longjmp.
//
void yyfatal(
yyscan_t yyscanner,
const char *error_message)
{
jmp_buf* recovery_state = (jmp_buf*) yr_thread_storage_get_value(
&yr_recovery_state_key);
jmp_buf* recovery_trampoline = (jmp_buf*) yr_thread_storage_get_value(
&yr_yyfatal_trampoline_tls);
longjmp(*recovery_state, 1);
longjmp(*recovery_trampoline, 1);
}
@@ -606,15 +612,19 @@ int yr_parse_re_string(
RE_ERROR* error)
{
yyscan_t yyscanner;
jmp_buf recovery_state;
jmp_buf recovery_trampoline;
RE_LEX_ENVIRONMENT lex_env;
lex_env.last_error = ERROR_SUCCESS;
lex_env.last_error_message[0] = '\0';
yr_thread_storage_set_value(&yr_recovery_state_key, &recovery_state);
yr_thread_storage_set_value(
&yr_yyfatal_trampoline_tls,
&recovery_trampoline);
if (setjmp(recovery_state) != 0)
// setjmp returns a non-zero value only when we are returning to this
// point via a call to longjmp to the recovery trampoline.
if (setjmp(recovery_trampoline) != 0)
return ERROR_INTERNAL_FATAL_ERROR;
FAIL_ON_ERROR(yr_re_ast_create(re_ast));
libyara/rules.c
+81 -97
View File
@@ -40,6 +40,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <yara/globals.h>
#include <yara/scan.h>
#include <yara/scanner.h>
#include <yara/compiler.h>
YR_API int yr_rules_define_integer_variable(
@@ -174,47 +175,6 @@ YR_API int yr_rules_define_string_variable(
}
#ifdef PROFILING_ENABLED
YR_API void yr_rules_print_profiling_info(
YR_RULES* rules)
{
YR_RULE* rule;
printf("\n===== PROFILING INFORMATION =====\n\n");
yr_rules_foreach(rules, rule)
{
printf(
"%s:%s: %" PRIu64 " (%0.3f%%)\n",
rule->ns->name,
rule->identifier,
rule->time_cost,
(float) rule->time_cost / rules->time_cost * 100);
}
printf("\n=================================\n");
}
#endif
YR_API void yr_rules_reset_profiling_info(
YR_RULES* rules)
{
#ifdef PROFILING_ENABLED
YR_RULE* rule;
yr_rules_foreach(rules, rule)
{
#ifdef _WIN32
InterlockedExchange64(&rule->time_cost, 0);
#else
__sync_fetch_and_and(&rule->time_cost, 0);
#endif
}
#endif
}
YR_API int yr_rules_scan_mem_blocks(
YR_RULES* rules,
YR_MEMORY_BLOCK_ITERATOR* iterator,
@@ -383,37 +343,48 @@ YR_API int yr_rules_scan_proc(
}
YR_API int yr_rules_load_stream(
YR_STREAM* stream,
int yr_rules_from_arena(
YR_ARENA* arena,
YR_RULES** rules)
{
YARA_RULES_FILE_HEADER* header;
YR_RULES* new_rules = (YR_RULES*) yr_malloc(sizeof(YR_RULES));
if (new_rules == NULL)
return ERROR_INSUFFICIENT_MEMORY;
FAIL_ON_ERROR_WITH_CLEANUP(
yr_arena_load_stream(stream, &new_rules->arena),
// cleanup
yr_free(new_rules));
YR_SUMMARY* summary = (YR_SUMMARY*) yr_arena_get_ptr(
arena, YR_SUMMARY_SECTION, 0);
header = (YARA_RULES_FILE_HEADER*)
yr_arena_base_address(new_rules->arena);
// Now YR_RULES relies on this arena, let's increment the arena's
// reference count so that if the original owner of the arena calls
// yr_arena_destroy the arena is not destroyed.
yr_arena_acquire(arena);
new_rules->code_start = header->code_start;
new_rules->externals_list_head = header->externals_list_head;
new_rules->rules_list_head = header->rules_list_head;
new_rules->ac_match_table = header->ac_match_table;
new_rules->ac_transition_table = header->ac_transition_table;
new_rules->ac_tables_size = header->ac_tables_size;
new_rules->arena = arena;
new_rules->num_rules = summary->num_rules;
new_rules->num_strings = summary->num_strings;
new_rules->num_namespaces = summary->num_namespaces;
memset(new_rules->tidx_mask, 0, sizeof(new_rules->tidx_mask));
new_rules->rules_list_head = yr_arena_get_ptr(
arena, YR_RULES_TABLE, 0);
FAIL_ON_ERROR_WITH_CLEANUP(
yr_mutex_create(&new_rules->mutex),
// cleanup
yr_free(new_rules));
new_rules->strings_list_head = yr_arena_get_ptr(
arena, YR_STRINGS_TABLE, 0);
new_rules->externals_list_head = yr_arena_get_ptr(
arena, YR_EXTERNAL_VARIABLES_TABLE, 0);
new_rules->ac_transition_table = yr_arena_get_ptr(
arena, YR_AC_TRANSITION_TABLE, 0);
new_rules->ac_match_table = yr_arena_get_ptr(
arena, YR_AC_STATE_MATCHES_TABLE, 0);
new_rules->ac_match_pool = yr_arena_get_ptr(
arena, YR_AC_STATE_MATCHES_POOL, 0);
new_rules->code_start = yr_arena_get_ptr(
arena, YR_CODE_SECTION, 0);
*rules = new_rules;
@@ -421,6 +392,28 @@ YR_API int yr_rules_load_stream(
}
YR_API int yr_rules_load_stream(
YR_STREAM* stream,
YR_RULES** rules)
{
YR_ARENA* arena;
// Load the arena's data the stream. We are the owners of the arena.
FAIL_ON_ERROR(yr_arena_load_stream(stream, &arena));
// Create the YR_RULES object from the arena, this makes YR_RULES owner
// of the arena too.
FAIL_ON_ERROR(yr_rules_from_arena(arena, rules));
// Release our ownership so that YR_RULES is the single owner. This way the
// arena is destroyed when YR_RULES is destroyed.
yr_arena_release(arena);
return ERROR_SUCCESS;
}
YR_API int yr_rules_load(
const char* filename,
YR_RULES** rules)
@@ -447,11 +440,6 @@ YR_API int yr_rules_save_stream(
YR_RULES* rules,
YR_STREAM* stream)
{
int i;
for (i = 0; i < YR_BITARRAY_NCHARS(YR_MAX_THREADS); ++i)
assert(rules->tidx_mask[i] == 0);
return yr_arena_save_stream(rules->arena, stream);
}
@@ -489,40 +477,37 @@ YR_API int yr_rules_get_stats(
YR_RULES* rules,
YR_RULES_STATS *stats)
{
YR_RULE* rule;
YR_STRING* string;
memset(stats, 0, sizeof(YR_RULES_STATS));
stats->ac_tables_size = yr_arena_get_current_offset(
rules->arena, YR_AC_TRANSITION_TABLE) / sizeof(YR_AC_TRANSITION);
uint32_t* match_list_lengths = (uint32_t*) yr_malloc(
sizeof(uint32_t) * rules->ac_tables_size);
float match_list_length_sum = 0;
int i, c = 0;
sizeof(uint32_t) * stats->ac_tables_size);
if (match_list_lengths == NULL)
return ERROR_INSUFFICIENT_MEMORY;
memset(stats, 0, sizeof(YR_RULES_STATS));
stats->num_rules = rules->num_rules;
stats->num_strings = rules->num_strings;
yr_rules_foreach(rules, rule)
float match_list_length_sum = 0;
int c = 0;
for (int i = 0; i < stats->ac_tables_size; i++)
{
stats->rules++;
yr_rule_strings_foreach(rule, string)
stats->strings++;
}
stats->ac_tables_size = rules->ac_tables_size;
for (i = 0; i < rules->ac_tables_size; i++)
{
YR_AC_MATCH* match = rules->ac_match_table[i].match;
int match_list_length = 0;
while (match != NULL)
if (rules->ac_match_table[i] != 0)
{
match_list_length++;
stats->ac_matches++;
match = match->next;
YR_AC_MATCH *m = &rules->ac_match_pool[rules->ac_match_table[i]-1];
while (m != NULL)
{
match_list_length++;
stats->ac_matches++;
m = m->next;
}
}
if (i == 0)
@@ -546,7 +531,7 @@ YR_API int yr_rules_get_stats(
// sort match_list_lengths in increasing order for computing percentiles.
qsort(match_list_lengths, c, sizeof(match_list_lengths[0]), _uint32_cmp);
for (i = 0; i < 100; i++)
for (int i = 0; i < 100; i++)
{
if (i < c)
stats->top_ac_match_list_lengths[i] = match_list_lengths[c-i-1];
@@ -558,7 +543,7 @@ YR_API int yr_rules_get_stats(
stats->ac_match_list_length_pctls[0] = match_list_lengths[0];
stats->ac_match_list_length_pctls[100] = match_list_lengths[c-1];
for (i = 1; i < 100; i++)
for (int i = 1; i < 100; i++)
stats->ac_match_list_length_pctls[i] = match_list_lengths[(c * i) / 100];
yr_free(match_list_lengths);
@@ -580,8 +565,7 @@ YR_API int yr_rules_destroy(
external++;
}
yr_mutex_destroy(&rules->mutex);
yr_arena_destroy(rules->arena);
yr_arena_release(rules->arena);
yr_free(rules);
return ERROR_SUCCESS;
@@ -592,11 +576,11 @@ YR_API void yr_rule_disable(
{
YR_STRING* string;
rule->g_flags |= RULE_GFLAGS_DISABLED;
rule->flags |= RULE_FLAGS_DISABLED;
yr_rule_strings_foreach(rule, string)
{
string->g_flags |= STRING_GFLAGS_DISABLED;
string->flags |= STRING_FLAGS_DISABLED;
}
}
@@ -606,10 +590,10 @@ YR_API void yr_rule_enable(
{
YR_STRING* string;
rule->g_flags &= ~RULE_GFLAGS_DISABLED;
rule->flags &= ~RULE_FLAGS_DISABLED;
yr_rule_strings_foreach(rule, string)
{
string->g_flags &= ~STRING_GFLAGS_DISABLED;
string->flags &= ~STRING_FLAGS_DISABLED;
}
}
libyara/scan.c
+71 -76
View File
@@ -210,7 +210,7 @@ static int _yr_scan_wicompare(
static void _yr_scan_update_match_chain_length(
int tidx,
YR_SCAN_CONTEXT* context,
YR_STRING* string,
YR_MATCH* match_to_update,
int chain_length)
@@ -225,7 +225,7 @@ static void _yr_scan_update_match_chain_length(
if (string->chained_to == NULL)
return;
match = string->chained_to->unconfirmed_matches[tidx].head;
match = context->unconfirmed_matches[string->chained_to->idx].head;
while (match != NULL)
{
@@ -235,7 +235,7 @@ static void _yr_scan_update_match_chain_length(
ending_offset + string->chain_gap_min <= match_to_update->offset)
{
_yr_scan_update_match_chain_length(
tidx, string->chained_to, match, chain_length + 1);
context, string->chained_to, match, chain_length + 1);
}
match = match->next;
@@ -350,7 +350,6 @@ static int _yr_scan_verify_chained_string_match(
uint64_t ending_offset;
int32_t full_chain_length;
int tidx = context->tidx;
bool add_match = false;
if (matching_string->chained_to == NULL)
@@ -358,7 +357,7 @@ static int _yr_scan_verify_chained_string_match(
// The matching string is the head of the chain, this match should be
// added to the list of unconfirmed matches. The match will remain
// unconfirmed until all the strings in the chain are found with the
// correct distances bewteen them.
// correct distances between them.
add_match = true;
}
else
@@ -368,16 +367,18 @@ static int _yr_scan_verify_chained_string_match(
// list of unconfirmed matches. Unconfirmed matches are sorted in ascending
// offset order. If no unconfirmed match exists, the lowest possible offset
// is the offset of the current match.
if (matching_string->unconfirmed_matches[tidx].head != NULL)
lowest_offset = matching_string->unconfirmed_matches[tidx].head->offset;
match = context->unconfirmed_matches[matching_string->idx].head;
if (match != NULL)
lowest_offset = match->offset;
else
lowest_offset = match_offset;
// Iterate over the list of unconfirmed matches for the string that
// preceeds the currently matching string. If we have a string chain like:
// precedes the currently matching string. If we have a string chain like:
// S1 <- S2 <- S3, and we just found a match for S2, we are iterating the
// list of unconfirmed matches of S1.
match = matching_string->chained_to->unconfirmed_matches[tidx].head;
match = context->unconfirmed_matches[matching_string->chained_to->idx].head;
while (match != NULL)
{
@@ -397,7 +398,8 @@ static int _yr_scan_verify_chained_string_match(
// negatively confirmed (i.e: we can be sure that this unconfirmed
// match can't be an actual match)
_yr_scan_remove_match_from_list(
match, &matching_string->chained_to->unconfirmed_matches[tidx]);
match,
&context->unconfirmed_matches[matching_string->chained_to->idx]);
}
else if (ending_offset + matching_string->chain_gap_max >= match_offset &&
ending_offset + matching_string->chain_gap_min <= match_offset)
@@ -427,12 +429,12 @@ static int _yr_scan_verify_chained_string_match(
// chained to some other string.
assert(matching_string->chained_to != NULL);
// Iterate over the list of unconfirmed matches of the preceeding string
// Iterate over the list of unconfirmed matches of the preceding string
// in the chain and update the chain_length field for each of them. This
// is a recursive operation that will update the chain_length field for
// every unconfirmed match in all the strings in the chain up to the head
// of the chain.
match = matching_string->chained_to->unconfirmed_matches[tidx].head;
match = context->unconfirmed_matches[matching_string->chained_to->idx].head;
while (match != NULL)
{
@@ -442,7 +444,7 @@ static int _yr_scan_verify_chained_string_match(
ending_offset + matching_string->chain_gap_min <= match_offset)
{
_yr_scan_update_match_chain_length(
tidx, matching_string->chained_to, match, 1);
context, matching_string->chained_to, match, 1);
}
match = match->next;
@@ -458,7 +460,7 @@ static int _yr_scan_verify_chained_string_match(
}
// "string" points now to the head of the strings chain.
match = string->unconfirmed_matches[tidx].head;
match = context->unconfirmed_matches[string->idx].head;
// Iterate over the list of unconfirmed matches of the head of the chain,
// and move to the list of confirmed matches those with a chain_length
@@ -471,24 +473,28 @@ static int _yr_scan_verify_chained_string_match(
if (match->chain_length == full_chain_length)
{
_yr_scan_remove_match_from_list(
match, &string->unconfirmed_matches[tidx]);
match,
&context->unconfirmed_matches[string->idx]);
match->match_length = (int32_t) \
(match_offset - match->offset + match_length);
match->data_length = yr_min(match->match_length, max_match_data);
FAIL_ON_ERROR(yr_arena_write_data(
context->matches_arena,
match->data = yr_notebook_alloc(
context->matches_notebook, match->data_length);
if (match->data == NULL)
return ERROR_INSUFFICIENT_MEMORY;
memcpy(
(void*) match->data,
match_data - match_offset + match->offset,
match->data_length,
(void**) &match->data));
match->data_length);
FAIL_ON_ERROR(_yr_scan_add_match_to_list(
match,
!STRING_IS_PRIVATE(string) ?
&string->matches[tidx] :
&string->private_matches[tidx],
&context->matches[string->idx],
false));
}
@@ -497,23 +503,10 @@ static int _yr_scan_verify_chained_string_match(
}
else // It's a part of a chain, but not the tail.
{
// If this is the first match for the string, put the string in the
// list of strings whose flags needs to be cleared after the scan.
if (matching_string->matches[tidx].count == 0 &&
matching_string->private_matches[tidx].count == 0 &&
matching_string->unconfirmed_matches[tidx].count == 0)
{
FAIL_ON_ERROR(yr_arena_write_data(
context->matching_strings_arena,
&matching_string,
sizeof(matching_string),
NULL));
}
new_match = yr_notebook_alloc(context->matches_notebook, sizeof(YR_MATCH));
FAIL_ON_ERROR(yr_arena_allocate_memory(
context->matches_arena,
sizeof(YR_MATCH),
(void**) &new_match));
if (new_match == NULL)
return ERROR_INSUFFICIENT_MEMORY;
new_match->base = match_base;
new_match->offset = match_offset;
@@ -521,6 +514,7 @@ static int _yr_scan_verify_chained_string_match(
new_match->chain_length = 0;
new_match->prev = NULL;
new_match->next = NULL;
new_match->private = STRING_IS_PRIVATE(matching_string);
// A copy of the matching data is written to the matches_arena, the
// amount of data copies is limited by YR_CONFIG_MAX_MATCH_DATA.
@@ -528,11 +522,16 @@ static int _yr_scan_verify_chained_string_match(
if (new_match->data_length > 0)
{
FAIL_ON_ERROR(yr_arena_write_data(
context->matches_arena,
new_match->data = yr_notebook_alloc(
context->matches_notebook, new_match->data_length);
if (new_match->data == NULL)
return ERROR_INSUFFICIENT_MEMORY;
memcpy(
(void*) new_match->data,
match_data,
new_match->data_length,
(void**) &new_match->data));
new_match->data_length);
}
else
{
@@ -544,7 +543,7 @@ static int _yr_scan_verify_chained_string_match(
// an actual match until finding the remaining parts of the chain.
FAIL_ON_ERROR(_yr_scan_add_match_to_list(
new_match,
&matching_string->unconfirmed_matches[tidx],
&context->unconfirmed_matches[matching_string->idx],
false));
}
}
@@ -565,7 +564,6 @@ static int _yr_scan_match_callback(
YR_MATCH* new_match;
int result = ERROR_SUCCESS;
int tidx = callback_args->context->tidx;
size_t match_offset = match_data - callback_args->data;
@@ -617,34 +615,28 @@ static int _yr_scan_match_callback(
FAIL_ON_ERROR(yr_get_configuration(
YR_CONFIG_MAX_MATCH_DATA,
&max_match_data))
&max_match_data));
if (string->matches[tidx].count == 0)
{
// If this is the first match for the string, put the string in the
// list of strings whose flags needs to be cleared after the scan.
new_match = yr_notebook_alloc(
callback_args->context->matches_notebook, sizeof(YR_MATCH));
FAIL_ON_ERROR(yr_arena_write_data(
callback_args->context->matching_strings_arena,
&string,
sizeof(string),
NULL));
}
FAIL_ON_ERROR(yr_arena_allocate_memory(
callback_args->context->matches_arena,
sizeof(YR_MATCH),
(void**) &new_match));
if (new_match == NULL)
return ERROR_INSUFFICIENT_MEMORY;
new_match->data_length = yr_min(match_length, max_match_data);
if (new_match->data_length > 0)
{
FAIL_ON_ERROR(yr_arena_write_data(
callback_args->context->matches_arena,
new_match->data = yr_notebook_alloc(
callback_args->context->matches_notebook, new_match->data_length);
if (new_match->data == NULL)
return ERROR_INSUFFICIENT_MEMORY;
memcpy(
(void*) new_match->data,
match_data,
new_match->data_length,
(void**) &new_match->data));
new_match->data_length);
}
else
{
@@ -658,10 +650,11 @@ static int _yr_scan_match_callback(
new_match->match_length = match_length;
new_match->prev = NULL;
new_match->next = NULL;
new_match->private = STRING_IS_PRIVATE(string);
FAIL_ON_ERROR(_yr_scan_add_match_to_list(
new_match,
!STRING_IS_PRIVATE(string) ? &string->matches[tidx] : &string->private_matches[tidx],
&callback_args->context->matches[string->idx],
STRING_IS_GREEDY_REGEXP(string)));
}
}
@@ -711,7 +704,9 @@ static int _yr_scan_verify_re_match(
else
exec = yr_re_exec;
if (STRING_IS_ASCII(ac_match->string))
if (STRING_IS_ASCII(ac_match->string) ||
STRING_IS_BASE64(ac_match->string) ||
STRING_IS_BASE64_WIDE(ac_match->string))
{
FAIL_ON_ERROR(exec(
context,
@@ -725,7 +720,9 @@ static int _yr_scan_verify_re_match(
&forward_matches));
}
if (STRING_IS_WIDE(ac_match->string) && forward_matches == -1)
if ((forward_matches == -1) &&
(STRING_IS_WIDE(ac_match->string) &&
!(STRING_IS_BASE64_WIDE(ac_match->string) || STRING_IS_BASE64_WIDE(ac_match->string))))
{
flags |= RE_FLAGS_WIDE;
FAIL_ON_ERROR(exec(
@@ -902,7 +899,7 @@ int yr_scan_verify_match(
if (context->flags & SCAN_FLAGS_FAST_MODE &&
STRING_IS_SINGLE_MATCH(string) &&
string->matches[context->tidx].head != NULL)
context->matches[string->idx].head != NULL)
return ERROR_SUCCESS;
if (STRING_IS_FIXED_OFFSET(string) &&
@@ -910,7 +907,7 @@ int yr_scan_verify_match(
return ERROR_SUCCESS;
#ifdef PROFILING_ENABLED
uint64_t start_time = yr_stopwatch_elapsed_us(&context->stopwatch);
uint64_t start_time = yr_stopwatch_elapsed_ns(&context->stopwatch);
#endif
if (STRING_IS_LITERAL(string))
@@ -924,15 +921,13 @@ int yr_scan_verify_match(
context, ac_match, data, data_size, data_base, offset);
}
#ifdef PROFILING_ENABLED
uint64_t finish_time = yr_stopwatch_elapsed_ns(&context->stopwatch);
context->time_cost[string->rule_idx] += (finish_time - start_time);
#endif
if (result != ERROR_SUCCESS)
context->last_error_string = string;
#ifdef PROFILING_ENABLED
uint64_t finish_time = yr_stopwatch_elapsed_us(&context->stopwatch);
string->rule->time_cost_per_thread[context->tidx] += (
finish_time - start_time);
#endif
return result;
}
libyara/scanner.c
+201 -138
View File
@@ -49,8 +49,8 @@ static int _yr_scanner_scan_mem_block(
YR_MEMORY_BLOCK* block)
{
YR_RULES* rules = scanner->rules;
YR_AC_TRANSITION_TABLE transition_table = rules->ac_transition_table;
YR_AC_MATCH_TABLE match_table = rules->ac_match_table;
YR_AC_TRANSITION* transition_table = rules->ac_transition_table;
uint32_t* match_table = rules->ac_match_table;
YR_AC_MATCH* match;
YR_AC_TRANSITION transition;
@@ -61,30 +61,39 @@ static int _yr_scanner_scan_mem_block(
while (i < block->size)
{
match = match_table[state].match;
if (i % 4096 == 0 && scanner->timeout > 0)
{
if (yr_stopwatch_elapsed_us(&scanner->stopwatch) > scanner->timeout)
if (yr_stopwatch_elapsed_ns(&scanner->stopwatch) > scanner->timeout)
return ERROR_SCAN_TIMEOUT;
}
while (match != NULL)
if (match_table[state] != 0)
{
if (match->backtrack <= i)
{
FAIL_ON_ERROR(yr_scan_verify_match(
scanner,
match,
block_data,
block->size,
block->base,
i - match->backtrack));
}
// If the entry corresponding to state N in the match table is zero, it
// means that there's no match associated to the state. If it's non-zero,
// its value is the 1-based index within ac_match_pool where the first
// match resides.
match = match->next;
match = &rules->ac_match_pool[match_table[state] - 1];
while (match != NULL)
{
if (match->backtrack <= i)
{
FAIL_ON_ERROR(yr_scan_verify_match(
scanner,
match,
block_data,
block->size,
block->base,
i - match->backtrack));
}
match = match->next;
}
}
index = block_data[i++] + 1;
transition = transition_table[state + index];
@@ -105,22 +114,25 @@ static int _yr_scanner_scan_mem_block(
state = YR_AC_NEXT_STATE(transition);
}
match = match_table[state].match;
while (match != NULL)
if (match_table[state] != 0)
{
if (match->backtrack <= i)
{
FAIL_ON_ERROR(yr_scan_verify_match(
scanner,
match,
block_data,
block->size,
block->base,
i - match->backtrack));
}
match = &rules->ac_match_pool[match_table[state] - 1];
match = match->next;
while (match != NULL)
{
if (match->backtrack <= i)
{
FAIL_ON_ERROR(yr_scan_verify_match(
scanner,
match,
block_data,
block->size,
block->base,
i - match->backtrack));
}
match = match->next;
}
}
return ERROR_SUCCESS;
@@ -130,40 +142,21 @@ static int _yr_scanner_scan_mem_block(
static void _yr_scanner_clean_matches(
YR_SCANNER* scanner)
{
YR_RULE* rule;
YR_STRING** string;
memset(
scanner->rule_matches_flags, 0,
sizeof(YR_BITMASK) * YR_BITMASK_SIZE(scanner->rules->num_rules));
int tidx = scanner->tidx;
memset(
scanner->ns_unsatisfied_flags, 0,
sizeof(YR_BITMASK) * YR_BITMASK_SIZE(scanner->rules->num_namespaces));
yr_rules_foreach(scanner->rules, rule)
{
rule->t_flags[tidx] &= ~RULE_TFLAGS_MATCH;
rule->ns->t_flags[tidx] &= ~NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL;
}
memset(
scanner->matches, 0,
sizeof(YR_MATCHES) * scanner->rules->num_strings);
if (scanner->matching_strings_arena != NULL)
{
string = (YR_STRING**) yr_arena_base_address(
scanner->matching_strings_arena);
while (string != NULL)
{
(*string)->matches[tidx].count = 0;
(*string)->matches[tidx].head = NULL;
(*string)->matches[tidx].tail = NULL;
(*string)->private_matches[tidx].count = 0;
(*string)->private_matches[tidx].head = NULL;
(*string)->private_matches[tidx].tail = NULL;
(*string)->unconfirmed_matches[tidx].count = 0;
(*string)->unconfirmed_matches[tidx].head = NULL;
(*string)->unconfirmed_matches[tidx].tail = NULL;
string = (YR_STRING**) yr_arena_next_address(
scanner->matching_strings_arena,
string,
sizeof(YR_STRING*));
}
}
memset(
scanner->unconfirmed_matches, 0,
sizeof(YR_MATCHES) * scanner->rules->num_strings);
}
@@ -184,10 +177,31 @@ YR_API int yr_scanner_create(
yr_scanner_destroy(new_scanner));
new_scanner->rules = rules;
new_scanner->tidx = -1;
new_scanner->entry_point = UNDEFINED;
new_scanner->entry_point = YR_UNDEFINED;
new_scanner->canary = rand();
// By default report both matching and non-matching rules.
new_scanner->flags = \
SCAN_FLAGS_REPORT_RULES_MATCHING |
SCAN_FLAGS_REPORT_RULES_NOT_MATCHING;
new_scanner->rule_matches_flags = (YR_BITMASK*) yr_calloc(
sizeof(YR_BITMASK), YR_BITMASK_SIZE(rules->num_rules));
new_scanner->ns_unsatisfied_flags = (YR_BITMASK*) yr_calloc(
sizeof(YR_BITMASK), YR_BITMASK_SIZE(rules->num_namespaces));
new_scanner->matches = (YR_MATCHES*) yr_calloc(
rules->num_strings, sizeof(YR_MATCHES));
new_scanner->unconfirmed_matches = (YR_MATCHES*) yr_calloc(
rules->num_strings, sizeof(YR_MATCHES));
#ifdef PROFILING_ENABLED
new_scanner->time_cost = (uint64_t*) yr_calloc(
rules->num_rules, sizeof(uint64_t));
#endif
external = rules->externals_list_head;
while (!EXTERNAL_VARIABLE_IS_NULL(external))
@@ -196,6 +210,7 @@ YR_API int yr_scanner_create(
FAIL_ON_ERROR_WITH_CLEANUP(
yr_object_from_external_variable(external, &object),
// cleanup
yr_scanner_destroy(new_scanner));
FAIL_ON_ERROR_WITH_CLEANUP(
@@ -204,6 +219,8 @@ YR_API int yr_scanner_create(
external->identifier,
NULL,
(void*) object),
// cleanup
yr_object_destroy(object);
yr_scanner_destroy(new_scanner));
yr_object_set_canary(object, new_scanner->canary);
@@ -238,6 +255,14 @@ YR_API void yr_scanner_destroy(
(YR_HASH_TABLE_FREE_VALUE_FUNC) yr_object_destroy);
}
#ifdef PROFILING_ENABLED
yr_free(scanner->time_cost);
#endif
yr_free(scanner->rule_matches_flags);
yr_free(scanner->ns_unsatisfied_flags);
yr_free(scanner->matches);
yr_free(scanner->unconfirmed_matches);
yr_free(scanner);
}
@@ -264,6 +289,16 @@ YR_API void yr_scanner_set_flags(
YR_SCANNER* scanner,
int flags)
{
// For backward compatibility, if neither SCAN_FLAGS_REPORT_RULES_MATCHING
// nor SCAN_FLAGS_REPORT_RULES_NOT_MATCHING are specified, both are assumed.
if (!(flags & SCAN_FLAGS_REPORT_RULES_MATCHING) &&
!(flags & SCAN_FLAGS_REPORT_RULES_NOT_MATCHING))
{
flags |= SCAN_FLAGS_REPORT_RULES_MATCHING |
SCAN_FLAGS_REPORT_RULES_NOT_MATCHING;
}
scanner->flags = flags;
}
@@ -345,14 +380,17 @@ YR_API int yr_scanner_scan_mem_blocks(
YR_RULE* rule;
YR_MEMORY_BLOCK* block;
int tidx = 0;
int result = ERROR_SUCCESS;
uint32_t max_match_data;
uint64_t elapsed_time;
int i, result = ERROR_SUCCESS;
if (scanner->callback == NULL)
return ERROR_CALLBACK_REQUIRED;
FAIL_ON_ERROR(yr_get_configuration(
YR_CONFIG_MAX_MATCH_DATA,
&max_match_data))
scanner->iterator = iterator;
rules = scanner->rules;
block = iterator->first(iterator);
@@ -360,39 +398,21 @@ YR_API int yr_scanner_scan_mem_blocks(
if (block == NULL)
return ERROR_SUCCESS;
yr_mutex_lock(&rules->mutex);
while (tidx < YR_MAX_THREADS && YR_BITARRAY_TEST(rules->tidx_mask, tidx))
{
tidx++;
}
if (tidx < YR_MAX_THREADS)
YR_BITARRAY_SET(rules->tidx_mask, tidx);
else
result = ERROR_TOO_MANY_SCAN_THREADS;
yr_mutex_unlock(&rules->mutex);
if (result != ERROR_SUCCESS)
return result;
scanner->tidx = tidx;
scanner->file_size = block->size;
yr_set_tidx(tidx);
// Create the notebook that will hold the YR_MATCH structures representing
// each match found. This notebook will also contain snippets of the matching
// data (the "data" field in YR_MATCH points to the snippet corresponding to
// the match). Each notebook's page can store up to 1024 matches.
result = yr_notebook_create(
1024 * (sizeof(YR_MATCH) + max_match_data),
&scanner->matches_notebook);
if (result != ERROR_SUCCESS)
goto _exit;
yr_stopwatch_start(&scanner->stopwatch);
result = yr_arena_create(1048576, 0, &scanner->matches_arena);
if (result != ERROR_SUCCESS)
goto _exit;
result = yr_arena_create(4096, 0, &scanner->matching_strings_arena);
if (result != ERROR_SUCCESS)
goto _exit;
while (block != NULL)
{
const uint8_t* data = block->fetch_data(block);
@@ -404,7 +424,7 @@ YR_API int yr_scanner_scan_mem_blocks(
continue;
}
if (scanner->entry_point == UNDEFINED)
if (scanner->entry_point == YR_UNDEFINED)
{
YR_TRYCATCH(
!(scanner->flags & SCAN_FLAGS_NO_TRYCATCH),
@@ -449,23 +469,27 @@ YR_API int yr_scanner_scan_mem_blocks(
if (result != ERROR_SUCCESS)
goto _exit;
yr_rules_foreach(rules, rule)
for (i = 0, rule = rules->rules_list_head;
!RULE_IS_NULL(rule);
i++, rule++)
{
int message;
int message = 0;
if (rule->t_flags[tidx] & RULE_TFLAGS_MATCH &&
!(rule->ns->t_flags[tidx] & NAMESPACE_TFLAGS_UNSATISFIED_GLOBAL))
if (yr_bitmask_is_set(scanner->rule_matches_flags, i) &&
yr_bitmask_is_not_set(scanner->ns_unsatisfied_flags, rule->ns->idx))
{
message = CALLBACK_MSG_RULE_MATCHING;
if (scanner->flags & SCAN_FLAGS_REPORT_RULES_MATCHING)
message = CALLBACK_MSG_RULE_MATCHING;
}
else
{
message = CALLBACK_MSG_RULE_NOT_MATCHING;
if (scanner->flags & SCAN_FLAGS_REPORT_RULES_NOT_MATCHING)
message = CALLBACK_MSG_RULE_NOT_MATCHING;
}
if (!RULE_IS_PRIVATE(rule))
if (message != 0 && !RULE_IS_PRIVATE(rule))
{
switch (scanner->callback(message, rule, scanner->user_data))
switch (scanner->callback(scanner, message, rule, scanner->user_data))
{
case CALLBACK_ABORT:
result = ERROR_SUCCESS;
@@ -478,46 +502,22 @@ YR_API int yr_scanner_scan_mem_blocks(
}
}
scanner->callback(CALLBACK_MSG_SCAN_FINISHED, NULL, scanner->user_data);
scanner->callback(
scanner,
CALLBACK_MSG_SCAN_FINISHED,
NULL,
scanner->user_data);
_exit:
elapsed_time = yr_stopwatch_elapsed_us(&scanner->stopwatch);
#ifdef PROFILING_ENABLED
yr_rules_foreach(rules, rule)
{
#ifdef _WIN32
InterlockedAdd64(&rule->time_cost, rule->time_cost_per_thread[tidx]);
#else
__sync_fetch_and_add(&rule->time_cost, rule->time_cost_per_thread[tidx]);
#endif
rule->time_cost_per_thread[tidx] = 0;
}
#endif
_yr_scanner_clean_matches(scanner);
if (scanner->matches_arena != NULL)
if (scanner->matches_notebook != NULL)
{
yr_arena_destroy(scanner->matches_arena);
scanner->matches_arena = NULL;
yr_notebook_destroy(scanner->matches_notebook);
scanner->matches_notebook = NULL;
}
if (scanner->matching_strings_arena != NULL)
{
yr_arena_destroy(scanner->matching_strings_arena);
scanner->matching_strings_arena = NULL;
}
yr_mutex_lock(&rules->mutex);
YR_BITARRAY_UNSET(rules->tidx_mask, tidx);
rules->time_cost += elapsed_time;
yr_mutex_unlock(&rules->mutex);
yr_set_tidx(-1);
return result;
}
@@ -634,5 +634,68 @@ YR_API YR_RULE* yr_scanner_last_error_rule(
if (scanner->last_error_string == NULL)
return NULL;
return scanner->last_error_string->rule;
return &scanner->rules->rules_list_head[scanner->last_error_string->rule_idx];
}
#ifdef PROFILING_ENABLED
struct RULE_COST
{
uint32_t idx;
uint64_t cost;
};
static int sort_by_cost_desc(
const struct RULE_COST* r1,
const struct RULE_COST* r2)
{
if (r1->cost < r2->cost)
return 1;
if (r1->cost > r2->cost)
return -1;
return 0;
}
YR_API int yr_scanner_print_profiling_info(
YR_SCANNER* scanner)
{
printf("\n===== PROFILING INFORMATION =====\n\n");
struct RULE_COST* rule_costs = yr_malloc(
scanner->rules->num_rules * sizeof(struct RULE_COST));
if (rule_costs == NULL)
return ERROR_INSUFFICIENT_MEMORY;
for (uint32_t i = 0; i < scanner->rules->num_rules; i++)
{
rule_costs[i].idx = i;
rule_costs[i].cost = scanner->time_cost[i];
}
qsort(
rule_costs,
scanner->rules->num_rules,
sizeof(struct RULE_COST),
(int (*)(const void *, const void *)) sort_by_cost_desc);
for (uint32_t i = 0; i < scanner->rules->num_rules; i++)
{
YR_RULE* rule = &scanner->rules->rules_list_head[rule_costs[i].idx];
printf(
"%s:%s: %" PRIu64 "\n",
rule->ns->name,
rule->identifier,
rule_costs[i].cost);
}
printf("\n=================================\n");
yr_free(rule_costs);
return ERROR_SUCCESS;
}
#endif
libyara/sizedstr.c
+65 -4
View File
@@ -27,9 +27,39 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <ctype.h>
#include <string.h>
#include <yara/globals.h>
#include <yara/mem.h>
#include <yara/sizedstr.h>
#include <yara/types.h>
int sized_string_cmp_nocase(
SIZED_STRING* s1,
SIZED_STRING* s2)
{
size_t i = 0;
while (s1->length > i &&
s2->length > i &&
yr_lowercase[(uint8_t) s1->c_string[i]] ==
yr_lowercase[(uint8_t) s2->c_string[i]])
{
i++;
}
if (i == s1->length && i == s2->length)
return 0;
else if (i == s1->length)
return -1;
else if (i == s2->length)
return 1;
else if (s1->c_string[i] < s2->c_string[i])
return -1;
else
return 1;
}
int sized_string_cmp(
@@ -85,12 +115,43 @@ SIZED_STRING* sized_string_new(
result = (SIZED_STRING*) yr_malloc(sizeof(SIZED_STRING) + length);
if (result == NULL)
return NULL;
result->length = length;
result->flags = 0;
strncpy(result->c_string, s, length);
result->c_string[length] = '\0';
// Copy the string and the null terminator.
strcpy(result->c_string, s);
return result;
}
//
// Convert a SIZED_STRING to a wide version. It is up to the caller to free
// the returned string.
//
SIZED_STRING* sized_string_convert_to_wide(
SIZED_STRING* s)
{
size_t i;
size_t j = 0;
SIZED_STRING* wide = (SIZED_STRING*) yr_malloc(
sizeof(SIZED_STRING) + s->length * 2);
if (wide == NULL)
return NULL;
for (i = 0; i <= s->length; i++)
{
wide->c_string[j++] = s->c_string[i];
wide->c_string[j++] = '\x00';
}
wide->length = s->length * 2;
wide->flags = s->flags | STRING_FLAGS_WIDE;
return wide;
}
libyara/stopwatch.c
+9 -12
View File
@@ -39,14 +39,14 @@ void yr_stopwatch_start(
}
uint64_t yr_stopwatch_elapsed_us(
uint64_t yr_stopwatch_elapsed_ns(
YR_STOPWATCH* sw)
{
LARGE_INTEGER li;
QueryPerformanceCounter(&li);
return (li.QuadPart - sw->start.QuadPart) * 1000000L / sw->frequency.QuadPart;
return (li.QuadPart - sw->start.QuadPart) * 1000000000L / sw->frequency.QuadPart;
}
@@ -60,14 +60,11 @@ void yr_stopwatch_start(
}
uint64_t yr_stopwatch_elapsed_us(
uint64_t yr_stopwatch_elapsed_ns(
YR_STOPWATCH* sw)
{
uint64_t now;
now = mach_absolute_time();
return (now - sw->start) * sw->timebase.numer /
(sw->timebase.denom * 1000ULL);
uint64_t now = mach_absolute_time();
return ((now - sw->start) * sw->timebase.numer) / sw->timebase.denom;
}
@@ -91,7 +88,7 @@ void yr_stopwatch_start(
}
uint64_t yr_stopwatch_elapsed_us(
uint64_t yr_stopwatch_elapsed_ns(
YR_STOPWATCH* stopwatch)
{
struct timespec ts_stop;
@@ -99,7 +96,7 @@ uint64_t yr_stopwatch_elapsed_us(
clock_gettime(CLOCK_MONOTONIC, &ts_stop);
timespecsub(&ts_stop, &stopwatch->ts_start, &ts_elapsed);
return ts_elapsed.tv_sec * 1000000L + ts_elapsed.tv_nsec / 1000;
return ts_elapsed.tv_sec * 1000000000L + ts_elapsed.tv_nsec;
}
@@ -125,7 +122,7 @@ void yr_stopwatch_start(
}
uint64_t yr_stopwatch_elapsed_us(
uint64_t yr_stopwatch_elapsed_ns(
YR_STOPWATCH* stopwatch)
{
struct timeval tv_stop;
@@ -133,7 +130,7 @@ uint64_t yr_stopwatch_elapsed_us(
gettimeofday(&tv_stop, NULL);
timevalsub(&tv_stop, &stopwatch->tv_start, &tv_elapsed);
return tv_elapsed.tv_sec * 1000000L + tv_elapsed.tv_usec;
return tv_elapsed.tv_sec * 1000000000L + tv_elapsed.tv_usec * 1000L;
}
#endif
sample.rules
+5 -1
View File
@@ -7,6 +7,10 @@ rule UPX : Packer
condition:
$a at pe.entry_point
}
fksjd
rule test {condition: false}
sandbox/BUILD.bazel
+4
View File
@@ -25,6 +25,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
load("@rules_cc//cc:defs.bzl", "cc_proto_library")
load("@rules_proto//proto:defs.bzl", "proto_library")
load(
"@com_google_sandboxed_api//sandboxed_api/bazel:sapi.bzl",
"sapi_library",
test-arena-stream
BIN
View File
Binary file not shown.
test-rules.yarc
BIN
View File
Binary file not shown.
tests/BUILD.bazel
+26
View File
@@ -49,6 +49,18 @@ cc_test(
srcs = ["test-alignment.c"],
copts = COPTS,
deps = ["@//:libyara"],
linkstatic=True,
)
cc_test(
name = "test_arena",
srcs = ["test-arena.c"],
copts = COPTS,
deps = [
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -59,6 +71,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -66,6 +79,7 @@ cc_test(
srcs = ["test-rules.c"],
copts = COPTS,
data = [
"data/base64",
"data/baz.yar",
"data/foo.yar",
"data/include/bar.yar",
@@ -78,6 +92,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -91,11 +106,13 @@ cc_test(
"data/tiny-idata-5200",
"data/tiny-overlay",
"data/weird_rich",
"data/mtxex.dll",
],
deps = [
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -107,6 +124,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -118,6 +136,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -128,6 +147,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -138,6 +158,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -148,6 +169,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
# This test fails when running under ASAN
@@ -159,6 +181,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -171,6 +194,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -182,6 +206,7 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
cc_test(
@@ -192,4 +217,5 @@ cc_test(
":util",
"@//:libyara",
],
linkstatic=True,
)
tests/data/base64
BIN
View File
Binary file not shown.
tests/data/mtxex.dll
Executable
BIN
View File
Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More