build: cbindgen

Rust headers are now generated using cbindgen. If cbindgen is present, they can
be generated during dist, otherwise they will be available for builds.

.github/workflows/builds.yml

@@ -70,6 +70,8 @@ jobs:
                 texlive-upquote \
                 texlive-capt-of \
                 texlive-needspace \
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - uses: actions/checkout@v1
       - name: Bundling libhtp
         run: git clone https://github.com/OISF/libhtp -b 0.5.x
@@ -139,6 +141,8 @@ jobs:
                 sudo \
                 which \
                 zlib-devel
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - name: Download suricata.tar.gz
         uses: actions/download-artifact@v1
         with:
@@ -177,6 +181,8 @@ jobs:
                 sudo \
                 which \
                 zlib-devel
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - name: Download suricata.tar.gz
         uses: actions/download-artifact@v1
         with:
@@ -241,6 +247,8 @@ jobs:
                 sudo \
                 which \
                 zlib-devel
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - uses: actions/checkout@v1
       - run: git clone https://github.com/OISF/libhtp -b 0.5.x
       - run: ./autogen.sh
@@ -306,6 +314,8 @@ jobs:
         run: |
           add-apt-repository -y ppa:npalix/coccinelle
           apt -y install coccinelle
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - uses: actions/checkout@v1
       - run: git clone https://github.com/OISF/libhtp -b 0.5.x
       - run: ./autogen.sh
@@ -357,6 +367,8 @@ jobs:
       - name: Install Rust
         run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.33.0 -y
       - run: echo "::add-path::/github/home/.cargo/bin"
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - name: Download suricata.tar.gz
         uses: actions/download-artifact@v1
         with:
@@ -429,6 +441,8 @@ jobs:
                 sudo \
                 zlib1g \
                 zlib1g-dev
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - uses: actions/checkout@v1
       - name: Bundling libhtp
         run: git clone https://github.com/OISF/libhtp -b 0.5.x
@@ -490,6 +504,8 @@ jobs:
       - name: Install Rust
         run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.33.0 -y
       - run: echo "::add-path::/github/home/.cargo/bin"
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - uses: actions/checkout@v1
       - name: Bundling libhtp
         run: git clone https://github.com/OISF/libhtp -b 0.5.x
@@ -537,6 +553,8 @@ jobs:
           pkg-config \
           rust \
           xz
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
       - run: pip install PyYAML
       - uses: actions/checkout@v1
       - run: git clone https://github.com/OISF/libhtp -b 0.5.x

.travis.yml

@@ -198,4 +198,6 @@ before_install:
     # match the requested version.
     rustup default "${RUST_VERSION}"
     rustc --version
+
+    cargo install --force cbindgen
   - ./qa/travis-libhtp.sh

appveyor.yml

@@ -58,6 +58,8 @@ install:
 
 build_script:
   - set Path=%MINGW_DIR%\bin;c:\msys64\usr\bin;%PCAP_PATH%;%Path%
+  - cargo install --force cbindgen
+  - set Path=C:\Users\appveyor\.cargo\bin;%Path%
   - git clone https://github.com/OISF/libhtp -b 0.5.x
   - bash autogen.sh
   - bash configure %CONFIGURE% --disable-shared --%RUST%-rust

configure.ac

@@ -2387,17 +2387,6 @@ fi
     rust_vendor_comment="# "
     have_rust_vendor="no"
 
-    # We may require Python if the Rust header stubs are not already
-    # generated.
-    if test "x$enable_python" != "xyes" && test ! -f rust/gen/c-headers/rust-core-gen.h; then
-        echo ""
-        echo "   ERROR! Rust support requires Python."
-        echo
-        echo "   Ubuntu: apt install python"
-        echo
-        exit 1
-    fi
-
     if test "x$cross_compiling" = "xyes"; then
       RUST_SURICATA_LIB_XC_DIR="${host_alias}/"
     else
@@ -2411,7 +2400,7 @@ fi
     fi
 
     RUST_LDADD="${RUST_SURICATA_LIB} ${RUST_LDADD}"
-    CFLAGS="${CFLAGS} -I\${srcdir}/../rust/gen/c-headers"
+    CFLAGS="${CFLAGS} -I\${srcdir}/../rust/gen"
     AC_SUBST(RUST_SURICATA_LIB)
     AC_SUBST(RUST_LDADD)
     if test "x$CARGO_HOME" = "x"; then
@@ -2470,6 +2459,27 @@ fi
         have_cargo_vendor=$have_cargo_vendor_bin
     fi
 
+    AC_CHECK_FILES([$srcdir/rust/gen], [have_rust_headers="yes"])
+    AC_DEFINE([HAVE_RUST_HEADERS],[1],[Enable Rust language])
+    AM_CONDITIONAL([HAVE_RUST_HEADERS], [test "x$have_rust_headers" = "xyes"])
+    if test "x$have_rust_headers" != "xyes"; then
+      AC_PATH_PROG(HAVE_CARGO_CBINDGEN, cbindgen, "no") 
+      if test "x$HAVE_CARGO_CBINDGEN" = "xno"; then
+        echo "  Warning: cbindgen not found, it is required to generate header files"
+        echo "  To install: cargo install --force cbindgen"
+        AC_MSG_ERROR([cbindgen required])
+      fi
+      required_version=0.9.0
+      cbindgen_version=$(cbindgen --version | cut -d' ' -f2-)
+      lowest_version=$(printf '%s\n' $required_version $cbindgen_version | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n | head -n1)
+      if test "x$lowest_version" != "x$required_version"; then
+        HAVE_CARGO_BINDGEN="no"
+        echo "  Warning: cbindgen must be at least version $required_version, version was $cbindgen_version, lowest version was $lowest_version"
+        echo "  To install: cargo install --force cbindgen"
+        AC_MSG_ERROR([incorrect cbindgen version])
+      fi
+    fi
+
     AM_CONDITIONAL([HAVE_CARGO_VENDOR], [test "x$have_cargo_vendor" != "xno"])
 
     AC_ARG_ENABLE(rust_strict,

rust/Makefile.am

@@ -1,6 +1,7 @@
 EXTRA_DIST =	src \
 		.cargo/config.in \
-		gen-c-headers.py
+                gen \
+                cbindgen.toml
 
 if HAVE_CARGO_VENDOR
 EXTRA_DIST +=	vendor
@@ -22,10 +23,9 @@ if RUST_CROSS_COMPILE
 RUST_TARGET = --target $(host_triplet)
 endif
 
-all-local:
-if HAVE_PYTHON
-	cd $(top_srcdir)/rust && $(HAVE_PYTHON) ./gen-c-headers.py
-endif
+build-rust:
+	cd $(top_srcdir)/rust
+
 if HAVE_CYGPATH
 	@rustup_home@ \
 		CARGO_HOME="$(CARGO_HOME)" \
@@ -40,12 +40,30 @@ else
 			--features "$(RUST_FEATURES)" $(RUST_TARGET)
 endif
 
+if HAVE_RUST_HEADERS
+
+all-local: build-rust
+	echo Rust headers available
+
+clean-local:
+	-rm -rf target
+
+distclean-local: clean-local
+	rm -rf vendor Cargo.lock
+
+else 
+
+all-local: gen-headers build-rust
+	$(info Generating rust headers)
+
 clean-local:
 	-rm -rf target gen
 
 distclean-local: clean-local
 	rm -rf vendor gen Cargo.lock
 
+endif
+
 check:
 	CARGO_HOME="$(CARGO_HOME)" @rustup_home@ \
 		CARGO_TARGET_DIR="$(abs_top_builddir)/rust/target" \
@@ -58,11 +76,6 @@ else
 vendor:
 endif
 
-# Can only include the headers if we have Python to generate them.
-if HAVE_PYTHON
-EXTRA_DIST +=	gen/c-headers
-gen/c-headers:
-	cd $(top_srcdir)/rust && $(HAVE_PYTHON) ./gen-c-headers.py
-else
-gen/c-headers:
-endif
+gen-headers: FORCE
+	cbindgen --config cbindgen.toml --output "gen/rust-bindings.h" > /dev/null
+FORCE:

rust/cbindgen.toml

@@ -0,0 +1,151 @@
+# The language to output bindings in
+#
+# possible values: "C", "C++"
+#
+# default: "C++"
+language = "C"
+
+
+# Options for wrapping the contents of the header:
+
+# An optional string of text to output at the beginning of the generated file
+# default: doesn't emit anything
+header = """/* Copyright (C) 2019 Open Information Security Foundation
+*
+* You can copy, redistribute or modify this Program under the terms of
+* the GNU General Public License version 2 as published by the Free
+* Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* version 2 along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+* 02110-1301, USA.
+*/"""
+
+# An optional name to use as an include guard
+# default: doesn't emit an include guard
+include_guard = "__RUST_BINDINGS_GEN_H_"
+
+# An optional string of text to output between major sections of the generated
+# file as a warning against manual editing
+#
+# default: doesn't emit anything
+autogen_warning = "/* DO NOT EDIT This file is autogenerated by cbindgen. Don't modify this manually. */"
+
+# Whether to include a comment with the version of cbindgen used to generate the file
+# default: false
+include_version = true
+
+# A list of headers to #include (with quotes)
+# default: []
+includes = ["rust.h"]
+
+# The desired length of a line to use when formatting lines
+# default: 100
+line_length = 80
+
+# The amount of spaces to indent by
+# default: 2
+tab_width = 4
+
+# How the generated documentation should be commented.
+#
+# possible values:
+# * "c": /* like this */
+# * "c99": // like this
+# * "c++": /// like this
+# * "doxy": like C, but with leading *'s on each line
+# * "auto": "c++" if that's the language, "doxy" otherwise
+#
+# default: "auto"
+documentation_style = "doxy"
+
+
+[export]
+# A list of additional items to always include in the generated bindings if they're
+# found but otherwise don't appear to be used by the public API.
+#
+# default: []
+include = ["AppLayerGetTxIterTuple"]
+
+# A list of items to not include in the generated bindings
+# default: []
+exclude = [
+    "AppLayerDecoderEvents",
+    "CLuaState",
+    "DetectEngineState",
+    "Flow",
+    "FileContainer",
+    "JsonT",
+    "IKEV2State",
+    "IKEV2Transaction",
+    "KRB5State",
+    "KRB5Transaction",
+    "NTPState",
+    "NTPTransaction",
+    "SNMPState",
+    "SNMPTransaction",
+    "SuricataContext",
+    "SuricataFileContext",
+    "TFTPState",
+    "TFTPTransaction"
+]
+
+# Types of items that we'll generate. If empty, then all types of item are emitted.
+#
+# possible items: (TODO: explain these in detail)
+# * "constants":
+# * "globals":
+# * "enums":
+# * "structs":
+# * "unions":
+# * "typedefs":
+# * "opaque":
+# * "functions":
+#
+# default: []
+item_types = ["structs","opaque","functions"]
+
+# Whether applying rules in export.rename prevents export.prefix from applying.
+#
+# e.g. given this toml:
+#
+# [export]
+# prefix = "capi_"
+[export.rename]
+"DNSState" = "RSDNSState"
+"DNSTransaction" = "RSDNSTransaction"
+"JsonT" = "json_t"
+"CLuaState" = "lua_State"
+
+#
+# You get the following results:
+#
+# renaming_overrides_prefixing = true:
+# "MyType" => "my_cool_type"
+#
+# renaming_overrides_prefixing = false:
+# "MyType => capi_my_cool_type"
+#
+# default: false
+renaming_overrides_prefixing = "true"
+
+[parse]
+# A black list of crate names that are not allowed to be parsed.
+# default: []
+exclude = ["libc"]
+
+[parse.expand]
+# A list of feature names that should be used when running `cargo expand`. This
+# combines with `default_features` like in your `Cargo.toml`. Note that the features
+# listed here are features for the current crate being built, *not* the crates
+# being expanded. The crate's `Cargo.toml` must take care of enabling the
+# appropriate features in its dependencies
+#
+# default: []
+features = ["cbindgen"]

rust/gen-c-headers.py

@@ -1,240 +0,0 @@
-#! /usr/bin/env python2
-
-# This script will scan Rust source files looking for extern "C"
-# functions and generate C header files from them with a filename
-# based on the Rust filename.
-#
-# Usage: From the top suricata source directory:
-#
-#    ./rust/gen-c-headers.py
-#
-
-from __future__ import print_function
-
-import sys
-import os
-import re
-from io import StringIO
-
-template = """/* Copyright (C) 2017 Open Information Security Foundation
- *
- * You can copy, redistribute or modify this Program under the terms of
- * the GNU General Public License version 2 as published by the Free
- * Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * version 2 along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-/*
- * DO NOT EDIT. This file is automatically generated.
- */
-
-#ifndef __%(name)s__
-#define __%(name)s__
-
-%(prototypes)s
-#endif /* ! __%(name)s__ */
-"""
-
-# Map of Rust types to C types.
-type_map = {
-    "bool": "bool",
-    "i8": "int8_t",
-    "i16" :"int16_t",
-    "i32" :"int32_t",
-    "i64" :"int64_t",
-
-    "u8": "uint8_t",
-    "u16" :"uint16_t",
-    "u32" :"uint32_t",
-    "u64" :"uint64_t",
-
-    "std::os::raw::c_void": "void",
-    "c_void": "void",
-
-    "std::os::raw::c_char": "char",
-    "c_char": "char",
-    "std::os::raw::c_int": "int",
-    "c_int": "int",
-    "std::os::raw::int8_t": "int8_t",
-    "std::os::raw::int32_t": "int32_t",
-
-    "std::os::raw::uint8_t": "uint8_t",
-    "std::os::raw::uint16_t": "uint16_t",
-    "std::os::raw::uint32_t": "uint32_t",
-    "std::os::raw::uint64_t": "uint64_t",
-
-    "SuricataContext": "SuricataContext",
-    "SuricataFileContext": "SuricataFileContext",
-    "FileContainer": "FileContainer",
-    "core::Flow": "Flow",
-    "Flow": "Flow",
-    "DNSState": "RSDNSState",
-    "DNSTransaction": "RSDNSTransaction",
-    "NFSState": "NFSState",
-    "NFSTransaction": "NFSTransaction",
-    "NTPState": "NTPState",
-    "NTPTransaction": "NTPTransaction",
-    "TFTPTransaction": "TFTPTransaction",
-    "TFTPState": "TFTPState",
-    "SMBState": "SMBState",
-    "SMBTransaction": "SMBTransaction",
-    "SNMPState": "SNMPState",
-    "SNMPTransaction": "SNMPTransaction",
-    "IKEV2State": "IKEV2State",
-    "IKEV2Transaction": "IKEV2Transaction",
-    "KRB5State": "KRB5State",
-    "KRB5Transaction": "KRB5Transaction",
-    "SIPState": "SIPState",
-    "SIPTransaction": "SIPTransaction",
-    "JsonT": "json_t",
-    "DetectEngineState": "DetectEngineState",
-    "core::DetectEngineState": "DetectEngineState",
-    "core::AppLayerDecoderEvents": "AppLayerDecoderEvents",
-    "AppLayerDecoderEvents": "AppLayerDecoderEvents",
-    "core::AppLayerEventType": "AppLayerEventType",
-    "applayer::AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple",
-    "AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple",
-    "AppLayerEventType": "AppLayerEventType",
-    "CLuaState": "lua_State",
-    "Store": "Store",
-    "AppProto": "AppProto",
-    "DetectDnsOpcode": "void *",
-}
-
-def convert_type(rs_type):
-    m = re.match("^[^\s]+$", rs_type)
-    if m:
-        if rs_type in type_map:
-            return type_map[rs_type]
-
-    m = re.match("^(.*)(\s[^\s]+)$", rs_type)
-    if m:
-        mod = m.group(1).strip()
-        rtype = m.group(2).strip()
-        if rtype in type_map:
-            if mod in [
-                    "*mut",
-                    "* mut",
-                    "&mut",
-                    "&'static mut",
-                    ]:
-                return "%s *" % (type_map[rtype])
-            elif mod in [
-                    "*const",
-                    "* const"]:
-                return "const %s *" % (type_map[rtype])
-            elif mod in [
-                    "*mut *const",
-                    "*mut*const"]:
-                return "const %s **" % (type_map[rtype])
-            else:
-                raise Exception("Unknown modifier '%s' in '%s'." % (
-                    mod, rs_type))
-        else:
-            raise Exception("Unknown type: %s" % (rtype))
-
-    raise Exception("Failed to parse Rust type: %s" % (rs_type))
-
-def make_output_filename(filename):
-    if filename.startswith("./"):
-        filename = filename[2:]
-    parts = filename.split(os.path.sep)[1:]
-    last = os.path.splitext(parts.pop())[0]
-    outpath = "./gen/c-headers/rust-%s-%s-gen.h" % (
-        "-".join(parts), last)
-    return outpath.replace("--", "-")
-
-def write_header(fileobj, filename):
-    filename = os.path.basename(filename).replace(
-        "-", "_").replace(".", "_").upper()
-    fileobj.write(file_header % {"name": filename})
-
-def should_regen(input_filename, output_filename):
-    """Check if a file should be regenerated. If the output doesn't exist,
-    or the input is newer than the output return True. Otherwise
-    return False.
-
-    """
-    if not os.path.exists(output_filename):
-        return True
-    if os.stat(input_filename).st_mtime > os.stat(output_filename).st_mtime:
-        return True
-    return False
-
-def gen_headers(filename):
-
-    output_filename = make_output_filename(filename)
-
-    if not should_regen(filename, output_filename):
-        return
-
-    buf = open(filename, "rb").read().decode("utf-8")
-    writer = StringIO()
-
-    for fn in re.findall(
-            r"^pub (unsafe )?extern \"C\" fn ([A_Za-z0-9_]+)\(([^{]+)?\)"
-            r"(\s+-> ([^{]+))?",
-            buf,
-            re.M | re.DOTALL):
-
-        args = []
-
-        fnName = fn[1]
-
-        for arg in fn[2].split(","):
-            if not arg.strip():
-                continue
-            arg_name, rs_type = arg.split(":", 1)
-            arg_name = arg_name.strip()
-            rs_type = rs_type.strip()
-            c_type = convert_type(rs_type)
-
-            if arg_name != "_":
-                args.append("%s %s" % (c_type, arg_name))
-            else:
-                args.append(c_type)
-
-        if not args:
-            args.append("void")
-
-        retType = fn[4].strip()
-        if retType == "":
-            returns = "void"
-        else:
-            returns = convert_type(retType)
-
-        writer.write(u"%s %s(%s);\n" % (returns, fnName, ", ".join(args)))
-
-    if writer.tell() > 0:
-        print("Writing %s" % (output_filename))
-        if not os.path.exists(os.path.dirname(output_filename)):
-            os.makedirs(os.path.dirname(output_filename))
-        with open(output_filename, "w") as output:
-            output.write(template % {
-                "prototypes": writer.getvalue(),
-                "name": os.path.basename(output_filename).replace(
-                    "-", "_").replace(".", "_").upper()
-            })
-
-def main():
-
-    rust_top = os.path.dirname(sys.argv[0])
-    os.chdir(rust_top)
-
-    for dirpath, dirnames, filenames in os.walk("./src"):
-        for filename in filenames:
-            if filename.endswith(".rs"):
-                path = os.path.join(dirpath, filename)
-                gen_headers(path)
-
-if __name__ == "__main__":
-    sys.exit(main())

src/Makefile.am

@@ -382,6 +382,7 @@ runmode-unix-socket.c runmode-unix-socket.h \
 runmode-windivert.c runmode-windivert.h \
 runmodes.c runmodes.h \
 rust.h \
+rust-context.h \
 source-af-packet.c source-af-packet.h \
 source-erf-dag.c source-erf-dag.h \
 source-erf-file.c source-erf-file.h \

src/app-layer-dhcp.c

@@ -25,7 +25,7 @@
 #include "util-unittest.h"
 #include "app-layer-parser.h"
 #include "app-layer-dhcp.h"
-#include "rust-dhcp-dhcp-gen.h"
+#include "rust.h"
 
 void RegisterDHCPParsers(void)
 {

src/app-layer-dns-common.h

@@ -121,8 +121,6 @@ enum {
 };
 
 /** Opaque Rust types. */
-typedef struct RSDNSState_ RSDNSState;
-typedef struct RSDNSTransaction_ RSDNSTransaction;
 
 /** \brief DNS packet header */
 typedef struct DNSHeader_ {

src/app-layer-dns-tcp.c

@@ -26,7 +26,7 @@
 #include "util-unittest.h"
 
 #include "app-layer-dns-tcp.h"
-#include "rust-dns-dns-gen.h"
+#include "rust.h"
 
 #ifdef UNITTESTS
 static void RustDNSTCPParserRegisterTests(void);

src/app-layer-dns-udp.c

@@ -26,7 +26,7 @@
 #include "util-unittest.h"
 
 #include "app-layer-dns-udp.h"
-#include "rust-dns-dns-gen.h"
+#include "rust.h"
 
 #ifdef UNITTESTS
 static void RustDNSUDPParserRegisterTests(void);

src/app-layer-ftp.c

@@ -59,9 +59,8 @@
 #include "util-mem.h"
 #include "util-misc.h"
 
-#include "rust-ftp-mod-gen.h"
-
 #include "output-json.h"
+#include "rust.h"
 
 typedef struct FTPThreadCtx_ {
     MpmThreadCtx *ftp_mpm_thread_ctx;

src/app-layer-ikev2.c

@@ -33,8 +33,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-ikev2.h"
-
-#include "rust-ikev2-ikev2-gen.h"
+#include "rust.h"
 
 void RegisterIKEV2Parsers(void)
 {

src/app-layer-krb5.c

@@ -33,8 +33,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-krb5.h"
-
-#include "rust-krb-krb5-gen.h"
+#include "rust.h"
 
 void RegisterKRB5Parsers(void)
 {

src/app-layer-nfs-tcp.c

@@ -38,7 +38,6 @@
 #include "app-layer-nfs-tcp.h"
 
 #include "rust.h"
-#include "rust-nfs-nfs-gen.h"
 
 /* The default port to probe for echo traffic if not provided in the
  * configuration file. */

src/app-layer-nfs-udp.c

@@ -35,7 +35,6 @@
 #include "app-layer-nfs-udp.h"
 
 #include "rust.h"
-#include "rust-nfs-nfs-gen.h"
 
 /* The default port to probe for echo traffic if not provided in the
  * configuration file. */

src/app-layer-ntp.c

@@ -33,8 +33,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-ntp.h"
-
-#include "rust-ntp-ntp-gen.h"
+#include "rust.h"
 
 void RegisterNTPParsers(void)
 {

src/app-layer-parser.h

@@ -29,6 +29,7 @@
 #include "detect-engine-state.h"
 #include "util-file.h"
 #include "stream-tcp-private.h"
+#include "rust.h"
 
 /* Flags for AppLayerParserState. */
 #define APP_LAYER_PARSER_EOF                    BIT_U8(0)
@@ -94,12 +95,6 @@ typedef int (*AppLayerParserFPtr)(Flow *f, void *protocol_state,
         const uint8_t *buf, uint32_t buf_len,
         void *local_storage, const uint8_t flags);
 
-typedef struct AppLayerGetTxIterTuple {
-    void *tx_ptr;
-    uint64_t tx_id;
-    bool has_next;
-} AppLayerGetTxIterTuple;
-
 typedef struct AppLayerGetTxIterState {
     union {
         void *ptr;

src/app-layer-rdp.c

@@ -30,7 +30,7 @@
 #include "app-layer-detect-proto.h"
 #include "app-layer-parser.h"
 #include "app-layer-rdp.h"
-#include "rust-rdp-rdp-gen.h"
+#include "rust.h"
 
 void RegisterRdpParsers(void) {
     SCLogDebug("Registering rdp parser");

src/app-layer-sip.c

@@ -33,7 +33,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-sip.h"
-#include "rust-sip-sip-gen.h"
+#include "rust.h"
 
 void RegisterSIPParsers(void)
 {

src/app-layer-sip.h

@@ -28,7 +28,5 @@ void RegisterSIPParsers(void);
 void SIPParserRegisterTests(void);
 
 /** Opaque Rust types. */
-typedef struct SIPState_ SIPState;
-typedef struct SIPTransaction_ SIPTransaction;
 
 #endif /* __APP_LAYER_SIP_H__ */

src/app-layer-smb.c

@@ -26,8 +26,6 @@
 
 #include "rust.h"
 #include "app-layer-smb.h"
-#include "rust-smb-smb-gen.h"
-#include "rust-smb-files-gen.h"
 #include "util-misc.h"
 
 #define MIN_REC_SIZE 32+4 // SMB hdr + nbss hdr

src/app-layer-snmp.c

@@ -34,8 +34,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-snmp.h"
-
-#include "rust-snmp-snmp-gen.h"
+#include "rust.h"
 
 void RegisterSNMPParsers(void)
 {

src/app-layer-template-rust.c

@@ -43,8 +43,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-template-rust.h"
-
-#include "rust-applayertemplate-template-gen.h"
+#include "rust.h"
 
 void RegisterTemplateRustParsers(void)
 {

src/app-layer-tftp.c

@@ -35,8 +35,7 @@
 #include "app-layer-parser.h"
 
 #include "app-layer-tftp.h"
-
-#include "rust-tftp-tftp-gen.h"
+#include "rust.h"
 
 /* The default port to probe if not provided in the configuration file. */
 #define TFTP_DEFAULT_PORT "69"

src/detect-dce-iface.c

@@ -48,7 +48,6 @@
 #include "stream-tcp.h"
 
 #include "rust.h"
-#include "rust-smb-detect-gen.h"
 
 #define PARSE_REGEX "^\\s*([0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{12})(?:\\s*,\\s*(<|>|=|!)([0-9]{1,5}))?(?:\\s*,\\s*(any_frag))?\\s*$"
 

src/detect-dce-opnum.c

@@ -49,7 +49,6 @@
 #include "stream-tcp.h"
 
 #include "rust.h"
-#include "rust-smb-detect-gen.h"
 
 #define PARSE_REGEX "^\\s*([0-9]{1,5}(\\s*-\\s*[0-9]{1,5}\\s*)?)(,\\s*[0-9]{1,5}(\\s*-\\s*[0-9]{1,5})?\\s*)*$"
 

src/detect-dce-stub-data.c

@@ -55,7 +55,6 @@
 #include "stream-tcp.h"
 
 #include "rust.h"
-#include "rust-smb-detect-gen.h"
 
 #define BUFFER_NAME "dce_stub_data"
 #define KEYWORD_NAME "dce_stub_data"

src/detect-dns-opcode.c

@@ -21,7 +21,7 @@
 #include "detect-engine.h"
 #include "detect-dns-opcode.h"
 #include "app-layer-dns-common.h"
-#include "rust-dns-detect-gen.h"
+#include "rust.h"
 
 static int dns_opcode_list_id = 0;
 

src/detect-dns-query.c

@@ -59,8 +59,7 @@
 #include "detect-engine-dns.h"
 
 #include "util-unittest-helper.h"
-
-#include "rust-dns-dns-gen.h"
+#include "rust.h"
 
 static int DetectDnsQuerySetup (DetectEngineCtx *, Signature *, const char *);
 static void DetectDnsQueryRegisterTests(void);

src/detect-krb5-cname.c

@@ -35,7 +35,6 @@
 
 #include "rust.h"
 #include "app-layer-krb5.h"
-#include "rust-krb-detect-gen.h"
 
 static int g_krb5_cname_buffer_id = 0;
 

src/detect-krb5-errcode.c

@@ -30,7 +30,7 @@
 #include "detect-krb5-errcode.h"
 
 #include "app-layer-krb5.h"
-#include "rust-krb-detect-gen.h"
+#include "rust.h"
 
 /**
  * \brief Regex for parsing our keyword options

src/detect-krb5-msgtype.c

@@ -30,7 +30,7 @@
 #include "detect-krb5-msgtype.h"
 
 #include "app-layer-krb5.h"
-#include "rust-krb-detect-gen.h"
+#include "rust.h"
 
 /**
  * \brief Regex for parsing our keyword options

src/detect-krb5-sname.c

@@ -35,7 +35,6 @@
 
 #include "rust.h"
 #include "app-layer-krb5.h"
-#include "rust-krb-detect-gen.h"
 
 static int g_krb5_sname_buffer_id = 0;
 

src/detect-nfs-procedure.c

@@ -45,7 +45,6 @@
 
 #include "app-layer-nfs-tcp.h"
 #include "rust.h"
-#include "rust-nfs-nfs-gen.h"
 
 /**
  *   [nfs_procedure]:[<|>]<proc>[<><proc>];

src/detect-nfs-version.c

@@ -45,7 +45,6 @@
 
 #include "app-layer-nfs-tcp.h"
 #include "rust.h"
-#include "rust-nfs-nfs-gen.h"
 
 /**
  *   [nfs_procedure]:[<|>]<proc>[<><proc>];

src/detect-sip-method.c

@@ -53,7 +53,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.method"
 #define KEYWORD_DOC  "sip-keywords.html#sip-method"

src/detect-sip-protocol.c

@@ -54,7 +54,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.protocol"
 #define KEYWORD_DOC  "sip-keywords.html#sip-protocol"

src/detect-sip-request-line.c

@@ -53,7 +53,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.request_line"
 #define KEYWORD_DOC  "sip-keywords.html#sip-request-line"

src/detect-sip-response-line.c

@@ -53,7 +53,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.response_line"
 #define KEYWORD_DOC  "sip-keywords.html#sip-response-line"

src/detect-sip-stat-code.c

@@ -54,7 +54,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.stat_code"
 #define KEYWORD_DOC  "sip-keywords.html#sip-stat-code"

src/detect-sip-stat-msg.c

@@ -54,7 +54,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.stat_msg"
 #define KEYWORD_DOC  "sip-keywords.html#sip-stat-msg"

src/detect-sip-uri.c

@@ -54,7 +54,6 @@
 
 #include "rust.h"
 #include "app-layer-sip.h"
-#include "rust-sip-detect-gen.h"
 
 #define KEYWORD_NAME "sip.uri"
 #define KEYWORD_DOC  "sip-keywords.html#sip-uri"

src/detect-smb-share.c

@@ -35,7 +35,6 @@
 
 #include "detect-smb-share.h"
 #include "rust.h"
-#include "rust-smb-detect-gen.h"
 
 #define BUFFER_NAME "smb_named_pipe"
 #define KEYWORD_NAME "smb.named_pipe"

src/detect-snmp-community.c

@@ -34,9 +34,7 @@
 #include "detect-engine-content-inspection.h"
 #include "detect-snmp-community.h"
 #include "app-layer-parser.h"
-
-#include "rust-snmp-snmp-gen.h"
-#include "rust-snmp-detect-gen.h"
+#include "rust.h"
 
 static int DetectSNMPCommunitySetup(DetectEngineCtx *, Signature *,
     const char *);

src/detect-snmp-pdu_type.c

@@ -29,9 +29,7 @@
 #include "detect-engine-content-inspection.h"
 #include "detect-snmp-pdu_type.h"
 #include "app-layer-parser.h"
-
-#include "rust-snmp-snmp-gen.h"
-#include "rust-snmp-detect-gen.h"
+#include "rust.h"
 
 /**
  *   [snmp.pdu_type]:<type>;

src/detect-snmp-version.c

@@ -29,9 +29,7 @@
 #include "detect-engine-content-inspection.h"
 #include "detect-snmp-version.h"
 #include "app-layer-parser.h"
-
-#include "rust-snmp-snmp-gen.h"
-#include "rust-snmp-detect-gen.h"
+#include "rust.h"
 
 /**
  *   [snmp.version]:[<|>|<=|>=]<version>;

src/detect-template-rust-buffer.c

@@ -38,7 +38,8 @@
 #include "detect-engine-content-inspection.h"
 #include "detect-template-rust-buffer.h"
 #include "app-layer-parser.h"
-#include "rust-applayertemplate-template-gen.h"
+#include "rust.h"
+
 
 static int DetectTemplateRustBufferSetup(DetectEngineCtx *, Signature *,
     const char *);

src/output-json-dhcp.c

@@ -44,8 +44,8 @@
 
 #include "app-layer-dhcp.h"
 #include "output-json-dhcp.h"
+#include "rust.h"
 
-#include "rust-dhcp-logger-gen.h"
 
 typedef struct LogDHCPFileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-dns.c

@@ -50,8 +50,7 @@
 
 #include "output-json.h"
 #include "output-json-dns.h"
-
-#include "rust-dns-log-gen.h"
+#include "rust.h"
 
 /* we can do query logging as well, but it's disabled for now as the
  * TX id handling doesn't expect it */

src/output-json-ikev2.c

@@ -48,7 +48,6 @@
 #include "output-json-ikev2.h"
 
 #include "rust.h"
-#include "rust-ikev2-log-gen.h"
 
 typedef struct LogIKEv2FileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-krb5.c

@@ -48,7 +48,6 @@
 #include "output-json-krb5.h"
 
 #include "rust.h"
-#include "rust-krb-log-gen.h"
 
 typedef struct LogKRB5FileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-nfs.c

@@ -47,7 +47,6 @@
 #include "output-json-nfs.h"
 
 #include "rust.h"
-#include "rust-nfs-log-gen.h"
 
 json_t *JsonNFSAddMetadataRPC(const Flow *f, uint64_t tx_id)
 {

src/output-json-rdp.c

@@ -41,7 +41,7 @@
 #include "app-layer-parser.h"
 #include "app-layer-rdp.h"
 #include "output-json-rdp.h"
-#include "rust-rdp-log-gen.h"
+#include "rust.h"
 
 typedef struct LogRdpFileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-sip.c

@@ -48,7 +48,6 @@
 #include "output-json-sip.h"
 
 #include "rust.h"
-#include "rust-sip-log-gen.h"
 
 typedef struct LogSIPFileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-smb.c

@@ -46,7 +46,6 @@
 #include "output-json-smb.h"
 
 #include "rust.h"
-#include "rust-smb-log-gen.h"
 
 json_t *JsonSMBAddMetadata(const Flow *f, uint64_t tx_id)
 {

src/output-json-snmp.c

@@ -48,7 +48,6 @@
 #include "output-json-snmp.h"
 
 #include "rust.h"
-#include "rust-snmp-log-gen.h"
 
 typedef struct LogSNMPFileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-template-rust.c

@@ -52,8 +52,7 @@
 
 #include "app-layer-template-rust.h"
 #include "output-json-template-rust.h"
-
-#include "rust-applayertemplate-logger-gen.h"
+#include "rust.h"
 
 typedef struct LogTemplateFileCtx_ {
     LogFileCtx *file_ctx;

src/output-json-tftp.c

@@ -49,7 +49,6 @@
 #include "output-json-tftp.h"
 
 #include "rust.h"
-#include "rust-tftp-log-gen.h"
 
 typedef struct LogTFTPFileCtx_ {
     LogFileCtx *file_ctx;

src/rust-context.h

@@ -0,0 +1,62 @@
+/* Copyright (C) 2017 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+#ifndef __RUST_CONTEXT_H__
+#define __RUST_CONTEXT_H__
+
+#include "detect-engine-state.h" //DetectEngineState
+#include "app-layer-krb5.h" //KRB5State, KRB5Transaction
+#include "app-layer-ikev2.h" //IKEV2State, IKEV2Transaction
+#include "app-layer-ntp.h" //NTPState, NTPTransaction
+#include "app-layer-snmp.h" //SNMPState, SNMPTransaction
+#include "app-layer-tftp.h" //TFTPState, TFTPTransaction
+
+typedef struct SuricataContext_ {
+    SCError (*SCLogMessage)(const SCLogLevel, const char *, const unsigned int,
+            const char *, const SCError, const char *message);
+    void (*DetectEngineStateFree)(DetectEngineState *);
+    void (*AppLayerDecoderEventsSetEventRaw)(AppLayerDecoderEvents **,
+            uint8_t);
+    void (*AppLayerDecoderEventsFreeEvents)(AppLayerDecoderEvents **);
+
+    int (*FileOpenFileWithId)(FileContainer *, const StreamingBufferConfig *,
+        uint32_t track_id, const uint8_t *name, uint16_t name_len,
+        const uint8_t *data, uint32_t data_len, uint16_t flags);
+    int (*FileCloseFileById)(FileContainer *, uint32_t track_id,
+            const uint8_t *data, uint32_t data_len, uint16_t flags);
+    int (*FileAppendDataById)(FileContainer *, uint32_t track_id,
+            const uint8_t *data, uint32_t data_len);
+    int (*FileAppendGAPById)(FileContainer *, uint32_t track_id,
+            const uint8_t *data, uint32_t data_len);
+    void (*FileContainerRecycle)(FileContainer *ffc);
+    void (*FilePrune)(FileContainer *ffc);
+    void (*FileSetTx)(FileContainer *, uint64_t);
+
+} SuricataContext;
+
+typedef struct SuricataFileContext_ {
+
+    const StreamingBufferConfig *sbcfg;
+
+} SuricataFileContext;
+
+struct _Store;
+typedef struct _Store Store;
+
+/** Opaque Rust types. */
+
+#endif /* !__RUST_CONTEXT_H__ */

src/rust.h

@@ -18,42 +18,7 @@
 #ifndef __RUST_H__
 #define __RUST_H__
 
-typedef struct SuricataContext_ {
-    SCError (*SCLogMessage)(const SCLogLevel, const char *, const unsigned int,
-            const char *, const SCError, const char *message);
-    void (*DetectEngineStateFree)(DetectEngineState *);
-    void (*AppLayerDecoderEventsSetEventRaw)(AppLayerDecoderEvents **,
-            uint8_t);
-    void (*AppLayerDecoderEventsFreeEvents)(AppLayerDecoderEvents **);
-
-    int (*FileOpenFileWithId)(FileContainer *, const StreamingBufferConfig *,
-        uint32_t track_id, const uint8_t *name, uint16_t name_len,
-        const uint8_t *data, uint32_t data_len, uint16_t flags);
-    int (*FileCloseFileById)(FileContainer *, uint32_t track_id,
-            const uint8_t *data, uint32_t data_len, uint16_t flags);
-    int (*FileAppendDataById)(FileContainer *, uint32_t track_id,
-            const uint8_t *data, uint32_t data_len);
-    int (*FileAppendGAPById)(FileContainer *, uint32_t track_id,
-            const uint8_t *data, uint32_t data_len);
-    void (*FileContainerRecycle)(FileContainer *ffc);
-    void (*FilePrune)(FileContainer *ffc);
-    void (*FileSetTx)(FileContainer *, uint64_t);
-
-} SuricataContext;
-
-typedef struct SuricataFileContext_ {
-
-    const StreamingBufferConfig *sbcfg;
-
-} SuricataFileContext;
-
-struct _Store;
-typedef struct _Store Store;
-
-/** Opaque Rust types. */
-typedef struct NFState_ NFSState;
-typedef struct NFSTransaction_ NFSTransaction;
-typedef struct SMBState_ SMBState;
-typedef struct SMBTransaction_ SMBTransaction;
+#include "rust-context.h"
+#include "rust-bindings.h"
 
 #endif /* !__RUST_H__ */

src/suricata-common.h

@@ -479,6 +479,10 @@ typedef enum {
 #include <lua.h>
 #include <lualib.h>
 #include <lauxlib.h>
+#else
+/* If we don't have Lua, create a typedef for lua_State so the
+ * exported Lua functions don't fail the build. */
+typedef void lua_State;
 #endif
 
 #ifndef HAVE_STRLCAT

src/suricata.c

@@ -177,7 +177,6 @@
 #include "util-lua.h"
 
 #include "rust.h"
-#include "rust-core-gen.h"
 
 /*
  * we put this here, because we only use it here in main.

src/util-debug.c

@@ -42,8 +42,8 @@
 
 #include "util-unittest.h"
 #include "util-syslog.h"
+#include "rust.h"
 
-#include "rust-log-gen.h"
 
 #include "conf.h"
 

src/util-lua-dns.c

@@ -47,6 +47,7 @@
 #include "util-proto-name.h"
 #include "util-logopenfile.h"
 #include "util-time.h"
+#include "rust.h"
 
 #ifdef HAVE_LUA
 
@@ -58,9 +59,6 @@
 #include "util-lua-common.h"
 #include "util-lua-dns.h"
 
-#include "rust-dns-dns-gen.h"
-#include "rust-dns-lua-gen.h"
-
 static int DnsGetDnsRrname(lua_State *luastate)
 {
     if (!(LuaStateNeedProto(luastate, ALPROTO_DNS)))