output/tx: allow registering logging at substate

Register HTTP/2 and DOH2 this way.

src/output-json-dns.c

@@ -694,7 +694,11 @@ void JsonDnsLogRegister (void)
 
 void JsonDoh2LogRegister(void)
 {
-    OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonDoH2Log", "eve-log.doh2",
+    OutputRegisterTxSubModuleWithProgressSubState(LOGGER_JSON_TX, "eve-log", "JsonDoH2Log::stream",
-            JsonDnsLogInitCtxSub, ALPROTO_DOH2, JsonDoh2Logger, LogDnsLogThreadInit,
+            "eve-log.doh2", JsonDnsLogInitCtxSub, ALPROTO_DOH2, HTTP2TxTypeStream, JsonDoh2Logger,
+            HTTP2ProgData, HTTP2ProgData, LogDnsLogThreadInit, LogDnsLogThreadDeinit);
+    OutputRegisterTxSubModuleWithProgressSubState(LOGGER_JSON_TX, "eve-log", "LogDoh2Log::global",
+            "eve-log.doh2", JsonDnsLogInitCtxSub, ALPROTO_DOH2, HTTP2TxTypeGlobal, JsonDoh2Logger,
+            HTTP2ProgGlobalComplete, HTTP2ProgGlobalComplete, LogDnsLogThreadInit,
             LogDnsLogThreadDeinit);
 }

src/output-tx.c

@@ -48,6 +48,7 @@ typedef struct OutputTxLoggerThreadData_ {
  * log module (e.g. fast.log) with different output ctx'. */
 typedef struct OutputTxLogger_ {
     AppProto alproto;
+    uint8_t sub_state;
     TxLogger LogFunc;
     TxLoggerCondition LogCondition;
     void *initdata;
@@ -63,10 +64,13 @@ typedef struct OutputTxLogger_ {
 
 static OutputTxLogger **list = NULL;
 
-int SCOutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc,
+static int SCOutputRegisterTxLoggerInternal(LoggerId id, const char *name, AppProto alproto,
-        void *initdata, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition,
+        const uint8_t sub_state, TxLogger LogFunc, void *initdata, int tc_log_progress,
-        ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
+        int ts_log_progress, TxLoggerCondition LogCondition, ThreadInitFunc ThreadInit,
+        ThreadDeinitFunc ThreadDeinit)
 {
+    BUG_ON(sub_state > 0 && (tc_log_progress < 0 || ts_log_progress < 0));
+
     if (list == NULL) {
         list = SCCalloc(g_alproto_max, sizeof(OutputTxLogger *));
         if (unlikely(list == NULL)) {
@@ -85,6 +89,7 @@ int SCOutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, Tx
         return -1;
 
     op->alproto = alproto;
+    op->sub_state = sub_state;
     op->LogFunc = LogFunc;
     op->LogCondition = LogCondition;
     op->initdata = initdata;
@@ -131,6 +136,23 @@ int SCOutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, Tx
     return 0;
 }
 
+int SCOutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc,
+        void *initdata, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition,
+        ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
+{
+    return SCOutputRegisterTxLoggerInternal(id, name, alproto, 0, LogFunc, initdata,
+            tc_log_progress, ts_log_progress, LogCondition, ThreadInit, ThreadDeinit);
+}
+
+int SCOutputRegisterTxLoggerForSubState(LoggerId id, const char *name, AppProto alproto,
+        const uint8_t sub_state, TxLogger LogFunc, void *initdata, int tc_log_progress,
+        int ts_log_progress, TxLoggerCondition LogCondition, ThreadInitFunc ThreadInit,
+        ThreadDeinitFunc ThreadDeinit)
+{
+    return SCOutputRegisterTxLoggerInternal(id, name, alproto, sub_state, LogFunc, initdata,
+            tc_log_progress, ts_log_progress, LogCondition, ThreadInit, ThreadDeinit);
+}
+
 extern bool g_file_logger_enabled;
 extern bool g_filedata_logger_enabled;
 
@@ -274,8 +296,8 @@ struct Ctx {
 
 static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data,
         const OutputTxLogger *logger, const OutputLoggerThreadStore *store, Packet *p, Flow *f,
-        void *alstate, void *tx, const uint64_t tx_id, const AppProto alproto, const bool eof,
+        void *alstate, void *tx, const uint64_t tx_id, AppLayerTxData *txd, const AppProto alproto,
-        const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx)
+        const bool eof, const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx)
 {
     DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
     DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
@@ -295,6 +317,13 @@ static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_
             SCLogDebug("pcap_cnt %" PRIu64 ", tx_id %" PRIu64 " logger %d. EOF %s",
                     PcapPacketCntGet(p), tx_id, logger->logger_id, eof ? "true" : "false");
 
+            if (logger->sub_state != txd->tx_type) {
+                SCLogDebug("logger:%s flow:%s: skip logger for wrong sub state: logger %u tx %u",
+                        AppProtoToString(logger->alproto), AppProtoToString(alproto),
+                        logger->sub_state, txd->tx_type);
+                goto next_logger;
+            }
+
             if (eof) {
                 SCLogDebug("EOF, so log now");
             } else {
@@ -503,8 +532,8 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
         struct Ctx ctx = { .tx_logged = txd->logged, .tx_logged_old = txd->logged };
         SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
 
-        OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, alproto,
+        OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, txd,
-                eof, tx_progress_ts, tx_progress_tc, &ctx);
+                alproto, eof, tx_progress_ts, tx_progress_tc, &ctx);
 
         SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
         if (ctx.tx_logged != ctx.tx_logged_old) {

src/output-tx.h

@@ -77,6 +77,9 @@ typedef bool (*TxLoggerCondition)(
 int SCOutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc,
         void *, int tc_log_progress, int ts_log_progress, TxLoggerCondition LogCondition,
         ThreadInitFunc, ThreadDeinitFunc);
+int SCOutputRegisterTxLoggerForSubState(LoggerId id, const char *name, AppProto alproto,
+        const uint8_t sub_state, TxLogger LogFunc, void *, int tc_log_progress, int ts_log_progress,
+        TxLoggerCondition LogCondition, ThreadInitFunc, ThreadDeinitFunc);
 
 /** Internal function: private API. */
 void OutputTxLoggerRegister (void);

src/output.c

@@ -300,9 +300,9 @@ error:
 }
 
 static void OutputRegisterTxSubModuleWrapper(LoggerId id, const char *parent_name, const char *name,
-        const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc,
+        const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto,
-        int tc_log_progress, int ts_log_progress, TxLoggerCondition TxLogCondition,
+        const uint8_t sub_state, TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress,
-        ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
+        TxLoggerCondition TxLogCondition, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
 {
     if (unlikely(TxLogFunc == NULL)) {
         goto error;
@@ -321,6 +321,7 @@ static void OutputRegisterTxSubModuleWrapper(LoggerId id, const char *parent_nam
     module->TxLogFunc = TxLogFunc;
     module->TxLogCondition = TxLogCondition;
     module->alproto = alproto;
+    module->sub_state = sub_state;
     module->tc_log_progress = tc_log_progress;
     module->ts_log_progress = ts_log_progress;
     module->ThreadInit = ThreadInit;
@@ -353,8 +354,8 @@ void OutputRegisterTxSubModuleWithCondition(LoggerId id, const char *parent_name
         const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc,
         TxLoggerCondition TxLogCondition, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
 {
-    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, TxLogFunc,
+    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, 0,
-            -1, -1, TxLogCondition, ThreadInit, ThreadDeinit);
+            TxLogFunc, -1, -1, TxLogCondition, ThreadInit, ThreadDeinit);
 }
 
 /**
@@ -378,8 +379,17 @@ void OutputRegisterTxSubModuleWithProgress(LoggerId id, const char *parent_name,
         int tc_log_progress, int ts_log_progress, ThreadInitFunc ThreadInit,
         ThreadDeinitFunc ThreadDeinit)
 {
-    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, TxLogFunc,
+    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, 0,
-            tc_log_progress, ts_log_progress, NULL, ThreadInit, ThreadDeinit);
+            TxLogFunc, tc_log_progress, ts_log_progress, NULL, ThreadInit, ThreadDeinit);
+}
+
+void OutputRegisterTxSubModuleWithProgressSubState(LoggerId id, const char *parent_name,
+        const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto,
+        const uint8_t sub_state, TxLogger TxLogFunc, uint8_t tc_log_progress,
+        uint8_t ts_log_progress, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
+{
+    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, sub_state,
+            TxLogFunc, tc_log_progress, ts_log_progress, NULL, ThreadInit, ThreadDeinit);
 }
 
 /**
@@ -402,8 +412,8 @@ void OutputRegisterTxSubModule(LoggerId id, const char *parent_name, const char
         const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc,
         ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit)
 {
-    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, TxLogFunc,
+    OutputRegisterTxSubModuleWrapper(id, parent_name, name, conf_name, InitFunc, alproto, 0,
-            -1, -1, NULL, ThreadInit, ThreadDeinit);
+            TxLogFunc, -1, -1, NULL, ThreadInit, ThreadDeinit);
 }
 
 /**
@@ -1087,9 +1097,14 @@ void OutputRegisterLoggers(void)
     JsonSmtpLogRegister();
     /* http log */
     JsonHttpLogRegister();
-    OutputRegisterTxSubModuleWithProgress(LOGGER_JSON_TX, "eve-log", "LogHttp2Log", "eve-log.http2",
+    OutputRegisterTxSubModuleWithProgressSubState(LOGGER_JSON_TX, "eve-log", "LogHttp2Log::stream",
-            OutputJsonLogInitSub, ALPROTO_HTTP2, JsonGenericDirFlowLogger, HTTP2ProgClosed,
+            "eve-log.http2", OutputJsonLogInitSub, ALPROTO_HTTP2, HTTP2TxTypeStream,
-            HTTP2ProgClosed, JsonLogThreadInit, JsonLogThreadDeinit);
+            JsonGenericDirFlowLogger, HTTP2ProgClosed, HTTP2ProgClosed, JsonLogThreadInit,
+            JsonLogThreadDeinit);
+    OutputRegisterTxSubModuleWithProgressSubState(LOGGER_JSON_TX, "eve-log", "LogHttp2Log::global",
+            "eve-log.http2", OutputJsonLogInitSub, ALPROTO_HTTP2, HTTP2TxTypeGlobal,
+            JsonGenericDirFlowLogger, HTTP2ProgGlobalComplete, HTTP2ProgGlobalComplete,
+            JsonLogThreadInit, JsonLogThreadDeinit);
     /* tls log */
     JsonTlsLogRegister();
     LogTlsStoreRegister();

src/output.h

@@ -74,6 +74,7 @@ typedef struct OutputModule_ {
     SCStreamingLogger StreamingLogFunc;
     StatsLogger StatsLogFunc;
     AppProto alproto;
+    uint8_t sub_state;
     enum SCOutputStreamingType stream_type;
     int tc_log_progress;
     int ts_log_progress;
@@ -121,6 +122,10 @@ void OutputRegisterTxSubModuleWithProgress(LoggerId id, const char *parent_name,
         const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto, TxLogger TxLogFunc,
         int tc_log_progress, int ts_log_progress, ThreadInitFunc ThreadInit,
         ThreadDeinitFunc ThreadDeinit);
+void OutputRegisterTxSubModuleWithProgressSubState(LoggerId id, const char *parent_name,
+        const char *name, const char *conf_name, OutputInitSubFunc InitFunc, AppProto alproto,
+        const uint8_t sub_state, TxLogger TxLogFunc, uint8_t tc_log_progress,
+        uint8_t ts_log_progress, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit);
 
 void OutputRegisterFileSubModule(LoggerId id, const char *parent_name, const char *name,
         const char *conf_name, OutputInitSubFunc InitFunc, SCFileLogger FileLogFunc,

src/runmodes.c

@@ -631,10 +631,18 @@ static void SetupOutput(
         SCOutputRegisterPacketLogger(module->logger_id, module->name, module->PacketLogFunc,
                 module->PacketConditionFunc, output_ctx, module->ThreadInit, module->ThreadDeinit);
     } else if (module->TxLogFunc) {
-        SCLogDebug("%s is a tx logger", module->name);
+        if (module->sub_state == 0) {
-        SCOutputRegisterTxLogger(module->logger_id, module->name, module->alproto,
+            SCLogDebug("%s is a tx logger", module->name);
-                module->TxLogFunc, output_ctx, module->tc_log_progress, module->ts_log_progress,
+            SCOutputRegisterTxLogger(module->logger_id, module->name, module->alproto,
-                module->TxLogCondition, module->ThreadInit, module->ThreadDeinit);
+                    module->TxLogFunc, output_ctx, module->tc_log_progress, module->ts_log_progress,
+                    module->TxLogCondition, module->ThreadInit, module->ThreadDeinit);
+        } else {
+            SCLogDebug("%s is a tx logger for sub state %u", module->name, module->sub_state);
+            SCOutputRegisterTxLoggerForSubState(module->logger_id, module->name, module->alproto,
+                    module->sub_state, module->TxLogFunc, output_ctx, module->tc_log_progress,
+                    module->ts_log_progress, module->TxLogCondition, module->ThreadInit,
+                    module->ThreadDeinit);
+        }
         /* Not used with wild card loggers */
         if (module->alproto != ALPROTO_UNKNOWN) {
             logger_bits[module->alproto] |= BIT_U32(module->logger_id);