fix: update correlator bundle to avoid issue with running it on windows 10 22H2

correlator/1.0.0/cmodule/engines/acts_engine.lua

@@ -3,6 +3,7 @@ require("strict")
 require("engines.base_engine")
 require("engines.corr_engine")
 
+local rex = require("rex_pcre2")
 local json = require("cjson.safe")
 
 CActsEngine = newclass("CActsEngine", CBaseEngine)
@@ -34,6 +35,7 @@ function CActsEngine:init(cfg)
         )
     end
 
+    self.excludes = {}
     self.proc_id_fields = {
         "object.process.id",
         "object.process.parent.id",
@@ -41,6 +43,13 @@ function CActsEngine:init(cfg)
         "subject.process.parent.id",
     }
 
+    self.topic_name = "raw_events"
+    if __imc.subscribe_to_topic then
+        assert(__imc.subscribe_to_topic(self.topic_name, __gid), "can't subscribe to topic")
+    else
+        __log.debug("topics mechanism unsupported on agent side")
+    end
+
     -- initialization of object after base class constructing
     self:update_config_cb()
 end
@@ -60,7 +69,7 @@ function CActsEngine:timer_cb()
     -- __log.debug("timer_cb CActsEngine")
     local acts_engine = CActsEngine:cast(self)
     acts_engine.correlator:pullEvents()
-    __metric.add_int_gauge_counter("corr_agent_mem_usage", collectgarbage("count")*1024)
+    __metric.add_int_gauge_counter("corr_agent_mem_usage", collectgarbage("count") * 1024)
     return 500
 end
 
@@ -70,6 +79,9 @@ function CActsEngine:quit_cb()
     __log.debug("quit_cb CActsEngine")
 
     -- here will be triggered before closing vxproto object and destroying the state
+    if __imc.unsubscribe_from_topic then
+        assert(__imc.unsubscribe_from_topic(self.topic_name, __gid), "can't unsubscribe from topic")
+    end
 end
 
 -- in: string
@@ -90,6 +102,10 @@ end
 -- out: nil
 function CActsEngine:update_config_cb()
     __log.debug("update_config_cb CActsEngine")
+
+    local acts_engine = CActsEngine:cast(self)
+    acts_engine:load_excludes()
+
     -- actual current configuration contains into next fields
     -- self.config.actions
     -- self.config.events
@@ -115,8 +131,10 @@ function CActsEngine:recv_data_cb(src, data)
         __metric.add_int_counter("corr_agent_events_recv_count", #odata)
         __metric.add_int_counter("corr_agent_events_recv_size", #data)
 
-        for _,v in ipairs(odata) do
-            v.body = json.encode(v.body)
+        for _, v in ipairs(odata) do
+            if v.mime == "application/x-pt-eventlog" or v.mime == "application/json" then
+                v.body = json.encode(v.body)
+            end
             while acts_engine.correlator:sendEvent(json.encode(v)) == false do
                 __api.await(50)
             end
@@ -157,6 +175,108 @@ function CActsEngine:recv_action_cb(src, data, name)
     return false
 end
 
+-- in: nil
+-- out: nil
+function CActsEngine:load_excludes()
+    self.excludes = {}
+    for _, exclude in ipairs(self.config.module.excludes) do
+        local compiled = {}
+        for _, cond in ipairs(exclude) do
+            if type(cond) ~= "table" then
+                __log.errorf("condition isn't a table (array) value: '%s'", json.encode(cond))
+                goto continue
+            end
+            if type(cond.fields) ~= "table" then
+                __log.errorf("fields isn't a table (array) value: '%s'", json.encode(cond.fields))
+                goto continue
+            end
+            if #cond.fields == 0 then
+                __log.errorf("fields is empty array")
+                goto continue
+            end
+            for _, field in ipairs(cond.fields) do
+                if type(field) ~= "string" then
+                    __log.errorf("field isn't a string value: '%s'", json.encode(field))
+                    goto continue
+                end
+            end
+            if type(cond.regex) ~= "string" then
+                __log.errorf("regex isn't a string value: '%s'", json.encode(cond.regex))
+                goto continue
+            end
+            local status, regex = pcall(rex.new, cond.regex)
+            if not status then
+                __log.errorf("can't compile regexp: '%s' with error '%s'", cond.regex, regex)
+                goto continue
+            end
+            table.insert(compiled, {
+                fields = cond.fields,
+                sregex = cond.regex,
+                cregex = regex,
+            })
+            ::continue::
+        end
+        if #compiled ~= 0 then
+            table.insert(self.excludes, compiled)
+        end
+    end
+    __log.infof("loaded %d excludes from '%s'", #self.excludes, json.encode(self.config.module.excludes))
+end
+
+-- in: table
+-- out: bool, table
+function CActsEngine:match_excludes(event)
+    local body = {}
+    for fname, fvalue in pairs(event) do
+        if type(fname) ~= "string" then
+            fname = tostring(fname) or ""
+        end
+        if type(fvalue) ~= "string" then
+            fvalue = json.encode(fvalue) or ""
+        end
+        body[fname] = fvalue
+    end
+
+    for _, exclude in ipairs(self.excludes) do
+        local matches = {}
+        for _, cond in ipairs(exclude) do
+            for _, fname in ipairs(cond.fields) do
+                local fvalue = body[fname]
+                if not fvalue then
+                    -- field '{fname}' isn't exist in body
+                    goto next_field
+                end
+                local match = cond.cregex:match(fvalue)
+                if match then
+                    table.insert(matches, {
+                        field = fname,
+                        match = match,
+                        regex = cond.sregex,
+                        value = fvalue,
+                    })
+                    -- field '{fname}' with value '{fvalue}' matches '{match}' by regex '{cond.sregex}'
+                    __log.debugf("field '%s' with value '%s' matches '%s' by regex '%s'", fname, fvalue, match, cond.sregex)
+                    goto next_condition
+                end
+                -- field '{fname}' with value '{fvalue}' doesn't match by regex '{cond.sregex}'
+                __log.debugf("field '%s' with value '%s' doesn't match by regex '%s'", fname, fvalue, cond.sregex)
+                ::next_field::
+            end
+            -- fields '{cond.fields}' don't match by regex '{cond.sregex}'
+            __log.debugf("fields '%s' don't match by regex '%s'", json.encode(cond.fields), cond.sregex)
+            goto next_exclude
+            ::next_condition::
+        end
+        if #matches == #exclude then
+            __log.debugf("all exclude conditions matched with event body '%s'", json.encode(matches))
+            return true, matches
+        end
+        ::next_exclude::
+    end
+
+    return false
+end
+
 -- in: table
 -- out: nil
 function CActsEngine:push_result(event)
@@ -170,11 +290,11 @@ function CActsEngine:push_result(event)
 
     if event_name == nil or event_name == "" then return end
 
-    local config_events = self.config["events"] or {events={}}
-    local config_event = config_events[event_name] or {fields={}}
-    local config_fields = self.config["fields"] or {properties={}}
+    local config_events = self.config["events"] or { events = {} }
+    local config_event = config_events[event_name] or { fields = {} }
+    local config_fields = self.config["fields"] or { properties = {} }
     local _fields = config_event["fields"] or {}
-    local defaults = {string = "", number = 0, integer = 0, object = {}, array = {}, boolean = false, null = nil}
+    local defaults = { string = "", number = 0, integer = 0, object = {}, array = {}, boolean = false, null = nil }
 
     for _, v in ipairs(self.proc_id_fields) do
         result[v] = tonumber(result[v])
@@ -184,5 +304,7 @@ function CActsEngine:push_result(event)
     end
 
     __log.debugf("perform logic to push result: '%s'", json.encode(result))
-    self:push_event(event_name,result)
+    if not self:match_excludes(result) then
+        self:push_event(event_name, result)
+    end
 end

correlator/1.0.0/cmodule/engines/corr_engine.lua

@@ -87,15 +87,15 @@ function CCorrEngine:init(receiveEvents, restore)
     self.callbacks = {
         receive = function(type, data, size)
             if type == 1 and receiveEvents then
-                receiveEvents(ffi.string(data,size))
+                receiveEvents(ffi.string(data, size))
             elseif type == 2 then
-                self.statistics = json.decode(ffi.string(data,size))
+                self.statistics = json.decode(ffi.string(data, size))
             elseif type == 3 then
-                __log.errorf("caught error from corr lib: '%s'", ffi.string(data,size))
+                __log.errorf("caught error from corr lib: '%s'", ffi.string(data, size))
             end
 
             return size
-        end
+        end,
     }
 
     self.statistics = {}
@@ -179,12 +179,12 @@ function CCorrEngine.get_file_hash_md5(filepath)
 end
 
 function CCorrEngine.copyFile(src, dst, force)
-    local f, err = io.open(src, 'rb')
+    local f, err = io.open(src, "rb")
     if not f then return nil, err end
 
     local t, ok
     if not force then
-        t = io.open(dst, 'rb')
+        t = io.open(dst, "rb")
         if t then
             f:close()
             t:close()
@@ -192,7 +192,7 @@ function CCorrEngine.copyFile(src, dst, force)
         end
     end
 
-    t, err = io.open(dst, 'w+b')
+    t, err = io.open(dst, "w+b")
     if not t then
         f:close()
         return nil, err

correlator/1.0.0/cmodule/main.lua

@@ -61,6 +61,7 @@ __api.add_cbs({
 
 __log.infof("module '%s' was started", __config.ctx.name)
 acts_engine:run()
+__api.del_cbs({ "data", "file", "action", "control" })
 __log.infof("module '%s' was stopped", __config.ctx.name)
 
 -- explicit destroy engine

correlator/1.0.0/cmodule/module.lua

@@ -50,9 +50,8 @@ function CModule:init(moduleName)
     self.api = {
         create = self.module.module_create,
         destroy = self.module.module_destroy,
-        version = nil,--self.module.Module__Version,
-
-        is_inited = false
+        version = nil, --self.module.Module__Version,
+        is_inited = false,
     }
     self.transport = ffi.new("api_module_transport[1]", {})
 end
@@ -116,7 +115,7 @@ function CModule:register(profile, callbacks)
     local ctmpdir = ffi.new("char[?]", 256)
     lk32.GetDllDirectoryA(256, ctmpdir)
     lk32.SetDllDirectoryA(__tmpdir)
-    self.api.is_inited = self.module_i.init(self.transport,self.profile, #profile)
+    self.api.is_inited = self.module_i.init(self.transport, self.profile, #profile)
     lk32.SetDllDirectoryA(ctmpdir)
 
     return self.api.is_inited, ""
@@ -131,7 +130,7 @@ function CModule:start()
     self.module_i.start(self.transport)
 end
 
-function CModule:send(type,data)
+function CModule:send(type, data)
     if self.transport[0].to_module == nil then
         return
     end

correlator/1.0.0/config/config_schema.json

@@ -1,6 +1,95 @@
 {
     "additionalProperties": false,
-    "properties": {},
+    "properties": {
+        "excludes": {
+            "items": {
+                "items": {
+                    "properties": {
+                        "fields": {
+                            "rules": {
+                                "required": true
+                            },
+                            "type": "array",
+                            "ui": {
+                                "columns": 6,
+                                "label": "dx: window.__$ncform.lang === 'en' ? 'Variables' : 'Переменные'",
+                                "widget": "select",
+                                "widgetConfig": {
+                                    "allowCreate": true,
+                                    "clearable": true,
+                                    "defaultFirstOption": false,
+                                    "enumSourceRemote": {
+                                        "otherParams": {
+                                            "filters[]": "{\"field\": \"module_name\", \"value\": [\"correlator\"]}",
+                                            "group": "name",
+                                            "lang": "en",
+                                            "page": 1,
+                                            "pageSize": -1,
+                                            "type": "init"
+                                        },
+                                        "remoteUrl": "/api/v1/options/fields",
+                                        "resField": "data.fields"
+                                    },
+                                    "filterLocal": true,
+                                    "filterable": true,
+                                    "itemDataKey": "item",
+                                    "itemLabelField": "name",
+                                    "itemValueField": "name",
+                                    "multiple": true
+                                }
+                            },
+                            "value": []
+                        },
+                        "regex": {
+                            "rules": {
+                                "customRule": [
+                                    {
+                                        "errMsg": "regexp format required",
+                                        "script": "dx: ((val) =\u003e { try { const v = new RegExp(val); return typeof(v) !== undefined } catch(e) { return false } })(__get({{$root}}, {{$path}}))"
+                                    }
+                                ],
+                                "required": true
+                            },
+                            "type": "string",
+                            "ui": {
+                                "columns": 6,
+                                "label": "dx: window.__$ncform.lang === 'en' ? 'Regular expression' : 'Регулярное выражение'"
+                            }
+                        }
+                    },
+                    "required": [],
+                    "type": "object",
+                    "ui": {
+                        "label": "dx: ((fields) =\u003e { const or = (window.__$ncform.lang === 'en' ? ' OR ' : ' ИЛИ '); const retIsArray = ((val, def) =\u003e (typeof val === 'object' \u0026\u0026 Array.isArray(val) ? val : def)); fields = retIsArray(fields, []); switch(fields.length) { case 0: return (window.__$ncform.lang === 'en' ? 'empty' : 'пустой список'); case 1: return fields[0]; default: return '(' + [...new Set(fields.map(f =\u003e '...' + f.split('.').slice(-1)[0]))].join(or) + ')'; } })(__get({{$root}}, {{$path}} + '.fields'))"
+                    }
+                },
+                "type": "array",
+                "ui": {
+                    "label": "dx: (() =\u003e { const and = (window.__$ncform.lang === 'en' ? ' AND ' : ' И '); const or = (window.__$ncform.lang === 'en' ? ' OR ' : ' ИЛИ '); const retIsArray = ((val, def) =\u003e (typeof val === 'object' \u0026\u0026 Array.isArray(val) ? val : def)); const getLabel = ((fields) =\u003e { fields = retIsArray(fields, []); switch(fields.length) { case 0: return (window.__$ncform.lang === 'en' ? 'empty' : 'пустой список'); case 1: return JSON.stringify(fields[0]); default: return '(' + [...new Set(fields.map(f =\u003e JSON.stringify('...' + f.split('.').slice(-1)[0])))].join(or) + ')'; } }); return retIsArray(__get({{$root}}, {{$path}}), []).map(e =\u003e getLabel(e.fields)).join(and);})()",
+                    "noLabelSpace": true,
+                    "showIdxLabel": false,
+                    "showLegend": false,
+                    "widget": "array-tabs",
+                    "widgetConfig": {
+                        "showOneIfEmpty": true
+                    }
+                }
+            },
+            "rules": {},
+            "type": "array",
+            "ui": {
+                "noLabelSpace": true,
+                "showIdxLabel": false,
+                "showLabel": false,
+                "showLegend": true,
+                "widgetConfig": {
+                    "collapsed": true,
+                    "disableReorder": true,
+                    "itemCollapse": true
+                }
+            }
+        }
+    },
     "required": [],
     "type": "object"
 }

correlator/1.0.0/config/current_config.json

@@ -1 +1,3 @@
-{}
+{
+    "excludes": []
+}

correlator/1.0.0/config/default_config.json

@@ -1 +1,3 @@
-{}
+{
+    "excludes": []
+}

correlator/1.0.0/config/locale.json

@@ -1,7 +1,18 @@
 {
     "action_config": {},
     "actions": {},
-    "config": {},
+    "config": {
+        "excludes": {
+            "en": {
+                "description": "Exclusion list",
+                "title": "Exclusions"
+            },
+            "ru": {
+                "description": "Список исключений",
+                "title": "Исключения"
+            }
+        }
+    },
     "event_config": {
         "Suspicious_Create_Process_BitsAdmin_RestrictionBypass": {},
         "Suspicious_Create_Process_NetSh_NetShell": {},

correlator/1.0.0/smodule/main.lua

@@ -25,4 +25,4 @@ __api.await(-1)
 
 __log.infof("module '%s' was stopped", __config.ctx.name)
 
-return 'success'
+return "success"