diff --git a/config.json b/config.json index b1a224c..b6b2be6 100644 --- a/config.json +++ b/config.json @@ -480,8 +480,8 @@ "subject.process.name", "subject.process.path" ], - "last_module_update": "2022-12-08 00:00:00", - "last_update": "2022-12-08 00:00:00" + "last_module_update": "2023-05-24 00:00:00", + "last_update": "2023-05-24 00:00:00" }, { "group_id": "", diff --git a/correlator/1.0.0/cmodule/clibs/windows/amd64/CorrelationInterface.dll b/correlator/1.0.0/cmodule/clibs/windows/amd64/CorrelationInterface.dll index d31503f..9c29918 100644 Binary files a/correlator/1.0.0/cmodule/clibs/windows/amd64/CorrelationInterface.dll and b/correlator/1.0.0/cmodule/clibs/windows/amd64/CorrelationInterface.dll differ diff --git a/correlator/1.0.0/cmodule/clibs/windows/amd64/correlator-module.dll b/correlator/1.0.0/cmodule/clibs/windows/amd64/correlator-module.dll index 78e7cbb..81716a3 100644 Binary files a/correlator/1.0.0/cmodule/clibs/windows/amd64/correlator-module.dll and b/correlator/1.0.0/cmodule/clibs/windows/amd64/correlator-module.dll differ diff --git a/correlator/1.0.0/cmodule/clibs/windows/amd64/libcrypto-1_1-x64.dll b/correlator/1.0.0/cmodule/clibs/windows/amd64/libcrypto-1_1-x64.dll new file mode 100644 index 0000000..5a8b729 Binary files /dev/null and b/correlator/1.0.0/cmodule/clibs/windows/amd64/libcrypto-1_1-x64.dll differ diff --git a/correlator/1.0.0/cmodule/clibs/windows/amd64/libssl-1_1-x64.dll b/correlator/1.0.0/cmodule/clibs/windows/amd64/libssl-1_1-x64.dll new file mode 100644 index 0000000..21c0c10 Binary files /dev/null and b/correlator/1.0.0/cmodule/clibs/windows/amd64/libssl-1_1-x64.dll differ diff --git a/correlator/1.0.0/cmodule/clibs/windows/amd64/log4cplus.dll b/correlator/1.0.0/cmodule/clibs/windows/amd64/log4cplus.dll index 568bc6b..900097d 100644 Binary files a/correlator/1.0.0/cmodule/clibs/windows/amd64/log4cplus.dll and b/correlator/1.0.0/cmodule/clibs/windows/amd64/log4cplus.dll differ diff --git a/correlator/1.0.0/cmodule/clibs/windows/amd64/normalizer-module.dll b/correlator/1.0.0/cmodule/clibs/windows/amd64/normalizer-module.dll index c1db0a0..f93547c 100644 Binary files a/correlator/1.0.0/cmodule/clibs/windows/amd64/normalizer-module.dll and b/correlator/1.0.0/cmodule/clibs/windows/amd64/normalizer-module.dll differ diff --git a/correlator/1.0.0/cmodule/engines/acts_engine.lua b/correlator/1.0.0/cmodule/engines/acts_engine.lua index 8d01171..c6f529e 100644 --- a/correlator/1.0.0/cmodule/engines/acts_engine.lua +++ b/correlator/1.0.0/cmodule/engines/acts_engine.lua @@ -3,6 +3,7 @@ require("strict") require("engines.base_engine") require("engines.corr_engine") +local rex = require("rex_pcre2") local json = require("cjson.safe") CActsEngine = newclass("CActsEngine", CBaseEngine) @@ -34,6 +35,7 @@ function CActsEngine:init(cfg) ) end + self.excludes = {} self.proc_id_fields = { "object.process.id", "object.process.parent.id", @@ -41,6 +43,13 @@ function CActsEngine:init(cfg) "subject.process.parent.id", } + self.topic_name = "raw_events" + if __imc.subscribe_to_topic then + assert(__imc.subscribe_to_topic(self.topic_name, __gid), "can't subscribe to topic") + else + __log.debug("topics mechanism unsupported on agent side") + end + -- initialization of object after base class constructing self:update_config_cb() end @@ -60,7 +69,7 @@ function CActsEngine:timer_cb() -- __log.debug("timer_cb CActsEngine") local acts_engine = CActsEngine:cast(self) acts_engine.correlator:pullEvents() - __metric.add_int_gauge_counter("corr_agent_mem_usage", collectgarbage("count")*1024) + __metric.add_int_gauge_counter("corr_agent_mem_usage", collectgarbage("count") * 1024) return 500 end @@ -70,6 +79,9 @@ function CActsEngine:quit_cb() __log.debug("quit_cb CActsEngine") -- here will be triggered before closing vxproto object and destroying the state + if __imc.unsubscribe_from_topic then + assert(__imc.unsubscribe_from_topic(self.topic_name, __gid), "can't unsubscribe from topic") + end end -- in: string @@ -90,6 +102,10 @@ end -- out: nil function CActsEngine:update_config_cb() __log.debug("update_config_cb CActsEngine") + + local acts_engine = CActsEngine:cast(self) + acts_engine:load_excludes() + -- actual current configuration contains into next fields -- self.config.actions -- self.config.events @@ -115,8 +131,10 @@ function CActsEngine:recv_data_cb(src, data) __metric.add_int_counter("corr_agent_events_recv_count", #odata) __metric.add_int_counter("corr_agent_events_recv_size", #data) - for _,v in ipairs(odata) do - v.body = json.encode(v.body) + for _, v in ipairs(odata) do + if v.mime == "application/x-pt-eventlog" or v.mime == "application/json" then + v.body = json.encode(v.body) + end while acts_engine.correlator:sendEvent(json.encode(v)) == false do __api.await(50) end @@ -157,6 +175,108 @@ function CActsEngine:recv_action_cb(src, data, name) return false end +-- in: nil +-- out: nil +function CActsEngine:load_excludes() + self.excludes = {} + for _, exclude in ipairs(self.config.module.excludes) do + local compiled = {} + for _, cond in ipairs(exclude) do + if type(cond) ~= "table" then + __log.errorf("condition isn't a table (array) value: '%s'", json.encode(cond)) + goto continue + end + if type(cond.fields) ~= "table" then + __log.errorf("fields isn't a table (array) value: '%s'", json.encode(cond.fields)) + goto continue + end + if #cond.fields == 0 then + __log.errorf("fields is empty array") + goto continue + end + for _, field in ipairs(cond.fields) do + if type(field) ~= "string" then + __log.errorf("field isn't a string value: '%s'", json.encode(field)) + goto continue + end + end + if type(cond.regex) ~= "string" then + __log.errorf("regex isn't a string value: '%s'", json.encode(cond.regex)) + goto continue + end + local status, regex = pcall(rex.new, cond.regex) + if not status then + __log.errorf("can't compile regexp: '%s' with error '%s'", cond.regex, regex) + goto continue + end + table.insert(compiled, { + fields = cond.fields, + sregex = cond.regex, + cregex = regex, + }) + ::continue:: + end + if #compiled ~= 0 then + table.insert(self.excludes, compiled) + end + end + __log.infof("loaded %d excludes from '%s'", #self.excludes, json.encode(self.config.module.excludes)) +end + +-- in: table +-- out: bool, table +function CActsEngine:match_excludes(event) + local body = {} + for fname, fvalue in pairs(event) do + if type(fname) ~= "string" then + fname = tostring(fname) or "" + end + if type(fvalue) ~= "string" then + fvalue = json.encode(fvalue) or "" + end + body[fname] = fvalue + end + + for _, exclude in ipairs(self.excludes) do + local matches = {} + for _, cond in ipairs(exclude) do + for _, fname in ipairs(cond.fields) do + local fvalue = body[fname] + if not fvalue then + -- field '{fname}' isn't exist in body + goto next_field + end + local match = cond.cregex:match(fvalue) + if match then + table.insert(matches, { + field = fname, + match = match, + regex = cond.sregex, + value = fvalue, + }) + -- field '{fname}' with value '{fvalue}' matches '{match}' by regex '{cond.sregex}' + __log.debugf("field '%s' with value '%s' matches '%s' by regex '%s'", fname, fvalue, match, cond.sregex) + goto next_condition + end + -- field '{fname}' with value '{fvalue}' doesn't match by regex '{cond.sregex}' + __log.debugf("field '%s' with value '%s' doesn't match by regex '%s'", fname, fvalue, cond.sregex) + ::next_field:: + end + -- fields '{cond.fields}' don't match by regex '{cond.sregex}' + __log.debugf("fields '%s' don't match by regex '%s'", json.encode(cond.fields), cond.sregex) + goto next_exclude + ::next_condition:: + end + if #matches == #exclude then + __log.debugf("all exclude conditions matched with event body '%s'", json.encode(matches)) + return true, matches + end + ::next_exclude:: + end + + return false +end + -- in: table -- out: nil function CActsEngine:push_result(event) @@ -170,11 +290,11 @@ function CActsEngine:push_result(event) if event_name == nil or event_name == "" then return end - local config_events = self.config["events"] or {events={}} - local config_event = config_events[event_name] or {fields={}} - local config_fields = self.config["fields"] or {properties={}} + local config_events = self.config["events"] or { events = {} } + local config_event = config_events[event_name] or { fields = {} } + local config_fields = self.config["fields"] or { properties = {} } local _fields = config_event["fields"] or {} - local defaults = {string = "", number = 0, integer = 0, object = {}, array = {}, boolean = false, null = nil} + local defaults = { string = "", number = 0, integer = 0, object = {}, array = {}, boolean = false, null = nil } for _, v in ipairs(self.proc_id_fields) do result[v] = tonumber(result[v]) @@ -184,5 +304,7 @@ function CActsEngine:push_result(event) end __log.debugf("perform logic to push result: '%s'", json.encode(result)) - self:push_event(event_name,result) + if not self:match_excludes(result) then + self:push_event(event_name, result) + end end diff --git a/correlator/1.0.0/cmodule/engines/corr_engine.lua b/correlator/1.0.0/cmodule/engines/corr_engine.lua index 7de8f8f..886bc6d 100644 --- a/correlator/1.0.0/cmodule/engines/corr_engine.lua +++ b/correlator/1.0.0/cmodule/engines/corr_engine.lua @@ -87,15 +87,15 @@ function CCorrEngine:init(receiveEvents, restore) self.callbacks = { receive = function(type, data, size) if type == 1 and receiveEvents then - receiveEvents(ffi.string(data,size)) + receiveEvents(ffi.string(data, size)) elseif type == 2 then - self.statistics = json.decode(ffi.string(data,size)) + self.statistics = json.decode(ffi.string(data, size)) elseif type == 3 then - __log.errorf("caught error from corr lib: '%s'", ffi.string(data,size)) + __log.errorf("caught error from corr lib: '%s'", ffi.string(data, size)) end return size - end + end, } self.statistics = {} @@ -179,12 +179,12 @@ function CCorrEngine.get_file_hash_md5(filepath) end function CCorrEngine.copyFile(src, dst, force) - local f, err = io.open(src, 'rb') + local f, err = io.open(src, "rb") if not f then return nil, err end local t, ok if not force then - t = io.open(dst, 'rb') + t = io.open(dst, "rb") if t then f:close() t:close() @@ -192,7 +192,7 @@ function CCorrEngine.copyFile(src, dst, force) end end - t, err = io.open(dst, 'w+b') + t, err = io.open(dst, "w+b") if not t then f:close() return nil, err diff --git a/correlator/1.0.0/cmodule/main.lua b/correlator/1.0.0/cmodule/main.lua index de2a522..880fa6a 100644 --- a/correlator/1.0.0/cmodule/main.lua +++ b/correlator/1.0.0/cmodule/main.lua @@ -61,6 +61,7 @@ __api.add_cbs({ __log.infof("module '%s' was started", __config.ctx.name) acts_engine:run() +__api.del_cbs({ "data", "file", "action", "control" }) __log.infof("module '%s' was stopped", __config.ctx.name) -- explicit destroy engine diff --git a/correlator/1.0.0/cmodule/module.lua b/correlator/1.0.0/cmodule/module.lua index 61b52d0..fd18492 100644 --- a/correlator/1.0.0/cmodule/module.lua +++ b/correlator/1.0.0/cmodule/module.lua @@ -50,9 +50,8 @@ function CModule:init(moduleName) self.api = { create = self.module.module_create, destroy = self.module.module_destroy, - version = nil,--self.module.Module__Version, - - is_inited = false + version = nil, --self.module.Module__Version, + is_inited = false, } self.transport = ffi.new("api_module_transport[1]", {}) end @@ -116,7 +115,7 @@ function CModule:register(profile, callbacks) local ctmpdir = ffi.new("char[?]", 256) lk32.GetDllDirectoryA(256, ctmpdir) lk32.SetDllDirectoryA(__tmpdir) - self.api.is_inited = self.module_i.init(self.transport,self.profile, #profile) + self.api.is_inited = self.module_i.init(self.transport, self.profile, #profile) lk32.SetDllDirectoryA(ctmpdir) return self.api.is_inited, "" @@ -131,7 +130,7 @@ function CModule:start() self.module_i.start(self.transport) end -function CModule:send(type,data) +function CModule:send(type, data) if self.transport[0].to_module == nil then return end diff --git a/correlator/1.0.0/config/config_schema.json b/correlator/1.0.0/config/config_schema.json index 01d441c..c6f21de 100644 --- a/correlator/1.0.0/config/config_schema.json +++ b/correlator/1.0.0/config/config_schema.json @@ -1,6 +1,95 @@ { "additionalProperties": false, - "properties": {}, + "properties": { + "excludes": { + "items": { + "items": { + "properties": { + "fields": { + "rules": { + "required": true + }, + "type": "array", + "ui": { + "columns": 6, + "label": "dx: window.__$ncform.lang === 'en' ? 'Variables' : 'Переменные'", + "widget": "select", + "widgetConfig": { + "allowCreate": true, + "clearable": true, + "defaultFirstOption": false, + "enumSourceRemote": { + "otherParams": { + "filters[]": "{\"field\": \"module_name\", \"value\": [\"correlator\"]}", + "group": "name", + "lang": "en", + "page": 1, + "pageSize": -1, + "type": "init" + }, + "remoteUrl": "/api/v1/options/fields", + "resField": "data.fields" + }, + "filterLocal": true, + "filterable": true, + "itemDataKey": "item", + "itemLabelField": "name", + "itemValueField": "name", + "multiple": true + } + }, + "value": [] + }, + "regex": { + "rules": { + "customRule": [ + { + "errMsg": "regexp format required", + "script": "dx: ((val) =\u003e { try { const v = new RegExp(val); return typeof(v) !== undefined } catch(e) { return false } })(__get({{$root}}, {{$path}}))" + } + ], + "required": true + }, + "type": "string", + "ui": { + "columns": 6, + "label": "dx: window.__$ncform.lang === 'en' ? 'Regular expression' : 'Регулярное выражение'" + } + } + }, + "required": [], + "type": "object", + "ui": { + "label": "dx: ((fields) =\u003e { const or = (window.__$ncform.lang === 'en' ? ' OR ' : ' ИЛИ '); const retIsArray = ((val, def) =\u003e (typeof val === 'object' \u0026\u0026 Array.isArray(val) ? val : def)); fields = retIsArray(fields, []); switch(fields.length) { case 0: return (window.__$ncform.lang === 'en' ? 'empty' : 'пустой список'); case 1: return fields[0]; default: return '(' + [...new Set(fields.map(f =\u003e '...' + f.split('.').slice(-1)[0]))].join(or) + ')'; } })(__get({{$root}}, {{$path}} + '.fields'))" + } + }, + "type": "array", + "ui": { + "label": "dx: (() =\u003e { const and = (window.__$ncform.lang === 'en' ? ' AND ' : ' И '); const or = (window.__$ncform.lang === 'en' ? ' OR ' : ' ИЛИ '); const retIsArray = ((val, def) =\u003e (typeof val === 'object' \u0026\u0026 Array.isArray(val) ? val : def)); const getLabel = ((fields) =\u003e { fields = retIsArray(fields, []); switch(fields.length) { case 0: return (window.__$ncform.lang === 'en' ? 'empty' : 'пустой список'); case 1: return JSON.stringify(fields[0]); default: return '(' + [...new Set(fields.map(f =\u003e JSON.stringify('...' + f.split('.').slice(-1)[0])))].join(or) + ')'; } }); return retIsArray(__get({{$root}}, {{$path}}), []).map(e =\u003e getLabel(e.fields)).join(and);})()", + "noLabelSpace": true, + "showIdxLabel": false, + "showLegend": false, + "widget": "array-tabs", + "widgetConfig": { + "showOneIfEmpty": true + } + } + }, + "rules": {}, + "type": "array", + "ui": { + "noLabelSpace": true, + "showIdxLabel": false, + "showLabel": false, + "showLegend": true, + "widgetConfig": { + "collapsed": true, + "disableReorder": true, + "itemCollapse": true + } + } + } + }, "required": [], "type": "object" } \ No newline at end of file diff --git a/correlator/1.0.0/config/current_config.json b/correlator/1.0.0/config/current_config.json index 9e26dfe..17d7298 100644 --- a/correlator/1.0.0/config/current_config.json +++ b/correlator/1.0.0/config/current_config.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "excludes": [] +} \ No newline at end of file diff --git a/correlator/1.0.0/config/default_config.json b/correlator/1.0.0/config/default_config.json index 9e26dfe..17d7298 100644 --- a/correlator/1.0.0/config/default_config.json +++ b/correlator/1.0.0/config/default_config.json @@ -1 +1,3 @@ -{} \ No newline at end of file +{ + "excludes": [] +} \ No newline at end of file diff --git a/correlator/1.0.0/config/locale.json b/correlator/1.0.0/config/locale.json index ae071e4..2ce59b8 100644 --- a/correlator/1.0.0/config/locale.json +++ b/correlator/1.0.0/config/locale.json @@ -1,7 +1,18 @@ { "action_config": {}, "actions": {}, - "config": {}, + "config": { + "excludes": { + "en": { + "description": "Exclusion list", + "title": "Exclusions" + }, + "ru": { + "description": "Список исключений", + "title": "Исключения" + } + } + }, "event_config": { "Suspicious_Create_Process_BitsAdmin_RestrictionBypass": {}, "Suspicious_Create_Process_NetSh_NetShell": {}, diff --git a/correlator/1.0.0/smodule/main.lua b/correlator/1.0.0/smodule/main.lua index 668e4f3..a4f28e4 100644 --- a/correlator/1.0.0/smodule/main.lua +++ b/correlator/1.0.0/smodule/main.lua @@ -25,4 +25,4 @@ __api.await(-1) __log.infof("module '%s' was stopped", __config.ctx.name) -return 'success' +return "success"