vxcontrol/soldr-modules
1
0
Fork 0
mirror of https://github.com/vxcontrol/soldr-modules.git synced 2026-07-01 12:47:17 -04:00
Code Issues 7 Packages Projects Releases 2 Wiki Activity

Feature/improve protocol (#43)

Browse Source 
* add args and sent msg validation for vxproto mock

* fix linter config

* remove unused/unpublished events for file_remover module

* add 'data' messages proxying similar to actions
This commit is contained in:
Mikhail Kochegarov
2023-01-16 18:19:36 +10:00
committed by GitHub
parent e26ea21191
commit d5324301ed
12 changed files with 202 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.luacheckrc
+8 -4
View File
@@ -6,11 +6,11 @@ self = false
files["**/*_spec.lua"].std = "+busted"
files["**/tests/*_spec.lua"].std = "+busted"
max_line_length=160
max_string_line_length=250
max_comment_line_length=320
max_line_length = 160
max_string_line_length = 250
max_comment_line_length = 320
exclude_files = {"tests_framework/*"}
exclude_files = { "tests_framework/*" }
read_globals = {
-- table
@@ -22,6 +22,9 @@ read_globals = {
-- assert
"assert.is_true",
"assert.is_false",
"assert.equal",
"assert.not_nil",
"assert.is_nil",
-- yaci API
"newclass",
@@ -109,5 +112,6 @@ globals = {
"teardown",
"setup",
"it",
"before_each"
}
file_remover/1.0.0/config/current_event_config.json
-38
View File
@@ -1,28 +1,4 @@
{
"fr_module_started": {
"actions": [
{
"fields": [],
"module_name": "this",
"name": "log_to_db",
"priority": 10
}
],
"fields": [],
"type": "atomic"
},
"fr_module_stopped": {
"actions": [
{
"fields": [],
"module_name": "this",
"name": "log_to_db",
"priority": 10
}
],
"fields": [],
"type": "atomic"
},
"fr_object_file_removed_failed": {
"actions": [
{
@@ -83,20 +59,6 @@
],
"type": "atomic"
},
"fr_remove_internal_error": {
"actions": [
{
"fields": [],
"module_name": "this",
"name": "log_to_db",
"priority": 10
}
],
"fields": [
"reason"
],
"type": "atomic"
},
"fr_subject_proc_image_removed_failed": {
"actions": [
{
file_remover/1.0.0/config/default_event_config.json
-38
View File
@@ -1,28 +1,4 @@
{
"fr_module_started": {
"actions": [
{
"fields": [],
"module_name": "this",
"name": "log_to_db",
"priority": 10
}
],
"fields": [],
"type": "atomic"
},
"fr_module_stopped": {
"actions": [
{
"fields": [],
"module_name": "this",
"name": "log_to_db",
"priority": 10
}
],
"fields": [],
"type": "atomic"
},
"fr_object_file_removed_failed": {
"actions": [
{
@@ -83,20 +59,6 @@
],
"type": "atomic"
},
"fr_remove_internal_error": {
"actions": [
{
"fields": [],
"module_name": "this",
"name": "log_to_db",
"priority": 10
}
],
"fields": [
"reason"
],
"type": "atomic"
},
"fr_subject_proc_image_removed_failed": {
"actions": [
{
file_remover/1.0.0/config/event_config_schema.json
-76
View File
@@ -1,50 +1,6 @@
{
"additionalProperties": false,
"properties": {
"fr_module_started": {
"allOf": [
{
"$ref": "#/definitions/events.atomic"
},
{
"properties": {
"fields": {
"default": [],
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"fields"
],
"type": "object"
}
]
},
"fr_module_stopped": {
"allOf": [
{
"$ref": "#/definitions/events.atomic"
},
{
"properties": {
"fields": {
"default": [],
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"fields"
],
"type": "object"
}
]
},
"fr_object_file_removed_failed": {
"allOf": [
{
@@ -169,35 +125,6 @@
}
]
},
"fr_remove_internal_error": {
"allOf": [
{
"$ref": "#/definitions/events.atomic"
},
{
"properties": {
"fields": {
"default": [
"reason"
],
"items": {
"enum": [
"reason"
],
"type": "string"
},
"maxItems": 1,
"minItems": 1,
"type": "array"
}
},
"required": [
"fields"
],
"type": "object"
}
]
},
"fr_subject_proc_image_removed_failed": {
"allOf": [
{
@@ -264,13 +191,10 @@
}
},
"required": [
"fr_module_started",
"fr_module_stopped",
"fr_object_file_removed_failed",
"fr_object_file_removed_successful",
"fr_object_proc_image_removed_failed",
"fr_object_proc_image_removed_successful",
"fr_remove_internal_error",
"fr_subject_proc_image_removed_failed",
"fr_subject_proc_image_removed_successful"
],
file_remover/1.0.0/config/info.json
-3
View File
@@ -26,13 +26,10 @@
"fr_remove_subject_proc_image"
],
"events": [
"fr_module_started",
"fr_module_stopped",
"fr_object_file_removed_failed",
"fr_object_file_removed_successful",
"fr_object_proc_image_removed_failed",
"fr_object_proc_image_removed_successful",
"fr_remove_internal_error",
"fr_subject_proc_image_removed_failed",
"fr_subject_proc_image_removed_successful"
],
file_remover/1.0.0/config/locale.json
-33
View File
@@ -96,26 +96,6 @@
}
},
"events": {
"fr_module_started": {
"en": {
"title": "The \"File remover\" module is started",
"description": "The module is started successfully"
},
"ru": {
"title": "Модуль «Удаление файлов» запущен",
"description": "Модуль успешно запущен"
}
},
"fr_module_stopped": {
"en": {
"title": "The \"File remover\" module is stopped",
"description": "The module is stopped successfully"
},
"ru": {
"title": "Модуль «Удаление файлов» остановлен",
"description": "Модуль успешно остановлен"
}
},
"fr_object_file_removed_failed": {
"en": {
"title": "The module failed to remove the object file: an error occurred",
@@ -156,16 +136,6 @@
"description": "'{{[object.process.fullpath]}}' был удален успешно"
}
},
"fr_remove_internal_error": {
"en": {
"title": "The module failed to remove the file: not enough data to perform the operation",
"description": "Internal error occurred by '{{ reason }}' for '{{ filepath }}'"
},
"ru": {
"title": "Файл не удален модулем: недостаточно данных для выполнения операции",
"description": "Возникла внутренняя ошибка удаления '{{ filepath }}' по причине '{{ reason }}'"
}
},
"fr_subject_proc_image_removed_failed": {
"en": {
"title": "The module failed to remove the executable file of the subject process: an error occurred",
@@ -193,13 +163,10 @@
"fr_remove_subject_proc_image": {}
},
"event_config": {
"fr_module_started": {},
"fr_module_stopped": {},
"fr_object_file_removed_failed": {},
"fr_object_file_removed_successful": {},
"fr_object_proc_image_removed_failed": {},
"fr_object_proc_image_removed_successful": {},
"fr_remove_internal_error": {},
"fr_subject_proc_image_removed_failed": {},
"fr_subject_proc_image_removed_successful": {}
},
tests/file_remover_server_spec.lua
+2 -2
View File
@@ -58,7 +58,7 @@ describe('file_remover server', function()
assert.equal(__mock.module_token, o.src)
assert.equal(_G.browser_dst, o.dst)
assert.not_nil(o.data)
assert.equal("Module.Сommon.ActionProxied", o.data.__msg_type)
assert.equal("Module.Common.ActionProxied", o.data.__msg_type)
assert.equal("123", o.data.__cid)
return true
end
@@ -201,7 +201,7 @@ describe('file_remover server', function()
assert.equal(__mock.module_token, o.src)
assert.equal(_G.browser_dst, o.dst)
assert.not_nil(o.data)
assert.equal("Module.Сommon.ActionProxied", o.data.__msg_type)
assert.equal("Module.Common.ActionProxied", o.data.__msg_type)
assert.not_nil(o.data.__cid)
return true
end
utils/mock/api.lua
+11 -3
View File
@@ -1,10 +1,12 @@
local glue = require("glue")
local socket = require("socket")
local protocol = require("protocol/protocol")
---------------------------------------------------
local api = {unsafe={}}
local api = { unsafe = {} }
---------------------------------------------------
function api.unsafe.lock() end
function api.unsafe.unlock() end
function api.add_cbs(cbs)
@@ -69,6 +71,12 @@ function api.send_text_to(dst, data, name)
end
function api.send_msg_to(dst, data, mtype)
assert(dst ~= nil and dst ~= "", "message destination must be defined")
assert(data ~= nil and data ~= "", "message data must be defined")
assert(mtype ~= nil and mtype ~= "", "message mtype must be defined")
assert(mtype >= protocol.message_type.debug and mtype <= protocol.message_type.error,
"message mtype must contain suppoted value")
__mock.trace("__api.send_msg_to", dst, data, mtype)
if __mock.callbacks.msg then
return __mock.callbacks.msg(__mock, dst, __mock.module_token, data, mtype)
@@ -157,11 +165,11 @@ end
function api.await(time)
__mock.trace("__api.await", time)
if coroutine.running() then
local stime, etime = socket.gettime()*1000, 0 -- in milliseconds
local stime, etime = socket.gettime() * 1000, 0 -- in milliseconds
while not api.is_close() and (etime < time or time == -1) do
coroutine.yield(false)
socket.sleep(0.1)
etime = socket.gettime()*1000 - stime
etime = socket.gettime() * 1000 - stime
end
end
end
utils/mock/core.lua
+7
View File
@@ -731,6 +731,13 @@ local args_file_data = read_file(args_file_path)
assert(type(args_file_data) == "string", "args.json file must be exist")
local args_file_json = cjson.decode(args_file_data)
assert(type(args_file_json) == "table", "args.json file must be JSON format")
for k, v in pairs(args_file_json) do
assert(type(k) == "string", "args.json root object must contain string keys")
assert(type(v) == "table", "args.json root object must contain table values")
for i, vv in ipairs(v) do
assert(type(i) == "number" and type(vv) == "string", "args.json values must contain array of strings")
end
end
__mock.args = args_file_json
---------------------------------------------------
utils/protocol/cmodule.lua
+68 -5
View File
@@ -7,9 +7,22 @@ require("engine")
require("protocol/actions_validator")
local cmodule = {}
cmodule.quit_handler = function() end
--- Module quit trigger handler.
-- @param reason Defines reason for a module stop (agent_stop, module_remove, module_update).
cmodule.quit_handler = function(_) end
--- New agent connected handler.
-- @param dst Connected agent token.
cmodule.agent_connected_handler = function(_) end
--- Agent disconnected handler.
-- @param dst Disconnected agent token.
cmodule.agent_disconnected_handler = function(_) end
--- Module configuration update handler.
-- @param previous_config Previous config object.
-- @param new_config New config object.
cmodule.update_config_handler = function(_, _) end
-- TODO: use common shared uuid library
@@ -130,9 +143,53 @@ cmodule.push_event = function(event_name, event_data, actions)
end
end
cmodule.start = function(action_handlers, background_process)
cmodule.start = function(action_handlers, data_handlers, background_process)
__api.add_cbs({
data = function(src, data)
local msg_data = cjson.decode(data) or {}
msg_data.__cid = msg_data.__cid or make_uuid()
local action_name = msg_data.name
local response = {
__retaddr = msg_data.__retaddr,
__cid = msg_data.__cid,
__aid = __aid,
__msg_type = protocol.message_name.data_response,
name = action_name,
-- NOTE(mkochegarov): Request data can be quite large for 'data' requests
-- that's why it was decided not to copy it back into response
-- request_data = cjson.decode(cjson.encode(msg_data)),
}
-- TODO: it is not possible to do a validation of the "data" messages
-- as we don't have schemas for them, once schemas are introduced
-- validation can be added here
local data_handler = (data_handlers or {})[action_name]
if data_handler == nil then
response.status = "error"
response.error = protocol.implementation_errors.data_handler_not_defined
__log.errorf("%s: action handler '%s' is not defined", response.error, action_name)
return __api.send_data_to(src, cjson.encode(response))
end
local data_handler_result, response_data = data_handler.handler(msg_data.data)
response.data = response_data
if data_handler_result then
response.status = "success"
else
response.status = "error"
response.error = protocol.implementation_errors.data_handler_error
response.reason = response_data.reason
__log.errorf("%s: %s", response.error, response.reason)
end
return __api.send_data_to(src, cjson.encode(response))
end,
action = function(src, data, action_name)
local action_data = cjson.decode(data) or {}
-- actions is a set of full action names (module_name.acton_name) that was already performed
@@ -208,7 +265,7 @@ cmodule.start = function(action_handlers, background_process)
end
if cmtype == "quit" then
if cmodule.quit_handler then
cmodule.quit_handler()
cmodule.quit_handler(data)
end
end
if cmtype == "agent_connected" then
@@ -229,8 +286,14 @@ cmodule.start = function(action_handlers, background_process)
if background_process ~= nil then
while not __api.is_close() do
background_process()
__api.await(1000)
local result, await_time = background_process()
if not result then
__log.errorf("module '%s' background process failed it's execution")
break
end
-- default await time is 1 second, which can be changed by a background_process()
await_time = await_time or 1000
__api.await(await_time)
end
else
__api.await(-1)
utils/protocol/protocol.lua
+6 -1
View File
@@ -5,7 +5,10 @@ local protocol = {}
----------------------------------------------
protocol.message_name = {}
protocol.message_name.action_response = "Module.Common.ActionResponse"
protocol.message_name.action_proxied = "Module.Сommon.ActionProxied"
protocol.message_name.action_proxied = "Module.Common.ActionProxied"
protocol.message_name.data_request = "Module.Common.DataRequest"
protocol.message_name.data_response = "Module.Common.DataResponse"
protocol.message_name.data_proxied = "Module.Common.DataProxied"
----------------------------------------------
-- Message types
@@ -37,11 +40,13 @@ protocol.validation_errors.validation_error = "Module.Common.ValidationError"
----------------------------------------------
protocol.implementation_errors = {}
protocol.implementation_errors.action_handler_not_defined = "Module.Common.ActionHandlerNotDefined"
protocol.implementation_errors.data_handler_not_defined = "Module.Common.DataHandlerNotDefined"
----------------------------------------------
-- Business logic errors
----------------------------------------------
protocol.business_logic_errors = {}
protocol.business_logic_errors.action_handler_error = "Module.Common.ActionHandlerError"
protocol.business_logic_errors.data_handler_error = "Module.Common.DataHandlerError"
return protocol
utils/protocol/smodule.lua
+100 -12
View File
@@ -5,11 +5,25 @@ require("engine")
require("protocol/actions_validator")
local smodule = {}
smodule.quit_handler = function() end
--- Module quit trigger handler.
-- @param reason Defines reason for a module stop (agent_stop, module_remove, module_update).
smodule.quit_handler = function(_) end
--- New agent connected handler.
-- @param dst Connected agent token.
smodule.agent_connected_handler = function(_) end
--- Agent disconnected handler.
-- @param dst Disconnected agent token.
smodule.agent_disconnected_handler = function(_) end
--- Module configuration update handler.
-- @param previous_config Previous config object.
-- @param new_config New config object.
smodule.update_config_handler = function(_, _) end
-- TODO: use common shared uuid library
local crc32 = require("crc32")
math.randomseed(crc32(tostring({})))
@@ -37,6 +51,23 @@ smodule.unload_dependencies = function()
collectgarbage("collect")
end
local function create_strict_strings_set(table, key_type)
local result = {}
for key, value in pairs(table or {}) do
if type(key) == "number" then
result[value] = value
result[value:gsub("[.]", "_")] = value
else
result[key] = key
result[key:gsub("[.]", "_")] = key
end
end
setmetatable(result, { __index = function(_, key)
error("unknown " .. key_type .. " '" .. key .. "' requested", 2)
end, })
return result
end
smodule.load_dependencies = function()
local action_config_schema = __config.get_action_config_schema()
local current_event_config = __config.get_current_event_config()
@@ -57,6 +88,10 @@ smodule.load_dependencies = function()
)
smodule.action_validator = CActionsValidator(
fields_schema, action_config_schema)
smodule.actions = create_strict_strings_set(smodule.action_validator.actions, "action")
smodule.events = create_strict_strings_set(smodule.event_engine.event_name_list, "event")
smodule.fields = create_strict_strings_set(smodule.action_validator.fields_validators, "field")
end
-- getting agent ID by dst token and agent type
@@ -102,22 +137,63 @@ smodule.push_event = function(agent_id, event_name, event_data, actions)
end
end
smodule.start = function(action_handlers, data_callback, background_process)
smodule.start = function(action_handlers, data_handlers, background_process)
__api.add_cbs({
data = function(src, data)
local msg_data = cjson.decode(data) or {}
local return_dst = msg_data.__retaddr
msg_data.__cid = msg_data.__cid or make_uuid()
-- If message is a valid response then it need to be proxied back to initial caller
local vxagent_id = get_agent_id_by_dst(src, "VXAgent")
if vxagent_id ~= "" and return_dst ~= nil and return_dst ~= "" then
msg_data.__retaddr = nil
return __api.send_data_to(return_dst, cjson.encode(msg_data))
end
-- msg from browser or external
if data_callback ~= nil then
return data_callback(src, nil)
local action_name = msg_data.name
local response = {
__retaddr = return_dst,
__cid = msg_data.__cid,
__aid = __aid,
__msg_type = protocol.message_name.internal_data_response,
name = action_name,
-- NOTE(mkochegarov): Request data can be quite large for 'data' requests
-- that's why it was decided not to copy it back into response
-- request_data = cjson.decode(cjson.encode(msg_data)),
}
-- TODO: it is not possible to do a validation of the "data" messages
-- as we don't have schemas for them, once schemas are introduced
-- validation can be added here
-- Server module can handle data request on it's own
local data_handler = (data_handlers or {})[action_name]
if data_handler ~= nil then
response.error, response.data = data_handler.handler(msg_data.data)
response.status = (response.error == nil) and "success" or "error"
return __api.send_data_to(src, cjson.encode(response))
end
return false
-- Server module can't handle action so it need to be proxied to the agent
local id, _ = get_agent_id_by_dst(src, "any")
local dst, _ = get_agent_src_by_id(id, "VXAgent")
if dst == "" then
response.status, response.error = "error", protocol.connection_errors.common
return __api.send_data_to(src, cjson.encode(response))
end
__log.debugf("data message '%s' was proxied", action_name)
__api.send_msg_to(src, cjson.encode({
__msg_type = protocol.message_name.data_proxied,
__cid = msg_data.__cid,
name = action_name,
}), protocol.message_type.info)
msg_data.__msg_type = msg_data.__msg_type or protocol.message_name.data_request
msg_data.__retaddr = src
return __api.send_data_to(dst, cjson.encode(msg_data))
end,
action = function(src, data, action_name)
@@ -138,13 +214,14 @@ smodule.start = function(action_handlers, data_callback, background_process)
response.status = "error"
response.error = error
response.reason = reason
__log.errorf("%s: %s", error, reason)
return __api.send_data_to(src, cjson.encode(response))
end
-- Server module can handle action on it's own
local action_handler = (action_handlers or {})[action_name]
if action_handler ~= nil then
response.error, response.response_data = action_handler(action_data.data)
response.error, response.data = action_handler.handler(action_data.data)
response.status = (response.error == nil) and "success" or "error"
return __api.send_data_to(src, cjson.encode(response))
end
@@ -154,6 +231,7 @@ smodule.start = function(action_handlers, data_callback, background_process)
local dst, _ = get_agent_src_by_id(id, "VXAgent")
if dst == "" then
response.status, response.error = "error", protocol.connection_errors.common
__log.errorf("%s: connected agent not found, src: %s", response.error, src)
return __api.send_data_to(src, cjson.encode(response))
else
response.__aid = id
@@ -166,6 +244,7 @@ smodule.start = function(action_handlers, data_callback, background_process)
name = action_name,
}), protocol.message_type.info)
action_data.__msg_type = action_data.__msg_type or protocol.message_name.action_request
action_data.__retaddr = src
return __api.send_action_to(dst, cjson.encode(action_data), action_name)
end,
@@ -182,7 +261,7 @@ smodule.start = function(action_handlers, data_callback, background_process)
end
if cmtype == "quit" then
if smodule.quit_handler then
smodule.quit_handler()
smodule.quit_handler(data)
end
end
if cmtype == "agent_connected" then
@@ -199,14 +278,18 @@ smodule.start = function(action_handlers, data_callback, background_process)
end,
})
smodule.load_dependencies()
__log.infof("module '%s' was started", __config.ctx.name)
if background_process ~= nil then
while not __api.is_close() do
background_process()
__api.await(1000)
local result, await_time = background_process()
if not result then
__log.errorf("module '%s' background process failed it's execution")
break
end
-- default await time is 1 second, which can be changed by a background_process()
await_time = await_time or 1000
__api.await(await_time)
end
else
__api.await(-1)
@@ -219,4 +302,9 @@ smodule.start = function(action_handlers, data_callback, background_process)
return "success"
end
-- NOTE: initial module dependencies are gonna be loaded automatically
-- even before "start" is called, that is done in this way to make it
-- possible to use data from actions/events/fields in module code
smodule.load_dependencies()
return smodule