Add option to install sigdb systemwise and custom user location for additional signatures (#2328)

.gitignore

@@ -124,5 +124,6 @@ subprojects/packagecache/
 subprojects/sdb/
 subprojects/tree-sitter-*/
 subprojects/zlib-*/
+subprojects/sigdb/
 subprojects/libdemangle/
 dist/windows/Output

librz/analysis/meson.build

@@ -299,6 +299,7 @@ pkgconfig_mod.generate(rz_analysis,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/asm/meson.build

@@ -335,6 +335,7 @@ pkgconfig_mod.generate(rz_asm,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/bin/meson.build

@@ -267,6 +267,7 @@ pkgconfig_mod.generate(rz_bin,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/bp/meson.build

@@ -50,6 +50,7 @@ pkgconfig_mod.generate(rz_bp,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/config/meson.build

@@ -31,7 +31,10 @@ pkgconfig_mod.generate(rz_config,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/cons/meson.build

@@ -42,7 +42,10 @@ pkgconfig_mod.generate(rz_cons,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/core/canalysis.c

@@ -5966,26 +5966,55 @@ RZ_API bool rz_core_analysis_everything(RzCore *core, bool experimental, char *d
 	return true;
 }
 
+static int core_sigdb_sorter(const RzSigDBEntry *a, const RzSigDBEntry *b) {
+	return strcmp(a->short_path, b->short_path);
+}
+
+static RzList *core_load_all_signatures_from_sigdb(RzCore *core, bool with_details) {
+	RzList *sysdb = NULL, *userdb = NULL;
+	char *system_sigdb = rz_path_system(RZ_SIGDB);
+	if (RZ_STR_ISNOTEMPTY(system_sigdb)) {
+		sysdb = rz_sign_sigdb_load_database(system_sigdb, with_details);
+	}
+	free(system_sigdb);
+
+	const char *user_sigdb = rz_config_get(core->config, "flirt.sigdb.path");
+	if (RZ_STR_ISEMPTY(user_sigdb)) {
+		return sysdb;
+	} else if (!rz_file_is_directory(user_sigdb)) {
+		RZ_LOG_ERROR("Invalid signature database path (flirt.sigdb.path)\n");
+		return sysdb;
+	} else {
+		userdb = rz_sign_sigdb_load_database(user_sigdb, with_details);
+	}
+	if (sysdb && userdb) {
+		rz_list_join(userdb, sysdb);
+		rz_list_free(sysdb);
+		rz_list_sort(userdb, (RzListComparator)core_sigdb_sorter);
+		sysdb = NULL;
+	}
+	return userdb ? userdb : sysdb;
+}
+
 /**
  * \brief Outputs the list of signatures found in the flirt.sigdb.path
  *
  * \param core The RzCore instance
  */
 RZ_API void rz_core_analysis_sigdb_print(RzCore *core) {
-	const char *sigdb_path = rz_config_get(core->config, "flirt.sigdb.path");
-	if (RZ_STR_ISEMPTY(sigdb_path) || !rz_file_is_directory(sigdb_path)) {
-		RZ_LOG_ERROR("Invalid signature database path (flirt.sigdb.path)\n");
+	RzList *sigdb = core_load_all_signatures_from_sigdb(core, true);
+	if (!sigdb) {
 		return;
 	}
 
 	RzTable *table = rz_table_new();
 	if (!table) {
+		rz_list_free(sigdb);
 		rz_warn_if_reached();
 		return;
 	}
 	rz_table_set_columnsf(table, "ssnsns", "bin", "arch", "bits", "name", "modules", "details");
 
-	RzList *sigdb = rz_sign_sigdb_load_database(sigdb_path, true);
 	RzSigDBEntry *sig = NULL;
 	RzListIter *iter = NULL;
 	ut64 bits, nmods;
@@ -6015,7 +6044,6 @@ RZ_API void rz_core_analysis_sigdb_print(RzCore *core) {
  */
 RZ_API bool rz_core_analysis_sigdb_apply(RzCore *core, int *n_applied, const char *filter) {
 	rz_return_val_if_fail(core, false);
-	const char *sigdb_path = NULL;
 	const char *bin = NULL;
 	const char *arch = NULL;
 	ut64 bits = 32;
@@ -6027,12 +6055,6 @@ RZ_API bool rz_core_analysis_sigdb_apply(RzCore *core, int *n_applied, const cha
 	int n_flags_new, n_flags_old;
 	ut8 arch_id = RZ_FLIRT_SIG_ARCH_ANY;
 
-	sigdb_path = rz_config_get(core->config, "flirt.sigdb.path");
-	if (RZ_STR_ISEMPTY(sigdb_path) || !rz_file_is_directory(sigdb_path)) {
-		RZ_LOG_INFO("Invalid signature database path (flirt.sigdb.path)\n");
-		return false;
-	}
-
 	if (RZ_STR_ISEMPTY(filter)) {
 		obj = core->bin ? rz_bin_cur_object(core->bin) : NULL;
 		if ((!obj || !obj->plugin)) {
@@ -6055,7 +6077,11 @@ RZ_API bool rz_core_analysis_sigdb_apply(RzCore *core, int *n_applied, const cha
 		return false;
 	}
 
-	sigdb = rz_sign_sigdb_load_database(sigdb_path, false);
+	sigdb = core_load_all_signatures_from_sigdb(core, false);
+	if (!sigdb) {
+		return false;
+	}
+
 	n_flags_old = rz_flag_count(core->flags, "flirt");
 	rz_list_foreach (sigdb, iter, sig) {
 		if (RZ_STR_ISEMPTY(filter)) {

librz/core/cconfig.c

@@ -3797,7 +3797,7 @@ RZ_API int rz_core_config_init(RzCore *core) {
 		"pe", "pilot", "srec", "w32run", "zip", "all", "none", NULL);
 	SETB("flirt.sig.deflate", false, "enables/disables FLIRT zlib compression when creating a signature file (available only for .sig files)");
 	SETI("flirt.node.optimize", RZ_FLIRT_NODE_OPTIMIZE_MAX, "FLIRT optimization option when creating a signature file (none: 0, normal: 1, smallest: 2)");
-	SETPREF("flirt.sigdb.path", "", "Rizin sigdb location on the filesystem.");
+	SETPREF("flirt.sigdb.path", "", "Additional user defined rizin sigdb location to load on the filesystem.");
 
 	rz_config_lock(cfg, true);
 	return true;

librz/core/meson.build

@@ -185,6 +185,7 @@ pkgconfig_mod.generate(
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/crypto/meson.build

@@ -70,6 +70,7 @@ pkgconfig_mod.generate(rz_crypto,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/debug/meson.build

@@ -154,6 +154,7 @@ pkgconfig_mod.generate(rz_debug,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/demangler/meson.build

@@ -45,6 +45,7 @@ pkgconfig_mod.generate(rz_demangler,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/diff/meson.build

@@ -30,7 +30,10 @@ pkgconfig_mod.generate(rz_diff,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/egg/meson.build

@@ -55,6 +55,7 @@ pkgconfig_mod.generate(rz_egg,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/flag/meson.build

@@ -32,7 +32,10 @@ pkgconfig_mod.generate(rz_flag,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/hash/meson.build

@@ -66,6 +66,7 @@ pkgconfig_mod.generate(rz_hash,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/il/meson.build

@@ -51,7 +51,10 @@ pkgconfig_mod.generate(rz_il,
     'rz_util',
     'rz_reg'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/include/rz_userconf.h.in

@@ -61,6 +61,7 @@
 #define RZ_DATADIR  "@DATADIR_RZ@"
 #define RZ_SDB      "@SDB@"
 #define RZ_ZIGNS    "@ZIGNS@"
+#define RZ_SIGDB    "@SIGDB@"
 #define RZ_THEMES   "@THEMES@"
 #define RZ_FORTUNES "@FORTUNES@"
 #define RZ_FLAGS    "@FLAGS@"

librz/io/meson.build

@@ -156,6 +156,7 @@ pkgconfig_mod.generate(rz_io,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/lang/meson.build

@@ -50,6 +50,7 @@ pkgconfig_mod.generate(rz_lang,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/magic/meson.build

@@ -44,7 +44,10 @@ pkgconfig_mod.generate(rz_magic,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 pkgconfig_magic_requires = ['rz_magic']
 

librz/main/meson.build

@@ -71,7 +71,10 @@ pkgconfig_mod.generate(rz_main,
     'rz_diff',
     'rz_syscall'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/main/rizin.c

@@ -156,6 +156,7 @@ static int main_help(int line) {
 		char *home_plugins = rz_path_home_prefix(RZ_PLUGINS);
 		char *system_plugins = rz_path_system(RZ_PLUGINS);
 		char *home_zigns = rz_path_home_prefix(RZ_ZIGNS);
+		char *system_sigdb = rz_path_system(RZ_SIGDB);
 		char *dirPrefix = rz_path_prefix(NULL);
 		// clang-format off
 		printf(
@@ -181,6 +182,7 @@ static int main_help(int line) {
 			" RZ_PREFIX    %s\n"
 			" RZ_INCDIR    %s\n"
 			" RZ_LIBDIR    %s\n"
+			" RZ_SIGDB     %s\n"
 			" RZ_LIBEXT    " RZ_LIB_EXT "\n",
 			system_rc,
 			home_rc, home_config_rc, home_config_rcdir,
@@ -193,7 +195,8 @@ static int main_help(int line) {
 			datahome,
 			dirPrefix,
 			incdir,
-			libdir);
+			libdir,
+			system_sigdb);
 		// clang-format on
 		free(datahome);
 		free(incdir);
@@ -208,6 +211,7 @@ static int main_help(int line) {
 		free(home_plugins);
 		free(system_plugins);
 		free(home_zigns);
+		free(system_sigdb);
 		free(dirPrefix);
 	}
 	return 0;
@@ -223,6 +227,7 @@ static int main_print_var(const char *var_name) {
 	char *cachehome = rz_path_home_cache();
 	char *homeplugins = rz_path_home_prefix(RZ_PLUGINS);
 	char *homezigns = rz_path_home_prefix(RZ_ZIGNS);
+	char *sigdbdir = rz_path_system(RZ_SIGDB);
 	char *plugins = rz_path_system(RZ_PLUGINS);
 	char *magicpath = rz_path_system(RZ_SDB_MAGIC);
 	const char *is_portable = RZ_IS_PORTABLE ? "1" : "0";
@@ -235,6 +240,7 @@ static int main_print_var(const char *var_name) {
 		{ "RZ_MAGICPATH", magicpath },
 		{ "RZ_INCDIR", incdir },
 		{ "RZ_LIBDIR", libdir },
+		{ "RZ_SIGDB", sigdbdir },
 		{ "RZ_LIBEXT", RZ_LIB_EXT },
 		{ "RZ_RCONFIGHOME", confighome },
 		{ "RZ_RDATAHOME", datahome },
@@ -270,6 +276,7 @@ static int main_print_var(const char *var_name) {
 	free(cachehome);
 	free(homeplugins);
 	free(homezigns);
+	free(sigdbdir);
 	free(plugins);
 	free(magicpath);
 	free(prefix);

librz/parse/meson.build

@@ -81,6 +81,7 @@ pkgconfig_mod.generate(rz_parse,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/reg/meson.build

@@ -35,6 +35,7 @@ pkgconfig_mod.generate(rz_reg,
   description: 'rizin foundation libraries',
   variables: [
     'plugindir=@0@'.format(rizin_plugins),
+    'datdir=@0@'.format(rizin_datdir_rz),
   ],
 )
 

librz/search/meson.build

@@ -33,7 +33,10 @@ pkgconfig_mod.generate(rz_search,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/signature/meson.build

@@ -52,7 +52,10 @@ pkgconfig_mod.generate(rz_sign,
     'rz_type',
     'rz_flag',
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/signature/sigdb.c

@@ -52,9 +52,9 @@ static bool sigdb_signature_resolve_details(RzSigDBEntry *entry, size_t path_len
 		goto skip_details;
 	}
 
-	buffer = rz_buf_new_slurp(entry->file_path);
+	buffer = rz_buf_new_file(entry->file_path, O_RDONLY, 0);
 	if (!buffer) {
-		RZ_LOG_WARN("sigdb: cannot open .sig file '%s'.\n", entry->file_path);
+		RZ_LOG_WARN("sigdb: cannot open signature file '%s'.\n", entry->file_path);
 		return false;
 	}
 

librz/socket/meson.build

@@ -49,7 +49,10 @@ pkgconfig_mod.generate(rz_socket,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/syscall/meson.build

@@ -27,7 +27,10 @@ pkgconfig_mod.generate(rz_syscall,
   requires: [
     'rz_util'
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/type/meson.build

@@ -54,7 +54,10 @@ pkgconfig_mod.generate(rz_type,
   requires: [
     'rz_util',
   ],
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

librz/util/meson.build

@@ -121,7 +121,10 @@ pkgconfig_mod.generate(rz_util,
   version: rizin_version,
   name: 'rz_util',
   filebase: 'rz_util',
-  description: 'rizin foundation libraries'
+  description: 'rizin foundation libraries',
+  variables: [
+    'datdir=@0@'.format(rizin_datdir_rz),
+  ],
 )
 
 if not is_static_libs_only

meson.build

@@ -195,6 +195,7 @@ if host_machine.system() == 'windows'
   rizin_wwwroot = rizin_datdir / 'www'
   rizin_sdb = rizin_datdir / ''
   rizin_zigns = rizin_datdir / 'zigns'
+  rizin_sigdb = rizin_datdir / 'sigdb'
   rizin_themes = rizin_datdir / 'cons'
   rizin_fortunes = rizin_datdir / 'fortunes'
   rizin_flags = rizin_datdir / 'flag'
@@ -206,6 +207,7 @@ else
   rizin_wwwroot = rizin_datdir_rz / 'www'
   rizin_sdb = rizin_datdir_rz
   rizin_zigns = rizin_datdir_rz / 'zigns'
+  rizin_sigdb = rizin_datdir_rz / 'sigdb'
   rizin_themes = rizin_datdir_rz / 'cons'
   rizin_fortunes = rizin_datdir_rz / 'fortunes'
   rizin_flags = rizin_datdir_rz / 'flag'
@@ -224,6 +226,7 @@ opts_overwrite = [
   'rizin_wwwroot',
   'rizin_sdb',
   'rizin_zigns',
+  'rizin_sigdb',
   'rizin_themes',
   'rizin_fortunes',
   'rizin_flags',
@@ -395,6 +398,7 @@ userconf.set('DATADIR', rizin_datdir)
 userconf.set('WWWROOT', rizin_wwwroot)
 userconf.set('SDB', rizin_sdb)
 userconf.set('ZIGNS', rizin_zigns)
+userconf.set('SIGDB', rizin_sigdb)
 userconf.set('THEMES', rizin_themes)
 userconf.set('FORTUNES', rizin_fortunes)
 userconf.set('FLAGS', rizin_flags)
@@ -669,6 +673,14 @@ endif
 
 subdir('librz')
 
+if get_option('install_sigdb')
+  r = run_command(py3_exe, check_meson_subproject_py, 'sigdb')
+  if r.returncode() == 1 and get_option('subprojects_check')
+    error('Subprojects are not updated. Please run `git clean -dxff subprojects/` to delete all local subprojects directories. If you want to compile against current subprojects then set option `subprojects_check=false`.')
+  endif
+  subproject('sigdb', default_options: ['sigdb_path=@0@'.format(get_option('prefix') / rizin_sigdb)])
+endif
+
 cli_option = get_option('cli')
 cli_enabled = cli_option.auto() ? not meson.is_subproject() : cli_option.enabled()
 if cli_enabled
@@ -704,6 +716,7 @@ summary({
   'wwwroot': rizin_wwwroot,
   'sdb': rizin_sdb,
   'zigns': rizin_zigns,
+  'sigdb': rizin_sigdb,
   'themes': rizin_themes,
   'fortunes': rizin_fortunes,
   'flags': rizin_flags,
@@ -714,6 +727,7 @@ summary({
 
 summary({
   'GPL code': get_option('use_gpl'),
+  'Install sigdb': get_option('install_sigdb'),
   'Swift demangler': get_option('use_swift_demangler'),
   'Debugger enabled': has_debugger,
   'Capstone version': capstone_dep.version(),

meson_options.txt

@@ -10,6 +10,7 @@ option('portable', type: 'boolean', value: false, description: 'Make rizin insta
 option('rizin_wwwroot', type: 'string', value: '', description: 'Install path for www files')
 option('rizin_sdb', type: 'string', value: '', description: '')
 option('rizin_zigns', type: 'string', value: '', description: 'Install path for zignatures files')
+option('rizin_sigdb', type: 'string', value: '', description: 'Install path for sigdb files')
 option('rizin_themes', type: 'string', value: '', description: 'Install path for themes (colors, etc.)')
 option('rizin_fortunes', type: 'string', value: '', description: 'Install path for fortunes displayed when starting')
 option('rizin_flags', type: 'string', value: '', description: 'Install path for flags tags')
@@ -34,6 +35,7 @@ option('use_sys_libuv', type: 'feature', value: 'auto', description: 'Whether to
 option('use_libuv', type: 'boolean', value: true, description: 'If true, libuv is used to handle remote features')
 option('use_swift_demangler', type: 'boolean', value: true, description: 'If false, disables the swift demangler')
 option('use_gpl', type: 'boolean', value: true, description: 'Set to false when you want to disable gpl code')
+option('install_sigdb', type: 'boolean', value: false, description: 'Downloads and installs rizin sigdb')
 option('debugger', type: 'boolean', value: true)
 
 option('enable_tests', type: 'boolean', value: true, description: 'Build unit tests in test/unit')

subprojects/sigdb.wrap

@@ -0,0 +1,3 @@
+[wrap-git]
+url = https://github.com/rizinorg/sigdb.git
+revision = main

sys/check_meson_subproject.py

@@ -48,7 +48,8 @@ try:
                     os.path.join(subproject_git_dir, "HEAD"), "r", encoding="utf8"
                 ) as f:
                     head = f.read().strip()
-                if head != revision:
+                # when using a branch name, head is 'refs/heads/<branch>'
+                if head != revision and revision not in head:
                     sys.exit(1)
 
         if not patch_directory: