From c3953d64d6e57eca052bae596042aa6401cace1b Mon Sep 17 00:00:00 2001 From: pancake Date: Thu, 20 May 2021 00:22:20 +0200 Subject: [PATCH] Use gperf on anal/d and improve build and checks ##anal * Add --without-gperf configure flag * Update sdb to support gperf.foreach and faster ls.sort() * Support cc and types sdb gperfs * add r_str_newvf * Honor HAVE_GPERF in more places * Add CI job to build and test cmds with gperf --- .github/workflows/ci.yml | 21 +- configure | 11 +- configure.acr | 7 +- libr/anal/Makefile | 44 ++- libr/anal/anal.c | 17 +- libr/anal/anplugs.c | 116 +++++++ libr/anal/cc.c | 4 +- libr/anal/d/Makefile | 18 +- libr/anal/meson.build | 1 + libr/anal/type.c | 1 + libr/asm/Makefile | 4 + libr/asm/d/Makefile | 2 + libr/config.h.in | 3 + libr/core/cbin.c | 86 ++--- libr/core/cfile.c | 2 +- libr/core/cmd_type.c | 4 +- libr/include/r_anal.h | 2 + libr/include/r_userconf.h.acr | 6 + libr/include/r_util/r_str.h | 2 + libr/parse/code.c | 7 +- libr/rules.mk | 6 +- libr/util/Makefile | 6 + libr/util/charset.c | 1 + libr/util/d/Makefile | 2 + libr/util/str.c | 13 + meson.build | 7 + shlr/sdb/src/ls.c | 35 +- shlr/sdb/src/main.c | 623 +++++++++++++++++++++------------- shlr/sdb/src/sdb.c | 3 + shlr/sdb/src/sdb.h | 5 + 30 files changed, 756 insertions(+), 303 deletions(-) create mode 100644 libr/anal/anplugs.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 304445ee0e..ae5f5591fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,23 @@ on: jobs: + build-acr-gperf: + name: linux-acr-gperf + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install meson ninja r2pipe wget + sudo apt install gperf + - name: Installing with symlinks + run: | + export CFLAGS="-O2 -Werror -Wno-unused-result -Wno-stringop-truncation" + sys/install.sh + - name: Running tests + run: r2r test/db/cmd build-resymlink: name: linux-acr-resymlink runs-on: ubuntu-20.04 @@ -280,7 +297,7 @@ jobs: # with: # path: rpmbuild/RPMS/*/*.rpm linux-asan-fuzz: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 continue-on-error: true if: contains(github.ref, 'master') || contains(github.ref, 'ci-') steps: @@ -306,7 +323,7 @@ jobs: export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 make -C test fuzz-tests linux-test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 # if: contains(github.ref, 'ci-') steps: - name: Checkout diff --git a/configure b/configure index 10c08cf3bb..ceb8f5d86b 100755 --- a/configure +++ b/configure @@ -24,6 +24,7 @@ LOADLIBS=1 WANT_DYLINK=1 HAVE_FORK=1 WANT_PTRACE_WRAP=1 +WANT_GPERF=1 WITH_LIBR=0 USE_CS5=0 USE_CS4=0 @@ -185,6 +186,7 @@ Optional Features: --without-dylink disable support for dynamic loading of plugins --without-fork disable fork --without-ptrace-wrap build ptrace-wrap support needed for the iaito debugger on Linux + --without-gperf do not statically link the sdb databases even if gperf is found --with-libr build libr.a and libr.dylib --with-capstone5 build next branch of capstone5 (default) --with-capstone4 build v4 branch of capstone @@ -273,7 +275,7 @@ echo "LANGS: c" echo "REQUIRED: libdl" echo "OPTIONAL: libmagic libz libzip libxxhash libssl liblibuv>=1.0.0" echo "PKG-CONFIG: capstone openssl libuv" -echo "FLAGS: --disable-debugger --with-sysmagic --disable-loadlibs --without-dylink --without-fork --without-ptrace-wrap --with-libr --with-capstone5 --with-capstone4 --with-syscapstone --with-syszip --with-sysxxhash --without-gpl --with-openssl --without-libuv --with-rpath --with-compiler=gcc --with-ostype=auto --with-libversion=xxx --without-jemalloc --with-checks-level=2" +echo "FLAGS: --disable-debugger --with-sysmagic --disable-loadlibs --without-dylink --without-fork --without-ptrace-wrap --without-gperf --with-libr --with-capstone5 --with-capstone4 --with-syscapstone --with-syszip --with-sysxxhash --without-gpl --with-openssl --without-libuv --with-rpath --with-compiler=gcc --with-ostype=auto --with-libversion=xxx --without-jemalloc --with-checks-level=2" exit 0 ;; --cache-file) @@ -329,6 +331,7 @@ echo "FLAGS: --disable-debugger --with-sysmagic --disable-loadlibs --without "--without-dylink") WANT_DYLINK="0"; ;; "--without-fork") HAVE_FORK="0"; ;; "--without-ptrace-wrap") WANT_PTRACE_WRAP="0"; ;; +"--without-gperf") WANT_GPERF="0"; ;; "--with-libr") WITH_LIBR="1"; ;; "--with-capstone5") USE_CS5="1"; ;; "--with-capstone4") USE_CS4="1"; ;; @@ -361,7 +364,7 @@ parse_options "$1" shift done -ENVWORDS="MANDIR INFODIR LIBDIR INCLUDEDIR LOCALSTATEDIR ETCDIR SYSCONFDIR DATADIR DOCDIR LIBEXECDIR SBINDIR BINDIR EPREFIX PREFIX SPREFIX TARGET HOST BUILD INSTALL INSTALL_LIB INSTALL_MAN INSTALL_PROGRAM INSTALL_PROGRAM_STRIP INSTALL_DIR INSTALL_SCRIPT INSTALL_DATA HOST_OS HOST_CPU BUILD_OS BUILD_CPU TARGET_OS TARGET_CPU VERSION VERSION_MAJOR VERSION_MINOR VERSION_PATCH VERSION_NUMBER PKGNAME VPATH CONTACT CONTACT_NAME CONTACT_MAIL CC CFLAGS CPPFLAGS LDFLAGS HAVE_LANG_C DEBUGGER HAVE_LIB_DL DL_LIBS PKGCONFIG HAVE_PATCH PATCH HAVE_GIT GIT HAVE_GPERF GPERF HAVE_LIB_MAGIC USE_MAGIC USE_LIB_MAGIC LIBMAGIC LOADLIBS WANT_DYLINK HAVE_FORK WANT_PTRACE_WRAP WITH_LIBR USE_CS5 USE_CS4 WITH_CAPSTONE CAPSTONE_CFLAGS CAPSTONE_LDFLAGS HAVE_PKGCFG_CAPSTONE USE_CAPSTONE HAVE_LIB_Z HAVE_LIB_ZIP USE_ZIP USE_LIB_ZIP LIBZIP HAVE_LIB_XXHASH USE_XXHASH USE_LIB_XXHASH LIBXXHASH WITH_GPL HAVE_DECL_ADDR_NO_RANDOMIZE HAVE_DECL___GLIBC__ HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET HAVE_CLOCK_NANOSLEEP HAVE_SIGACTION HAVE_CLOCK_GETTIME CLOCK_LDFLAGS SUPPORT_GNU99 HAVE_LIB_GMP HAVE_LIB_SSL SSL_CFLAGS SSL_LDFLAGS HAVE_PKGCFG_OPENSSL HAVE_OPENSSL WANT_OPENSSL HAVE_LIBUV_VERSION_1_0_0 LIBUV_CFLAGS LIBUV_LDFLAGS HAVE_PKGCFG_LIBUV HAVE_LIBUV WANT_LIBUV USE_RPATH USERCC USEROSTYPE LIBVERSION HAVE_JEMALLOC HAVE_PTRACE USE_PTRACE_WRAP R_CHECKS_LEVEL" +ENVWORDS="MANDIR INFODIR LIBDIR INCLUDEDIR LOCALSTATEDIR ETCDIR SYSCONFDIR DATADIR DOCDIR LIBEXECDIR SBINDIR BINDIR EPREFIX PREFIX SPREFIX TARGET HOST BUILD INSTALL INSTALL_LIB INSTALL_MAN INSTALL_PROGRAM INSTALL_PROGRAM_STRIP INSTALL_DIR INSTALL_SCRIPT INSTALL_DATA HOST_OS HOST_CPU BUILD_OS BUILD_CPU TARGET_OS TARGET_CPU VERSION VERSION_MAJOR VERSION_MINOR VERSION_PATCH VERSION_NUMBER PKGNAME VPATH CONTACT CONTACT_NAME CONTACT_MAIL CC CFLAGS CPPFLAGS LDFLAGS HAVE_LANG_C DEBUGGER HAVE_LIB_DL DL_LIBS PKGCONFIG HAVE_PATCH PATCH HAVE_GIT GIT HAVE_GPERF GPERF HAVE_LIB_MAGIC USE_MAGIC USE_LIB_MAGIC LIBMAGIC LOADLIBS WANT_DYLINK HAVE_FORK WANT_PTRACE_WRAP WANT_GPERF WITH_LIBR USE_CS5 USE_CS4 WITH_CAPSTONE CAPSTONE_CFLAGS CAPSTONE_LDFLAGS HAVE_PKGCFG_CAPSTONE USE_CAPSTONE HAVE_LIB_Z HAVE_LIB_ZIP USE_ZIP USE_LIB_ZIP LIBZIP HAVE_LIB_XXHASH USE_XXHASH USE_LIB_XXHASH LIBXXHASH WITH_GPL HAVE_DECL_ADDR_NO_RANDOMIZE HAVE_DECL___GLIBC__ HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET HAVE_CLOCK_NANOSLEEP HAVE_SIGACTION HAVE_CLOCK_GETTIME CLOCK_LDFLAGS SUPPORT_GNU99 HAVE_LIB_GMP HAVE_LIB_SSL SSL_CFLAGS SSL_LDFLAGS HAVE_PKGCFG_OPENSSL HAVE_OPENSSL WANT_OPENSSL HAVE_LIBUV_VERSION_1_0_0 LIBUV_CFLAGS LIBUV_LDFLAGS HAVE_PKGCFG_LIBUV HAVE_LIBUV WANT_LIBUV USE_RPATH USERCC USEROSTYPE LIBVERSION HAVE_JEMALLOC HAVE_PTRACE USE_PTRACE_WRAP R_CHECKS_LEVEL" create_environ @@ -542,6 +545,8 @@ LIBMAGIC="-lmagic" else USE_LIB_MAGIC="0" LIBMAGIC=""; fi +if [ "$WANT_GPERF" = "0" ]; then +HAVE_GPERF="0"; fi if [ "$WITH_CAPSTONE" = "1" ]; then if [ -z "${PKGCONFIG}" ]; then pkg-config --version >/dev/null 2>&1 ; if [ 0 = 0 ]; then PKGCONFIG=pkg-config ; else PKGCONFIG=pkgconf ; fi; fi type ${PKGCONFIG} > /dev/null 2>&1 || echo "ERROR: Cannot find valid PKGCONFIG, pkg-config or pkgconf in PATH" @@ -821,7 +826,7 @@ do_remove if [ "$QUIET" = 0 ]; then echo echo "Final report:" -for A in R_CHECKS_LEVEL PREFIX HAVE_LIB_GMP HAVE_OPENSSL HAVE_LIBUV USE_CAPSTONE HAVE_PTRACE USE_PTRACE_WRAP HAVE_FORK VERSION USE_LIB_ZIP USE_LIB_MAGIC USE_LIB_XXHASH DEBUGGER CC USERCC HAVE_ARC4RANDOM_UNIFORM PKGCONFIG HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET WANT_DYLINK USEROSTYPE LIBVERSION BUILD HOST TARGET CFLAGS LDFLAGS ; do +for A in R_CHECKS_LEVEL PREFIX HAVE_LIB_GMP HAVE_OPENSSL HAVE_LIBUV USE_CAPSTONE HAVE_PTRACE USE_PTRACE_WRAP HAVE_FORK VERSION USE_LIB_ZIP USE_LIB_MAGIC USE_LIB_XXHASH DEBUGGER CC USERCC HAVE_ARC4RANDOM_UNIFORM PKGCONFIG HAVE_GPERF HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET WANT_DYLINK USEROSTYPE LIBVERSION BUILD HOST TARGET CFLAGS LDFLAGS ; do eval VAL="\$${A}" [ -z "${VAL}" ] && VAL="(null)" echo " - ${A} = ${VAL}" diff --git a/configure.acr b/configure.acr index 0b9e30141e..b361f5d309 100644 --- a/configure.acr +++ b/configure.acr @@ -31,6 +31,11 @@ ARG_DISABLE LOADLIBS loadlibs disable loading plugins ; ARG_WITHOUT WANT_DYLINK dylink disable support for dynamic loading of plugins ; ARG_WITHOUT HAVE_FORK fork disable fork ; ARG_WITHOUT WANT_PTRACE_WRAP ptrace-wrap build ptrace-wrap support needed for the iaito debugger on Linux ; +ARG_WITHOUT WANT_GPERF gperf do not statically link the sdb databases even if gperf is found ; + +IFNOT WANT_GPERF { + HAVE_GPERF = 0 ; +} ARG_WITH WITH_LIBR libr build libr.a and libr.dylib ; @@ -239,7 +244,7 @@ ARG_WITH R_CHECKS_LEVEL=2 checks-level value between 0 and 3 to enable different REPORT R_CHECKS_LEVEL PREFIX HAVE_LIB_GMP HAVE_OPENSSL HAVE_LIBUV USE_CAPSTONE HAVE_PTRACE USE_PTRACE_WRAP HAVE_FORK VERSION USE_LIB_ZIP USE_LIB_MAGIC USE_LIB_XXHASH DEBUGGER - CC USERCC HAVE_ARC4RANDOM_UNIFORM PKGCONFIG + CC USERCC HAVE_ARC4RANDOM_UNIFORM PKGCONFIG HAVE_GPERF HAVE_EXPLICIT_BZERO HAVE_EXPLICIT_MEMSET WANT_DYLINK USEROSTYPE LIBVERSION BUILD HOST TARGET CFLAGS LDFLAGS ; diff --git a/libr/anal/Makefile b/libr/anal/Makefile index 3fc0761308..7e8c55371b 100644 --- a/libr/anal/Makefile +++ b/libr/anal/Makefile @@ -20,7 +20,11 @@ LDFLAGS+=$(LINK) .PHONY: all plugins libs ${EXTRA_CLEAN} -all: plugins +# all: plugins +main: + $(MAKE) -C d + $(MAKE) all + plugins: ${LIBSO} ${LIBAR} @${MAKE} -C p all @@ -34,13 +38,49 @@ OBJLIBS+=cond.o value.o cc.o class.o diff.o type.o type_pdb.o dwarf_process.o OBJLIBS+=hint.o anal.o data.o xrefs.o esil.o sign.o esil_plugin.o OBJLIBS+=esil_handler.o switch.o cycles.o esil_dfg.o esil_cfg.o OBJLIBS+=esil_stats.o esil_trace.o flirt.o labels.o -OBJLIBS+=pin.o vtable.o rtti.o codemeta.o +OBJLIBS+=pin.o vtable.o rtti.o codemeta.o anplugs.o OBJLIBS+=rtti_msvc.o rtti_itanium.o jmptbl.o function.o ASMOBJS+=$(LTOP)/asm/arch/xtensa/gnu/xtensa-modules.o ASMOBJS+=$(LTOP)/asm/arch/xtensa/gnu/xtensa-isa.o ASMOBJS+=$(LTOP)/asm/arch/xtensa/gnu/elf32-xtensa.o OBJS=${STATIC_OBJS} ${OBJLIBS} ${ASMOBJS} +ifeq ($(HAVE_GPERF),1) + +OBJS+=d/cc-arm-16.o +OBJS+=d/cc-arm-32.o +OBJS+=d/cc-arm-64.o +OBJS+=d/cc-avr-8.o +OBJS+=d/cc-hexagon-32.o +OBJS+=d/cc-m68k-32.o +OBJS+=d/cc-mips-32.o +OBJS+=d/cc-mips-64.o +OBJS+=d/cc-ppc-32.o +OBJS+=d/cc-ppc-64.o +OBJS+=d/cc-riscv-64.o +OBJS+=d/cc-s390-64.o +OBJS+=d/cc-sparc-32.o +OBJS+=d/cc-v850-32.o +OBJS+=d/cc-x86-16.o +OBJS+=d/cc-x86-32.o +OBJS+=d/cc-x86-64.o +OBJS+=d/cc-xtensa-32.o +OBJS+=d/spec.o +OBJS+=d/types-16.o +OBJS+=d/types-32.o +OBJS+=d/types-64.o +OBJS+=d/types-android.o +OBJS+=d/types-arm-ios-16.o +OBJS+=d/types-arm-ios-32.o +OBJS+=d/types-arm-ios-64.o +OBJS+=d/types-darwin.o +OBJS+=d/types-linux.o +OBJS+=d/types-x86-macos-64.o +#OBJS+=d/types-windows.o +#OBJS+=d/types-x86-windows-32.o +#OBJS+=d/types-x86-windows-64.o +OBJS+=d/types.o +endif test tests: tests-esil diff --git a/libr/anal/anal.c b/libr/anal/anal.c index b91bdcae00..61aaa63e96 100644 --- a/libr/anal/anal.c +++ b/libr/anal/anal.c @@ -1,4 +1,4 @@ -/* radare - LGPL - Copyright 2009-2020 - pancake, nibble */ +/* radare - LGPL - Copyright 2009-2021 - pancake, nibble */ #include #include @@ -285,8 +285,19 @@ static void sdb_concat_by_path(Sdb *s, const char *path) { R_API bool r_anal_set_os(RAnal *anal, const char *os) { Sdb *types = anal->sdb_types; const char *dir_prefix = r_sys_prefix (NULL); - const char *dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s.sdb"), - dir_prefix, os); + SdbGperf *gp = r_anal_get_gperf_types (os); + if (gp) { + Sdb *gd = sdb_new0 (); + sdb_open_gperf (gd, gp); + sdb_reset (anal->sdb_types); + sdb_merge (anal->sdb_types, gd); + sdb_close (gd); + sdb_free (gd); + return r_anal_set_triplet (anal, os, NULL, -1); + } + + char *ff = r_str_newf ("types-%s.sdb", os); + char *dbpath = r_file_new (dir_prefix, R2_SDB_FCNSIGN, ff); if (r_file_exists (dbpath)) { sdb_concat_by_path (types, dbpath); } diff --git a/libr/anal/anplugs.c b/libr/anal/anplugs.c new file mode 100644 index 0000000000..2d36a86665 --- /dev/null +++ b/libr/anal/anplugs.c @@ -0,0 +1,116 @@ +/* radare - LGPL - Copyright 2021 - pancake */ + +#include +#include +#include +#include + +#if HAVE_GPERF +extern SdbGperf gperf_cc_arm_16; +extern SdbGperf gperf_cc_arm_32; +extern SdbGperf gperf_cc_arm_64; +extern SdbGperf gperf_cc_avr_8; +extern SdbGperf gperf_cc_hexagon_32; +extern SdbGperf gperf_cc_m68k_32; +extern SdbGperf gperf_cc_mips_32; +extern SdbGperf gperf_cc_mips_64; +extern SdbGperf gperf_cc_ppc_32; +extern SdbGperf gperf_cc_ppc_64; +extern SdbGperf gperf_cc_riscv_64; +extern SdbGperf gperf_cc_s390_64; +extern SdbGperf gperf_cc_sparc_32; +extern SdbGperf gperf_cc_v850_32; +extern SdbGperf gperf_cc_x86_16; +extern SdbGperf gperf_cc_x86_32; +extern SdbGperf gperf_cc_x86_64; +extern SdbGperf gperf_cc_xtensa_32; +extern SdbGperf gperf_spec; +extern SdbGperf gperf_types_16; +extern SdbGperf gperf_types_32; +extern SdbGperf gperf_types_64; +extern SdbGperf gperf_types_android; +extern SdbGperf gperf_types_arm_ios_16; +extern SdbGperf gperf_types_arm_ios_32; +extern SdbGperf gperf_types_arm_ios_64; +extern SdbGperf gperf_types_darwin; +extern SdbGperf gperf_types_linux; +extern SdbGperf gperf_types_x86_macos_64; +extern SdbGperf gperf_types; +// #OBJS+=d/types_windows.o +// #OBJS+=d/types_x86_windows_32.o +// #OBJS+=d/types_x86_windows_64.o + +static const SdbGperf *gperfs_cc[] = { + &gperf_cc_arm_16, + &gperf_cc_arm_32, + &gperf_cc_arm_64, + &gperf_cc_avr_8, + &gperf_cc_hexagon_32, + &gperf_cc_m68k_32, + &gperf_cc_mips_32, + &gperf_cc_mips_64, + &gperf_cc_ppc_32, + &gperf_cc_ppc_64, + &gperf_cc_riscv_64, + &gperf_cc_s390_64, + &gperf_cc_sparc_32, + &gperf_cc_v850_32, + &gperf_cc_x86_16, + &gperf_cc_x86_32, + &gperf_cc_x86_64, + &gperf_cc_xtensa_32, + NULL +}; +static const SdbGperf *gperfs_types[] = { + &gperf_spec, + &gperf_types_16, + &gperf_types_32, + &gperf_types_64, + &gperf_types_android, + &gperf_types_arm_ios_16, + &gperf_types_arm_ios_32, + &gperf_types_arm_ios_64, + &gperf_types_darwin, + &gperf_types_linux, + &gperf_types_x86_macos_64, + &gperf_types, + NULL +}; + +R_API SdbGperf *r_anal_get_gperf_cc(const char *k) { + SdbGperf **gp = (SdbGperf**)gperfs_cc; + while (*gp) { + SdbGperf *g = *gp; + if (!strcmp (k, g->name)) { + return *gp; + } + gp++; + } + return NULL; +} + +R_API SdbGperf *r_anal_get_gperf_types(const char *k) { + SdbGperf **gp = (SdbGperf**)gperfs_types; + char *s = strdup (k); + r_str_replace_char (s, '-', '_'); + while (*gp) { + SdbGperf *g = *gp; + if (!strcmp (s, g->name)) { + free (s); + return *gp; + } + gp++; + } + free (s); + return NULL; +} +#else +R_API SdbGperf *r_anal_get_gperf_cc(const char *k) { + return NULL; +} + +R_API SdbGperf *r_anal_get_gperf_types(const char *k) { + return NULL; +} +#endif + diff --git a/libr/anal/cc.c b/libr/anal/cc.c index 67c75eb022..35c811cdd6 100644 --- a/libr/anal/cc.c +++ b/libr/anal/cc.c @@ -185,7 +185,7 @@ R_API void r_anal_cc_set_self(RAnal *anal, const char *convention, const char *s return; } RStrBuf sb; - sdb_set (anal->sdb_cc, r_strbuf_initf (&sb, "cc.%s.self", convention), self, 0); + sdb_set (DB, r_strbuf_initf (&sb, "cc.%s.self", convention), self, 0); r_strbuf_fini (&sb); } @@ -202,7 +202,7 @@ R_API void r_anal_cc_set_error(RAnal *anal, const char *convention, const char * return; } RStrBuf sb; - sdb_set (anal->sdb_cc, r_strbuf_initf (&sb, "cc.%s.error", convention), error, 0); + sdb_set (DB, r_strbuf_initf (&sb, "cc.%s.error", convention), error, 0); r_strbuf_fini (&sb); } diff --git a/libr/anal/d/Makefile b/libr/anal/d/Makefile index 609e9e8056..631bf11687 100755 --- a/libr/anal/d/Makefile +++ b/libr/anal/d/Makefile @@ -6,9 +6,6 @@ F+= types-32 F+= types-64 F+= types-android F+= types-darwin -F+= types-windows -F+= types-x86-windows-32 -F+= types-x86-windows-64 F+= types-linux F+= types-x86-macos-64 F+= types-arm-ios-16 @@ -34,6 +31,11 @@ F+= cc-hexagon-32 F+= cc-v850-32 F+= cc-s390-64 +# those are extremely large and make GNU/gperf take forever +F+= types-windows +F+= types-x86-windows-32 +F+= types-x86-windows-64 + WIN_TYPES= WIN_TYPES+= bcrypt WIN_TYPES+= cfgmgr32 @@ -99,16 +101,22 @@ ifneq ($(SILENT),) @rm -f $@ @cat $< $(EXTRA_FILES) | ${SDB} $@ = @test -f $@ +ifeq ($(HAVE_GPERF),1) + -if [ -z "`echo $@| grep windows`" ]; then $(SDB) -C $@ ; fi +endif else rm -f $@ cat $< $(EXTRA_FILES) | ${SDB} $@ = test -f $@ +ifeq ($(HAVE_GPERF),1) + -if [ -z "`echo $@| grep windows`" ]; then $(SDB) -C $@ ; fi +endif endif clean: - rm -f *.sdb + rm -f *.sdb *.c -${SDB}: +$(SDB): @echo "Cannot find ${SDB}" @false diff --git a/libr/anal/meson.build b/libr/anal/meson.build index cca6bb5f8c..4958ba9e30 100644 --- a/libr/anal/meson.build +++ b/libr/anal/meson.build @@ -1,5 +1,6 @@ r_anal_sources = [ 'anal.c', + 'anplugs.c', 'bb.c', 'block.c', 'function.c', diff --git a/libr/anal/type.c b/libr/anal/type.c index c69cc6863a..52ec15a475 100644 --- a/libr/anal/type.c +++ b/libr/anal/type.c @@ -53,6 +53,7 @@ R_API void r_anal_remove_parsed_type(RAnal *anal, const char *name) { } } +// RENAME TO r_anal_types_save(); // parses the string and imports the types R_API void r_anal_save_parsed_type(RAnal *anal, const char *parsed) { r_return_if_fail (anal && parsed); diff --git a/libr/asm/Makefile b/libr/asm/Makefile index 7d5999431a..6e61db2361 100644 --- a/libr/asm/Makefile +++ b/libr/asm/Makefile @@ -24,7 +24,9 @@ SDB=$(SDB_PATH)/sdb gperfs: PATH=$(SDB_PATH):$$PATH $(MAKE) -C d all +ifeq ($(HAVE_GPERF),1) PATH=$(SDB_PATH):$$PATH cd d && for a in *.sdb ; do test $$a -nt `echo $$a.c|sed -e 's,.sdb,,'`; if [ $$? = 0 ]; then $(SDB) -C $$a ; fi ; done +endif plugins: ${LIBSO} ${LIBAR} $(MAKE) -C d all @@ -47,6 +49,7 @@ STATIC_OBJS=$(subst ..,p/..,$(subst asm_,p/asm_,$(STATIC_OBJ))) OBJS=${STATIC_OBJS} asm.o acode.o aop.o binutils_as.o aplugs.o OBJS+=${SHARED2_OBJ} +ifeq ($(HAVE_GPERF),1) OBJS+=d/z80.o OBJS+=d/6502.o OBJS+=d/i4004.o @@ -72,5 +75,6 @@ OBJS+=d/arm.o OBJS+=d/msp430.o OBJS+=d/propeller.o OBJS+=d/pic18c.o +endif include $(TOP)/libr/rules.mk diff --git a/libr/asm/d/Makefile b/libr/asm/d/Makefile index 397db6f10a..d050f9588b 100644 --- a/libr/asm/d/Makefile +++ b/libr/asm/d/Makefile @@ -13,7 +13,9 @@ all: ${F_SDB} %.sdb:%.sdb.txt ${SDB} $@ = < $< test -f $@ +ifeq ($(HAVE_GPERF),1) ${SDB} -C $@ +endif clean: rm -f *.sdb *.c *.h *.gperf a.out diff --git a/libr/config.h.in b/libr/config.h.in index bf49aa75b3..412e581ce1 100644 --- a/libr/config.h.in +++ b/libr/config.h.in @@ -1,6 +1,9 @@ #ifndef _INCLUDE_CONFIG_H_ #define _INCLUDE_CONFIG_H_ +#ifndef HAVE_GPERF +#define HAVE_GPERF 0 +#endif #define R_DEBUG 0 #define R_RTDEBUG 1 diff --git a/libr/core/cbin.c b/libr/core/cbin.c index e0a78e72e4..721eaabf5d 100644 --- a/libr/core/cbin.c +++ b/libr/core/cbin.c @@ -683,9 +683,32 @@ static void sdb_concat_by_path(Sdb *s, const char *path) { sdb_free (db); } +static void load_types_from(RCore *core, const char *fmt, ...) { + const char *dir_prefix = r_config_get (core->config, "dir.prefix"); + va_list ap; + va_start (ap, fmt); + char *s = r_str_newvf (fmt, ap); + SdbGperf *gp = r_anal_get_gperf_types (s); + if (gp) { + Sdb *gd = sdb_new0 (); + sdb_open_gperf (gd, gp); + sdb_reset (core->anal->sdb_types); + sdb_merge (core->anal->sdb_types, gd); + sdb_close (gd); + sdb_free (gd); + } else { + char *dbpath = r_str_newf ("%s/%s/%s.sdb", dir_prefix, R2_SDB_FCNSIGN, s); + if (r_file_exists (dbpath)) { + sdb_concat_by_path (core->anal->sdb_types, dbpath); + } + free (dbpath); + } + free (s); + va_end (ap); +} + R_API void r_core_anal_type_init(RCore *core) { r_return_if_fail (core && core->anal); - const char *dir_prefix = r_config_get (core->config, "dir.prefix"); int bits = core->rasm->bits; Sdb *types = core->anal->sdb_types; // make sure they are empty this is initializing @@ -694,45 +717,14 @@ R_API void r_core_anal_type_init(RCore *core) { const char *os = r_config_get (core->config, "asm.os"); // spaguetti ahead - const char *dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types.sdb"), dir_prefix); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s.sdb"), - dir_prefix, anal_arch); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s.sdb"), - dir_prefix, os); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%d.sdb"), - dir_prefix, bits); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s-%d.sdb"), - dir_prefix, os, bits); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s-%d.sdb"), - dir_prefix, anal_arch, bits); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s-%s.sdb"), - dir_prefix, anal_arch, os); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } - dbpath = sdb_fmt (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "types-%s-%s-%d.sdb"), - dir_prefix, anal_arch, os, bits); - if (r_file_exists (dbpath)) { - sdb_concat_by_path (types, dbpath); - } + load_types_from (core, "types"); + load_types_from (core, "types-%s", anal_arch); + load_types_from (core, "types-%s", os); + load_types_from (core, "types-%d", bits); + load_types_from (core, "types-%s-%d", os, bits); + load_types_from (core, "types-%s-%d", anal_arch, bits); + load_types_from (core, "types-%s-%s", anal_arch, os); + load_types_from (core, "types-%s-%s-%d", anal_arch, os, bits); } R_API void r_core_anal_cc_init(RCore *core) { @@ -743,6 +735,20 @@ R_API void r_core_anal_cc_init(RCore *core) { if (!anal_arch) { return; } +#if HAVE_GPERF + char *k = r_str_newf ("cc_%s_%d", anal_arch, bits); + SdbGperf *gp = r_anal_get_gperf_cc (k); + free (k); + if (gp) { + Sdb *gd = sdb_new0 (); + sdb_open_gperf (gd, gp); + sdb_reset (core->anal->sdb_cc); + sdb_merge (core->anal->sdb_cc, gd); + sdb_close (gd); + sdb_free (gd); + return; + } +#endif char *dbpath = r_str_newf (R_JOIN_3_PATHS ("%s", R2_SDB_FCNSIGN, "cc-%s-%d.sdb"), dir_prefix, anal_arch, bits); char *dbhomepath = r_str_newf (R_JOIN_3_PATHS ("~", R2_HOME_SDB_FCNSIGN, "cc-%s-%d.sdb"), diff --git a/libr/core/cfile.c b/libr/core/cfile.c index 06ba9682b9..d56993d1b1 100644 --- a/libr/core/cfile.c +++ b/libr/core/cfile.c @@ -434,7 +434,7 @@ static int r_core_file_do_load_for_io_plugin(RCore *r, ut64 baseaddr, ut64 loada } binfile = r_bin_cur (r->bin); if (r_core_bin_set_env (r, binfile)) { - if (!r->anal->sdb_cc->path) { + if (!sdb_const_get (r->anal->sdb_cc, "default.cc", 0)) { R_LOG_WARN ("No calling convention defined for this file, analysis may be inaccurate.\n"); } } diff --git a/libr/core/cmd_type.c b/libr/core/cmd_type.c index 255c8bdd57..eef27a06c2 100644 --- a/libr/core/cmd_type.c +++ b/libr/core/cmd_type.c @@ -1331,7 +1331,7 @@ static int cmd_type(void *data, const char *input) { char *error_msg = NULL; char *out = r_parse_c_string (core->anal, tmp, &error_msg); if (out) { - // r_cons_strcat (out); + // r_cons_strcat (out); r_anal_save_parsed_type (core->anal, out); free (out); } @@ -1822,7 +1822,7 @@ static int cmd_type(void *data, const char *input) { r_cons_println (res); } } else { - eprintf ("This is not an typedef\n"); + eprintf ("This is not a typedef\n"); } free (s); break; diff --git a/libr/include/r_anal.h b/libr/include/r_anal.h index f2cd36c277..7433e4ff05 100644 --- a/libr/include/r_anal.h +++ b/libr/include/r_anal.h @@ -2129,6 +2129,8 @@ R_API RAnalEsilCFG *r_anal_esil_cfg_expr(RAnalEsilCFG *cfg, RAnal *anal, const u R_API RAnalEsilCFG *r_anal_esil_cfg_op(RAnalEsilCFG *cfg, RAnal *anal, RAnalOp *op); R_API void r_anal_esil_cfg_merge_blocks(RAnalEsilCFG *cfg); R_API void r_anal_esil_cfg_free(RAnalEsilCFG *cfg); +R_API SdbGperf *r_anal_get_gperf_cc(const char *k); +R_API SdbGperf *r_anal_get_gperf_types(const char *k); R_API RAnalEsilDFGNode *r_anal_esil_dfg_node_new(RAnalEsilDFG *edf, const char *c); R_API RAnalEsilDFG *r_anal_esil_dfg_new(RReg *regs); diff --git a/libr/include/r_userconf.h.acr b/libr/include/r_userconf.h.acr index 52aecb2d04..f2ee5dc175 100644 --- a/libr/include/r_userconf.h.acr +++ b/libr/include/r_userconf.h.acr @@ -12,6 +12,12 @@ #define HAVE_CLOCK_NANOSLEEP @HAVE_CLOCK_NANOSLEEP@ #define HAVE_SIGACTION @HAVE_SIGACTION@ +#if (HAVE_GPERF) == 1 +#define HAVE_GPERF @HAVE_GPERF@ +#else +#define HAVE_GPERF 0 +#endif + #ifdef R_MESON_VERSION #define R2_PREFIX "@PREFIX@" #define R2_ETCDIR "@ETCDIR@" diff --git a/libr/include/r_util/r_str.h b/libr/include/r_util/r_str.h index db7b520a33..c9551d0505 100644 --- a/libr/include/r_util/r_str.h +++ b/libr/include/r_util/r_str.h @@ -2,6 +2,7 @@ #define R_STR_H #include +#include #include "r_str_util.h" #include "r_list.h" @@ -114,6 +115,7 @@ R_API bool r_str_is_printable_limited(const char *str, int size); R_API bool r_str_is_printable_incl_newlines(const char *str); R_API char *r_str_appendlen(char *ptr, const char *string, int slen); R_API char *r_str_newf(const char *fmt, ...) R_PRINTF_CHECK(1, 2); +R_API char *r_str_newvf(const char *fmt, va_list ap); R_API int r_str_distance(const char *a, const char *b); R_API char *r_str_newlen(const char *str, int len); R_API const char *r_str_sysbits(const int v); diff --git a/libr/parse/code.c b/libr/parse/code.c index 0f76f1c7fb..aab87b1227 100644 --- a/libr/parse/code.c +++ b/libr/parse/code.c @@ -1,4 +1,4 @@ -/* radare - LGPL - Copyright 2013-2019 - pancake */ +/* radare - LGPL - Copyright 2013-2021 - pancake */ #include "r_util.h" #include "r_types.h" @@ -105,7 +105,7 @@ R_API char *r_parse_c_file(RAnal *anal, const char *path, const char *dir, char } tcc_set_callback (T, &__appendString, &str); tcc_set_error_func (T, (void *)error_msg, __errorFunc); - sdb_foreach (anal->sdb_types, __typeLoad, anal); + sdb_foreach (anal->sdb_types, __typeLoad, anal); // why is this needed?? char *d = strdup (dir); RList *dirs = r_str_split_list (d, ":", 0); RListIter *iter; @@ -136,8 +136,7 @@ R_API char *r_parse_c_string(RAnal *anal, const char *code, char **error_msg) { tcc_set_error_func (T, (void *)error_msg, __errorFunc); sdb_foreach (anal->sdb_types, __typeLoad, NULL); if (tcc_compile_string (T, code) != 0) { - free (str); - str = NULL; + R_FREE (str); } tcc_delete (T); return str; diff --git a/libr/rules.mk b/libr/rules.mk index de6d952518..820d668ba0 100644 --- a/libr/rules.mk +++ b/libr/rules.mk @@ -72,9 +72,11 @@ else all: ${LIBSO} ${LIBAR} ${EXTRA_TARGETS} endif ifneq ($(SILENT),) - @-if [ -f p/Makefile ]; then (cd p && ${MAKE}) ; fi + @-if [ -f p/Makefile ]; then ${MAKE} -C p ; fi + @-if [ -f d/Makefile ]; then ${MAKE} -C d ; fi else - @-if [ -f p/Makefile ] ; then (echo "DIR ${NAME}/p"; cd p && ${MAKE}) ; fi + @-if [ -f p/Makefile ] ; then (echo "DIR ${NAME}/p"; ${MAKE} -C p) ; fi + @-if [ -f d/Makefile ] ; then (echo "DIR ${NAME}/d"; ${MAKE} -C d) ; fi endif ifeq ($(WITH_LIBS),1) diff --git a/libr/util/Makefile b/libr/util/Makefile index 3fa2ee15cb..c0b81a30fb 100644 --- a/libr/util/Makefile +++ b/libr/util/Makefile @@ -24,6 +24,12 @@ OBJS+=punycode.o pkcs7.o x509.o asn1.o astr.o json_parser.o json_indent.o skipli OBJS+=pj.o rbtree.o intervaltree.o qrcode.o vector.o skyline.o str_constpool.o str_trim.o OBJS+=ascii_table.o protobuf.o graph_drawable.o axml.o +ifeq (${HAVE_GPERF},1) +OBJS+=d/ascii.o +OBJS+=d/pokered.o +OBJS+=d/ebcdic37.o +endif + ifeq (${HAVE_LIB_GMP},1) OBJS+=big-gmp.o else diff --git a/libr/util/charset.c b/libr/util/charset.c index a0c31f1dcb..2d7c302a5a 100644 --- a/libr/util/charset.c +++ b/libr/util/charset.c @@ -1,6 +1,7 @@ /* radare - LGPL - Copyright 2020-2021 - gogo, pancake */ #include +#include #define USE_RUNES 0 diff --git a/libr/util/d/Makefile b/libr/util/d/Makefile index 9e17167d70..097fe62818 100644 --- a/libr/util/d/Makefile +++ b/libr/util/d/Makefile @@ -12,7 +12,9 @@ alle: $(F_SDB) %.sdb: %.sdb.txt $(SDB) $@ = < $< +ifeq ($(HAVE_GPERF),1) $(SDB) -C $@ +endif test -f $@ install: diff --git a/libr/util/str.c b/libr/util/str.c index b8c19fb68f..e240230a95 100644 --- a/libr/util/str.c +++ b/libr/util/str.c @@ -716,6 +716,19 @@ R_API char *r_str_trunc_ellipsis(const char *str, int len) { return buf; } +R_API char *r_str_newvf(const char *fmt, va_list ap) { + va_list ap2; + va_copy (ap2, ap); + int ret = vsnprintf (NULL, 0, fmt, ap2); + ret++; + char *p = calloc (1, ret); + if (p) { + (void)vsnprintf (p, ret, fmt, ap); + } + va_end (ap2); + return p; +} + R_API char *r_str_newf(const char *fmt, ...) { va_list ap, ap2; diff --git a/meson.build b/meson.build index 38193d9e4c..6c800125af 100644 --- a/meson.build +++ b/meson.build @@ -219,6 +219,13 @@ conf_data.set('plugins_debug', '&r_debug_plugin_' + ', &r_debug_plugin_'.join(de conf_data.set('plugins_egg', '&r_egg_plugin_' + ', &r_egg_plugin_'.join(egg_plugins) + ', 0') conf_data.set('plugins_lang', '&r_lang_plugin_' + ', &r_lang_plugin_'.join(lang_plugins) + ', 0') conf_data.set('plugins_parse', '&r_parse_plugin_' + ', &r_parse_plugin_'.join(parse_plugins) + ', 0') + +# userconf_h = configure_file( +# input: 'libr/include/r_userconf.h.acr', +# output: 'r_userconf.h', +# configuration: conf_data +#) + config_h = configure_file( input: 'libr/config.h.in', output: 'config.h', diff --git a/shlr/sdb/src/ls.c b/shlr/sdb/src/ls.c index 44b92d4fa6..b7cce872b6 100644 --- a/shlr/sdb/src/ls.c +++ b/shlr/sdb/src/ls.c @@ -1,8 +1,18 @@ -/* sdb - MIT - Copyright 2007-2017 - pancake, alvaro */ +/* sdb - MIT - Copyright 2007-2021 - pancake, alvaro */ #include #include "ls.h" +#if 0 + 1 128= 7s / 32 + 2 64 = 7s / 30s + 3 32 = 6s / 30s + 4 24 = 6s / 30s + 5 4 = 7s / 30s +#endif + +#define MERGELIMIT 24 + SDB_API SdbList *ls_newf(SdbListFree freefn) { SdbList *list = ls_new (); if (list) { @@ -68,7 +78,6 @@ static SdbListIter *_merge(SdbListIter *first, SdbListIter *second, SdbListCompa } static SdbListIter * _sdb_list_split(SdbListIter *head) { - SdbListIter *tmp; SdbListIter *fast; SdbListIter *slow; if (!head || !head->n) { @@ -76,24 +85,32 @@ static SdbListIter * _sdb_list_split(SdbListIter *head) { } slow = head; fast = head; + int count = 0; while (fast && fast->n && fast->n->n) { fast = fast->n->n; slow = slow->n; + count++; } - tmp = slow->n; + if (count < MERGELIMIT) { + return NULL; + } + SdbListIter *tmp = slow->n; slow->n = NULL; return tmp; } static SdbListIter * _merge_sort(SdbListIter *head, SdbListComparator cmp) { - SdbListIter *second; if (!head || !head->n) { return head; } - second = _sdb_list_split (head); - head = _merge_sort (head, cmp); - second = _merge_sort (second, cmp); - return _merge (head, second, cmp); + SdbListIter *second = _sdb_list_split (head); + if (second) { + head = _merge_sort (head, cmp); + second = _merge_sort (second, cmp); + return _merge (head, second, cmp); + } + ls_insertion_sort_iter (head, cmp); + return head; } SDB_API bool ls_merge_sort(SdbList *list, SdbListComparator cmp) { @@ -118,7 +135,7 @@ SDB_API bool ls_sort(SdbList *list, SdbListComparator cmp) { if (!cmp || list->cmp == cmp) { return false; } - if (list->length > 43) { + if (list->length > MERGELIMIT) { ls_merge_sort (list, cmp); } else { ls_insertion_sort (list, cmp); diff --git a/shlr/sdb/src/main.c b/shlr/sdb/src/main.c index b8f17e013f..7e275e1ea3 100644 --- a/shlr/sdb/src/main.c +++ b/shlr/sdb/src/main.c @@ -12,9 +12,42 @@ #define MODE_ZERO '0' #define MODE_JSON 'j' #define MODE_CGEN 'c' +#define MODE_TEXT 0 // default in plaintext #define MODE_DFLT 0 -static int save = 0; +typedef enum { + text, + zero, + json, + cgen, + diff, + perf +} MainFormat; + +typedef enum { + nope = 0, + dash, + eqeq, + dobl +} MainCreate; + +typedef struct { + int argc; + const char **argv; + int argi; + int db0; + bool failed; + const char *db; + const char *db2; + const char *grep; + ut32 options; + int textmode; + MainCreate create; + MainFormat format; +} MainOptions; + +static bool save = false; +static bool textmode = false; static Sdb *s = NULL; static ut32 options = SDB_OPTION_FS | SDB_OPTION_NOSTAMP; @@ -28,7 +61,7 @@ static void terminate(int sig UNUSED) { exit (1); } sdb_free (s); - exit (sig<2?sig:0); + exit (sig < 2? sig: 0); } static void write_null(void) { @@ -175,7 +208,7 @@ static char* get_name(const char*name) { char *n = strdup (name); char *v, *d = n; // local db beacuse is readonly and we dont need to finalize in case of ^C - for (v=(char*)n; *v; v++) { + for (v = (char*)n; *v; v++) { if (*v == '.') { break; } @@ -190,7 +223,7 @@ static char* get_cname(const char*name) { char *v, *d = n; // local db beacuse is readonly and we dont need to finalize in case of ^C for (v=(char*)n; *v; v++) { - if (*v == '/') { + if (*v == '/' || *v == '-') { *d++ = '_'; continue; } @@ -203,10 +236,13 @@ static char* get_cname(const char*name) { return n; } -static char *escape(const char *b) { +static char *escape(const char *b, int ch) { char *a = calloc ((1 + strlen (b)), 4); char *c = a; while (*b) { + if (*b == ch) { + *c = '_'; + } else switch (*b) { case '"': *c++ = '\\'; @@ -234,9 +270,9 @@ static char *escape(const char *b) { return a; } -static void sdb_grep_dump_cb(int fmt, const char *k, const char *v, const char *comma) { - switch (fmt) { - case MODE_JSON: +static void sdb_dump_cb(MainOptions *mo, const char *k, const char *v, const char *comma) { + switch (mo->format) { + case json: if (!strcmp (v, "true") || !strcmp (v, "false")) { printf ("%s\"%s\":%s", comma, k, v); } else if (sdb_isnum (v)) { @@ -247,29 +283,21 @@ static void sdb_grep_dump_cb(int fmt, const char *k, const char *v, const char * printf ("%s\"%s\":\"%s\"", comma, k, v); } break; - case MODE_CGEN: + case perf: + case cgen: { - char *a = strdup (k); - char *b = strdup (v); - char *p = b; - while (*p) { - *p = (*p == '"')? '\'': *p; - p++; - } - for (p = a; *p; p++) { - if (*p == ',') { - eprintf ("Warning: Keys cant contain a comma in gperf.\n"); - *p = '.'; + char *a = escape (k, ','); + char *b = escape (v, 0); + if (textmode) { + printf (" \"%s=%s\"\n", a, b); + } else { + printf ("%s,\"%s\"\n", a, b); } - } - char *e = escape (b); - printf ("%s,\"%s\"\n", a, e); - free (e); - free (a); - free (b); + free (a); + free (b); } break; - case MODE_ZERO: + case zero: printf ("%s=%s", k, v); break; default: @@ -278,78 +306,57 @@ static void sdb_grep_dump_cb(int fmt, const char *k, const char *v, const char * } } -static int sdb_grep_dump(const char *dbname, int fmt, bool grep, const char *expgrep) { - char *v, k[SDB_MAX_KEY] = { 0 }; - const char *comma = ""; - Sdb *db = sdb_new (NULL, dbname, 0); - if (!db) { - return 1; - } - char *cname = get_cname (dbname); - char *name = get_name (dbname); - sdb_config (db, options); - sdb_dump_begin (db); - switch (fmt) { - case MODE_CGEN: - printf ("%%{\n"); - printf ("// gperf -aclEDCIG --null-strings -H sdb_hash_c_%s -N sdb_get_c_%s -t %s.gperf > %s.c\n", cname, cname, cname, cname); +static void cgen_header(const char *cname) { + if (textmode) { printf ("// gcc -DMAIN=1 %s.c ; ./a.out > %s.h\n", cname, cname); printf ("#include \n"); - printf ("#include \n"); - printf ("%%}\n"); + printf ("#include \n"); printf ("\n"); - printf ("struct kv { const char *name; const char *value; };\n"); - printf ("%%%%\n"); - break; - case MODE_JSON: - printf ("{"); - break; + printf ("static const char *textdb = \"\"\\\n"); + return; } + printf ("%%{\n"); + printf ("// gperf -aclEDCIG --null-strings -H sdb_hash_c_%s -N sdb_get_c_%s -t %s.gperf > %s.c\n", cname, cname, cname, cname); + printf ("// gcc -DMAIN=1 %s.c ; ./a.out > %s.h\n", cname, cname); + printf ("#include \n"); + printf ("#include \n"); + printf ("#include \n"); + printf ("%%}\n"); + printf ("\n"); + printf ("struct kv { const char *name; const char *value; };\n"); + printf ("%%%%\n"); +} - if (db->fd == -1) { - SdbList *l = sdb_foreach_list (db, true); - SdbKv *kv; - SdbListIter *it; - ls_foreach (l, it, kv) { - if (grep && !strstr (k, expgrep) && !strstr (v, expgrep)) { - continue; - } - sdb_grep_dump_cb (fmt, sdbkv_key (kv), sdbkv_value (kv), comma); - comma = ","; - } - ls_free (l); - } else { - while (sdb_dump_dupnext (db, k, &v, NULL)) { - if (grep && !strstr (k, expgrep) && !strstr (v, expgrep)) { - free (v); - continue; - } - sdb_grep_dump_cb (fmt, k, v, comma); - comma = ","; - free (v); - } +// TODO rename gperf with cgen +static void cgen_footer(const char *name, const char *cname) { + if (textmode) { + printf ("};\n"); + printf ("TODO\n"); + return; } - switch (fmt) { - case MODE_ZERO: - fflush (stdout); - write_null (); - break; - case MODE_CGEN: - printf ("%%%%\n"); - printf ("// SDB-CGEN V"SDB_VERSION"\n"); - printf ("// %p\n", cname); - printf ("const char* gperf_%s_get(const char *s) {\n", cname); - printf ("\tconst struct kv *o = sdb_get_c_%s (s, strlen(s));\n", cname); - printf ("\treturn o? o->value: NULL;\n"); - printf ("}\n"); - printf ("const unsigned int gperf_%s_hash(const char *s) {\n", cname); - printf ("\treturn sdb_hash_c_%s(s, strlen (s));\n", cname); - printf ("}\n"); - printf ( -"struct {const char*name;void*get;void*hash;} gperf_%s = {\n" + printf ("%%%%\n"); + printf ("// SDB-CGEN V"SDB_VERSION"\n"); + printf ("// %p\n", cname); + printf ("typedef int (*GperfForeachCallback)(void *user, const char *k, const char *v);\n"); + printf ("int gperf_%s_foreach(GperfForeachCallback cb, void *user) {\n", cname); + printf ("\tint i;for (i=0;iname, w->value)) return 0;\n"); + printf ("}\n"); + printf ("return 1;}\n"); + printf ("const char* gperf_%s_get(const char *s) {\n", cname); + printf ("\tconst struct kv *o = sdb_get_c_%s (s, strlen(s));\n", cname); + printf ("\treturn o? o->value: NULL;\n"); + printf ("}\n"); + printf ("const unsigned int gperf_%s_hash(const char *s) {\n", cname); + printf ("\treturn sdb_hash_c_%s(s, strlen (s));\n", cname); + printf ("}\n"); + printf ( +"struct {const char*name;void*get;void*hash;void *foreach;} gperf_%s = {\n" "\t.name = \"%s\",\n" "\t.get = &gperf_%s_get,\n" -"\t.hash = &gperf_%s_hash\n" +"\t.hash = &gperf_%s_hash,\n" +"\t.foreach = &gperf_%s_foreach\n" "};\n" "\n" "#if MAIN\n" @@ -382,16 +389,87 @@ static int sdb_grep_dump(const char *dbname, int fmt, bool grep, const char *exp " printf (\"#endif\\n\");\n" "}\n" "#endif\n", - cname, cname, cname, - cname, name, name, - cname, cname, - cname, cname - ); - printf ("\n"); + cname, cname, cname, cname, cname, + name, name, + cname, cname, cname, cname + ); + printf ("\n"); +} + +static int sdb_dump(MainOptions *mo) { + const char *dbname = mo->db; + bool grep = mo->grep; + const char *expgrep = mo->grep; + + char *v, k[SDB_MAX_KEY] = { 0 }; + const char *comma = ""; + Sdb *db = sdb_new (NULL, dbname, 0); + if (!db) { + return 1; + } + char *cname = get_cname (dbname); + char *name = get_name (dbname); + sdb_config (db, options); + sdb_dump_begin (db); + switch (mo->format) { + case cgen: + case perf: + cgen_header (cname); break; - case MODE_JSON: + case json: + printf ("{"); + break; + default: + break; + } + + if (db->fd == -1) { + SdbList *l = sdb_foreach_list (db, true); + if (mo->format == cgen && ls_length (l) > SDB_MAX_GPERF_KEYS) { + ls_free (l); + eprintf ("Error: gperf doesn't work with datasets with more than 15.000 keys.\n"); + return -1; + } + SdbKv *kv; + SdbListIter *it; + ls_foreach (l, it, kv) { + if (grep && !strstr (k, expgrep) && !strstr (v, expgrep)) { + continue; + } + sdb_dump_cb (mo, sdbkv_key (kv), sdbkv_value (kv), comma); + comma = ","; + } + ls_free (l); + } else { + int count = 0; + while (sdb_dump_dupnext (db, k, &v, NULL)) { + if (grep && !strstr (k, expgrep) && !strstr (v, expgrep)) { + free (v); + continue; + } + sdb_dump_cb (mo, k, v, comma); + comma = ","; + free (v); + if (mo->format == cgen && count++ > SDB_MAX_GPERF_KEYS) { + eprintf ("Error: gperf doesn't work with datasets with more than 15.000 keys.\n"); + return -1; + } + } + } + switch (mo->format) { + case zero: + fflush (stdout); + write_null (); + break; + case perf: + case cgen: + cgen_footer (name, cname); + break; + case json: printf ("}\n"); break; + default: + break; } sdb_free (db); free (cname); @@ -399,14 +477,6 @@ static int sdb_grep_dump(const char *dbname, int fmt, bool grep, const char *exp return 0; } -static int sdb_grep(const char *db, int fmt, const char *grep) { - return sdb_grep_dump (db, fmt, true, grep); -} - -static int sdb_dump(const char *db, int fmt) { - return sdb_grep_dump (db, fmt, false, NULL); -} - static int insertkeys(Sdb *s, const char **args, int nargs, int mode) { int must_save = 0; if (args && nargs > 0) { @@ -464,19 +534,21 @@ static int createdb(const char *f, const char **args, int nargs) { } static int showusage(int o) { - printf ("usage: sdb [-0cCdDehjJv|-D A B] [-|db] " + printf ("usage: sdb [-0cCdDehjJtv|-D A B] [-|db] " "[.file]|[-=]|==||[-+][(idx)key[:json|=value] ..]\n"); if (o == 2) { printf (" -0 terminate results with \\x00\n" " -c count the number of keys database\n" " -C create foo.{c,h} for embedding (uses gperf)\n" - " -G print database in gperf format\n" " -d decode base64 from stdin\n" " -D diff two databases\n" " -e encode stdin as base64\n" + " -g [..] grep expression\n" + " -G print database in gperf format\n" " -h show this help\n" " -j output in json\n" " -J enable journaling\n" + " -t use textmode (for -C)\n" " -v show version information\n"); return 0; } @@ -585,12 +657,34 @@ static int showcount(const char *db) { return 0; } -static int gen_gperf(const char *file, const char *name) { - int (*_system)(const char *cmd); +static int sdb_system(const char *cmd) { + static int (*sys)(const char *cmd) = NULL; + if (!sys) { +#if USE_DLSYSTEM + sys = dlsym (NULL, "system"); + if (!sys) { + sys = puts; + return -1; + } +#else + sys = system; +#endif + } + return sys (cmd); +} + +static int gen_gperf(MainOptions *mo, const char *file, const char *name) { const size_t buf_size = 4096; char *buf = malloc (buf_size); + if (!buf) { + return -1; + } size_t out_size = strlen (file) + 32; char *out = malloc (out_size); + if (!out) { + free (buf); + return -1; + } snprintf (out, out_size, "%s.gperf", name); int wd = open (out, O_RDWR, 0644); if (wd == -1) { @@ -599,39 +693,26 @@ static int gen_gperf(const char *file, const char *name) { ftruncate (wd, 0); } int rc = -1; -#if USE_DLSYSTEM - _system = dlsym (NULL, "system"); - if (!_system) { - _system = puts; - return -1; - } -#else - _system = system; -#endif if (wd != -1) { dup2 (1, 999); dup2 (wd, 1); - rc = sdb_dump (file, MODE_CGEN); + // mo->format = cgen; + rc = sdb_dump (mo); // file, MODE_CGEN, false, NULL); fflush (stdout); close (wd); dup2 (999, 1); -#if 0 } else { - snprintf (buf, bufsz, "sdb -G %s > %s.gperf\n", file, name); - rc = _system (buf); -#endif - } else { - eprintf ("Cannot create .gperf%c", 10); + eprintf ("Cannot create .gperf\n"); } if (rc == 0) { char *cname = get_cname (name); snprintf (buf, buf_size, "gperf -aclEDCIG --null-strings -H sdb_hash_c_%s" " -N sdb_get_c_%s -t %s.gperf > %s.c\n", cname, cname, name, name); free (cname); - rc = _system (buf); + rc = sdb_system (buf); if (rc == 0) { snprintf (buf, buf_size, "gcc -DMAIN=1 %s.c ; ./a.out > %s.h\n", name, name); - rc = _system (buf); + rc = sdb_system (buf); if (rc == 0) { eprintf ("Generated %s.c and %s.h\n", name, name); } @@ -646,116 +727,192 @@ static int gen_gperf(const char *file, const char *name) { return rc; } +static const char *main_argparse_getarg(MainOptions *mo) { + mo->argi++; + mo->db0++; + if (mo->argi >= mo->argc) { + return NULL; + } + return mo->argv[mo->argi]; +} + +static bool main_argparse_flag(MainOptions *mo, char flag) { + switch (flag) { + case '0': + mo->format = zero; + break; + case 'h': + return showusage (2); + case 'v': + return showversion (); + case 'e': + return base64encode (); + case 'd': + return base64decode (); + case 'j': + mo->format = json; + if (mo->argi + 1 >= mo->argc) { + return jsonIndent (); + } + break; + case 'c': + if (mo->argc < 3) { + return showusage (1); + } else { + const char *db = main_argparse_getarg (mo); + if (!db) { + return showusage (1); + } + return showcount (db); + } + break; + case 'g': + mo->grep = main_argparse_getarg (mo); + if (!mo->grep) { + eprintf ("Missing argument for -g\n"); + return false; + } + break; + case 'D': + if (mo->argi + 2 >= mo->argc) { + return showusage (0); + } + mo->format = diff; + break; + case 'G': + mo->format = perf; + break; + case 'C': + mo->format = cgen; + break; + case 't': + mo->textmode = true; + break; + case 'J': + mo->options |= SDB_OPTION_JOURNAL; + // expect argument + break; + default: + return false; + } + return true; +} + +static MainOptions *main_argparse(int argc, const char **argv) { + MainOptions *mo = (MainOptions*)calloc (sizeof (MainOptions), 1); + if (!mo) { + return NULL; + } + mo->argc = argc; + mo->argv = argv; + mo->options = SDB_OPTION_FS | SDB_OPTION_NOSTAMP; + mo->failed = true; + int i; + for (i = 1; i < argc; i++) { + mo->argi = i; + if (argv[i][0] == '-' && argv[i][1]) { + int j = 1; + while (argv[i][j]) { + if (!main_argparse_flag (mo, argv[i][j])) { + // invalid flag + return NULL; + } + j++; + } + } else { + mo->db0 = i; + mo->db = argv[i]; + if (i + 1 < argc) { + switch (argv[i+1][0]) { + case '-': + if (!argv[i + 1][1]) { + mo->create = dash; + } + break; + case '=': + if (argv[i + 1][1]) { + mo->create = dobl; + } else { + mo->create = eqeq; + } + break; + default: + mo->db2 = argv[i]; + break; + } + } + break; + } + } + return mo; +} + int main(int argc, const char **argv) { char *line; - const char *arg, *grep = NULL; int i, fmt = MODE_DFLT; - int db0 = 1, argi = 1; - bool interactive = false; /* terminate flags */ if (argc < 2) { return showusage (1); } - arg = argv[1]; - if (arg[0] == '-') {// && arg[1] && arg[2]==0) { - switch (arg[1]) { - case 0: - /* no-op */ - break; - case '0': - fmt = MODE_ZERO; - db0++; - argi++; - if (db0 >= argc) { - return showusage (1); - } - break; - case 'g': - db0 += 2; - if (db0 >= argc) { - return showusage (1); - } - grep = argv[2]; - argi += 2; - break; - case 'J': - options |= SDB_OPTION_JOURNAL; - db0++; - argi++; - if (db0 >= argc) { - return showusage (1); - } - break; - case 'G': - if (argc > 2) { - return sdb_dump (argv[db0 + 1], MODE_CGEN); - } - return showusage (1); - case 'c': - return (argc < 3)? showusage (1): showcount (argv[2]); - case 'C': - if (argc > 2) { - const char *file = argv[db0 + 1]; - char *name = strdup (file); - char *p = strchr (name, '.'); - if (p) *p = 0; - int rc = gen_gperf (file, name); - free (name); - return rc; - } - return showusage (1); - case 'v': return showversion (); - case 'h': return showusage (2); - case 'e': return base64encode (); - case 'd': return base64decode (); - case 'D': - if (argc == 4) { - return dbdiff (argv[2], argv[3]) ? 0 : 1; - } - return showusage (0); - case 'j': - if (argc > 2) { - return sdb_dump (argv[db0 + 1], MODE_JSON); - } - return jsonIndent (); - default: - eprintf ("Invalid flag %s\n", arg); - break; - } + MainOptions *mo = main_argparse (argc, argv); + if (!mo) { + return 1; } - /* sdb - */ - if (argi == 1 && !strcmp (argv[argi], "-")) { - /* no database */ - argv[argi] = ""; - if (argc == db0 + 1) { - interactive = true; - /* if no argument passed */ - argv[argi] = "-"; - argc++; - argi++; + // -j json return sdb_dump (argv[db0 + 1], MODE_JSON); + // -G sdb_dump (argv[db0 + 1], MODE_CGEN); // gperf + // -C print C/H files + // -t text + switch (mo->format) { + case diff: + if (mo->db && mo->db2) { + return dbdiff (mo->db, mo->db2)? 0: 1; } + return showusage (1); + case perf: + return sdb_dump (mo); + case cgen: + { + if (mo->db0 >= argc) { + return showusage (1); + } + const char *file = mo->argv[mo->db0]; + char *name = strdup (file); + char *p = strchr (name, '.'); + if (p) { + *p = 0; + } + int rc = gen_gperf (mo, file, name); + free (name); + return rc; + } + default: + break; } - /* sdb dbname */ - if (argc - 1 == db0) { - if (grep) { - return sdb_grep (argv[db0], fmt, grep); + + if (!mo->db && !mo->create) { + return showusage (1); + } + if (!mo->create && mo->db) { + if (!strcmp (mo->db, "-")) { + mo->create = dash; + mo->db = NULL; } - return sdb_dump (argv[db0], fmt); } #if USE_MMAN signal (SIGINT, terminate); signal (SIGHUP, synchronize); #endif int ret = 0; - if (interactive || !strcmp (argv[db0 + 1], "-")) { - if ((s = sdb_new (NULL, argv[db0], 0))) { + switch (mo->create) { + case dash: // "-" + if ((s = sdb_new (NULL, mo->db, 0))) { sdb_config (s, options); - int kvs = db0 + 2; + int kvs = mo->db0 + 2; if (kvs < argc) { - save |= insertkeys (s, argv + argi + 2, argc - kvs, '-'); + save |= insertkeys (s, argv + mo->argi + 2, argc - kvs, '-'); } for (; (line = slurp (stdin, NULL));) { save |= sdb_query (s, line); @@ -766,23 +923,35 @@ int main(int argc, const char **argv) { free (line); } } - } else if (!strcmp (argv[db0 + 1], "=")) { - ret = createdb (argv[db0], NULL, 0); - } else if (!strcmp (argv[db0 + 1], "==")) { - ret = createdb (argv[db0], argv + db0 + 2, argc - (db0 + 2)); - } else { - s = sdb_new (NULL, argv[db0], 0); + break; + case eqeq: // "=" + ret = createdb (mo->db, NULL, 0); + break; + case dobl: // "==" + { + int delta = mo->db0 + 2; + ret = createdb (mo->db, mo->argv + delta, mo->argc - delta); + } + break; + default: + // "sdb test.db" + s = sdb_new (NULL, mo->db, 0); if (!s) { return 1; } sdb_config (s, options); - for (i = db0 + 1; i < argc; i++) { - save |= sdb_query (s, argv[i]); - if (fmt) { - fflush (stdout); - write_null (); + if (mo->argi + 1 < mo->argc) { + for (i = mo->db0 + 1; i < argc; i++) { + save |= sdb_query (s, mo->argv[i]); + if (fmt) { + fflush (stdout); + write_null (); + } } + } else { + return sdb_dump (mo); } + break; } terminate (ret); return ret; diff --git a/shlr/sdb/src/sdb.c b/shlr/sdb/src/sdb.c index abfcf61a0b..dac8d683e0 100644 --- a/shlr/sdb/src/sdb.c +++ b/shlr/sdb/src/sdb.c @@ -802,6 +802,9 @@ SDB_API bool sdb_foreach(Sdb* s, SdbForeachCallback cb, void *user) { return false; } s->depth++; + if (s->gp) { + return s->gp->foreach ((GperfForeachCallback)cb, user); + } bool result = sdb_foreach_cdb (s, cb, NULL, user); if (!result) { return sdb_foreach_end (s, false); diff --git a/shlr/sdb/src/sdb.h b/shlr/sdb/src/sdb.h index 1865f7375d..7efa7ffdc3 100644 --- a/shlr/sdb/src/sdb.h +++ b/shlr/sdb/src/sdb.h @@ -24,6 +24,9 @@ extern "C" { #define SDB_MIN_KEY 1 #define SDB_MAX_KEY 0xff +// ftp://ftp.gnu.org/old-gnu/Manuals/gperf-2.7/html_node/gperf_17.es.html +#define SDB_MAX_GPERF_KEYS 15000 + #if !defined(SZT_ADD_OVFCHK) #define SZT_ADD_OVFCHK(x, y) ((SIZE_MAX - (x)) <= (y)) #endif @@ -81,10 +84,12 @@ extern char *strdup (const char *); #define SDB_KSZ 0xff #define SDB_VSZ 0xffffff +typedef int (*GperfForeachCallback)(void *user, const char *k, const char *v); typedef struct sdb_gperf_t { const char *name; const char *(*get)(const char *k); unsigned int *(*hash)(const char *k); + bool (*foreach)(GperfForeachCallback cb, void *user); } SdbGperf; typedef struct sdb_t {