Update to sdb-1.7.0 (#18337)

This commit is contained in:
pancake 2021-02-08 22:21:56 +01:00 committed by GitHub
parent f1722cd46d
commit bd02bfd5c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 263 additions and 34 deletions

View File

@ -11,8 +11,10 @@ SDB_OBJS+=cdb_make.o
SDB_OBJS+=ht_uu.o
SDB_OBJS+=ht_up.o
SDB_OBJS+=ht_pp.o
SDB_OBJS+=ht_pu.o
SDB_OBJS+=sdbht.o
SDB_OBJS+=json.o
SDB_OBJS+=text.o
SDB_OBJS+=lock.o
SDB_OBJS+=ls.o
SDB_OBJS+=ns.o

View File

@ -462,6 +462,7 @@ sdb_files = [
'shlr/sdb/src/ht_uu.c',
'shlr/sdb/src/ht_up.c',
'shlr/sdb/src/ht_pp.c',
'shlr/sdb/src/ht_pu.c',
'shlr/sdb/src/journal.c',
'shlr/sdb/src/json.c',
'shlr/sdb/src/lock.c',
@ -498,6 +499,7 @@ sdb_inc_files = [
'shlr/sdb/src/ht_pp.h',
'shlr/sdb/src/ht_up.h',
'shlr/sdb/src/ht_uu.h',
'shlr/sdb/src/ht_pu.h',
'shlr/sdb/src/ls.h',
'shlr/sdb/src/sdb.h',
'shlr/sdb/src/sdbht.h',

View File

@ -6,7 +6,7 @@ INCDIR=${PREFIX}/include
VAPIDIR=${DATADIR}/vala/vapi/
MANDIR=${DATADIR}/man/man1
SDBVER=1.6.0
SDBVER=1.7.0
BUILD_MEMCACHE=0

View File

@ -42,8 +42,9 @@ libsdb_sources = [
'src/disk.c',
'src/fmt.c',
'src/ht_uu.c',
'src/ht_pp.c',
'src/ht_up.c',
'src/ht_pp.c',
'src/ht_pu.c',
'src/journal.c',
'src/json.c',
#'src/json/api.c',
@ -95,6 +96,7 @@ if not meson.is_subproject()
'src/ht_pp.h',
'src/ht_up.h',
'src/ht_uu.h',
'src/ht_up.h',
'src/ls.h',
'src/sdb.h',
'src/sdbht.h',

View File

@ -3,7 +3,7 @@ include ../config.mk
CFLAGS+=-g
OBJ=cdb.o buffer.o cdb_make.o ls.o sdbht.o ht_uu.o sdb.o num.o base64.o match.o
OBJ+=json.o ns.o lock.o util.o disk.o query.o array.o fmt.o journal.o text.o
OBJ+=dict.o ht_pp.o ht_up.o set.o diff.o
OBJ+=dict.o ht_pp.o ht_up.o ht_pu.o set.o diff.o
SOBJ=$(subst .o,.o.o,${OBJ})
WITHPIC?=1
BIN=sdb${EXT_EXE}

View File

@ -1,4 +1,4 @@
/* radare2 - BSD 3 Clause License - 2016 - crowell */
/* radare2 - BSD 3 Clause License - 2016-2021 - crowell */
#ifndef HT_TYPE
#error HT_TYPE should be defined before including this header
@ -28,7 +28,7 @@
#define VALUE_TYPE void *
#define KEY_TO_HASH(x) ((ut32)(x))
#define HT_NULL_VALUE 0
#else
#elif HT_TYPE == 3
#define HtName_(name) name##UU
#define Ht_(name) ht_uu_##name
#define HT_(name) HtUU##name
@ -36,6 +36,14 @@
#define VALUE_TYPE ut64
#define KEY_TO_HASH(x) ((ut32)(x))
#define HT_NULL_VALUE 0
#else
#define HtName_(name) name##PU
#define Ht_(name) ht_pu_##name
#define HT_(name) HtPU##name
#define KEY_TYPE void *
#define VALUE_TYPE ut64
#define KEY_TO_HASH(x) ((ut32)(uintptr_t)(x))
#define HT_NULL_VALUE 0
#endif
#include "ls.h"
@ -106,3 +114,5 @@ SDB_API void Ht_(foreach)(HtName_(Ht) *ht, HT_(ForeachCallback) cb, void *user);
SDB_API HT_(Kv)* Ht_(find_kv)(HtName_(Ht)* ht, const KEY_TYPE key, bool* found);
SDB_API bool Ht_(insert_kv)(HtName_(Ht) *ht, HT_(Kv) *kv, bool update);
#undef HT_TYPE

View File

@ -5,12 +5,13 @@
* This header provides an hashtable HtPP that has void* as key and void* as
* value. The API functions starts with "ht_pp_" and the types starts with "HtPP".
*/
#undef HT_TYPE
#define HT_TYPE 1
//#include "sdbht.h"
#include "ht_inc.h"
SDB_API HtName_(Ht)* Ht_(new0)(void);
SDB_API HtName_(Ht)* Ht_(new)(HT_(DupValue) valdup, HT_(KvFreeFunc) pair_free, HT_(CalcSizeV) valueSize);
SDB_API HtName_(Ht)* Ht_(new_size)(ut32 initial_size, HT_(DupValue) valdup, HT_(KvFreeFunc) pair_free, HT_(CalcSizeV) valueSize);
#undef HT_TYPE
#endif

22
shlr/sdb/src/ht_pu.c Normal file
View File

@ -0,0 +1,22 @@
#include "sdb.h"
#include "ht_pu.h"
#include "ht_inc.c"
static void free_kv_key(HT_(Kv) *kv) {
free (kv->key);
}
// creates a default HtPU that has strings as keys
SDB_API HtName_(Ht)* Ht_(new0)(void) {
HT_(Options) opt = {
.cmp = (HT_(ListComparator))strcmp,
.hashfn = (HT_(HashFunction))sdb_hash,
.dupkey = (HT_(DupKey))strdup,
.dupvalue = NULL,
.calcsizeK = (HT_(CalcSizeK))strlen,
.calcsizeV = NULL,
.freefn = free_kv_key,
.elem_size = sizeof (HT_(Kv)),
};
return Ht_(new_opt) (&opt);
}

14
shlr/sdb/src/ht_pu.h Normal file
View File

@ -0,0 +1,14 @@
#ifndef SDB_HT_PU_H
#define SDB_HT_PU_H
/*
* This header provides an hashtable HtPU that has void* as key and ut64 as
* value. The API functions starts with "ht_pu_" and the types starts with "HtPU".
*/
#define HT_TYPE 4
#include "ht_inc.h"
SDB_API HtName_(Ht)* Ht_(new0)(void);
#undef HT_TYPE
#endif

View File

@ -1,3 +1,4 @@
#include "sdb.h"
#include "ht_up.h"
#include "ht_inc.c"

View File

@ -5,8 +5,10 @@
* This header provides an hashtable HtUP that has ut64 as key and void* as
* value. The API functions starts with "ht_up_" and the types starts with "HtUP".
*/
#undef HT_TYPE
#define HT_TYPE 2
#include "ht_inc.h"
#include "sdbht.h"
SDB_API HtName_(Ht)* Ht_(new0)(void);
SDB_API HtName_(Ht)* Ht_(new)(HT_(DupValue) valdup, HT_(KvFreeFunc) pair_free, HT_(CalcSizeV) valueSize);

View File

@ -1,7 +1,8 @@
#include "sdb.h"
#include "ht_uu.h"
#include "ht_inc.c"
// creates a default HtUU that has strings as keys
SDB_API HtName_(Ht)* Ht_(new0)(void) {
HT_(Options) opt = {
.cmp = NULL,

View File

@ -5,10 +5,11 @@
* This header provides an hashtable Ht that has ut64 as key and ut64 as
* value. The API functions starts with "ht_" and the types starts with "Ht".
*/
#define HT_TYPE 0
#undef HT_TYPE
#define HT_TYPE 3
#include "ht_inc.h"
#include "sdbht.h"
SDB_API HtName_(Ht)* Ht_(new0)(void);
#undef HT_TYPE
#endif

View File

@ -1,4 +1,4 @@
/* sdb - MIT - Copyright 2012-2016 - pancake */
/* sdb - MIT - Copyright 2012-2021 - pancake */
#include <stdarg.h>
#include "sdb.h"
@ -51,7 +51,7 @@ SDB_API int sdb_json_num_dec(Sdb *s, const char *k, const char *p, int n, ut32 c
return cur - n;
}
SDB_API int sdb_json_num_get (Sdb *s, const char *k, const char *p, ut32 *cas) {
SDB_API int sdb_json_num_get(Sdb *s, const char *k, const char *p, ut32 *cas) {
char *v = sdb_get (s, k, cas);
if (v) {
Rangstr rs = json_get (v, p);
@ -94,17 +94,17 @@ static bool isstring(const char *s) {
}
// JSON only supports base16 numbers
SDB_API int sdb_json_num_set (Sdb *s, const char *k, const char *p, int v, ut32 cas) {
SDB_API int sdb_json_num_set(Sdb *s, const char *k, const char *p, int v, ut32 cas) {
char *_str, str[64];
_str = sdb_itoa (v, str, 10);
return sdb_json_set (s, k, p, _str, cas);
}
SDB_API int sdb_json_unset (Sdb *s, const char *k, const char *p, ut32 cas) {
SDB_API int sdb_json_unset(Sdb *s, const char *k, const char *p, ut32 cas) {
return sdb_json_set (s, k, p, NULL, cas);
}
SDB_API bool sdb_json_set (Sdb *s, const char *k, const char *p, const char *v, ut32 cas) {
SDB_API bool sdb_json_set(Sdb *s, const char *k, const char *p, const char *v, ut32 cas) {
int l, idx, len[3], jslen = 0;
char *b, *str = NULL;
const char *beg[3];

View File

@ -1,4 +1,4 @@
/* sdb - MIT - Copyright 2012-2016 - pancake */
/* sdb - MIT - Copyright 2012-2021 - pancake */
#include <stdio.h>
#include <string.h>
@ -20,8 +20,6 @@ SDB_API const char *sdb_lock_file(const char *f) {
return buf;
}
#define os_getpid() getpid()
SDB_API bool sdb_lock(const char *s) {
int fd;
char *pid, pidstr[64];
@ -32,7 +30,7 @@ SDB_API bool sdb_lock(const char *s) {
if (fd == -1) {
return false;
}
pid = sdb_itoa (getpid(), pidstr, 10);
pid = sdb_itoa (getpid (), pidstr, 10);
if (pid) {
if ((write (fd, pid, strlen (pid)) < 0)
|| (write (fd, "\n", 1) < 0)) {
@ -52,7 +50,7 @@ SDB_API int sdb_lock_wait(const char *s) {
#if __SDB_WINDOWS__
Sleep (500); // hack
#else
// TODO use lockf() here .. flock is not much useful (fd, LOCK_EX);
// TODO use lockf() here .. flock is not much useful (fd, LOCK_EX);
sleep (1); // hack
#endif
}

View File

@ -1,4 +1,4 @@
/* sdb - MIT - Copyright 2011-2020 - pancake */
/* sdb - MIT - Copyright 2011-2021 - pancake */
#include <signal.h>
#include <stdio.h>
@ -8,6 +8,7 @@
#define MODE_ZERO '0'
#define MODE_JSON 'j'
#define MODE_CGEN 'c'
#define MODE_DFLT 0
static int save = 0;
@ -169,17 +170,39 @@ static void synchronize(int sig UNUSED) {
static int sdb_grep_dump(const char *dbname, int fmt, bool grep,
const char *expgrep) {
char cname[SDB_MAX_KEY];
char *v, k[SDB_MAX_KEY] = { 0 };
char *d = cname;
const char *comma = "";
// local db beacuse is readonly and we dont need to finalize in case of ^C
for (v=(char*)dbname;*v;v++) {
if (*v == '.') {
break;
}
*d++ = *v;
}
*d++ = 0;
Sdb *db = sdb_new (NULL, dbname, 0);
if (!db) {
return 1;
}
sdb_config (db, options);
sdb_dump_begin (db);
if (fmt == MODE_JSON) {
switch (fmt) {
case MODE_JSON:
printf ("{");
break;
case MODE_CGEN:
printf ("%%{\n");
printf ("// gperf -aclEDCIG --null-strings -H sdb_hash_c_%s -N sdb_get_c_%s -t %s.gperf > %s.c\n", cname, cname, cname, cname);
printf ("// gcc -DMAIN=1 %s.c ; ./a.out > %s.h\n", cname, cname);
printf ("#include <stdio.h>\n");
printf ("#include <ctype.h>\n");
printf ("%%}\n");
printf ("\n");
printf ("struct kv { const char *name; const char *value; };\n");
printf ("%%%%\n");
break;
}
while (sdb_dump_dupnext (db, k, &v, NULL)) {
if (grep && !strstr (k, expgrep) && !strstr (v, expgrep)) {
@ -199,6 +222,9 @@ static int sdb_grep_dump(const char *dbname, int fmt, bool grep,
}
comma = ",";
break;
case MODE_CGEN:
printf ("%s,\"%s\"\n", k, v);
break;
case MODE_ZERO:
printf ("%s=%s", k, v);
break;
@ -213,6 +239,49 @@ static int sdb_grep_dump(const char *dbname, int fmt, bool grep,
fflush (stdout);
write_null ();
break;
case MODE_CGEN:
printf ("%%%%\n");
printf ("// SDB-CGEN V"SDB_VERSION"\n");
printf ("const char* %s_get(const char *s) {\n", cname);
printf ("\tconst struct kv *o = sdb_get_c_%s (s, strlen(s));\n", cname);
printf ("\treturn o? o->value: NULL;\n");
printf ("}\n");
printf ("const unsigned int %s_hash(const char *s) {\n", cname);
printf ("\treturn sdb_hash_c_%s(s, strlen (s));\n", cname);
printf ("}\n");
printf (
"#if MAIN\n"
"int main () {\n"
" char line[1024];\n"
" FILE *fd = fopen (\"%s.gperf\", \"r\");\n"
" int mode = 0;\n"
" printf (\"#ifndef INCLUDE_%s_H\\n\");\n"
" printf (\"#define INCLUDE_%s_H 1\\n\");\n"
" while (!feof (fd)) {\n"
" *line = 0;\n"
" fgets(line,sizeof(line),fd);\n"
" if (mode == 1) {\n"
" char *comma = strchr (line, ',');\n"
" if(comma) {\n"
" *comma = 0;\n"
" char *up = strdup(line);\n"
" char *p = up;while(*p){*p=toupper(*p);p++;}\n"
" printf (\"#define %s_%%s %%d\\n\",\n"
" line, sdb_hash_c_%s (line, comma-line));\n"
" }\n"
" }\n"
" if (*line == '%%' && line[1] == '%%')\n"
" mode++;\n"
" }\n"
" printf (\"#endif\\n\");\n"
"}\n"
"#endif\n",
cname, cname,
cname, cname,
cname
);
printf ("\n");
break;
case MODE_JSON:
printf ("}\n");
break;
@ -286,11 +355,13 @@ static int createdb(const char *f, const char **args, int nargs) {
}
static int showusage(int o) {
printf ("usage: sdb [-0cdehjJv|-D A B] [-|db] "
printf ("usage: sdb [-0cCdDehjJv|-D A B] [-|db] "
"[.file]|[-=]|==||[-+][(idx)key[:json|=value] ..]\n");
if (o == 2) {
printf (" -0 terminate results with \\x00\n"
" -c count the number of keys database\n"
" -C create foo.c and foo.h for sdb embedding\n"
" -G output database for gperf processing\n"
" -d decode base64 from stdin\n"
" -D diff two databases\n"
" -e encode stdin as base64\n"
@ -394,7 +465,7 @@ static bool dbdiff(const char *a, const char *b) {
return equal;
}
int showcount(const char *db) {
static int showcount(const char *db) {
ut32 d;
s = sdb_new (NULL, db, 0);
if (sdb_stats (s, &d, NULL)) {
@ -405,6 +476,47 @@ int showcount(const char *db) {
return 0;
}
static int gen_gperf(const char *file, const char *name) {
const size_t bufsz = 4096;
char *buf = malloc (bufsz);
char *out = malloc (strlen (file) +32);
snprintf(out, strlen (file) + 32, "%s.gperf", name);
int wd = open (out, O_RDWR);
ftruncate (wd, 0);
int rc = 0;
if (wd != -1) {
dup2 (1, 999);
dup2 (wd, 1);
rc = sdb_dump (file, MODE_CGEN);
fflush (stdout);
close (wd);
dup2 (999, 1);
} else {
snprintf (buf, bufsz, "sdb -G %s > %s.gperf\n", file, name);
rc = system (buf);
}
if (rc == 0) {
snprintf (buf, bufsz, "gperf -aclEDCIG --null-strings -H sdb_hash_c_%s"
" -N sdb_get_c_%s -t %s.gperf > %s.c\n", name, name, name, name);
rc = system (buf);
if (rc == 0) {
snprintf (buf, bufsz, "gcc -DMAIN=1 %s.c ; ./a.out > %s.h\n", name, name);
rc = system (buf);
if (rc == 0) {
eprintf ("Generated %s.c and %s.h\n", name, name);
}
} else {
eprintf ("%s\n", buf);
eprintf ("Cannot run gperf\n");
}
} else {
eprintf ("Outdated sdb binary in PATH?\n");
}
free (buf);
return rc;
}
int main(int argc, const char **argv) {
char *line;
const char *arg, *grep = NULL;
@ -447,7 +559,24 @@ int main(int argc, const char **argv) {
return showusage (1);
}
break;
case 'c': return (argc < 3)? showusage (1): showcount (argv[2]);
case 'G':
if (argc > 2) {
return sdb_dump (argv[db0 + 1], MODE_CGEN);
}
return showusage (1);
case 'c':
return (argc < 3)? showusage (1): showcount (argv[2]);
case 'C':
if (argc > 2) {
const char *file = argv[db0 + 1];
char *name = strdup (file);
char *p = strchr (name, '.');
if (p) *p = 0;
int rc = gen_gperf (file, name);
free (name);
return rc;
}
return showusage (1);
case 'v': return showversion ();
case 'h': return showusage (2);
case 'e': return base64encode ();

View File

@ -753,11 +753,11 @@ next_quote:
*json++ = 0;
ok = sdb_json_set (s, cmd, json, val, 0);
} else {
while (*val && isspace ((unsigned char)*val)) {
while (*val && isspace (*val)) {
val++;
}
int i = strlen (cmd) - 1;
while (i >= 0 && isspace ((unsigned char)cmd[i])) {
while (i >= 0 && isspace (cmd[i])) {
cmd[i] = '\0';
i--;
}

View File

@ -1,4 +1,4 @@
.Dd May 7, 2015
.Dd Feb 7, 2021
.Dt SDB 1
.Os
.Sh NAME
@ -17,6 +17,10 @@ SDB is a simple disk and memory string-based key-value database. It is based on
Decode stdin as base64 and prints to result to stdout
.It Fl D
Find differences between two databases
.It Fl C
Create the C and H files for a perfect hash implementation of the contents of the given Sdb
.It Fl G
Create the gperf file used by -C
.It Fl e
Encode stdin in base64 and prints to stdout
.It Fl h

View File

@ -1,4 +1,4 @@
/* sdb - MIT - Copyright 2011-2018 - pancake */
/* sdb - MIT - Copyright 2011-2021 - pancake */
#include <stdio.h>
#include <fcntl.h>
@ -60,13 +60,14 @@ SDB_API Sdb* sdb_new(const char *path, const char *name, int lock) {
s->db.fd = -1;
s->fd = -1;
s->refs = 1;
s->ht = sdb_ht_new ();
if (path && !*path) {
path = NULL;
}
if (name && *name && strcmp (name, "-")) {
if (path && *path) {
int plen = strlen (path);
int nlen = strlen (name);
size_t plen = strlen (path);
size_t nlen = strlen (name);
s->dir = malloc (plen + nlen + 2);
if (!s->dir) {
free (s);
@ -112,7 +113,6 @@ SDB_API Sdb* sdb_new(const char *path, const char *name, int lock) {
if (!s->ns) {
goto fail;
}
s->ht = sdb_ht_new ();
s->lock = lock;
// if open fails ignore
if (global_hook) {
@ -369,7 +369,7 @@ SDB_API bool sdb_exists(Sdb* s, const char *key) {
ut32 pos;
char ch;
bool found;
int klen = strlen (key) + 1;
size_t klen = strlen (key) + 1;
if (!s) {
return false;
}
@ -396,6 +396,9 @@ SDB_API int sdb_open(Sdb *s, const char *file) {
return -1;
}
if (file) {
if (sdb_text_check (s, file)) {
return sdb_text_load (s, file);
}
if (s->fd != -1) {
close (s->fd);
s->fd = -1;

View File

@ -218,6 +218,7 @@ SDB_API bool sdb_text_save_fd(Sdb *s, int fd, bool sort);
SDB_API bool sdb_text_save(Sdb *s, const char *file, bool sort);
SDB_API bool sdb_text_load_buf(Sdb *s, char *buf, size_t sz);
SDB_API bool sdb_text_load(Sdb *s, const char *file);
SDB_API bool sdb_text_check(Sdb *s, const char *file);
/* iterate */
SDB_API void sdb_dump_begin(Sdb* s);

View File

@ -1,4 +1,4 @@
/* sdb - MIT - Copyright 2020 - thestr4ng3r */
/* sdb - MIT - Copyright 2020-2021 - thestr4ng3r */
#include "sdb.h"
@ -265,7 +265,7 @@ static void load_process_line(LoadCtx *ctx) {
ctx->state = STATE_NEWLINE;
}
static inline char unescape_raw_char (char c) {
static inline char unescape_raw_char(char c) {
switch (c) {
case 'n':
return '\n';
@ -431,3 +431,38 @@ beach:
return r;
}
SDB_API bool sdb_text_check(Sdb *s, const char *file) {
char buf[64];
int fd = open (file, O_RDONLY | O_BINARY);
if (fd < 0) {
return false;
}
struct stat st;
if (fstat (fd, &st) || !st.st_size) {
close (fd);
return false;
}
int count = read (fd, buf, R_MIN (st.st_size, (off_t)sizeof (buf)));
close (fd);
bool is_ascii = true;
bool has_eq = false;
bool has_nl = false;
buf[sizeof (buf) - 1] = 0;
char *p = buf;
while (*p) {
if (*p == '=') {
has_eq = true;
} else if (*p == '\n') {
if (!has_eq) {
break;
}
has_nl = true;
} else if (!has_eq) {
if (*p < 10 || *p > '~') {
is_ascii = false;
}
}
p++;
}
return count > 4 && is_ascii && has_nl && has_eq;
}

View File

@ -39,6 +39,7 @@
#if __WIN32__ || __MINGW__ || __WINDOWS__ || _MSC_VER
#define __SDB_WINDOWS__ 1
#define DIRSEP '\\'
#define lseek _lseek
#include <windows.h>
#include <io.h>
#else