libretro/RetroArch
1
0
Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2025-02-12 12:39:56 +00:00
Code Issues Actions 25 Packages Projects Releases Wiki Activity

Merge pull request #2667 from aliaspider/master

Browse Source 
(libretro-db) c_converter: improve performance when merging multiple .dat files.
This commit is contained in:
Twinaphex 2016-01-24 04:21:21 +01:00
commit 6a7674dca1
3 changed files with 102 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libretro-db/Makefile
View File

@ -33,7 +33,6 @@ PLAIN_CONVERTER_C = \
bintree.c \
query.c \
plain_converter.c \
$(LIBRETRO_COMMON_DIR)/hash/rhash.c \
$(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \
$(LIBRETRO_COMMON_DIR)/file/retro_file.c \
$(LIBRETRO_COMMON_DIR)/compat/compat_strl.c

174
libretro-db/c_converter.c
View File

@ -38,9 +38,12 @@ typedef struct
const char* fname;
} dat_converter_token_t;
typedef struct dat_converter_map_t dat_converter_map_t;
typedef struct dat_converter_list_t dat_converter_list_t;
typedef union dat_converter_list_item_t dat_converter_list_item_t;
typedef struct dat_converter_search_tree_t dat_converter_search_tree_t;
typedef struct dat_converter_bt_node_t dat_converter_bt_node_t;
struct dat_converter_map_t
{
@ -57,6 +60,7 @@ struct dat_converter_list_t
{
dat_converter_list_enum type;
dat_converter_list_item_t* values;
dat_converter_bt_node_t* bt_root;
int count;
int capacity;
};
@ -69,6 +73,20 @@ union dat_converter_list_item_t
dat_converter_list_t* list;
};
struct dat_converter_bt_node_t
{
int index;
dat_converter_bt_node_t* right;
dat_converter_bt_node_t* left;
};
int dat_converter_cmp_func(const void *a, const void *b, void *ctx)
{
return strcmp(a, b);
}
dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type)
{
dat_converter_list_t* list = malloc(sizeof(*list));
@ -76,9 +94,19 @@ dat_converter_list_t* dat_converter_list_create(dat_converter_list_enum type)
list->count = 0;
list->capacity = (1 << 2);
list->values = (dat_converter_list_item_t*)malloc(sizeof(*list->values) * list->capacity);
list->bt_root = NULL;
return list;
}
void dat_converter_bt_node_free(dat_converter_bt_node_t* node)
{
if(!node)
return;
dat_converter_bt_node_free(node->left);
dat_converter_bt_node_free(node->right);
free(node);
}
void dat_converter_list_free(dat_converter_list_t* list)
{
if (!list)
@ -95,6 +123,7 @@ void dat_converter_list_free(dat_converter_list_t* list)
if (list->values[list->count].map.type == DAT_CONVERTER_LIST_MAP)
dat_converter_list_free(list->values[list->count].map.value.list);
}
dat_converter_bt_node_free(list->bt_root);
break;
default:
break;
@ -104,6 +133,35 @@ void dat_converter_list_free(dat_converter_list_t* list)
free(list);
}
dat_converter_bt_node_t* dat_converter_bt_node_insert(dat_converter_list_t* list, dat_converter_bt_node_t** node, dat_converter_map_t* map)
{
assert(map->key);
assert(list->type == DAT_CONVERTER_MAP_LIST);
if(!*node)
{
*node = calloc(1, sizeof(dat_converter_bt_node_t));
return *node;
}
int diff = strcmp(list->values[(*node)->index].map.key, map->key);
if(diff < 0)
return dat_converter_bt_node_insert(list, &(*node)->left, map);
else if(diff > 0)
return dat_converter_bt_node_insert(list, &(*node)->right, map);
/* found match */
if(list->values[(*node)->index].map.type == DAT_CONVERTER_LIST_MAP)
dat_converter_list_free(list->values[(*node)->index].map.value.list);
list->values[(*node)->index].map = *map;
return NULL;
}
void dat_converter_list_append(dat_converter_list_t* dst, void* item)
{
if (dst->count == dst->capacity)
@ -127,26 +185,19 @@ void dat_converter_list_append(dat_converter_list_t* dst, void* item)
}
case DAT_CONVERTER_MAP_LIST:
{
int i;
dat_converter_map_t* map = (dat_converter_map_t*) item;
if(!map->key)
{
dst->values[dst->count].map = *map;
break;
}
for (i = 0; i < dst->count; i++)
{
if (map->key && dst->values[i].map.key && !strcmp(dst->values[i].map.key, map->key))
break;
}
if (i == dst->count)
dst->values[dst->count].map = *map;
else
{
if (dst->values[i].map.type == DAT_CONVERTER_LIST_MAP)
dat_converter_list_free(dst->values[i].map.value.list);
dst->values[i].map = *map;
return;
dat_converter_bt_node_t* new_node = dat_converter_bt_node_insert(dst, &dst->bt_root, map);
if(new_node)
{
dst->values[dst->count].map = *map;
new_node->index = dst->count;
}
else
return;
}
break;
}
@ -162,26 +213,6 @@ void dat_converter_list_append(dat_converter_list_t* dst, void* item)
dst->count++;
}
void dat_converter_token_list_dump(dat_converter_list_t* list)
{
int i;
switch (list->type)
{
case DAT_CONVERTER_TOKEN_LIST:
for (i = 0; i < list->count; i++)
printf("\x1B[31m(%6i)@%s:%5i:%-5i\x1B[0m\"%s\"\n", i,
list->values[i].token.fname,
list->values[i].token.line_no,
list->values[i].token.column,
list->values[i].token.label);
break;
default:
return;
}
}
dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path)
{
@ -240,7 +271,6 @@ dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path)
src++;
token.column++;
}
*src = '\0';
token.label = NULL;
dat_converter_list_append(token_list, &token);
@ -251,14 +281,13 @@ dat_converter_list_t* dat_converter_lexer(char* src, const char* dat_path)
dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token)
{
dat_converter_list_t* parsed_table = dat_converter_list_create(DAT_CONVERTER_MAP_LIST);
const char* key = NULL;
dat_converter_map_t map = {0};
dat_converter_list_item_t* current = *start_token;
while (current->token.label)
{
if (!key)
if (!map.key)
{
if (!strcmp(current->token.label, ")"))
{
@ -276,7 +305,7 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token)
}
else
{
key = current->token.label;
map.key = current->token.label;
current++;
}
}
@ -285,9 +314,7 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token)
if (!strcmp(current->token.label, "("))
{
current++;
dat_converter_map_t map;
map.type = DAT_CONVERTER_LIST_MAP;
map.key = key;
map.value.list = dat_parser_table(&current);
dat_converter_list_append(parsed_table, &map);
}
@ -301,14 +328,12 @@ dat_converter_list_t* dat_parser_table(dat_converter_list_item_t** start_token)
}
else
{
dat_converter_map_t map;
map.type = DAT_CONVERTER_STRING_MAP;
map.key = key;
map.value.string = current->token.label;
dat_converter_list_append(parsed_table, &map);
current++;
}
key = NULL;
map.key = NULL;
}
}
@ -368,11 +393,14 @@ const char* dat_converter_get_match_key(dat_converter_list_t* list, const char*
return NULL;
}
dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_list_t* lexer_list, const char* match_key)
dat_converter_list_t* dat_converter_parser(dat_converter_list_t* target, dat_converter_list_t* lexer_list, const char* match_key)
{
const char* key = NULL;
bool skip = true;
dat_converter_list_item_t* current = lexer_list->values;
dat_converter_map_t map;
map.key = NULL;
map.type = DAT_CONVERTER_LIST_MAP;
if (!target)
{
@ -386,11 +414,11 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_lis
while (current->token.label)
{
if (!key)
if (!map.key)
{
if (!strcmp(current->token.label, "game"))
skip = false;
key = current->token.label;
map.key = current->token.label;
current++;
}
else
@ -398,12 +426,9 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_lis
if (!strcmp(current->token.label, "("))
{
current++;
dat_converter_map_t map;
map.type = DAT_CONVERTER_LIST_MAP;
map.key = NULL;
map.value.list = dat_parser_table(&current);
if (!skip)
{
{
if (match_key)
{
map.key = dat_converter_get_match_key(map.value.list, match_key);
@ -413,13 +438,16 @@ dat_converter_list_t* dat_parser(dat_converter_list_t* target, dat_converter_lis
dat_converter_exit(1);
}
}
else
map.key = NULL;
dat_converter_list_append(target, &map);
skip = true;
}
else
dat_converter_list_free(map.value.list);
key = NULL;
map.key = NULL;
}
else
{
@ -641,7 +669,6 @@ int main(int argc, char** argv)
const char* rdb_path;
const char* match_key = NULL;
RFILE* rdb_file;
int i;
if (argc < 2)
{
@ -664,33 +691,37 @@ int main(int argc, char** argv)
int dat_count = argc;
char** dat_buffers = (char**)malloc(dat_count * sizeof(*dat_buffers));
dat_converter_list_t** dat_lexer_lists = (dat_converter_list_t**)malloc(dat_count * sizeof(*dat_lexer_lists));
char** dat_buffer = dat_buffers;
dat_converter_list_t* dat_parser_list = NULL;
for (i = 0; i < dat_count; i++)
while(argc)
{
dat_converter_list_t* dat_lexer_list;
size_t dat_file_size;
FILE* dat_file = fopen(argv[i], "r");
FILE* dat_file = fopen(*argv, "r");
if (!dat_file)
{
printf("could not open dat file '%s': %s\n", argv[i], strerror(errno));
printf("could not open dat file '%s': %s\n", *argv, strerror(errno));
dat_converter_exit(1);
}
fseek(dat_file, 0, SEEK_END);
dat_file_size = ftell(dat_file);
fseek(dat_file, 0, SEEK_SET);
dat_buffers[i] = (char*)malloc(dat_file_size + 1);
fread(dat_buffers[i], 1, dat_file_size, dat_file);
*dat_buffer = (char*)malloc(dat_file_size + 1);
fread(*dat_buffer, 1, dat_file_size, dat_file);
fclose(dat_file);
dat_buffers[i][dat_file_size] = '\0';
(*dat_buffer)[dat_file_size] = '\0';
printf("Parsing dat file '%s'...\n", argv[i]);
dat_lexer_lists[i] = dat_converter_lexer(dat_buffers[i], argv[i]);
dat_parser_list = dat_parser(dat_parser_list, dat_lexer_lists[i], match_key);
// dat_converter_token_list_dump(*dat_lexer_dst);
printf("Parsing dat file '%s'...\n", *argv);
dat_lexer_list = dat_converter_lexer(*dat_buffer, *argv);
dat_parser_list = dat_converter_parser(dat_parser_list, dat_lexer_list, match_key);
dat_converter_list_free(dat_lexer_list);
argc--;
argv++;
dat_buffer++;
}
rdb_file = retro_fopen(rdb_path, RFILE_MODE_WRITE, -1);
@ -710,16 +741,11 @@ int main(int argc, char** argv)
retro_fclose(rdb_file);
for (i = 0; i < dat_count; i++)
{
dat_converter_list_free(dat_lexer_lists[i]);
free(dat_buffers[i]);
}
free(dat_lexer_lists);
free(dat_buffers);
dat_converter_list_free(dat_parser_list);
while(dat_count--)
free(dat_buffers[dat_count]);
free(dat_buffers);
return 0;
}

4
libretro-db/c_converter_test.sh
View File

@ -18,7 +18,7 @@ for dat_file in $DAT_dir/*.dat ; do
name=`echo "$name" | sed "s/\.dat//"`
./lua_converter "$lua_RDB_outdir/$name.rdb" dat_converter.lua "$dat_file"
done
#./lua_converter "$lua_RDB_outdir/merged.rdb" dat_converter.lua rom.crc $DAT_dir/N*.dat
./lua_converter "$lua_RDB_outdir/merged.rdb" dat_converter.lua rom.sha1 $DAT_dir/N*.dat
echo
echo "==========================================================="
@ -34,7 +34,7 @@ for dat_file in $DAT_dir/*.dat ; do
name=`echo "$name" | sed "s/\.dat//"`
./c_converter "$c_RDB_outdir/$name.rdb" "$dat_file"
done
#./c_converter "$c_RDB_outdir/merged.rdb" rom.crc $DAT_dir/N*.dat
./c_converter "$c_RDB_outdir/merged.rdb" rom.sha1 $DAT_dir/N*.dat
echo
echo "==========================================================="