Get rid of leiradel's db converter files

This commit is contained in:
twinaphex 2016-01-26 08:36:44 +01:00
parent beb985dbaa
commit 767822bb46
7 changed files with 3 additions and 915 deletions

View File

@ -2,7 +2,7 @@ DEBUG = 0
LIBRETRO_COMMON_DIR := ../libretro-common
INCFLAGS = -I. -I$(LIBRETRO_COMMON_DIR)/include
TARGETS = rmsgpack_test libretrodb_tool plain_converter lua_converter c_converter
TARGETS = rmsgpack_test libretrodb_tool lua_converter c_converter
ifeq ($(DEBUG), 1)
CFLAGS = -g -O0 -Wall
@ -24,22 +24,6 @@ LUA_CONVERTER_C = \
LUA_CONVERTER_OBJS := $(LUA_CONVERTER_C:.c=.o)
PLAIN_CONVERTER_C = \
lexer.c \
parser.c \
rmsgpack.c \
rmsgpack_dom.c \
libretrodb.c \
bintree.c \
query.c \
plain_converter.c \
$(LIBRETRO_COMMON_DIR)/hash/rhash.c \
$(LIBRETRO_COMMON_DIR)/compat/compat_fnmatch.c \
$(LIBRETRO_COMMON_DIR)/file/retro_file.c \
$(LIBRETRO_COMMON_DIR)/compat/compat_strl.c
PLAIN_CONVERTER_OBJS := $(PLAIN_CONVERTER_C:.c=.o)
C_CONVERTER_C = \
rmsgpack.c \
rmsgpack_dom.c \
@ -97,9 +81,6 @@ all: $(TARGETS)
%.o: %.c
$(CC) $(INCFLAGS) $< -c $(CFLAGS) -o $@
plain_converter: $(PLAIN_CONVERTER_OBJS)
$(CC) $(INCFLAGS) $(PLAIN_CONVERTER_OBJS) -o $@
lua_converter: $(LUA_CONVERTER_OBJS)
$(CC) $(INCFLAGS) $(LUA_CONVERTER_OBJS) $(LUA_FLAGS) -o $@
@ -120,4 +101,4 @@ check: testlib.so tests.lua
lua ./tests.lua
clean:
rm -rf $(TARGETS) $(PLAIN_CONVERTER_OBJS) $(LUA_CONVERTER_OBJS) $(C_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) testlib.so
rm -rf $(TARGETS) $(LUA_CONVERTER_OBJS) $(C_CONVERTER_OBJS) $(RARCHDB_TOOL_OBJS) $(RMSGPACK_OBJS) $(TESTLIB_OBJS) testlib.so

View File

@ -1,5 +1,4 @@
#!/bin/sh
rdb_file="$1"
shift 1
./lua_converter "$rdb_file" dat_converter.lua "$@"
#./plain_converter "$rdb_file" "$@"
./c_converter "$rdb_file" dat_converter.lua "$@"

View File

@ -1,138 +0,0 @@
#include <string.h>
#include <ctype.h>
#include "lexer.h"
static void skip( lx_state_t* lexer )
{
if ( lexer->current < lexer->end )
{
lexer->current++;
}
}
int lx_next( lx_state_t* lexer )
{
/* skip spaces */
for ( ;; )
{
if ( isspace( *lexer->current ) )
{
if ( *lexer->current == '\n' )
{
lexer->line++;
}
skip( lexer );
}
else if ( *lexer->current != 0 )
{
break;
}
else
{
/* return LX_EOF if we've reached the end of the input */
lexer->lexeme.str = "<eof>";
lexer->lexeme.len = 5;
lexer->token = LX_EOF;
return 0;
}
}
lexer->lexeme.str = lexer->current;
/* if the last token was the "version" tag, parse anything until a blank */
if ( lexer->last_was_version )
{
do
{
skip( lexer );
}
while ( !isspace( *lexer->current ) );
lexer->lexeme.len = lexer->current - lexer->lexeme.str;
lexer->token = LX_VERSION;
lexer->last_was_version = 0;
return 0;
}
/* if the character is alphabetic or '_', the token is an identifier */
if ( isalpha( *lexer->current ) || *lexer->current == '_' )
{
/* get all alphanumeric and '_' characters */
do
{
skip( lexer );
}
while ( isalnum( *lexer->current ) || *lexer->current == '_' );
lexer->lexeme.len = lexer->current - lexer->lexeme.str;
lexer->token = LX_TAG;
lexer->last_was_version = !strncmp( lexer->lexeme.str, "version", 7 );
return 0;
}
/* if the character is an hex digit, the token is a number */
if ( isxdigit( *lexer->current ) )
{
do
{
skip( lexer );
}
while ( isxdigit( *lexer->current ) );
lexer->lexeme.len = lexer->current - lexer->lexeme.str;
lexer->token = LX_NUMBER;
return 0;
}
/* if the character is a quote, it's a string */
if ( *lexer->current == '"' )
{
/* get anything until another quote */
do
{
skip( lexer );
if ( *lexer->current == '"' && lexer->current[ -1 ] != '\\' )
{
break;
}
}
while ( lexer->current < lexer->end );
if ( lexer->current == lexer->end )
{
return LX_UNTERMINATED_STRING;
}
skip( lexer );
lexer->lexeme.str++;
lexer->lexeme.len = lexer->current - lexer->lexeme.str - 1;
lexer->token = LX_STRING;
return 0;
}
/* otherwise the token is a symbol */
lexer->lexeme.len = 1;
switch ( *lexer->current++ )
{
case '(': lexer->token = LX_LPAREN; return 0;
case ')': lexer->token = LX_RPAREN; return 0;
}
return LX_INVALID_CHARACTER;
}
void lx_new( lx_state_t* lexer, const char* source, unsigned srclen )
{
lexer->line = 1;
lexer->current = source;
lexer->end = source + srclen;
lexer->lexeme.str = NULL;
lexer->lexeme.len = 0;
lexer->token = 0;
lexer->last_was_version = 0;
}

View File

@ -1,48 +0,0 @@
#ifndef LEXER_H
#define LEXER_H
enum
{
LX_UNTERMINATED_STRING = -1,
LX_INVALID_CHARACTER = -2,
};
enum
{
LX_EOF = 256,
LX_TAG,
LX_NUMBER,
LX_STRING,
LX_VERSION,
LX_LPAREN,
LX_RPAREN,
};
typedef struct
{
const char* str;
unsigned len;
}
lx_string_t;
typedef struct
{
/* source code */
int line;
/* lexer state */
const char* current;
const char* end;
int last_was_version;
/* lookahead */
int token;
lx_string_t lexeme;
}
lx_state_t;
int lx_next( lx_state_t* lexer );
void lx_new( lx_state_t* lexer, const char* source, unsigned srclen );
#endif /* LEXER_H */

View File

@ -1,157 +0,0 @@
#include <malloc.h>
#include <string.h>
#include "parser.h"
static void libretrodb_parser_match_any(pr_state_t* parser)
{
switch (lx_next( &parser->lexer))
{
case LX_UNTERMINATED_STRING:
longjmp( parser->env, PR_UNTERMINATED_STRING );
break;
case LX_INVALID_CHARACTER:
longjmp( parser->env, PR_INVALID_CHARACTER );
break;
}
}
static void libretrodb_parser_match( pr_state_t* parser, int token )
{
if (parser->lexer.token != token)
longjmp( parser->env, PR_UNEXPECTED_TOKEN );
libretrodb_parser_match_any( parser );
}
static void libretrodb_parser_match_tag( pr_state_t* parser, const char* tag )
{
if ( parser->lexer.token != LX_TAG || strncmp( parser->lexer.lexeme.str, tag, strlen( tag ) ) )
longjmp( parser->env, PR_UNEXPECTED_TOKEN );
libretrodb_parser_match_any( parser );
}
static int libretrodb_parser_cmp_keys( const pr_key_t* key1, const pr_key_t* key2 )
{
if ( !key1 && !key2 )
return 1;
if ( ( key1 && !key2 ) || ( !key1 && key2 ) )
return 0;
if ( key1->key.len != key2->key.len || strncmp( key1->key.str, key2->key.str, key1->key.len ) )
return 0;
return libretrodb_parser_cmp_keys( key1->prev, key2->prev );
}
static pr_pair_t* libretrodb_parser_find_key( pr_node_t* node, const pr_key_t* key )
{
int i;
for ( i = 0; i < node->count; i++ )
{
const pr_key_t* other = &node->pairs[ i ].key;
if (libretrodb_parser_cmp_keys(key, other))
return node->pairs + i;
}
return node->pairs + node->count++;
}
static void parse_value( pr_state_t* parser, const pr_key_t* key, pr_node_t* node )
{
pr_pair_t* pair = libretrodb_parser_find_key( node, key );
int token = parser->lexer.token;
pair->key = *key;
pair->value = parser->lexer.lexeme;
if ( token == LX_STRING || token == LX_NUMBER || token == LX_VERSION || token == LX_TAG )
libretrodb_parser_match_any( parser );
else
longjmp( parser->env, PR_UNEXPECTED_TOKEN );
}
static void libretrodb_parse_map( pr_state_t* parser, const pr_key_t* prev, int skip )
{
pr_node_t dummy;
pr_node_t* node;
pr_key_t key;
if ( skip )
{
dummy.count = 0;
node = &dummy;
}
else
node = parser->node;
libretrodb_parser_match( parser, LX_LPAREN );
while ( parser->lexer.token != LX_RPAREN )
{
key.key = parser->lexer.lexeme;
key.prev = prev;
libretrodb_parser_match( parser, LX_TAG );
if ( parser->lexer.token == LX_LPAREN )
libretrodb_parse_map( parser, &key, skip );
else
parse_value( parser, &key, node );
}
libretrodb_parser_match_any( parser );
}
static void libretrodb_parser_parse_clrmamepro( pr_state_t* parser )
{
static const pr_key_t clrmamepro = { { "clrmamepro", 10 }, NULL };
libretrodb_parser_match_tag( parser, clrmamepro.key.str );
libretrodb_parse_map( parser, &clrmamepro, 1 );
}
static void libretrodb_parser_parse_game( pr_state_t* parser )
{
static const pr_key_t game = { { "game", 4 }, NULL };
libretrodb_parser_match_tag( parser, game.key.str );
pr_node_t* node = (pr_node_t*)malloc( sizeof( pr_node_t ) );
if ( node == NULL )
longjmp( parser->env, PR_OUT_OF_MEMORY );
node->count = 0;
parser->node = node;
*parser->prev = node;
parser->prev = &node->next;
libretrodb_parse_map( parser, NULL, 0 );
}
void pr_new( pr_state_t* parser, const char* source, unsigned srclen )
{
lx_new( &parser->lexer, source, srclen );
parser->prev = &parser->first;
}
int pr_parse( pr_state_t* parser )
{
int res;
if ( ( res = setjmp( parser->env ) ) == 0 )
{
libretrodb_parser_match_any( parser );
libretrodb_parser_parse_clrmamepro( parser );
while ( parser->lexer.token != LX_EOF )
libretrodb_parser_parse_game( parser );
}
*parser->prev = NULL;
return res;
}

View File

@ -1,56 +0,0 @@
#ifndef PARSER_H
#define PARSER_H
#include <setjmp.h>
#include "lexer.h"
#define PR_MAX_FIELDS 64
enum
{
PR_UNTERMINATED_STRING = -1,
PR_INVALID_CHARACTER = -2,
PR_UNEXPECTED_TOKEN = -3,
PR_OUT_OF_MEMORY = -4,
};
typedef struct pr_key_t pr_key_t;
struct pr_key_t
{
lx_string_t key;
const pr_key_t* prev;
};
typedef struct
{
pr_key_t key;
lx_string_t value;
}
pr_pair_t;
typedef struct pr_node_t pr_node_t;
struct pr_node_t
{
pr_pair_t pairs[ PR_MAX_FIELDS ];
unsigned count;
pr_node_t* next;
};
typedef struct
{
lx_state_t lexer;
pr_node_t* node;
pr_node_t* first;
pr_node_t** prev;
jmp_buf env;
}
pr_state_t;
void pr_new( pr_state_t* parser, const char* source, unsigned srclen );
int pr_parse( pr_state_t* parser );
#endif /* PARSER_H */

View File

@ -1,493 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <malloc.h>
#include <errno.h>
#include "parser.h"
#include "libretrodb.h"
static int cmpkeystr_( const pr_key_t* key, const char** str )
{
int res = 1;
if ( key->prev )
res = cmpkeystr_( key->prev, str );
res = res && !strncmp( key->key.str, *str, key->key.len );
( *str ) += key->key.len + 1;
return res;
}
static int cmpkeystr( const pr_key_t* key, const char* str )
{
return cmpkeystr_( key, &str );
}
static int cmpkeys( const pr_key_t* key1, const pr_key_t* key2 )
{
if ( !key1 && !key2 )
{
return 1;
}
if ( ( key1 && !key2 ) || ( !key1 && key2 ) )
{
return 0;
}
if ( key1->key.len != key2->key.len || strncmp( key1->key.str, key2->key.str, key1->key.len ) )
{
return 0;
}
return cmpkeys( key1->prev, key2->prev );
}
static const char* printchar( pr_state_t* parser )
{
static char k[ 16 ];
if ( *parser->lexer.current < 32 )
{
snprintf( k, sizeof( k ), "\\x%02x", (unsigned char)*parser->lexer.current );
k[ sizeof( k ) - 1 ] = 0;
}
else
{
k[ 0 ] = *parser->lexer.current;
k[ 1 ] = 0;
}
return k;
}
#if 0
static void printkey( const pr_key_t* key )
{
int i;
if ( key->prev )
{
printkey( key->prev );
printf( "." );
}
for ( i = 0; i < key->key.len; i++ )
{
printf( "%c", key->key.str[ i ] );
}
}
static void printstr( const lx_string_t* str )
{
int i;
for ( i = 0; i < str->len; i++ )
{
printf( "%c", str->str[ i ] );
}
}
#endif
static const char* printtoken( pr_state_t* parser )
{
static char k[ 256 ];
char* aux = k;
const char* end = aux + sizeof( k ) - 1;
while ( parser->lexer.lexeme.len-- && aux < end )
{
*aux++ = *parser->lexer.lexeme.str++;
}
*aux = 0;
return k;
}
static char* dup_string( const lx_string_t* str )
{
char* dup = (char*)malloc( str->len + 1 );
if ( dup )
{
memcpy( (void*)dup, (const void*)str->str, str->len );
dup[ str->len ] = 0;
}
return dup;
}
static unsigned char* dup_hexa( const lx_string_t* str )
{
unsigned len = str->len / 2;
unsigned char* dup = (unsigned char*)malloc( len );
unsigned char* aux = dup;
char byte[ 3 ];
byte[ 2 ] = 0;
if ( dup )
{
const char* s = str->str;
while ( len-- )
{
byte[ 0 ] = *s++;
byte[ 1 ] = *s++;
*aux++ = strtol( byte, NULL, 16 );
}
}
return dup;
}
static void* dup_memory( const lx_string_t* str )
{
void* dup = (unsigned char*)malloc( str->len );
if ( dup )
{
memcpy( dup, (void*)str->str, str->len );
}
return dup;
}
static void merge( pr_node_t* game, pr_node_t* game2, const char* key )
{
unsigned i, j, k, found, merged;
pr_node_t* game1;
pr_node_t* newgame;
pr_key_t* key1;
lx_string_t* value1;
pr_key_t* key2;
lx_string_t* value2;
for ( ; game2; game2 = game2->next )
{
value2 = NULL;
for ( i = 0; i < game2->count; i++ )
{
key2 = &game2->pairs[ i ].key;
if ( cmpkeystr( key2, key ) )
{
value2 = &game2->pairs[ i ].value;
break;
}
}
if ( !value2 )
{
continue;
}
merged = 0;
for ( game1 = game, value1 = NULL; game1; game1 = game1->next )
{
for ( i = 0; i < game1->count; i++ )
{
key1 = &game1->pairs[ i ].key;
if ( cmpkeystr( key1, key ) )
{
value1 = &game1->pairs[ i ].value;
if ( value1->len == value2->len && !strncmp( value1->str, value2->str, value1->len ) )
{
merged = 1;
for ( k = 0; k < game2->count; k++ )
{
key2 = &game2->pairs[ k ].key;
found = game1->count;
for ( j = 0; j < game1->count; j++ )
{
key1 = &game1->pairs[ j ].key;
if ( cmpkeys( key1, key2 ) )
{
found = j;
break;
}
}
if ( found == game1->count )
{
game1->pairs[ found ] = game2->pairs[ k ];
game1->count++;
}
else if ( game2->pairs[ k ].value.len )
{
/* only overwrite if value is not empty */
game1->pairs[ found ] = game2->pairs[ k ];
}
}
}
}
}
}
if ( !merged )
{
/* add */
newgame = (pr_node_t*)malloc( sizeof( pr_node_t ) );
*newgame = *game2;
newgame->next = game->next;
game->next = newgame;
}
}
}
typedef struct
{
const char* key;
const char* new_key;
int type;
}
mapping_t;
enum
{
TYPE_STRING,
TYPE_UINT,
TYPE_HEXA,
TYPE_RAW
}
type_t;
static int provider( void* ctx, struct rmsgpack_dom_value* out )
{
static mapping_t map[] =
{
{ "name", "name", TYPE_STRING },
{ "description", "description", TYPE_STRING },
{ "rom.name", "rom_name", TYPE_STRING },
{ "rom.size", "size", TYPE_UINT },
{ "users", "users", TYPE_UINT },
{ "releasemonth", "releasemonth", TYPE_UINT },
{ "releaseyear", "releaseyear", TYPE_UINT },
{ "rumble", "rumble", TYPE_UINT },
{ "analog", "analog", TYPE_UINT },
{ "famitsu_rating", "famitsu_rating", TYPE_UINT },
{ "edge_rating", "edge_rating", TYPE_UINT },
{ "edge_issue", "edge_issue", TYPE_UINT },
{ "edge_review", "edge_review", TYPE_STRING },
{ "enhancement_hw", "enhancement_hw", TYPE_STRING },
{ "barcode", "barcode", TYPE_STRING },
{ "esrb_rating", "esrb_rating", TYPE_STRING },
{ "elspa_rating", "elspa_rating", TYPE_STRING },
{ "pegi_rating", "pegi_rating", TYPE_STRING },
{ "cero_rating", "cero_rating", TYPE_STRING },
{ "franchise", "franchise", TYPE_STRING },
{ "developer", "developer", TYPE_STRING },
{ "publisher", "publisher", TYPE_STRING },
{ "origin", "origin", TYPE_STRING },
{ "rom.crc", "crc", TYPE_HEXA },
{ "rom.md5", "md5", TYPE_HEXA },
{ "rom.sha1", "sha1", TYPE_HEXA },
{ "serial", "serial", TYPE_RAW },
{ "rom.serial", "serial", TYPE_RAW },
};
unsigned i, j, index, fields;
pr_node_t** game_ptr = (pr_node_t**)ctx;
pr_node_t* game = *game_ptr;
if ( game == NULL )
{
return 1;
}
*game_ptr = game->next;
out->type = RDT_MAP;
out->val.map.items = calloc( game->count, sizeof( struct rmsgpack_dom_pair ) );
index = fields = 0;
for ( i = 0; i < game->count; i++ )
{
out->val.map.items[ i ].key.type = RDT_STRING;
for ( j = 0; j < sizeof( map ) / sizeof( map[ 0 ] ); j++ )
{
if ( ( fields & ( 1 << j ) ) == 0 && cmpkeystr( &game->pairs[ i ].key, map[ j ].key ) )
{
fields |= 1 << j; /* avoid overrides */
out->val.map.items[ index ].key.val.string.len = strlen( map[ j ].new_key );
out->val.map.items[ index ].key.val.string.buff = strdup( map[ j ].new_key );
switch ( map[ j ].type )
{
case TYPE_STRING:
out->val.map.items[ index ].value.type = RDT_STRING;
out->val.map.items[ index ].value.val.string.len = game->pairs[ i ].value.len;
out->val.map.items[ index ].value.val.string.buff = dup_string( &game->pairs[ i ].value );
index++;
break;
case TYPE_HEXA:
out->val.map.items[ index ].value.type = RDT_BINARY;
out->val.map.items[ index ].value.val.binary.len = game->pairs[ i ].value.len / 2;
/* FIXME/TODO - warning: pointer targets in assignment differ in signedness */
out->val.map.items[ index ].value.val.binary.buff = dup_hexa( &game->pairs[ i ].value );
index++;
break;
case TYPE_UINT:
out->val.map.items[ index ].value.type = RDT_UINT;
out->val.map.items[ index ].value.val.uint_ = strtol( game->pairs[ i ].value.str, NULL, 10 );
index++;
break;
case TYPE_RAW:
out->val.map.items[ index ].value.type = RDT_BINARY;
out->val.map.items[ index ].value.val.binary.len = game->pairs[ i ].value.len;
out->val.map.items[ index ].value.val.binary.buff = dup_memory( &game->pairs[ i ].value );
index++;
break;
default:
fprintf( stderr, "Unhandled type in mapping %u (%s => %s)\n", j, map[ j ].key, map[ j ].new_key );
break;
}
break;
}
}
}
out->val.map.len = index;
return 0;
}
static const char* read_file( const char* filename, unsigned* size )
{
char* source;
RFILE* file = retro_fopen( filename, RFILE_MODE_READ, -1 );
if ( file == NULL )
{
fprintf( stderr, "Error opening file: %s\n", strerror( errno ) );
return NULL;
}
retro_fseek( file, 0, SEEK_END );
*size = retro_ftell( file );
retro_fseek( file, 0, SEEK_SET );
source = (char*)malloc( *size + 1 );
if ( source == NULL )
{
retro_fclose( file );
fprintf( stderr, "Out of memory\n" );
return NULL;
}
retro_fread( file, (void*)source, *size );
retro_fclose( file );
source[ *size ] = 0;
return source;
}
static int parse( pr_state_t* parser, const char* filename, const char* source, unsigned size, const char* key )
{
pr_node_t** prev;
pr_node_t* game;
int i, found;
printf( "Parsing dat file '%s'...\n", filename );
switch ( pr_parse( parser ) )
{
case PR_UNTERMINATED_STRING: fprintf( stderr, "%s:%u: Unterminated string\n", filename, parser->lexer.line ); return -1;
case PR_INVALID_CHARACTER: fprintf( stderr, "%s:%u: Invalid character '%s'\n", filename, parser->lexer.line, printchar( parser ) ); return -1;
case PR_UNEXPECTED_TOKEN: fprintf( stderr, "%s:%u: Unexpected token \"%s\"\n", filename, parser->lexer.line, printtoken( parser ) ); return -1;
case PR_OUT_OF_MEMORY: fprintf( stderr, "%s:%u: Out of memory\n", filename, parser->lexer.line ); return -1;
}
prev = &parser->first;
game = parser->first;
while ( game )
{
found = 0;
for ( i = 0; i < game->count; i++ )
{
if ( cmpkeystr( &game->pairs[ i ].key, key ) )
{
found = 1;
break;
}
}
if ( found )
{
prev = &game->next;
}
else
{
*prev = game->next;
}
game = game->next;
}
return 0;
}
int main( int argc, const char* argv[] )
{
const char* source;
unsigned size;
pr_state_t parser;
pr_node_t* game;
RFILE* out;
int i, res = 1;
if ( argc < 4 )
{
fprintf( stderr, "usage:\nplain_converter <db file> <key name> <dat files...>\n\n" );
return 1;
}
source = read_file( argv[ 3 ], &size );
pr_new( &parser, source, size );
if ( source && parse( &parser, argv[ 3 ], source, size, argv[ 2 ] ) == 0 )
{
game = parser.first;
for ( i = 4; i < argc; i++ )
{
source = read_file( argv[ i ], &size );
pr_new( &parser, source, size );
if ( source && parse( &parser, argv[ i ], source, size, argv[ 2 ] ) == 0 )
{
merge( game, parser.first, argv[ 2 ] );
}
}
out = retro_fopen( argv[ 1 ], RFILE_MODE_WRITE, -1 );
if ( out != NULL )
{
res = libretrodb_create( out, &provider, (void*)&game );
retro_fclose( out );
res = 0;
}
}
/* HACK Don't free anything, let the OS take care of that */
return res;
}