mirror of
https://github.com/YohannDR/mzm.git
synced 2024-11-26 22:40:42 +00:00
Atoi-less C extractor (#15)
* [C extractor] Custom atoi implementations Additional minor changes: - Better C extractor compilation script - Change spaces to tabs for indentation in C extractor - Minor changes in the C extractor - Fix typo in database.txt * Update README to use the C extractor * [C extractor] Better comments/documentation
This commit is contained in:
Bolu
GitHub
parent
2ec14130d9
commit
764254b754
@ -35,5 +35,7 @@ This produces the following ROMs:
|
||||
## Build
|
||||
|
||||
- Run `make clean` if necessary
|
||||
- Run the `extractor` script in tools (`./tools/extractor`) if necessary
|
||||
- Run the data extractor if necessary:
|
||||
* Compile: `cd tools && . compile_c_extractor.sh && cd ..`
|
||||
* Run: `tools/c_extractor`
|
||||
- Run `make` (using the -j option is recommended to speed up the process)
|
||||
|
||||
@ -479,7 +479,7 @@ sprites/FallingChozoPillar.pal;16;0x31cb00;2
|
||||
sprites/MechaRidley.gfx.lz;2525;0x31fcf4;4
|
||||
sprites/MechaRidleyWeapons.gfx.lz;1024;0x322468;4
|
||||
sprites/MechaRidleyDestroyed.gfx.lz;336;0x323468;4
|
||||
sprites/MechaRidley.pal;129;03239a8;2
|
||||
sprites/MechaRidley.pal;129;0x3239a8;2
|
||||
sprites/MechaRidley_323aaa.pal;12;0x323aaa;2
|
||||
sprites/MechaRidleyGreenGlow.pal;64;0x323ac2;2
|
||||
sprites/MechaRidley_323b42.pal;4;0x323b42;2
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
#!/bin/bash
|
||||
gcc -Wall -O3 -fopenmp extractor.c
|
||||
gcc -Wall -O3 -fopenmp extractor.c -o c_extractor
|
||||
|
||||
@ -14,55 +14,123 @@ const char *romPath = "mzm_us_baserom.gba";
|
||||
const char *databasePath = "database.txt";
|
||||
const char *dataPath = "data/";
|
||||
|
||||
void create_directories(const char *filePath)
|
||||
/*
|
||||
* Custom implementation of atoi() for decimal numbers, since atoi() seems to
|
||||
* segfault on some systems for some reason.
|
||||
*
|
||||
* This function converts a string into the 32-bit integer number it represents.
|
||||
*
|
||||
* This function is not general, and makes some assumptions about its input
|
||||
* string that are true for this particular case:
|
||||
* - it assumes it represents a decimal number
|
||||
* - it assumes the represented number fits in the 32-bit range
|
||||
* - it assumes the string only contains valid numeric characters, and ends
|
||||
* with a null terminator. It assumes it does not contain newline characters.
|
||||
*/
|
||||
int stoi_dec(const char *s)
|
||||
{
|
||||
// NOTE: filePath contains the full path of a given extracted data asset.
|
||||
// This function creates its full directory hierarchy.
|
||||
const char *lastSlash = strrchr(filePath, '/');
|
||||
size_t dirPathLen = lastSlash - filePath + 1;
|
||||
int n = 0;
|
||||
for (int i = 0; s[i] != '\0'; ++i)
|
||||
{
|
||||
n *= 10;
|
||||
n += s[i] - '0';
|
||||
}
|
||||
|
||||
char *dirPath = (char *)malloc((dirPathLen + 1) * sizeof(char));
|
||||
if (dirPath == NULL)
|
||||
{
|
||||
fprintf(stderr, "Memory allocation error for path %s.\n", filePath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// Retrieve the directory filePath from the fila filePath:
|
||||
strncpy(dirPath, filePath, dirPathLen);
|
||||
dirPath[dirPathLen] = '\0';
|
||||
|
||||
// Create directory hierarchy:
|
||||
char *p = strdup(dirPath);
|
||||
char *sep = strchr(p + 1, '/');
|
||||
while(sep != NULL)
|
||||
{
|
||||
*sep = '\0';
|
||||
mkdir(p, 0755);
|
||||
*sep = '/';
|
||||
sep = strchr(sep + 1, '/');
|
||||
}
|
||||
|
||||
free(p);
|
||||
free(dirPath);
|
||||
return n;
|
||||
}
|
||||
|
||||
/*
|
||||
* Custom implementation of atoi() for hexadecimal numbers, since atoi() seems
|
||||
* to segfault on some systems for some reason.
|
||||
*
|
||||
* This function converts a string into the 32-bit integer number it represents.
|
||||
*
|
||||
* This function is not general, and makes some assumptions about its input
|
||||
* string that are true for this particular case:
|
||||
* - it assumes it represents a hexadecimal number
|
||||
* - it assumes the string starts with "0x"
|
||||
* - it assumes the represented number fits in the 32-bit range
|
||||
* - it assumes the string only contains valid hex characters, and ends
|
||||
* with a null terminator. It assumes it does not contain newline characters.
|
||||
*/
|
||||
int stoi_hex(const char *s)
|
||||
{
|
||||
int n = 0;
|
||||
for (int i = 2; s[i] != '\0'; ++i)
|
||||
{
|
||||
n *= 16;
|
||||
int digit = s[i] - '0';
|
||||
if (digit > 9) digit += -('a' - '0') + 10;
|
||||
n += digit;
|
||||
}
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
/*
|
||||
* Creates the entire directory hierarchy for the given file path, if it does
|
||||
* not exist yet.
|
||||
*/
|
||||
void create_directories(const char *filePath)
|
||||
{
|
||||
// NOTE: filePath contains the full path of a given extracted data asset.
|
||||
// This function creates its full directory hierarchy.
|
||||
const char *lastSlash = strrchr(filePath, '/');
|
||||
size_t dirPathLen = lastSlash - filePath + 1;
|
||||
|
||||
char *dirPath = (char *)malloc((dirPathLen + 1) * sizeof(char));
|
||||
if (dirPath == NULL)
|
||||
{
|
||||
fprintf(stderr, "Memory allocation error for path %s.\n", filePath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// Retrieve the directory path from the complete file path:
|
||||
strncpy(dirPath, filePath, dirPathLen);
|
||||
dirPath[dirPathLen] = '\0';
|
||||
|
||||
// Create directory hierarchy:
|
||||
char *p = strdup(dirPath);
|
||||
char *sep = strchr(p + 1, '/');
|
||||
while(sep != NULL)
|
||||
{
|
||||
*sep = '\0';
|
||||
mkdir(p, 0755);
|
||||
*sep = '/';
|
||||
sep = strchr(sep + 1, '/');
|
||||
}
|
||||
|
||||
free(p);
|
||||
free(dirPath);
|
||||
}
|
||||
|
||||
/*
|
||||
* Function passed to nftw() to recursively remove all files and directories
|
||||
* under a given path.
|
||||
* It basically does the "removing" part of "rm -rf".
|
||||
* fpath is a specific file or empty directory. It cannot be a non-empty
|
||||
* directory (i.e. does not work recursively). nftw() passes the files and
|
||||
* directories in the correct order so that a path (parent of fpath) is
|
||||
* recursively removed.
|
||||
* All the other arguments (sb, typeflag, ftwbuf) are needed as specified by
|
||||
* nftw(), but are unused in this function.
|
||||
*/
|
||||
int remove_file_or_directory(const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf)
|
||||
{
|
||||
int rv = remove(fpath);
|
||||
int rv = remove(fpath);
|
||||
|
||||
if (rv)
|
||||
{
|
||||
// If an error is found, print to stderr with errno value:
|
||||
perror(fpath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (rv)
|
||||
{
|
||||
// If an error is found, print to stderr with errno value:
|
||||
perror(fpath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
return rv;
|
||||
return rv;
|
||||
}
|
||||
|
||||
/*
|
||||
* Splits a string (s) by a given delimiter (del) into a a preallocated
|
||||
* Splits a string (s) by a given delimiter (del) into a preallocated
|
||||
* buffer (result), without allocating new memory. result should be properly
|
||||
* allocated to store all of the substrings.
|
||||
* s is edited as a byproduct of calling this function. None of the elements
|
||||
@ -70,154 +138,152 @@ int remove_file_or_directory(const char *fpath, const struct stat *sb, int typef
|
||||
*/
|
||||
void str_split(char *s, char del, char **result)
|
||||
{
|
||||
size_t idx = 0;
|
||||
char *start = s;
|
||||
char *p = s;
|
||||
while (*p != '\0')
|
||||
{
|
||||
if (*p == del)
|
||||
{
|
||||
size_t len = p - start;
|
||||
result[idx] = start;
|
||||
result[idx][len] = '\0';
|
||||
idx++;
|
||||
start = p + 1;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
// Last substring:
|
||||
size_t len = p - start;
|
||||
result[idx] = start;
|
||||
result[idx][len] = '\0';
|
||||
size_t idx = 0;
|
||||
char *start = s;
|
||||
char *p = s;
|
||||
while (*p != '\0')
|
||||
{
|
||||
if (*p == del)
|
||||
{
|
||||
*p = '\0'; // Change delimiter to NULL terminator in-place.
|
||||
result[idx] = start;
|
||||
idx++;
|
||||
start = p + 1;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
// Last substring:
|
||||
result[idx] = start;
|
||||
}
|
||||
|
||||
int main()
|
||||
int main(void)
|
||||
{
|
||||
FILE *romFile = fopen(romPath, "rb");
|
||||
if (!romFile)
|
||||
{
|
||||
fprintf(stderr, "The ROM file (%s) does not exist.\n", romPath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
fclose(romFile);
|
||||
FILE *romFile = fopen(romPath, "rb");
|
||||
if (!romFile)
|
||||
{
|
||||
fprintf(stderr, "The ROM file (%s) does not exist.\n", romPath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
fclose(romFile);
|
||||
|
||||
FILE *databaseFile = fopen(databasePath, "r");
|
||||
if (!databaseFile)
|
||||
{
|
||||
fprintf(stderr, "The database file (%s) does not exist.\n", databasePath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
FILE *databaseFile = fopen(databasePath, "r");
|
||||
if (!databaseFile)
|
||||
{
|
||||
fprintf(stderr, "The database file (%s) does not exist.\n", databasePath);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// Declare important variables:
|
||||
int nthreads = omp_get_num_procs();
|
||||
char *database;
|
||||
size_t databaseSize = 0;
|
||||
// Declare important variables:
|
||||
int nthreads = omp_get_num_procs();
|
||||
char *database;
|
||||
size_t databaseSize = 0;
|
||||
|
||||
#pragma omp parallel sections
|
||||
{
|
||||
// Concurrently delete previously extracted data, if any:
|
||||
#pragma omp section
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(dataPath, &st) == 0)
|
||||
{
|
||||
printf("Deleting old files...\n");
|
||||
nftw(dataPath, remove_file_or_directory, 10, FTW_DEPTH | FTW_PHYS);
|
||||
}
|
||||
}
|
||||
#pragma omp parallel sections
|
||||
{
|
||||
// Concurrently delete previously extracted data, if any:
|
||||
#pragma omp section
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(dataPath, &st) == 0)
|
||||
{
|
||||
printf("Deleting old files...\n");
|
||||
// Basically a "rm -rf dataPath":
|
||||
nftw(dataPath, remove_file_or_directory, 10, FTW_DEPTH | FTW_PHYS);
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrently parse the database file:
|
||||
#pragma omp section
|
||||
{
|
||||
printf("Using %d parallel threads.\n", nthreads);
|
||||
// Concurrently parse the database file:
|
||||
#pragma omp section
|
||||
{
|
||||
printf("Using %d parallel threads.\n", nthreads);
|
||||
|
||||
// Get number of database entries:
|
||||
char line[MAX_PATH_LENGTH];
|
||||
while (fgets(line, sizeof(line), databaseFile) != NULL)
|
||||
{
|
||||
if (line[0] == '#' || line[0] == '\n')
|
||||
continue;
|
||||
databaseSize++;
|
||||
}
|
||||
rewind(databaseFile);
|
||||
// Get number of database entries:
|
||||
char line[MAX_PATH_LENGTH];
|
||||
while (fgets(line, sizeof(line), databaseFile) != NULL)
|
||||
{
|
||||
if (line[0] == '#' || line[0] == '\n')
|
||||
continue;
|
||||
databaseSize++;
|
||||
}
|
||||
rewind(databaseFile);
|
||||
|
||||
// Create and populate database array:
|
||||
database = (char *)malloc(databaseSize * MAX_PATH_LENGTH * sizeof(char));
|
||||
if (database == NULL)
|
||||
{
|
||||
fprintf(stderr, "Error allocating buffer for the database\n.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
// Create and populate database array:
|
||||
database = (char *)malloc(databaseSize * MAX_PATH_LENGTH * sizeof(char));
|
||||
if (database == NULL)
|
||||
{
|
||||
fprintf(stderr, "Error allocating buffer for the database\n.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
size_t i = 0;
|
||||
while (fgets(line, sizeof(line), databaseFile) != NULL)
|
||||
{
|
||||
if (line[0] == '#' || line[0] == '\n')
|
||||
continue;
|
||||
strcpy(&database[i++ * MAX_PATH_LENGTH], line);
|
||||
}
|
||||
}
|
||||
}
|
||||
fclose(databaseFile);
|
||||
size_t i = 0;
|
||||
while (fgets(line, sizeof(line), databaseFile) != NULL)
|
||||
{
|
||||
if (line[0] == '#' || line[0] == '\n')
|
||||
continue;
|
||||
strcpy(&database[i++ * MAX_PATH_LENGTH], line);
|
||||
}
|
||||
}
|
||||
}
|
||||
fclose(databaseFile);
|
||||
|
||||
printf("Extracting data...\n");
|
||||
// Parallely extract all data across all available CPU threads:
|
||||
#pragma omp parallel
|
||||
{
|
||||
//int tid = omp_get_thread_num();
|
||||
//printf("Thread %d started.\n", tid);
|
||||
printf("Extracting data...\n");
|
||||
// Parallely extract all data across all available CPU threads:
|
||||
#pragma omp parallel
|
||||
{
|
||||
//int tid = omp_get_thread_num();
|
||||
//printf("Thread %d started.\n", tid);
|
||||
|
||||
// Each thread has an open copy of the ROM:
|
||||
FILE *romFile = fopen(romPath, "rb");
|
||||
if (!romFile)
|
||||
{
|
||||
fprintf(stderr, "Error opening ROM file.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
// Each thread has an open copy of the ROM:
|
||||
FILE *romFile = fopen(romPath, "rb");
|
||||
if (!romFile)
|
||||
{
|
||||
fprintf(stderr, "Error opening ROM file.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
#pragma omp for
|
||||
for (size_t i = 0; i < databaseSize; ++i)
|
||||
{
|
||||
char *split[4];
|
||||
str_split(&database[i * MAX_PATH_LENGTH], ';', split);
|
||||
#pragma omp for
|
||||
for (size_t i = 0; i < databaseSize; ++i)
|
||||
{
|
||||
char *split[4];
|
||||
str_split(&database[i * MAX_PATH_LENGTH], ';', split);
|
||||
|
||||
// Extract data:
|
||||
char *name = split[0];
|
||||
int length = atoi(split[1]);
|
||||
int size = atoi(split[3]);
|
||||
int address = strtol(split[2], NULL, 16);
|
||||
// Extract data:
|
||||
char *name = split[0];
|
||||
int length = stoi_dec(split[1]);
|
||||
int size = split[3][0] - '0';
|
||||
int address = stoi_hex(split[2]);
|
||||
|
||||
printf("Extracting %s\n", name);
|
||||
char filePath[MAX_PATH_LENGTH];
|
||||
snprintf(filePath, sizeof(filePath), "%s%s", dataPath, name);
|
||||
printf("Extracting %s\n", name);
|
||||
char filePath[MAX_PATH_LENGTH];
|
||||
snprintf(filePath, sizeof(filePath), "%s%s", dataPath, name);
|
||||
|
||||
create_directories(filePath);
|
||||
char *data = (char *)malloc(length * size);
|
||||
if (!data)
|
||||
{
|
||||
fprintf(stderr, "Error allocating data buffer.\n");
|
||||
fclose(romFile);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
create_directories(filePath);
|
||||
char *data = (char *)malloc(length * size);
|
||||
if (!data)
|
||||
{
|
||||
fprintf(stderr, "Error allocating data buffer.\n");
|
||||
fclose(romFile);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
FILE *outFile = fopen(filePath, "wb");
|
||||
if (!outFile)
|
||||
{
|
||||
fprintf(stderr, "Error opening output file %s.\n", filePath);
|
||||
fclose(romFile);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
FILE *outFile = fopen(filePath, "wb");
|
||||
if (!outFile)
|
||||
{
|
||||
fprintf(stderr, "Error opening output file %s.\n", filePath);
|
||||
fclose(romFile);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
fseek(romFile, address, SEEK_SET);
|
||||
fread(data, size, length, romFile);
|
||||
fwrite(data, size, length, outFile);
|
||||
fseek(romFile, address, SEEK_SET);
|
||||
fread(data, size, length, romFile);
|
||||
fwrite(data, size, length, outFile);
|
||||
|
||||
fclose(outFile);
|
||||
free(data);
|
||||
}
|
||||
fclose(romFile);
|
||||
}
|
||||
free(database);
|
||||
fclose(outFile);
|
||||
free(data);
|
||||
}
|
||||
fclose(romFile);
|
||||
}
|
||||
free(database);
|
||||
|
||||
return 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user