libretro/RetroArch
1
0
Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2024-10-07 14:23:29 +00:00
Code Issues Actions 24 Packages Projects Releases Wiki Activity

First set of cleanups to archive handling

Browse Source 
- Don't read the entire archive file into memory on systems without HAVE_MMAP
- Support ZIP files between 2GB ~ 4GB on 32bit architectures
- Remove zip exclusive members from `file_archive_transfer`
- Add `zip_context_t` for zip like the already existing `sevenzip_context_t`
- Unify cleanups of zip and 7z in `file_archive_file_backend::archive_parse_file_free`
- Handle some sizes and offsets as int64_t (preparation for future Zip64 support)
- Handle compression methods in `zlib_stream_decompress_data_to_file_init` not in `file_archive_perform_mode`
- Return progress of 7z iterating with `file_archive_parse_file_progress`
This commit is contained in:
Bernhard Schelling 2020-07-12 00:51:35 +09:00
parent 347d1964ba
commit 0ddb073784
6 changed files with 395 additions and 518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
database_info.c
View File

@ -389,16 +389,6 @@ database_info_handle_t *database_info_dir_init(const char *dir,
db->list_ptr = 0;
db->list = list;
db->state.type = ARCHIVE_TRANSFER_NONE;
db->state.archive_size = 0;
db->state.start_delta = 0;
db->state.handle = NULL;
db->state.stream = NULL;
db->state.footer = NULL;
db->state.directory = NULL;
db->state.data = NULL;
db->state.backend = NULL;
return db;
}
@ -430,16 +420,6 @@ database_info_handle_t *database_info_file_init(const char *path,
db->list_ptr = 0;
db->list = list;
db->state.type = ARCHIVE_TRANSFER_NONE;
db->state.archive_size = 0;
db->state.start_delta = 0;
db->state.handle = NULL;
db->state.stream = NULL;
db->state.footer = NULL;
db->state.directory = NULL;
db->state.data = NULL;
db->state.backend = NULL;
return db;
}

1
database_info.h
View File

@ -76,7 +76,6 @@ typedef struct
enum database_type type;
size_t list_ptr;
struct string_list *list;
file_archive_transfer_t state;
} database_info_handle_t;
typedef struct

337
libretro-common/file/archive_file.c
View File

@ -24,19 +24,6 @@
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#ifdef HAVE_MMAP
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#endif
#include <compat/strl.h>
#include <file/archive_file.h>
#include <file/file_path.h>
@ -45,119 +32,6 @@
#include <lists/string_list.h>
#include <string/stdstring.h>
struct file_archive_file_data
{
#ifdef HAVE_MMAP
int fd;
#endif
void *data;
size_t size;
};
static size_t file_archive_size(file_archive_file_data_t *data)
{
if (!data)
return 0;
return data->size;
}
static const uint8_t *file_archive_data(file_archive_file_data_t *data)
{
if (!data)
return NULL;
return (const uint8_t*)data->data;
}
#ifdef HAVE_MMAP
/* Closes, unmaps and frees. */
static void file_archive_free(file_archive_file_data_t *data)
{
if (!data)
return;
if (data->data)
munmap(data->data, data->size);
if (data->fd >= 0)
close(data->fd);
free(data);
}
static file_archive_file_data_t* file_archive_open(const char *path)
{
file_archive_file_data_t *data = (file_archive_file_data_t*)
malloc(sizeof(*data));
if (!data)
return NULL;
data->fd = open(path, O_RDONLY);
data->data = NULL;
data->size = 0;
/* Failed to open archive. */
if (data->fd < 0)
goto error;
data->size = path_get_size(path);
if (!data->size)
return data;
data->data = mmap(NULL,
data->size, PROT_READ, MAP_SHARED, data->fd, 0);
if (data->data == MAP_FAILED)
{
data->data = NULL;
/* Failed to mmap() file */
goto error;
}
return data;
error:
file_archive_free(data);
return NULL;
}
#else
/* Closes, unmaps and frees. */
static void file_archive_free(file_archive_file_data_t *data)
{
if (!data)
return;
if(data->data)
free(data->data);
free(data);
}
static file_archive_file_data_t* file_archive_open(const char *path)
{
int64_t ret = -1;
bool read_from_file = false;
file_archive_file_data_t *data = (file_archive_file_data_t*)
malloc(sizeof(*data));
if (!data)
return NULL;
data->data = NULL;
data->size = 0;
read_from_file = filestream_read_file(
path, &data->data, &ret);
/* Failed to open archive? */
if (!read_from_file || ret < 0)
goto error;
data->size = ret;
return data;
error:
file_archive_free(data);
return NULL;
}
#endif
static int file_archive_get_file_list_cb(
const char *path,
const char *valid_exts,
@ -252,7 +126,7 @@ static int file_archive_extract_cb(const char *name, const char *valid_exts,
if (file_archive_perform_mode(new_path,
valid_exts, cdata, cmode, csize, size,
0, userdata))
checksum, userdata))
userdata->found_file = true;
return 0;
@ -280,14 +154,18 @@ static int file_archive_parse_file_init(file_archive_transfer_t *state,
if (!state->backend)
return -1;
state->handle = file_archive_open(path);
if (!state->handle)
state->archive_file = filestream_open(path,
RETRO_VFS_FILE_ACCESS_READ,
RETRO_VFS_FILE_ACCESS_HINT_NONE);
/* Failed to open archive. */
if (!state->archive_file)
return -1;
state->archive_size = (int32_t)file_archive_size(state->handle);
state->data = file_archive_data(state->handle);
state->footer = 0;
state->directory = 0;
state->archive_size = filestream_get_size(state->archive_file);
state->step_current = 0;
state->step_total = 0;
return state->backend->archive_parse_file_init(state, path);
}
@ -295,35 +173,25 @@ static int file_archive_parse_file_init(file_archive_transfer_t *state,
/**
* file_archive_decompress_data_to_file:
* @path : filename path of archive.
* @valid_exts : Valid extensions of archive to be parsed.
* If NULL, allow all.
* @cdata : input data.
* @csize : size of input data.
* @size : output file size
* @checksum : CRC32 checksum from input data.
*
* Decompress data to file.
* Write data to file.
*
* Returns: true (1) on success, otherwise false (0).
**/
static int file_archive_decompress_data_to_file(
file_archive_transfer_t *transfer,
file_archive_file_handle_t *handle,
int ret,
const char *path,
const char *valid_exts,
const uint8_t *cdata,
uint32_t csize,
uint32_t size,
uint32_t checksum)
{
if (!handle || ret == -1)
{
ret = 0;
goto end;
}
if (!handle)
return 0;
#if 0
handle->real_checksum = handle->backend->stream_crc_calculate(
handle->real_checksum = transfer->backend->stream_crc_calculate(
0, handle->data, size);
if (handle->real_checksum != checksum)
{
@ -334,38 +202,14 @@ static int file_archive_decompress_data_to_file(
#endif
if (!filestream_write_file(path, handle->data, size))
{
ret = false;
goto end;
}
return 0;
end:
if (handle)
{
if (handle->backend)
{
if (handle->backend->stream_free)
{
#ifdef HAVE_7ZIP
if (handle->backend != &sevenzip_backend)
#endif
{
handle->backend->stream_free(handle->stream);
if (handle->data)
free(handle->data);
}
}
}
}
return ret;
return 1;
}
void file_archive_parse_file_iterate_stop(file_archive_transfer_t *state)
{
if (!state || !state->handle)
if (!state || !state->archive_file)
return;
state->type = ARCHIVE_TRANSFER_DEINIT;
@ -392,7 +236,7 @@ int file_archive_parse_file_iterate(
{
if (userdata)
{
userdata->context = state->stream;
userdata->transfer = state;
strlcpy(userdata->archive_path, file,
sizeof(userdata->archive_path));
}
@ -402,14 +246,13 @@ int file_archive_parse_file_iterate(
state->type = ARCHIVE_TRANSFER_DEINIT_ERROR;
break;
case ARCHIVE_TRANSFER_ITERATE:
if (file_archive_get_file_backend(file))
if (state->backend)
{
const struct file_archive_file_backend *backend =
file_archive_get_file_backend(file);
int ret =
backend->archive_parse_file_iterate_step(state,
valid_exts, userdata, file_cb);
int ret = state->backend->archive_parse_file_iterate_step(
state->context, valid_exts, userdata, file_cb);
if (ret == 1)
state->step_current++; /* found another file */
if (ret != 1)
state->type = ARCHIVE_TRANSFER_DEINIT;
if (ret == -1)
@ -422,25 +265,21 @@ int file_archive_parse_file_iterate(
case ARCHIVE_TRANSFER_DEINIT_ERROR:
*returnerr = false;
case ARCHIVE_TRANSFER_DEINIT:
if (state->handle)
if (state->context)
{
file_archive_free(state->handle);
state->handle = NULL;
if (state->backend->archive_parse_file_free)
state->backend->archive_parse_file_free(state->context);
state->context = NULL;
}
if (state->stream && state->backend)
if (state->archive_file)
{
if (state->backend->stream_free)
state->backend->stream_free(state->stream);
if (state->stream)
free(state->stream);
state->stream = NULL;
if (userdata)
userdata->context = NULL;
filestream_close(state->archive_file);
state->archive_file = NULL;
}
if (userdata)
userdata->transfer = NULL;
break;
}
@ -471,12 +310,11 @@ static bool file_archive_walk(const char *file, const char *valid_exts,
bool returnerr = true;
state.type = ARCHIVE_TRANSFER_INIT;
state.archive_file = NULL;
state.archive_size = 0;
state.handle = NULL;
state.stream = NULL;
state.footer = NULL;
state.directory = NULL;
state.data = NULL;
state.context = NULL;
state.step_total = 0;
state.step_current = 0;
state.backend = NULL;
for (;;)
@ -491,17 +329,10 @@ static bool file_archive_walk(const char *file, const char *valid_exts,
int file_archive_parse_file_progress(file_archive_transfer_t *state)
{
ptrdiff_t delta = 0;
if (!state || state->archive_size == 0)
if (!state || state->step_total == 0)
return 0;
delta = state->directory - state->data;
if (!state->start_delta)
state->start_delta = delta;
return (int)(((delta - state->start_delta) * 100) / (state->archive_size - state->start_delta));
return (int)((state->step_current * 100) / (state->step_total));
}
/**
@ -537,15 +368,10 @@ bool file_archive_extract_file(
userdata.list = NULL;
userdata.found_file = false;
userdata.list_only = false;
userdata.context = NULL;
userdata.crc = 0;
userdata.transfer = NULL;
userdata.dec = NULL;
userdata.decomp_state.opt_file = NULL;
userdata.decomp_state.needle = NULL;
userdata.decomp_state.size = 0;
userdata.decomp_state.found = false;
if (!list)
{
ret = false;
@ -599,15 +425,10 @@ struct string_list *file_archive_get_file_list(const char *path,
userdata.list = string_list_new();
userdata.found_file = false;
userdata.list_only = true;
userdata.context = NULL;
userdata.crc = 0;
userdata.transfer = NULL;
userdata.dec = NULL;
userdata.decomp_state.opt_file = NULL;
userdata.decomp_state.needle = NULL;
userdata.decomp_state.size = 0;
userdata.decomp_state.found = false;
if (!userdata.list)
goto error;
@ -627,45 +448,29 @@ bool file_archive_perform_mode(const char *path, const char *valid_exts,
const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size,
uint32_t crc32, struct archive_extract_userdata *userdata)
{
switch (cmode)
if (!userdata->transfer || !userdata->transfer->backend)
return false;
int ret = 0;
file_archive_file_handle_t handle;
handle.data = NULL;
handle.real_checksum = 0;
if (!userdata->transfer->backend->stream_decompress_data_to_file_init(
userdata->transfer->context, &handle, cdata, cmode, csize, size))
return false;
do
{
case ARCHIVE_MODE_UNCOMPRESSED:
if (!filestream_write_file(path, cdata, size))
return false;
break;
ret = userdata->transfer->backend->stream_decompress_data_to_file_iterate(
userdata->transfer->context, &handle);
}while (ret == 0);
case ARCHIVE_MODE_COMPRESSED:
{
int ret = 0;
file_archive_file_handle_t handle;
handle.stream = userdata->context;
handle.data = NULL;
handle.real_checksum = 0;
handle.backend = file_archive_get_file_backend(userdata->archive_path);
if (!handle.backend)
return false;
if (!handle.backend->stream_decompress_data_to_file_init(&handle,
cdata, csize, size))
return false;
do
{
ret = handle.backend->stream_decompress_data_to_file_iterate(
handle.stream);
}while (ret == 0);
if (!file_archive_decompress_data_to_file(&handle,
ret, path, valid_exts,
cdata, csize, size, crc32))
return false;
}
break;
default:
return false;
}
if (ret == -1 || !file_archive_decompress_data_to_file(
userdata->transfer, &handle, path,
size, crc32))
return false;
return true;
}
@ -841,12 +646,11 @@ uint32_t file_archive_get_file_crc32(const char *path)
}
state.type = ARCHIVE_TRANSFER_INIT;
state.archive_file = NULL;
state.archive_size = 0;
state.handle = NULL;
state.stream = NULL;
state.footer = NULL;
state.directory = NULL;
state.data = NULL;
state.context = NULL;
state.step_total = 0;
state.step_current = 0;
state.backend = NULL;
/* Initialize and open archive first.
@ -881,8 +685,5 @@ uint32_t file_archive_get_file_crc32(const char *path)
file_archive_parse_file_iterate_stop(&state);
if (userdata.crc)
return userdata.crc;
return 0;
return userdata.crc;
}

90
libretro-common/file/archive_file_7z.c
View File

@ -55,10 +55,10 @@ struct sevenzip_context_t
CSzArEx db;
size_t temp_size;
uint32_t block_index;
uint32_t index;
uint32_t parse_index;
uint32_t decompress_index;
uint32_t packIndex;
uint8_t *output;
file_archive_file_handle_t *handle;
};
static void *sevenzip_stream_alloc_impl(void *p, size_t size)
@ -97,14 +97,13 @@ static void* sevenzip_stream_new(void)
sevenzip_context->allocTempImp.Free = sevenzip_stream_free_impl;
sevenzip_context->block_index = 0xFFFFFFFF;
sevenzip_context->output = NULL;
sevenzip_context->handle = NULL;
return sevenzip_context;
}
static void sevenzip_stream_free(void *data)
static void sevenzip_parse_file_free(void *context)
{
struct sevenzip_context_t *sevenzip_context = (struct sevenzip_context_t*)data;
struct sevenzip_context_t *sevenzip_context = (struct sevenzip_context_t*)context;
if (!sevenzip_context)
return;
@ -113,11 +112,12 @@ static void sevenzip_stream_free(void *data)
{
IAlloc_Free(&sevenzip_context->allocImp, sevenzip_context->output);
sevenzip_context->output = NULL;
sevenzip_context->handle->data = NULL;
}
SzArEx_Free(&sevenzip_context->db, &sevenzip_context->allocImp);
File_Close(&sevenzip_context->archiveStream.file);
free(sevenzip_context);
}
/* Extract the relative path (needle) from a 7z archive
@ -125,7 +125,7 @@ static void sevenzip_stream_free(void *data)
* If optional_outfile is set, extract to that instead
* and don't allocate buffer.
*/
static int sevenzip_file_read(
static int64_t sevenzip_file_read(
const char *path,
const char *needle, void **buf,
const char *optional_outfile)
@ -136,7 +136,7 @@ static int sevenzip_file_read(
ISzAlloc allocTempImp;
CSzArEx db;
uint8_t *output = 0;
long outsize = -1;
int64_t outsize = -1;
/*These are the allocation routines.
* Currently using the non-standard 7zip choices. */
@ -255,7 +255,7 @@ static int sevenzip_file_read(
if (res != SZ_OK)
break; /* This goes to the error section. */
outsize = outSizeProcessed;
outsize = (int64_t)outSizeProcessed;
if (optional_outfile)
{
@ -275,7 +275,7 @@ static int sevenzip_file_read(
* We would however need to realloc anyways, because RetroArch
* expects a \0 at the end, therefore we allocate new,
* copy and free the old one. */
*buf = malloc(outsize + 1);
*buf = malloc((size_t)(outsize + 1));
((char*)(*buf))[outsize] = '\0';
memcpy(*buf,output + offset,outsize);
}
@ -301,28 +301,29 @@ static int sevenzip_file_read(
SzArEx_Free(&db, &allocImp);
File_Close(&archiveStream.file);
return (int)outsize;
return outsize;
}
static bool sevenzip_stream_decompress_data_to_file_init(
file_archive_file_handle_t *handle,
const uint8_t *cdata, uint32_t csize, uint32_t size)
void *context, file_archive_file_handle_t *handle,
const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size)
{
struct sevenzip_context_t *sevenzip_context =
(struct sevenzip_context_t*)handle->stream;
(struct sevenzip_context_t*)context;
if (!sevenzip_context)
return false;
sevenzip_context->handle = handle;
sevenzip_context->decompress_index = (uint32_t)(size_t)cdata;
return true;
}
static int sevenzip_stream_decompress_data_to_file_iterate(void *data)
static int sevenzip_stream_decompress_data_to_file_iterate(
void *context, file_archive_file_handle_t *handle)
{
struct sevenzip_context_t *sevenzip_context =
(struct sevenzip_context_t*)data;
(struct sevenzip_context_t*)context;
SRes res = SZ_ERROR_FAIL;
size_t output_size = 0;
@ -330,7 +331,7 @@ static int sevenzip_stream_decompress_data_to_file_iterate(void *data)
size_t outSizeProcessed = 0;
res = SzArEx_Extract(&sevenzip_context->db,
&sevenzip_context->lookStream.s, sevenzip_context->index,
&sevenzip_context->lookStream.s, sevenzip_context->decompress_index,
&sevenzip_context->block_index, &sevenzip_context->output,
&output_size, &offset, &outSizeProcessed,
&sevenzip_context->allocImp, &sevenzip_context->allocTempImp);
@ -338,8 +339,8 @@ static int sevenzip_stream_decompress_data_to_file_iterate(void *data)
if (res != SZ_OK)
return 0;
if (sevenzip_context->handle)
sevenzip_context->handle->data = sevenzip_context->output + offset;
if (handle)
handle->data = sevenzip_context->output + offset;
return 1;
}
@ -347,16 +348,21 @@ static int sevenzip_stream_decompress_data_to_file_iterate(void *data)
static int sevenzip_parse_file_init(file_archive_transfer_t *state,
const char *file)
{
struct sevenzip_context_t *sevenzip_context =
(struct sevenzip_context_t*)sevenzip_stream_new();
uint8_t magic_buf[SEVENZIP_MAGIC_LEN];
struct sevenzip_context_t *sevenzip_context = NULL;
if (state->archive_size < SEVENZIP_MAGIC_LEN)
goto error;
if (string_is_not_equal_fast(state->data, SEVENZIP_MAGIC, SEVENZIP_MAGIC_LEN))
filestream_seek(state->archive_file, 0, SEEK_SET);
if (filestream_read(state->archive_file, magic_buf, SEVENZIP_MAGIC_LEN) != SEVENZIP_MAGIC_LEN)
goto error;
state->stream = sevenzip_context;
if (string_is_not_equal_fast(magic_buf, SEVENZIP_MAGIC, SEVENZIP_MAGIC_LEN))
goto error;
sevenzip_context = (struct sevenzip_context_t*)sevenzip_stream_new();
state->context = sevenzip_context;
#if defined(_WIN32) && defined(USE_WINDOWS_FILE) && !defined(LEGACY_WIN32)
if (!string_is_empty(file))
@ -392,27 +398,28 @@ static int sevenzip_parse_file_init(file_archive_transfer_t *state,
&sevenzip_context->allocImp, &sevenzip_context->allocTempImp) != SZ_OK)
goto error;
state->step_total = sevenzip_context->db.db.NumFiles;
return 0;
error:
if (sevenzip_context)
sevenzip_stream_free(sevenzip_context);
sevenzip_parse_file_free(sevenzip_context);
return -1;
}
static int sevenzip_parse_file_iterate_step_internal(
file_archive_transfer_t *state, char *filename,
struct sevenzip_context_t *sevenzip_context, char *filename,
const uint8_t **cdata, unsigned *cmode,
uint32_t *size, uint32_t *csize, uint32_t *checksum,
unsigned *payback, struct archive_extract_userdata *userdata)
{
struct sevenzip_context_t *sevenzip_context = (struct sevenzip_context_t*)state->stream;
const CSzFileItem *file = sevenzip_context->db.db.Files + sevenzip_context->index;
const CSzFileItem *file = sevenzip_context->db.db.Files + sevenzip_context->parse_index;
if (sevenzip_context->index < sevenzip_context->db.db.NumFiles)
if (sevenzip_context->parse_index < sevenzip_context->db.db.NumFiles)
{
size_t len = SzArEx_GetFileNameUtf16(&sevenzip_context->db,
sevenzip_context->index, NULL);
sevenzip_context->parse_index, NULL);
uint64_t compressed_size = 0;
if (sevenzip_context->packIndex < sevenzip_context->db.db.NumPackStreams)
@ -432,7 +439,7 @@ static int sevenzip_parse_file_iterate_step_internal(
infile[0] = '\0';
SzArEx_GetFileNameUtf16(&sevenzip_context->db, sevenzip_context->index,
SzArEx_GetFileNameUtf16(&sevenzip_context->db, sevenzip_context->parse_index,
temp);
if (temp)
@ -447,10 +454,12 @@ static int sevenzip_parse_file_iterate_step_internal(
strlcpy(filename, infile, PATH_MAX_LENGTH);
*cmode = ARCHIVE_MODE_COMPRESSED;
*cmode = 0; /* unused for 7zip */
*checksum = file->Crc;
*size = (uint32_t)file->Size;
*csize = (uint32_t)compressed_size;
*cdata = (uint8_t *)(size_t)sevenzip_context->parse_index;
}
}
else
@ -461,7 +470,7 @@ static int sevenzip_parse_file_iterate_step_internal(
return 1;
}
static int sevenzip_parse_file_iterate_step(file_archive_transfer_t *state,
static int sevenzip_parse_file_iterate_step(void *context,
const char *valid_exts,
struct archive_extract_userdata *userdata, file_archive_file_cb file_cb)
{
@ -471,12 +480,12 @@ static int sevenzip_parse_file_iterate_step(file_archive_transfer_t *state,
uint32_t csize = 0;
unsigned cmode = 0;
unsigned payload = 0;
struct sevenzip_context_t *sevenzip_context = NULL;
struct sevenzip_context_t *sevenzip_context = (struct sevenzip_context_t*)context;
int ret;
userdata->current_file_path[0] = '\0';
ret = sevenzip_parse_file_iterate_step_internal(state,
ret = sevenzip_parse_file_iterate_step_internal(sevenzip_context,
userdata->current_file_path,
&cdata, &cmode, &size, &csize,
&checksum, &payload, userdata);
@ -491,9 +500,7 @@ static int sevenzip_parse_file_iterate_step(file_archive_transfer_t *state,
csize, size, checksum, userdata))
return 0;
sevenzip_context = (struct sevenzip_context_t*)state->stream;
sevenzip_context->index += payload;
sevenzip_context->parse_index += payload;
return 1;
}
@ -505,13 +512,12 @@ static uint32_t sevenzip_stream_crc32_calculate(uint32_t crc,
}
const struct file_archive_file_backend sevenzip_backend = {
sevenzip_stream_new,
sevenzip_stream_free,
sevenzip_parse_file_init,
sevenzip_parse_file_iterate_step,
sevenzip_parse_file_free,
sevenzip_stream_decompress_data_to_file_init,
sevenzip_stream_decompress_data_to_file_iterate,
sevenzip_stream_crc32_calculate,
sevenzip_file_read,
sevenzip_parse_file_init,
sevenzip_parse_file_iterate_step,
"7z"
};

404
libretro-common/file/archive_file_zlib.c
View File

@ -41,6 +41,22 @@
#define END_OF_CENTRAL_DIR_SIGNATURE 0x06054b50
#endif
enum file_archive_compression_mode
{
ZIP_MODE_STORED = 0,
ZIP_MODE_DEFLATED = 8,
};
typedef struct
{
RFILE *file;
uint8_t *directory;
uint8_t *directory_entry;
uint8_t *directory_end;
void *current_stream;
uint8_t *compressed_data;
} zip_context_t;
static INLINE uint32_t read_le(const uint8_t *data, unsigned size)
{
unsigned i;
@ -53,69 +69,119 @@ static INLINE uint32_t read_le(const uint8_t *data, unsigned size)
return val;
}
static void *zlib_stream_new(void)
static void zip_context_free_stream(zip_context_t *zip_context)
{
return zlib_inflate_backend.stream_new();
}
static void zlib_stream_free(void *stream)
{
zlib_inflate_backend.stream_free(stream);
if (zip_context->current_stream)
{
zlib_inflate_backend.stream_free(zip_context->current_stream);
zip_context->current_stream = NULL;
}
if (zip_context->compressed_data)
{
free(zip_context->compressed_data);
zip_context->compressed_data = NULL;
}
}
static bool zlib_stream_decompress_data_to_file_init(
file_archive_file_handle_t *handle,
const uint8_t *cdata, uint32_t csize, uint32_t size)
void *context, file_archive_file_handle_t *handle,
const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size)
{
if (!handle)
return false;
zip_context_t *zip_context = (zip_context_t *)context;
uint8_t local_header_buf[4];
uint32_t offsetNL, offsetEL;
handle->stream = zlib_inflate_backend.stream_new();
/* free previous stream if left unfinished */
zip_context_free_stream(zip_context);
if (!handle->stream)
/* allocate memory for the compressed data */
zip_context->compressed_data = (uint8_t*)malloc(csize);
if (!zip_context->compressed_data)
goto error;
if (zlib_inflate_backend.define)
zlib_inflate_backend.define(handle->stream, "window_bits", (uint32_t)-MAX_WBITS);
handle->data = (uint8_t*)malloc(size);
if (!handle->data)
/* seek past most of the local directory header */
filestream_seek(zip_context->file, (int64_t)(size_t)cdata + 26, SEEK_SET);
if (filestream_read(zip_context->file, local_header_buf, 4) != 4)
goto error;
zlib_inflate_backend.set_in(handle->stream,
(const uint8_t*)cdata, csize);
zlib_inflate_backend.set_out(handle->stream,
handle->data, size);
offsetNL = read_le(local_header_buf, 2); /* file name length */
offsetEL = read_le(local_header_buf + 2, 2); /* extra field length */
return true;
/* skip over name and extra data */
filestream_seek(zip_context->file, offsetNL + offsetEL, SEEK_CUR);
if (filestream_read(zip_context->file, zip_context->compressed_data, csize) != csize)
goto error;
switch (cmode)
{
case ZIP_MODE_STORED:
handle->data = zip_context->compressed_data;
zip_context->compressed_data = NULL;
return true;
case ZIP_MODE_DEFLATED:
zip_context->current_stream = zlib_inflate_backend.stream_new();
if (!zip_context->current_stream)
goto error;
if (zlib_inflate_backend.define)
zlib_inflate_backend.define(zip_context->current_stream, "window_bits", (uint32_t)-MAX_WBITS);
handle->data = (uint8_t*)malloc(size);
if (!handle->data)
goto error;
zlib_inflate_backend.set_in(zip_context->current_stream,
zip_context->compressed_data, csize);
zlib_inflate_backend.set_out(zip_context->current_stream,
handle->data, size);
return true;
}
error:
if (handle->stream)
zlib_inflate_backend.stream_free(handle->stream);
if (handle->data)
{
free(handle->data);
handle->data = NULL;
}
zip_context_free_stream(zip_context);
return false;
}
static int zlib_stream_decompress_data_to_file_iterate(void *stream)
static int zlib_stream_decompress_data_to_file_iterate(
void *context, file_archive_file_handle_t *handle)
{
zip_context_t *zip_context = (zip_context_t *)context;
bool zstatus;
uint32_t rd, wn;
enum trans_stream_error terror;
if (!stream)
return -1;
if (!zip_context->current_stream)
{
/* file was uncompressed or decompression finished before */
return 1;
}
zstatus = zlib_inflate_backend.trans(stream, false, &rd, &wn, &terror);
if (!zstatus && terror != TRANS_STREAM_ERROR_BUFFER_FULL)
return -1;
zstatus = zlib_inflate_backend.trans(zip_context->current_stream, false, &rd, &wn, &terror);
if (zstatus && !terror)
{
/* successfully decompressed entire file */
zip_context_free_stream(zip_context);
return 1;
}
if (!zstatus && terror != TRANS_STREAM_ERROR_BUFFER_FULL)
{
/* error during stream processing */
zip_context_free_stream(zip_context);
return -1;
}
/* still more data to process */
return 0;
}
@ -126,49 +192,52 @@ static uint32_t zlib_stream_crc32_calculate(uint32_t crc,
}
static bool zip_file_decompressed_handle(
file_archive_file_handle_t *handle,
const uint8_t *cdata, uint32_t csize,
file_archive_transfer_t *transfer,
file_archive_file_handle_t* handle,
const uint8_t *cdata, unsigned cmode, uint32_t csize,
uint32_t size, uint32_t crc32)
{
zip_context_t *zip_context = (zip_context_t *)transfer->context;
int ret = 0;
handle->backend = &zlib_backend;
transfer->backend = &zlib_backend;
if (!handle->backend->stream_decompress_data_to_file_init(
handle, cdata, csize, size))
if (!transfer->backend->stream_decompress_data_to_file_init(
transfer->context, handle, cdata, cmode, csize, size))
return false;
do
{
ret = handle->backend->stream_decompress_data_to_file_iterate(
handle->stream);
ret = transfer->backend->stream_decompress_data_to_file_iterate(
transfer->context, handle);
}while (ret == 0);
#if 0
handle->real_checksum = handle->backend->stream_crc_calculate(0,
handle->real_checksum = transfer->backend->stream_crc_calculate(0,
handle->data, size);
if (handle->real_checksum != crc32)
goto error;
#endif
{
if (handle->data)
free(handle->data);
if (handle->stream)
free(handle->stream);
handle->data = NULL;
return false;
}
#endif
return true;
#if 0
error:
if (handle->stream)
free(handle->stream);
if (handle->data)
free(handle->data);
handle->stream = NULL;
handle->data = NULL;
return false;
#endif
}
typedef struct
{
char *opt_file;
char *needle;
void **buf;
size_t size;
bool found;
} decomp_state_t;
/* Extract the relative path (needle) from a
* ZIP archive (path) and allocate a buffer for it to write it in.
*
@ -182,212 +251,251 @@ static int zip_file_decompressed(
uint32_t csize, uint32_t size,
uint32_t crc32, struct archive_extract_userdata *userdata)
{
decomp_state_t* decomp_state = (decomp_state_t*)userdata->cb_data;
char last_char = name[strlen(name) - 1];
/* Ignore directories. */
if (last_char == '/' || last_char == '\\')
return 1;
if (strstr(name, userdata->decomp_state.needle))
if (strstr(name, decomp_state->needle))
{
bool goto_error = false;
file_archive_file_handle_t handle = {0};
userdata->decomp_state.found = true;
if (zip_file_decompressed_handle(&handle,
cdata, csize, size, crc32))
if (zip_file_decompressed_handle(userdata->transfer,
&handle, cdata, cmode, csize, size, crc32))
{
if (userdata->decomp_state.opt_file != 0)
if (decomp_state->opt_file != 0)
{
/* Called in case core has need_fullpath enabled. */
char *buf = (char*)malloc(size);
bool success = filestream_write_file(decomp_state->opt_file, handle.data, size);
if (buf)
{
memcpy(buf, handle.data, size);
free(handle.data);
handle.data = NULL;
if (!filestream_write_file(userdata->decomp_state.opt_file, buf, size))
goto_error = true;
}
decomp_state->size = 0;
free(buf);
userdata->decomp_state.size = 0;
if (!success)
return -1;
}
else
{
/* Called in case core has need_fullpath disabled.
* Will copy decompressed content directly into
* Will move decompressed content directly into
* RetroArch's ROM buffer. */
*userdata->decomp_state.buf = malloc(size);
memcpy(*userdata->decomp_state.buf, handle.data, size);
*decomp_state->buf = handle.data;
handle.data = NULL;
userdata->decomp_state.size = size;
decomp_state->size = size;
}
}
if (handle.data)
free(handle.data);
if (goto_error)
return 0;
decomp_state->found = true;
}
return 1;
}
static int zip_file_read(
static int64_t zip_file_read(
const char *path,
const char *needle, void **buf,
const char *optional_outfile)
{
file_archive_transfer_t zlib;
file_archive_transfer_t state = {ARCHIVE_TRANSFER_INIT};
decomp_state_t decomp = {0};
struct archive_extract_userdata userdata = {{0}};
bool returnerr = true;
int ret = 0;
zlib.type = ARCHIVE_TRANSFER_INIT;
zlib.archive_size = 0;
zlib.start_delta = 0;
zlib.handle = NULL;
zlib.stream = NULL;
zlib.footer = NULL;
zlib.directory = NULL;
zlib.data = NULL;
zlib.backend = NULL;
userdata.decomp_state.needle = NULL;
userdata.decomp_state.opt_file = NULL;
userdata.decomp_state.found = false;
userdata.decomp_state.buf = buf;
if (needle)
userdata.decomp_state.needle = strdup(needle);
decomp.needle = strdup(needle);
if (optional_outfile)
userdata.decomp_state.opt_file = strdup(optional_outfile);
decomp.opt_file = strdup(optional_outfile);
userdata.transfer = &state;
userdata.cb_data = &decomp;
decomp.buf = buf;
do
{
ret = file_archive_parse_file_iterate(&zlib, &returnerr, path,
ret = file_archive_parse_file_iterate(&state, &returnerr, path,
"", zip_file_decompressed, &userdata);
if (!returnerr)
break;
} while (ret == 0 && !userdata.decomp_state.found);
}while (ret == 0 && !decomp.found);
file_archive_parse_file_iterate_stop(&zlib);
file_archive_parse_file_iterate_stop(&state);
if (userdata.decomp_state.opt_file)
free(userdata.decomp_state.opt_file);
if (userdata.decomp_state.needle)
free(userdata.decomp_state.needle);
if (decomp.opt_file)
free(decomp.opt_file);
if (decomp.needle)
free(decomp.needle);
if (!userdata.decomp_state.found)
if (!decomp.found)
return -1;
return (int)userdata.decomp_state.size;
return (int64_t)decomp.size;
}
static int zip_parse_file_init(file_archive_transfer_t *state,
const char *file)
{
if (state->archive_size < 22)
uint8_t footer_buf[1024];
uint8_t* footer = footer_buf;
int64_t read_pos = state->archive_size;
int64_t read_block = MIN(read_pos, sizeof(footer_buf));
int64_t directory_size, directory_offset;
zip_context_t *zip_context = NULL;
/* Minimal ZIP file size is 22 bytes */
if (read_block < 22)
return -1;
state->footer = state->data + state->archive_size - 22;
for (;; state->footer--)
/* Find the end of central directory record by scanning
* the file from the end towards the beginning.
*/
for (;;)
{
if (state->footer <= state->data + 22)
return -1;
if (read_le(state->footer, 4) == END_OF_CENTRAL_DIR_SIGNATURE)
if (--footer < footer_buf)
{
unsigned comment_len = read_le(state->footer + 20, 2);
if (state->footer + 22 + comment_len == state->data + state->archive_size)
break;
if (read_pos <= 0)
return -1; /* reached beginning of file */
/* Read 21 bytes of overlaps except on the first block. */
if (read_pos == state->archive_size)
read_pos = read_pos - read_block;
else
read_pos = MAX(read_pos - read_block + 21, 0);
/* Seek to read_pos and read read_block bytes. */
filestream_seek(state->archive_file, read_pos, SEEK_SET);
if (filestream_read(state->archive_file, footer_buf, read_block) != read_block)
return -1;
footer = footer_buf + read_block - 22;
}
if (read_le(footer, 4) == END_OF_CENTRAL_DIR_SIGNATURE)
{
unsigned comment_len = read_le(footer + 20, 2);
if (read_pos + (footer - footer_buf) + 22 + comment_len == state->archive_size)
break; /* found it! */
}
}
state->directory = state->data + read_le(state->footer + 16, 4);
/* Read directory info and do basic sanity checks. */
directory_size = read_le(footer + 12, 4);
directory_offset = read_le(footer + 16, 4);
if (directory_size > state->archive_size
|| directory_offset > state->archive_size)
return -1;
/* This is a ZIP file, allocate one block of memory for both the
* context and the entire directory, then read the directory.
*/
zip_context = (zip_context_t*)malloc(sizeof(zip_context_t) + (size_t)directory_size);
zip_context->file = state->archive_file;
zip_context->directory = (uint8_t*)(zip_context + 1);
zip_context->directory_entry = zip_context->directory;
zip_context->directory_end = zip_context->directory + (size_t)directory_size;
zip_context->current_stream = NULL;
zip_context->compressed_data = NULL;
filestream_seek(state->archive_file, directory_offset, SEEK_SET);
if (filestream_read(state->archive_file, zip_context->directory, directory_size) != directory_size)
{
free(zip_context);
return -1;
}
state->context = zip_context;
state->step_total = read_le(footer + 10, 2); /* total entries */;
return 0;
}
static int zip_parse_file_iterate_step_internal(
file_archive_transfer_t *state, char *filename,
zip_context_t * zip_context, char *filename,
const uint8_t **cdata,
unsigned *cmode, uint32_t *size, uint32_t *csize,
uint32_t *checksum, unsigned *payback)
{
uint8_t* entry = zip_context->directory_entry;
uint32_t offset;
uint32_t namelength, extralength, commentlength,
offsetNL, offsetEL;
uint32_t signature = read_le(state->directory + 0, 4);
uint32_t namelength, extralength, commentlength;
if (entry < zip_context->directory || entry >= zip_context->directory_end)
return 0;
uint32_t signature = read_le(zip_context->directory_entry + 0, 4);
if (signature != CENTRAL_FILE_HEADER_SIGNATURE)
return 0;
*cmode = read_le(state->directory + 10, 2); /* compression mode, 0 = store, 8 = deflate */
*checksum = read_le(state->directory + 16, 4); /* CRC32 */
*csize = read_le(state->directory + 20, 4); /* compressed size */
*size = read_le(state->directory + 24, 4); /* uncompressed size */
*cmode = read_le(zip_context->directory_entry + 10, 2); /* compression mode, 0 = store, 8 = deflate */
*checksum = read_le(zip_context->directory_entry + 16, 4); /* CRC32 */
*csize = read_le(zip_context->directory_entry + 20, 4); /* compressed size */
*size = read_le(zip_context->directory_entry + 24, 4); /* uncompressed size */
namelength = read_le(state->directory + 28, 2); /* file name length */
extralength = read_le(state->directory + 30, 2); /* extra field length */
commentlength = read_le(state->directory + 32, 2); /* file comment length */
namelength = read_le(zip_context->directory_entry + 28, 2); /* file name length */
extralength = read_le(zip_context->directory_entry + 30, 2); /* extra field length */
commentlength = read_le(zip_context->directory_entry + 32, 2); /* file comment length */
if (namelength >= PATH_MAX_LENGTH)
return -1;
memcpy(filename, state->directory + 46, namelength); /* file name */
memcpy(filename, zip_context->directory_entry + 46, namelength); /* file name */
filename[namelength] = '\0';
offset = read_le(state->directory + 42, 4); /* relative offset of local file header */
offsetNL = read_le(state->data + offset + 26, 2); /* file name length */
offsetEL = read_le(state->data + offset + 28, 2); /* extra field length */
offset = read_le(zip_context->directory_entry + 42, 4); /* relative offset of local file header */
*cdata = state->data + offset + 30 + offsetNL + offsetEL;
*cdata = (uint8_t *)(size_t)offset; /* store file offset in data pointer */
*payback = 46 + namelength + extralength + commentlength;
*payback = 46 + namelength + extralength + commentlength;
return 1;
}
static int zip_parse_file_iterate_step(file_archive_transfer_t *state,
static int zip_parse_file_iterate_step(void *context,
const char *valid_exts, struct archive_extract_userdata *userdata,
file_archive_file_cb file_cb)
{
zip_context_t *zip_context = (zip_context_t *)context;
const uint8_t *cdata = NULL;
uint32_t checksum = 0;
uint32_t size = 0;
uint32_t csize = 0;
unsigned cmode = 0;
unsigned payload = 0;
int ret = zip_parse_file_iterate_step_internal(
state, userdata->current_file_path, &cdata, &cmode, &size, &csize, &checksum, &payload);
int ret = zip_parse_file_iterate_step_internal(zip_context,
userdata->current_file_path, &cdata, &cmode, &size, &csize, &checksum, &payload);
if (ret != 1)
return ret;
userdata->crc = checksum;
if (file_cb && !file_cb(userdata->current_file_path, valid_exts,
cdata, cmode,
if (file_cb && !file_cb(userdata->current_file_path, valid_exts, cdata, cmode,
csize, size, checksum, userdata))
return 0;
state->directory += payload;
zip_context->directory_entry += payload;
return 1;
}
static void zip_parse_file_free(void *context)
{
zip_context_t *zip_context = (zip_context_t *)context;
zip_context_free_stream(zip_context);
free(zip_context);
}
const struct file_archive_file_backend zlib_backend = {
zlib_stream_new,
zlib_stream_free,
zip_parse_file_init,
zip_parse_file_iterate_step,
zip_parse_file_free,
zlib_stream_decompress_data_to_file_init,
zlib_stream_decompress_data_to_file_iterate,
zlib_stream_crc32_calculate,
zip_file_read,
zip_parse_file_init,
zip_parse_file_iterate_step,
"zlib"
};

61
libretro-common/include/file/archive_file.h
View File

@ -39,8 +39,6 @@
RETRO_BEGIN_DECLS
struct archive_extract_userdata;
enum file_archive_transfer_type
{
ARCHIVE_TRANSFER_NONE = 0,
@ -52,42 +50,20 @@ enum file_archive_transfer_type
typedef struct file_archive_handle
{
void *stream;
uint8_t *data;
uint32_t real_checksum;
const struct file_archive_file_backend *backend;
} file_archive_file_handle_t;
typedef struct file_archive_file_data file_archive_file_data_t;
typedef struct file_archive_transfer
{
enum file_archive_transfer_type type;
int32_t archive_size;
ptrdiff_t start_delta;
file_archive_file_data_t *handle;
void *stream;
const uint8_t *footer;
const uint8_t *directory;
const uint8_t *data;
struct RFILE *archive_file;
int64_t archive_size;
void *context;
unsigned step_total, step_current;
const struct file_archive_file_backend *backend;
} file_archive_transfer_t;
enum file_archive_compression_mode
{
ARCHIVE_MODE_UNCOMPRESSED = 0,
ARCHIVE_MODE_COMPRESSED = 8
};
struct decomp_state_t
{
char *opt_file;
char *needle;
void **buf;
size_t size;
bool found;
};
typedef struct
{
char *source_file;
@ -104,6 +80,7 @@ typedef struct
struct archive_extract_userdata
{
/* These are set or read by the archive processing */
char archive_path[PATH_MAX_LENGTH];
char current_file_path[PATH_MAX_LENGTH];
char *first_extracted_file_path;
@ -113,10 +90,11 @@ struct archive_extract_userdata
struct string_list *list;
bool found_file;
bool list_only;
void *context;
uint32_t crc;
struct decomp_state_t decomp_state;
file_archive_transfer_t *transfer;
/* Not used by the processing, free to use outside or in iterate callback */
decompress_state_t *dec;
void* cb_data;
};
/* Returns true when parsing should continue. False to stop. */
@ -126,22 +104,27 @@ typedef int (*file_archive_file_cb)(const char *name, const char *valid_exts,
struct file_archive_file_backend
{
void *(*stream_new)(void);
void (*stream_free)(void *);
bool (*stream_decompress_data_to_file_init)(
file_archive_file_handle_t *, const uint8_t *, uint32_t, uint32_t);
int (*stream_decompress_data_to_file_iterate)(void *);
uint32_t (*stream_crc_calculate)(uint32_t, const uint8_t *, size_t);
int (*compressed_file_read)(const char *path, const char *needle, void **buf,
const char *optional_outfile);
int (*archive_parse_file_init)(
file_archive_transfer_t *state,
const char *file);
int (*archive_parse_file_iterate_step)(
file_archive_transfer_t *state,
void *context,
const char *valid_exts,
struct archive_extract_userdata *userdata,
file_archive_file_cb file_cb);
void (*archive_parse_file_free)(
void *context);
bool (*stream_decompress_data_to_file_init)(
void *context, file_archive_file_handle_t *handle,
const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size);
int (*stream_decompress_data_to_file_iterate)(
void *context,
file_archive_file_handle_t *handle);
uint32_t (*stream_crc_calculate)(uint32_t, const uint8_t *, size_t);
int64_t (*compressed_file_read)(const char *path, const char *needle, void **buf,
const char *optional_outfile);
const char *ident;
};