From 2ddc97ff8012beec4014bd200b5d228c8c11be86 Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Tue, 27 Oct 2026 08:47:26 +0100 Subject: [PATCH] archive_file: bzip2 zip members and .bz2 files Zip members stored with method 12 are decoded with rbzip2, whole, from the mapping, from what the stream lends or else read, as Zstandard members are. A .bz2 file stands for the file named as it is without the extension: a stream does not say how long it decodes to, so the member is decoded when it is listed, its size and CRC then known, and kept for the extraction that follows. "#" after ".bz2" names its member. HAVE_RBZIP2, on unless configured off, builds both and the decoder; RVZ's bzip2 groups take the decoder alone where it is off. file_archive_whole() is the whole-file read the .zst and .bz2 backends share. archive_bz2_test lists, reads and extracts bzip2 zip members and a .bz2 file, and refuses members whose CRC is wrong. --- .../Linux-libretro-common-samples.yml | 1 + Makefile.common | 15 +- config.features.h | 6 + griffin/griffin.c | 11 +- libretro-common/file/archive_file.c | 26 +- libretro-common/file/archive_file_bz2.c | 204 ++++++++++++ libretro-common/file/archive_file_zlib.c | 50 ++- libretro-common/file/archive_file_zstd.c | 25 +- libretro-common/file/file_path.c | 3 + libretro-common/include/file/archive_file.h | 14 + .../samples/file/archive_bz2/Makefile | 62 ++++ .../file/archive_bz2/archive_bz2_test.c | 300 ++++++++++++++++++ menu/menu_displaylist.c | 3 + qb/config.params.sh | 1 + retroarch.c | 3 + tasks/task_database.c | 1 + tasks/task_decompress.c | 3 + 17 files changed, 688 insertions(+), 40 deletions(-) create mode 100644 libretro-common/file/archive_file_bz2.c create mode 100644 libretro-common/samples/file/archive_bz2/Makefile create mode 100644 libretro-common/samples/file/archive_bz2/archive_bz2_test.c diff --git a/.github/workflows/Linux-libretro-common-samples.yml b/.github/workflows/Linux-libretro-common-samples.yml index 971adfd4c0..c2d1bf707b 100644 --- a/.github/workflows/Linux-libretro-common-samples.yml +++ b/.github/workflows/Linux-libretro-common-samples.yml @@ -110,6 +110,7 @@ jobs: archive_zip_test archive_zstd_test archive_zst_test + archive_bz2_test archive_rar_test archive_stream_test config_file_test diff --git a/Makefile.common b/Makefile.common index c1cfbfc074..5887b03060 100644 --- a/Makefile.common +++ b/Makefile.common @@ -2209,6 +2209,14 @@ ifeq ($(HAVE_RRAR),1) $(LIBRETRO_COMM_DIR)/formats/rar/rrar_ppmd7.o endif +# The built-in bzip2 decoder: one C file, no dependency. +HAVE_RBZIP2 ?= 1 +ifeq ($(HAVE_RBZIP2),1) + DEFINES += -DHAVE_RBZIP2 + OBJ += $(LIBRETRO_COMM_DIR)/encodings/encoding_rbzip2.o \ + $(LIBRETRO_COMM_DIR)/file/archive_file_bz2.o +endif + # The built-in Zstandard codec: one C file, no dependency. ifeq ($(HAVE_RZSTD),1) DEFINES += -DHAVE_RZSTD @@ -2786,8 +2794,11 @@ ifeq ($(HAVE_NETWORKING), 1) HAVE_CHEEVOS_RVZ ?= $(if $(filter 1,$(HAVE_RZSTD)),1,0) ifeq ($(HAVE_CHEEVOS_RVZ), 1) DEFINES += -DHAVE_CHEEVOS_RVZ - OBJ += $(LIBRETRO_COMM_DIR)/formats/rvz/rvz.o \ - $(LIBRETRO_COMM_DIR)/encodings/encoding_rbzip2.o + OBJ += $(LIBRETRO_COMM_DIR)/formats/rvz/rvz.o + # bzip2 groups, decoded as the archive layer does + ifneq ($(HAVE_RBZIP2), 1) + OBJ += $(LIBRETRO_COMM_DIR)/encodings/encoding_rbzip2.o + endif ifneq ($(HAVE_CRYPTO), 1) OBJ += $(LIBRETRO_COMM_DIR)/crypto/crypto.o endif diff --git a/config.features.h b/config.features.h index ff7aa6f127..fa96dbbb75 100644 --- a/config.features.h +++ b/config.features.h @@ -302,6 +302,12 @@ #define SUPPORTS_ZSTD false #endif +#ifdef HAVE_RBZIP2 +#define SUPPORTS_BZIP2 true +#else +#define SUPPORTS_BZIP2 false +#endif + #ifdef HAVE_DYLIB #define SUPPORTS_DYLIB true #else diff --git a/griffin/griffin.c b/griffin/griffin.c index 3006e9ab59..12a1bd463a 100644 --- a/griffin/griffin.c +++ b/griffin/griffin.c @@ -47,6 +47,11 @@ #define HAVE_RRAR 1 #endif +/* Nor has the bzip2 decoder. */ +#ifndef HAVE_RBZIP2 +#define HAVE_RBZIP2 1 +#endif + #if defined(HAVE_OPENGL) && defined(HAVE_ANGLE) #ifndef HAVE_OPENGLES #define HAVE_OPENGLES 1 @@ -139,6 +144,11 @@ ARCHIVE FILE #include "../libretro-common/file/archive_file_zstd.c" #endif +#ifdef HAVE_RBZIP2 +#include "../libretro-common/file/archive_file_bz2.c" +#include "../libretro-common/encodings/encoding_rbzip2.c" +#endif + #ifdef HAVE_RRAR #include "../libretro-common/file/archive_file_rar.c" #include "../libretro-common/formats/rar/rrar_archive.c" @@ -226,7 +236,6 @@ ACHIEVEMENTS #if defined(HAVE_CHEEVOS_RVZ) && defined(HAVE_RZSTD) #include "../libretro-common/formats/rvz/rvz.c" -#include "../libretro-common/encodings/encoding_rbzip2.c" #ifndef HAVE_CRYPTO /* its AES, without the rest of the crypto library */ #include "../libretro-common/crypto/crypto.c" diff --git a/libretro-common/file/archive_file.c b/libretro-common/file/archive_file.c index b6384fd005..2d27944fff 100644 --- a/libretro-common/file/archive_file.c +++ b/libretro-common/file/archive_file.c @@ -157,6 +157,25 @@ unsigned file_archive_open_hints(int64_t size) : RETRO_VFS_FILE_ACCESS_HINT_NONE; } +const uint8_t *file_archive_whole(RFILE *file, int64_t size, void **owned) +{ + const uint8_t *src = NULL; + *owned = NULL; + filestream_seek(file, 0, RETRO_VFS_SEEK_POSITION_START); + if (filestream_view(file, size, &src) == size) + return src; + if (!(*owned = malloc((size_t)size))) + return NULL; + filestream_seek(file, 0, RETRO_VFS_SEEK_POSITION_START); + if (filestream_read(file, *owned, size) != size) + { + free(*owned); + *owned = NULL; + return NULL; + } + return (const uint8_t*)*owned; +} + static int file_archive_parse_file_init(file_archive_transfer_t *state, const char *file) { @@ -924,7 +943,7 @@ const struct file_archive_file_backend *file_archive_get_rar_file_backend(void) const struct file_archive_file_backend* file_archive_get_file_backend(const char *path) { -#if defined(HAVE_7ZIP) || defined(HAVE_ZLIB) || defined(HAVE_RZSTD) || defined(HAVE_RRAR) || defined(HAVE_COMPRESSION) +#if defined(HAVE_7ZIP) || defined(HAVE_ZLIB) || defined(HAVE_RZSTD) || defined(HAVE_RRAR) || defined(HAVE_RBZIP2) || defined(HAVE_COMPRESSION) char newpath[PATH_MAX_LENGTH]; const char *file_ext = NULL; char *last = NULL; @@ -955,6 +974,11 @@ const struct file_archive_file_backend* file_archive_get_file_backend(const char return &zstd_backend; #endif +#ifdef HAVE_RBZIP2 + if (string_is_equal_noncase(file_ext, "bz2")) + return &bz2_backend; +#endif + #ifdef HAVE_RRAR if (string_is_equal_noncase(file_ext, "rar")) return &rar_backend; diff --git a/libretro-common/file/archive_file_bz2.c b/libretro-common/file/archive_file_bz2.c new file mode 100644 index 0000000000..c7f94b0cb3 --- /dev/null +++ b/libretro-common/file/archive_file_bz2.c @@ -0,0 +1,204 @@ +/* Copyright (C) 2010-2026 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (archive_file_bz2.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* .bz2 files: one bzip2 stream, standing for the file named as the + * .bz2 is without its extension. A stream does not say how long it + * decodes to, so its member is decoded when it is listed, its size + * and CRC are then known, and what was decoded is kept for the + * extraction that usually follows. */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +struct bz2_file_context +{ + file_archive_transfer_t *state; + uint8_t *data; /* the member, once decoded */ + size_t size; + char inner_filename[PATH_MAX_LENGTH]; + bool parsed; +}; + +/* The member's name: the .bz2's without its extension */ +static void bz2_inner_filename(const char *path, char *s, size_t len) +{ + char *ext; + strlcpy(s, path_basename(path), len); + if ((ext = strrchr(s, '.')) && string_is_equal_noncase(ext, ".bz2")) + *ext = '\0'; +} + +/* All of @file decoded into *@out (@size octets in it) */ +static int bz2_decode_file(RFILE *file, int64_t size, uint8_t **out, + size_t *out_len) +{ + void *owned = NULL; + const uint8_t *src; + rbzip2_t *bz; + int e = RBZIP2_ERROR_DATA; + + *out = NULL; + *out_len = 0; + if (size <= 0 || !(bz = rbzip2_new())) + return -1; + if ((src = file_archive_whole(file, size, &owned))) + e = rbzip2_decode_alloc(bz, src, (size_t)size, out, out_len); + free(owned); + rbzip2_free(bz); + return e == RBZIP2_OK ? 0 : -1; +} + +static int bz2_parse_file_init(file_archive_transfer_t *state, + const char *file) +{ + uint8_t magic[4]; + struct bz2_file_context *ctx; + + state->context = NULL; + if (state->archive_size < 4) + return -1; + filestream_seek(state->archive_file, 0, RETRO_VFS_SEEK_POSITION_START); + if ( filestream_read(state->archive_file, magic, 4) != 4 + || memcmp(magic, "BZh", 3) || magic[3] < '1' || magic[3] > '9') + return -1; + if (!(ctx = (struct bz2_file_context*)calloc(1, sizeof(*ctx)))) + return -1; + bz2_inner_filename(file, ctx->inner_filename, sizeof(ctx->inner_filename)); + ctx->state = state; + state->context = ctx; + state->step_total = 1; + return 0; +} + +static int bz2_parse_file_iterate_step(void *context, + const char *valid_exts, + struct archive_extract_userdata *userdata, file_archive_file_cb file_cb) +{ + struct bz2_file_context *ctx = (struct bz2_file_context*)context; + uint32_t crc; + + if (!ctx || ctx->parsed) + return 0; + ctx->parsed = true; + + if ( bz2_decode_file(ctx->state->archive_file, + ctx->state->archive_size, &ctx->data, &ctx->size) != 0 + || ctx->size > UINT32_MAX) + return -1; + + crc = encoding_crc32(0, ctx->data, ctx->size); + strlcpy(userdata->current_file_path, ctx->inner_filename, + sizeof(userdata->current_file_path)); + userdata->crc = crc; + userdata->size = (uint64_t)ctx->size; + + if (file_cb && !file_cb(ctx->inner_filename, valid_exts, NULL, 0, + (uint32_t)ctx->state->archive_size, (uint32_t)ctx->size, + crc, userdata)) + return 0; + return 1; +} + +static void bz2_parse_file_free(void *context) +{ + struct bz2_file_context *ctx = (struct bz2_file_context*)context; + if (!ctx) + return; + free(ctx->data); + free(ctx); +} + +static bool bz2_stream_decompress_data_to_file_init( + void *context, file_archive_file_handle_t *handle, + const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size) +{ + (void)handle; + (void)cdata; + (void)cmode; + (void)csize; + (void)size; + return context != NULL; +} + +/* The member was decoded when it was listed */ +static int bz2_stream_decompress_data_to_file_iterate( + void *context, file_archive_file_handle_t *handle) +{ + struct bz2_file_context *ctx = (struct bz2_file_context*)context; + if (!ctx || !handle || !ctx->data) + return -1; + handle->data = ctx->data; + return 1; +} + +/* The member of @path, named @needle, into *@buf (NUL after it) or the + * file @optional_outfile; returns its size, or -1. */ +static int64_t bz2_file_read(const char *path, const char *needle, + void **buf, const char *optional_outfile) +{ + char inner[PATH_MAX_LENGTH]; + uint8_t *data = NULL; + size_t size = 0; + RFILE *file; + int e; + + bz2_inner_filename(path, inner, sizeof(inner)); + if (!string_is_equal(inner, needle)) + return -1; + if (!(file = filestream_open(path, RETRO_VFS_FILE_ACCESS_READ, + file_archive_open_hints(path_get_size(path))))) + return -1; + e = bz2_decode_file(file, filestream_get_size(file), &data, &size); + filestream_close(file); + if (e != 0) + return -1; + + if (optional_outfile) + { + bool ok = filestream_write_file(optional_outfile, data, (int64_t)size); + free(data); + return ok ? (int64_t)size : -1; + } + data[size] = '\0'; /* rbzip2_decode_alloc() leaves the room */ + *buf = data; + return (int64_t)size; +} + +const struct file_archive_file_backend bz2_backend = { + bz2_parse_file_init, + bz2_parse_file_iterate_step, + bz2_parse_file_free, + bz2_stream_decompress_data_to_file_init, + bz2_stream_decompress_data_to_file_iterate, + encoding_crc32, + bz2_file_read, + "bzip2" +}; diff --git a/libretro-common/file/archive_file_zlib.c b/libretro-common/file/archive_file_zlib.c index c320c46d8b..6d2dde41ad 100644 --- a/libretro-common/file/archive_file_zlib.c +++ b/libretro-common/file/archive_file_zlib.c @@ -61,6 +61,10 @@ #include #endif +#ifdef HAVE_RBZIP2 +#include +#endif + #ifndef CENTRAL_FILE_HEADER_SIGNATURE #define CENTRAL_FILE_HEADER_SIGNATURE 0x02014b50 #endif @@ -79,6 +83,7 @@ enum file_archive_compression_mode { ZIP_MODE_STORED = 0, ZIP_MODE_DEFLATED = 8, + ZIP_MODE_BZIP2 = 12, ZIP_MODE_ZSTD = 93 }; @@ -103,6 +108,9 @@ typedef struct #endif uint8_t *tmpbuf; size_t tmpbuf_len; +#ifdef HAVE_RBZIP2 + rbzip2_t *bz; /* made at the first bzip2 member, kept */ +#endif /* The whole member, or a ZIP_STREAM_WINDOW of it when streaming */ uint8_t *decompressed_data; uint32_t win_len; /* size of decompressed_data */ @@ -547,25 +555,22 @@ static int zlib_stream_decompress_data_to_file_iterate( return 0; /* still more data to process */ } -#ifdef ZIP_HAVE_ZSTD - else if (zip_context->cmode == ZIP_MODE_ZSTD) +#if defined(ZIP_HAVE_ZSTD) || defined(HAVE_RBZIP2) + else if ( zip_context->cmode == ZIP_MODE_ZSTD + || zip_context->cmode == ZIP_MODE_BZIP2) { - size_t result; - int zerr; + /* Decoded whole, from the mapping, from what the stream lends + * when it holds all of the member, or else read */ + const uint8_t *src = NULL; + size_t result = 0; + int zerr = 1; #ifdef VFS_HAVE_FILE_MAPPING if (state->archive_mmap_data) - { - zerr = (rzstd_decode( - zip_context->decompressed_data, zip_context->usize, - state->archive_mmap_data + (size_t)zip_context->fdoffset, - zip_context->csize, &result) != RZSTD_PROCESS_END); - } + src = state->archive_mmap_data + (size_t)zip_context->fdoffset; else #endif { - /* The whole frame, lent if the stream holds it all, else read */ - const uint8_t *src = NULL; filestream_seek(state->archive_file, zip_context->fdoffset, RETRO_VFS_SEEK_POSITION_START); @@ -582,12 +587,25 @@ static int zlib_stream_decompress_data_to_file_iterate( return -1; src = zip_context->tmpbuf; } + } +#ifdef ZIP_HAVE_ZSTD + if (zip_context->cmode == ZIP_MODE_ZSTD) zerr = (rzstd_decode( zip_context->decompressed_data, zip_context->usize, src, zip_context->csize, &result) != RZSTD_PROCESS_END); +#endif +#ifdef HAVE_RBZIP2 + if (zip_context->cmode == ZIP_MODE_BZIP2) + { + if (!zip_context->bz && !(zip_context->bz = rbzip2_new())) + return -1; + zerr = rbzip2_decode(zip_context->bz, + zip_context->decompressed_data, zip_context->usize, + src, zip_context->csize, &result) != RBZIP2_OK; } +#endif if ( zerr || result != zip_context->usize @@ -855,6 +873,9 @@ static int zip_parse_file_init(file_archive_transfer_t *state, #endif zip_context->tmpbuf = NULL; zip_context->tmpbuf_len = 0; +#ifdef HAVE_RBZIP2 + zip_context->bz = NULL; +#endif zip_context->entry_crc = 0; zip_context->expect_crc = 0; zip_context->crc = 0; @@ -929,7 +950,7 @@ again: if (signature != CENTRAL_FILE_HEADER_SIGNATURE) return 0; - *cmode = read_le(zip_context->directory_entry + 10, 2); /* compression mode, 0 = store, 8 = deflate */ + *cmode = read_le(zip_context->directory_entry + 10, 2); /* compression mode: 0 store, 8 deflate, 12 bzip2, 93 zstd */ *checksum = read_le(zip_context->directory_entry + 16, 4); /* CRC32 */ *csize = read_le(zip_context->directory_entry + 20, 4); /* compressed size */ *size = read_le(zip_context->directory_entry + 24, 4); /* uncompressed size */ @@ -1011,6 +1032,9 @@ static void zip_parse_file_free(void *context) { zip_context_t *zip_context = (zip_context_t *)context; zip_context_free_stream(zip_context, false); +#ifdef HAVE_RBZIP2 + rbzip2_free(zip_context->bz); +#endif free(zip_context); } diff --git a/libretro-common/file/archive_file_zstd.c b/libretro-common/file/archive_file_zstd.c index 0a831ab8f9..bec6b44ee1 100644 --- a/libretro-common/file/archive_file_zstd.c +++ b/libretro-common/file/archive_file_zstd.c @@ -67,27 +67,6 @@ static void zstd_derive_inner_filename(const char *path, char *s, size_t len) ((char*)ext)[0] = '\0'; } -/* The whole of @file, lent where the stream can (a mapping, a - * read-ahead window), else read into *@owned for the caller to free */ -static const uint8_t *zstd_whole(RFILE *file, int64_t size, void **owned) -{ - const uint8_t *src = NULL; - *owned = NULL; - filestream_seek(file, 0, RETRO_VFS_SEEK_POSITION_START); - if (filestream_view(file, size, &src) == size) - return src; - if (!(*owned = malloc((size_t)size))) - return NULL; - filestream_seek(file, 0, RETRO_VFS_SEEK_POSITION_START); - if (filestream_read(file, *owned, size) != size) - { - free(*owned); - *owned = NULL; - return NULL; - } - return (const uint8_t*)*owned; -} - static int zstd_parse_file_init(file_archive_transfer_t *state, const char *file) { @@ -229,7 +208,7 @@ static int zstd_stream_decompress_data_to_file_iterate( return 1; } - if (!(src = zstd_whole(ctx->state->archive_file, ctx->archive_size, + if (!(src = file_archive_whole(ctx->state->archive_file, ctx->archive_size, &owned))) return -1; @@ -286,7 +265,7 @@ static int64_t zstd_file_read( /* The frame is decoded where the stream lends it, so the stream * stays open until it is */ if ( (file_size = filestream_get_size(file)) <= 0 - || !(src = zstd_whole(file, file_size, &owned))) + || !(src = file_archive_whole(file, file_size, &owned))) goto done; content_size = rzstd_frame_content_size(src, (size_t)file_size); diff --git a/libretro-common/file/file_path.c b/libretro-common/file/file_path.c index d85289a498..d551ef5d72 100644 --- a/libretro-common/file/file_path.c +++ b/libretro-common/file/file_path.c @@ -204,6 +204,9 @@ const char *path_get_archive_delim(const char *path) /* Check ".rar" */ else if (c3 == 'r' && c2 == 'a' && c1 == 'r') return delim; + /* Check ".bz2" */ + else if (c3 == 'b' && c2 == 'z' && delim[-1] == '2') + return delim; } } } diff --git a/libretro-common/include/file/archive_file.h b/libretro-common/include/file/archive_file.h index f36b0e807e..8043a8a2a8 100644 --- a/libretro-common/include/file/archive_file.h +++ b/libretro-common/include/file/archive_file.h @@ -288,6 +288,19 @@ const struct file_archive_file_backend* file_archive_get_file_backend(const char **/ unsigned file_archive_open_hints(int64_t size); +/** + * file_archive_whole: + * @size : octets in @file + * @owned : receives a buffer for the caller to + * free, or NULL + * + * Returns: all of @file, where the stream lends it whole (a mapping, a + * read-ahead window) or else read into *@owned; NULL on failure. What + * is lent holds until the next call on @file. + **/ +const uint8_t *file_archive_whole(struct RFILE *file, int64_t size, + void **owned); + /** * file_archive_get_file_crc32: * @path : filename path of archive @@ -312,6 +325,7 @@ uint32_t file_archive_get_file_crc32_and_size(const char *path, uint64_t *size); extern const struct file_archive_file_backend zlib_backend; extern const struct file_archive_file_backend sevenzip_backend; extern const struct file_archive_file_backend zstd_backend; +extern const struct file_archive_file_backend bz2_backend; extern const struct file_archive_file_backend rar_backend; /* ---- incremental archive entry source ---- diff --git a/libretro-common/samples/file/archive_bz2/Makefile b/libretro-common/samples/file/archive_bz2/Makefile new file mode 100644 index 0000000000..9621ee4aad --- /dev/null +++ b/libretro-common/samples/file/archive_bz2/Makefile @@ -0,0 +1,62 @@ +include ../../sample_objdir.mk + +TARGET := archive_bz2_test + +LIBRETRO_COMM_DIR := ../../.. + +SOURCES := \ + archive_bz2_test.c \ + $(LIBRETRO_COMM_DIR)/compat/fopen_utf8.c \ + $(LIBRETRO_COMM_DIR)/compat/compat_strl.c \ + $(LIBRETRO_COMM_DIR)/compat/compat_strcasestr.c \ + $(LIBRETRO_COMM_DIR)/compat/compat_posix_string.c \ + $(LIBRETRO_COMM_DIR)/encodings/encoding_crc32.c \ + $(LIBRETRO_COMM_DIR)/encodings/encoding_rbzip2.c \ + $(LIBRETRO_COMM_DIR)/encodings/encoding_utf.c \ + $(LIBRETRO_COMM_DIR)/encodings/encoding_deflate.c \ + $(LIBRETRO_COMM_DIR)/features/features_cpu.c \ + $(LIBRETRO_COMM_DIR)/file/archive_file.c \ + $(LIBRETRO_COMM_DIR)/file/archive_file_zlib.c \ + $(LIBRETRO_COMM_DIR)/file/archive_file_bz2.c \ + $(LIBRETRO_COMM_DIR)/file/file_path.c \ + $(LIBRETRO_COMM_DIR)/file/file_path_io.c \ + $(LIBRETRO_COMM_DIR)/lists/string_list.c \ + $(LIBRETRO_COMM_DIR)/streams/file_stream.c \ + $(LIBRETRO_COMM_DIR)/streams/trans_stream.c \ + $(LIBRETRO_COMM_DIR)/streams/trans_stream_pipe.c \ + $(LIBRETRO_COMM_DIR)/streams/trans_stream_deflate.c \ + $(LIBRETRO_COMM_DIR)/string/rstrtod.c \ + $(LIBRETRO_COMM_DIR)/string/stdstring.c \ + $(LIBRETRO_COMM_DIR)/vfs/vfs_implementation.c \ + $(LIBRETRO_COMM_DIR)/time/rtime.c + +OBJS := $(call sample_objs,$(SOURCES:.c=.o)) + +override CPPFLAGS += -DHAVE_COMPRESSION -DHAVE_RBZIP2 -I$(LIBRETRO_COMM_DIR)/include +CFLAGS += -Wall -pedantic -std=gnu99 -g1 + +ifneq ($(SANITIZER),) + CFLAGS := -fsanitize=$(SANITIZER) -fno-omit-frame-pointer $(CFLAGS) + LDFLAGS := -fsanitize=$(SANITIZER) $(LDFLAGS) +endif + +all: $(TARGET) + +%.o: %.c Makefile ../../encodings/rbzip2/bz_write.h + $(CC) -c -o $@ $< $(CPPFLAGS) $(CFLAGS) + +$(SAMPLE_OBJDIR)/%.o: $$(call sample_src,$$*).c Makefile + @mkdir -p $(@D) + $(CC) -c -o $@ $< $(CPPFLAGS) $(CFLAGS) + +$(TARGET): $(OBJS) + $(CC) -o $@ $^ $(LDFLAGS) + +check: $(TARGET) + ./$(TARGET) + +clean: + rm -rf $(SAMPLE_OBJDIR) + rm -f $(TARGET) $(OBJS) + +.PHONY: all check clean diff --git a/libretro-common/samples/file/archive_bz2/archive_bz2_test.c b/libretro-common/samples/file/archive_bz2/archive_bz2_test.c new file mode 100644 index 0000000000..fd0da11e39 --- /dev/null +++ b/libretro-common/samples/file/archive_bz2/archive_bz2_test.c @@ -0,0 +1,300 @@ +/* Copyright (C) 2010-2026 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (archive_bz2_test.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* bzip2 in the archive layer: zip members stored with method 12, and + * .bz2 files, which stand for the file named as they are without the + * extension. Each is listed, read whole with + * file_archive_compressed_read() and extracted through a transfer, and + * must come out as it went in; a zip member whose recorded CRC is wrong + * is refused, and a "#" after ".bz2" names a member. The streams are + * bz_write.h's, as the rbzip2 sample makes them. */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../../encodings/rbzip2/bz_write.h" + +#define ZIP_PATH "archive_bz2_test.zip" +#define BZ2_PATH "archive_bz2_test.bin.bz2" +#define OUT_PATH "archive_bz2_test.out" +#define N_MEMBERS 3 + +static unsigned failures; + +static void fail(const char *what, const char *detail) +{ + printf("[FAIL] %s: %s\n", what, detail); + failures++; +} + +static void put16(uint8_t *p, unsigned v) +{ + p[0] = (uint8_t)v; + p[1] = (uint8_t)(v >> 8); +} + +static void put32(uint8_t *p, uint32_t v) +{ + put16(p, v & 0xffff); + put16(p + 2, v >> 16); +} + +static void fill(uint8_t *d, size_t n, unsigned k) +{ + uint32_t x = 7 + k; + size_t i; + for (i = 0; i < n; i++) + { + x = x * 1103515245u + 12345u; + d[i] = (uint8_t)(k == 1 ? (x >> 16) : 'a' + (x >> 16) % (3 + k) + (i >> 10) % 3); + } +} + +static bool write_file(const char *path, const uint8_t *d, size_t n) +{ + FILE *f = fopen(path, "wb"); + bool ok; + if (!f) + return false; + ok = fwrite(d, 1, n, f) == n; + fclose(f); + return ok; +} + +/* the member's octets, as read back every way there is */ +static void check_member(const char *what, const char *archive, + const char *name, const uint8_t *want, size_t n, uint32_t crc) +{ + char path[256]; + void *buf = NULL; + int64_t len = 0; + + snprintf(path, sizeof(path), "%s#%s", archive, name); + if ( !file_archive_compressed_read(path, &buf, NULL, &len) + || len != (int64_t)n || memcmp(buf, want, n)) + fail(what, "read whole"); + free(buf); + if (file_archive_get_file_crc32(path) != crc) + fail(what, "listed CRC"); +} + +static int extract_cb(const char *name, const char *valid_exts, + const uint8_t *cdata, unsigned cmode, uint32_t csize, uint32_t size, + uint32_t checksum, struct archive_extract_userdata *userdata) +{ + char out[64]; + snprintf(out, sizeof(out), "%s.%s", OUT_PATH, path_basename(name)); + if (file_archive_perform_mode(out, valid_exts, cdata, cmode, + csize, size, checksum, userdata)) + (*(unsigned*)userdata->cb_data)++; + return 1; +} + +static unsigned extract_all(const char *archive) +{ + struct archive_extract_userdata userdata; + file_archive_transfer_t state; + unsigned extracted = 0; + bool returnerr = true; + memset(&state, 0, sizeof(state)); + memset(&userdata, 0, sizeof(userdata)); + state.type = ARCHIVE_TRANSFER_INIT; + userdata.cb_data = &extracted; + while (file_archive_parse_file_iterate(&state, &returnerr, archive, + NULL, extract_cb, &userdata) == 0) { } + return extracted; +} + +static void check_extracted(const char *what, const char *name, + const uint8_t *want, size_t n) +{ + char out[64]; + void *got = NULL; + int64_t got_len = 0; + snprintf(out, sizeof(out), "%s.%s", OUT_PATH, name); + if ( !filestream_read_file(out, &got, &got_len) + || got_len != (int64_t)n || memcmp(got, want, n)) + fail(what, "extracted"); + free(got); + remove(out); +} + +static void test_zip(bool wrong_crc) +{ + static const size_t sizes[N_MEMBERS] = { 70000, 333, 260000 }; + uint8_t *data[N_MEMBERS], *packed[N_MEMBERS], *zip; + size_t plen[N_MEMBERS], at = 0, cd, cap = 64; + uint32_t crc[N_MEMBERS], off[N_MEMBERS]; + char name[16]; + unsigned i; + + for (i = 0; i < N_MEMBERS; i++) + { + size_t c = sizes[i] + sizes[i] / 2 + 4096; + data[i] = (uint8_t*)malloc(sizes[i]); + packed[i] = (uint8_t*)malloc(c); + fill(data[i], sizes[i], i); + crc[i] = encoding_crc32(0, data[i], sizes[i]) ^ (wrong_crc ? 1u : 0u); + plen[i] = bz_write(packed[i], c, data[i], sizes[i], 1 + i * 4, i); + cap += 30 + 46 + 2 * 16 + plen[i]; + } + zip = (uint8_t*)malloc(cap + 22); + for (i = 0; i < N_MEMBERS; i++) + { + sprintf(name, "m%u.bin", i); + off[i] = (uint32_t)at; + memset(zip + at, 0, 30); + put32(zip + at, 0x04034b50u); + put16(zip + at + 4, 46); + put16(zip + at + 8, 12); + put32(zip + at + 14, crc[i]); + put32(zip + at + 18, (uint32_t)plen[i]); + put32(zip + at + 22, (uint32_t)sizes[i]); + put16(zip + at + 26, (unsigned)strlen(name)); + memcpy(zip + at + 30, name, strlen(name)); + at += 30 + strlen(name); + memcpy(zip + at, packed[i], plen[i]); + at += plen[i]; + } + cd = at; + for (i = 0; i < N_MEMBERS; i++) + { + sprintf(name, "m%u.bin", i); + memset(zip + at, 0, 46); + put32(zip + at, 0x02014b50u); + put16(zip + at + 4, 46); + put16(zip + at + 6, 46); + put16(zip + at + 10, 12); + put32(zip + at + 16, crc[i]); + put32(zip + at + 20, (uint32_t)plen[i]); + put32(zip + at + 24, (uint32_t)sizes[i]); + put16(zip + at + 28, (unsigned)strlen(name)); + put32(zip + at + 42, off[i]); + memcpy(zip + at + 46, name, strlen(name)); + at += 46 + strlen(name); + } + memset(zip + at, 0, 22); + put32(zip + at, 0x06054b50u); + put16(zip + at + 8, N_MEMBERS); + put16(zip + at + 10, N_MEMBERS); + put32(zip + at + 12, (uint32_t)(at - cd)); + put32(zip + at + 16, (uint32_t)cd); + at += 22; + if (!write_file(ZIP_PATH, zip, at)) + fail("zip", "cannot write"); + + if (wrong_crc) + { + void *buf = NULL; + int64_t len = 0; + if (extract_all(ZIP_PATH) != 0) + fail("zip, CRCs wrong", "extracted"); + if (file_archive_compressed_read(ZIP_PATH "#m1.bin", &buf, NULL, &len)) + fail("zip, CRCs wrong", "read whole"); + free(buf); + for (i = 0; i < N_MEMBERS; i++) + { + sprintf(name, "m%u.bin", i); + { + char out[64]; + snprintf(out, sizeof(out), "%s.%s", OUT_PATH, name); + remove(out); + } + } + } + else + { + if (extract_all(ZIP_PATH) != N_MEMBERS) + fail("zip", "not every member extracted"); + for (i = 0; i < N_MEMBERS; i++) + { + sprintf(name, "m%u.bin", i); + check_extracted("zip", name, data[i], sizes[i]); + check_member("zip", ZIP_PATH, name, data[i], sizes[i], crc[i]); + } + } + remove(ZIP_PATH); + for (i = 0; i < N_MEMBERS; i++) + { + free(data[i]); + free(packed[i]); + } + free(zip); +} + +static void test_bz2(void) +{ + size_t n = 300000, c = n + n / 2 + 4096, plen; + uint8_t *data = (uint8_t*)malloc(n); + uint8_t *bz = (uint8_t*)malloc(c); + struct string_list *list; + + fill(data, n, 2); + plen = bz_write(bz, c, data, n, 2, 9); + if (!write_file(BZ2_PATH, bz, plen)) + fail(".bz2", "cannot write"); + if ( !(list = file_archive_get_file_list(BZ2_PATH, NULL)) + || list->size != 1 + || strcmp(list->elems[0].data, "archive_bz2_test.bin")) + fail(".bz2", "listed as its name without the extension"); + string_list_free(list); + check_member(".bz2", BZ2_PATH, "archive_bz2_test.bin", data, n, + encoding_crc32(0, data, n)); + if (extract_all(BZ2_PATH) != 1) + fail(".bz2", "not extracted"); + check_extracted(".bz2", "archive_bz2_test.bin", data, n); + { + void *buf = NULL; + int64_t len = 0; + if (file_archive_compressed_read(BZ2_PATH "#other.bin", &buf, NULL, &len)) + fail(".bz2", "a member it does not have read"); + free(buf); + } + remove(BZ2_PATH); + free(data); + free(bz); +} + +int main(void) +{ + const char *p = "roms/set.BZ2#game.bin"; + if (path_get_archive_delim(p) != p + 12) + fail("path_get_archive_delim", "\".bz2#\" not a member"); + test_zip(false); + test_zip(true); + test_bz2(); + if (failures) + { + printf("%u check(s) failed\n", failures); + return 1; + } + printf("archive_bz2_test: zip members and .bz2 files read and extracted\n"); + return 0; +} diff --git a/menu/menu_displaylist.c b/menu/menu_displaylist.c index 1176201334..b78c65f1b3 100644 --- a/menu/menu_displaylist.c +++ b/menu/menu_displaylist.c @@ -3726,6 +3726,9 @@ static unsigned menu_displaylist_parse_system_info(file_list_t *list) #ifdef HAVE_RZSTD {SUPPORTS_ZSTD, "Zstandard"}, #endif +#ifdef HAVE_RBZIP2 + {SUPPORTS_BZIP2, "bzip2"}, +#endif #ifdef HAVE_RRAR {SUPPORTS_RAR, "RAR"}, #endif diff --git a/qb/config.params.sh b/qb/config.params.sh index b69f1a8052..81bfb53d01 100644 --- a/qb/config.params.sh +++ b/qb/config.params.sh @@ -267,6 +267,7 @@ HAVE_ZLIB=no # zlib support (uses the built-in DEFLATE codec when HAVE_7ZIP=yes # 7z archive support HAVE_RRAR=yes # RAR archive support HAVE_RZSTD=yes # Built-in Zstandard codec +HAVE_RBZIP2=yes # Built-in bzip2 decoder (.bz2, bzip2 zip members) HAVE_CHD=yes # CHD disc image support HAVE_RCHD=yes # Read CHD images with the built-in reader instead of libchdr CXX_CHD=no diff --git a/retroarch.c b/retroarch.c index 7dc557040d..0ae1157665 100644 --- a/retroarch.c +++ b/retroarch.c @@ -7684,6 +7684,9 @@ static void retroarch_print_features(void) #ifdef HAVE_RZSTD _len += _PSUPP_BUF(buf, _len, SUPPORTS_ZSTD, "zstd", "Zstandard support"); #endif +#ifdef HAVE_RBZIP2 + _len += _PSUPP_BUF(buf, _len, SUPPORTS_BZIP2, "bzip2", "bzip2 support"); +#endif #ifdef HAVE_RRAR _len += _PSUPP_BUF(buf, _len, SUPPORTS_RAR, "rar", "RAR support"); #endif diff --git a/tasks/task_database.c b/tasks/task_database.c index 1bf894d7f9..5f08c1017f 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -1018,6 +1018,7 @@ static enum msg_file_type extension_to_file_type(const char *ext) || string_is_equal(ext_lower, "zip") || string_is_equal(ext_lower, "apk") || string_is_equal(ext_lower, "zst") + || string_is_equal(ext_lower, "bz2") ) return FILE_TYPE_COMPRESSED; if (string_is_equal(ext_lower, "cue")) diff --git a/tasks/task_decompress.c b/tasks/task_decompress.c index ca0711ab5a..4ca1ec690e 100644 --- a/tasks/task_decompress.c +++ b/tasks/task_decompress.c @@ -486,6 +486,9 @@ void *task_push_decompress( #ifdef HAVE_RZSTD && strcasecmp(ext, "zst") != 0 #endif +#ifdef HAVE_RBZIP2 + && strcasecmp(ext, "bz2") != 0 +#endif #ifdef HAVE_RRAR && strcasecmp(ext, "rar") != 0 #endif