From 7e5c815578d00c7b344094bc6e19d360ffa248ed Mon Sep 17 00:00:00 2001 From: libretroadmin Date: Sat, 7 Nov 2026 14:15:49 +0100 Subject: [PATCH] formats/zar: lookups bisect, directories found and listed with places, a writer A lookup finds directories as well as files, the root at the empty path, and searches each directory's sorted entries by bisection instead of scanning them. A listing's entries carry their place: a file's offset and size, a directory's first entry and count. zar_index_offset() tells a caller where the index begins, so it can keep that part of the file in memory and answer lookups and listings without I/O. zar_write.c builds an archive front to back through a write callback, compressing blocks with rzstd where HAVE_RZSTD is defined and finishing with the footer's SHA-256, so an emulator can make .zar/.wua files without the reference ZArchive library. Its output reads back with the reference reader, and the reference writer's output with this one. --- libretro-common/formats/zar/zar.c | 328 ++++++----- libretro-common/formats/zar/zar_write.c | 550 ++++++++++++++++++ libretro-common/include/formats/zar.h | 77 ++- libretro-common/samples/formats/zar/Makefile | 14 + .../samples/formats/zar/zar_test.c | 326 +++++++++-- tasks/task_database_cue.c | 5 +- 6 files changed, 1090 insertions(+), 210 deletions(-) create mode 100644 libretro-common/formats/zar/zar_write.c diff --git a/libretro-common/formats/zar/zar.c b/libretro-common/formats/zar/zar.c index 819ccb7214..7514266c3b 100644 --- a/libretro-common/formats/zar/zar.c +++ b/libretro-common/formats/zar/zar.c @@ -33,7 +33,8 @@ * for a file), then a file's place and size in the stream (48 bits * each, their top 16 bits in the last word) or a directory's first * child and count. A name is its length, in one octet or two, and its - * octets. */ + * octets. A directory's entries are sorted by name, A-Z folded to a-z, + * octets compared unsigned and a prefix first. */ #include #include @@ -67,10 +68,13 @@ #define ZAR_OP_DIR 4 /* the lookup */ -#define ZAR_L_DIR 0 /* the directory's own node */ -#define ZAR_L_BATCH 1 /* a run of its entries */ -#define ZAR_L_NAME 2 /* one entry's name */ -#define ZAR_L_LIST 3 /* a listed entry's name, in full */ +#define ZAR_L_ROOT 0 /* the root's node */ +#define ZAR_L_HAVE 1 /* a node in hand, the path's next part to find */ +#define ZAR_L_PROBE 2 /* the middle entry of the range left */ +#define ZAR_L_NAME 3 /* its name */ +/* the listing */ +#define ZAR_L_BATCH 4 /* a run of its entries */ +#define ZAR_L_LIST 5 /* a listed entry's name, in full */ /* where the range asked for goes */ #define ZAR_TO_BUF 0 @@ -88,7 +92,6 @@ struct zar uint64_t rec_group; /* the record held, ZAR_NONE for none */ uint64_t blk_index; /* the decoded block held, ZAR_NONE for none */ uint64_t rd_off; /* in the stream */ - zar_file_t file; /* what the lookup found */ uint8_t *rd_dst; uint8_t *packed; /* a compressed block, as stored */ uint8_t *block; /* a decoded block, for partial reads */ @@ -103,6 +106,8 @@ struct zar uint32_t nodes; uint32_t dir_first; uint32_t dir_count; + uint32_t lo; /* the entries the name may be among */ + uint32_t hi; uint32_t dir_index; /* the first entry of the run in hand */ uint32_t batch; /* entries in the run */ uint32_t entry; /* the one whose name is looked at */ @@ -111,12 +116,12 @@ struct zar int stage; int to; uint8_t buf[ZAR_BATCH * ZAR_NODE]; + uint8_t node[ZAR_NODE]; /* the lookup's, in hand */ uint8_t name[ZAR_NAME_MAX + 3]; /* length, octets, terminator */ uint8_t rec[ZAR_RECORD]; bool fetching; bool failed; - bool want_dir; /* a listing's directory is looked up */ - bool listing; /* and found */ + bool listing; /* a listing's directory is found */ }; static uint32_t zar_be32(const uint8_t *p) @@ -148,8 +153,6 @@ zar_t *zar_new(uint64_t file_size) z->rec_group = ZAR_NONE; z->blk_index = ZAR_NONE; z->rd_off = 0; - z->file.offset = 0; - z->file.size = 0; z->rd_dst = NULL; z->packed = NULL; z->block = NULL; @@ -164,6 +167,8 @@ zar_t *zar_new(uint64_t file_size) z->nodes = 0; z->dir_first = 0; z->dir_count = 0; + z->lo = 0; + z->hi = 0; z->dir_index = 0; z->batch = 0; z->entry = 0; @@ -173,7 +178,6 @@ zar_t *zar_new(uint64_t file_size) z->to = ZAR_TO_BUF; z->fetching = false; z->failed = false; - z->want_dir = false; z->listing = false; return z; } @@ -246,8 +250,7 @@ static int zar_open_run(zar_t *z, zar_request_t *req) z->sec_size[i] = size; } } - if ( !z->sec_size[ZAR_SEC_RECORDS] - || z->sec_size[ZAR_SEC_RECORDS] > 0xFFFFFFFFu + if ( z->sec_size[ZAR_SEC_RECORDS] > 0xFFFFFFFFu || z->sec_size[ZAR_SEC_RECORDS] % ZAR_RECORD || z->sec_size[ZAR_SEC_NAMES] > 0x7FFFFFFFu || !z->sec_size[ZAR_SEC_TREE] @@ -281,13 +284,11 @@ int zar_lookup_begin(zar_t *z, const char *path) if (z->op == ZAR_OP_OPEN) return ZAR_ERROR_STATE; z->op = ZAR_OP_LOOKUP; - z->stage = ZAR_L_DIR; + z->stage = ZAR_L_ROOT; z->path = path; - z->dir_first = 0; z->got = 0; z->fetching = false; z->failed = false; - z->want_dir = false; z->listing = false; return ZAR_OK; } @@ -296,153 +297,147 @@ int zar_dir_begin(zar_t *z, const char *path) { int e = zar_lookup_begin(z, path); if (e == ZAR_OK) - { - z->op = ZAR_OP_DIR; - z->want_dir = true; - } + z->op = ZAR_OP_DIR; return e; } -static int zar_upper(int c) +static int zar_lower(int c) { - return (c >= 'a' && c <= 'z') ? c - ('a' - 'A') : c; + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; } -/* The tree, from the root: each directory's entries looked at a run at - * a time; an entry's name read only as far as it can match */ +/* A tree node as the caller sees it */ +static int zar_node_file(const zar_t *z, const uint8_t *n, zar_file_t *out) +{ + if (n[0] & 0x80) + { + out->offset = zar_be32(n + 4) + | ((uint64_t)(zar_be32(n + 12) & 0xFFFFu) << 32); + out->size = zar_be32(n + 8) + | ((uint64_t)(zar_be32(n + 12) & 0xFFFF0000u) << 16); + out->dir = 0; + if ( out->offset > z->blocks * ZAR_BLOCK + || out->size > z->blocks * ZAR_BLOCK - out->offset) + return ZAR_ERROR_DATA; + } + else + { + out->offset = zar_be32(n + 4); + out->size = zar_be32(n + 8); + out->dir = 1; + if ( out->offset > z->nodes + || out->size > z->nodes - out->offset) + return ZAR_ERROR_DATA; + } + return ZAR_OK; +} + +/* Asks for the name @n points at, at most @want octets of it past its + * length; *@hdr and *@nlen say where it starts and how long it is */ +static int zar_name(zar_t *z, zar_request_t *req, const uint8_t *n, + size_t want, size_t *cap, size_t *hdr, size_t *nlen) +{ + uint32_t name_at = zar_be32(n) & 0x7FFFFFFFu; + if ( name_at == ZAR_NO_NAME + || name_at >= z->sec_size[ZAR_SEC_NAMES]) + return ZAR_ERROR_DATA; + *cap = 2 + want; + if (*cap > z->sec_size[ZAR_SEC_NAMES] - name_at) + *cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at); + if (z->got < *cap) + return zar_ask(z, req, ZAR_TO_NAME, + z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, *cap); + z->got = 0; + *nlen = z->name[0] & 0x7F; + *hdr = 1; + if (z->name[0] & 0x80) + { + if (*cap < 2) + return ZAR_ERROR_DATA; + *nlen |= (size_t)z->name[1] << 7; + *hdr = 2; + } + return ZAR_OK; +} + +/* Down the tree from the root, a binary search of each directory's + * sorted entries for the path's next part */ static int zar_lookup_run(zar_t *z, zar_request_t *req, zar_file_t *out) { for (;;) { - const char *p; - const uint8_t *n; - size_t plen; - size_t cap; - size_t hdr; - size_t nlen; - size_t i; - uint32_t name_at; + const char *p; + size_t plen; + size_t cap; + size_t hdr; + size_t nlen; + size_t i; + int c; + int e; while (*z->path == '/' || *z->path == '\\') z->path++; p = z->path; for (plen = 0; p[plen] && p[plen] != '/' && p[plen] != '\\'; plen++) { } - /* a listing of the root looks up no name */ - if ( (!plen && !(z->want_dir && z->stage == ZAR_L_DIR)) - || plen > ZAR_NAME_MAX) - return ZAR_ERROR_NOT_FOUND; switch (z->stage) { - case ZAR_L_DIR: - /* the directory node z->dir_first names */ - if (z->dir_first >= z->nodes) - return ZAR_ERROR_DATA; + case ZAR_L_ROOT: + if (z->got < ZAR_NODE) + return zar_ask(z, req, ZAR_TO_BUF, + z->sec_off[ZAR_SEC_TREE], z->node, ZAR_NODE); + z->got = 0; + z->stage = ZAR_L_HAVE; + break; + case ZAR_L_HAVE: + if ((e = zar_node_file(z, z->node, out)) != ZAR_OK) + return e; + if (!plen) + return ZAR_OK; + if (!out->dir || plen > ZAR_NAME_MAX) + return ZAR_ERROR_NOT_FOUND; + z->dir_first = (uint32_t)out->offset; + z->lo = 0; + z->hi = (uint32_t)out->size; + z->stage = ZAR_L_PROBE; + break; + case ZAR_L_PROBE: + if (z->lo >= z->hi) + return ZAR_ERROR_NOT_FOUND; + z->entry = z->dir_first + z->lo + (z->hi - z->lo) / 2; if (z->got < ZAR_NODE) return zar_ask(z, req, ZAR_TO_BUF, z->sec_off[ZAR_SEC_TREE] - + (uint64_t)z->dir_first * ZAR_NODE, z->buf, ZAR_NODE); - z->got = 0; - if (z->buf[0] & 0x80) - return ZAR_ERROR_NOT_FOUND; - z->dir_first = zar_be32(z->buf + 4); - z->dir_count = zar_be32(z->buf + 8); - if ( z->dir_first > z->nodes - || z->dir_count > z->nodes - z->dir_first) - return ZAR_ERROR_DATA; - z->dir_index = 0; - z->batch = 0; - z->stage = ZAR_L_BATCH; - if (!plen) - return ZAR_OK; - break; - case ZAR_L_BATCH: - if (z->dir_index >= z->dir_count) - return ZAR_ERROR_NOT_FOUND; - z->batch = z->dir_count - z->dir_index; - if (z->batch > ZAR_BATCH) - z->batch = ZAR_BATCH; - if (z->got < (size_t)z->batch * ZAR_NODE) - return zar_ask(z, req, ZAR_TO_BUF, - z->sec_off[ZAR_SEC_TREE] - + ((uint64_t)z->dir_first + z->dir_index) * ZAR_NODE, - z->buf, (size_t)z->batch * ZAR_NODE); + + (uint64_t)z->entry * ZAR_NODE, z->buf, ZAR_NODE); z->got = 0; - z->entry = 0; z->stage = ZAR_L_NAME; break; case ZAR_L_NAME: - if (z->entry >= z->batch) - { - z->dir_index += z->batch; - z->stage = ZAR_L_BATCH; - break; - } - n = z->buf + z->entry * ZAR_NODE; - name_at = zar_be32(n) & 0x7FFFFFFFu; - if ( name_at == ZAR_NO_NAME - || name_at >= z->sec_size[ZAR_SEC_NAMES]) + if ((e = zar_name(z, req, z->buf, plen, + &cap, &hdr, &nlen)) != ZAR_OK) + return e; + if (hdr + (nlen < plen ? nlen : plen) > cap) return ZAR_ERROR_DATA; - cap = 2 + plen; - if (cap > z->sec_size[ZAR_SEC_NAMES] - name_at) - cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at); - if (z->got < cap) - return zar_ask(z, req, ZAR_TO_NAME, - z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, cap); - z->got = 0; - nlen = z->name[0] & 0x7F; - hdr = 1; - if (z->name[0] & 0x80) + c = 0; + for (i = 0; i < plen && i < nlen && !c; i++) + c = zar_lower((unsigned char)p[i]) + - zar_lower(z->name[hdr + i]); + if (!c) + c = (plen > nlen) - (plen < nlen); + if (c) { - if (cap < 2) - return ZAR_ERROR_DATA; - nlen |= (size_t)z->name[1] << 7; - hdr = 2; - } - if (nlen != plen || hdr + nlen > cap) - { - z->entry++; - break; - } - for (i = 0; i < plen; i++) - if ( zar_upper((unsigned char)p[i]) - != zar_upper(z->name[hdr + i])) - break; - if (i < plen) - { - z->entry++; + uint32_t mid = z->entry - z->dir_first; + if (c < 0) + z->hi = mid; + else + z->lo = mid + 1; + z->stage = ZAR_L_PROBE; break; } + memcpy(z->node, z->buf, ZAR_NODE); z->path += plen; - while (*z->path == '/' || *z->path == '\\') - z->path++; - if (*z->path || z->want_dir) - { - /* down into this entry, which must be a directory */ - if (n[0] & 0x80) - return ZAR_ERROR_NOT_FOUND; - z->dir_first = zar_be32(n + 4); - z->dir_count = zar_be32(n + 8); - if ( z->dir_first > z->nodes - || z->dir_count > z->nodes - z->dir_first) - return ZAR_ERROR_DATA; - z->dir_index = 0; - z->batch = 0; - z->stage = ZAR_L_BATCH; - if (!*z->path) - return ZAR_OK; - break; - } - if (!(n[0] & 0x80)) - return ZAR_ERROR_NOT_FOUND; - out->offset = zar_be32(n + 4) - | ((uint64_t)(zar_be32(n + 12) & 0xFFFFu) << 32); - out->size = zar_be32(n + 8) - | ((uint64_t)(zar_be32(n + 12) & 0xFFFF0000u) << 16); - if ( out->offset > z->blocks * ZAR_BLOCK - || out->size > z->blocks * ZAR_BLOCK - out->offset) - return ZAR_ERROR_DATA; - return ZAR_OK; + z->stage = ZAR_L_HAVE; + break; default: return ZAR_ERROR_STATE; } @@ -472,7 +467,7 @@ static int zar_list_run(zar_t *z, zar_request_t *req, zar_dirent_t *ent) size_t cap; size_t hdr; size_t nlen; - uint32_t name_at; + int e; switch (z->stage) { @@ -498,33 +493,18 @@ static int zar_list_run(zar_t *z, zar_request_t *req, zar_dirent_t *ent) z->stage = ZAR_L_BATCH; break; } - n = z->buf + z->entry * ZAR_NODE; - name_at = zar_be32(n) & 0x7FFFFFFFu; - if ( name_at == ZAR_NO_NAME - || name_at >= z->sec_size[ZAR_SEC_NAMES]) - return ZAR_ERROR_DATA; - cap = 2 + ZAR_NAME_MAX; - if (cap > z->sec_size[ZAR_SEC_NAMES] - name_at) - cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at); - if (z->got < cap) - return zar_ask(z, req, ZAR_TO_NAME, - z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, cap); - z->got = 0; - nlen = z->name[0] & 0x7F; - hdr = 1; - if (z->name[0] & 0x80) - { - if (cap < 2) - return ZAR_ERROR_DATA; - nlen |= (size_t)z->name[1] << 7; - hdr = 2; - } + n = z->buf + z->entry * ZAR_NODE; + if ((e = zar_name(z, req, n, ZAR_NAME_MAX, + &cap, &hdr, &nlen)) != ZAR_OK) + return e; if (nlen > ZAR_NAME_MAX || hdr + nlen > cap) return ZAR_ERROR_DATA; + if ((e = zar_node_file(z, n, &ent->file)) != ZAR_OK) + return e; z->name[hdr + nlen] = '\0'; ent->name = (const char*)z->name + hdr; ent->len = nlen; - ent->is_dir = !(n[0] & 0x80); + ent->is_dir = (int)ent->file.dir; z->entry++; return ZAR_OK; default: @@ -544,20 +524,48 @@ int zar_dir_step(zar_t *z, zar_request_t *req, zar_dirent_t *ent) e = ZAR_ERROR_DATA; else if (z->listing) e = zar_list_run(z, req, ent); - else if ((e = zar_lookup_run(z, req, NULL)) == ZAR_OK) + else { - z->listing = true; - e = zar_list_run(z, req, ent); + zar_file_t d; + if ((e = zar_lookup_run(z, req, &d)) == ZAR_OK) + { + if (!d.dir) + e = ZAR_ERROR_NOT_FOUND; + else + { + z->dir_first = (uint32_t)d.offset; + z->dir_count = (uint32_t)d.size; + z->dir_index = 0; + z->batch = 0; + z->got = 0; + z->stage = ZAR_L_BATCH; + z->listing = true; + e = zar_list_run(z, req, ent); + } + } } if (e != ZAR_PENDING && e != ZAR_OK) z->op = ZAR_OP_NONE; return e; } +uint64_t zar_index_offset(const zar_t *z) +{ + uint64_t off; + if (!z || !z->nodes) + return 0; + off = z->sec_off[ZAR_SEC_RECORDS]; + if (z->sec_off[ZAR_SEC_NAMES] < off) + off = z->sec_off[ZAR_SEC_NAMES]; + if (z->sec_off[ZAR_SEC_TREE] < off) + off = z->sec_off[ZAR_SEC_TREE]; + return off; +} + int zar_read_begin(zar_t *z, const zar_file_t *file, uint64_t off, void *dst, size_t len) { - if (!z || !file || (!dst && len)) + if (!z || !file || file->dir || (!dst && len)) return ZAR_ERROR_PARAM; if (z->op == ZAR_OP_OPEN) return ZAR_ERROR_STATE; diff --git a/libretro-common/formats/zar/zar_write.c b/libretro-common/formats/zar/zar_write.c new file mode 100644 index 0000000000..02ec98ee50 --- /dev/null +++ b/libretro-common/formats/zar/zar_write.c @@ -0,0 +1,550 @@ +/* Copyright (C) 2010-2026 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (zar_write.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* The archive zar.c reads, written front to back: the blocks as the + * files' octets fill them, then the records, the names, the tree with + * each directory's entries sorted, and the footer, whose hash is + * SHA-256 over the whole file with the hash itself zeroed. */ + +#include +#include + +#include +#include +#include +#ifdef HAVE_RZSTD +#include +#endif + +#define ZAR_W_BLOCK 65536u +#define ZAR_W_RECORD 40 +#define ZAR_W_NODE 16 +#define ZAR_W_FOOTER 144 +#define ZAR_W_NONE 0xFFFFFFFFu +#define ZAR_W_NAME 0x7FFFu +#define ZAR_W_MAX ((((uint64_t)1) << 48) - 1) /* a file's offset or size */ + +typedef struct zar_wnode +{ + uint64_t offset; /* a file's, in the stream */ + uint64_t size; + uint32_t name; /* in the names, ZAR_W_NONE for the root */ + uint32_t child; /* a directory's last entry added */ + uint32_t sibling; /* the entry added before this one */ + uint32_t first; /* a directory's first entry's node, written */ + uint32_t count; + uint8_t file; +} zar_wnode_t; + +struct zar_writer +{ + struct sha256_state sha; + zar_write_fn write; + void *ud; + uint8_t *block; + uint8_t *enc; + uint8_t *names; + uint8_t *recs; + zar_wnode_t *nodes; + uint64_t out; /* octets written */ + uint64_t in; /* octets of the stream */ + size_t fill; /* in the block */ + size_t enc_cap; + size_t names_len; + size_t names_cap; + size_t recs_len; + size_t recs_cap; + uint64_t blocks; + uint32_t nnodes; + uint32_t nodes_cap; + uint32_t current; /* the file appended to */ + int level; + bool failed; +}; + +static void zar_w_be32(uint8_t *p, uint32_t v) +{ + p[0] = (uint8_t)(v >> 24); + p[1] = (uint8_t)(v >> 16); + p[2] = (uint8_t)(v >> 8); + p[3] = (uint8_t)v; +} + +static void zar_w_be64(uint8_t *p, uint64_t v) +{ + zar_w_be32(p, (uint32_t)(v >> 32)); + zar_w_be32(p + 4, (uint32_t)v); +} + +static bool zar_w_grow(void **p, size_t *cap, size_t need, size_t size) +{ + size_t n; + void *q; + if (need <= *cap) + return true; + n = *cap ? *cap : 64; + while (n < need) + n *= 2; + if (!(q = realloc(*p, n * size))) + return false; + *p = q; + *cap = n; + return true; +} + +static int zar_w_out(zar_writer_t *w, const void *data, size_t len) +{ + if (w->failed) + return ZAR_ERROR_WRITE; + if (len && !w->write(w->ud, data, len)) + { + w->failed = true; + return ZAR_ERROR_WRITE; + } + sha256_stream_update(&w->sha, (const uint8_t*)data, len); + w->out += len; + return ZAR_OK; +} + +static uint32_t zar_w_node(zar_writer_t *w, uint32_t parent, + uint32_t name, bool file) +{ + zar_wnode_t *n; + size_t cap = w->nodes_cap; + if (!zar_w_grow((void**)&w->nodes, &cap, (size_t)w->nnodes + 1, + sizeof(zar_wnode_t))) + return ZAR_W_NONE; + w->nodes_cap = (uint32_t)cap; + n = &w->nodes[w->nnodes]; + n->offset = w->in; + n->size = 0; + n->name = name; + n->child = ZAR_W_NONE; + n->sibling = ZAR_W_NONE; + n->first = 0; + n->count = 0; + n->file = file ? 1 : 0; + if (parent != ZAR_W_NONE) + { + n->sibling = w->nodes[parent].child; + w->nodes[parent].child = w->nnodes; + w->nodes[parent].count++; + } + return w->nnodes++; +} + +zar_writer_t *zar_writer_new(zar_write_fn write, void *ud, int level) +{ + zar_writer_t *w; + if (!write || !(w = (zar_writer_t*)calloc(1, sizeof(*w)))) + return NULL; + w->write = write; + w->ud = ud; + w->level = level; + w->current = ZAR_W_NONE; + sha256_stream_init(&w->sha, 0); + if ( !(w->block = (uint8_t*)malloc(ZAR_W_BLOCK)) + || zar_w_node(w, ZAR_W_NONE, ZAR_W_NONE, false) != 0) + { + zar_writer_free(w); + return NULL; + } + return w; +} + +void zar_writer_free(zar_writer_t *w) +{ + if (!w) + return; + free(w->block); + free(w->enc); + free(w->names); + free(w->recs); + free(w->nodes); + free(w); +} + +static int zar_w_lower(int c) +{ + return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; +} + +static const uint8_t *zar_w_name(const zar_writer_t *w, uint32_t at, + size_t *len) +{ + const uint8_t *p = w->names + at; + if (p[0] & 0x80) + { + *len = (p[0] & 0x7F) | ((size_t)p[1] << 7); + return p + 2; + } + *len = p[0]; + return p + 1; +} + +/* the order zar.c's search expects */ +static int zar_w_cmp(const uint8_t *a, size_t alen, + const uint8_t *b, size_t blen) +{ + size_t i; + for (i = 0; i < alen && i < blen; i++) + { + int c = zar_w_lower(a[i]) - zar_w_lower(b[i]); + if (c) + return c; + } + return (alen > blen) - (alen < blen); +} + +static uint32_t zar_w_find(const zar_writer_t *w, uint32_t dir, + const char *name, size_t len) +{ + uint32_t c; + for (c = w->nodes[dir].child; c != ZAR_W_NONE; c = w->nodes[c].sibling) + { + size_t nlen; + const uint8_t *n = zar_w_name(w, w->nodes[c].name, &nlen); + if (nlen == len && !zar_w_cmp(n, nlen, (const uint8_t*)name, len)) + return c; + } + return ZAR_W_NONE; +} + +static uint32_t zar_w_add_name(zar_writer_t *w, const char *name, + size_t len) +{ + uint32_t at = (uint32_t)w->names_len; + if ( w->names_len + len + 2 > 0x7FFFFFFFu + || !zar_w_grow((void**)&w->names, &w->names_cap, + w->names_len + len + 2, 1)) + return ZAR_W_NONE; + if (len >= 0x80) + { + w->names[w->names_len++] = (uint8_t)((len & 0x7F) | 0x80); + w->names[w->names_len++] = (uint8_t)(len >> 7); + } + else + w->names[w->names_len++] = (uint8_t)len; + memcpy(w->names + w->names_len, name, len); + w->names_len += len; + return at; +} + +/* Walks @path from the root, adding the directories it lacks; the last + * part is added as a file when @file */ +static int zar_w_path(zar_writer_t *w, const char *path, bool file, + uint32_t *out) +{ + uint32_t at = 0; + for (;;) + { + size_t len; + uint32_t c; + bool last; + + while (*path == '/' || *path == '\\') + path++; + for (len = 0; path[len] && path[len] != '/' && path[len] != '\\'; len++) { } + if (!len) + break; + if (len > ZAR_W_NAME) + return ZAR_ERROR_PARAM; + last = !path[len]; + if (!last) + { + const char *q = path + len; + while (*q == '/' || *q == '\\') + q++; + last = !*q; + } + c = zar_w_find(w, at, path, len); + if (c != ZAR_W_NONE) + { + if (w->nodes[c].file || (last && file)) + return ZAR_ERROR_PARAM; + } + else + { + uint32_t name = zar_w_add_name(w, path, len); + if ( name == ZAR_W_NONE + || (c = zar_w_node(w, at, name, last && file)) == ZAR_W_NONE) + return ZAR_ERROR_MEM; + } + at = c; + path += len; + } + if (file && !at) + return ZAR_ERROR_PARAM; + *out = at; + return ZAR_OK; +} + +int zar_writer_dir(zar_writer_t *w, const char *path) +{ + uint32_t at; + if (!w || !path) + return ZAR_ERROR_PARAM; + return zar_w_path(w, path, false, &at); +} + +int zar_writer_file(zar_writer_t *w, const char *path) +{ + uint32_t at; + int e; + if (!w || !path) + return ZAR_ERROR_PARAM; + w->current = ZAR_W_NONE; + if ((e = zar_w_path(w, path, true, &at)) == ZAR_OK) + w->current = at; + return e; +} + +static int zar_w_block(zar_writer_t *w, const uint8_t *src) +{ + const uint8_t *data = src; + size_t len = ZAR_W_BLOCK; + size_t r; + int e; + +#ifdef HAVE_RZSTD + { + size_t wrote = 0; + if (!w->enc) + { + w->enc_cap = rzstd_compress_bound(ZAR_W_BLOCK); + if (!(w->enc = (uint8_t*)malloc(w->enc_cap))) + return ZAR_ERROR_MEM; + } + if ( rzstd_encode(w->enc, w->enc_cap, src, ZAR_W_BLOCK, + w->level, &wrote) == RZSTD_PROCESS_END + && wrote && wrote < ZAR_W_BLOCK) + { + data = w->enc; + len = wrote; + } + } +#endif + + r = (size_t)(w->blocks % 16); + if (!r) + { + if (!zar_w_grow((void**)&w->recs, &w->recs_cap, + w->recs_len + ZAR_W_RECORD, 1)) + return ZAR_ERROR_MEM; + memset(w->recs + w->recs_len, 0, ZAR_W_RECORD); + zar_w_be64(w->recs + w->recs_len, w->out); + w->recs_len += ZAR_W_RECORD; + } + w->recs[w->recs_len - ZAR_W_RECORD + 8 + r * 2] = (uint8_t)((len - 1) >> 8); + w->recs[w->recs_len - ZAR_W_RECORD + 8 + r * 2 + 1] = (uint8_t)(len - 1); + w->blocks++; + if ((e = zar_w_out(w, data, len)) != ZAR_OK) + return e; + return ZAR_OK; +} + +static int zar_w_append(zar_writer_t *w, const uint8_t *src, size_t len) +{ + int e; + w->in += len; + while (len) + { + size_t take; + /* a whole block straight from the caller */ + if (!w->fill && len >= ZAR_W_BLOCK) + { + if ((e = zar_w_block(w, src)) != ZAR_OK) + return e; + src += ZAR_W_BLOCK; + len -= ZAR_W_BLOCK; + continue; + } + take = ZAR_W_BLOCK - w->fill; + if (take > len) + take = len; + memcpy(w->block + w->fill, src, take); + w->fill += take; + src += take; + len -= take; + if (w->fill == ZAR_W_BLOCK) + { + w->fill = 0; + if ((e = zar_w_block(w, w->block)) != ZAR_OK) + return e; + } + } + return ZAR_OK; +} + +int zar_writer_append(zar_writer_t *w, const void *data, size_t len) +{ + if (!w || (!data && len) || w->current == ZAR_W_NONE) + return ZAR_ERROR_PARAM; + if (w->failed) + return ZAR_ERROR_WRITE; + if (len > ZAR_W_MAX - w->in) + return ZAR_ERROR_PARAM; + w->nodes[w->current].size += len; + return zar_w_append(w, (const uint8_t*)data, len); +} + +/* Sorts @ids by name, @tmp as long */ +static void zar_w_sort(const zar_writer_t *w, uint32_t *ids, + uint32_t *tmp, uint32_t n) +{ + uint32_t width; + for (width = 1; width < n; width *= 2) + { + uint32_t lo; + for (lo = 0; lo < n; lo += 2 * width) + { + uint32_t mid = lo + width < n ? lo + width : n; + uint32_t hi = lo + 2 * width < n ? lo + 2 * width : n; + uint32_t i = lo; + uint32_t j = mid; + uint32_t k = lo; + while (i < mid && j < hi) + { + size_t alen, blen; + const uint8_t *a = zar_w_name(w, w->nodes[ids[i]].name, &alen); + const uint8_t *b = zar_w_name(w, w->nodes[ids[j]].name, &blen); + tmp[k++] = zar_w_cmp(b, blen, a, alen) < 0 ? ids[j++] : ids[i++]; + } + while (i < mid) + tmp[k++] = ids[i++]; + while (j < hi) + tmp[k++] = ids[j++]; + } + memcpy(ids, tmp, n * sizeof(*ids)); + } +} + +int zar_writer_finish(zar_writer_t *w) +{ + uint8_t ft[ZAR_W_FOOTER]; + uint8_t digest[32]; + uint64_t sec[6][2]; + uint32_t *order; + uint32_t *tmp; + uint32_t k; + uint32_t len; + int e = ZAR_OK; + + if (!w) + return ZAR_ERROR_PARAM; + w->current = ZAR_W_NONE; + if (w->fill) + { + memset(w->block + w->fill, 0, ZAR_W_BLOCK - w->fill); + w->fill = 0; + if ((e = zar_w_block(w, w->block)) != ZAR_OK) + return e; + } + sec[0][0] = 0; + sec[0][1] = w->out; + memset(ft, 0, sizeof(ft)); + if ((e = zar_w_out(w, ft, (size_t)((8 - w->out % 8) % 8))) != ZAR_OK) + return e; + + sec[1][0] = w->out; + sec[1][1] = w->recs_len; + if ((e = zar_w_out(w, w->recs, w->recs_len)) != ZAR_OK) + return e; + sec[2][0] = w->out; + sec[2][1] = w->names_len; + if ((e = zar_w_out(w, w->names, w->names_len)) != ZAR_OK) + return e; + + /* breadth first: each directory's entries together and sorted */ + order = (uint32_t*)malloc(w->nnodes * sizeof(uint32_t)); + tmp = (uint32_t*)malloc(w->nnodes * sizeof(uint32_t)); + if (!order || !tmp) + { + free(order); + free(tmp); + return ZAR_ERROR_MEM; + } + order[0] = 0; + len = 1; + for (k = 0; k < len; k++) + { + zar_wnode_t *d = &w->nodes[order[k]]; + uint32_t c; + uint32_t at; + if (d->file) + continue; + d->first = len; + at = len + d->count; + for (c = d->child; c != ZAR_W_NONE; c = w->nodes[c].sibling) + order[--at] = c; + zar_w_sort(w, order + len, tmp, d->count); + len += d->count; + } + sec[3][0] = w->out; + sec[3][1] = (uint64_t)w->nnodes * ZAR_W_NODE; + for (k = 0; k < w->nnodes && e == ZAR_OK; k++) + { + uint8_t n[ZAR_W_NODE]; + const zar_wnode_t *d = &w->nodes[order[k]]; + zar_w_be32(n, (d->name == ZAR_W_NONE ? 0x7FFFFFFFu : d->name) + | (d->file ? 0x80000000u : 0)); + if (d->file) + { + zar_w_be32(n + 4, (uint32_t)d->offset); + zar_w_be32(n + 8, (uint32_t)d->size); + zar_w_be32(n + 12, (uint32_t)((d->offset >> 32) & 0xFFFFu) + | (uint32_t)((d->size >> 16) & 0xFFFF0000u)); + } + else + { + zar_w_be32(n + 4, d->first); + zar_w_be32(n + 8, d->count); + zar_w_be32(n + 12, 0); + } + e = zar_w_out(w, n, sizeof(n)); + } + free(order); + free(tmp); + if (e != ZAR_OK) + return e; + + sec[4][0] = sec[5][0] = w->out; + sec[4][1] = sec[5][1] = 0; + for (k = 0; k < 6; k++) + { + zar_w_be64(ft + k * 16, sec[k][0]); + zar_w_be64(ft + k * 16 + 8, sec[k][1]); + } + zar_w_be64(ft + 128, w->out + ZAR_W_FOOTER); + zar_w_be32(ft + 136, 0x61BF3A01u); + zar_w_be32(ft + 140, 0x169F52D6u); + sha256_stream_update(&w->sha, ft, sizeof(ft)); + sha256_stream_final(&w->sha, digest); + memcpy(ft + 96, digest, sizeof(digest)); + if (w->failed || !w->write(w->ud, ft, sizeof(ft))) + { + w->failed = true; + return ZAR_ERROR_WRITE; + } + w->out += sizeof(ft); + return ZAR_OK; +} diff --git a/libretro-common/include/formats/zar.h b/libretro-common/include/formats/zar.h index 53723f9e4f..db13d12020 100644 --- a/libretro-common/include/formats/zar.h +++ b/libretro-common/include/formats/zar.h @@ -47,10 +47,11 @@ RETRO_BEGIN_DECLS * while ((err = zar_open_step(z, &req)) == ZAR_PENDING) * zar_feed(z, pull(req.offset, req.dst, req.length)); * - * Nothing is read that an operation does not look at: a directory's - * entries a run at a time, a name only when its length matches, a - * block's place from its one record. A read of a block kept as it is - * asks for its octets straight into the read's destination; a + * Nothing is read that an operation does not look at: a lookup reads + * the entries and names its binary search of each directory probes, a + * listing a directory's entries a run at a time, a block's place comes + * from its one record. A read of a block kept as it is asks for its + * octets straight into the read's destination; a * compressed block is decoded into the destination when the read * wants all of it, and otherwise into a block kept for the reads that * follow. */ @@ -64,6 +65,7 @@ RETRO_BEGIN_DECLS #define ZAR_ERROR_STATE (-4) #define ZAR_ERROR_NOT_FOUND (-5) #define ZAR_ERROR_UNSUPPORTED (-6) +#define ZAR_ERROR_WRITE (-7) typedef struct zar zar_t; @@ -80,11 +82,13 @@ typedef struct zar_request uint8_t *dst; /* where they go */ } zar_request_t; -/* A file in the archive: its octets in the stream */ +/* A file in the archive, or a directory: a file's octets in the + * stream, a directory's first entry's node and its entry count */ typedef struct zar_file { uint64_t offset; uint64_t size; + unsigned dir; } zar_file_t; /* @file_size: the archive's size, which its footer must agree with */ @@ -115,7 +119,8 @@ int zar_feed(zar_t *z, size_t len); /** * zar_lookup_begin: * @path : '/' or '\' separated, matched without regard to ASCII case; - * it must stay valid until the lookup is over + * it must stay valid until the lookup is over. A file or a + * directory; an empty path is the root. * * Returns: ZAR_OK, or ZAR_ERROR_STATE before the archive is open. **/ @@ -137,6 +142,7 @@ typedef struct zar_dirent const char *name; size_t len; int is_dir; + zar_file_t file; } zar_dirent_t; /** @@ -153,20 +159,26 @@ int zar_dir_begin(zar_t *z, const char *path); /** * zar_dir_step: * - * Returns: ZAR_OK with @ent filled in for each entry in the archive's - * order, ZAR_END past the last one, ZAR_PENDING with @req filled in + * Returns: ZAR_OK with @ent filled in for each entry in name order, + * ZAR_END past the last one, ZAR_PENDING with @req filled in * when it needs a range of the file, ZAR_ERROR_NOT_FOUND when @path is * no directory, or another negative code. **/ int zar_dir_step(zar_t *z, zar_request_t *req, zar_dirent_t *ent); +/* Where the octets past the stream's blocks start, from which a caller + * can keep the rest of the file in memory and answer every request a + * lookup or listing makes without I/O; 0 before the archive is open */ +uint64_t zar_index_offset(const zar_t *z); + /** * zar_read_begin: * * Arms a read of @len octets of @file at @off into @dst, short at the * end of the file, abandoning a read or lookup in progress. * - * Returns: ZAR_OK, or ZAR_ERROR_STATE before the archive is open. + * Returns: ZAR_OK, ZAR_ERROR_PARAM for a directory, or ZAR_ERROR_STATE + * before the archive is open. **/ int zar_read_begin(zar_t *z, const zar_file_t *file, uint64_t off, void *dst, size_t len); @@ -183,6 +195,53 @@ int zar_read_step(zar_t *z, zar_request_t *req); /* octets the armed read has put at its @dst so far */ size_t zar_read_progress(const zar_t *z); +/* The writer, in zar_write.c: an archive built front to back, every + * octet handed to @write as it is made. Blocks are compressed with + * Zstandard at @level (1 to 9) where that shrinks them and HAVE_RZSTD + * is defined, and kept as they are otherwise. Paths are matched without + * regard to ASCII case, as the reader matches them. */ + +typedef struct zar_writer zar_writer_t; + +/* Returns nonzero when all @len octets were written */ +typedef int (*zar_write_fn)(void *ud, const void *data, size_t len); + +zar_writer_t *zar_writer_new(zar_write_fn write, void *ud, int level); + +void zar_writer_free(zar_writer_t *w); + +/** + * zar_writer_dir: + * + * Adds directory @path and those above it that are missing. + * + * Returns: ZAR_OK, or ZAR_ERROR_PARAM when a part of it is a file. + **/ +int zar_writer_dir(zar_writer_t *w, const char *path); + +/** + * zar_writer_file: + * + * Adds file @path, the directories above it that are missing, and + * makes it the one zar_writer_append() adds to. + * + * Returns: ZAR_OK, or ZAR_ERROR_PARAM when it is there already or a + * part above it is a file. + **/ +int zar_writer_file(zar_writer_t *w, const char *path); + +int zar_writer_append(zar_writer_t *w, const void *data, size_t len); + +/** + * zar_writer_finish: + * + * Writes the rest of the archive. + * + * Returns: ZAR_OK, ZAR_ERROR_WRITE when @write failed, or + * ZAR_ERROR_MEM. + **/ +int zar_writer_finish(zar_writer_t *w); + RETRO_END_DECLS #endif diff --git a/libretro-common/samples/formats/zar/Makefile b/libretro-common/samples/formats/zar/Makefile index eb83eb218a..47a75701ca 100644 --- a/libretro-common/samples/formats/zar/Makefile +++ b/libretro-common/samples/formats/zar/Makefile @@ -8,6 +8,20 @@ LIBRETRO_COMM_DIR := ../../.. SOURCES := \ zar_test.c \ $(LIBRETRO_COMM_DIR)/formats/zar/zar.c \ + $(LIBRETRO_COMM_DIR)/formats/zar/zar_write.c \ + $(LIBRETRO_COMM_DIR)/hash/lrc_hash.c \ + $(LIBRETRO_COMM_DIR)/features/features_cpu.c \ + $(LIBRETRO_COMM_DIR)/streams/file_stream.c \ + $(LIBRETRO_COMM_DIR)/vfs/vfs_implementation.c \ + $(LIBRETRO_COMM_DIR)/file/file_path.c \ + $(LIBRETRO_COMM_DIR)/file/file_path_io.c \ + $(LIBRETRO_COMM_DIR)/string/stdstring.c \ + $(LIBRETRO_COMM_DIR)/string/rstrtod.c \ + $(LIBRETRO_COMM_DIR)/encodings/encoding_utf.c \ + $(LIBRETRO_COMM_DIR)/compat/compat_strl.c \ + $(LIBRETRO_COMM_DIR)/compat/compat_posix_string.c \ + $(LIBRETRO_COMM_DIR)/compat/fopen_utf8.c \ + $(LIBRETRO_COMM_DIR)/time/rtime.c \ $(LIBRETRO_COMM_DIR)/encodings/encoding_rzstd.c \ $(LIBRETRO_COMM_DIR)/encodings/encoding_xxh64.c diff --git a/libretro-common/samples/formats/zar/zar_test.c b/libretro-common/samples/formats/zar/zar_test.c index 7d94007053..8d3bfe46c4 100644 --- a/libretro-common/samples/formats/zar/zar_test.c +++ b/libretro-common/samples/formats/zar/zar_test.c @@ -4,15 +4,16 @@ * * The archives hold blocks compressed with Zstandard beside blocks * kept as they are (data that does not compress), a file crossing - * them, a directory with more entries than the reader asks for at once + * them, a directory with more entries than a listing asks for at once * and a nested one. Ranges are supplied short now and then. Every - * request is checked: a lookup asks for a directory's entries a run at - * a time and for each name only as far as it can match, exactly that - * many octets; a read of a block kept as it is lands in the read's own - * destination; a compressed block is fetched once for the partial - * reads that follow it. A directory is listed in the archive's order, - * each name read whole. A footer that is not one, a size that - * disagrees and a block that does not decode must fail. */ + * request is checked: a lookup bisects each directory's sorted entries + * and reads each probed name only as far as it can match; a read of a + * block kept as it is lands in the read's own destination; a + * compressed block is fetched once for the partial reads that follow + * it. A directory is listed in name order, each name read whole, each + * entry's place the one a lookup gives. The writer's archive reads + * back, sorted, with its footer's hash. A footer that is not one, a + * size that disagrees and a block that does not decode must fail. */ #include #include @@ -21,6 +22,7 @@ #include #include #include +#include #define BLOCK 65536 #define FILLERS 38 @@ -85,8 +87,8 @@ static size_t pull(file_t *f, const zar_request_t *req) return n; } -/* the tree: the root holds FILLERS files, media/ and default.xex last; - * media/ holds a.bin and intro.wmv */ +/* the tree, each directory in name order: the root holds default.xex, + * FILLERS files and media/, which holds a.bin and intro.wmv */ #define N_NODES (1 + FILLERS + 2 + 2) typedef struct @@ -101,6 +103,7 @@ typedef struct static uint8_t *stream; /* the files, laid end to end */ static size_t stream_size; +static size_t names_size; /* The archive of the tree, with @bad_block's compressed octets broken * when it is not -1 */ @@ -126,15 +129,15 @@ static void make_archive(file_t *f, int bad_block) nodes[0].count = FILLERS + 2; for (i = 0; i < FILLERS; i++) { - sprintf(nodes[1 + i].name, "file%02u.bin", (unsigned)i); - nodes[1 + i].file = true; + sprintf(nodes[2 + i].name, "file%02u.bin", (unsigned)i); + nodes[2 + i].file = true; } - strcpy(nodes[1 + FILLERS].name, "media"); - nodes[1 + FILLERS].first = 1 + FILLERS + 2; - nodes[1 + FILLERS].count = 2; - strcpy(nodes[2 + FILLERS].name, "default.xex"); - nodes[2 + FILLERS].file = true; - nodes[2 + FILLERS].size = XEX_SIZE; + strcpy(nodes[2 + FILLERS].name, "media"); + nodes[2 + FILLERS].first = 1 + FILLERS + 2; + nodes[2 + FILLERS].count = 2; + strcpy(nodes[1].name, "default.xex"); + nodes[1].file = true; + nodes[1].size = XEX_SIZE; strcpy(nodes[3 + FILLERS].name, "a.bin"); nodes[3 + FILLERS].file = true; nodes[3 + FILLERS].size = 5; @@ -142,7 +145,7 @@ static void make_archive(file_t *f, int bad_block) nodes[4 + FILLERS].file = true; nodes[4 + FILLERS].size = 100; for (i = 0; i < FILLERS; i++) - nodes[1 + i].size = 10; + nodes[2 + i].size = 10; /* the stream: default.xex first - block 0 compresses, block 1 does * not, block 2 compresses - then the rest */ @@ -151,12 +154,12 @@ static void make_archive(file_t *f, int bad_block) pos = 0; for (i = 0; i < XEX_SIZE; i++) stream[i] = (i / BLOCK == 1) ? (uint8_t)rnd() : (uint8_t)(i * 7 / 64); - nodes[2 + FILLERS].offset = 0; + nodes[1].offset = 0; pos = XEX_SIZE; for (i = 0; i < N_NODES; i++) { size_t k; - if (!nodes[i].file || i == 2 + FILLERS) + if (!nodes[i].file || i == 1) continue; nodes[i].offset = pos; for (k = 0; k < nodes[i].size; k++) @@ -242,6 +245,7 @@ static void make_archive(file_t *f, int bad_block) be64(ft + 0, 0); be64(ft + 8, bsize); be64(ft + 16, rec_at); be64(ft + 24, 40); be64(ft + 32, names_at); be64(ft + 40, nsize); + names_size = nsize; be64(ft + 48, tree_at); be64(ft + 56, sizeof(tree)); be64(ft + 64, end); be64(ft + 72, 0); be64(ft + 80, end); be64(ft + 88, 0); @@ -310,31 +314,53 @@ static void lookup_lane(void) f.asked = 0; check( lookup(&f, z, "\\DEFAULT.XEX", &zf) == ZAR_OK && zf.offset == 0 && zf.size == XEX_SIZE, "lookup by name"); - /* the root's node, its entries in two runs, each name as far as an - * eleven-octet name can match */ - check(f.asked == 16 + 32 * 16 + 8 * 16 + (FILLERS + 2) * 13, - "lookup reads only what it needs"); + /* the root's node, then six probes of the forty entries, each an + * entry's node and its name as far as an eleven-octet name can go */ + check(f.asked == 16 + 6 * (16 + 13), "lookup reads only what it needs"); check( lookup(&f, z, "media/Intro.WMV", &zf) == ZAR_OK && zf.size == 100, "nested lookup"); check( lookup(&f, z, "media/nope", &zf) == ZAR_ERROR_NOT_FOUND && lookup(&f, z, "default.xex/x", &zf) == ZAR_ERROR_NOT_FOUND - && lookup(&f, z, "media", &zf) == ZAR_ERROR_NOT_FOUND - && lookup(&f, z, "nope", &zf) == ZAR_ERROR_NOT_FOUND, + && lookup(&f, z, "nope", &zf) == ZAR_ERROR_NOT_FOUND + && lookup(&f, z, "file99.bin", &zf) == ZAR_ERROR_NOT_FOUND + && lookup(&f, z, "aaa", &zf) == ZAR_ERROR_NOT_FOUND + && lookup(&f, z, "zzz", &zf) == ZAR_ERROR_NOT_FOUND, "missing files"); + check( lookup(&f, z, "MEDIA/", &zf) == ZAR_OK + && zf.dir && zf.size == 2, "a directory"); + check( lookup(&f, z, "", &zf) == ZAR_OK + && zf.dir && zf.offset == 1 && zf.size == FILLERS + 2, "the root"); + { + unsigned i; + bool all = true; + for (i = 0; i < FILLERS; i++) + { + char name[32]; + sprintf(name, "/File%02u.BIN", i); + if ( lookup(&f, z, name, &zf) != ZAR_OK + || zf.dir || zf.size != 10) + all = false; + } + check(all, "every entry found by its search"); + } + check(zar_index_offset(z) == f.size - 144 - (16 * N_NODES) + - names_size - 40, "where the index starts"); zar_free(z); free(f.data); } -/* Lists @path, checking each entry against @names; returns how many - * matched in order, or -1 when the listing fails */ +/* Lists @path, checking each entry against @names; a file's place is + * the one a lookup of it gives. Returns how many matched in order, or + * -1 when the listing fails */ static int list(file_t *f, zar_t *z, const char *path, const char *const *names, const int *dirs, int n) { zar_request_t req; zar_dirent_t ent; + zar_file_t seen[64]; int err, i = 0; - if (zar_dir_begin(z, path) != ZAR_OK) + if (n > 64 || zar_dir_begin(z, path) != ZAR_OK) return -1; for (;;) { @@ -343,11 +369,25 @@ static int list(file_t *f, zar_t *z, const char *path, if (err != ZAR_OK) break; if ( i >= n || strcmp(ent.name, names[i]) - || ent.len != strlen(names[i]) || !ent.is_dir != !dirs[i]) + || ent.len != strlen(names[i]) || !ent.is_dir != !dirs[i] + || !ent.file.dir != !dirs[i]) return -1; - i++; + seen[i++] = ent.file; } - return err == ZAR_END ? i : -1; + if (err != ZAR_END) + return -1; + for (err = 0; err < i; err++) + { + char full[64]; + zar_file_t zf; + if (dirs[err]) + continue; + sprintf(full, "%s/%s", path, names[err]); + if ( lookup(f, z, full, &zf) != ZAR_OK + || zf.offset != seen[err].offset || zf.size != seen[err].size) + return -1; + } + return i; } static void list_lane(void) @@ -362,16 +402,16 @@ static void list_lane(void) zar_file_t zf; int i, pass; + root[0] = "default.xex"; + rdirs[0] = 0; for (i = 0; i < FILLERS; i++) { sprintf(fill[i], "file%02u.bin", (unsigned)i); - root[i] = fill[i]; - rdirs[i] = 0; + root[1 + i] = fill[i]; + rdirs[1 + i] = 0; } - root[FILLERS] = "media"; - rdirs[FILLERS] = 1; - root[FILLERS + 1] = "default.xex"; - rdirs[FILLERS + 1] = 0; + root[FILLERS + 1] = "media"; + rdirs[FILLERS + 1] = 1; for (pass = 0; pass < 2; pass++) { @@ -385,6 +425,9 @@ static void list_lane(void) check( list(&f, z, "nope", media, mdirs, 2) == -1 && list(&f, z, "default.xex", media, mdirs, 2) == -1, "a missing directory and a file are not listed"); + check( lookup(&f, z, "media", &zf) == ZAR_OK && zf.dir + && zar_read_begin(z, &zf, 0, f.data, 1) == ZAR_ERROR_PARAM, + "a directory is not read"); /* a lookup after a listing */ check( lookup(&f, z, "media/a.bin", &zf) == ZAR_OK && zf.size == 5, "lookup after a listing"); @@ -498,12 +541,217 @@ static void broken_lanes(void) free(f.data); } +/* the writer's output, grown in memory */ +typedef struct +{ + uint8_t *data; + size_t size; + size_t cap; + size_t fail_at; /* a write past this fails; 0 never */ +} sink_t; + +static int sink_write(void *ud, const void *data, size_t len) +{ + sink_t *k = (sink_t*)ud; + if (k->fail_at && k->size + len > k->fail_at) + return 0; + if (k->size + len > k->cap) + { + size_t n = k->cap ? k->cap * 2 : 65536; + uint8_t *q; + while (n < k->size + len) + n *= 2; + if (!(q = (uint8_t*)realloc(k->data, n))) + return 0; + k->data = q; + k->cap = n; + } + memcpy(k->data + k->size, data, len); + k->size += len; + return 1; +} + +static int fold(int c) +{ + return (c >= 'A' && c <= 'Z') ? c + 32 : c; +} + +static int name_cmp(const char *a, const char *b) +{ + for (; *a && *b; a++, b++) + if (fold((unsigned char)*a) != fold((unsigned char)*b)) + return fold((unsigned char)*a) - fold((unsigned char)*b); + return (*a != 0) - (*b != 0); +} + +#define W_FILES 9 + +static void writer_lane(void) +{ + static const char *const paths[W_FILES] = { + "content/a.bin", "content/Empty", "content/B_x", "content/ba", + "content/b[", "content/A_", "content/sub/deep/z", "meta/meta.xml", + NULL /* the long name */ + }; + char longname[8 + 200 + 1]; + size_t sizes[W_FILES] = { 3 * BLOCK + 777, 0, 1, 2, 3, 4, BLOCK, 50, 9 }; + uint8_t *data[W_FILES]; + sink_t k; + zar_writer_t *w; + file_t f; + zar_t *z; + zar_file_t zf; + zar_dirent_t e; + uint8_t digest[32]; + uint8_t saved[32]; + struct sha256_state sha; + unsigned i; + size_t j; + size_t total = 0; + bool ok = true; + + memset(&k, 0, sizeof(k)); + memcpy(longname, "content/", 8); + memset(longname + 8, 'q', 200); + longname[208] = '\0'; + check((w = zar_writer_new(sink_write, &k, 3)) != NULL, "writer"); + check( zar_writer_dir(w, "meta") == ZAR_OK + && zar_writer_dir(w, "content/sub/deep") == ZAR_OK + && zar_writer_dir(w, "/CONTENT/") == ZAR_OK, "directories"); + check(zar_writer_append(w, "x", 1) == ZAR_ERROR_PARAM, + "nothing to append to"); + for (i = 0; i < W_FILES; i++) + { + const char *path = paths[i] ? paths[i] : longname; + data[i] = (uint8_t*)malloc(sizes[i] + 1); + for (j = 0; j < sizes[i]; j++) + data[i][j] = (i == 0 && j / BLOCK == 1) ? (uint8_t)rnd() + : (uint8_t)(i * 13 + j / 100); + total += sizes[i]; + if (zar_writer_file(w, path) != ZAR_OK) + ok = false; + /* in uneven pieces, a whole block among them */ + for (j = 0; j < sizes[i]; ) + { + size_t n = (j == 0 && sizes[i] > BLOCK) ? 1000 + : (j == 1000 ? 3 * BLOCK : 1 + rnd() % 5000); + if (n > sizes[i] - j) + n = sizes[i] - j; + if (zar_writer_append(w, data[i] + j, n) != ZAR_OK) + ok = false; + j += n; + } + } + check(ok, "files written"); + check( zar_writer_file(w, "CONTENT/A.BIN") == ZAR_ERROR_PARAM + && zar_writer_file(w, "content/a.bin/x") == ZAR_ERROR_PARAM + && zar_writer_dir(w, "meta/meta.xml") == ZAR_ERROR_PARAM + && zar_writer_file(w, "content") == ZAR_ERROR_PARAM + && zar_writer_file(w, "") == ZAR_ERROR_PARAM, + "what is there already, or under a file"); + check(zar_writer_finish(w) == ZAR_OK, "finished"); + zar_writer_free(w); + check(k.size < total, "compressed"); + + /* the footer's hash: the file's, with the hash zeroed */ + memcpy(saved, k.data + k.size - 48, 32); + memset(k.data + k.size - 48, 0, 32); + sha256_stream_init(&sha, 0); + sha256_stream_update(&sha, k.data, k.size); + sha256_stream_final(&sha, digest); + check(!memcmp(saved, digest, 32), "the footer's hash"); + + f.data = k.data; + f.size = k.size; + f.shorten = true; + f.requests = 0; + f.asked = 0; + check(open_zar(&f, &z) == ZAR_OK, "reads back"); + ok = true; + for (i = 0; i < W_FILES; i++) + { + char upper[256]; + unsigned outside = 0; + uint8_t *got = (uint8_t*)malloc(sizes[i] + 1); + const char *path = paths[i] ? paths[i] : longname; + for (j = 0; path[j]; j++) + upper[j] = (char)((path[j] >= 'a' && path[j] <= 'z') + ? path[j] - 32 : path[j]); + upper[j] = '\0'; + if ( lookup(&f, z, upper, &zf) != ZAR_OK || zf.dir + || zf.size != sizes[i] + || read_file(&f, z, &zf, 0, got, sizes[i] + 1, &outside) != ZAR_OK + || zar_read_progress(z) != sizes[i] + || memcmp(got, data[i], sizes[i])) + ok = false; + free(got); + free(data[i]); + } + check(ok, "every file found and read back"); + + /* each directory's entries in the order the search needs */ + check( lookup(&f, z, "content", &zf) == ZAR_OK && zf.dir + && zf.size == 8, "a directory written"); + { + zar_request_t req; + char prev[256]; + int err; + unsigned n = 0; + prev[0] = '\0'; + ok = zar_dir_begin(z, "content") == ZAR_OK; + for (;;) + { + while ((err = zar_dir_step(z, &req, &e)) == ZAR_PENDING) + zar_feed(z, pull(&f, &req)); + if (err != ZAR_OK) + break; + if (n++ && name_cmp(prev, e.name) >= 0) + ok = false; + strcpy(prev, e.name); + } + check(ok && err == ZAR_END && n == 8, "entries sorted"); + ok = zar_dir_begin(z, "content/sub/deep") == ZAR_OK; + while ((err = zar_dir_step(z, &req, &e)) == ZAR_PENDING) + zar_feed(z, pull(&f, &req)); + check(ok && err == ZAR_OK && !strcmp(e.name, "z") && e.file.size == BLOCK, + "a nested directory"); + } + zar_free(z); + free(k.data); + + /* a write that fails */ + memset(&k, 0, sizeof(k)); + k.fail_at = 100; + w = zar_writer_new(sink_write, &k, 3); + zar_writer_file(w, "x"); + zar_writer_append(w, "abc", 3); + check(zar_writer_finish(w) == ZAR_ERROR_WRITE, "a failed write"); + zar_writer_free(w); + free(k.data); + + /* nothing but an empty file */ + memset(&k, 0, sizeof(k)); + w = zar_writer_new(sink_write, &k, 3); + zar_writer_file(w, "empty"); + check(zar_writer_finish(w) == ZAR_OK, "an archive without blocks"); + zar_writer_free(w); + f.data = k.data; + f.size = k.size; + f.shorten = false; + check( open_zar(&f, &z) == ZAR_OK + && lookup(&f, z, "empty", &zf) == ZAR_OK && !zf.size, + "reads back without blocks"); + zar_free(z); + free(k.data); +} + int main(void) { lookup_lane(); list_lane(); read_lane(); broken_lanes(); + writer_lane(); free(stream); if (failures) diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index c3b67bbb8f..fa978c4535 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -2492,7 +2492,7 @@ int task_database_zar_get_serial(const char *name, char *s, size_t len, while ((err = zar_lookup_step(zs.z, &req, &zs.file)) == ZAR_PENDING) zar_feed(zs.z, task_database_pull(&zs.src, req.offset, req.dst, req.length)); - if (err != ZAR_OK || !(x = xdvdfs_new(zs.file.size))) + if (err != ZAR_OK || zs.file.dir || !(x = xdvdfs_new(zs.file.size))) goto end; exe.offset = 0; @@ -2704,7 +2704,8 @@ int task_database_wua_get_serial(const char *name, char *s, size_t len, while ((err = zar_lookup_step(z, &req, &zs.file)) == ZAR_PENDING) zar_feed(z, task_database_pull(&zs.src, req.offset, req.dst, req.length)); - if (err != ZAR_OK || !(xml = (char*)malloc(WIIU_META_HEAD + 1))) + if ( err != ZAR_OK || zs.file.dir + || !(xml = (char*)malloc(WIIU_META_HEAD + 1))) goto end; n = task_database_zar_pull(&zs, 0, (uint8_t*)xml, WIIU_META_HEAD); xml[n] = '\0';