libretro/RetroArch
1
0 Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2026-11-07 08:36:02 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity

formats/zar: lookups bisect, directories found and listed with places, a writer

Browse Source 
A lookup finds directories as well as files, the root at the empty
path, and searches each directory's sorted entries by bisection instead
of scanning them. A listing's entries carry their place: a file's
offset and size, a directory's first entry and count. zar_index_offset()
tells a caller where the index begins, so it can keep that part of the
file in memory and answer lookups and listings without I/O.

zar_write.c builds an archive front to back through a write callback,
compressing blocks with rzstd where HAVE_RZSTD is defined and finishing
with the footer's SHA-256, so an emulator can make .zar/.wua files
without the reference ZArchive library. Its output reads back with the
reference reader, and the reference writer's output with this one.
This commit is contained in:
libretroadmin committed 2026-11-07 15:39:37 +01:00
1 parent 4a0d8c8c92
commit 7e5c815578
6 files changed
+1090 -210
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

libretro-common/formats/zar/zar.c
+168 -160
View File
@@ -33,7 +33,8 @@
* for a file), then a file's place and size in the stream (48 bits
* each, their top 16 bits in the last word) or a directory's first
* child and count. A name is its length, in one octet or two, and its
* octets. */
* octets. A directory's entries are sorted by name, A-Z folded to a-z,
* octets compared unsigned and a prefix first. */
#include <stdlib.h>
#include <string.h>
@@ -67,10 +68,13 @@
#define ZAR_OP_DIR 4
/* the lookup */
#define ZAR_L_DIR 0 /* the directory's own node */
#define ZAR_L_BATCH 1 /* a run of its entries */
#define ZAR_L_NAME 2 /* one entry's name */
#define ZAR_L_LIST 3 /* a listed entry's name, in full */
#define ZAR_L_ROOT 0 /* the root's node */
#define ZAR_L_HAVE 1 /* a node in hand, the path's next part to find */
#define ZAR_L_PROBE 2 /* the middle entry of the range left */
#define ZAR_L_NAME 3 /* its name */
/* the listing */
#define ZAR_L_BATCH 4 /* a run of its entries */
#define ZAR_L_LIST 5 /* a listed entry's name, in full */
/* where the range asked for goes */
#define ZAR_TO_BUF 0
@@ -88,7 +92,6 @@ struct zar
uint64_t rec_group; /* the record held, ZAR_NONE for none */
uint64_t blk_index; /* the decoded block held, ZAR_NONE for none */
uint64_t rd_off; /* in the stream */
zar_file_t file; /* what the lookup found */
uint8_t *rd_dst;
uint8_t *packed; /* a compressed block, as stored */
uint8_t *block; /* a decoded block, for partial reads */
@@ -103,6 +106,8 @@ struct zar
uint32_t nodes;
uint32_t dir_first;
uint32_t dir_count;
uint32_t lo; /* the entries the name may be among */
uint32_t hi;
uint32_t dir_index; /* the first entry of the run in hand */
uint32_t batch; /* entries in the run */
uint32_t entry; /* the one whose name is looked at */
@@ -111,12 +116,12 @@ struct zar
int stage;
int to;
uint8_t buf[ZAR_BATCH * ZAR_NODE];
uint8_t node[ZAR_NODE]; /* the lookup's, in hand */
uint8_t name[ZAR_NAME_MAX + 3]; /* length, octets, terminator */
uint8_t rec[ZAR_RECORD];
bool fetching;
bool failed;
bool want_dir; /* a listing's directory is looked up */
bool listing; /* and found */
bool listing; /* a listing's directory is found */
};
static uint32_t zar_be32(const uint8_t *p)
@@ -148,8 +153,6 @@ zar_t *zar_new(uint64_t file_size)
z->rec_group = ZAR_NONE;
z->blk_index = ZAR_NONE;
z->rd_off = 0;
z->file.offset = 0;
z->file.size = 0;
z->rd_dst = NULL;
z->packed = NULL;
z->block = NULL;
@@ -164,6 +167,8 @@ zar_t *zar_new(uint64_t file_size)
z->nodes = 0;
z->dir_first = 0;
z->dir_count = 0;
z->lo = 0;
z->hi = 0;
z->dir_index = 0;
z->batch = 0;
z->entry = 0;
@@ -173,7 +178,6 @@ zar_t *zar_new(uint64_t file_size)
z->to = ZAR_TO_BUF;
z->fetching = false;
z->failed = false;
z->want_dir = false;
z->listing = false;
return z;
}
@@ -246,8 +250,7 @@ static int zar_open_run(zar_t *z, zar_request_t *req)
z->sec_size[i] = size;
}
}
if ( !z->sec_size[ZAR_SEC_RECORDS]
|| z->sec_size[ZAR_SEC_RECORDS] > 0xFFFFFFFFu
if ( z->sec_size[ZAR_SEC_RECORDS] > 0xFFFFFFFFu
|| z->sec_size[ZAR_SEC_RECORDS] % ZAR_RECORD
|| z->sec_size[ZAR_SEC_NAMES] > 0x7FFFFFFFu
|| !z->sec_size[ZAR_SEC_TREE]
@@ -281,13 +284,11 @@ int zar_lookup_begin(zar_t *z, const char *path)
if (z->op == ZAR_OP_OPEN)
return ZAR_ERROR_STATE;
z->op = ZAR_OP_LOOKUP;
z->stage = ZAR_L_DIR;
z->stage = ZAR_L_ROOT;
z->path = path;
z->dir_first = 0;
z->got = 0;
z->fetching = false;
z->failed = false;
z->want_dir = false;
z->listing = false;
return ZAR_OK;
}
@@ -296,153 +297,147 @@ int zar_dir_begin(zar_t *z, const char *path)
{
int e = zar_lookup_begin(z, path);
if (e == ZAR_OK)
{
z->op = ZAR_OP_DIR;
z->want_dir = true;
}
z->op = ZAR_OP_DIR;
return e;
}
static int zar_upper(int c)
static int zar_lower(int c)
{
return (c >= 'a' && c <= 'z') ? c - ('a' - 'A') : c;
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
}
/* The tree, from the root: each directory's entries looked at a run at
* a time; an entry's name read only as far as it can match */
/* A tree node as the caller sees it */
static int zar_node_file(const zar_t *z, const uint8_t *n, zar_file_t *out)
{
if (n[0] & 0x80)
{
out->offset = zar_be32(n + 4)
| ((uint64_t)(zar_be32(n + 12) & 0xFFFFu) << 32);
out->size = zar_be32(n + 8)
| ((uint64_t)(zar_be32(n + 12) & 0xFFFF0000u) << 16);
out->dir = 0;
if ( out->offset > z->blocks * ZAR_BLOCK
|| out->size > z->blocks * ZAR_BLOCK - out->offset)
return ZAR_ERROR_DATA;
}
else
{
out->offset = zar_be32(n + 4);
out->size = zar_be32(n + 8);
out->dir = 1;
if ( out->offset > z->nodes
|| out->size > z->nodes - out->offset)
return ZAR_ERROR_DATA;
}
return ZAR_OK;
}
/* Asks for the name @n points at, at most @want octets of it past its
* length; *@hdr and *@nlen say where it starts and how long it is */
static int zar_name(zar_t *z, zar_request_t *req, const uint8_t *n,
size_t want, size_t *cap, size_t *hdr, size_t *nlen)
{
uint32_t name_at = zar_be32(n) & 0x7FFFFFFFu;
if ( name_at == ZAR_NO_NAME
|| name_at >= z->sec_size[ZAR_SEC_NAMES])
return ZAR_ERROR_DATA;
*cap = 2 + want;
if (*cap > z->sec_size[ZAR_SEC_NAMES] - name_at)
*cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at);
if (z->got < *cap)
return zar_ask(z, req, ZAR_TO_NAME,
z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, *cap);
z->got = 0;
*nlen = z->name[0] & 0x7F;
*hdr = 1;
if (z->name[0] & 0x80)
{
if (*cap < 2)
return ZAR_ERROR_DATA;
*nlen |= (size_t)z->name[1] << 7;
*hdr = 2;
}
return ZAR_OK;
}
/* Down the tree from the root, a binary search of each directory's
* sorted entries for the path's next part */
static int zar_lookup_run(zar_t *z, zar_request_t *req, zar_file_t *out)
{
for (;;)
{
const char *p;
const uint8_t *n;
size_t plen;
size_t cap;
size_t hdr;
size_t nlen;
size_t i;
uint32_t name_at;
const char *p;
size_t plen;
size_t cap;
size_t hdr;
size_t nlen;
size_t i;
int c;
int e;
while (*z->path == '/' || *z->path == '\\')
z->path++;
p = z->path;
for (plen = 0; p[plen] && p[plen] != '/' && p[plen] != '\\'; plen++) { }
/* a listing of the root looks up no name */
if ( (!plen && !(z->want_dir && z->stage == ZAR_L_DIR))
|| plen > ZAR_NAME_MAX)
return ZAR_ERROR_NOT_FOUND;
switch (z->stage)
{
case ZAR_L_DIR:
/* the directory node z->dir_first names */
if (z->dir_first >= z->nodes)
return ZAR_ERROR_DATA;
case ZAR_L_ROOT:
if (z->got < ZAR_NODE)
return zar_ask(z, req, ZAR_TO_BUF,
z->sec_off[ZAR_SEC_TREE], z->node, ZAR_NODE);
z->got = 0;
z->stage = ZAR_L_HAVE;
break;
case ZAR_L_HAVE:
if ((e = zar_node_file(z, z->node, out)) != ZAR_OK)
return e;
if (!plen)
return ZAR_OK;
if (!out->dir || plen > ZAR_NAME_MAX)
return ZAR_ERROR_NOT_FOUND;
z->dir_first = (uint32_t)out->offset;
z->lo = 0;
z->hi = (uint32_t)out->size;
z->stage = ZAR_L_PROBE;
break;
case ZAR_L_PROBE:
if (z->lo >= z->hi)
return ZAR_ERROR_NOT_FOUND;
z->entry = z->dir_first + z->lo + (z->hi - z->lo) / 2;
if (z->got < ZAR_NODE)
return zar_ask(z, req, ZAR_TO_BUF,
z->sec_off[ZAR_SEC_TREE]
+ (uint64_t)z->dir_first * ZAR_NODE, z->buf, ZAR_NODE);
z->got = 0;
if (z->buf[0] & 0x80)
return ZAR_ERROR_NOT_FOUND;
z->dir_first = zar_be32(z->buf + 4);
z->dir_count = zar_be32(z->buf + 8);
if ( z->dir_first > z->nodes
|| z->dir_count > z->nodes - z->dir_first)
return ZAR_ERROR_DATA;
z->dir_index = 0;
z->batch = 0;
z->stage = ZAR_L_BATCH;
if (!plen)
return ZAR_OK;
break;
case ZAR_L_BATCH:
if (z->dir_index >= z->dir_count)
return ZAR_ERROR_NOT_FOUND;
z->batch = z->dir_count - z->dir_index;
if (z->batch > ZAR_BATCH)
z->batch = ZAR_BATCH;
if (z->got < (size_t)z->batch * ZAR_NODE)
return zar_ask(z, req, ZAR_TO_BUF,
z->sec_off[ZAR_SEC_TREE]
+ ((uint64_t)z->dir_first + z->dir_index) * ZAR_NODE,
z->buf, (size_t)z->batch * ZAR_NODE);
+ (uint64_t)z->entry * ZAR_NODE, z->buf, ZAR_NODE);
z->got = 0;
z->entry = 0;
z->stage = ZAR_L_NAME;
break;
case ZAR_L_NAME:
if (z->entry >= z->batch)
{
z->dir_index += z->batch;
z->stage = ZAR_L_BATCH;
break;
}
n = z->buf + z->entry * ZAR_NODE;
name_at = zar_be32(n) & 0x7FFFFFFFu;
if ( name_at == ZAR_NO_NAME
|| name_at >= z->sec_size[ZAR_SEC_NAMES])
if ((e = zar_name(z, req, z->buf, plen,
&cap, &hdr, &nlen)) != ZAR_OK)
return e;
if (hdr + (nlen < plen ? nlen : plen) > cap)
return ZAR_ERROR_DATA;
cap = 2 + plen;
if (cap > z->sec_size[ZAR_SEC_NAMES] - name_at)
cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at);
if (z->got < cap)
return zar_ask(z, req, ZAR_TO_NAME,
z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, cap);
z->got = 0;
nlen = z->name[0] & 0x7F;
hdr = 1;
if (z->name[0] & 0x80)
c = 0;
for (i = 0; i < plen && i < nlen && !c; i++)
c = zar_lower((unsigned char)p[i])
- zar_lower(z->name[hdr + i]);
if (!c)
c = (plen > nlen) - (plen < nlen);
if (c)
{
if (cap < 2)
return ZAR_ERROR_DATA;
nlen |= (size_t)z->name[1] << 7;
hdr = 2;
}
if (nlen != plen || hdr + nlen > cap)
{
z->entry++;
break;
}
for (i = 0; i < plen; i++)
if ( zar_upper((unsigned char)p[i])
!= zar_upper(z->name[hdr + i]))
break;
if (i < plen)
{
z->entry++;
uint32_t mid = z->entry - z->dir_first;
if (c < 0)
z->hi = mid;
else
z->lo = mid + 1;
z->stage = ZAR_L_PROBE;
break;
}
memcpy(z->node, z->buf, ZAR_NODE);
z->path += plen;
while (*z->path == '/' || *z->path == '\\')
z->path++;
if (*z->path || z->want_dir)
{
/* down into this entry, which must be a directory */
if (n[0] & 0x80)
return ZAR_ERROR_NOT_FOUND;
z->dir_first = zar_be32(n + 4);
z->dir_count = zar_be32(n + 8);
if ( z->dir_first > z->nodes
|| z->dir_count > z->nodes - z->dir_first)
return ZAR_ERROR_DATA;
z->dir_index = 0;
z->batch = 0;
z->stage = ZAR_L_BATCH;
if (!*z->path)
return ZAR_OK;
break;
}
if (!(n[0] & 0x80))
return ZAR_ERROR_NOT_FOUND;
out->offset = zar_be32(n + 4)
| ((uint64_t)(zar_be32(n + 12) & 0xFFFFu) << 32);
out->size = zar_be32(n + 8)
| ((uint64_t)(zar_be32(n + 12) & 0xFFFF0000u) << 16);
if ( out->offset > z->blocks * ZAR_BLOCK
|| out->size > z->blocks * ZAR_BLOCK - out->offset)
return ZAR_ERROR_DATA;
return ZAR_OK;
z->stage = ZAR_L_HAVE;
break;
default:
return ZAR_ERROR_STATE;
}
@@ -472,7 +467,7 @@ static int zar_list_run(zar_t *z, zar_request_t *req, zar_dirent_t *ent)
size_t cap;
size_t hdr;
size_t nlen;
uint32_t name_at;
int e;
switch (z->stage)
{
@@ -498,33 +493,18 @@ static int zar_list_run(zar_t *z, zar_request_t *req, zar_dirent_t *ent)
z->stage = ZAR_L_BATCH;
break;
}
n = z->buf + z->entry * ZAR_NODE;
name_at = zar_be32(n) & 0x7FFFFFFFu;
if ( name_at == ZAR_NO_NAME
|| name_at >= z->sec_size[ZAR_SEC_NAMES])
return ZAR_ERROR_DATA;
cap = 2 + ZAR_NAME_MAX;
if (cap > z->sec_size[ZAR_SEC_NAMES] - name_at)
cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at);
if (z->got < cap)
return zar_ask(z, req, ZAR_TO_NAME,
z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, cap);
z->got = 0;
nlen = z->name[0] & 0x7F;
hdr = 1;
if (z->name[0] & 0x80)
{
if (cap < 2)
return ZAR_ERROR_DATA;
nlen |= (size_t)z->name[1] << 7;
hdr = 2;
}
n = z->buf + z->entry * ZAR_NODE;
if ((e = zar_name(z, req, n, ZAR_NAME_MAX,
&cap, &hdr, &nlen)) != ZAR_OK)
return e;
if (nlen > ZAR_NAME_MAX || hdr + nlen > cap)
return ZAR_ERROR_DATA;
if ((e = zar_node_file(z, n, &ent->file)) != ZAR_OK)
return e;
z->name[hdr + nlen] = '\0';
ent->name = (const char*)z->name + hdr;
ent->len = nlen;
ent->is_dir = !(n[0] & 0x80);
ent->is_dir = (int)ent->file.dir;
z->entry++;
return ZAR_OK;
default:
@@ -544,20 +524,48 @@ int zar_dir_step(zar_t *z, zar_request_t *req, zar_dirent_t *ent)
e = ZAR_ERROR_DATA;
else if (z->listing)
e = zar_list_run(z, req, ent);
else if ((e = zar_lookup_run(z, req, NULL)) == ZAR_OK)
else
{
z->listing = true;
e = zar_list_run(z, req, ent);
zar_file_t d;
if ((e = zar_lookup_run(z, req, &d)) == ZAR_OK)
{
if (!d.dir)
e = ZAR_ERROR_NOT_FOUND;
else
{
z->dir_first = (uint32_t)d.offset;
z->dir_count = (uint32_t)d.size;
z->dir_index = 0;
z->batch = 0;
z->got = 0;
z->stage = ZAR_L_BATCH;
z->listing = true;
e = zar_list_run(z, req, ent);
}
}
}
if (e != ZAR_PENDING && e != ZAR_OK)
z->op = ZAR_OP_NONE;
return e;
}
uint64_t zar_index_offset(const zar_t *z)
{
uint64_t off;
if (!z || !z->nodes)
return 0;
off = z->sec_off[ZAR_SEC_RECORDS];
if (z->sec_off[ZAR_SEC_NAMES] < off)
off = z->sec_off[ZAR_SEC_NAMES];
if (z->sec_off[ZAR_SEC_TREE] < off)
off = z->sec_off[ZAR_SEC_TREE];
return off;
}
int zar_read_begin(zar_t *z, const zar_file_t *file, uint64_t off,
void *dst, size_t len)
{
if (!z || !file || (!dst && len))
if (!z || !file || file->dir || (!dst && len))
return ZAR_ERROR_PARAM;
if (z->op == ZAR_OP_OPEN)
return ZAR_ERROR_STATE;
libretro-common/formats/zar/zar_write.c
+550
View File
@@ -0,0 +1,550 @@
/* Copyright (C) 2010-2026 The RetroArch team
*
* ---------------------------------------------------------------------------------------
* The following license statement only applies to this file (zar_write.c).
* ---------------------------------------------------------------------------------------
*
* Permission is hereby granted, free of charge,
* to any person obtaining a copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
* and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
/* The archive zar.c reads, written front to back: the blocks as the
* files' octets fill them, then the records, the names, the tree with
* each directory's entries sorted, and the footer, whose hash is
* SHA-256 over the whole file with the hash itself zeroed. */
#include <stdlib.h>
#include <string.h>
#include <boolean.h>
#include <lrc_hash.h>
#include <formats/zar.h>
#ifdef HAVE_RZSTD
#include <encodings/rzstd.h>
#endif
#define ZAR_W_BLOCK 65536u
#define ZAR_W_RECORD 40
#define ZAR_W_NODE 16
#define ZAR_W_FOOTER 144
#define ZAR_W_NONE 0xFFFFFFFFu
#define ZAR_W_NAME 0x7FFFu
#define ZAR_W_MAX ((((uint64_t)1) << 48) - 1) /* a file's offset or size */
typedef struct zar_wnode
{
uint64_t offset; /* a file's, in the stream */
uint64_t size;
uint32_t name; /* in the names, ZAR_W_NONE for the root */
uint32_t child; /* a directory's last entry added */
uint32_t sibling; /* the entry added before this one */
uint32_t first; /* a directory's first entry's node, written */
uint32_t count;
uint8_t file;
} zar_wnode_t;
struct zar_writer
{
struct sha256_state sha;
zar_write_fn write;
void *ud;
uint8_t *block;
uint8_t *enc;
uint8_t *names;
uint8_t *recs;
zar_wnode_t *nodes;
uint64_t out; /* octets written */
uint64_t in; /* octets of the stream */
size_t fill; /* in the block */
size_t enc_cap;
size_t names_len;
size_t names_cap;
size_t recs_len;
size_t recs_cap;
uint64_t blocks;
uint32_t nnodes;
uint32_t nodes_cap;
uint32_t current; /* the file appended to */
int level;
bool failed;
};
static void zar_w_be32(uint8_t *p, uint32_t v)
{
p[0] = (uint8_t)(v >> 24);
p[1] = (uint8_t)(v >> 16);
p[2] = (uint8_t)(v >> 8);
p[3] = (uint8_t)v;
}
static void zar_w_be64(uint8_t *p, uint64_t v)
{
zar_w_be32(p, (uint32_t)(v >> 32));
zar_w_be32(p + 4, (uint32_t)v);
}
static bool zar_w_grow(void **p, size_t *cap, size_t need, size_t size)
{
size_t n;
void *q;
if (need <= *cap)
return true;
n = *cap ? *cap : 64;
while (n < need)
n *= 2;
if (!(q = realloc(*p, n * size)))
return false;
*p = q;
*cap = n;
return true;
}
static int zar_w_out(zar_writer_t *w, const void *data, size_t len)
{
if (w->failed)
return ZAR_ERROR_WRITE;
if (len && !w->write(w->ud, data, len))
{
w->failed = true;
return ZAR_ERROR_WRITE;
}
sha256_stream_update(&w->sha, (const uint8_t*)data, len);
w->out += len;
return ZAR_OK;
}
static uint32_t zar_w_node(zar_writer_t *w, uint32_t parent,
uint32_t name, bool file)
{
zar_wnode_t *n;
size_t cap = w->nodes_cap;
if (!zar_w_grow((void**)&w->nodes, &cap, (size_t)w->nnodes + 1,
sizeof(zar_wnode_t)))
return ZAR_W_NONE;
w->nodes_cap = (uint32_t)cap;
n = &w->nodes[w->nnodes];
n->offset = w->in;
n->size = 0;
n->name = name;
n->child = ZAR_W_NONE;
n->sibling = ZAR_W_NONE;
n->first = 0;
n->count = 0;
n->file = file ? 1 : 0;
if (parent != ZAR_W_NONE)
{
n->sibling = w->nodes[parent].child;
w->nodes[parent].child = w->nnodes;
w->nodes[parent].count++;
}
return w->nnodes++;
}
zar_writer_t *zar_writer_new(zar_write_fn write, void *ud, int level)
{
zar_writer_t *w;
if (!write || !(w = (zar_writer_t*)calloc(1, sizeof(*w))))
return NULL;
w->write = write;
w->ud = ud;
w->level = level;
w->current = ZAR_W_NONE;
sha256_stream_init(&w->sha, 0);
if ( !(w->block = (uint8_t*)malloc(ZAR_W_BLOCK))
|| zar_w_node(w, ZAR_W_NONE, ZAR_W_NONE, false) != 0)
{
zar_writer_free(w);
return NULL;
}
return w;
}
void zar_writer_free(zar_writer_t *w)
{
if (!w)
return;
free(w->block);
free(w->enc);
free(w->names);
free(w->recs);
free(w->nodes);
free(w);
}
static int zar_w_lower(int c)
{
return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c;
}
static const uint8_t *zar_w_name(const zar_writer_t *w, uint32_t at,
size_t *len)
{
const uint8_t *p = w->names + at;
if (p[0] & 0x80)
{
*len = (p[0] & 0x7F) | ((size_t)p[1] << 7);
return p + 2;
}
*len = p[0];
return p + 1;
}
/* the order zar.c's search expects */
static int zar_w_cmp(const uint8_t *a, size_t alen,
const uint8_t *b, size_t blen)
{
size_t i;
for (i = 0; i < alen && i < blen; i++)
{
int c = zar_w_lower(a[i]) - zar_w_lower(b[i]);
if (c)
return c;
}
return (alen > blen) - (alen < blen);
}
static uint32_t zar_w_find(const zar_writer_t *w, uint32_t dir,
const char *name, size_t len)
{
uint32_t c;
for (c = w->nodes[dir].child; c != ZAR_W_NONE; c = w->nodes[c].sibling)
{
size_t nlen;
const uint8_t *n = zar_w_name(w, w->nodes[c].name, &nlen);
if (nlen == len && !zar_w_cmp(n, nlen, (const uint8_t*)name, len))
return c;
}
return ZAR_W_NONE;
}
static uint32_t zar_w_add_name(zar_writer_t *w, const char *name,
size_t len)
{
uint32_t at = (uint32_t)w->names_len;
if ( w->names_len + len + 2 > 0x7FFFFFFFu
|| !zar_w_grow((void**)&w->names, &w->names_cap,
w->names_len + len + 2, 1))
return ZAR_W_NONE;
if (len >= 0x80)
{
w->names[w->names_len++] = (uint8_t)((len & 0x7F) | 0x80);
w->names[w->names_len++] = (uint8_t)(len >> 7);
}
else
w->names[w->names_len++] = (uint8_t)len;
memcpy(w->names + w->names_len, name, len);
w->names_len += len;
return at;
}
/* Walks @path from the root, adding the directories it lacks; the last
* part is added as a file when @file */
static int zar_w_path(zar_writer_t *w, const char *path, bool file,
uint32_t *out)
{
uint32_t at = 0;
for (;;)
{
size_t len;
uint32_t c;
bool last;
while (*path == '/' || *path == '\\')
path++;
for (len = 0; path[len] && path[len] != '/' && path[len] != '\\'; len++) { }
if (!len)
break;
if (len > ZAR_W_NAME)
return ZAR_ERROR_PARAM;
last = !path[len];
if (!last)
{
const char *q = path + len;
while (*q == '/' || *q == '\\')
q++;
last = !*q;
}
c = zar_w_find(w, at, path, len);
if (c != ZAR_W_NONE)
{
if (w->nodes[c].file || (last && file))
return ZAR_ERROR_PARAM;
}
else
{
uint32_t name = zar_w_add_name(w, path, len);
if ( name == ZAR_W_NONE
|| (c = zar_w_node(w, at, name, last && file)) == ZAR_W_NONE)
return ZAR_ERROR_MEM;
}
at = c;
path += len;
}
if (file && !at)
return ZAR_ERROR_PARAM;
*out = at;
return ZAR_OK;
}
int zar_writer_dir(zar_writer_t *w, const char *path)
{
uint32_t at;
if (!w || !path)
return ZAR_ERROR_PARAM;
return zar_w_path(w, path, false, &at);
}
int zar_writer_file(zar_writer_t *w, const char *path)
{
uint32_t at;
int e;
if (!w || !path)
return ZAR_ERROR_PARAM;
w->current = ZAR_W_NONE;
if ((e = zar_w_path(w, path, true, &at)) == ZAR_OK)
w->current = at;
return e;
}
static int zar_w_block(zar_writer_t *w, const uint8_t *src)
{
const uint8_t *data = src;
size_t len = ZAR_W_BLOCK;
size_t r;
int e;
#ifdef HAVE_RZSTD
{
size_t wrote = 0;
if (!w->enc)
{
w->enc_cap = rzstd_compress_bound(ZAR_W_BLOCK);
if (!(w->enc = (uint8_t*)malloc(w->enc_cap)))
return ZAR_ERROR_MEM;
}
if ( rzstd_encode(w->enc, w->enc_cap, src, ZAR_W_BLOCK,
w->level, &wrote) == RZSTD_PROCESS_END
&& wrote && wrote < ZAR_W_BLOCK)
{
data = w->enc;
len = wrote;
}
}
#endif
r = (size_t)(w->blocks % 16);
if (!r)
{
if (!zar_w_grow((void**)&w->recs, &w->recs_cap,
w->recs_len + ZAR_W_RECORD, 1))
return ZAR_ERROR_MEM;
memset(w->recs + w->recs_len, 0, ZAR_W_RECORD);
zar_w_be64(w->recs + w->recs_len, w->out);
w->recs_len += ZAR_W_RECORD;
}
w->recs[w->recs_len - ZAR_W_RECORD + 8 + r * 2] = (uint8_t)((len - 1) >> 8);
w->recs[w->recs_len - ZAR_W_RECORD + 8 + r * 2 + 1] = (uint8_t)(len - 1);
w->blocks++;
if ((e = zar_w_out(w, data, len)) != ZAR_OK)
return e;
return ZAR_OK;
}
static int zar_w_append(zar_writer_t *w, const uint8_t *src, size_t len)
{
int e;
w->in += len;
while (len)
{
size_t take;
/* a whole block straight from the caller */
if (!w->fill && len >= ZAR_W_BLOCK)
{
if ((e = zar_w_block(w, src)) != ZAR_OK)
return e;
src += ZAR_W_BLOCK;
len -= ZAR_W_BLOCK;
continue;
}
take = ZAR_W_BLOCK - w->fill;
if (take > len)
take = len;
memcpy(w->block + w->fill, src, take);
w->fill += take;
src += take;
len -= take;
if (w->fill == ZAR_W_BLOCK)
{
w->fill = 0;
if ((e = zar_w_block(w, w->block)) != ZAR_OK)
return e;
}
}
return ZAR_OK;
}
int zar_writer_append(zar_writer_t *w, const void *data, size_t len)
{
if (!w || (!data && len) || w->current == ZAR_W_NONE)
return ZAR_ERROR_PARAM;
if (w->failed)
return ZAR_ERROR_WRITE;
if (len > ZAR_W_MAX - w->in)
return ZAR_ERROR_PARAM;
w->nodes[w->current].size += len;
return zar_w_append(w, (const uint8_t*)data, len);
}
/* Sorts @ids by name, @tmp as long */
static void zar_w_sort(const zar_writer_t *w, uint32_t *ids,
uint32_t *tmp, uint32_t n)
{
uint32_t width;
for (width = 1; width < n; width *= 2)
{
uint32_t lo;
for (lo = 0; lo < n; lo += 2 * width)
{
uint32_t mid = lo + width < n ? lo + width : n;
uint32_t hi = lo + 2 * width < n ? lo + 2 * width : n;
uint32_t i = lo;
uint32_t j = mid;
uint32_t k = lo;
while (i < mid && j < hi)
{
size_t alen, blen;
const uint8_t *a = zar_w_name(w, w->nodes[ids[i]].name, &alen);
const uint8_t *b = zar_w_name(w, w->nodes[ids[j]].name, &blen);
tmp[k++] = zar_w_cmp(b, blen, a, alen) < 0 ? ids[j++] : ids[i++];
}
while (i < mid)
tmp[k++] = ids[i++];
while (j < hi)
tmp[k++] = ids[j++];
}
memcpy(ids, tmp, n * sizeof(*ids));
}
}
int zar_writer_finish(zar_writer_t *w)
{
uint8_t ft[ZAR_W_FOOTER];
uint8_t digest[32];
uint64_t sec[6][2];
uint32_t *order;
uint32_t *tmp;
uint32_t k;
uint32_t len;
int e = ZAR_OK;
if (!w)
return ZAR_ERROR_PARAM;
w->current = ZAR_W_NONE;
if (w->fill)
{
memset(w->block + w->fill, 0, ZAR_W_BLOCK - w->fill);
w->fill = 0;
if ((e = zar_w_block(w, w->block)) != ZAR_OK)
return e;
}
sec[0][0] = 0;
sec[0][1] = w->out;
memset(ft, 0, sizeof(ft));
if ((e = zar_w_out(w, ft, (size_t)((8 - w->out % 8) % 8))) != ZAR_OK)
return e;
sec[1][0] = w->out;
sec[1][1] = w->recs_len;
if ((e = zar_w_out(w, w->recs, w->recs_len)) != ZAR_OK)
return e;
sec[2][0] = w->out;
sec[2][1] = w->names_len;
if ((e = zar_w_out(w, w->names, w->names_len)) != ZAR_OK)
return e;
/* breadth first: each directory's entries together and sorted */
order = (uint32_t*)malloc(w->nnodes * sizeof(uint32_t));
tmp = (uint32_t*)malloc(w->nnodes * sizeof(uint32_t));
if (!order || !tmp)
{
free(order);
free(tmp);
return ZAR_ERROR_MEM;
}
order[0] = 0;
len = 1;
for (k = 0; k < len; k++)
{
zar_wnode_t *d = &w->nodes[order[k]];
uint32_t c;
uint32_t at;
if (d->file)
continue;
d->first = len;
at = len + d->count;
for (c = d->child; c != ZAR_W_NONE; c = w->nodes[c].sibling)
order[--at] = c;
zar_w_sort(w, order + len, tmp, d->count);
len += d->count;
}
sec[3][0] = w->out;
sec[3][1] = (uint64_t)w->nnodes * ZAR_W_NODE;
for (k = 0; k < w->nnodes && e == ZAR_OK; k++)
{
uint8_t n[ZAR_W_NODE];
const zar_wnode_t *d = &w->nodes[order[k]];
zar_w_be32(n, (d->name == ZAR_W_NONE ? 0x7FFFFFFFu : d->name)
| (d->file ? 0x80000000u : 0));
if (d->file)
{
zar_w_be32(n + 4, (uint32_t)d->offset);
zar_w_be32(n + 8, (uint32_t)d->size);
zar_w_be32(n + 12, (uint32_t)((d->offset >> 32) & 0xFFFFu)
| (uint32_t)((d->size >> 16) & 0xFFFF0000u));
}
else
{
zar_w_be32(n + 4, d->first);
zar_w_be32(n + 8, d->count);
zar_w_be32(n + 12, 0);
}
e = zar_w_out(w, n, sizeof(n));
}
free(order);
free(tmp);
if (e != ZAR_OK)
return e;
sec[4][0] = sec[5][0] = w->out;
sec[4][1] = sec[5][1] = 0;
for (k = 0; k < 6; k++)
{
zar_w_be64(ft + k * 16, sec[k][0]);
zar_w_be64(ft + k * 16 + 8, sec[k][1]);
}
zar_w_be64(ft + 128, w->out + ZAR_W_FOOTER);
zar_w_be32(ft + 136, 0x61BF3A01u);
zar_w_be32(ft + 140, 0x169F52D6u);
sha256_stream_update(&w->sha, ft, sizeof(ft));
sha256_stream_final(&w->sha, digest);
memcpy(ft + 96, digest, sizeof(digest));
if (w->failed || !w->write(w->ud, ft, sizeof(ft)))
{
w->failed = true;
return ZAR_ERROR_WRITE;
}
w->out += sizeof(ft);
return ZAR_OK;
}
libretro-common/include/formats/zar.h
+68 -9
View File
@@ -47,10 +47,11 @@ RETRO_BEGIN_DECLS
* while ((err = zar_open_step(z, &req)) == ZAR_PENDING)
* zar_feed(z, pull(req.offset, req.dst, req.length));
*
* Nothing is read that an operation does not look at: a directory's
* entries a run at a time, a name only when its length matches, a
* block's place from its one record. A read of a block kept as it is
* asks for its octets straight into the read's destination; a
* Nothing is read that an operation does not look at: a lookup reads
* the entries and names its binary search of each directory probes, a
* listing a directory's entries a run at a time, a block's place comes
* from its one record. A read of a block kept as it is asks for its
* octets straight into the read's destination; a
* compressed block is decoded into the destination when the read
* wants all of it, and otherwise into a block kept for the reads that
* follow. */
@@ -64,6 +65,7 @@ RETRO_BEGIN_DECLS
#define ZAR_ERROR_STATE (-4)
#define ZAR_ERROR_NOT_FOUND (-5)
#define ZAR_ERROR_UNSUPPORTED (-6)
#define ZAR_ERROR_WRITE (-7)
typedef struct zar zar_t;
@@ -80,11 +82,13 @@ typedef struct zar_request
uint8_t *dst; /* where they go */
} zar_request_t;
/* A file in the archive: its octets in the stream */
/* A file in the archive, or a directory: a file's octets in the
* stream, a directory's first entry's node and its entry count */
typedef struct zar_file
{
uint64_t offset;
uint64_t size;
unsigned dir;
} zar_file_t;
/* @file_size: the archive's size, which its footer must agree with */
@@ -115,7 +119,8 @@ int zar_feed(zar_t *z, size_t len);
/**
* zar_lookup_begin:
* @path : '/' or '\' separated, matched without regard to ASCII case;
* it must stay valid until the lookup is over
* it must stay valid until the lookup is over. A file or a
* directory; an empty path is the root.
*
* Returns: ZAR_OK, or ZAR_ERROR_STATE before the archive is open.
**/
@@ -137,6 +142,7 @@ typedef struct zar_dirent
const char *name;
size_t len;
int is_dir;
zar_file_t file;
} zar_dirent_t;
/**
@@ -153,20 +159,26 @@ int zar_dir_begin(zar_t *z, const char *path);
/**
* zar_dir_step:
*
* Returns: ZAR_OK with @ent filled in for each entry in the archive's
* order, ZAR_END past the last one, ZAR_PENDING with @req filled in
* Returns: ZAR_OK with @ent filled in for each entry in name order,
* ZAR_END past the last one, ZAR_PENDING with @req filled in
* when it needs a range of the file, ZAR_ERROR_NOT_FOUND when @path is
* no directory, or another negative code.
**/
int zar_dir_step(zar_t *z, zar_request_t *req, zar_dirent_t *ent);
/* Where the octets past the stream's blocks start, from which a caller
* can keep the rest of the file in memory and answer every request a
* lookup or listing makes without I/O; 0 before the archive is open */
uint64_t zar_index_offset(const zar_t *z);
/**
* zar_read_begin:
*
* Arms a read of @len octets of @file at @off into @dst, short at the
* end of the file, abandoning a read or lookup in progress.
*
* Returns: ZAR_OK, or ZAR_ERROR_STATE before the archive is open.
* Returns: ZAR_OK, ZAR_ERROR_PARAM for a directory, or ZAR_ERROR_STATE
* before the archive is open.
**/
int zar_read_begin(zar_t *z, const zar_file_t *file, uint64_t off,
void *dst, size_t len);
@@ -183,6 +195,53 @@ int zar_read_step(zar_t *z, zar_request_t *req);
/* octets the armed read has put at its @dst so far */
size_t zar_read_progress(const zar_t *z);
/* The writer, in zar_write.c: an archive built front to back, every
* octet handed to @write as it is made. Blocks are compressed with
* Zstandard at @level (1 to 9) where that shrinks them and HAVE_RZSTD
* is defined, and kept as they are otherwise. Paths are matched without
* regard to ASCII case, as the reader matches them. */
typedef struct zar_writer zar_writer_t;
/* Returns nonzero when all @len octets were written */
typedef int (*zar_write_fn)(void *ud, const void *data, size_t len);
zar_writer_t *zar_writer_new(zar_write_fn write, void *ud, int level);
void zar_writer_free(zar_writer_t *w);
/**
* zar_writer_dir:
*
* Adds directory @path and those above it that are missing.
*
* Returns: ZAR_OK, or ZAR_ERROR_PARAM when a part of it is a file.
**/
int zar_writer_dir(zar_writer_t *w, const char *path);
/**
* zar_writer_file:
*
* Adds file @path, the directories above it that are missing, and
* makes it the one zar_writer_append() adds to.
*
* Returns: ZAR_OK, or ZAR_ERROR_PARAM when it is there already or a
* part above it is a file.
**/
int zar_writer_file(zar_writer_t *w, const char *path);
int zar_writer_append(zar_writer_t *w, const void *data, size_t len);
/**
* zar_writer_finish:
*
* Writes the rest of the archive.
*
* Returns: ZAR_OK, ZAR_ERROR_WRITE when @write failed, or
* ZAR_ERROR_MEM.
**/
int zar_writer_finish(zar_writer_t *w);
RETRO_END_DECLS
#endif
libretro-common/samples/formats/zar/Makefile
+14
View File
@@ -8,6 +8,20 @@ LIBRETRO_COMM_DIR := ../../..
SOURCES := \
zar_test.c \
$(LIBRETRO_COMM_DIR)/formats/zar/zar.c \
$(LIBRETRO_COMM_DIR)/formats/zar/zar_write.c \
$(LIBRETRO_COMM_DIR)/hash/lrc_hash.c \
$(LIBRETRO_COMM_DIR)/features/features_cpu.c \
$(LIBRETRO_COMM_DIR)/streams/file_stream.c \
$(LIBRETRO_COMM_DIR)/vfs/vfs_implementation.c \
$(LIBRETRO_COMM_DIR)/file/file_path.c \
$(LIBRETRO_COMM_DIR)/file/file_path_io.c \
$(LIBRETRO_COMM_DIR)/string/stdstring.c \
$(LIBRETRO_COMM_DIR)/string/rstrtod.c \
$(LIBRETRO_COMM_DIR)/encodings/encoding_utf.c \
$(LIBRETRO_COMM_DIR)/compat/compat_strl.c \
$(LIBRETRO_COMM_DIR)/compat/compat_posix_string.c \
$(LIBRETRO_COMM_DIR)/compat/fopen_utf8.c \
$(LIBRETRO_COMM_DIR)/time/rtime.c \
$(LIBRETRO_COMM_DIR)/encodings/encoding_rzstd.c \
$(LIBRETRO_COMM_DIR)/encodings/encoding_xxh64.c
libretro-common/samples/formats/zar/zar_test.c
+287 -39
View File
@@ -4,15 +4,16 @@
*
* The archives hold blocks compressed with Zstandard beside blocks
* kept as they are (data that does not compress), a file crossing
* them, a directory with more entries than the reader asks for at once
* them, a directory with more entries than a listing asks for at once
* and a nested one. Ranges are supplied short now and then. Every
* request is checked: a lookup asks for a directory's entries a run at
* a time and for each name only as far as it can match, exactly that
* many octets; a read of a block kept as it is lands in the read's own
* destination; a compressed block is fetched once for the partial
* reads that follow it. A directory is listed in the archive's order,
* each name read whole. A footer that is not one, a size that
* disagrees and a block that does not decode must fail. */
* request is checked: a lookup bisects each directory's sorted entries
* and reads each probed name only as far as it can match; a read of a
* block kept as it is lands in the read's own destination; a
* compressed block is fetched once for the partial reads that follow
* it. A directory is listed in name order, each name read whole, each
* entry's place the one a lookup gives. The writer's archive reads
* back, sorted, with its footer's hash. A footer that is not one, a
* size that disagrees and a block that does not decode must fail. */
#include <stdio.h>
#include <stdlib.h>
@@ -21,6 +22,7 @@
#include <boolean.h>
#include <formats/zar.h>
#include <encodings/rzstd.h>
#include <lrc_hash.h>
#define BLOCK 65536
#define FILLERS 38
@@ -85,8 +87,8 @@ static size_t pull(file_t *f, const zar_request_t *req)
return n;
}
/* the tree: the root holds FILLERS files, media/ and default.xex last;
* media/ holds a.bin and intro.wmv */
/* the tree, each directory in name order: the root holds default.xex,
* FILLERS files and media/, which holds a.bin and intro.wmv */
#define N_NODES (1 + FILLERS + 2 + 2)
typedef struct
@@ -101,6 +103,7 @@ typedef struct
static uint8_t *stream; /* the files, laid end to end */
static size_t stream_size;
static size_t names_size;
/* The archive of the tree, with @bad_block's compressed octets broken
* when it is not -1 */
@@ -126,15 +129,15 @@ static void make_archive(file_t *f, int bad_block)
nodes[0].count = FILLERS + 2;
for (i = 0; i < FILLERS; i++)
{
sprintf(nodes[1 + i].name, "file%02u.bin", (unsigned)i);
nodes[1 + i].file = true;
sprintf(nodes[2 + i].name, "file%02u.bin", (unsigned)i);
nodes[2 + i].file = true;
}
strcpy(nodes[1 + FILLERS].name, "media");
nodes[1 + FILLERS].first = 1 + FILLERS + 2;
nodes[1 + FILLERS].count = 2;
strcpy(nodes[2 + FILLERS].name, "default.xex");
nodes[2 + FILLERS].file = true;
nodes[2 + FILLERS].size = XEX_SIZE;
strcpy(nodes[2 + FILLERS].name, "media");
nodes[2 + FILLERS].first = 1 + FILLERS + 2;
nodes[2 + FILLERS].count = 2;
strcpy(nodes[1].name, "default.xex");
nodes[1].file = true;
nodes[1].size = XEX_SIZE;
strcpy(nodes[3 + FILLERS].name, "a.bin");
nodes[3 + FILLERS].file = true;
nodes[3 + FILLERS].size = 5;
@@ -142,7 +145,7 @@ static void make_archive(file_t *f, int bad_block)
nodes[4 + FILLERS].file = true;
nodes[4 + FILLERS].size = 100;
for (i = 0; i < FILLERS; i++)
nodes[1 + i].size = 10;
nodes[2 + i].size = 10;
/* the stream: default.xex first - block 0 compresses, block 1 does
* not, block 2 compresses - then the rest */
@@ -151,12 +154,12 @@ static void make_archive(file_t *f, int bad_block)
pos = 0;
for (i = 0; i < XEX_SIZE; i++)
stream[i] = (i / BLOCK == 1) ? (uint8_t)rnd() : (uint8_t)(i * 7 / 64);
nodes[2 + FILLERS].offset = 0;
nodes[1].offset = 0;
pos = XEX_SIZE;
for (i = 0; i < N_NODES; i++)
{
size_t k;
if (!nodes[i].file || i == 2 + FILLERS)
if (!nodes[i].file || i == 1)
continue;
nodes[i].offset = pos;
for (k = 0; k < nodes[i].size; k++)
@@ -242,6 +245,7 @@ static void make_archive(file_t *f, int bad_block)
be64(ft + 0, 0); be64(ft + 8, bsize);
be64(ft + 16, rec_at); be64(ft + 24, 40);
be64(ft + 32, names_at); be64(ft + 40, nsize);
names_size = nsize;
be64(ft + 48, tree_at); be64(ft + 56, sizeof(tree));
be64(ft + 64, end); be64(ft + 72, 0);
be64(ft + 80, end); be64(ft + 88, 0);
@@ -310,31 +314,53 @@ static void lookup_lane(void)
f.asked = 0;
check( lookup(&f, z, "\\DEFAULT.XEX", &zf) == ZAR_OK
&& zf.offset == 0 && zf.size == XEX_SIZE, "lookup by name");
/* the root's node, its entries in two runs, each name as far as an
* eleven-octet name can match */
check(f.asked == 16 + 32 * 16 + 8 * 16 + (FILLERS + 2) * 13,
"lookup reads only what it needs");
/* the root's node, then six probes of the forty entries, each an
* entry's node and its name as far as an eleven-octet name can go */
check(f.asked == 16 + 6 * (16 + 13), "lookup reads only what it needs");
check( lookup(&f, z, "media/Intro.WMV", &zf) == ZAR_OK
&& zf.size == 100, "nested lookup");
check( lookup(&f, z, "media/nope", &zf) == ZAR_ERROR_NOT_FOUND
&& lookup(&f, z, "default.xex/x", &zf) == ZAR_ERROR_NOT_FOUND
&& lookup(&f, z, "media", &zf) == ZAR_ERROR_NOT_FOUND
&& lookup(&f, z, "nope", &zf) == ZAR_ERROR_NOT_FOUND,
&& lookup(&f, z, "nope", &zf) == ZAR_ERROR_NOT_FOUND
&& lookup(&f, z, "file99.bin", &zf) == ZAR_ERROR_NOT_FOUND
&& lookup(&f, z, "aaa", &zf) == ZAR_ERROR_NOT_FOUND
&& lookup(&f, z, "zzz", &zf) == ZAR_ERROR_NOT_FOUND,
"missing files");
check( lookup(&f, z, "MEDIA/", &zf) == ZAR_OK
&& zf.dir && zf.size == 2, "a directory");
check( lookup(&f, z, "", &zf) == ZAR_OK
&& zf.dir && zf.offset == 1 && zf.size == FILLERS + 2, "the root");
{
unsigned i;
bool all = true;
for (i = 0; i < FILLERS; i++)
{
char name[32];
sprintf(name, "/File%02u.BIN", i);
if ( lookup(&f, z, name, &zf) != ZAR_OK
|| zf.dir || zf.size != 10)
all = false;
}
check(all, "every entry found by its search");
}
check(zar_index_offset(z) == f.size - 144 - (16 * N_NODES)
- names_size - 40, "where the index starts");
zar_free(z);
free(f.data);
}
/* Lists @path, checking each entry against @names; returns how many
* matched in order, or -1 when the listing fails */
/* Lists @path, checking each entry against @names; a file's place is
* the one a lookup of it gives. Returns how many matched in order, or
* -1 when the listing fails */
static int list(file_t *f, zar_t *z, const char *path,
const char *const *names, const int *dirs, int n)
{
zar_request_t req;
zar_dirent_t ent;
zar_file_t seen[64];
int err, i = 0;
if (zar_dir_begin(z, path) != ZAR_OK)
if (n > 64 || zar_dir_begin(z, path) != ZAR_OK)
return -1;
for (;;)
{
@@ -343,11 +369,25 @@ static int list(file_t *f, zar_t *z, const char *path,
if (err != ZAR_OK)
break;
if ( i >= n || strcmp(ent.name, names[i])
|| ent.len != strlen(names[i]) || !ent.is_dir != !dirs[i])
|| ent.len != strlen(names[i]) || !ent.is_dir != !dirs[i]
|| !ent.file.dir != !dirs[i])
return -1;
i++;
seen[i++] = ent.file;
}
return err == ZAR_END ? i : -1;
if (err != ZAR_END)
return -1;
for (err = 0; err < i; err++)
{
char full[64];
zar_file_t zf;
if (dirs[err])
continue;
sprintf(full, "%s/%s", path, names[err]);
if ( lookup(f, z, full, &zf) != ZAR_OK
|| zf.offset != seen[err].offset || zf.size != seen[err].size)
return -1;
}
return i;
}
static void list_lane(void)
@@ -362,16 +402,16 @@ static void list_lane(void)
zar_file_t zf;
int i, pass;
root[0] = "default.xex";
rdirs[0] = 0;
for (i = 0; i < FILLERS; i++)
{
sprintf(fill[i], "file%02u.bin", (unsigned)i);
root[i] = fill[i];
rdirs[i] = 0;
root[1 + i] = fill[i];
rdirs[1 + i] = 0;
}
root[FILLERS] = "media";
rdirs[FILLERS] = 1;
root[FILLERS + 1] = "default.xex";
rdirs[FILLERS + 1] = 0;
root[FILLERS + 1] = "media";
rdirs[FILLERS + 1] = 1;
for (pass = 0; pass < 2; pass++)
{
@@ -385,6 +425,9 @@ static void list_lane(void)
check( list(&f, z, "nope", media, mdirs, 2) == -1
&& list(&f, z, "default.xex", media, mdirs, 2) == -1,
"a missing directory and a file are not listed");
check( lookup(&f, z, "media", &zf) == ZAR_OK && zf.dir
&& zar_read_begin(z, &zf, 0, f.data, 1) == ZAR_ERROR_PARAM,
"a directory is not read");
/* a lookup after a listing */
check( lookup(&f, z, "media/a.bin", &zf) == ZAR_OK
&& zf.size == 5, "lookup after a listing");
@@ -498,12 +541,217 @@ static void broken_lanes(void)
free(f.data);
}
/* the writer's output, grown in memory */
typedef struct
{
uint8_t *data;
size_t size;
size_t cap;
size_t fail_at; /* a write past this fails; 0 never */
} sink_t;
static int sink_write(void *ud, const void *data, size_t len)
{
sink_t *k = (sink_t*)ud;
if (k->fail_at && k->size + len > k->fail_at)
return 0;
if (k->size + len > k->cap)
{
size_t n = k->cap ? k->cap * 2 : 65536;
uint8_t *q;
while (n < k->size + len)
n *= 2;
if (!(q = (uint8_t*)realloc(k->data, n)))
return 0;
k->data = q;
k->cap = n;
}
memcpy(k->data + k->size, data, len);
k->size += len;
return 1;
}
static int fold(int c)
{
return (c >= 'A' && c <= 'Z') ? c + 32 : c;
}
static int name_cmp(const char *a, const char *b)
{
for (; *a && *b; a++, b++)
if (fold((unsigned char)*a) != fold((unsigned char)*b))
return fold((unsigned char)*a) - fold((unsigned char)*b);
return (*a != 0) - (*b != 0);
}
#define W_FILES 9
static void writer_lane(void)
{
static const char *const paths[W_FILES] = {
"content/a.bin", "content/Empty", "content/B_x", "content/ba",
"content/b[", "content/A_", "content/sub/deep/z", "meta/meta.xml",
NULL /* the long name */
};
char longname[8 + 200 + 1];
size_t sizes[W_FILES] = { 3 * BLOCK + 777, 0, 1, 2, 3, 4, BLOCK, 50, 9 };
uint8_t *data[W_FILES];
sink_t k;
zar_writer_t *w;
file_t f;
zar_t *z;
zar_file_t zf;
zar_dirent_t e;
uint8_t digest[32];
uint8_t saved[32];
struct sha256_state sha;
unsigned i;
size_t j;
size_t total = 0;
bool ok = true;
memset(&k, 0, sizeof(k));
memcpy(longname, "content/", 8);
memset(longname + 8, 'q', 200);
longname[208] = '\0';
check((w = zar_writer_new(sink_write, &k, 3)) != NULL, "writer");
check( zar_writer_dir(w, "meta") == ZAR_OK
&& zar_writer_dir(w, "content/sub/deep") == ZAR_OK
&& zar_writer_dir(w, "/CONTENT/") == ZAR_OK, "directories");
check(zar_writer_append(w, "x", 1) == ZAR_ERROR_PARAM,
"nothing to append to");
for (i = 0; i < W_FILES; i++)
{
const char *path = paths[i] ? paths[i] : longname;
data[i] = (uint8_t*)malloc(sizes[i] + 1);
for (j = 0; j < sizes[i]; j++)
data[i][j] = (i == 0 && j / BLOCK == 1) ? (uint8_t)rnd()
: (uint8_t)(i * 13 + j / 100);
total += sizes[i];
if (zar_writer_file(w, path) != ZAR_OK)
ok = false;
/* in uneven pieces, a whole block among them */
for (j = 0; j < sizes[i]; )
{
size_t n = (j == 0 && sizes[i] > BLOCK) ? 1000
: (j == 1000 ? 3 * BLOCK : 1 + rnd() % 5000);
if (n > sizes[i] - j)
n = sizes[i] - j;
if (zar_writer_append(w, data[i] + j, n) != ZAR_OK)
ok = false;
j += n;
}
}
check(ok, "files written");
check( zar_writer_file(w, "CONTENT/A.BIN") == ZAR_ERROR_PARAM
&& zar_writer_file(w, "content/a.bin/x") == ZAR_ERROR_PARAM
&& zar_writer_dir(w, "meta/meta.xml") == ZAR_ERROR_PARAM
&& zar_writer_file(w, "content") == ZAR_ERROR_PARAM
&& zar_writer_file(w, "") == ZAR_ERROR_PARAM,
"what is there already, or under a file");
check(zar_writer_finish(w) == ZAR_OK, "finished");
zar_writer_free(w);
check(k.size < total, "compressed");
/* the footer's hash: the file's, with the hash zeroed */
memcpy(saved, k.data + k.size - 48, 32);
memset(k.data + k.size - 48, 0, 32);
sha256_stream_init(&sha, 0);
sha256_stream_update(&sha, k.data, k.size);
sha256_stream_final(&sha, digest);
check(!memcmp(saved, digest, 32), "the footer's hash");
f.data = k.data;
f.size = k.size;
f.shorten = true;
f.requests = 0;
f.asked = 0;
check(open_zar(&f, &z) == ZAR_OK, "reads back");
ok = true;
for (i = 0; i < W_FILES; i++)
{
char upper[256];
unsigned outside = 0;
uint8_t *got = (uint8_t*)malloc(sizes[i] + 1);
const char *path = paths[i] ? paths[i] : longname;
for (j = 0; path[j]; j++)
upper[j] = (char)((path[j] >= 'a' && path[j] <= 'z')
? path[j] - 32 : path[j]);
upper[j] = '\0';
if ( lookup(&f, z, upper, &zf) != ZAR_OK || zf.dir
|| zf.size != sizes[i]
|| read_file(&f, z, &zf, 0, got, sizes[i] + 1, &outside) != ZAR_OK
|| zar_read_progress(z) != sizes[i]
|| memcmp(got, data[i], sizes[i]))
ok = false;
free(got);
free(data[i]);
}
check(ok, "every file found and read back");
/* each directory's entries in the order the search needs */
check( lookup(&f, z, "content", &zf) == ZAR_OK && zf.dir
&& zf.size == 8, "a directory written");
{
zar_request_t req;
char prev[256];
int err;
unsigned n = 0;
prev[0] = '\0';
ok = zar_dir_begin(z, "content") == ZAR_OK;
for (;;)
{
while ((err = zar_dir_step(z, &req, &e)) == ZAR_PENDING)
zar_feed(z, pull(&f, &req));
if (err != ZAR_OK)
break;
if (n++ && name_cmp(prev, e.name) >= 0)
ok = false;
strcpy(prev, e.name);
}
check(ok && err == ZAR_END && n == 8, "entries sorted");
ok = zar_dir_begin(z, "content/sub/deep") == ZAR_OK;
while ((err = zar_dir_step(z, &req, &e)) == ZAR_PENDING)
zar_feed(z, pull(&f, &req));
check(ok && err == ZAR_OK && !strcmp(e.name, "z") && e.file.size == BLOCK,
"a nested directory");
}
zar_free(z);
free(k.data);
/* a write that fails */
memset(&k, 0, sizeof(k));
k.fail_at = 100;
w = zar_writer_new(sink_write, &k, 3);
zar_writer_file(w, "x");
zar_writer_append(w, "abc", 3);
check(zar_writer_finish(w) == ZAR_ERROR_WRITE, "a failed write");
zar_writer_free(w);
free(k.data);
/* nothing but an empty file */
memset(&k, 0, sizeof(k));
w = zar_writer_new(sink_write, &k, 3);
zar_writer_file(w, "empty");
check(zar_writer_finish(w) == ZAR_OK, "an archive without blocks");
zar_writer_free(w);
f.data = k.data;
f.size = k.size;
f.shorten = false;
check( open_zar(&f, &z) == ZAR_OK
&& lookup(&f, z, "empty", &zf) == ZAR_OK && !zf.size,
"reads back without blocks");
zar_free(z);
free(k.data);
}
int main(void)
{
lookup_lane();
list_lane();
read_lane();
broken_lanes();
writer_lane();
free(stream);
if (failures)
tasks/task_database_cue.c
+3 -2
View File
@@ -2492,7 +2492,7 @@ int task_database_zar_get_serial(const char *name, char *s, size_t len,
while ((err = zar_lookup_step(zs.z, &req, &zs.file)) == ZAR_PENDING)
zar_feed(zs.z, task_database_pull(&zs.src, req.offset, req.dst,
req.length));
if (err != ZAR_OK || !(x = xdvdfs_new(zs.file.size)))
if (err != ZAR_OK || zs.file.dir || !(x = xdvdfs_new(zs.file.size)))
goto end;
exe.offset = 0;
@@ -2704,7 +2704,8 @@ int task_database_wua_get_serial(const char *name, char *s, size_t len,
while ((err = zar_lookup_step(z, &req, &zs.file)) == ZAR_PENDING)
zar_feed(z, task_database_pull(&zs.src, req.offset, req.dst,
req.length));
if (err != ZAR_OK || !(xml = (char*)malloc(WIIU_META_HEAD + 1)))
if ( err != ZAR_OK || zs.file.dir
|| !(xml = (char*)malloc(WIIU_META_HEAD + 1)))
goto end;
n = task_database_zar_pull(&zs, 0, (uint8_t*)xml, WIIU_META_HEAD);
xml[n] = '\0';