diff --git a/.github/workflows/Linux-libretro-common-samples.yml b/.github/workflows/Linux-libretro-common-samples.yml index 33d923207a..5f33709172 100644 --- a/.github/workflows/Linux-libretro-common-samples.yml +++ b/.github/workflows/Linux-libretro-common-samples.yml @@ -237,6 +237,7 @@ jobs: rvz_test ciso_test xdvdfs_test + wux_test zar_test filestream_printf_test strlcpy_append_test diff --git a/Makefile.common b/Makefile.common index 82f00cdf91..160a804389 100644 --- a/Makefile.common +++ b/Makefile.common @@ -608,7 +608,8 @@ ifeq ($(HAVE_LIBRETRODB), 1) tasks/task_database_cue.o \ $(LIBRETRO_COMM_DIR)/formats/ciso/ciso.o \ $(LIBRETRO_COMM_DIR)/formats/xdvdfs/xdvdfs.o \ - $(LIBRETRO_COMM_DIR)/formats/zar/zar.o + $(LIBRETRO_COMM_DIR)/formats/zar/zar.o \ + $(LIBRETRO_COMM_DIR)/formats/wux/wux.o ifeq ($(HAVE_MENU), 1) OBJ += menu/menu_explore.o \ diff --git a/griffin/griffin.c b/griffin/griffin.c index 2476c4b06e..3e75149048 100644 --- a/griffin/griffin.c +++ b/griffin/griffin.c @@ -1853,6 +1853,7 @@ DATA RUNLOOP #include "../tasks/task_database_cue.c" #include "../libretro-common/formats/xdvdfs/xdvdfs.c" #include "../libretro-common/formats/zar/zar.c" +#include "../libretro-common/formats/wux/wux.c" #endif #if defined(HAVE_LIBRETRODB) || defined(HAVE_CHEEVOS) #include "../libretro-common/formats/ciso/ciso.c" diff --git a/libretro-common/formats/wux/wux.c b/libretro-common/formats/wux/wux.c new file mode 100644 index 0000000000..2c70c66e93 --- /dev/null +++ b/libretro-common/formats/wux/wux.c @@ -0,0 +1,329 @@ +/* Copyright (C) 2010-2026 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (wux.c). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +/* No file I/O: every range of the file is asked of the caller (see + * wux.h). + * + * The file, all little endian: + * + * 0 "WUX0" + * 4 0x1099D02E + * 8 sector size, 0x100 through 0x0FFFFFFF + * 16 the WUD's size, 64-bit + * 24 flags + * 32 the index: per WUD sector, a 32-bit sector number in the array + * then the sector array, from the index's end rounded up to the + * sector size + * + * A sector the WUD holds more than once is kept once, so the index may + * name a sector many times; a run of the WUD is one request only + * where the array keeps its sectors in the same order. */ + +#include +#include + +#include +#include + +#define WUX_HEAD_SIZE 32 +#define WUX_MAGIC1 0x1099D02Eu +#define WUX_SECTOR_MIN 0x100u +#define WUX_SECTOR_MAX 0x10000000u +/* index entries asked for at once */ +#define WUX_WINDOW 256 + +#define WUX_STAGE_HEAD 0 +#define WUX_STAGE_OPEN 1 +#define WUX_STAGE_FAILED 2 + +#define WUX_TO_HEAD 0 +#define WUX_TO_INDEX 1 +#define WUX_TO_DATA 2 + +struct wux +{ + uint64_t file_size; + uint64_t disc_size; + uint64_t array_at; /* the sector array, in the file */ + uint64_t rd_off; + uint8_t *rd_dst; + size_t rd_len; + size_t rd_done; + size_t got; /* of the header or the index window */ + size_t fetch_len; + uint32_t sector; + uint32_t sectors; /* the WUD's, as many as the index has */ + uint32_t win_first; /* the first sector the window holds */ + uint32_t win_count; /* how many it holds, 0 for none */ + uint32_t win_want; /* how many are being read into it */ + int stage; + int shift; /* log2 of the sector size, -1 when not a power of two */ + int to; + uint8_t win[WUX_WINDOW * 4]; + bool fetching; + bool failed; + bool rd_armed; +}; + +static uint32_t wux_le32(const uint8_t *p) +{ + return (uint32_t)p[0] + | ((uint32_t)p[1] << 8) + | ((uint32_t)p[2] << 16) + | ((uint32_t)p[3] << 24); +} + +int wux_is_wux(const uint8_t *head) +{ + return head + && head[0] == 'W' && head[1] == 'U' && head[2] == 'X' && head[3] == '0' + && wux_le32(head + 4) == WUX_MAGIC1; +} + +wux_t *wux_new(uint64_t file_size) +{ + wux_t *w = (wux_t*)malloc(sizeof(wux_t)); + if (!w) + return NULL; + w->file_size = file_size; + w->disc_size = 0; + w->array_at = 0; + w->rd_off = 0; + w->rd_dst = NULL; + w->rd_len = 0; + w->rd_done = 0; + w->got = 0; + w->fetch_len = 0; + w->sector = 0; + w->sectors = 0; + w->win_first = 0; + w->win_count = 0; + w->win_want = 0; + w->stage = WUX_STAGE_HEAD; + w->shift = -1; + w->to = WUX_TO_HEAD; + w->fetching = false; + w->failed = false; + w->rd_armed = false; + return w; +} + +void wux_free(wux_t *w) +{ + free(w); +} + +static int wux_ask(wux_t *w, wux_request_t *req, int to, uint64_t off, + uint8_t *dst, size_t len) +{ + req->offset = off; + req->length = len; + req->dst = dst; + w->to = to; + w->fetch_len = len; + w->fetching = true; + return WUX_PENDING; +} + +/* The header is read into the index window, which nothing else holds + * until the image is open. */ +static int wux_parse(wux_t *w) +{ + const uint8_t *h = w->win; + uint32_t ss; + uint64_t n, at; + + if (!wux_is_wux(h)) + return WUX_ERROR_DATA; + ss = wux_le32(h + 8); + w->disc_size = (uint64_t)wux_le32(h + 16) + | ((uint64_t)wux_le32(h + 20) << 32); + if (ss < WUX_SECTOR_MIN || ss >= WUX_SECTOR_MAX || !w->disc_size) + return WUX_ERROR_DATA; + n = w->disc_size / ss + (w->disc_size % ss ? 1 : 0); + if (n > 0xFFFFFFFFu) + return WUX_ERROR_DATA; + at = WUX_HEAD_SIZE + n * 4; + at = (at + ss - 1) / ss * ss; + if (at > w->file_size) + return WUX_ERROR_DATA; + w->sector = ss; + w->sectors = (uint32_t)n; + w->array_at = at; + w->shift = -1; + if (!(ss & (ss - 1))) + for (w->shift = 0; (1u << w->shift) != ss; w->shift++) { } + return WUX_OK; +} + +int wux_open_step(wux_t *w, wux_request_t *req) +{ + int e; + if (!w || !req) + return WUX_ERROR_PARAM; + if (w->stage == WUX_STAGE_OPEN) + return WUX_OK; + if (w->stage == WUX_STAGE_FAILED || w->failed) + { + w->stage = WUX_STAGE_FAILED; + return WUX_ERROR_DATA; + } + if (w->got < WUX_HEAD_SIZE) + return wux_ask(w, req, WUX_TO_HEAD, w->got, w->win + w->got, + WUX_HEAD_SIZE - w->got); + w->got = 0; + e = wux_parse(w); + w->stage = (e == WUX_OK) ? WUX_STAGE_OPEN : WUX_STAGE_FAILED; + return e; +} + +int wux_feed(wux_t *w, size_t len) +{ + if (!w || !w->fetching) + return WUX_ERROR_STATE; + w->fetching = false; + if (!len) + { + w->failed = true; + return WUX_ERROR_DATA; + } + if (len > w->fetch_len) + len = w->fetch_len; + if (w->to == WUX_TO_DATA) + w->rd_done += len; + else + w->got += len; + return WUX_OK; +} + +uint64_t wux_disc_size(const wux_t *w) +{ + return w ? w->disc_size : 0; +} + +int wux_read_begin(wux_t *w, uint64_t off, void *dst, size_t len) +{ + if (!w || (!dst && len)) + return WUX_ERROR_PARAM; + if (w->stage != WUX_STAGE_OPEN) + return WUX_ERROR_STATE; + if (off >= w->disc_size) + len = 0; + else if (len > w->disc_size - off) + len = (size_t)(w->disc_size - off); + /* a window left half read is not kept */ + if (w->fetching && w->to == WUX_TO_INDEX) + w->win_count = 0; + w->got = 0; + w->fetching = false; + w->failed = false; + w->rd_dst = (uint8_t*)dst; + w->rd_off = off; + w->rd_len = len; + w->rd_done = 0; + w->rd_armed = true; + return WUX_OK; +} + +int wux_read_step(wux_t *w, wux_request_t *req) +{ + uint64_t ss; + int shift; + if (!w || !req) + return WUX_ERROR_PARAM; + if (!w->rd_armed) + return WUX_ERROR_STATE; + if (w->failed) + { + w->rd_armed = false; + return WUX_ERROR_DATA; + } + ss = w->sector; + shift = w->shift; + while (w->rd_done < w->rd_len) + { + uint64_t off = w->rd_off + w->rd_done; + uint64_t left = w->rd_len - w->rd_done; + uint64_t in, run; + uint32_t s, e, stop, idx; + + /* 64-bit division is a library call on the 32-bit targets */ + if (shift >= 0) + { + s = (uint32_t)(off >> shift); + in = off & (ss - 1); + } + else + { + s = (uint32_t)(off / ss); + in = off - (uint64_t)s * ss; + } + + /* the index window, read from @s on when it does not hold it */ + if ( !w->win_count + || s < w->win_first + || s - w->win_first >= w->win_count) + { + if (!w->got) + { + w->win_count = 0; + w->win_first = s; + w->win_want = w->sectors - s; + if (w->win_want > WUX_WINDOW) + w->win_want = WUX_WINDOW; + } + if (w->got < (size_t)w->win_want * 4) + return wux_ask(w, req, WUX_TO_INDEX, + WUX_HEAD_SIZE + (uint64_t)w->win_first * 4 + w->got, + w->win + w->got, (size_t)w->win_want * 4 - w->got); + w->got = 0; + w->win_count = w->win_want; + } + + /* as far as the array keeps the WUD's sectors in order, within + * the read and the window */ + idx = wux_le32(w->win + (size_t)(s - w->win_first) * 4); + stop = w->win_first + w->win_count; + e = s + 1; + run = ss - in; + while (run < left && e < stop + && wux_le32(w->win + (size_t)(e - w->win_first) * 4) + == idx + (e - s)) + { + run += ss; + e++; + } + if (run > left) + run = left; + + return wux_ask(w, req, WUX_TO_DATA, + w->array_at + (uint64_t)idx * ss + in, + w->rd_dst + w->rd_done, (size_t)run); + } + w->rd_armed = false; + return WUX_OK; +} + +size_t wux_read_progress(const wux_t *w) +{ + return w ? w->rd_done : 0; +} diff --git a/libretro-common/formats/zar/zar.c b/libretro-common/formats/zar/zar.c index edbbc6d54a..819ccb7214 100644 --- a/libretro-common/formats/zar/zar.c +++ b/libretro-common/formats/zar/zar.c @@ -64,11 +64,13 @@ #define ZAR_OP_NONE 1 #define ZAR_OP_LOOKUP 2 #define ZAR_OP_READ 3 +#define ZAR_OP_DIR 4 /* the lookup */ #define ZAR_L_DIR 0 /* the directory's own node */ #define ZAR_L_BATCH 1 /* a run of its entries */ #define ZAR_L_NAME 2 /* one entry's name */ +#define ZAR_L_LIST 3 /* a listed entry's name, in full */ /* where the range asked for goes */ #define ZAR_TO_BUF 0 @@ -109,10 +111,12 @@ struct zar int stage; int to; uint8_t buf[ZAR_BATCH * ZAR_NODE]; - uint8_t name[ZAR_NAME_MAX + 2]; + uint8_t name[ZAR_NAME_MAX + 3]; /* length, octets, terminator */ uint8_t rec[ZAR_RECORD]; bool fetching; bool failed; + bool want_dir; /* a listing's directory is looked up */ + bool listing; /* and found */ }; static uint32_t zar_be32(const uint8_t *p) @@ -169,6 +173,8 @@ zar_t *zar_new(uint64_t file_size) z->to = ZAR_TO_BUF; z->fetching = false; z->failed = false; + z->want_dir = false; + z->listing = false; return z; } @@ -281,9 +287,22 @@ int zar_lookup_begin(zar_t *z, const char *path) z->got = 0; z->fetching = false; z->failed = false; + z->want_dir = false; + z->listing = false; return ZAR_OK; } +int zar_dir_begin(zar_t *z, const char *path) +{ + int e = zar_lookup_begin(z, path); + if (e == ZAR_OK) + { + z->op = ZAR_OP_DIR; + z->want_dir = true; + } + return e; +} + static int zar_upper(int c) { return (c >= 'a' && c <= 'z') ? c - ('a' - 'A') : c; @@ -308,7 +327,9 @@ static int zar_lookup_run(zar_t *z, zar_request_t *req, zar_file_t *out) z->path++; p = z->path; for (plen = 0; p[plen] && p[plen] != '/' && p[plen] != '\\'; plen++) { } - if (!plen || plen > ZAR_NAME_MAX) + /* a listing of the root looks up no name */ + if ( (!plen && !(z->want_dir && z->stage == ZAR_L_DIR)) + || plen > ZAR_NAME_MAX) return ZAR_ERROR_NOT_FOUND; switch (z->stage) @@ -332,6 +353,8 @@ static int zar_lookup_run(zar_t *z, zar_request_t *req, zar_file_t *out) z->dir_index = 0; z->batch = 0; z->stage = ZAR_L_BATCH; + if (!plen) + return ZAR_OK; break; case ZAR_L_BATCH: if (z->dir_index >= z->dir_count) @@ -393,7 +416,7 @@ static int zar_lookup_run(zar_t *z, zar_request_t *req, zar_file_t *out) z->path += plen; while (*z->path == '/' || *z->path == '\\') z->path++; - if (*z->path) + if (*z->path || z->want_dir) { /* down into this entry, which must be a directory */ if (n[0] & 0x80) @@ -404,7 +427,10 @@ static int zar_lookup_run(zar_t *z, zar_request_t *req, zar_file_t *out) || z->dir_count > z->nodes - z->dir_first) return ZAR_ERROR_DATA; z->dir_index = 0; + z->batch = 0; z->stage = ZAR_L_BATCH; + if (!*z->path) + return ZAR_OK; break; } if (!(n[0] & 0x80)) @@ -436,6 +462,98 @@ int zar_lookup_step(zar_t *z, zar_request_t *req, zar_file_t *out) return e; } +/* The directory's entries a run at a time, each one's name read in + * full */ +static int zar_list_run(zar_t *z, zar_request_t *req, zar_dirent_t *ent) +{ + for (;;) + { + const uint8_t *n; + size_t cap; + size_t hdr; + size_t nlen; + uint32_t name_at; + + switch (z->stage) + { + case ZAR_L_BATCH: + if (z->dir_index >= z->dir_count) + return ZAR_END; + z->batch = z->dir_count - z->dir_index; + if (z->batch > ZAR_BATCH) + z->batch = ZAR_BATCH; + if (z->got < (size_t)z->batch * ZAR_NODE) + return zar_ask(z, req, ZAR_TO_BUF, + z->sec_off[ZAR_SEC_TREE] + + ((uint64_t)z->dir_first + z->dir_index) * ZAR_NODE, + z->buf, (size_t)z->batch * ZAR_NODE); + z->got = 0; + z->entry = 0; + z->stage = ZAR_L_LIST; + break; + case ZAR_L_LIST: + if (z->entry >= z->batch) + { + z->dir_index += z->batch; + z->stage = ZAR_L_BATCH; + break; + } + n = z->buf + z->entry * ZAR_NODE; + name_at = zar_be32(n) & 0x7FFFFFFFu; + if ( name_at == ZAR_NO_NAME + || name_at >= z->sec_size[ZAR_SEC_NAMES]) + return ZAR_ERROR_DATA; + cap = 2 + ZAR_NAME_MAX; + if (cap > z->sec_size[ZAR_SEC_NAMES] - name_at) + cap = (size_t)(z->sec_size[ZAR_SEC_NAMES] - name_at); + if (z->got < cap) + return zar_ask(z, req, ZAR_TO_NAME, + z->sec_off[ZAR_SEC_NAMES] + name_at, z->name, cap); + z->got = 0; + nlen = z->name[0] & 0x7F; + hdr = 1; + if (z->name[0] & 0x80) + { + if (cap < 2) + return ZAR_ERROR_DATA; + nlen |= (size_t)z->name[1] << 7; + hdr = 2; + } + if (nlen > ZAR_NAME_MAX || hdr + nlen > cap) + return ZAR_ERROR_DATA; + z->name[hdr + nlen] = '\0'; + ent->name = (const char*)z->name + hdr; + ent->len = nlen; + ent->is_dir = !(n[0] & 0x80); + z->entry++; + return ZAR_OK; + default: + return ZAR_ERROR_STATE; + } + } +} + +int zar_dir_step(zar_t *z, zar_request_t *req, zar_dirent_t *ent) +{ + int e; + if (!z || !req || !ent) + return ZAR_ERROR_PARAM; + if (z->op != ZAR_OP_DIR) + return ZAR_ERROR_STATE; + if (z->failed) + e = ZAR_ERROR_DATA; + else if (z->listing) + e = zar_list_run(z, req, ent); + else if ((e = zar_lookup_run(z, req, NULL)) == ZAR_OK) + { + z->listing = true; + e = zar_list_run(z, req, ent); + } + if (e != ZAR_PENDING && e != ZAR_OK) + z->op = ZAR_OP_NONE; + return e; +} + int zar_read_begin(zar_t *z, const zar_file_t *file, uint64_t off, void *dst, size_t len) { diff --git a/libretro-common/include/formats/wux.h b/libretro-common/include/formats/wux.h new file mode 100644 index 0000000000..bf83935a71 --- /dev/null +++ b/libretro-common/include/formats/wux.h @@ -0,0 +1,140 @@ +/* Copyright (C) 2010-2026 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (wux.h). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __LIBRETRO_SDK_FORMAT_WUX_H +#define __LIBRETRO_SDK_FORMAT_WUX_H + +#include +#include + +#include + +RETRO_BEGIN_DECLS + +/* WUX, the Wii U disc image format that keeps each distinct sector + * once, read as the WUD (the plain disc image) it stands for: every + * read returns the octets the WUD holds at that offset. + * + * The reader does no file I/O of its own and holds no file. Each range + * of the file it needs is described to the caller as a request, which + * names where the octets go: the caller puts up to @length octets from + * @offset there, however it gets them, and says how many with + * wux_feed(). A short supply is legal and asked for again; none at all + * fails the operation. + * + * wux_request_t req; + * int err; + * + * while ((err = wux_open_step(w, &req)) == WUX_PENDING) + * wux_feed(w, pull(req.offset, req.dst, req.length)); + * + * wux_read_begin(w, offset, dst, len); + * while ((err = wux_read_step(w, &req)) == WUX_PENDING) + * wux_feed(w, pull(req.offset, req.dst, req.length)); + * + * Opening reads the 32-octet header alone. The sector index is read a + * window at a time, only where a read needs it, and a read's data + * requests point into the read's own destination, one per run of + * sectors the file keeps in order, so nothing is copied twice. */ + +#define WUX_OK 0 +#define WUX_PENDING 1 +#define WUX_ERROR_DATA (-1) +#define WUX_ERROR_PARAM (-2) +#define WUX_ERROR_MEM (-3) +#define WUX_ERROR_STATE (-4) + +/* the octets wux_is_wux() looks at */ +#define WUX_MAGIC_SIZE 8 + +typedef struct wux wux_t; + +/** + * wux_request_t: + * + * A range of the file the reader needs, and where to put it. Valid + * until the next call on the same wux_t. + */ +typedef struct wux_request +{ + uint64_t offset; /* in the file */ + size_t length; /* octets wanted; fewer may be supplied */ + uint8_t *dst; /* where they go */ +} wux_request_t; + +/* Whether the first WUX_MAGIC_SIZE octets of a file are a WUX's */ +int wux_is_wux(const uint8_t *head); + +/* @file_size: the file's size, which the header must fit */ +wux_t *wux_new(uint64_t file_size); + +void wux_free(wux_t *w); + +/** + * wux_open_step: + * + * Reads the file's header. + * + * Returns: WUX_OK once the image is open, WUX_PENDING with @req filled + * in when it needs a range of the file, or a negative WUX_ERROR_* code + * when the file is not one this reads. + **/ +int wux_open_step(wux_t *w, wux_request_t *req); + +/** + * wux_feed: + * @len : octets put at the last request's @dst + * + * Returns: WUX_OK, WUX_ERROR_DATA when @len is 0 (the operation is + * then over), or WUX_ERROR_STATE with no request outstanding. + **/ +int wux_feed(wux_t *w, size_t len); + +/* the size of the WUD the image stands for */ +uint64_t wux_disc_size(const wux_t *w); + +/** + * wux_read_begin: + * + * Arms a read of @len octets of the WUD at @off into @dst, short at + * its end. Nothing is read until wux_read_step(). An armed read + * abandons one still in progress. + * + * Returns: WUX_OK, or WUX_ERROR_STATE before the image is open. + **/ +int wux_read_begin(wux_t *w, uint64_t off, void *dst, size_t len); + +/** + * wux_read_step: + * + * Returns: WUX_OK when the read is complete, WUX_PENDING with @req + * filled in when it needs a range of the file, or a negative + * WUX_ERROR_* code. + **/ +int wux_read_step(wux_t *w, wux_request_t *req); + +/* octets the armed read has put at its @dst so far */ +size_t wux_read_progress(const wux_t *w); + +RETRO_END_DECLS + +#endif diff --git a/libretro-common/include/formats/zar.h b/libretro-common/include/formats/zar.h index 872bbd15c4..53723f9e4f 100644 --- a/libretro-common/include/formats/zar.h +++ b/libretro-common/include/formats/zar.h @@ -57,6 +57,7 @@ RETRO_BEGIN_DECLS #define ZAR_OK 0 #define ZAR_PENDING 1 +#define ZAR_END 2 /* a listing is over */ #define ZAR_ERROR_DATA (-1) #define ZAR_ERROR_PARAM (-2) #define ZAR_ERROR_MEM (-3) @@ -129,6 +130,36 @@ int zar_lookup_begin(zar_t *z, const char *path); **/ int zar_lookup_step(zar_t *z, zar_request_t *req, zar_file_t *out); +/* A directory's entry. @name is terminated, and valid until the next + * call on the same zar_t. */ +typedef struct zar_dirent +{ + const char *name; + size_t len; + int is_dir; +} zar_dirent_t; + +/** + * zar_dir_begin: + * @path : the directory, as for zar_lookup_begin(); "" for the root + * + * Arms a listing of a directory, abandoning a read, lookup or listing + * in progress. + * + * Returns: ZAR_OK, or ZAR_ERROR_STATE before the archive is open. + **/ +int zar_dir_begin(zar_t *z, const char *path); + +/** + * zar_dir_step: + * + * Returns: ZAR_OK with @ent filled in for each entry in the archive's + * order, ZAR_END past the last one, ZAR_PENDING with @req filled in + * when it needs a range of the file, ZAR_ERROR_NOT_FOUND when @path is + * no directory, or another negative code. + **/ +int zar_dir_step(zar_t *z, zar_request_t *req, zar_dirent_t *ent); + /** * zar_read_begin: * diff --git a/libretro-common/samples/formats/wux/Makefile b/libretro-common/samples/formats/wux/Makefile new file mode 100644 index 0000000000..c3ece0a0aa --- /dev/null +++ b/libretro-common/samples/formats/wux/Makefile @@ -0,0 +1,40 @@ +include ../../sample_objdir.mk + +TARGET := wux_test +.DEFAULT_GOAL := all + +LIBRETRO_COMM_DIR := ../../.. + +SOURCES := \ + wux_test.c \ + $(LIBRETRO_COMM_DIR)/formats/wux/wux.c + +OBJS := $(call sample_objs,$(SOURCES:.c=.o)) + +CFLAGS += -Wall -pedantic -std=gnu99 -g1 -O2 -I$(LIBRETRO_COMM_DIR)/include + +ifneq ($(SANITIZER),) + CFLAGS := -fsanitize=$(SANITIZER) -fno-omit-frame-pointer $(CFLAGS) + LDFLAGS := -fsanitize=$(SANITIZER) $(LDFLAGS) +endif + +all: $(TARGET) + +%.o: %.c + $(CC) -c -o $@ $< $(CFLAGS) + +$(SAMPLE_OBJDIR)/%.o: $$(call sample_src,$$*).c + @mkdir -p $(@D) + $(CC) -c -o $@ $< $(CFLAGS) + +$(TARGET): $(OBJS) + $(CC) -o $@ $^ $(LDFLAGS) + +check: $(TARGET) + ./$(TARGET) + +clean: + rm -rf $(SAMPLE_OBJDIR) + rm -f $(TARGET) $(OBJS) + +.PHONY: all check clean diff --git a/libretro-common/samples/formats/wux/wux_test.c b/libretro-common/samples/formats/wux/wux_test.c new file mode 100644 index 0000000000..268c27d1d3 --- /dev/null +++ b/libretro-common/samples/formats/wux/wux_test.c @@ -0,0 +1,323 @@ +/* wux_test: writes WUX images of disc images it makes up and checks + * that formats/wux reads them back as the disc image, octet for octet. + * + * The images keep a sector the disc holds more than once, once, and + * keep the array out of the disc's order in places, with sector sizes + * a power of two and not. Ranges are supplied short now and then. + * Every request is checked: opening reads the header alone; the index + * is read a window at a time, only where a read needs it; a read's + * data lands in its own destination, one request per run of sectors + * the array keeps in order. A header that is not one, a sector size + * out of range, an array past the file's end and nothing supplied must + * fail. */ + +#include +#include +#include + +#include +#include + +#define SECTORS 700 /* more than the reader's index window */ + +static int failures = 0; +static uint32_t rng_state = 0x5EED1234u; + +static uint32_t rnd(void) +{ + rng_state ^= rng_state << 13; + rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; + return rng_state; +} + +static void check(int ok, const char *what) +{ + if (!ok) + { + failures++; + printf("FAIL %s\n", what); + } +} + +static void le32(uint8_t *p, uint32_t v) +{ + p[0] = (uint8_t)v; + p[1] = (uint8_t)(v >> 8); + p[2] = (uint8_t)(v >> 16); + p[3] = (uint8_t)(v >> 24); +} + +/* a WUX, the disc it stands for, and how the reader asked for it */ +typedef struct +{ + uint8_t *data; + size_t size; + uint8_t *disc; + size_t disc_size; + uint32_t ss; + bool shorten; + unsigned requests; + unsigned index_requests; + size_t asked; +} file_t; + +static size_t pull(file_t *f, const wux_request_t *req) +{ + size_t n = req->length; + f->requests++; + f->asked += req->length; + if (req->offset >= f->size) + return 0; + if (n > f->size - req->offset) + n = f->size - (size_t)req->offset; + if (f->shorten && n > 1 && (rnd() & 3) == 0) + n = 1 + rnd() % (n - 1); + memcpy(req->dst, f->data + req->offset, n); + return n; +} + +/* The disc: @sectors of @ss octets, the last @tail long when not 0. + * Every seventh sector repeats the one before it, and sectors 100 + * through 119 are kept in reverse order; the rest are in order. */ +static void make_wux(file_t *f, uint32_t ss, uint32_t sectors, uint32_t tail) +{ + uint32_t *where = (uint32_t*)malloc(sectors * sizeof(uint32_t)); + uint32_t *order = (uint32_t*)malloc(sectors * sizeof(uint32_t)); + uint32_t kept = 0; + uint32_t s, k; + size_t at; + + f->ss = ss; + f->disc_size = (size_t)ss * sectors - (tail ? ss - tail : 0); + f->disc = (uint8_t*)malloc((size_t)ss * sectors); + for (s = 0; s < sectors; s++) + { + if (s % 7 == 6) + memcpy(f->disc + (size_t)s * ss, f->disc + (size_t)(s - 1) * ss, ss); + else + for (k = 0; k < ss; k++) + f->disc[(size_t)s * ss + k] = (uint8_t)rnd(); + } + memset(f->disc + f->disc_size, 0, (size_t)ss * sectors - f->disc_size); + + /* the distinct sectors, in the array's order */ + for (s = 0; s < sectors; s++) + if (s % 7 != 6) + order[kept++] = s; + for (k = 0; k < 10; k++) + { + uint32_t a = kept, b = kept, t; + for (s = 0; s < kept; s++) + { + if (order[s] == 100 + k) + a = s; + if (order[s] == 119 - k) + b = s; + } + /* a repeated sector is not in the array */ + if (a < kept && b < kept) + { + t = order[a]; order[a] = order[b]; order[b] = t; + } + } + for (k = 0; k < kept; k++) + where[order[k]] = k; + for (s = 0; s < sectors; s++) + if (s % 7 == 6) + where[s] = where[s - 1]; + + at = 32 + (size_t)sectors * 4; + at = (at + ss - 1) / ss * ss; + f->size = at + (size_t)kept * ss; + f->data = (uint8_t*)calloc(1, f->size); + memcpy(f->data, "WUX0", 4); + le32(f->data + 4, 0x1099D02Eu); + le32(f->data + 8, ss); + le32(f->data + 16, (uint32_t)f->disc_size); + le32(f->data + 20, 0); + for (s = 0; s < sectors; s++) + le32(f->data + 32 + s * 4, where[s]); + for (k = 0; k < kept; k++) + memcpy(f->data + at + (size_t)k * ss, f->disc + (size_t)order[k] * ss, ss); + f->shorten = false; + f->requests = 0; + f->index_requests = 0; + f->asked = 0; + free(where); + free(order); +} + +static void free_wux(file_t *f) +{ + free(f->data); + free(f->disc); +} + +static int open_wux(file_t *f, wux_t **out) +{ + wux_request_t req; + int err; + *out = wux_new(f->size); + while ((err = wux_open_step(*out, &req)) == WUX_PENDING) + wux_feed(*out, pull(f, &req)); + return err; +} + +/* counts the data requests that land outside @dst */ +static int read_disc(file_t *f, wux_t *w, uint64_t off, uint8_t *dst, + size_t len, unsigned *outside) +{ + wux_request_t req; + int err; + if ((err = wux_read_begin(w, off, dst, len)) != WUX_OK) + return err; + while ((err = wux_read_step(w, &req)) == WUX_PENDING) + { + if (req.offset < 32 + (uint64_t)SECTORS * 4 && req.offset >= 32) + f->index_requests++; + else if (req.dst < dst || req.dst + req.length > dst + len) + (*outside)++; + wux_feed(w, pull(f, &req)); + } + return err; +} + +static void read_lane(uint32_t ss, uint32_t tail) +{ + file_t f; + wux_t *w; + uint8_t *got; + unsigned outside = 0; + unsigned i; + int pass; + char what[64]; + + for (pass = 0; pass < 2; pass++) + { + make_wux(&f, ss, SECTORS, tail); + f.shorten = pass == 1; + got = (uint8_t*)malloc(f.disc_size + 16); + check(open_wux(&f, &w) == WUX_OK, "opens"); + if (!pass) + check(f.asked == 32 && f.requests == 1, "open reads the header alone"); + check(wux_disc_size(w) == f.disc_size, "disc size"); + + sprintf(what, "whole disc, sector size %u", (unsigned)ss); + check( read_disc(&f, w, 0, got, f.disc_size + 16, &outside) == WUX_OK + && wux_read_progress(w) == f.disc_size + && !memcmp(got, f.disc, f.disc_size), what); + + for (i = 0; i < 400; i++) + { + uint64_t off = rnd() % f.disc_size; + size_t len = 1 + rnd() % (5 * ss); + size_t want = len > f.disc_size - off + ? (size_t)(f.disc_size - off) : len; + if ( read_disc(&f, w, off, got, len, &outside) != WUX_OK + || wux_read_progress(w) != want + || memcmp(got, f.disc + off, want)) + { + check(0, "random reads"); + break; + } + } + check(!outside, "data lands in the read's destination"); + check( read_disc(&f, w, f.disc_size, got, 10, &outside) == WUX_OK + && !wux_read_progress(w), "a read past the end is empty"); + wux_free(w); + free(got); + free_wux(&f); + } + + /* requests: sectors 0 through 5 are in order and 6 repeats 5, so + * the first seven sectors are one window and two runs */ + make_wux(&f, ss, SECTORS, tail); + got = (uint8_t*)malloc(7 * ss); + open_wux(&f, &w); + f.requests = 0; + f.asked = 0; + check( read_disc(&f, w, 0, got, 7 * ss, &outside) == WUX_OK + && !memcmp(got, f.disc, 7 * ss) + && f.requests == 3 && f.asked == 256 * 4 + 7 * ss, + "one index window, one request per run"); + /* the same window serves the next read */ + f.requests = 0; + check( read_disc(&f, w, 2 * ss + 3, got, 10, &outside) == WUX_OK + && f.requests == 1, "the window is kept"); + /* the last sectors: a window of what is left */ + f.asked = 0; + check( read_disc(&f, w, f.disc_size - 1, got, 1, &outside) == WUX_OK + && got[0] == f.disc[f.disc_size - 1] + && f.asked == (size_t)(SECTORS - (SECTORS - 1)) * 4 + 1, + "the last window is as long as the index"); + wux_free(w); + free(got); + free_wux(&f); +} + +static void broken_lanes(void) +{ + file_t f; + wux_t *w; + wux_request_t req; + uint8_t buf[16]; + unsigned outside = 0; + + make_wux(&f, 0x100, 16, 0); + check(wux_is_wux(f.data), "is a WUX"); + f.data[3] = '1'; + check(!wux_is_wux(f.data), "is not a WUX"); + check(open_wux(&f, &w) == WUX_ERROR_DATA, "bad magic"); + check( wux_read_begin(w, 0, buf, 1) == WUX_ERROR_STATE, + "read before open"); + wux_free(w); + f.data[3] = '0'; + + le32(f.data + 8, 0x80); + check(open_wux(&f, &w) == WUX_ERROR_DATA, "sector size too small"); + wux_free(w); + le32(f.data + 8, 0x10000000u); + check(open_wux(&f, &w) == WUX_ERROR_DATA, "sector size too large"); + wux_free(w); + le32(f.data + 8, 0x100); + + le32(f.data + 20, 1); + check(open_wux(&f, &w) == WUX_ERROR_DATA, "array past the file's end"); + wux_free(w); + le32(f.data + 20, 0); + + /* a sector named past the array */ + le32(f.data + 32, 0x7FFF); + check( open_wux(&f, &w) == WUX_OK + && read_disc(&f, w, 0, buf, 16, &outside) == WUX_ERROR_DATA, + "a sector past the file's end"); + wux_free(w); + free_wux(&f); + + make_wux(&f, 0x100, 16, 0); + w = wux_new(f.size); + check(wux_feed(w, 1) == WUX_ERROR_STATE, "feed with nothing asked"); + check(wux_open_step(w, &req) == WUX_PENDING, "open asks"); + check( wux_feed(w, 0) == WUX_ERROR_DATA + && wux_open_step(w, &req) == WUX_ERROR_DATA, + "nothing supplied fails"); + wux_free(w); + free_wux(&f); +} + +int main(void) +{ + read_lane(0x8000, 0); + read_lane(0x100, 0x40); + read_lane(0x180, 0); + broken_lanes(); + + if (failures) + { + printf("%d failure(s)\n", failures); + return 1; + } + printf("wux_test: all passed\n"); + return 0; +} diff --git a/libretro-common/samples/formats/zar/zar_test.c b/libretro-common/samples/formats/zar/zar_test.c index 4a5e29c594..7d94007053 100644 --- a/libretro-common/samples/formats/zar/zar_test.c +++ b/libretro-common/samples/formats/zar/zar_test.c @@ -10,7 +10,8 @@ * a time and for each name only as far as it can match, exactly that * many octets; a read of a block kept as it is lands in the read's own * destination; a compressed block is fetched once for the partial - * reads that follow it. A footer that is not one, a size that + * reads that follow it. A directory is listed in the archive's order, + * each name read whole. A footer that is not one, a size that * disagrees and a block that does not decode must fail. */ #include @@ -325,6 +326,73 @@ static void lookup_lane(void) free(f.data); } +/* Lists @path, checking each entry against @names; returns how many + * matched in order, or -1 when the listing fails */ +static int list(file_t *f, zar_t *z, const char *path, + const char *const *names, const int *dirs, int n) +{ + zar_request_t req; + zar_dirent_t ent; + int err, i = 0; + if (zar_dir_begin(z, path) != ZAR_OK) + return -1; + for (;;) + { + while ((err = zar_dir_step(z, &req, &ent)) == ZAR_PENDING) + zar_feed(z, pull(f, &req)); + if (err != ZAR_OK) + break; + if ( i >= n || strcmp(ent.name, names[i]) + || ent.len != strlen(names[i]) || !ent.is_dir != !dirs[i]) + return -1; + i++; + } + return err == ZAR_END ? i : -1; +} + +static void list_lane(void) +{ + static const char *const media[] = { "a.bin", "intro.wmv" }; + static const int mdirs[] = { 0, 0 }; + const char *root[FILLERS + 2]; + int rdirs[FILLERS + 2]; + char fill[FILLERS][16]; + file_t f; + zar_t *z; + zar_file_t zf; + int i, pass; + + for (i = 0; i < FILLERS; i++) + { + sprintf(fill[i], "file%02u.bin", (unsigned)i); + root[i] = fill[i]; + rdirs[i] = 0; + } + root[FILLERS] = "media"; + rdirs[FILLERS] = 1; + root[FILLERS + 1] = "default.xex"; + rdirs[FILLERS + 1] = 0; + + for (pass = 0; pass < 2; pass++) + { + make_archive(&f, -1); + f.shorten = pass == 1; + open_zar(&f, &z); + check(list(&f, z, "", root, rdirs, FILLERS + 2) == FILLERS + 2, + "the root, in order"); + check(list(&f, z, "\\MEDIA\\", media, mdirs, 2) == 2, + "a directory, in order"); + check( list(&f, z, "nope", media, mdirs, 2) == -1 + && list(&f, z, "default.xex", media, mdirs, 2) == -1, + "a missing directory and a file are not listed"); + /* a lookup after a listing */ + check( lookup(&f, z, "media/a.bin", &zf) == ZAR_OK + && zf.size == 5, "lookup after a listing"); + zar_free(z); + free(f.data); + } +} + static void read_lane(void) { file_t f; @@ -433,6 +501,7 @@ static void broken_lanes(void) int main(void) { lookup_lane(); + list_lane(); read_lane(); broken_lanes(); free(stream); diff --git a/libretro-db/query.c b/libretro-db/query.c index 36e2037dae..6a8d60b222 100644 --- a/libretro-db/query.c +++ b/libretro-db/query.c @@ -303,6 +303,21 @@ static struct rmsgpack_dom_value query_func_glob( input.val.string.buff, 0 ) == 0; + /* A field kept as binary, as a serial is: matched as text when it + * holds no NUL and is short enough to terminate here */ + else if (input.type == RDT_BINARY && input.val.binary.buff) + { + char text[256]; + uint32_t len = input.val.binary.len; + if ( len < sizeof(text) + && !memchr(input.val.binary.buff, '\0', len)) + { + memcpy(text, input.val.binary.buff, len); + text[len] = '\0'; + res.val.bool_ = rl_fnmatch( + argv[0].a.value.val.string.buff, text, 0) == 0; + } + } return res; } diff --git a/msg_hash.h b/msg_hash.h index de10f0023d..eb9811f62b 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -251,6 +251,8 @@ enum msg_file_type FILE_TYPE_XEX, FILE_TYPE_XCP, FILE_TYPE_ZAR, + FILE_TYPE_WUD, + FILE_TYPE_WUA, FILE_TYPE_DIRECT_LOAD, diff --git a/samples/tasks/database/Makefile b/samples/tasks/database/Makefile index 8f277957f5..88f606b8d3 100644 --- a/samples/tasks/database/Makefile +++ b/samples/tasks/database/Makefile @@ -105,6 +105,7 @@ SOURCES_C := \ $(LIBRETRO_COMM_DIR)/formats/ciso/ciso.c \ $(LIBRETRO_COMM_DIR)/formats/xdvdfs/xdvdfs.c \ $(LIBRETRO_COMM_DIR)/formats/zar/zar.c \ + $(LIBRETRO_COMM_DIR)/formats/wux/wux.c \ $(LIBRETRO_COMM_DIR)/formats/m3u/rm3u.c \ $(LIBRETRO_COMM_DIR)/formats/logiqx_dat/logiqx_dat.c \ $(CORE_DIR)/libretro-db/bintree.c \ diff --git a/samples/tasks/database/database_scan_test.c b/samples/tasks/database/database_scan_test.c index d7c238cdb5..ed124f857d 100644 --- a/samples/tasks/database/database_scan_test.c +++ b/samples/tasks/database/database_scan_test.c @@ -2698,6 +2698,252 @@ end: free(d); } +/* A Wii U disc of @sectors 32 KiB sectors whose header says @code; + * with @wux, as a WUX that keeps the empty sectors once */ +static int write_wiiu_disc(const char *path, const char *code, + unsigned sectors, bool wux) +{ + uint8_t *d = (uint8_t*)calloc(sectors + 1, 0x8000); + size_t n = (size_t)sectors * 0x8000; + int ok = 0; + unsigned i; + + if (!d) + return 0; + memcpy(d, "WUP-P-", 6); + memcpy(d + 6, code, 4); + memcpy(d + 10, "-0-00", 5); + if (wux) + { + /* the header, the index (sector 0, then the empty one for the + * rest), padding to a sector, then the two sectors */ + uint8_t *w = (uint8_t*)calloc(3, 0x8000); + if (!w) + { + free(d); + return 0; + } + memcpy(w, "WUX0", 4); + w[4] = 0x2E; w[5] = 0xD0; w[6] = 0x99; w[7] = 0x10; + w[9] = 0x80; + w[16] = (uint8_t)n; w[17] = (uint8_t)(n >> 8); + w[18] = (uint8_t)(n >> 16); w[19] = (uint8_t)(n >> 24); + for (i = 1; i < sectors; i++) + w[32 + i * 4] = 1; + memcpy(w + 0x8000, d, 0x8000); + ok = write_bytes(path, w, 3 * 0x8000); + free(w); + } + else + ok = write_bytes(path, d, n); + free(d); + return ok; +} + +/* A Wii U archive of two titles - an update listed first, then the + * base game - each with meta/meta.xml; the update's names @upd_code, + * the game's @code and @company. Its one block is kept as it is. */ +static int write_wua(const char *path, const char *code, + const char *company, const char *upd_code) +{ + static const char *const dirs[2] = { + "0005000E101C9500_v32", "00050000101C9500_v0" }; + uint8_t *out = (uint8_t*)calloc(1, 65536 + 4096); + uint8_t tree[7 * 16]; + uint8_t names[128]; + uint32_t name_at[7]; + size_t nsize = 0; + size_t xml_len[2]; + size_t at; + uint8_t *ft; + int ok = 0; + unsigned i; + + if (!out) + return 0; + /* the stream: the update's meta.xml, then the game's at 2048 */ + for (i = 0; i < 2; i++) + xml_len[i] = (size_t)sprintf((char*)out + i * 2048, + "\n" + "\n" + " 33\n" + " WUP-%c-%s" + "\n" + " WUP" + "\n" + " %s" + "\n\n", + i ? 'P' : 'N', i ? code : upd_code, i ? company : "0001"); + + /* names: the two titles, "meta" twice, "meta.xml" twice */ + name_at[0] = 0x7FFFFFFFu; + for (i = 1; i < 7; i++) + { + const char *nm = (i < 3) ? dirs[i - 1] : (i < 5) ? "meta" : "meta.xml"; + size_t l = strlen(nm); + name_at[i] = (uint32_t)nsize; + names[nsize++] = (uint8_t)l; + memcpy(names + nsize, nm, l); + nsize += l; + } + /* root: the titles; each title: its meta; each meta: its meta.xml */ + memset(tree, 0, sizeof(tree)); + for (i = 0; i < 7; i++) + xbox_put_be32(tree + i * 16, name_at[i] | (i >= 5 ? 0x80000000u : 0)); + xbox_put_be32(tree + 0 * 16 + 4, 1); xbox_put_be32(tree + 0 * 16 + 8, 2); + xbox_put_be32(tree + 1 * 16 + 4, 3); xbox_put_be32(tree + 1 * 16 + 8, 1); + xbox_put_be32(tree + 2 * 16 + 4, 4); xbox_put_be32(tree + 2 * 16 + 8, 1); + xbox_put_be32(tree + 3 * 16 + 4, 5); xbox_put_be32(tree + 3 * 16 + 8, 1); + xbox_put_be32(tree + 4 * 16 + 4, 6); xbox_put_be32(tree + 4 * 16 + 8, 1); + xbox_put_be32(tree + 5 * 16 + 4, 0); + xbox_put_be32(tree + 5 * 16 + 8, (uint32_t)xml_len[0]); + xbox_put_be32(tree + 6 * 16 + 4, 2048); + xbox_put_be32(tree + 6 * 16 + 8, (uint32_t)xml_len[1]); + + /* the block, the record, names, tree, two empty sections, footer */ + at = 65536; + memset(out + at, 0, 40); + out[at + 8] = 0xFF; + out[at + 9] = 0xFF; + memcpy(out + at + 40, names, nsize); + memcpy(out + at + 40 + nsize, tree, sizeof(tree)); + ft = out + at + 40 + nsize + sizeof(tree); + memset(ft, 0, 144); + xbox_put_be32(ft + 12, 65536); + xbox_put_be32(ft + 20, (uint32_t)at); + xbox_put_be32(ft + 28, 40); + xbox_put_be32(ft + 36, (uint32_t)(at + 40)); + xbox_put_be32(ft + 44, (uint32_t)nsize); + xbox_put_be32(ft + 52, (uint32_t)(at + 40 + nsize)); + xbox_put_be32(ft + 60, sizeof(tree)); + xbox_put_be32(ft + 68, (uint32_t)(ft - out)); + xbox_put_be32(ft + 84, (uint32_t)(ft - out)); + xbox_put_be32(ft + 132, (uint32_t)(ft - out + 144)); + xbox_put_be32(ft + 136, 0x61BF3A01u); + xbox_put_be32(ft + 140, 0x169F52D6u); + ok = write_bytes(path, out, (size_t)(ft - out + 144)); + free(out); + return ok; +} + +/* Wii U content takes the serial lookup by its product code, in the + * Wii U database alone: a release's serial adds its maker's two + * characters, or nothing for a title sold as a download. A WUD, a WUX + * and a Wii U disc named .iso match a disc release; an archive matches + * the release of the maker its meta.xml names, and the download when + * there is none, reading its base title rather than the update listed + * before it. Another database holding the code alone is not + * consulted. */ +static void wiiu_lane(const char *root, const char *pl_dir, + const char *db_dir) +{ + static const db_rec_t wiiu_recs[] = { + { "Kart Download (USA)", "AMKE", 0, 0, NULL, true }, + { "Kart Disc (USA)", "AMKE01", 0, 0, NULL, true }, + { "Kart Other (USA)", "AMKEXY", 0, 0, NULL, true }, + { "Kart Long (USA)", "AMKE01Z", 0, 0, NULL, true }, + { "Pikmin Disc (Europe)", "AC3P01", 0, 0, NULL, true }, + { "Zelda Disc (USA)", "ALZE01", 0, 0, NULL, true }, + { "Splat Disc (USA)", "WSPE01", 0, 0, NULL, true }, + { "Splat Download (USA)", "WSPE", 0, 0, NULL, true }, + { "Mii Other (Japan)", "WMIJ01", 0, 0, NULL, true }, + { "Mii Maker (Japan)", "WMIJ8P", 0, 0, NULL, true }, + { "Update Trap (USA)", "UPDE", 0, 0, NULL, true } + }; + static const db_rec_t decoy_recs[] = { + { "Decoy Pikmin", "AC3P", 0, 0, NULL, true }, + { "Decoy Zelda", "ALZE", 0, 0, NULL, true }, + { "Decoy Kart", "AMKE", 0, 0, NULL, true } + }; + char *w_db = (char*)malloc(512); + char *w_in = (char*)malloc(512); + char *p = (char*)malloc(1024); + bool started; + + if (!w_db || !w_in || !p) + { + check(0, "wiiu lane", "out of memory"); + goto end; + } + + sprintf(w_db, "%s/scan_db_wiiu", root); + sprintf(w_in, "%s/scan_wiiu", root); + path_mkdir(w_db); + path_mkdir(w_in); + + sprintf(p, "%s/Nintendo - Wii U.rdb", w_db); + if (!write_db_set(p, wiiu_recs, ARRAY_SIZE(wiiu_recs))) + { check(0, "fixture", "could not write database"); goto end; } + sprintf(p, "%s/Test Decoy.rdb", w_db); + if (!write_db_set(p, decoy_recs, ARRAY_SIZE(decoy_recs))) + { check(0, "fixture", "could not write database"); goto end; } + + sprintf(p, "%s/80_kart.wud", w_in); + if (!write_wiiu_disc(p, "AMKE", 4, false)) + { check(0, "fixture", "could not write image"); goto end; } + sprintf(p, "%s/81_pikmin.wux", w_in); + if (!write_wiiu_disc(p, "AC3P", 4, true)) + { check(0, "fixture", "could not write image"); goto end; } + sprintf(p, "%s/82_zelda.iso", w_in); + if (!write_wiiu_disc(p, "ALZE", 4, false)) + { check(0, "fixture", "could not write image"); goto end; } + sprintf(p, "%s/83_splat.wua", w_in); + if (!write_wua(p, "WSPE", "0099", "UPDE")) + { check(0, "fixture", "could not write archive"); goto end; } + sprintf(p, "%s/84_mii.wua", w_in); + if (!write_wua(p, "WMIJ", "008P", "UPDE")) + { check(0, "fixture", "could not write archive"); goto end; } + + strlcpy(config_get_ptr()->paths.path_content_database, w_db, + sizeof(config_get_ptr()->paths.path_content_database)); + started = run_scan(w_in); + strlcpy(config_get_ptr()->paths.path_content_database, db_dir, + sizeof(config_get_ptr()->paths.path_content_database)); + if (!started) + { + check(0, "wiiu scan started", "task_push_dbscan refused"); + goto end; + } + + check(scan_completed, "wiiu scan completed", + scan_completed ? "callback fired" : "timed out"); + sprintf(p, "%s/Nintendo - Wii U.lpl", pl_dir); + check( file_contains(p, "80_kart.wud") + && file_contains(p, "Kart Disc (USA)") + && file_contains(p, "AMKE01|serial") + && !file_contains(p, "Kart Download") + && !file_contains(p, "Kart Other") + && !file_contains(p, "Kart Long"), + "a WUD matches a disc release of its product code", + "80_kart.wud"); + check( file_contains(p, "81_pikmin.wux") + && file_contains(p, "Pikmin Disc (Europe)"), + "a WUX is read through its index", "81_pikmin.wux"); + check( file_contains(p, "82_zelda.iso") + && file_contains(p, "Zelda Disc (USA)"), + "a Wii U disc named .iso", "82_zelda.iso"); + check( file_contains(p, "83_splat.wua") + && file_contains(p, "Splat Download (USA)") + && !file_contains(p, "Splat Disc"), + "an archive whose maker has no release takes the download", + "83_splat.wua"); + check( file_contains(p, "84_mii.wua") + && file_contains(p, "Mii Maker (Japan)") + && !file_contains(p, "Mii Other"), + "an archive's maker picks the release", "84_mii.wua"); + check(!file_contains(p, "Update Trap"), + "an archive's base title, not its update", "UPDE"); + sprintf(p, "%s/Test Decoy.lpl", pl_dir); + check(!path_is_valid(p) || !file_contains(p, "Decoy"), + "a product code is looked up in the Wii U database alone", + "Test Decoy.lpl"); + +end: + free(w_db); + free(w_in); + free(p); +} + /* The folder a playlist refreshes from, its default core and * whether it recurses. False when the playlist holds no scan record. */ static bool scan_record(const char *lpl, char *dir, char *core, @@ -11714,6 +11960,20 @@ int main(int argc, char **argv) } sprintf(p, "%s/x360_libretro.so", core_dir); { FILE *f = fopen(p, "wb"); if (f) { fputs("\177ELF", f); fclose(f); } } + sprintf(p, "%s/wiiu_libretro.info", info_dir); + { + FILE *f = fopen(p, "w"); + if (!f) + { check(0, "fixture", "could not write core info"); return 1; } + fprintf(f, + "display_name = \"Wii U Test\"\n" + "corename = \"WiiUTest\"\n" + "supported_extensions = \"wud|wux|wua|iso\"\n" + "database = \"Nintendo - Wii U\"\n"); + fclose(f); + } + sprintf(p, "%s/wiiu_libretro.so", core_dir); + { FILE *f = fopen(p, "wb"); if (f) { fputs("\177ELF", f); fclose(f); } } sprintf(p, "%s/atari_libretro.info", info_dir); { FILE *f = fopen(p, "w"); @@ -12635,6 +12895,7 @@ int main(int argc, char **argv) ciso_lane(root, pl_dir, db_dir); xbox_lane(root, pl_dir, db_dir); x360_lane(root, pl_dir, db_dir); + wiiu_lane(root, pl_dir, db_dir); single_lane(root, pl_dir, crc_a, sz_a); counts_lane(root, pl_dir, crc_a, sz_a, crc_d, sz_d); failed_lane(root, pl_dir, db_dir, crc_a, sz_a); diff --git a/tasks/task_database.c b/tasks/task_database.c index a57ff2cf5f..053ce293ae 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -350,6 +350,11 @@ typedef struct database_state_handle /* The XDVDFS_REGION_* an Xbox title's executable says it is for, * or 0: picks among the releases its serial is shared by. */ uint8_t regions; + /* A Wii U serial is a product code, which a release's serial may + * follow with its maker's two characters: those, when the content + * says them, pick among the releases the code is shared by. */ + bool wiiu; + char wiiu_maker[3]; bool crc_done; /* The index build in flight, run by task_database_index_step() * until index_ready, then taken by the lookup that began it. */ @@ -1220,6 +1225,11 @@ static enum msg_file_type extension_to_file_type(const char *ext) return FILE_TYPE_XCP; if (string_is_equal(ext_lower, "zar")) return FILE_TYPE_ZAR; + if ( string_is_equal(ext_lower, "wud") + || string_is_equal(ext_lower, "wux")) + return FILE_TYPE_WUD; + if (string_is_equal(ext_lower, "wua")) + return FILE_TYPE_WUA; if (string_is_equal(ext_lower, "pbp")) return FILE_TYPE_PBP; if (string_is_equal(ext_lower, "lutro")) @@ -2130,7 +2140,9 @@ static int task_database_iterate_playlist( database_state_handle_t *db_state, database_info_handle_t *db, const char *name) { - db_state->regions = 0; + db_state->regions = 0; + db_state->wiiu = false; + db_state->wiiu_maker[0] = '\0'; switch (extension_to_file_type(path_get_extension(name))) { case FILE_TYPE_COMPRESSED: @@ -2187,18 +2199,36 @@ static int task_database_iterate_playlist( case FILE_TYPE_ISO: db_state->serial[0] = '\0'; db->type = DATABASE_TYPE_SERIAL_LOOKUP_SIZEHINT; - /* An Xbox disc is looked for only when nothing else is found, - * so other discs read nothing more for it. Its size says - * nothing: an XISO is trimmed. */ - if ( !intfstream_file_get_serial(name, 0, INT64_MAX, db_state->serial, sizeof(db_state->serial),&db_state->size) - && task_database_xbox_get_serial(name, true, db_state->serial, sizeof(db_state->serial),&db_state->size,&db_state->regions)) - db->type = DATABASE_TYPE_SERIAL_LOOKUP; + /* A Wii U disc, then an Xbox one, is looked for only when + * nothing else is found, so other discs read nothing more for + * them. The size says nothing: an XISO is trimmed, and a Wii U + * database has no sizes. */ + if (!intfstream_file_get_serial(name, 0, INT64_MAX, db_state->serial, sizeof(db_state->serial),&db_state->size)) + { + if (task_database_wiiu_get_serial(name, db_state->serial, sizeof(db_state->serial),&db_state->size)) + { + db_state->wiiu = true; + db->type = DATABASE_TYPE_SERIAL_LOOKUP; + } + else if (task_database_xbox_get_serial(name, true, db_state->serial, sizeof(db_state->serial),&db_state->size,&db_state->regions)) + db->type = DATABASE_TYPE_SERIAL_LOOKUP; + } break; case FILE_TYPE_XISO: db_state->serial[0] = '\0'; task_database_xbox_get_serial(name, true, db_state->serial, sizeof(db_state->serial),&db_state->size,&db_state->regions); db->type = DATABASE_TYPE_SERIAL_LOOKUP; break; + case FILE_TYPE_WUD: + db_state->serial[0] = '\0'; + db_state->wiiu = task_database_wiiu_get_serial(name, db_state->serial, sizeof(db_state->serial),&db_state->size) != 0; + db->type = DATABASE_TYPE_SERIAL_LOOKUP; + break; + case FILE_TYPE_WUA: + db_state->serial[0] = '\0'; + db_state->wiiu = task_database_wua_get_serial(name, db_state->serial, sizeof(db_state->serial),&db_state->size,db_state->wiiu_maker) != 0; + db->type = DATABASE_TYPE_SERIAL_LOOKUP; + break; case FILE_TYPE_ZAR: db_state->serial[0] = '\0'; task_database_zar_get_serial(name, db_state->serial, sizeof(db_state->serial),&db_state->size,&db_state->regions); @@ -3618,6 +3648,48 @@ static uint8_t task_database_xbox_regions(const char *list) return regions; } +/* The database a Wii U product code is looked up in */ +static bool task_database_is_wiiu_db(const char *rdb) +{ + return rdb && string_is_equal(path_basename(rdb), "Nintendo - Wii U.rdb"); +} + +/* Of the records a Wii U product code's stem query found, the one the + * content is: the maker's release when the content names its maker; + * otherwise, for a game from an archive, the record of the code alone, + * as a title sold as a download is; for a disc, a disc release, whose + * serial adds its maker. Returns the record's index, or the count when + * none is a release of the code. */ +static size_t task_database_wiiu_pick(const database_info_list_t *info, + const char *code, const char *maker) +{ + size_t i; + size_t pick = info->count; + size_t code_len = strlen(code); + int best = 3; + for (i = 0; i < info->count && best; i++) + { + const char *serial = info->list[i].serial; + size_t len; + int rank = 2; + if (!serial || strncmp(serial, code, code_len)) + continue; + len = strlen(serial); + if (len != code_len && len != code_len + 2) + continue; + if (*maker && len == code_len + 2 && !strcmp(serial + code_len, maker)) + rank = 0; + else if (len == (*maker ? code_len : code_len + 2)) + rank = 1; + if (rank < best) + { + best = rank; + pick = i; + } + } + return pick; +} + static int task_database_iterate_serial_lookup( manual_scan_handle_t *_db, database_state_handle_t *db_state, @@ -3668,6 +3740,12 @@ static int task_database_iterate_serial_lookup( path_contains_compressed_file)) return SCAN_VERDICT_CONTINUE; + /* A Wii U product code means nothing to another database */ + if ( db_state->wiiu + && !task_database_is_wiiu_db( + db_state->list->elems[db_state->list_index].data)) + return database_info_list_iterate_next(db_state); + /* If size boundaries are not filled for this DB, run the queries */ if ( !(db_state->flags[db_state->list_index] & DB_STATE_FLAG_SIZE_CHECKED) && !task_database_fill_db_min_max(db_state, &si)) @@ -3752,7 +3830,10 @@ static int task_database_iterate_serial_lookup( db_state->serial_index[db_state->list_index] = si; } - if (db_state->serial_index[db_state->list_index]) + /* the index answers whole serials, not a product code's + * stem */ + if ( db_state->serial_index[db_state->list_index] + && !db_state->wiiu) hits = database_info_list_new_serial( db_state->serial_index[db_state->list_index], rdb, db_state->serial, DB_EXTRACT_SCAN_FIELDS @@ -3770,6 +3851,23 @@ static int task_database_iterate_serial_lookup( goto serial_query_done; } + /* A Wii U product code, four characters the probe checked: + * the releases that are it, with a maker or without */ + if (db_state->wiiu) + { + static const char head[] = "{'serial':glob('"; + static const char tail[] = "*')}"; + size_t code_len = strlen(db_state->serial); + if (code_len + sizeof(head) + sizeof(tail) > sizeof(query_buf)) + return database_info_list_iterate_next(db_state); + memcpy(query_buf, head, sizeof(head) - 1); + memcpy(query_buf + sizeof(head) - 1, db_state->serial, code_len); + memcpy(query_buf + sizeof(head) - 1 + code_len, tail, + sizeof(tail)); + database_info_list_iterate_new(db_state, query_buf); + goto serial_query_done; + } + serial_buf = bin_to_hex_alloc( (uint8_t*)db_state->serial, strlen(db_state->serial) * sizeof(uint8_t)); @@ -3852,6 +3950,20 @@ serial_query_done: } } } + if (db_state->info && db_state->wiiu && !db_state->entry_index) + { + size_t i = task_database_wiiu_pick(db_state->info, db_state->serial, + db_state->wiiu_maker); + if (i < db_state->info->count) + { + /* the playlist names the release found */ + strlcpy(db_state->serial, db_state->info->list[i].serial, + sizeof(db_state->serial)); + db_state->entry_index = i; + return database_info_list_iterate_found_match(_db, + db_state, db, NULL); + } + } if (db_state->info) { /* A 3DS product code is shared by a title's revisions and diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index 13fa10b25a..c3b67bbb8f 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "task_database_cue.h" @@ -2513,6 +2514,223 @@ end: return _len != 0; } +/* A Wii U product code's four characters, from "WUP-?-" and them; + * @code gets them, terminated */ +static bool wiiu_product_code(const uint8_t *p, size_t n, char *code) +{ + size_t i; + if ( n < 10 + || p[0] != 'W' || p[1] != 'U' || p[2] != 'P' || p[3] != '-' + || p[4] < 'A' || p[4] > 'Z' || p[5] != '-') + return false; + for (i = 0; i < 4; i++) + { + uint8_t c = p[6 + i]; + if (!((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'))) + return false; + code[i] = (char)c; + } + code[4] = '\0'; + return true; +} + +/* The disc header a Wii U product code is read from */ +#define WIIU_HEAD 10 + +int task_database_wiiu_get_serial(const char *name, char *s, size_t len, + uint64_t *filesize) +{ + task_database_src_t src; + wux_request_t req; + uint8_t head[WIIU_HEAD]; + char code[5]; + size_t got; + int err; + int rv = 0; + int64_t size; + wux_t *w = NULL; + /* The descriptor path: the few octets the probe needs land where + * it asks */ + intfstream_t *fd = intfstream_open_file(name, + RETRO_VFS_FILE_ACCESS_READ, + RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK); + + if (!fd) + return 0; + size = intfstream_get_size(fd); + *filesize = (uint64_t)size; + src.fd = fd; + src.pos = 0; + if (size < WIIU_HEAD) + goto end; + for (got = 0; got < WIIU_HEAD; ) + { + size_t n = task_database_pull(&src, got, head + got, WIIU_HEAD - got); + if (!n) + goto end; + got += n; + } + + /* A WUX: the same octets of the disc it stands for */ + if (wux_is_wux(head)) + { + if (!(w = wux_new((uint64_t)size))) + goto end; + while ((err = wux_open_step(w, &req)) == WUX_PENDING) + wux_feed(w, task_database_pull(&src, req.offset, req.dst, + req.length)); + if ( err != WUX_OK + || wux_read_begin(w, 0, head, WIIU_HEAD) != WUX_OK) + goto end; + while ((err = wux_read_step(w, &req)) == WUX_PENDING) + wux_feed(w, task_database_pull(&src, req.offset, req.dst, + req.length)); + if (err != WUX_OK || wux_read_progress(w) != WIIU_HEAD) + goto end; + } + + /* a disc says "WUP-P-" */ + if (head[4] == 'P' && wiiu_product_code(head, WIIU_HEAD, code)) + rv = strlcpy(s, code, len) < len; + +end: + wux_free(w); + intfstream_close(fd); + free(fd); + return rv; +} + +/* The text of the first <@tag> element of @xml, which is terminated; + * NULL when there is none. *@n gets its length. */ +static const char *wiiu_xml_value(const char *xml, const char *tag, + size_t *n) +{ + size_t tlen = strlen(tag); + const char *p = xml; + while ((p = strchr(p, '<'))) + { + const char *v; + p++; + if (strncmp(p, tag, tlen) || (p[tlen] != '>' && p[tlen] != ' ')) + continue; + if (!(v = strchr(p + tlen, '>'))) + return NULL; + v++; + for (*n = 0; v[*n] && v[*n] != '<'; (*n)++) { } + return v[*n] ? v : NULL; + } + return NULL; +} + +/* meta.xml's head, which names the product and company first */ +#define WIIU_META_HEAD 2048 + +int task_database_wua_get_serial(const char *name, char *s, size_t len, + uint64_t *filesize, char *maker) +{ + task_database_zar_src_t zs; + zar_request_t req; + zar_dirent_t ent; + char path[64]; + char code[5]; + char *xml = NULL; + const char *v; + size_t n; + int err; + int rv = 0; + int64_t size; + zar_t *z = NULL; + /* The descriptor path: each range lands where the reader asks */ + intfstream_t *fd = intfstream_open_file(name, + RETRO_VFS_FILE_ACCESS_READ, + RETRO_VFS_FILE_ACCESS_HINT_SEQUENTIAL_BULK); + + if (!fd) + return 0; + maker[0] = '\0'; + path[0] = '\0'; + size = intfstream_get_size(fd); + *filesize = (uint64_t)size; + zs.src.fd = fd; + zs.src.pos = 0; + if (size <= 0 || !(z = zar_new((uint64_t)size))) + goto end; + zs.z = z; + + while ((err = zar_open_step(z, &req)) == ZAR_PENDING) + zar_feed(z, task_database_pull(&zs.src, req.offset, req.dst, + req.length)); + if (err != ZAR_OK || zar_dir_begin(z, "") != ZAR_OK) + goto end; + + /* A title's directory is its ID, 16 hex digits, then "_v" and its + * version. The first base title (00050000...) is the game; any + * other title stands in when there is none, as an update or the + * content of one names the same product. */ + for (;;) + { + size_t i; + while ((err = zar_dir_step(z, &req, &ent)) == ZAR_PENDING) + zar_feed(z, task_database_pull(&zs.src, req.offset, req.dst, + req.length)); + if (err != ZAR_OK) + break; + if ( !ent.is_dir || ent.len < 18 || ent.len > 40 + || ent.name[16] != '_' || (ent.name[17] | 0x20) != 'v') + continue; + for (i = 0; i < 16; i++) + { + char c = ent.name[i]; + if (!( (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') + || (c >= 'a' && c <= 'f'))) + break; + } + if (i < 16) + continue; + if (!path[0] || !strncmp(ent.name, "00050000", 8)) + { + memcpy(path, ent.name, ent.len); + path[ent.len] = '\0'; + if (!strncmp(ent.name, "00050000", 8)) + break; + } + } + if (!path[0]) + goto end; + strlcat(path, "/meta/meta.xml", sizeof(path)); + + if (zar_lookup_begin(z, path) != ZAR_OK) + goto end; + while ((err = zar_lookup_step(z, &req, &zs.file)) == ZAR_PENDING) + zar_feed(z, task_database_pull(&zs.src, req.offset, req.dst, + req.length)); + if (err != ZAR_OK || !(xml = (char*)malloc(WIIU_META_HEAD + 1))) + goto end; + n = task_database_zar_pull(&zs, 0, (uint8_t*)xml, WIIU_META_HEAD); + xml[n] = '\0'; + + if ( !(v = wiiu_xml_value(xml, "product_code", &n)) + || !wiiu_product_code((const uint8_t*)v, n, code)) + goto end; + /* the maker: the company code's last two characters */ + if ( (v = wiiu_xml_value(xml, "company_code", &n)) && n >= 2 + && ISALNUM((unsigned char)v[n - 2]) + && ISALNUM((unsigned char)v[n - 1])) + { + maker[0] = (char)TOUPPER((unsigned char)v[n - 2]); + maker[1] = (char)TOUPPER((unsigned char)v[n - 1]); + maker[2] = '\0'; + } + rv = strlcpy(s, code, len) < len; + +end: + free(xml); + zar_free(z); + intfstream_close(fd); + free(fd); + return rv; +} + /* Opens what the scanner hashes for @name: the file itself, the data * track a CUE or GDI sheet names, or a CHD's data track. The stream * comes back at the start of the span, *span is its length or -1 to diff --git a/tasks/task_database_cue.h b/tasks/task_database_cue.h index f1f1d7c20f..119803fb37 100644 --- a/tasks/task_database_cue.h +++ b/tasks/task_database_cue.h @@ -80,6 +80,15 @@ int task_database_xbox_get_serial(const char *name, bool disc, char *s, /* The Xbox 360 serial of the game a ZArchive holds, @regions as above */ int task_database_zar_get_serial(const char *name, char *s, size_t len, uint64_t *filesize, uint8_t *regions); +/* The product code of a Wii U disc, a WUD or a WUX (or a disc image + * by another name), four characters: a release's serial may add the + * maker's two */ +int task_database_wiiu_get_serial(const char *name, char *s, size_t len, + uint64_t *filesize); +/* The same for the game a Wii U archive (.wua) holds, @maker getting + * the maker's two characters from its company code, or "" */ +int task_database_wua_get_serial(const char *name, char *s, size_t len, + uint64_t *filesize, char *maker); bool is_chd_file_cdi(const char *path); /* @regions as for task_database_xbox_get_serial(), for an Xbox disc */ int task_database_chd_get_serial(const char *name, char *serial, size_t len,