From c712f9898242f23fc686ec03f2e944d41dd973af Mon Sep 17 00:00:00 2001 From: LibretroAdmin Date: Tue, 27 Oct 2026 06:51:15 +0100 Subject: [PATCH] rmsgpack: read a value from a window the stream lends rmsgpack_read() takes each field from a window the stream lends with the new intfstream_view(), and reads only what the window does not hold. Buffered streams lend their window and file streams lend what filestream_view() lends. The stream is left where the value ends. Reading every entry of a 150,000-entry database through a cursor goes from 1.31 to 1.61 million entries a second, with the same values. The cursor is what the Explore menu and database lookups read through. rmsgpack_window_test writes values of every shape: integers of every width and sign, and strings and binaries up to past the reader's window. It reads them back through buffered streams with 4 KiB and 64-octet windows, a file stream mapped and not, and a memory stream, which lends nothing. Every value must match and the position must be right after each. It passes under ASan and UBSan. --- .../workflows/Linux-libretro-db-samples.yml | 1 + .../include/streams/interface_stream.h | 14 + libretro-common/streams/interface_stream.c | 36 ++ libretro-db/.gitignore | 2 + libretro-db/rmsgpack.c | 395 +++++++++--------- libretro-db/samples/rmsgpack/Makefile | 16 +- .../samples/rmsgpack/rmsgpack_window_test.c | 231 ++++++++++ 7 files changed, 487 insertions(+), 208 deletions(-) create mode 100644 libretro-db/samples/rmsgpack/rmsgpack_window_test.c diff --git a/.github/workflows/Linux-libretro-db-samples.yml b/.github/workflows/Linux-libretro-db-samples.yml index 24c180b502..4d90e54c57 100644 --- a/.github/workflows/Linux-libretro-db-samples.yml +++ b/.github/workflows/Linux-libretro-db-samples.yml @@ -93,6 +93,7 @@ jobs: # failure. These are built AND executed. declare -a RUN_TARGETS=( rmsgpack_overflow_test + rmsgpack_window_test libretrodb_leak_test ) diff --git a/libretro-common/include/streams/interface_stream.h b/libretro-common/include/streams/interface_stream.h index 8c540350c1..37c91fe604 100644 --- a/libretro-common/include/streams/interface_stream.h +++ b/libretro-common/include/streams/interface_stream.h @@ -98,6 +98,20 @@ void intfstream_rewind(intfstream_internal_t *intf); int64_t intfstream_tell(intfstream_internal_t *intf); +/** + * intfstream_view: + * + * intfstream_read() without the copy: points @ptr at up to @len + * octets at the stream position, where the stream holds them, and + * moves past them as a read of that many would. + * + * Returns: the octets lent, 0 at the end, or -1 when the stream holds + * nothing there it can lend; the caller then reads. Lent octets stay + * valid until the next call on @intf and must not be written. + */ +int64_t intfstream_view(intfstream_internal_t *intf, uint64_t len, + const uint8_t **ptr); + int intfstream_eof(intfstream_internal_t *intf); void intfstream_putc(intfstream_internal_t *intf, int c); diff --git a/libretro-common/streams/interface_stream.c b/libretro-common/streams/interface_stream.c index 80054c97c5..be3b75d5ff 100644 --- a/libretro-common/streams/interface_stream.c +++ b/libretro-common/streams/interface_stream.c @@ -483,6 +483,42 @@ int64_t intfstream_read(intfstream_internal_t *intf, void *s, uint64_t len) return -1; } +int64_t intfstream_view(intfstream_internal_t *intf, uint64_t len, + const uint8_t **ptr) +{ + if (!intf || !ptr) + return -1; + + switch (intf->type) + { + case INTFSTREAM_FILE: + return filestream_view(intf->file.fp, (int64_t)len, ptr); + case INTFSTREAM_BUFFERED: + { + uint64_t p = intf->buffered.pos; + uint64_t n = len; + if (p >= intf->buffered.size || !n) + return 0; + if (n > intf->buffered.size - p) + n = intf->buffered.size - p; + if (n > intf->buffered.cap) + n = intf->buffered.cap; + if ( !(p >= intf->buffered.lo && p < intf->buffered.hi) + && !intfstream_buffered_fill(intf, p, n)) + return -1; + if (n > intf->buffered.hi - p) + n = intf->buffered.hi - p; + *ptr = intf->buffered.buf + (p - intf->buffered.lo); + intf->buffered.pos = p + n; + return (int64_t)n; + } + default: + break; + } + + return -1; +} + int64_t intfstream_write(intfstream_internal_t *intf, const void *s, uint64_t len) { diff --git a/libretro-db/.gitignore b/libretro-db/.gitignore index 09ef21d176..a507ca7af8 100644 --- a/libretro-db/.gitignore +++ b/libretro-db/.gitignore @@ -2,3 +2,5 @@ /c_converter /libretrodb_tool /rmsgpack_test +/samples/rmsgpack/rmsgpack_overflow_test +/samples/rmsgpack/rmsgpack_window_test diff --git a/libretro-db/rmsgpack.c b/libretro-db/rmsgpack.c index eea9de7974..c806262eee 100644 --- a/libretro-db/rmsgpack.c +++ b/libretro-db/rmsgpack.c @@ -30,6 +30,8 @@ #include #include +#include +#include #include "rmsgpack.h" @@ -410,237 +412,211 @@ int rmsgpack_read_uint(intfstream_t *fd, uint64_t *s, size_t len) return 0; } -static int rmsgpack_read_int(intfstream_t *fd, int64_t *s, size_t len) +/* ---- reading a value ---- + * + * A value is read from a window the stream lends where it can + * (intfstream_view), so a field is taken from memory rather than + * through a read call of its own; what the window does not hold is + * read. The stream is left where the value ends. */ + +#define RMSGPACK_VIEW 65536 + +struct rmsgpack_src { - union { uint64_t u64; uint32_t u32; uint16_t u16; uint8_t u8; } tmp; + intfstream_t *fd; + const uint8_t *p; /* lent octets not yet taken */ + const uint8_t *end; + uint64_t pos; /* the stream position p stands for */ + uint64_t size; + bool lent; /* the stream is past pos */ + bool view; /* the stream lends at all */ +}; - tmp.u64 = 0; - if (rmsgpack_read_exact(fd, &tmp, len) == -1) - return -1; +static void rmsgpack_src_init(struct rmsgpack_src *s, intfstream_t *fd) +{ + int64_t pos = intfstream_tell(fd); + int64_t size = intfstream_get_size(fd); + s->fd = fd; + s->p = NULL; + s->end = NULL; + s->pos = pos > 0 ? (uint64_t)pos : 0; + /* a stream that does not know its size bounds nothing */ + s->size = size >= 0 ? (uint64_t)size : ~(uint64_t)0; + s->lent = false; + s->view = true; +} - switch (len) +/* Puts the stream back on the position */ +static void rmsgpack_src_done(struct rmsgpack_src *s) +{ + if (s->lent) + intfstream_seek(s->fd, (int64_t)s->pos, RETRO_VFS_SEEK_POSITION_START); + s->lent = false; + s->p = s->end = NULL; +} + +/* Whether @len octets at the position are lent */ +static bool rmsgpack_src_window(struct rmsgpack_src *s, size_t len) +{ + const uint8_t *ptr = NULL; + int64_t n; + if ((size_t)(s->end - s->p) >= len) + return true; + if (!s->view) + return false; + if ( s->lent + && intfstream_seek(s->fd, (int64_t)s->pos, + RETRO_VFS_SEEK_POSITION_START) < 0) + return false; + n = intfstream_view(s->fd, len > RMSGPACK_VIEW ? len : RMSGPACK_VIEW, &ptr); + s->lent = n > 0; + if (n < 0) + s->view = false; + if (n <= 0) { - case 1: - *s = (int8_t)tmp.u8; - break; - case 2: - *s = (int16_t)swap_if_little16(tmp.u16); - break; - case 4: - *s = (int32_t)swap_if_little32(tmp.u32); - break; - case 8: - *s = (int64_t)swap_if_little64(tmp.u64); - break; + s->p = s->end = NULL; + return false; } + s->p = ptr; + s->end = ptr + n; + return (size_t)n >= len; +} + +/* @len octets at the position into @dst, and past them */ +static int rmsgpack_src_read(struct rmsgpack_src *s, void *dst, size_t len) +{ + if (!len) + return 0; + if (rmsgpack_src_window(s, len)) + { + memcpy(dst, s->p, len); + s->p += len; + s->pos += len; + return 0; + } + /* more than a window: read where the position is */ + rmsgpack_src_done(s); + if (rmsgpack_read_exact(s->fd, dst, len) == -1) + return -1; + s->pos += len; return 0; } -static int rmsgpack_read_buff(intfstream_t *fd, size_t size, char **pbuff, uint64_t *len) +/* A big endian number of @len octets (1, 2, 4 or 8) */ +static int rmsgpack_src_uint(struct rmsgpack_src *s, uint64_t *v, size_t len) { - ssize_t read_len; - uint64_t tmp_len = 0; - int64_t remaining; - int64_t here; - int64_t total; + uint8_t tmp[8]; + const uint8_t *q = tmp; + size_t i; + if (rmsgpack_src_window(s, len)) + { + q = s->p; + s->p += len; + s->pos += len; + } + else if (rmsgpack_src_read(s, tmp, len) == -1) + return -1; + *v = 0; + for (i = 0; i < len; i++) + *v = (*v << 8) | q[i]; + return 0; +} - if (rmsgpack_read_uint(fd, &tmp_len, size) == -1) - return -1; +/* @len octets, the payload of a string or binary, in a fresh + * allocation ending in a 0 */ +/* the octets the stream has left */ +static uint64_t rmsgpack_src_left(const struct rmsgpack_src *s) +{ + return s->size > s->pos ? s->size - s->pos : 0; +} - /* Pre-patch tmp_len was an attacker-controlled uint64 fed - * directly into malloc((size_t)(tmp_len + 1)). Three - * problems: - * - tmp_len = UINT64_MAX wraps to malloc(0) returning a - * small block; the subsequent intfstream_read with - * (size_t)UINT64_MAX bytes would heap-overflow on any - * stream that could deliver them. - * - tmp_len = 0xFFFFFFFE forces a ~4 GiB malloc on 64-bit - * (which Linux overcommit happily grants) for a 5-byte - * STR32 input, OOM-killing the process. On 32-bit the - * (size_t) cast truncates and creates a heap overflow. - * - Even when malloc returns NULL the code dereferenced - * *pbuff at line 396 and (*pbuff)[read_len] at line 404. - * - * Fix: bound tmp_len against the remaining bytes in the - * stream (a buffer can't legitimately claim more bytes than - * are left to read), reject the SIZE_MAX edge to avoid the - * +1 wrap, and NULL-check the malloc. */ - here = intfstream_tell(fd); - total = intfstream_get_size(fd); - if (here < 0 || total < 0 || here > total) +static int rmsgpack_src_buff(struct rmsgpack_src *s, uint64_t len, char **pbuff) +{ + /* a payload claims no more than the stream has left */ + if (len > rmsgpack_src_left(s) || len >= (uint64_t)((size_t)-1)) return -1; - remaining = total - here; - if (tmp_len > (uint64_t)remaining) + if (!(*pbuff = (char*)malloc((size_t)len + 1))) return -1; - if (tmp_len >= (uint64_t)((size_t)-1)) - return -1; - - *pbuff = (char *)malloc((size_t)(tmp_len + 1) * sizeof(char)); - if (!*pbuff) - return -1; - - if (rmsgpack_read_exact(fd, *pbuff, (size_t)tmp_len) == -1) + if (rmsgpack_src_read(s, *pbuff, (size_t)len) == -1) { free(*pbuff); *pbuff = NULL; return -1; } - - read_len = (ssize_t)tmp_len; - *len = (uint64_t)read_len; - (*pbuff)[read_len] = 0; - + (*pbuff)[len] = 0; return 0; } -static int rmsgpack_read_depth(intfstream_t *fd, +static int rmsgpack_src_value(struct rmsgpack_src *s, struct rmsgpack_read_callbacks *callbacks, void *data, unsigned depth); -static int rmsgpack_read_map(intfstream_t *fd, uint32_t len, - struct rmsgpack_read_callbacks *callbacks, void *data, - unsigned depth) +/* A map or array of @len entries (@pairs: a map) */ +static int rmsgpack_src_items(struct rmsgpack_src *s, uint64_t len, + bool pairs, struct rmsgpack_read_callbacks *callbacks, void *data, + unsigned depth) { - int rv; - unsigned i; - int64_t here, total; + int rv; + uint64_t i; - /* Pre-patch len was an attacker-controlled uint32 from the - * file (MAP16 or MAP32 header) handed straight to the callback - * which calloc'd 'len * sizeof(rmsgpack_dom_pair)' (~80 bytes - * per pair). A 5-byte MAP32 header '0xdf 0x10 0x00 0x00 0x00' - * (len = 2^28) demanded a ~21 GiB calloc, OOM-killing the - * process. Even where calloc succeeded, the subsequent - * iteration recursed into rmsgpack_read len times, doubling - * the dom reader stack on each recursion until it ran out. - * - * A map cannot legitimately claim more entries than the - * stream has bytes left to encode them: the smallest possible - * key+value pair is 2 bytes (one type byte each). Reject - * len > remaining_bytes / 2. */ - here = intfstream_tell(fd); - total = intfstream_get_size(fd); - if (here >= 0 && total >= 0 && here <= total) + /* No container claims more entries than the stream has octets + * left for: an entry is one octet at least, a pair two. */ + if (len > rmsgpack_src_left(s) / (pairs ? 2u : 1u)) + return -1; + if (pairs) { - uint64_t remaining = (uint64_t)(total - here); - if ((uint64_t)len > remaining / 2u) - return -1; + if ( callbacks->read_map_start + && (rv = callbacks->read_map_start((uint32_t)len, data)) < 0) + return rv; } - - if ( ( callbacks->read_map_start) - && (rv = callbacks->read_map_start(len, data)) < 0) + else if (callbacks->read_array_start + && (rv = callbacks->read_array_start((uint32_t)len, data)) < 0) return rv; - for (i = 0; i < len; i++) - { - if ((rv = rmsgpack_read_depth(fd, callbacks, data, depth)) < 0) + for (i = 0; i < (pairs ? 2 * len : len); i++) + if ((rv = rmsgpack_src_value(s, callbacks, data, depth)) < 0) return rv; - if ((rv = rmsgpack_read_depth(fd, callbacks, data, depth)) < 0) - return rv; - } - return 0; } -static int rmsgpack_read_array(intfstream_t *fd, uint32_t len, +static int rmsgpack_src_value(struct rmsgpack_src *s, struct rmsgpack_read_callbacks *callbacks, void *data, unsigned depth) { - int rv; - unsigned i; - int64_t here, total; - - /* Same primitive as rmsgpack_read_map above. Smallest - * possible array element is 1 byte (a fixint), so len cannot - * legitimately exceed the number of bytes left in the - * stream. */ - here = intfstream_tell(fd); - total = intfstream_get_size(fd); - if (here >= 0 && total >= 0 && here <= total) - { - uint64_t remaining = (uint64_t)(total - here); - if ((uint64_t)len > remaining) - return -1; - } - - if ( ( callbacks->read_array_start) - && (rv = callbacks->read_array_start(len, data)) < 0) - return rv; - - for (i = 0; i < len; i++) - { - if ((rv = rmsgpack_read_depth(fd, callbacks, data, depth)) < 0) - return rv; - } - - return 0; -} - -int rmsgpack_read(intfstream_t *fd, - struct rmsgpack_read_callbacks *callbacks, void *data) -{ - return rmsgpack_read_depth(fd, callbacks, data, 0); -} - -static int rmsgpack_read_depth(intfstream_t *fd, - struct rmsgpack_read_callbacks *callbacks, void *data, - unsigned depth) -{ - int rv; uint64_t tmp_len = 0; uint64_t tmp_uint = 0; - int64_t tmp_int = 0; - uint8_t type = 0; - char *buff = NULL; + uint8_t type; + char *buff = NULL; if (depth >= RMSGPACK_MAX_DEPTH) return -1; depth++; - if (rmsgpack_read_exact(fd, &type, sizeof(uint8_t)) == -1) + if (s->p < s->end) + { + type = *s->p++; + s->pos++; + } + else if (rmsgpack_src_read(s, &type, 1) == -1) return -1; if (type < MPF_FIXMAP) - { - if (callbacks->read_int) - return callbacks->read_int(type, data); - return 0; - } + return callbacks->read_int ? callbacks->read_int(type, data) : 0; else if (type < MPF_FIXARRAY) - { - tmp_len = type - MPF_FIXMAP; - return rmsgpack_read_map(fd, (uint32_t)tmp_len, callbacks, data, depth); - } + return rmsgpack_src_items(s, type - MPF_FIXMAP, true, callbacks, data, depth); else if (type < MPF_FIXSTR) - { - tmp_len = type - MPF_FIXARRAY; - return rmsgpack_read_array(fd, (uint32_t)tmp_len, callbacks, data, depth); - } + return rmsgpack_src_items(s, type - MPF_FIXARRAY, false, callbacks, data, depth); else if (type < MPF_NIL) { - ssize_t _len = 0; - tmp_len = type - MPF_FIXSTR; - if (!(buff = (char *)malloc((size_t)(tmp_len + 1) * sizeof(char)))) + if (rmsgpack_src_buff(s, type - MPF_FIXSTR, &buff) == -1) return -1; - if (rmsgpack_read_exact(fd, buff, (size_t)tmp_len) == -1) - { - free(buff); - return -1; - } - _len = (ssize_t)tmp_len; - buff[_len] = '\0'; if (callbacks->read_string) - return callbacks->read_string(buff, (uint32_t)_len, data); - if (buff) - free(buff); + return callbacks->read_string(buff, (uint32_t)(type - MPF_FIXSTR), data); + free(buff); return 0; } else if (type > MPF_MAP32) - { - if (callbacks->read_int) - return callbacks->read_int(type - 0xff - 1, data); - return 0; - } + return callbacks->read_int ? callbacks->read_int(type - 0xff - 1, data) : 0; switch (type) { @@ -659,19 +635,26 @@ static int rmsgpack_read_depth(intfstream_t *fd, case _MPF_BIN8: case _MPF_BIN16: case _MPF_BIN32: - if ((rv = rmsgpack_read_buff(fd, (size_t)(1 << (type - _MPF_BIN8)), - &buff, &tmp_len)) < 0) - return rv; - if (callbacks->read_bin) + case _MPF_STR8: + case _MPF_STR16: + case _MPF_STR32: + { + bool str = type >= _MPF_STR8; + if ( rmsgpack_src_uint(s, &tmp_len, + (size_t)1 << (type - (str ? _MPF_STR8 : _MPF_BIN8))) == -1 + || rmsgpack_src_buff(s, tmp_len, &buff) == -1) + return -1; + if (str && callbacks->read_string) + return callbacks->read_string(buff, (uint32_t)tmp_len, data); + if (!str && callbacks->read_bin) return callbacks->read_bin(buff, (uint32_t)tmp_len, data); break; + } case _MPF_UINT8: case _MPF_UINT16: case _MPF_UINT32: case _MPF_UINT64: - tmp_len = UINT64_C(1) << (type - _MPF_UINT8); - tmp_uint = 0; - if (rmsgpack_read_uint(fd, &tmp_uint, (size_t)tmp_len) == -1) + if (rmsgpack_src_uint(s, &tmp_uint, (size_t)1 << (type - _MPF_UINT8)) == -1) return -1; if (callbacks->read_uint) return callbacks->read_uint(tmp_uint, data); @@ -680,38 +663,46 @@ static int rmsgpack_read_depth(intfstream_t *fd, case _MPF_INT16: case _MPF_INT32: case _MPF_INT64: - tmp_len = UINT64_C(1) << (type - _MPF_INT8); - tmp_int = 0; - if (rmsgpack_read_int(fd, &tmp_int, (size_t)tmp_len) == -1) + { + size_t len = (size_t)1 << (type - _MPF_INT8); + int64_t v; + if (rmsgpack_src_uint(s, &tmp_uint, len) == -1) return -1; + /* sign extended from its width */ + if (len < 8 && (tmp_uint >> (8 * len - 1)) & 1) + tmp_uint |= ~(uint64_t)0 << (8 * len); + v = (int64_t)tmp_uint; if (callbacks->read_int) - return callbacks->read_int(tmp_int, data); - break; - case _MPF_STR8: - case _MPF_STR16: - case _MPF_STR32: - if ((rv = rmsgpack_read_buff(fd, (size_t)(1 << (type - _MPF_STR8)), &buff, &tmp_len)) < 0) - return rv; - if (callbacks->read_string) - return callbacks->read_string(buff, (uint32_t)tmp_len, data); + return callbacks->read_int(v, data); break; + } case _MPF_ARRAY16: case _MPF_ARRAY32: - if (rmsgpack_read_uint(fd, &tmp_len, 2<<(type - _MPF_ARRAY16)) != -1) - return rmsgpack_read_array(fd, (uint32_t)tmp_len, callbacks, data, depth); - return -1; + if (rmsgpack_src_uint(s, &tmp_len, (size_t)2 << (type - _MPF_ARRAY16)) == -1) + return -1; + return rmsgpack_src_items(s, tmp_len, false, callbacks, data, depth); case _MPF_MAP16: case _MPF_MAP32: - if (rmsgpack_read_uint(fd, &tmp_len, 2<<(type - _MPF_MAP16)) != -1) - return rmsgpack_read_map(fd, (uint32_t)tmp_len, callbacks, data, depth); - return -1; + if (rmsgpack_src_uint(s, &tmp_len, (size_t)2 << (type - _MPF_MAP16)) == -1) + return -1; + return rmsgpack_src_items(s, tmp_len, true, callbacks, data, depth); } - if (buff) - free(buff); + free(buff); return 0; } +int rmsgpack_read(intfstream_t *fd, + struct rmsgpack_read_callbacks *callbacks, void *data) +{ + struct rmsgpack_src s; + int rv; + rmsgpack_src_init(&s, fd); + rv = rmsgpack_src_value(&s, callbacks, data, 0); + rmsgpack_src_done(&s); + return rv; +} + /** * rmsgpack_skip_bytes: * diff --git a/libretro-db/samples/rmsgpack/Makefile b/libretro-db/samples/rmsgpack/Makefile index 668422eae1..f2e1e18ce4 100644 --- a/libretro-db/samples/rmsgpack/Makefile +++ b/libretro-db/samples/rmsgpack/Makefile @@ -1,10 +1,10 @@ -TARGET := rmsgpack_overflow_test +TARGET := rmsgpack_overflow_test +TARGET2 := rmsgpack_window_test LIBRETRODB_DIR := ../.. LIBRETRO_COMM_DIR := ../../../libretro-common -SOURCES := \ - rmsgpack_overflow_test.c \ +COMMON := \ $(LIBRETRODB_DIR)/rmsgpack.c \ $(LIBRETRODB_DIR)/rmsgpack_dom.c \ $(LIBRETRO_COMM_DIR)/streams/interface_stream.c \ @@ -22,7 +22,8 @@ SOURCES := \ $(LIBRETRO_COMM_DIR)/time/rtime.c \ $(LIBRETRO_COMM_DIR)/vfs/vfs_implementation.c -OBJS := $(SOURCES:.c=.o) +OBJS := rmsgpack_overflow_test.o $(COMMON:.c=.o) +OBJS2 := rmsgpack_window_test.o $(COMMON:.c=.o) CFLAGS += -Wall -pedantic -std=gnu99 -g \ -I$(LIBRETRO_COMM_DIR)/include \ @@ -37,7 +38,7 @@ ifneq ($(SANITIZER),) LDFLAGS := -fsanitize=$(SANITIZER) $(LDFLAGS) endif -all: $(TARGET) +all: $(TARGET) $(TARGET2) %.o: %.c $(CC) -c -o $@ $< $(CFLAGS) @@ -45,7 +46,10 @@ all: $(TARGET) $(TARGET): $(OBJS) $(CC) -o $@ $^ $(LDFLAGS) +$(TARGET2): $(OBJS2) + $(CC) -o $@ $^ $(LDFLAGS) + clean: - rm -f $(TARGET) $(OBJS) + rm -f $(TARGET) $(TARGET2) rmsgpack_overflow_test.o rmsgpack_window_test.o $(COMMON:.c=.o) .PHONY: clean diff --git a/libretro-db/samples/rmsgpack/rmsgpack_window_test.c b/libretro-db/samples/rmsgpack/rmsgpack_window_test.c new file mode 100644 index 0000000000..fe91ea2155 --- /dev/null +++ b/libretro-db/samples/rmsgpack/rmsgpack_window_test.c @@ -0,0 +1,231 @@ +/* rmsgpack_read() takes a value from a window the stream lends where + * it can, and reads what the window does not hold. This writes a + * stream of values - maps, arrays, integers of every width and sign, + * strings and binaries from empty to well over the window - and reads + * it back through a buffered stream with a window smaller than some of + * them, a file stream, and a memory stream, which lends nothing. Every + * value must come back as written, and after each the stream must + * stand where the value ends. */ + +#include +#include +#include +#include + +#include +#include + +#include "../../rmsgpack.h" +#include "../../rmsgpack_dom.h" + +#define N_VALUES 400 +#define BIG (100 * 1024) + +static int failures = 0; +static uint8_t big[BIG]; + +/* A value whose shape depends on @i */ +static void write_value(intfstream_t *fd, unsigned i) +{ + switch (i % 7) + { + case 0: + rmsgpack_write_map_header(fd, 3); + rmsgpack_write_string(fd, "name", 4); + rmsgpack_write_string(fd, (const char*)big, i % 300); + rmsgpack_write_string(fd, "size", 4); + rmsgpack_write_uint(fd, (uint64_t)i * 0x10001u); + rmsgpack_write_string(fd, "crc", 3); + rmsgpack_write_bin(fd, big + i, 4); + break; + case 1: + rmsgpack_write_int(fd, -(int64_t)i); + break; + case 2: + rmsgpack_write_int(fd, -((int64_t)1 << (i % 63))); + break; + case 3: + /* longer than the buffered stream's window, and than the + * reader's own */ + rmsgpack_write_string(fd, (const char*)big, (uint32_t)(i * 997 % BIG)); + break; + case 4: + rmsgpack_write_array_header(fd, 3); + rmsgpack_write_uint(fd, (uint64_t)1 << (i % 64)); + rmsgpack_write_bool(fd, (int)(i & 1)); + rmsgpack_write_nil(fd); + break; + case 5: + rmsgpack_write_bin(fd, big, (uint32_t)(i * 131 % 70000)); + break; + default: + rmsgpack_write_uint(fd, i); + break; + } +} + +/* A map's value by its key */ +static const struct rmsgpack_dom_value *field( + const struct rmsgpack_dom_value *map, const char *key) +{ + struct rmsgpack_dom_value k; + if (map->type != RDT_MAP) + return NULL; + k.type = RDT_STRING; + k.val.string.len = (uint32_t)strlen(key); + k.val.string.buff = (char*)key; + return rmsgpack_dom_value_map_value(map, &k); +} + +static int64_t expect_int(unsigned i) +{ + return i % 7 == 1 ? -(int64_t)i : -((int64_t)1 << (i % 63)); +} + +static void check_value(const struct rmsgpack_dom_value *v, unsigned i, + const char *how) +{ + int ok = 0; + switch (i % 7) + { + case 0: + { + const struct rmsgpack_dom_value *name = field(v, "name"); + const struct rmsgpack_dom_value *size = field(v, "size"); + const struct rmsgpack_dom_value *crc = field(v, "crc"); + ok = v->type == RDT_MAP && v->val.map.len == 3 + && name && name->type == RDT_STRING + && name->val.string.len == i % 300 + && !memcmp(name->val.string.buff, big, i % 300) + /* a small unsigned number is written as a fixint, and read + * back as signed */ + && size && (size->type == RDT_UINT || size->type == RDT_INT) + && size->val.uint_ == (uint64_t)i * 0x10001u + && crc && crc->type == RDT_BINARY && crc->val.binary.len == 4 + && !memcmp(crc->val.binary.buff, big + i, 4); + break; + } + case 1: + case 2: + ok = (v->type == RDT_INT || v->type == RDT_UINT) + && v->val.int_ == expect_int(i); + break; + case 3: + ok = v->type == RDT_STRING + && v->val.string.len == i * 997 % BIG + && !memcmp(v->val.string.buff, big, v->val.string.len); + break; + case 4: + ok = v->type == RDT_ARRAY && v->val.array.len == 3 + && v->val.array.items[0].val.uint_ == (uint64_t)1 << (i % 64) + && v->val.array.items[1].type == RDT_BOOL + && v->val.array.items[1].val.bool_ == (int)(i & 1) + && v->val.array.items[2].type == RDT_NULL; + break; + case 5: + ok = v->type == RDT_BINARY + && v->val.binary.len == i * 131 % 70000 + && !memcmp(v->val.binary.buff, big, v->val.binary.len); + break; + default: + ok = (v->type == RDT_UINT || v->type == RDT_INT) && v->val.uint_ == i; + break; + } + if (!ok) + { + fprintf(stderr, "FAIL: %s: value %u (shape %u)\n", how, i, i % 7); + failures++; + } +} + +static void read_back(intfstream_t *fd, const int64_t *ends, const char *how) +{ + unsigned i; + if (!fd) + { + fprintf(stderr, "FAIL: %s: cannot open\n", how); + failures++; + return; + } + for (i = 0; i < N_VALUES; i++) + { + struct rmsgpack_dom_value v; + if (rmsgpack_dom_read(fd, &v) < 0) + { + fprintf(stderr, "FAIL: %s: value %u did not read\n", how, i); + failures++; + break; + } + check_value(&v, i, how); + rmsgpack_dom_value_free(&v); + if (intfstream_tell(fd) != ends[i]) + { + fprintf(stderr, "FAIL: %s: after value %u at %lld, not %lld\n", how, i, + (long long)intfstream_tell(fd), (long long)ends[i]); + failures++; + break; + } + } + intfstream_close(fd); + free(fd); +} + +int main(void) +{ + static int64_t ends[N_VALUES]; + const char *path = "rmsgpack_window_test.bin"; + intfstream_t *fd; + uint8_t *mem; + int64_t size; + unsigned i; + + for (i = 0; i < BIG; i++) + big[i] = (uint8_t)(i * 7 + (i >> 11)); + + if (!(fd = intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_WRITE, + RETRO_VFS_FILE_ACCESS_HINT_NONE))) + return 2; + for (i = 0; i < N_VALUES; i++) + { + write_value(fd, i); + ends[i] = intfstream_tell(fd); + } + size = ends[N_VALUES - 1]; + intfstream_close(fd); + free(fd); + + /* a window smaller than the long values, and one smaller still */ + read_back(intfstream_open_buffered(path, 4096), ends, "buffered, 4 KiB window"); + read_back(intfstream_open_buffered(path, 64), ends, "buffered, 64 octet window"); + read_back(intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_READ, + RETRO_VFS_FILE_ACCESS_HINT_NONE), ends, "file"); + read_back(intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_READ, + RETRO_VFS_FILE_ACCESS_HINT_FREQUENT_ACCESS), ends, "file, mapped"); + + /* a memory stream lends nothing: every value is read */ + if ((mem = (uint8_t*)malloc((size_t)size))) + { + FILE *f = fopen(path, "rb"); + if (f && fread(mem, 1, (size_t)size, f) == (size_t)size) + read_back(intfstream_open_memory(mem, RETRO_VFS_FILE_ACCESS_READ, + RETRO_VFS_FILE_ACCESS_HINT_NONE, (uint64_t)size), ends, "memory"); + else + { + fprintf(stderr, "FAIL: cannot read the file back\n"); + failures++; + } + if (f) + fclose(f); + free(mem); + } + remove(path); + + if (failures) + { + fprintf(stderr, "FAIL: %d checks\n", failures); + return 1; + } + printf("ok: %u values through buffered, file, mapped and memory streams\n", + N_VALUES); + return 0; +}