rmsgpack: read a value from a window the stream lends

rmsgpack_read() takes each field from a window the stream lends with
the new intfstream_view(), and reads only what the window does not
hold. Buffered streams lend their window and file streams lend what
filestream_view() lends. The stream is left where the value ends.

Reading every entry of a 150,000-entry database through a cursor
goes from 1.31 to 1.61 million entries a second, with the same
values. The cursor is what the Explore menu and database lookups
read through.

rmsgpack_window_test writes values of every shape: integers of every
width and sign, and strings and binaries up to past the reader's
window. It reads them back through buffered streams with 4 KiB and
64-octet windows, a file stream mapped and not, and a memory stream,
which lends nothing. Every value must match and the position must be
right after each. It passes under ASan and UBSan.

.github/workflows/Linux-libretro-db-samples.yml

@@ -93,6 +93,7 @@ jobs:
           # failure.  These are built AND executed.
           declare -a RUN_TARGETS=(
             rmsgpack_overflow_test
+            rmsgpack_window_test
             libretrodb_leak_test
           )
 

libretro-common/include/streams/interface_stream.h

@@ -98,6 +98,20 @@ void intfstream_rewind(intfstream_internal_t *intf);
 
 int64_t intfstream_tell(intfstream_internal_t *intf);
 
+/**
+ * intfstream_view:
+ *
+ * intfstream_read() without the copy: points @ptr at up to @len
+ * octets at the stream position, where the stream holds them, and
+ * moves past them as a read of that many would.
+ *
+ * Returns: the octets lent, 0 at the end, or -1 when the stream holds
+ * nothing there it can lend; the caller then reads. Lent octets stay
+ * valid until the next call on @intf and must not be written.
+ */
+int64_t intfstream_view(intfstream_internal_t *intf, uint64_t len,
+      const uint8_t **ptr);
+
 int intfstream_eof(intfstream_internal_t *intf);
 
 void intfstream_putc(intfstream_internal_t *intf, int c);

libretro-common/streams/interface_stream.c

@@ -483,6 +483,42 @@ int64_t intfstream_read(intfstream_internal_t *intf, void *s, uint64_t len)
    return -1;
 }
 
+int64_t intfstream_view(intfstream_internal_t *intf, uint64_t len,
+      const uint8_t **ptr)
+{
+   if (!intf || !ptr)
+      return -1;
+
+   switch (intf->type)
+   {
+      case INTFSTREAM_FILE:
+         return filestream_view(intf->file.fp, (int64_t)len, ptr);
+      case INTFSTREAM_BUFFERED:
+         {
+            uint64_t p = intf->buffered.pos;
+            uint64_t n = len;
+            if (p >= intf->buffered.size || !n)
+               return 0;
+            if (n > intf->buffered.size - p)
+               n = intf->buffered.size - p;
+            if (n > intf->buffered.cap)
+               n = intf->buffered.cap;
+            if (   !(p >= intf->buffered.lo && p < intf->buffered.hi)
+                && !intfstream_buffered_fill(intf, p, n))
+               return -1;
+            if (n > intf->buffered.hi - p)
+               n = intf->buffered.hi - p;
+            *ptr               = intf->buffered.buf + (p - intf->buffered.lo);
+            intf->buffered.pos = p + n;
+            return (int64_t)n;
+         }
+      default:
+         break;
+   }
+
+   return -1;
+}
+
 int64_t intfstream_write(intfstream_internal_t *intf,
       const void *s, uint64_t len)
 {

libretro-db/.gitignore

@@ -2,3 +2,5 @@
 /c_converter
 /libretrodb_tool
 /rmsgpack_test
+/samples/rmsgpack/rmsgpack_overflow_test
+/samples/rmsgpack/rmsgpack_window_test

libretro-db/rmsgpack.c

@@ -30,6 +30,8 @@
 #include <string.h>
 
 #include <retro_endianness.h>
+#include <boolean.h>
+#include <streams/file_stream.h>
 
 #include "rmsgpack.h"
 
@@ -410,237 +412,211 @@ int rmsgpack_read_uint(intfstream_t *fd, uint64_t *s, size_t len)
    return 0;
 }
 
-static int rmsgpack_read_int(intfstream_t *fd, int64_t *s, size_t len)
+/* ---- reading a value ----
+ *
+ * A value is read from a window the stream lends where it can
+ * (intfstream_view), so a field is taken from memory rather than
+ * through a read call of its own; what the window does not hold is
+ * read. The stream is left where the value ends. */
+
+#define RMSGPACK_VIEW 65536
+
+struct rmsgpack_src
 {
-   union { uint64_t u64; uint32_t u32; uint16_t u16; uint8_t u8; } tmp;
+   intfstream_t  *fd;
+   const uint8_t *p;     /* lent octets not yet taken */
+   const uint8_t *end;
+   uint64_t       pos;   /* the stream position p stands for */
+   uint64_t       size;
+   bool           lent;  /* the stream is past pos */
+   bool           view;  /* the stream lends at all */
+};
 
-   tmp.u64 = 0;
-   if (rmsgpack_read_exact(fd, &tmp, len) == -1)
-      return -1;
+static void rmsgpack_src_init(struct rmsgpack_src *s, intfstream_t *fd)
+{
+   int64_t pos  = intfstream_tell(fd);
+   int64_t size = intfstream_get_size(fd);
+   s->fd   = fd;
+   s->p    = NULL;
+   s->end  = NULL;
+   s->pos  = pos  > 0 ? (uint64_t)pos  : 0;
+   /* a stream that does not know its size bounds nothing */
+   s->size = size >= 0 ? (uint64_t)size : ~(uint64_t)0;
+   s->lent = false;
+   s->view = true;
+}
 
-   switch (len)
+/* Puts the stream back on the position */
+static void rmsgpack_src_done(struct rmsgpack_src *s)
+{
+   if (s->lent)
+      intfstream_seek(s->fd, (int64_t)s->pos, RETRO_VFS_SEEK_POSITION_START);
+   s->lent = false;
+   s->p    = s->end = NULL;
+}
+
+/* Whether @len octets at the position are lent */
+static bool rmsgpack_src_window(struct rmsgpack_src *s, size_t len)
+{
+   const uint8_t *ptr = NULL;
+   int64_t        n;
+   if ((size_t)(s->end - s->p) >= len)
+      return true;
+   if (!s->view)
+      return false;
+   if (     s->lent
+         && intfstream_seek(s->fd, (int64_t)s->pos,
+            RETRO_VFS_SEEK_POSITION_START) < 0)
+      return false;
+   n       = intfstream_view(s->fd, len > RMSGPACK_VIEW ? len : RMSGPACK_VIEW, &ptr);
+   s->lent = n > 0;
+   if (n < 0)
+      s->view = false;
+   if (n <= 0)
    {
-      case 1:
-         *s = (int8_t)tmp.u8;
-         break;
-      case 2:
-         *s = (int16_t)swap_if_little16(tmp.u16);
-         break;
-      case 4:
-         *s = (int32_t)swap_if_little32(tmp.u32);
-         break;
-      case 8:
-         *s = (int64_t)swap_if_little64(tmp.u64);
-         break;
+      s->p = s->end = NULL;
+      return false;
    }
+   s->p   = ptr;
+   s->end = ptr + n;
+   return (size_t)n >= len;
+}
+
+/* @len octets at the position into @dst, and past them */
+static int rmsgpack_src_read(struct rmsgpack_src *s, void *dst, size_t len)
+{
+   if (!len)
+      return 0;
+   if (rmsgpack_src_window(s, len))
+   {
+      memcpy(dst, s->p, len);
+      s->p   += len;
+      s->pos += len;
+      return 0;
+   }
+   /* more than a window: read where the position is */
+   rmsgpack_src_done(s);
+   if (rmsgpack_read_exact(s->fd, dst, len) == -1)
+      return -1;
+   s->pos += len;
    return 0;
 }
 
-static int rmsgpack_read_buff(intfstream_t *fd, size_t size, char **pbuff, uint64_t *len)
+/* A big endian number of @len octets (1, 2, 4 or 8) */
+static int rmsgpack_src_uint(struct rmsgpack_src *s, uint64_t *v, size_t len)
 {
-   ssize_t read_len;
-   uint64_t tmp_len   = 0;
-   int64_t  remaining;
-   int64_t  here;
-   int64_t  total;
+   uint8_t        tmp[8];
+   const uint8_t *q = tmp;
+   size_t         i;
+   if (rmsgpack_src_window(s, len))
+   {
+      q       = s->p;
+      s->p   += len;
+      s->pos += len;
+   }
+   else if (rmsgpack_src_read(s, tmp, len) == -1)
+      return -1;
+   *v = 0;
+   for (i = 0; i < len; i++)
+      *v = (*v << 8) | q[i];
+   return 0;
+}
 
-   if (rmsgpack_read_uint(fd, &tmp_len, size) == -1)
-      return -1;
+/* @len octets, the payload of a string or binary, in a fresh
+ * allocation ending in a 0 */
+/* the octets the stream has left */
+static uint64_t rmsgpack_src_left(const struct rmsgpack_src *s)
+{
+   return s->size > s->pos ? s->size - s->pos : 0;
+}
 
-   /* Pre-patch tmp_len was an attacker-controlled uint64 fed
-    * directly into malloc((size_t)(tmp_len + 1)).  Three
-    * problems:
-    *   - tmp_len = UINT64_MAX wraps to malloc(0) returning a
-    *     small block; the subsequent intfstream_read with
-    *     (size_t)UINT64_MAX bytes would heap-overflow on any
-    *     stream that could deliver them.
-    *   - tmp_len = 0xFFFFFFFE forces a ~4 GiB malloc on 64-bit
-    *     (which Linux overcommit happily grants) for a 5-byte
-    *     STR32 input, OOM-killing the process.  On 32-bit the
-    *     (size_t) cast truncates and creates a heap overflow.
-    *   - Even when malloc returns NULL the code dereferenced
-    *     *pbuff at line 396 and (*pbuff)[read_len] at line 404.
-    *
-    * Fix: bound tmp_len against the remaining bytes in the
-    * stream (a buffer can't legitimately claim more bytes than
-    * are left to read), reject the SIZE_MAX edge to avoid the
-    * +1 wrap, and NULL-check the malloc. */
-   here  = intfstream_tell(fd);
-   total = intfstream_get_size(fd);
-   if (here < 0 || total < 0 || here > total)
+static int rmsgpack_src_buff(struct rmsgpack_src *s, uint64_t len, char **pbuff)
+{
+   /* a payload claims no more than the stream has left */
+   if (len > rmsgpack_src_left(s) || len >= (uint64_t)((size_t)-1))
       return -1;
-   remaining = total - here;
-   if (tmp_len > (uint64_t)remaining)
+   if (!(*pbuff = (char*)malloc((size_t)len + 1)))
       return -1;
-   if (tmp_len >= (uint64_t)((size_t)-1))
-      return -1;
-
-   *pbuff             = (char *)malloc((size_t)(tmp_len + 1) * sizeof(char));
-   if (!*pbuff)
-      return -1;
-
-   if (rmsgpack_read_exact(fd, *pbuff, (size_t)tmp_len) == -1)
+   if (rmsgpack_src_read(s, *pbuff, (size_t)len) == -1)
    {
       free(*pbuff);
       *pbuff = NULL;
       return -1;
    }
-
-   read_len           = (ssize_t)tmp_len;
-   *len               = (uint64_t)read_len;
-   (*pbuff)[read_len] = 0;
-
+   (*pbuff)[len] = 0;
    return 0;
 }
 
-static int rmsgpack_read_depth(intfstream_t *fd,
+static int rmsgpack_src_value(struct rmsgpack_src *s,
       struct rmsgpack_read_callbacks *callbacks, void *data,
       unsigned depth);
 
-static int rmsgpack_read_map(intfstream_t *fd, uint32_t len,
-        struct rmsgpack_read_callbacks *callbacks, void *data,
-        unsigned depth)
+/* A map or array of @len entries (@pairs: a map) */
+static int rmsgpack_src_items(struct rmsgpack_src *s, uint64_t len,
+      bool pairs, struct rmsgpack_read_callbacks *callbacks, void *data,
+      unsigned depth)
 {
-   int rv;
-   unsigned i;
-   int64_t  here, total;
+   int      rv;
+   uint64_t i;
 
-   /* Pre-patch len was an attacker-controlled uint32 from the
-    * file (MAP16 or MAP32 header) handed straight to the callback
-    * which calloc'd 'len * sizeof(rmsgpack_dom_pair)' (~80 bytes
-    * per pair).  A 5-byte MAP32 header '0xdf 0x10 0x00 0x00 0x00'
-    * (len = 2^28) demanded a ~21 GiB calloc, OOM-killing the
-    * process.  Even where calloc succeeded, the subsequent
-    * iteration recursed into rmsgpack_read len times, doubling
-    * the dom reader stack on each recursion until it ran out.
-    *
-    * A map cannot legitimately claim more entries than the
-    * stream has bytes left to encode them: the smallest possible
-    * key+value pair is 2 bytes (one type byte each).  Reject
-    * len > remaining_bytes / 2. */
-   here  = intfstream_tell(fd);
-   total = intfstream_get_size(fd);
-   if (here >= 0 && total >= 0 && here <= total)
+   /* No container claims more entries than the stream has octets
+    * left for: an entry is one octet at least, a pair two. */
+   if (len > rmsgpack_src_left(s) / (pairs ? 2u : 1u))
+      return -1;
+   if (pairs)
    {
-      uint64_t remaining = (uint64_t)(total - here);
-      if ((uint64_t)len > remaining / 2u)
-         return -1;
+      if (     callbacks->read_map_start
+            && (rv = callbacks->read_map_start((uint32_t)len, data)) < 0)
+         return rv;
    }
-
-   if (     (     callbacks->read_map_start)
-         && (rv = callbacks->read_map_start(len, data)) < 0)
+   else if (callbacks->read_array_start
+         && (rv = callbacks->read_array_start((uint32_t)len, data)) < 0)
       return rv;
 
-   for (i = 0; i < len; i++)
-   {
-      if ((rv = rmsgpack_read_depth(fd, callbacks, data, depth)) < 0)
+   for (i = 0; i < (pairs ? 2 * len : len); i++)
+      if ((rv = rmsgpack_src_value(s, callbacks, data, depth)) < 0)
          return rv;
-      if ((rv = rmsgpack_read_depth(fd, callbacks, data, depth)) < 0)
-         return rv;
-   }
-
    return 0;
 }
 
-static int rmsgpack_read_array(intfstream_t *fd, uint32_t len,
+static int rmsgpack_src_value(struct rmsgpack_src *s,
       struct rmsgpack_read_callbacks *callbacks, void *data,
       unsigned depth)
 {
-   int rv;
-   unsigned i;
-   int64_t  here, total;
-
-   /* Same primitive as rmsgpack_read_map above.  Smallest
-    * possible array element is 1 byte (a fixint), so len cannot
-    * legitimately exceed the number of bytes left in the
-    * stream. */
-   here  = intfstream_tell(fd);
-   total = intfstream_get_size(fd);
-   if (here >= 0 && total >= 0 && here <= total)
-   {
-      uint64_t remaining = (uint64_t)(total - here);
-      if ((uint64_t)len > remaining)
-         return -1;
-   }
-
-   if (     (     callbacks->read_array_start)
-         && (rv = callbacks->read_array_start(len, data)) < 0)
-      return rv;
-
-   for (i = 0; i < len; i++)
-   {
-      if ((rv = rmsgpack_read_depth(fd, callbacks, data, depth)) < 0)
-         return rv;
-   }
-
-   return 0;
-}
-
-int rmsgpack_read(intfstream_t *fd,
-      struct rmsgpack_read_callbacks *callbacks, void *data)
-{
-   return rmsgpack_read_depth(fd, callbacks, data, 0);
-}
-
-static int rmsgpack_read_depth(intfstream_t *fd,
-      struct rmsgpack_read_callbacks *callbacks, void *data,
-      unsigned depth)
-{
-   int rv;
    uint64_t tmp_len  = 0;
    uint64_t tmp_uint = 0;
-   int64_t tmp_int   = 0;
-   uint8_t type      = 0;
-   char *buff        = NULL;
+   uint8_t  type;
+   char    *buff     = NULL;
 
    if (depth >= RMSGPACK_MAX_DEPTH)
       return -1;
    depth++;
 
-   if (rmsgpack_read_exact(fd, &type, sizeof(uint8_t)) == -1)
+   if (s->p < s->end)
+   {
+      type = *s->p++;
+      s->pos++;
+   }
+   else if (rmsgpack_src_read(s, &type, 1) == -1)
       return -1;
 
    if (type < MPF_FIXMAP)
-   {
-      if (callbacks->read_int)
-         return callbacks->read_int(type, data);
-      return 0;
-   }
+      return callbacks->read_int ? callbacks->read_int(type, data) : 0;
    else if (type < MPF_FIXARRAY)
-   {
-      tmp_len = type - MPF_FIXMAP;
-      return rmsgpack_read_map(fd, (uint32_t)tmp_len, callbacks, data, depth);
-   }
+      return rmsgpack_src_items(s, type - MPF_FIXMAP, true, callbacks, data, depth);
    else if (type < MPF_FIXSTR)
-   {
-      tmp_len = type - MPF_FIXARRAY;
-      return rmsgpack_read_array(fd, (uint32_t)tmp_len, callbacks, data, depth);
-   }
+      return rmsgpack_src_items(s, type - MPF_FIXARRAY, false, callbacks, data, depth);
    else if (type < MPF_NIL)
    {
-      ssize_t _len = 0;
-      tmp_len      = type - MPF_FIXSTR;
-      if (!(buff = (char *)malloc((size_t)(tmp_len + 1) * sizeof(char))))
+      if (rmsgpack_src_buff(s, type - MPF_FIXSTR, &buff) == -1)
          return -1;
-      if (rmsgpack_read_exact(fd, buff, (size_t)tmp_len) == -1)
-      {
-         free(buff);
-         return -1;
-      }
-      _len       = (ssize_t)tmp_len;
-      buff[_len] = '\0';
       if (callbacks->read_string)
-         return callbacks->read_string(buff, (uint32_t)_len, data);
-      if (buff)
-         free(buff);
+         return callbacks->read_string(buff, (uint32_t)(type - MPF_FIXSTR), data);
+      free(buff);
       return 0;
    }
    else if (type > MPF_MAP32)
-   {
-      if (callbacks->read_int)
-         return callbacks->read_int(type - 0xff - 1, data);
-      return 0;
-   }
+      return callbacks->read_int ? callbacks->read_int(type - 0xff - 1, data) : 0;
 
    switch (type)
    {
@@ -659,19 +635,26 @@ static int rmsgpack_read_depth(intfstream_t *fd,
       case _MPF_BIN8:
       case _MPF_BIN16:
       case _MPF_BIN32:
-         if ((rv = rmsgpack_read_buff(fd, (size_t)(1 << (type - _MPF_BIN8)),
-                     &buff, &tmp_len)) < 0)
-            return rv;
-         if (callbacks->read_bin)
+      case _MPF_STR8:
+      case _MPF_STR16:
+      case _MPF_STR32:
+      {
+         bool str = type >= _MPF_STR8;
+         if (     rmsgpack_src_uint(s, &tmp_len,
+                     (size_t)1 << (type - (str ? _MPF_STR8 : _MPF_BIN8))) == -1
+               || rmsgpack_src_buff(s, tmp_len, &buff) == -1)
+            return -1;
+         if (str && callbacks->read_string)
+            return callbacks->read_string(buff, (uint32_t)tmp_len, data);
+         if (!str && callbacks->read_bin)
             return callbacks->read_bin(buff, (uint32_t)tmp_len, data);
          break;
+      }
       case _MPF_UINT8:
       case _MPF_UINT16:
       case _MPF_UINT32:
       case _MPF_UINT64:
-         tmp_len  = UINT64_C(1) << (type - _MPF_UINT8);
-         tmp_uint = 0;
-         if (rmsgpack_read_uint(fd, &tmp_uint, (size_t)tmp_len) == -1)
+         if (rmsgpack_src_uint(s, &tmp_uint, (size_t)1 << (type - _MPF_UINT8)) == -1)
             return -1;
          if (callbacks->read_uint)
             return callbacks->read_uint(tmp_uint, data);
@@ -680,38 +663,46 @@ static int rmsgpack_read_depth(intfstream_t *fd,
       case _MPF_INT16:
       case _MPF_INT32:
       case _MPF_INT64:
-         tmp_len = UINT64_C(1) << (type - _MPF_INT8);
-         tmp_int = 0;
-         if (rmsgpack_read_int(fd, &tmp_int, (size_t)tmp_len) == -1)
+      {
+         size_t  len = (size_t)1 << (type - _MPF_INT8);
+         int64_t v;
+         if (rmsgpack_src_uint(s, &tmp_uint, len) == -1)
             return -1;
+         /* sign extended from its width */
+         if (len < 8 && (tmp_uint >> (8 * len - 1)) & 1)
+            tmp_uint |= ~(uint64_t)0 << (8 * len);
+         v = (int64_t)tmp_uint;
          if (callbacks->read_int)
-            return callbacks->read_int(tmp_int, data);
-         break;
-      case _MPF_STR8:
-      case _MPF_STR16:
-      case _MPF_STR32:
-         if ((rv = rmsgpack_read_buff(fd, (size_t)(1 << (type - _MPF_STR8)), &buff, &tmp_len)) < 0)
-            return rv;
-         if (callbacks->read_string)
-            return callbacks->read_string(buff, (uint32_t)tmp_len, data);
+            return callbacks->read_int(v, data);
          break;
+      }
       case _MPF_ARRAY16:
       case _MPF_ARRAY32:
-         if (rmsgpack_read_uint(fd, &tmp_len, 2<<(type - _MPF_ARRAY16)) != -1)
-            return rmsgpack_read_array(fd, (uint32_t)tmp_len, callbacks, data, depth);
-         return -1;
+         if (rmsgpack_src_uint(s, &tmp_len, (size_t)2 << (type - _MPF_ARRAY16)) == -1)
+            return -1;
+         return rmsgpack_src_items(s, tmp_len, false, callbacks, data, depth);
       case _MPF_MAP16:
       case _MPF_MAP32:
-         if (rmsgpack_read_uint(fd, &tmp_len, 2<<(type - _MPF_MAP16)) != -1)
-            return rmsgpack_read_map(fd, (uint32_t)tmp_len, callbacks, data, depth);
-         return -1;
+         if (rmsgpack_src_uint(s, &tmp_len, (size_t)2 << (type - _MPF_MAP16)) == -1)
+            return -1;
+         return rmsgpack_src_items(s, tmp_len, true, callbacks, data, depth);
    }
 
-   if (buff)
-      free(buff);
+   free(buff);
    return 0;
 }
 
+int rmsgpack_read(intfstream_t *fd,
+      struct rmsgpack_read_callbacks *callbacks, void *data)
+{
+   struct rmsgpack_src s;
+   int                 rv;
+   rmsgpack_src_init(&s, fd);
+   rv = rmsgpack_src_value(&s, callbacks, data, 0);
+   rmsgpack_src_done(&s);
+   return rv;
+}
+
 /**
  * rmsgpack_skip_bytes:
  *

libretro-db/samples/rmsgpack/Makefile

@@ -1,10 +1,10 @@
-TARGET := rmsgpack_overflow_test
+TARGET  := rmsgpack_overflow_test
+TARGET2 := rmsgpack_window_test
 
 LIBRETRODB_DIR    := ../..
 LIBRETRO_COMM_DIR := ../../../libretro-common
 
-SOURCES := \
-	rmsgpack_overflow_test.c \
+COMMON := \
 	$(LIBRETRODB_DIR)/rmsgpack.c \
 	$(LIBRETRODB_DIR)/rmsgpack_dom.c \
 	$(LIBRETRO_COMM_DIR)/streams/interface_stream.c \
@@ -22,7 +22,8 @@ SOURCES := \
 	$(LIBRETRO_COMM_DIR)/time/rtime.c \
 	$(LIBRETRO_COMM_DIR)/vfs/vfs_implementation.c
 
-OBJS := $(SOURCES:.c=.o)
+OBJS  := rmsgpack_overflow_test.o $(COMMON:.c=.o)
+OBJS2 := rmsgpack_window_test.o $(COMMON:.c=.o)
 
 CFLAGS  += -Wall -pedantic -std=gnu99 -g \
            -I$(LIBRETRO_COMM_DIR)/include \
@@ -37,7 +38,7 @@ ifneq ($(SANITIZER),)
    LDFLAGS := -fsanitize=$(SANITIZER) $(LDFLAGS)
 endif
 
-all: $(TARGET)
+all: $(TARGET) $(TARGET2)
 
 %.o: %.c
 	$(CC) -c -o $@ $< $(CFLAGS)
@@ -45,7 +46,10 @@ all: $(TARGET)
 $(TARGET): $(OBJS)
 	$(CC) -o $@ $^ $(LDFLAGS)
 
+$(TARGET2): $(OBJS2)
+	$(CC) -o $@ $^ $(LDFLAGS)
+
 clean:
-	rm -f $(TARGET) $(OBJS)
+	rm -f $(TARGET) $(TARGET2) rmsgpack_overflow_test.o rmsgpack_window_test.o $(COMMON:.c=.o)
 
 .PHONY: clean

libretro-db/samples/rmsgpack/rmsgpack_window_test.c

@@ -0,0 +1,231 @@
+/* rmsgpack_read() takes a value from a window the stream lends where
+ * it can, and reads what the window does not hold. This writes a
+ * stream of values - maps, arrays, integers of every width and sign,
+ * strings and binaries from empty to well over the window - and reads
+ * it back through a buffered stream with a window smaller than some of
+ * them, a file stream, and a memory stream, which lends nothing. Every
+ * value must come back as written, and after each the stream must
+ * stand where the value ends. */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include <streams/interface_stream.h>
+#include <streams/file_stream.h>
+
+#include "../../rmsgpack.h"
+#include "../../rmsgpack_dom.h"
+
+#define N_VALUES 400
+#define BIG      (100 * 1024)
+
+static int     failures = 0;
+static uint8_t big[BIG];
+
+/* A value whose shape depends on @i */
+static void write_value(intfstream_t *fd, unsigned i)
+{
+   switch (i % 7)
+   {
+      case 0:
+         rmsgpack_write_map_header(fd, 3);
+         rmsgpack_write_string(fd, "name", 4);
+         rmsgpack_write_string(fd, (const char*)big, i % 300);
+         rmsgpack_write_string(fd, "size", 4);
+         rmsgpack_write_uint(fd, (uint64_t)i * 0x10001u);
+         rmsgpack_write_string(fd, "crc", 3);
+         rmsgpack_write_bin(fd, big + i, 4);
+         break;
+      case 1:
+         rmsgpack_write_int(fd, -(int64_t)i);
+         break;
+      case 2:
+         rmsgpack_write_int(fd, -((int64_t)1 << (i % 63)));
+         break;
+      case 3:
+         /* longer than the buffered stream's window, and than the
+          * reader's own */
+         rmsgpack_write_string(fd, (const char*)big, (uint32_t)(i * 997 % BIG));
+         break;
+      case 4:
+         rmsgpack_write_array_header(fd, 3);
+         rmsgpack_write_uint(fd, (uint64_t)1 << (i % 64));
+         rmsgpack_write_bool(fd, (int)(i & 1));
+         rmsgpack_write_nil(fd);
+         break;
+      case 5:
+         rmsgpack_write_bin(fd, big, (uint32_t)(i * 131 % 70000));
+         break;
+      default:
+         rmsgpack_write_uint(fd, i);
+         break;
+   }
+}
+
+/* A map's value by its key */
+static const struct rmsgpack_dom_value *field(
+      const struct rmsgpack_dom_value *map, const char *key)
+{
+   struct rmsgpack_dom_value k;
+   if (map->type != RDT_MAP)
+      return NULL;
+   k.type            = RDT_STRING;
+   k.val.string.len  = (uint32_t)strlen(key);
+   k.val.string.buff = (char*)key;
+   return rmsgpack_dom_value_map_value(map, &k);
+}
+
+static int64_t expect_int(unsigned i)
+{
+   return i % 7 == 1 ? -(int64_t)i : -((int64_t)1 << (i % 63));
+}
+
+static void check_value(const struct rmsgpack_dom_value *v, unsigned i,
+      const char *how)
+{
+   int ok = 0;
+   switch (i % 7)
+   {
+      case 0:
+      {
+         const struct rmsgpack_dom_value *name = field(v, "name");
+         const struct rmsgpack_dom_value *size = field(v, "size");
+         const struct rmsgpack_dom_value *crc  = field(v, "crc");
+         ok = v->type == RDT_MAP && v->val.map.len == 3
+            && name && name->type == RDT_STRING
+            && name->val.string.len == i % 300
+            && !memcmp(name->val.string.buff, big, i % 300)
+            /* a small unsigned number is written as a fixint, and read
+             * back as signed */
+            && size && (size->type == RDT_UINT || size->type == RDT_INT)
+            && size->val.uint_ == (uint64_t)i * 0x10001u
+            && crc && crc->type == RDT_BINARY && crc->val.binary.len == 4
+            && !memcmp(crc->val.binary.buff, big + i, 4);
+         break;
+      }
+      case 1:
+      case 2:
+         ok = (v->type == RDT_INT || v->type == RDT_UINT)
+            && v->val.int_ == expect_int(i);
+         break;
+      case 3:
+         ok = v->type == RDT_STRING
+            && v->val.string.len == i * 997 % BIG
+            && !memcmp(v->val.string.buff, big, v->val.string.len);
+         break;
+      case 4:
+         ok = v->type == RDT_ARRAY && v->val.array.len == 3
+            && v->val.array.items[0].val.uint_ == (uint64_t)1 << (i % 64)
+            && v->val.array.items[1].type == RDT_BOOL
+            && v->val.array.items[1].val.bool_ == (int)(i & 1)
+            && v->val.array.items[2].type == RDT_NULL;
+         break;
+      case 5:
+         ok = v->type == RDT_BINARY
+            && v->val.binary.len == i * 131 % 70000
+            && !memcmp(v->val.binary.buff, big, v->val.binary.len);
+         break;
+      default:
+         ok = (v->type == RDT_UINT || v->type == RDT_INT) && v->val.uint_ == i;
+         break;
+   }
+   if (!ok)
+   {
+      fprintf(stderr, "FAIL: %s: value %u (shape %u)\n", how, i, i % 7);
+      failures++;
+   }
+}
+
+static void read_back(intfstream_t *fd, const int64_t *ends, const char *how)
+{
+   unsigned i;
+   if (!fd)
+   {
+      fprintf(stderr, "FAIL: %s: cannot open\n", how);
+      failures++;
+      return;
+   }
+   for (i = 0; i < N_VALUES; i++)
+   {
+      struct rmsgpack_dom_value v;
+      if (rmsgpack_dom_read(fd, &v) < 0)
+      {
+         fprintf(stderr, "FAIL: %s: value %u did not read\n", how, i);
+         failures++;
+         break;
+      }
+      check_value(&v, i, how);
+      rmsgpack_dom_value_free(&v);
+      if (intfstream_tell(fd) != ends[i])
+      {
+         fprintf(stderr, "FAIL: %s: after value %u at %lld, not %lld\n", how, i,
+               (long long)intfstream_tell(fd), (long long)ends[i]);
+         failures++;
+         break;
+      }
+   }
+   intfstream_close(fd);
+   free(fd);
+}
+
+int main(void)
+{
+   static int64_t ends[N_VALUES];
+   const char    *path = "rmsgpack_window_test.bin";
+   intfstream_t  *fd;
+   uint8_t       *mem;
+   int64_t        size;
+   unsigned       i;
+
+   for (i = 0; i < BIG; i++)
+      big[i] = (uint8_t)(i * 7 + (i >> 11));
+
+   if (!(fd = intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_WRITE,
+               RETRO_VFS_FILE_ACCESS_HINT_NONE)))
+      return 2;
+   for (i = 0; i < N_VALUES; i++)
+   {
+      write_value(fd, i);
+      ends[i] = intfstream_tell(fd);
+   }
+   size = ends[N_VALUES - 1];
+   intfstream_close(fd);
+   free(fd);
+
+   /* a window smaller than the long values, and one smaller still */
+   read_back(intfstream_open_buffered(path, 4096), ends, "buffered, 4 KiB window");
+   read_back(intfstream_open_buffered(path, 64), ends, "buffered, 64 octet window");
+   read_back(intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_READ,
+            RETRO_VFS_FILE_ACCESS_HINT_NONE), ends, "file");
+   read_back(intfstream_open_file(path, RETRO_VFS_FILE_ACCESS_READ,
+            RETRO_VFS_FILE_ACCESS_HINT_FREQUENT_ACCESS), ends, "file, mapped");
+
+   /* a memory stream lends nothing: every value is read */
+   if ((mem = (uint8_t*)malloc((size_t)size)))
+   {
+      FILE *f = fopen(path, "rb");
+      if (f && fread(mem, 1, (size_t)size, f) == (size_t)size)
+         read_back(intfstream_open_memory(mem, RETRO_VFS_FILE_ACCESS_READ,
+                  RETRO_VFS_FILE_ACCESS_HINT_NONE, (uint64_t)size), ends, "memory");
+      else
+      {
+         fprintf(stderr, "FAIL: cannot read the file back\n");
+         failures++;
+      }
+      if (f)
+         fclose(f);
+      free(mem);
+   }
+   remove(path);
+
+   if (failures)
+   {
+      fprintf(stderr, "FAIL: %d checks\n", failures);
+      return 1;
+   }
+   printf("ok: %u values through buffered, file, mapped and memory streams\n",
+         N_VALUES);
+   return 0;
+}