rtga, rbmp: fix blank BMP/TGA thumbnails - resolve the frontier when the length is known

Regression from the partial-buffer work: BMP and TGA thumbnails
stopped appearing in the file browser entirely.

task_image raises the decoder's frontier before the first process()
call, and when a file finishes reading before any decode has run it
raises it to (size_t)-1 outright ("the frontier is now the whole
buffer").  set_avail turned that into a pointer immediately -
buff_data + SIZE_MAX - which wraps.  true_end was still NULL at that
point, so the clamp meant to catch exactly this was skipped, and every
later comparison against the wrapped end was nonsense: the header
check saw a negative length, returned IMAGE_PROCESS_WAIT for bytes
that had already arrived, and the image never decoded.  Small files -
which is every file-browser icon - take that path every time.

set_avail now records the requested frontier and resolves it against
the buffer once process() learns the length, so the call is safe in
any order and (size_t)-1 simply means "all of it".

samples/formats/image_avail covers the ordering: for four fixtures it
raises the frontier before the first process() call, with both
(size_t)-1 and the true length, and requires the decode to complete
and be byte-exact against the untouched whole-buffer path.  Reverting
just the two decoders makes it fail 8 checks - which is what it should
have been doing before this shipped.

libretro-common/formats/bmp/rbmp.c

@@ -87,6 +87,7 @@ struct rbmp
     * A caller that never calls set_avail leaves avail_set false and
     * nothing below changes behaviour. */
    unsigned char *true_end;
+   size_t         avail_req;    /* frontier asked for, clamped at IDLE */
    bool           avail_set;
    bool           need_more;
 };
@@ -908,17 +909,28 @@ bool rbmp_header_ready(const uint8_t *data, size_t len)
 
 void rbmp_set_avail(rbmp_t *rbmp, size_t avail)
 {
-   unsigned char *wall;
-   if (!rbmp || !rbmp->buff_data)
+   if (!rbmp)
       return;
    rbmp->avail_set = true;
-   wall            = rbmp->buff_data + avail;
-   if (rbmp->true_end && wall > rbmp->true_end)
-      wall = rbmp->true_end;
-   /* Monotonic: a caller raising the frontier each tick must not be
-    * able to lower it. */
-   if (wall > rbmp->s.img_buffer_end)
-      rbmp->s.img_buffer_end = wall;
+   /* Only ever forward.  The request is recorded rather than turned
+    * into a pointer here: a caller may raise the frontier before the
+    * first process() call, when the buffer length - and so the end it
+    * has to be clamped against - is not yet known.  task_image does
+    * exactly that, with (size_t)-1, the moment a small file finishes
+    * reading before any decode has run; computing buff_data + that
+    * wraps the pointer and every later comparison against it is
+    * nonsense. */
+   if (avail > rbmp->avail_req)
+      rbmp->avail_req = avail;
+   if (rbmp->true_end && rbmp->buff_data)
+   {
+      unsigned char *wall = (rbmp->avail_req
+            > (size_t)(rbmp->true_end - rbmp->buff_data))
+                          ? rbmp->true_end
+                          : rbmp->buff_data + rbmp->avail_req;
+      if (wall > rbmp->s.img_buffer_end)
+         rbmp->s.img_buffer_end = wall;
+   }
 }
 
 bool rbmp_need_more(rbmp_t *rbmp)
@@ -951,6 +963,11 @@ int rbmp_process_image(rbmp_t *rbmp, void **buf_data,
       rbmp->true_end              = rbmp->buff_data + (int)size;
       if (!rbmp->avail_set)
          rbmp->s.img_buffer_end   = rbmp->true_end;
+      else
+         /* Now the length is known, resolve the recorded frontier. */
+         rbmp->s.img_buffer_end   = (rbmp->avail_req >= size)
+                                  ? rbmp->true_end
+                                  : rbmp->buff_data + rbmp->avail_req;
 
       /* rbmp_begin walks the file header, the DIB header (whose size
        * the file states), any bitfield masks and the whole palette,

libretro-common/formats/tga/rtga.c

@@ -87,6 +87,7 @@ struct rtga
     * calls set_avail, and then true_end == the end passed to
     * process() and nothing below changes behaviour. */
    uint8_t *true_end;
+   size_t   avail_req;           /* frontier asked for, clamped at IDLE */
    bool     avail_set;
    bool     need_more;           /* last slice stopped at the wall */
 };
@@ -808,17 +809,25 @@ bool rtga_header_ready(const uint8_t *data, size_t len)
 
 void rtga_set_avail(rtga_t *rtga, size_t avail)
 {
-   uint8_t *wall;
-   if (!rtga || !rtga->buff_data)
+   if (!rtga)
       return;
    rtga->avail_set = true;
-   wall            = rtga->buff_data + avail;
-   /* Monotonic, and never past the file: a caller raising the
-    * frontier each tick must not be able to lower it. */
-   if (rtga->true_end && wall > rtga->true_end)
-      wall = rtga->true_end;
-   if (wall > rtga->s.img_buffer_end)
-      rtga->s.img_buffer_end = wall;
+   /* Recorded, not resolved: a caller may raise the frontier before
+    * the first process() call, when the buffer length is not yet
+    * known.  task_image does exactly that with (size_t)-1 when a
+    * small file finishes reading before any decode has run, and
+    * buff_data + that wraps the pointer. */
+   if (avail > rtga->avail_req)
+      rtga->avail_req = avail;
+   if (rtga->true_end && rtga->buff_data)
+   {
+      uint8_t *wall = (rtga->avail_req
+            > (size_t)(rtga->true_end - rtga->buff_data))
+                    ? rtga->true_end
+                    : rtga->buff_data + rtga->avail_req;
+      if (wall > rtga->s.img_buffer_end)
+         rtga->s.img_buffer_end = wall;
+   }
 }
 
 bool rtga_need_more(rtga_t *rtga)
@@ -850,9 +859,10 @@ int rtga_process_image(rtga_t *rtga, void **buf_data,
       rtga->s.img_buffer          = rtga->buff_data;
       rtga->s.img_buffer_original = rtga->buff_data;
       rtga->true_end              = rtga->buff_data + (int)size;
-      rtga->s.img_buffer_end      = rtga->avail_set
-                                  ? rtga->s.img_buffer_end
-                                  : rtga->true_end;
+      rtga->s.img_buffer_end      = !rtga->avail_set ? rtga->true_end
+                                  : ((rtga->avail_req >= size)
+                                     ? rtga->true_end
+                                     : rtga->buff_data + rtga->avail_req);
       /* The header, the colour map and the first pixel must be
        * resident before anything can be parsed at all; below that
        * the caller has to come back with more. */

samples/formats/image_avail/image_avail_test.c

@@ -499,6 +499,93 @@ int main(void)
       codec = CODEC_TGA;
    }
 
+   /* The task layer raises the frontier BEFORE the first process()
+    * call, and when a file finishes reading before any decode has run
+    * it raises it to (size_t)-1 outright (task_image.c: "the frontier
+    * is now the whole buffer").  At that moment the decoder does not
+    * know the buffer length, so a set_avail that turns the request
+    * into a pointer there computes buff_data + SIZE_MAX and wraps;
+    * every later comparison against that end is nonsense, the header
+    * check sees a negative length and waits for bytes that already
+    * arrived.  That shipped: BMP and TGA thumbnails went blank in the
+    * file browser, and small files - every file-browser icon - take
+    * this path every time.
+    *
+    * Both orderings must decode exactly as the untouched path does. */
+   {
+      struct { const char *name; fixture_t f; int bmp; } cs[4];
+      unsigned c;
+      cs[0].name = "TGA raw 32bpp"; cs[0].f = fx_tga_raw(31, 17, 32, 0);   cs[0].bmp = 0;
+      cs[1].name = "TGA indexed";   cs[1].f = fx_tga_indexed(23, 11, 200); cs[1].bmp = 0;
+      cs[2].name = "BMP 24bpp";     cs[2].f = fx_bmp(31, 17, 24, 0, 0);    cs[2].bmp = 1;
+      cs[3].name = "BMP 8bpp";      cs[3].f = fx_bmp(23, 11, 8, 0, 200);   cs[3].bmp = 1;
+
+      printf("-- frontier set before the first process() call --\n");
+      for (c = 0; c < 4; c++)
+      {
+         fixture_t f = cs[c].f;
+         size_t    avails[2];
+         unsigned  k, w0 = 0, h0 = 0;
+         uint32_t *ref;
+
+         codec = cs[c].bmp ? CODEC_BMP : CODEC_TGA;
+         ref   = decode_whole(f.buf, f.len, &w0, &h0);
+
+         avails[0] = (size_t)-1;   /* what task_image actually passes */
+         avails[1] = f.len;        /* the honest equivalent           */
+
+         for (k = 0; k < 2; k++)
+         {
+            void    *out = NULL;
+            unsigned w = 0, h = 0;
+            int      ret = IMAGE_PROCESS_NEXT, guard = 0;
+            char     what[176];
+
+            if (cs[c].bmp)
+            {
+               rbmp_t *b = rbmp_alloc();
+               rbmp_set_buf_ptr(b, f.buf);
+               rbmp_set_avail(b, avails[k]);      /* before any process */
+               while (ret == IMAGE_PROCESS_NEXT && ++guard < 100000)
+                  ret = rbmp_process_image(b, &out, f.len, &w, &h, true);
+               rbmp_free(b);
+            }
+            else
+            {
+               rtga_t *t = rtga_alloc();
+               rtga_set_buf_ptr(t, f.buf);
+               rtga_set_avail(t, avails[k]);
+               while (ret == IMAGE_PROCESS_NEXT && ++guard < 100000)
+                  ret = rtga_process_image(t, &out, f.len, &w, &h, true);
+               rtga_free(t);
+            }
+
+            snprintf(what, sizeof(what),
+                  "%s: set_avail(%s) before process still completes",
+                  cs[c].name, k ? "len" : "(size_t)-1");
+            CHECK(ret == IMAGE_PROCESS_END, what);
+
+            snprintf(what, sizeof(what),
+                  "%s: set_avail(%s) before process is byte-exact",
+                  cs[c].name, k ? "len" : "(size_t)-1");
+            if (ret == IMAGE_PROCESS_END && ref && w == w0 && h == h0)
+            {
+               size_t n = (size_t)w0 * h0, i, bad = 0;
+               for (i = 0; i < n; i++)
+                  if (((uint32_t*)out)[i] != ref[i])
+                     bad++;
+               CHECK(bad == 0, what);
+            }
+            else
+               CHECK(0, what);
+            free(out);
+         }
+         free(ref);
+         fx_free(&cs[c].f);
+      }
+      codec = CODEC_TGA;
+   }
+
    /* A truncated file is not a stall: the frontier reaches the real
     * end of the data and the decode has to settle, not wait forever. */
    {