rwav: keep MS ADPCM's step and prediction inside an int

A block states its opening step, and a run of large nibbles grows it
by up to 768/256 a sample with no bound, so a crafted block overflowed
the adaptation's multiply within a few dozen samples: undefined
behaviour, in practice a step that wraps back to small values.  The
step is now capped at INT_MAX / 768, the largest the next adaptation
and the nibble times step both still hold, as reference decoders cap
it.  Past a step of 32767 every non-zero nibble already clamps, so
only streams that were saturated garbage decode differently.

The prediction sums two products of a 16-bit sample and a coefficient
the file states, which reach 2^31 at -32768 each; it is summed in 64
bits and divided as before.

The fixtures decode to the same samples.  test_rwav adds a block of
the largest step and +7 nibbles, which UBSan flagged before and which
now stays pinned at the top.

libretro-common/formats/wav/rwav.c

@@ -555,6 +555,30 @@ static const short rwav_ms_adapt[16] = {
    230,230,230,230,307,409,512,614,768,614,512,409,307,230,230,230
 };
 
+/* MS ADPCM's step, kept inside an int.  The file states each block's
+ * opening delta, and a run of large nibbles grows it by 768/256 a
+ * sample without bound, so a crafted block overflows the multiply
+ * within a few dozen samples; the cap is the largest delta that the
+ * next adaptation and the nibble times delta (at most 8x) still hold,
+ * as reference decoders cap it.  No real encoder gets near it: past
+ * 32767 every sample already clamps. */
+#define RWAV_MS_DELTA_MAX (0x7FFFFFFF / 768)
+static int rwav_ms_adapt_delta(int nib, int delta)
+{
+   if (delta > RWAV_MS_DELTA_MAX)
+      delta = RWAV_MS_DELTA_MAX;
+   delta = (rwav_ms_adapt[nib] * delta) / 256;
+   return (delta < 16) ? 16 : delta;
+}
+
+/* The prediction from the two previous samples.  Samples are 16-bit
+ * but the coefficients are the file's, so two products at -32768 each
+ * sum to 2^31: summed wide, then divided as before. */
+static int rwav_ms_predict(int s1, int co1, int s2, int co2)
+{
+   return (int)(((int64_t)s1 * co1 + (int64_t)s2 * co2) / 256);
+}
+
 /* Decode one block into 'out', which must hold samplesperblock frames.
  * Returns frames produced, 0 on a malformed block. */
 /* Emit only the frames in [first, first + count) of the block, written
@@ -668,14 +692,12 @@ static unsigned rwav_decode_block(const rwav_t *wav, const unsigned char *b,
             unsigned byte = b[off];
             int      nib  = (c & 1) ? (byte & 0x0f) : (byte >> 4);
             int      sn   = (nib & 8) ? nib - 16 : nib;
-            int      p    = (s1[c] * co1[c] + s2[c] * co2[c]) / 256
+            int      p    = rwav_ms_predict(s1[c], co1[c], s2[c], co2[c])
                           + sn * delta[c];
             p        = rwav_clamp16(p);
             s2[c]    = s1[c];
             s1[c]    = p;
-            delta[c] = (rwav_ms_adapt[nib] * delta[c]) / 256;
-            if (delta[c] < 16)
-               delta[c] = 16;
+            delta[c] = rwav_ms_adapt_delta(nib, delta[c]);
             RWAV_PUT(i, c, p);
             if (c & 1)
                off++;
@@ -690,14 +712,12 @@ static unsigned rwav_decode_block(const rwav_t *wav, const unsigned char *b,
                unsigned byte = b[off];
                int nib = byte & 0x0f;
                int sn  = (nib & 8) ? nib - 16 : nib;
-               int p   = (s1[0] * co1[0] + s2[0] * co2[0]) / 256
+               int p   = rwav_ms_predict(s1[0], co1[0], s2[0], co2[0])
                        + sn * delta[0];
                p        = rwav_clamp16(p);
                s2[0]    = s1[0];
                s1[0]    = p;
-               delta[0] = (rwav_ms_adapt[nib] * delta[0]) / 256;
-               if (delta[0] < 16)
-                  delta[0] = 16;
+               delta[0] = rwav_ms_adapt_delta(nib, delta[0]);
                RWAV_PUT(i, 0, p);
                i++;
             }

libretro-common/test/formats/test_rwav.c

@@ -2516,6 +2516,43 @@ START_TEST (test_rwav_adpcm_block_frames_bound)
 }
 END_TEST
 
+/* MS ADPCM grows its step by up to 768/256 a sample.  A block of the
+ * largest nibbles drives it far past any real file's within a few
+ * dozen samples; decoding one must stay defined (UBSan watches the
+ * multiplies) and keep clamping, not wrap back to small values. */
+START_TEST (test_rwav_ms_adpcm_delta_growth)
+{
+   rwav_t   h;
+   size_t   i, frames, data_at;
+   short   *out;
+   uint8_t *buf = (uint8_t*)malloc(sizeof(wav_ms));
+   ck_assert(buf != NULL);
+   memcpy(buf, wav_ms, sizeof(wav_ms));
+   ck_assert_int_eq(rwav_parse(&h, buf, sizeof(wav_ms)),
+         RWAV_ITERATE_DONE);
+   data_at = h.dataoffset;
+   /* every block: predictor 0, the largest opening delta, then nibbles
+    * of 7 (+7, the second-largest adaptation) */
+   for (i = 0; i + h.blockalign <= h.subchunk2size; i += h.blockalign)
+   {
+      uint8_t *b = buf + data_at + i;
+      b[0] = 0;
+      b[1] = 0xFF;
+      b[2] = 0x7F;
+      memset(b + 7, 0x77, h.blockalign - 7);
+   }
+   out = (short*)malloc(h.numsamples * h.numchannels * sizeof(short));
+   ck_assert(out != NULL);
+   frames = rwav_decode_s16(&h, buf, 0, h.numsamples, out);
+   ck_assert_uint_eq(frames, h.numsamples);
+   /* once the step is large every +7 saturates: the block's last
+    * sample is pinned at the top */
+   ck_assert_int_eq(out[h.samplesperblock - 1], 32767);
+   free(out);
+   free(buf);
+}
+END_TEST
+
 static Suite *rwav_suite(void)
 {
    Suite *s       = suite_create(SUITE_NAME);
@@ -2526,6 +2563,7 @@ static Suite *rwav_suite(void)
    tcase_add_test(tc_core, test_rwav_windowed_equals_whole);
    tcase_add_test(tc_core, test_rwav_truncation_robustness);
    tcase_add_test(tc_core, test_rwav_adpcm_block_frames_bound);
+   tcase_add_test(tc_core, test_rwav_ms_adpcm_delta_growth);
    suite_add_tcase(s, tc_core);
    return s;
 }