audio_transfer: make the AAC/MP4 arm windowable

The AAC arm could not be told that its buffer was partial.
audio_transfer_set_avail had arms for Vorbis and Opus only, so an
M4A stream handed a window read wherever its sample tables pointed -
into reserved, uncommitted pages.  That is what forced the thumbnail
preview to read whole containers into memory, and what capped it at
GFX_THUMB_ANIM_ABS_MAX_FILE: a multi-gigabyte recording could not
have audio at all.

Everything needed already existed one layer down and was simply never
wired up: rmp4_open_memory_avail bounds the box walk, rmp4_set_avail
raises the bound, rmp4_read_packet already withholds a sample whose
extent is past it with RMP4_READ_AGAIN, and rmp4_consumed reports the
compressed frontier.  This connects them:

  - struct audio_transfer_aac gains 'avail', taken exactly as the
    Vorbis and Opus arms take theirs.
  - set_avail gets an AAC arm, forwarding to rmp4_set_avail and, for
    Matroska-carried AAC, rwebm_set_avail.
  - the MP4 open becomes bounded.  Unbounded, it walks the box tree to
    the end of the file - for a leading-moov movie that is a read past
    the resident head before a single packet is pulled.
  - the pull reports 2 for RMP4_READ_AGAIN, and the ADTS walk reports
    it rather than stepping past the prefix.
  - buffer_tell reports rmp4_consumed / rwebm_tell.  Without it a
    windowed M4A could be given a bound that no feeder could ever
    move, since audio_driver_mixer_stream_byte_tell already routes
    M4A here and the arm returned 0.

Every consumer of the new code had to be taught it, and two of them
would otherwise have hung rather than misbehaved: read_s16 and
read_f32 test 'r == 0' to stop, so a 2 fell through with pend_frames
at 0, took 0 frames, and spun forever on a window the feeder had not
caught up with.  The seek walk had the same shape plus an
uninitialised read, 2 being > 0 there: it used pdata/plen the pull
never set.  A stall now refuses the seek, which is what the windowing
contract in this file already specifies - the caller retries once the
bound has moved.

audio_mixer's set_avail call was compiled out unless WebM with Vorbis
or Opus was built, so an avail passed for an M4A stream was silently
dropped; the guard now admits RAAC+RMP4 as well.

Verified for PCM equivalence, not just for playing: the same file
decoded whole-file and through a window raised in steps produces
byte-identical s16 over 144000 frames, with the window deliberately
starved (1 KiB steps) to force the backpressure path - 4 stalls, still
byte-exact.  A windowed decode that dropped or repeated one access
unit per boundary would pass a does-it-play test and fail this one.
ASan+UBSan+LSan and strict UBSan clean.  Builds at C89 with RAAC
alone, with RAAC+RMP4, with the full set, and with none.

libretro-common/audio/audio_mixer.c

@@ -1656,10 +1656,14 @@ static bool audio_mixer_play_stream(
    audio_transfer_set_buffer_ptr(xfer, type,
          (void*)sound->types.stream.data, sound->types.stream.size);
 
-#if defined(HAVE_RWEBM) && (defined(HAVE_ROPUS) || defined(HAVE_RVORBIS))
-   /* Windowed WebM: bound the container header parse to the resident
-    * head, so opening does not walk the segment to the end of a file
-    * whose middle is reserved rather than populated. */
+#if (defined(HAVE_RWEBM) && (defined(HAVE_ROPUS) || defined(HAVE_RVORBIS))) \
+ || (defined(HAVE_RAAC) && defined(HAVE_RMP4))
+   /* Windowed container: bound the header parse to the resident head,
+    * so opening does not walk to the end of a file whose middle is
+    * reserved rather than populated.  MP4/AAC takes this too now that
+    * its arm honours the bound - the guard used to admit only the
+    * WebM-backed arms, so an M4A stream could be handed an avail that
+    * was then compiled out and silently ignored. */
    if (sound->avail)
       audio_transfer_set_avail(xfer, type, sound->avail);
 #endif
@@ -1785,9 +1789,10 @@ static bool audio_mixer_play_stream_s16(
       return false;
    audio_transfer_set_buffer_ptr(xfer, type,
          (void*)sound->types.stream.data, sound->types.stream.size);
-#if defined(HAVE_RWEBM) && (defined(HAVE_ROPUS) || defined(HAVE_RVORBIS))
-   /* Windowed WebM: bound the header parse to the head (see the f32
-    * path). */
+#if (defined(HAVE_RWEBM) && (defined(HAVE_ROPUS) || defined(HAVE_RVORBIS))) \
+ || (defined(HAVE_RAAC) && defined(HAVE_RMP4))
+   /* Windowed container: bound the header parse to the head (see the
+    * f32 path). */
    if (sound->avail)
       audio_transfer_set_avail(xfer, type, sound->avail);
 #endif

libretro-common/formats/audio_transfer.c

@@ -882,6 +882,10 @@ struct audio_transfer_aac
     * track. */
    const uint8_t *buf;
    size_t      buf_size;
+   /* Resident prefix of 'buf' for a windowed caller; 0 means all of
+    * it.  Raised through audio_transfer_set_avail as the window
+    * slides, exactly as the Vorbis and Opus arms take it. */
+   size_t      avail;
    int         adts;        /* buffer is an ADTS stream                  */
    size_t      adts_pos;    /* byte cursor of the next ADTS frame        */
 #ifdef HAVE_RMP4
@@ -1535,6 +1539,28 @@ void audio_transfer_set_avail(void *data, enum audio_type_enum type,
             rwebm_set_avail(op->demux, avail);
          return;
       }
+#endif
+#ifdef HAVE_RAAC
+      case AUDIO_TYPE_AAC:
+      {
+         struct audio_transfer_aac *ac = (struct audio_transfer_aac*)data;
+         if (!ac)
+            return;
+         ac->avail = avail;
+#ifdef HAVE_RMP4
+         /* MP4: the sample tables address the whole file, so the
+          * demuxer must be told which of those bytes have actually
+          * arrived - it withholds a packet whose extent is past the
+          * bound rather than reading reserved pages. */
+         if (ac->demux)
+            rmp4_set_avail(ac->demux, avail);
+#endif
+#ifdef HAVE_RWEBM
+         if (ac->wdemux)
+            rwebm_set_avail(ac->wdemux, avail);
+#endif
+         return;
+      }
 #endif
       default:
          break;
@@ -3252,7 +3278,16 @@ bool audio_transfer_start(void *data, enum audio_type_enum type)
          {
             const rmp4_track *at = NULL;
             int i;
-            if (!(ac->demux = rmp4_open_memory(ac->buf, ac->buf_size)))
+            /* Bounded open: a windowed caller has only a prefix
+             * resident, and the unbounded open walks the box tree to
+             * the end of the file - which for a leading-moov movie is
+             * a reserved-page read past the head.  avail == 0 is the
+             * whole-file caller and behaves exactly as before. */
+            if (!(ac->demux = rmp4_open_memory_avail(ac->buf,
+                        ac->buf_size,
+                        (ac->avail && ac->avail < ac->buf_size)
+                           ? ac->avail : ac->buf_size,
+                        NULL, NULL, NULL)))
                return false;
             ac->track_idx = -1;
             for (i = 0; i < rmp4_num_tracks(ac->demux); i++)
@@ -3941,6 +3976,10 @@ static int audio_transfer_aac_pull(struct audio_transfer_aac *ac,
       const uint8_t **pdata, uint32_t *plen)
 {
    {
+      /* Windowed ADTS: stop at the resident prefix rather than the
+       * buffer end, and say "again" instead of "end of stream". */
+      if (ac->adts && ac->avail && ac->adts_pos + 7 > ac->avail)
+         return 2;
       if (ac->adts)
       {
          /* walk the next ADTS frame: 12-bit sync, CRC flag choosing a
@@ -3987,7 +4026,14 @@ static int audio_transfer_aac_pull(struct audio_transfer_aac *ac,
          rmp4_packet pkt;
          for (;;)
          {
-            if (rmp4_read_packet(ac->demux, &pkt) != 1)
+            int r = rmp4_read_packet(ac->demux, &pkt);
+            /* The sample's bytes are not resident yet.  This is NOT
+             * end of stream: reporting 0 here would have the mixer
+             * loop or stop the voice mid-file the moment the feeder
+             * fell a window behind. */
+            if (r == RMP4_READ_AGAIN)
+               return 2;
+            if (r != 1)
                return 0;
             if (pkt.track == ac->track_idx)
                break;
@@ -4054,7 +4100,11 @@ static int64_t audio_transfer_aac_seek_to(struct audio_transfer_aac *ac,
       uint32_t plen;
       int64_t d = flen;
       int r = audio_transfer_aac_pull(ac, &pdata, &plen);
-      if (r <= 0)
+      /* r == 2 left pdata/plen unset: a target past the resident wall
+       * is refused rather than approximated, per the windowing
+       * contract, and the caller retries once the feeder has raised
+       * the bound past it. */
+      if (r == 2 || r <= 0)
          return -1;              /* asked past the end of the stream   */
       /* the encoder delay is dropped rather than emitted, so it does
        * not count towards the position */
@@ -4074,7 +4124,7 @@ static int64_t audio_transfer_aac_seek_to(struct audio_transfer_aac *ac,
       int r = audio_transfer_aac_fill(ac);
       if (r < 0)
          return -1;
-      if (r == 0)
+      if (r == 0 || r == 2)   /* 2: refused, see the walk above */
          return -1;
       take = frame - pos;
       if (take > (int64_t)ac->pend_frames)
@@ -4095,6 +4145,8 @@ static int audio_transfer_aac_fill(struct audio_transfer_aac *ac)
       int r;
       uint64_t skip;
       r = audio_transfer_aac_pull(ac, &pdata, &plen);
+      if (r == 2)
+         return 2;               /* not yet resident; no frames, no EOF */
       if (r <= 0)
          return r;
       r = raac_decode_f32(ac->handle, pdata, plen, ac->pend_f32);
@@ -4249,7 +4301,11 @@ int audio_transfer_read_s16(void *data, enum audio_type_enum type,
             int r = audio_transfer_aac_fill(ac);
             if (r < 0)
                return AUDIO_PROCESS_ERROR;
-            if (r == 0)
+            /* 2 is "not resident yet".  It must break like end of
+             * stream and not fall through: pend_frames is 0 there, so
+             * take would be 0 and this loop would spin forever on a
+             * window the feeder has not caught up with. */
+            if (r == 0 || r == 2)
                break;
             take = frames - produced;
             if (take > ac->pend_frames)
@@ -4544,7 +4600,7 @@ int audio_transfer_read_f32(void *data, enum audio_type_enum type,
             int r = audio_transfer_aac_fill(ac);
             if (r < 0)
                return AUDIO_PROCESS_ERROR;
-            if (r == 0)
+            if (r == 0 || r == 2)   /* see the s16 arm: 2 must break */
                break;
             take = frames - produced;
             if (take > ac->pend_frames)
@@ -4705,8 +4761,20 @@ size_t audio_transfer_buffer_tell(void *data, enum audio_type_enum type)
           * set. */
          if (ac->handle && ac->packets)
             return ac->pkt_offset;
-         /* The MP4 and Matroska paths read where their demuxer points
-          * and expose no cursor of their own. */
+#ifdef HAVE_RMP4
+         /* MP4: the packets are decoded where the demuxer points at
+          * them in the caller's buffer, so its consumed offset is the
+          * compressed frontier a feeder needs - the same quantity
+          * rwebm_tell reports for the WebM arms.  Without this a
+          * windowed M4A stream could be given a bound but the feeder
+          * could never learn where to move it. */
+         if (ac->handle && ac->demux)
+            return rmp4_consumed(ac->demux);
+#endif
+#ifdef HAVE_RWEBM
+         if (ac->handle && ac->wdemux)
+            return rwebm_tell(ac->wdemux);
+#endif
          return 0;
       }
 #endif