retroarch: carry the running audio across a content switch

The driver stage of a staged content load leaves the audio driver out
when its init would open the same device the same way for the next
core - same driver, device, output rate, latency, resampler and
threading, no DSP filter, time stretch or core audio callback, and the
core's float and multi-channel entries as they were. The device,
pipeline and mixer keep running through the rebuild of the rest; once
the video driver has settled the rates, the new core's rate is set in
place and the stream reset a core swap needs is done. When the rate
cannot be taken in place, the driver is opened again, the device kept
across as before. The driver lookup of the stage leaves a carried
driver's table alone.

The content_load harness's close keeps its tone playing with the
device never stopped, and a load with time stretch on sets the device
aside and takes it back.

audio/audio_driver.c

@@ -1144,6 +1144,26 @@ static struct
  * through the threaded wrapper. */
 static unsigned audio_driver_open_latency;
 static bool     audio_driver_open_wrapper;
+/* The rest of what the open device and pipeline were set up with that
+ * a content switch can change - a core's override, its float or
+ * multi-channel entries, its own audio callback - for
+ * audio_driver_session_carry_ok() to compare against. */
+static struct
+{
+   unsigned out_rate;
+   unsigned resampler_quality;
+   bool     hq_oversampling;
+   bool     virtual_surround;
+   bool     threaded_pipeline;
+   bool     priority;
+   bool     fast_cores;
+   bool     core_float;
+   bool     core_multi;
+   bool     callback;
+   char     ident[sizeof(((settings_t*)0)->arrays.audio_driver)];
+   char     device[sizeof(((settings_t*)0)->arrays.audio_device)];
+   char     resampler[sizeof(((settings_t*)0)->arrays.audio_resampler)];
+} audio_driver_opened;
 
 void audio_driver_keep_device(bool keep)
 {
@@ -1245,6 +1265,96 @@ static bool audio_driver_take_kept(audio_driver_state_t *audio_st,
    return true;
 }
 
+/* The pause tail continues the waveform, and across a core swap or a
+ * driver change there is no waveform to continue. */
+static void audio_driver_core_stream_reset(audio_driver_state_t *audio_st)
+{
+   audio_st->last_out[0]              = 0.0f;
+   audio_st->last_out[1]              = 0.0f;
+   audio_st->fade_in_frames           = 0;
+   audio_st->fade_in_pending          = false;
+   audio_st->pause_mute_frames        = 0;
+   audio_st->pause_hist_pos           = 0;
+   audio_st->pause_hist_fill          = 0;
+   audio_st->resume_topup_pending     = false;
+   retro_atomic_store_release_int(&audio_st->core_silenced, 0);
+}
+
+bool audio_driver_session_carry_ok(void)
+{
+   audio_driver_state_t *audio_st = &audio_driver_st;
+   const audio_driver_t *audio    = audio_st->current_audio;
+   settings_t *settings           = config_get_ptr();
+   const char *env                = getenv("RETROARCH_AUDIO_KEEP");
+   unsigned setting_latency       = settings->uints.audio_latency;
+   unsigned core_latency          = runloop_state_get_ptr()->audio_latency;
+   unsigned latency               = (core_latency > setting_latency)
+         ? core_latency : setting_latency;
+
+   if (     (env && env[0] == '0')
+         || !audio
+         || !audio_st->context_audio_data
+         || !(AUDIO_FLAGS_GET(audio_st) & AUDIO_FLAG_ACTIVE)
+         || !settings->bools.audio_enable
+#ifdef HAVE_THREADS
+         || (     audio_st->pipe_threaded
+               && retro_atomic_load_acquire_int(
+                     &audio_st->pipe_consumer_gone))
+#endif
+         /* What the next core brings that the open stream was not
+          * set up for */
+         || audio_st->callback.callback
+         || audio_driver_opened.callback
+         || audio_st->core_float != audio_driver_opened.core_float
+         || audio_st->core_multi != audio_driver_opened.core_multi
+#ifdef HAVE_DSP_FILTER
+         || *settings->paths.path_audio_dsp_plugin
+         || retro_atomic_load_acquire_int(&audio_st->dsp_caps)
+#endif
+         || settings->bools.audio_time_stretch
+         || settings->bools.audio_time_stretch_lowpass
+         /* What its override may have changed */
+         || latency != audio_driver_open_latency
+         || settings->uints.audio_output_sample_rate
+               != audio_driver_opened.out_rate
+         || settings->uints.audio_resampler_quality
+               != audio_driver_opened.resampler_quality
+         || settings->bools.audio_resampler_hq_oversampling
+               != audio_driver_opened.hq_oversampling
+         || settings->bools.audio_headphone_virtual_surround
+               != audio_driver_opened.virtual_surround
+         || settings->bools.audio_threaded_pipeline
+               != audio_driver_opened.threaded_pipeline
+         || settings->bools.audio_thread_priority
+               != audio_driver_opened.priority
+         || settings->bools.thread_prefer_fast_cores
+               != audio_driver_opened.fast_cores
+         || !string_is_equal(settings->arrays.audio_driver,
+               audio_driver_opened.ident)
+         || !string_is_equal(settings->arrays.audio_device,
+               audio_driver_opened.device)
+         || !string_is_equal(settings->arrays.audio_resampler,
+               audio_driver_opened.resampler))
+      return false;
+   return true;
+}
+
+bool audio_driver_session_carry(double input)
+{
+   audio_driver_state_t *audio_st = &audio_driver_st;
+
+   if (!audio_driver_set_input_rate(input))
+      return false;
+   audio_driver_core_stream_reset(audio_st);
+   audio_st->core_layout = AUDIO_LAYOUT_STEREO;
+#ifdef HAVE_REWIND
+   audio_st->rewind_ptr  = audio_st->rewind_size;
+#endif
+   RARCH_LOG("[Audio] Driver \"%s\" carried across the content switch; the device keeps running.\n",
+         audio_driver_get_ident());
+   return true;
+}
+
 static bool audio_driver_deinit_internal(bool audio_enable)
 {
    audio_driver_state_t *audio_st = &audio_driver_st;
@@ -1308,17 +1418,7 @@ static bool audio_driver_deinit_internal(bool audio_enable)
 #endif
    audio_st->transport_lpf_only = false;
 
-   /* The pause tail continues the waveform, and across a core swap or a
-    * driver change there is no waveform to continue. */
-   audio_st->last_out[0]              = 0.0f;
-   audio_st->last_out[1]              = 0.0f;
-   audio_st->fade_in_frames           = 0;
-   audio_st->fade_in_pending          = false;
-   audio_st->pause_mute_frames        = 0;
-   audio_st->pause_hist_pos           = 0;
-   audio_st->pause_hist_fill          = 0;
-   audio_st->resume_topup_pending     = false;
-   retro_atomic_store_release_int(&audio_st->core_silenced, 0);
+   audio_driver_core_stream_reset(audio_st);
 
 #ifdef HAVE_THREADS
    audio_pipeline_stretch_free(audio_st->pipe_transport);
@@ -4849,6 +4949,22 @@ bool audio_driver_init_internal(void *settings_data, bool audio_cb_inited)
    audio_driver_open_latency = audio_latency;
    audio_driver_open_wrapper = audio_cb_inited
       || (AUDIO_FLAGS_GET(&audio_driver_st) & AUDIO_FLAG_PIPELINE_THREADED);
+   audio_driver_opened.out_rate          = settings->uints.audio_output_sample_rate;
+   audio_driver_opened.resampler_quality = settings->uints.audio_resampler_quality;
+   audio_driver_opened.hq_oversampling   = settings->bools.audio_resampler_hq_oversampling;
+   audio_driver_opened.virtual_surround  = settings->bools.audio_headphone_virtual_surround;
+   audio_driver_opened.threaded_pipeline = settings->bools.audio_threaded_pipeline;
+   audio_driver_opened.priority          = settings->bools.audio_thread_priority;
+   audio_driver_opened.fast_cores        = settings->bools.thread_prefer_fast_cores;
+   audio_driver_opened.core_float        = audio_driver_st.core_float;
+   audio_driver_opened.core_multi        = audio_driver_st.core_multi;
+   audio_driver_opened.callback          = audio_cb_inited;
+   strlcpy(audio_driver_opened.ident, settings->arrays.audio_driver,
+         sizeof(audio_driver_opened.ident));
+   strlcpy(audio_driver_opened.device, settings->arrays.audio_device,
+         sizeof(audio_driver_opened.device));
+   strlcpy(audio_driver_opened.resampler, settings->arrays.audio_resampler,
+         sizeof(audio_driver_opened.resampler));
    if (audio_driver_take_kept(&audio_driver_st, settings, audio_latency,
             audio_driver_open_wrapper))
       ;

audio/audio_driver.h

@@ -1694,6 +1694,19 @@ void audio_driver_keep_device(bool keep);
 /* Frees a kept device that was not taken back. */
 void audio_driver_drop_kept(void);
 
+/* Asked before a content switch's driver stage: whether the running
+ * audio can be carried into the next session as it is - device,
+ * pipeline and mixer left up, so menu sounds play through the stage -
+ * because its init would open the same device the same way for the
+ * next core. Off with RETROARCH_AUDIO_KEEP=0, as keeping is. */
+bool audio_driver_session_carry_ok(void);
+
+/* Carries the running audio into the next session at @input Hz: the
+ * rate in place, and the stream reset that a core swap asks for.
+ * Returns false when the rate cannot be taken in place; the caller
+ * then reinits the audio driver. */
+bool audio_driver_session_carry(double input);
+
 const char *audio_driver_get_ident(void);
 
 /* The speaker path in effect, in a few words, for the statistics

driver.h

@@ -118,7 +118,15 @@ enum driver_lifetime_flags
     * unloading ends still ends - but a driver that can outlive the
     * video driver may be left running across it, as across a reset.
     */
-   DRIVER_LIFETIME_SESSION_SWITCH = 1 << 1
+   DRIVER_LIFETIME_SESSION_SWITCH = 1 << 1,
+
+   /**
+    * Passed to \c drivers_init along a content switch that left the
+    * audio driver out of the teardown: the running audio is carried
+    * into the new session - its rate set in place once the video
+    * driver has settled the rates - instead of opened again.
+    */
+   DRIVER_LIFETIME_AUDIO_CARRIED = 1 << 2
 };
 
 enum driver_ctl_state

retroarch.c

@@ -1758,6 +1758,7 @@ void drivers_init(
       *location_st                = location_state_get_ptr();
    bool video_is_threaded         = VIDEO_DRIVER_IS_THREADED_INTERNAL(video_st);
    gfx_display_t *p_disp          = disp_get_ptr();
+   double carried_input           = 0.0;
 #if defined(HAVE_GFX_WIDGETS)
    bool video_font_enable         = settings->bools.video_font_enable;
    bool menu_enable_widgets       = settings->bools.menu_enable_widgets;
@@ -1870,9 +1871,29 @@ void drivers_init(
       }
    }
 
-   /* Adjust rates */
+   /* Adjust rates. Running audio carried across a content switch takes
+    * its new rate in place, below, instead of having it set under it. */
    if (flags & (DRIVER_VIDEO_MASK | DRIVER_AUDIO_MASK))
-      driver_adjust_system_rates(runloop_st, video_st, settings, NULL);
+      driver_adjust_system_rates(runloop_st, video_st, settings,
+            (lifetime_flags & DRIVER_LIFETIME_AUDIO_CARRIED)
+            ? &carried_input : NULL);
+
+   if (     (lifetime_flags & DRIVER_LIFETIME_AUDIO_CARRIED)
+         && !audio_driver_session_carry(carried_input))
+   {
+      /* Not in place after all: open it again for the new session,
+       * the device kept across when it can be */
+      audio_driver_keep_device(true);
+      audio_driver_deinit_for_reinit();
+      audio_st->context_audio_data = NULL;
+      AUDIO_FLAGS_SET(audio_st, AUDIO_FLAG_ACTIVE);
+      if (!audio_driver_find_driver(settings->arrays.audio_driver,
+               "audio driver", verbosity_enabled))
+         retroarch_fail(1, "audio_driver_find()");
+      if (carried_input > 0.0)
+         audio_st->input = (float)carried_input;
+      flags |= DRIVER_AUDIO_MASK;
+   }
 
    /* Initialize audio driver */
    if (flags & DRIVER_AUDIO_MASK)
@@ -9234,14 +9255,18 @@ static bool retroarch_main_init_fail(void)
 /* Picks the driver of every kind from the configuration; the video,
  * input and menu ones are required.  Reports through retroarch_fail(),
  * so it runs under a live error_sjlj_context. */
+/* @audio: false when the running audio driver is carried across, so
+ * the table it runs on - the threaded wrapper's, say - is not swapped
+ * from under it. */
 static void retroarch_find_drivers(settings_t *settings,
-      bool verbosity_enabled)
+      bool verbosity_enabled, bool audio)
 {
 #ifdef HAVE_MENU
    struct menu_state *menu_st    = menu_state_get_ptr();
 #endif
-   if (!(audio_driver_find_driver(settings->arrays.audio_driver,
-         "audio driver", verbosity_enabled)))
+   if (     audio
+         && !(audio_driver_find_driver(settings->arrays.audio_driver,
+               "audio driver", verbosity_enabled)))
       retroarch_fail(1, "audio_driver_find()");
    if (!video_driver_find_driver(settings,
          "video driver", verbosity_enabled))
@@ -9616,7 +9641,7 @@ bool retroarch_main_init_core(int argc, char *argv[],
     * staged load has the previous session's drivers up at this point
     * and finds after freeing them, in retroarch_main_init_drivers(). */
    if (find_drivers)
-      retroarch_find_drivers(settings, verbosity_enabled);
+      retroarch_find_drivers(settings, verbosity_enabled, true);
 
 #ifdef HAVE_CLOUDSYNC
    /* Content from the command line loads its save RAM below, before
@@ -9728,6 +9753,8 @@ bool retroarch_main_init_drivers(bool staged,
       *input_st                  = input_state_get_ptr();
    settings_t *settings          = config_get_ptr();
    recording_state_t *rec_st     = recording_state_get_ptr();
+   int init_flags                = DRIVERS_CMD_ALL;
+   int init_lifetime             = 0;
 
    if (setjmp(rarch_st.error_sjlj_context) > 0)
    {
@@ -9747,17 +9774,27 @@ bool retroarch_main_init_drivers(bool staged,
        * once it is back. */
       struct video_hw_request hw_request;
       video_driver_hw_request_take(&hw_request);
+      /* Audio the next session would open the same way stays up
+       * through the stage, so what plays over the switch - the menu's
+       * sounds - is not paused for the rebuild of the rest. */
+      if (audio_driver_session_carry_ok())
+      {
+         init_flags     &= ~DRIVER_AUDIO_MASK;
+         init_lifetime   = DRIVER_LIFETIME_AUDIO_CARRIED;
+      }
       /* drivers_init() is a few lines down: an input driver that can
        * outlive the video driver is left running across the two */
-      driver_uninit(DRIVERS_CMD_ALL, DRIVER_LIFETIME_SESSION_SWITCH);
+      driver_uninit(init_flags, DRIVER_LIFETIME_SESSION_SWITCH);
       video_driver_hw_request_restore(&hw_request);
 
       video_state_get_ptr()->main_flags |=  VIDEO_FLAG_ACTIVE;
       AUDIO_FLAGS_SET(audio_state_get_ptr(), AUDIO_FLAG_ACTIVE);
-      retroarch_find_drivers(settings, verbosity_enabled);
+      retroarch_find_drivers(settings, verbosity_enabled,
+            (init_flags & DRIVER_AUDIO_MASK) != 0);
    }
 
-   drivers_init(settings, DRIVERS_CMD_ALL, (enum driver_lifetime_flags)0, verbosity_enabled);
+   drivers_init(settings, init_flags,
+         (enum driver_lifetime_flags)init_lifetime, verbosity_enabled);
 #ifdef HAVE_COMMAND
    input_driver_refresh_command(input_st, settings);
 #endif

samples/tasks/content_load/content_load_test.c

@@ -41,8 +41,8 @@
  *    before a playlist Run or the one after it, nothing in between;
  *  - menu sound close: a menu sound playing when Close Content is
  *    confirmed plays on through the staged close and past it - the
- *    device never handed audio while stopped, the sound not reloaded
- *    under itself - and the tone handed over whole;
+ *    device carried through the driver stage, never stopped, the
+ *    sound not reloaded under itself - and the tone handed over whole;
  *  - quit waits for a save: a quit from the frame loop while a state
  *    is being written returns to the loop at once, which presents
  *    frames - the core not run - until the state is written, and only
@@ -466,6 +466,7 @@ static void lane_audio_kept(void)
    struct load_frame log[LOAD_FRAMES];
    audio_driver_t saved = audio_null;
    unsigned had         = failures;
+   bool time_stretch    = config_get_ptr()->bools.audio_time_stretch;
    unsigned n;
 
    audio_null.init               = kept_init;
@@ -493,6 +494,25 @@ static void lane_audio_kept(void)
          "across the load the audio device was closed %u and opened %u time(s)",
          kept_frees, kept_opens);
 
+   /* Time stretch keeps the rate from changing in place: the device
+    * is set aside through the driver stage and taken back instead. */
+   kept_opens = kept_frees = 0;
+   configuration_set_bool(config_get_ptr(),
+         config_get_ptr()->bools.audio_time_stretch, true);
+   open_menu();
+   CHECK(task_push_load_contentless_core_from_menu(core_path),
+         "the time-stretch load was not started");
+   n = run_load(log, LOAD_FRAMES);
+   configuration_set_bool(config_get_ptr(),
+         config_get_ptr()->bools.audio_time_stretch, time_stretch);
+   CHECK(n < LOAD_FRAMES && core_is_up(),
+         "the time-stretch load did not bring the core up within %u frames",
+         LOAD_FRAMES);
+   CHECK(kept_opens == 0 && kept_frees == 0
+         && audio_state_get_ptr()->context_audio_data == &kept_token,
+         "with time stretch, across the load the audio device was closed"
+         " %u and opened %u time(s)", kept_frees, kept_opens);
+
    kept_opens = kept_frees = 0;
    putenv((char*)"RETROARCH_AUDIO_KEEP=0");
    open_menu();
@@ -698,7 +718,7 @@ static void lane_menu_sound_close(void)
          && runloop_state_get_ptr()->current_core_type == CORE_TYPE_DUMMY,
          "the close did not finish");
 
-   CHECK(tap_stops <= 1 && tap_writes_stopped == 0,
+   CHECK(tap_stops == 0 && tap_writes_stopped == 0,
          "the device was stopped %u time(s) across the close, and handed"
          " %u write(s) while stopped", tap_stops, tap_writes_stopped);
    worst = tap_worst_step(0);