video: software filters draw into memory the driver lends

A filtered frame was drawn into the frontend's buffer and copied into
the driver's frame memory. It now goes straight there where the driver
lends memory that reads back at cached speed, on the video thread too,
and into the frontend's buffer otherwise.

The Vulkan lend left a staging frame's optimal texture behind when a
format change made the frame texture streamed, sampling the old format.

gfx/drivers/vulkan.c

@@ -14710,6 +14710,9 @@ static bool vulkan_get_current_sw_framebuffer(void *data,
                chain->texture.format, /* Ensure we use the non-remapped format. */
                NULL, NULL, VULKAN_TEXTURE_DYNAMIC);
       }
+      /* Sampled whenever it exists, and of the old format */
+      else if (chain->texture_optimal.memory != VK_NULL_HANDLE)
+         vulkan_texture_defer_copy(vk, &chain->texture_optimal);
    }
 
    /* If the driver picked a row pitch wider than width*bpp for this linear

gfx/video_driver.c

@@ -6960,6 +6960,14 @@ void video_driver_cached_frame_invalidate(void)
  * the cached pointer is one of its own allocations. */
 void video_driver_cached_frame_retire(void)
 {
+#ifdef HAVE_THREADS
+   {
+      size_t keeper = retro_atomic_load_acquire_size(
+            &video_driver_st.frame_cache_keeper);
+      if (keeper && keeper == (size_t)sthread_get_current_thread_id())
+         return;
+   }
+#endif
    video_driver_cached_frame_invalidate();
 #ifdef FRAME_CACHE_HAZARDS
    /* Bump first, then scan: a reader that armed before the bump is
@@ -10033,14 +10041,19 @@ void video_driver_frame(const void *data, unsigned width,
    {
       unsigned output_dims  = 0;
       unsigned output_pitch = 0;
+      void    *output       = NULL;
 
       rarch_softfilter_get_output_size(video_st->state_filter,
             &output_dims, dims);
 
       output_pitch = VIDEO_SCALE_W(output_dims) * video_st->state_out_bpp;
+      if (!(output = video_driver_filter_lend(video_st->poke,
+                  video_st->data, output_dims, video_st->state_out_bpp,
+                  &output_pitch)))
+         output    = video_st->state_buffer;
 
       rarch_softfilter_process(video_st->state_filter,
-            video_st->state_buffer, output_pitch,
+            output, output_pitch,
             data, dims, pitch);
 
       if (     video_info.post_filter_record
@@ -10048,10 +10061,10 @@ void video_driver_frame(const void *data, unsigned width,
             && recording_st->driver
             && recording_st->driver->push_video)
          recording_dump_frame(
-               video_st->state_buffer, output_dims, output_pitch,
+               output, output_dims, output_pitch,
                runloop_idle, runloop_st->perfcnt_enable);
 
-      data   = video_st->state_buffer;
+      data   = output;
       dims   = output_dims;
       pitch  = output_pitch;
    }

gfx/video_driver.h

@@ -1826,6 +1826,10 @@ typedef struct
     * the main thread only, once the wrapper is built and once it has
     * been torn down and the thread joined. */
    bool thread_wrapper_active;
+#ifdef HAVE_THREADS
+   /* The id of a thread whose retires leave the cached frame be */
+   retro_atomic_size_t frame_cache_keeper;
+#endif
 #if defined(HAVE_THREADS) && !defined(RETRO_ATOMIC_HAS_PTR)
    /* Only the backend without pointer atomics has a lock here, and
     * only for the window-title fallback protocol; on every other
@@ -2590,6 +2594,34 @@ float video_driver_get_aspect_ratio(void);
 /* The aspect ratio, as the bits of the float in an atomic word. A
  * union, not a cast through a pointer, so it stays inside what the
  * standard allows and MSVC's aliasing rules. */
+/* Where a software filter draws: memory the driver lends, read back at
+ * cached speed as filters may, in the format and size of the frames it
+ * makes. NULL when there is none; *pitch is then left alone. */
+static INLINE void *video_driver_filter_lend(
+      const video_poke_interface_t *poke, void *driver_data,
+      unsigned dims, unsigned bpp, unsigned *pitch)
+{
+   struct retro_framebuffer fb;
+   if (     !poke || !poke->get_current_software_framebuffer || !driver_data)
+      return NULL;
+   fb.data         = NULL;
+   fb.width        = VIDEO_SCALE_W(dims);
+   fb.height       = VIDEO_SCALE_H(dims);
+   fb.pitch        = 0;
+   fb.format       = RETRO_PIXEL_FORMAT_UNKNOWN;
+   fb.access_flags = RETRO_MEMORY_ACCESS_READ | RETRO_MEMORY_ACCESS_WRITE;
+   fb.memory_flags = 0;
+   if (     !poke->get_current_software_framebuffer(driver_data, &fb)
+         || !fb.data
+         || !(fb.memory_flags & RETRO_MEMORY_TYPE_CACHED)
+         || fb.format != (bpp == sizeof(uint32_t)
+            ? RETRO_PIXEL_FORMAT_XRGB8888 : RETRO_PIXEL_FORMAT_RGB565)
+         || fb.pitch < (size_t)fb.width * bpp)
+      return NULL;
+   *pitch = (unsigned)fb.pitch;
+   return fb.data;
+}
+
 static INLINE float video_driver_aspect_ratio_of(
       const retro_atomic_int_t *bits)
 {

gfx/video_thread_wrapper.c

@@ -2256,11 +2256,13 @@ void video_thread_defer_convert(enum video_thread_convert kind)
  * video_thread_defer_filter() arrives raw, in the core's format, with
  * the filter it was staged with. One replaced since is retired behind
  * the frames that name it (video_thread_retire_filter()). */
-static void video_thread_filter(void *filter, void *buffer,
-      unsigned out_bpp, const void **data, unsigned *dims, unsigned *pitch)
+static void video_thread_filter(thread_video_t *lend_thr, void *filter,
+      void *buffer, unsigned out_bpp, const void **data, unsigned *dims,
+      unsigned *pitch)
 {
    unsigned out_dims = 0;
    unsigned out_pitch;
+   void    *lent     = NULL;
 
    if (!*data || !filter || !buffer)
       return;
@@ -2268,6 +2270,19 @@ static void video_thread_filter(void *filter, void *buffer,
    rarch_softfilter_get_output_size((rarch_softfilter_t*)filter,
          &out_dims, *dims);
    out_pitch = VIDEO_SCALE_W(out_dims) * out_bpp;
+   /* Remaking its frame texture for this, a driver retires the cached
+    * frame. That is the main thread's, and never in this memory. */
+   if (lend_thr)
+   {
+      video_driver_state_t *video_st = lend_thr->video_st;
+      retro_atomic_store_release_size(&video_st->frame_cache_keeper,
+            (size_t)sthread_get_current_thread_id());
+      lent = video_driver_filter_lend(lend_thr->poke, lend_thr->driver_data,
+            out_dims, out_bpp, &out_pitch);
+      retro_atomic_store_release_size(&video_st->frame_cache_keeper, 0);
+   }
+   if (lent)
+      buffer = lent;
    rarch_softfilter_process((rarch_softfilter_t*)filter, buffer,
          out_pitch, *data, *dims, *pitch);
 
@@ -2998,7 +3013,7 @@ static void video_thread_loop(void *data)
                            &fdata, fdims, &fpitch);
 #ifdef HAVE_VIDEO_FILTER
                   if (fdata && thr->frame.slot[slot].filter_bpp)
-                     video_thread_filter(thr->frame.slot[slot].filter,
+                     video_thread_filter(thr, thr->frame.slot[slot].filter,
                            thr->frame.slot[slot].filter_buf,
                            thr->frame.slot[slot].filter_out_bpp,
                            &fdata, &fdims, &fpitch);
@@ -3908,7 +3923,7 @@ static VIDEO_NOINLINE bool video_thread_frame_replay(thread_video_t *thr,
          video_thread_convert(thr, convert, &frame_, dims, &pitch);
 #ifdef HAVE_VIDEO_FILTER
       if (filter_bpp)
-         video_thread_filter(thr->video_st->state_filter,
+         video_thread_filter(NULL, thr->video_st->state_filter,
                thr->video_st->state_buffer, thr->video_st->state_out_bpp,
                &frame_, &dims, &pitch);
 #endif

samples/gfx/video_filter_menu/video_filter_menu_test.c

@@ -24,6 +24,9 @@
  *    with the format and scale the frames now have, nothing is set up
  *    again, and a change the frames do not make tells it nothing.
  *
+ * With the null driver a stand-in poke lends memory for the filter to
+ * draw into: taken when cached, declined when not.
+ *
  * HARNESS_VIDEO_DRIVER=gl, glcore, vulkan or (on macOS) metal, under a
  * display (Xvfb on Linux), runs the same steps on that driver, which
  * takes them in place: nothing is set up again, and after each step
@@ -529,6 +532,101 @@ static void lane_told_in_place(void)
             "told each change once, with what the frames now are\n");
 }
 
+/* A filter draws straight into memory the driver lends, at the driver's
+ * pitch, while that memory reads back at cached speed; otherwise into
+ * the frontend's own buffer. */
+#define LEND_PITCH (512 * sizeof(uint32_t))
+static uint32_t     lend_mem[512 * 400];
+static unsigned     lend_flags;
+static const void  *lend_seen;
+static unsigned     lend_seen_pitch;
+static bool (*lend_inner)(void*, const void*, unsigned, uint64_t, unsigned,
+      const char*, video_frame_info_t*);
+
+static bool lend_framebuffer(void *data, struct retro_framebuffer *fb)
+{
+   (void)data;
+   fb->data         = lend_mem;
+   fb->pitch        = LEND_PITCH;
+   fb->format       = RETRO_PIXEL_FORMAT_XRGB8888;
+   fb->memory_flags = lend_flags;
+   return fb->width <= 512 && fb->height <= 400;
+}
+
+static bool lend_frame(void *data, const void *frame, unsigned dims,
+      uint64_t frame_count, unsigned pitch, const char *msg,
+      video_frame_info_t *video_info)
+{
+   lend_seen       = frame;
+   lend_seen_pitch = pitch;
+   return lend_inner(data, frame, dims, frame_count, pitch, msg, video_info);
+}
+
+static void lend_drain(void)
+{
+   pump(4);
+#ifdef HAVE_THREADS
+   if (video_state_get_ptr()->thread_wrapper_active)
+      video_thread_wait_idle();
+#endif
+}
+
+static void lane_filter_lent(void)
+{
+   video_driver_state_t *video_st = video_state_get_ptr();
+   unsigned had = failures;
+   size_t i;
+   bool drawn   = false;
+
+   choose_filter("widen32.filt");       /* 480x320, XRGB8888 */
+   menu_open(false);
+   pump(2);
+   if (video_st->thread_wrapper_active)
+   {
+#ifdef HAVE_THREADS
+      /* Drawn on the video thread, into what its driver lends */
+      thread_video_t *thr = (thread_video_t*)video_st->data;
+      video_thread_wait_idle();
+      if (thr->poke)
+         fake_poke = *thr->poke;
+      fake_poke.get_current_software_framebuffer = lend_framebuffer;
+      thr->poke  = &fake_poke;
+      lend_inner = thr->driver->frame;
+      hold_driver       = *thr->driver;
+      hold_driver.frame = lend_frame;
+      video_thread_run_blocking(hold_install, &hold_driver);
+#endif
+   }
+   else
+   {
+      hook_install();
+      fake_poke                                  = *video_st->poke;
+      fake_poke.get_current_software_framebuffer = lend_framebuffer;
+      video_st->poke                             = &fake_poke;
+      lend_inner                                 = hooked_video.frame;
+      hooked_video.frame                         = lend_frame;
+   }
+
+   lend_flags = RETRO_MEMORY_TYPE_CACHED;
+   lend_drain();
+   CHECK(lend_seen == lend_mem, "a filter draws into cached lent memory");
+   CHECK(lend_seen_pitch == LEND_PITCH, "at the pitch the driver lent");
+   for (i = 0; i < sizeof(lend_mem) / sizeof(lend_mem[0]); i++)
+      drawn = drawn || lend_mem[i];
+   CHECK(drawn, "and the picture is there");
+
+   lend_flags = 0;
+   lend_drain();
+   CHECK(lend_seen == video_st->state_buffer,
+         "uncached lent memory is left alone");
+   CHECK(lend_seen_pitch == 480 * sizeof(uint32_t),
+         "and the frontend's buffer is packed");
+
+   menu_open(true);
+   if (failures == had)
+      fprintf(stderr, "[pass] a filter draws where the driver lends\n");
+}
+
 /* ------------------------------------------------------------------ */
 
 int main(int argc, char *argv[])
@@ -670,7 +768,10 @@ int main(int argc, char *argv[])
    lane_swap_unwaited();
 #endif
    if (!real_driver)
+   {
       lane_told_in_place();
+      lane_filter_lent();
+   }
 
 #ifdef __APPLE__
    /* main_exit() terminates the Cocoa application; the status goes