Vulkan: the XFB texture handed to the frontend in place

As on Direct3D 11, a picture whose draw would be a plain copy is not
drawn: the XFB image is handed over with set_image and the frontend
reads it where it is. It goes over in shader-read layout, submitted on
the core's thread before the frontend is told, through a 2D view of its
one layer made for that picture: the frontend's shaders read a 2D
image, and a view is not kept for the next picture, as an image's
handle can come back for another image once it is destroyed. The view
goes once the core's next submission completes, which follows every
frontend frame that read it.

The image is not one of the output's, so no sync index is waited on.
The frontend keeps reading it in the frames that repeat it, so before
each repeat the output puts it back in shader-read layout if the core
used it in another one meanwhile, a copy's source say: FrameOutput gains
Repeat(), called at the end of a run that reports a repeat.

Source/Core/DolphinLibretro/Output.cpp

@@ -11,12 +11,22 @@
 #endif
 #ifdef HAS_VULKAN
 #include "VideoBackends/Vulkan/CommandBufferManager.h"
+#include "VideoBackends/Vulkan/StateTracker.h"
+#include "VideoBackends/Vulkan/VulkanContext.h"
 #endif
 
 namespace Libretro
 {
 namespace Video
 {
+static FrameOutput* s_active;
+
+void OutputRepeat()
+{
+  if (s_active)
+    s_active->Repeat();
+}
+
 #if defined(_WIN32) || defined(HAS_VULKAN)
 /* The sync indices a mask names: one past the highest set bit. */
 static u32 SlotCount(unsigned mask)
@@ -206,9 +216,33 @@ void DX12Output::Present()
 
 #ifdef HAS_VULKAN
 VKOutput::VKOutput(retro_hw_render_interface_vulkan* iface, u32 width, u32 height)
-    : m_iface(iface), m_width(width), m_height(height), m_current(0)
+    : m_direct(nullptr), m_iface(iface), m_width(width), m_height(height), m_current(0)
 {
   memset(m_images, 0, sizeof(m_images));
+  memset(&m_direct_image, 0, sizeof(m_direct_image));
+  s_active = this;
+}
+
+VKOutput::~VKOutput()
+{
+  if (s_active == this)
+    s_active = nullptr;
+  DropDirect();
+}
+
+void VKOutput::DropDirect()
+{
+  /* the frontend's frames that read it were submitted before anything
+   * the core submits next, so it goes once that completes */
+  if (m_direct_image.image_view != VK_NULL_HANDLE)
+  {
+    if (Vulkan::g_command_buffer_mgr)
+      Vulkan::g_command_buffer_mgr->DeferImageViewDestruction(m_direct_image.image_view);
+    else
+      vkDestroyImageView(Vulkan::g_vulkan_context->GetDevice(), m_direct_image.image_view, nullptr);
+  }
+  memset(&m_direct_image, 0, sizeof(m_direct_image));
+  m_direct = nullptr;
 }
 
 bool VKOutput::Create()
@@ -289,9 +323,68 @@ void VKOutput::Present()
                               VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
   Vulkan::g_command_buffer_mgr->SubmitCommandBuffer(false, false, true);
 
+  DropDirect();
   m_iface->set_image(m_iface->handle, &m_images[m_current], 0, nullptr, m_iface->queue_index);
   video_cb(RETRO_HW_FRAME_BUFFER_VALID, m_width, m_height, 0);
 }
+
+bool VKOutput::PresentTexture(const AbstractTexture* texture, u32 width, u32 height)
+{
+  Vulkan::VKTexture* vk_texture =
+      const_cast<Vulkan::VKTexture*>(static_cast<const Vulkan::VKTexture*>(texture));
+  retro_vulkan_image image;
+  VkImageView view = VK_NULL_HANDLE;
+
+  memset(&image, 0, sizeof(image));
+  image.image_layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+  image.create_info.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO;
+  image.create_info.image = vk_texture->GetImage();
+  image.create_info.viewType = VK_IMAGE_VIEW_TYPE_2D;
+  image.create_info.format = vk_texture->GetVkFormat();
+  image.create_info.components.r = VK_COMPONENT_SWIZZLE_R;
+  image.create_info.components.g = VK_COMPONENT_SWIZZLE_G;
+  image.create_info.components.b = VK_COMPONENT_SWIZZLE_B;
+  image.create_info.components.a = VK_COMPONENT_SWIZZLE_A;
+  image.create_info.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
+  image.create_info.subresourceRange.baseMipLevel = 0;
+  image.create_info.subresourceRange.levelCount = 1;
+  image.create_info.subresourceRange.baseArrayLayer = 0;
+  image.create_info.subresourceRange.layerCount = 1;
+  /* a view per picture: an image's handle can come back for another one
+   * once it is destroyed, so none is kept for the next */
+  if (vkCreateImageView(Vulkan::g_vulkan_context->GetDevice(), &image.create_info, nullptr,
+                        &view) != VK_SUCCESS)
+  {
+    return false;
+  }
+  image.image_view = view;
+
+  vk_texture->TransitionToLayout(Vulkan::g_command_buffer_mgr->GetCurrentCommandBuffer(),
+                                 VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+  Vulkan::g_command_buffer_mgr->SubmitCommandBuffer(false, false, true);
+
+  DropDirect();
+  m_direct_image = image;
+  m_direct = vk_texture;
+  m_iface->set_image(m_iface->handle, &m_direct_image, 0, nullptr, m_iface->queue_index);
+  video_cb(RETRO_HW_FRAME_BUFFER_VALID, width, height, 0);
+  return true;
+}
+
+void VKOutput::Repeat()
+{
+  /* the core may have read the texture since in a layout of its own, a
+   * copy's source say: back to the one the frontend reads it in, ahead
+   * of the frontend's next frame on the queue */
+  if (!m_direct || m_direct->GetLayout() == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
+    return;
+  Vulkan::StateTracker::GetInstance()->EndRenderPass();
+  Vulkan::g_command_buffer_mgr->WaitForWorkerThreadIdle();
+  m_direct->TransitionToLayout(Vulkan::g_command_buffer_mgr->GetCurrentCommandBuffer(),
+                               VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+  Vulkan::g_command_buffer_mgr->SubmitCommandBuffer(false, false, false);
+  Vulkan::StateTracker::GetInstance()->InvalidateCachedState();
+}
 #endif
 }  // namespace Video
 }  // namespace Libretro

Source/Core/DolphinLibretro/Output.h

@@ -9,6 +9,10 @@ namespace Libretro
 {
 namespace Video
 {
+/* The output in use, told when the frontend is about to show the last
+ * picture again. */
+void OutputRepeat();
+
 /* What each sync index of the frontend can name: the frontend reads the
  * picture where it is drawn, so there is one target per index the
  * frontend can be reading from while the next picture is drawn. */
@@ -77,18 +81,28 @@ class VKOutput final : public FrameOutput
 {
 public:
   VKOutput(retro_hw_render_interface_vulkan* iface, u32 width, u32 height);
+  ~VKOutput() override;
   bool Create();
 
   AbstractFramebuffer* Acquire() override;
   void Present() override;
+  bool PresentTexture(const AbstractTexture* texture, u32 width, u32 height) override;
+  void Repeat() override;
   u32 GetWidth() const override { return m_width; }
   u32 GetHeight() const override { return m_height; }
   AbstractTextureFormat GetFormat() const override { return AbstractTextureFormat::RGBA8; }
 
 private:
+  void DropDirect();
+
   retro_vulkan_image m_images[OUTPUT_MAX_SLOTS];
   std::unique_ptr<Vulkan::VKTexture> m_textures[OUTPUT_MAX_SLOTS];
   std::unique_ptr<Vulkan::VKFramebuffer> m_framebuffers[OUTPUT_MAX_SLOTS];
+  /* A texture handed on where it is, with a 2D view of its one layer of
+   * its own, made for it: the frontend reads it until another picture is
+   * handed on, so it is kept in shader-read layout meanwhile. */
+  retro_vulkan_image m_direct_image;
+  Vulkan::VKTexture* m_direct;
   retro_hw_render_interface_vulkan* m_iface;
   u32 m_width;
   u32 m_height;

Source/Core/DolphinLibretro/Video.cpp

@@ -855,7 +855,10 @@ static void CountedVideoRefresh(const void* data, unsigned width, unsigned heigh
 void EndFrame()
 {
   if (!s_frames_sent && s_last_width && s_last_height)
+  {
+    OutputRepeat();
     s_frontend_video_cb(nullptr, s_last_width, s_last_height, 0);
+  }
   s_frames_sent = 0;
 }
 }  // namespace Video

Source/Core/VideoBackends/Vulkan/VKGfx.cpp

@@ -242,6 +242,19 @@ void VKGfx::PresentBackbuffer()
   StateTracker::GetInstance()->InvalidateCachedState();
 }
 
+bool VKGfx::PresentTexture(const AbstractTexture* texture, u32 width, u32 height)
+{
+  bool presented;
+
+  /* the output makes the texture readable outside a render pass, submits
+   * on this thread and hands it on */
+  StateTracker::GetInstance()->EndRenderPass();
+  g_command_buffer_mgr->WaitForWorkerThreadIdle();
+  presented = m_output->PresentTexture(texture, width, height);
+  StateTracker::GetInstance()->InvalidateCachedState();
+  return presented;
+}
+
 void VKGfx::ExecuteCommandBuffer(bool submit_off_thread, bool wait_for_completion)
 {
   StateTracker::GetInstance()->EndRenderPass();

Source/Core/VideoBackends/Vulkan/VKGfx.h

@@ -77,6 +77,7 @@ public:
                              u32 groupsize_z, u32 groups_x, u32 groups_y, u32 groups_z) override;
   bool BindBackbuffer(const ClearColor& clear_color = {}) override;
   void PresentBackbuffer() override;
+  bool PresentTexture(const AbstractTexture* texture, u32 width, u32 height) override;
 
   SurfaceInfo GetSurfaceInfo() const override;
 

Source/Core/VideoCommon/FrameOutput.h

@@ -34,6 +34,10 @@ public:
     return false;
   }
 
+  /* Whoever shows the pictures is about to show the last one handed on
+   * again: what it reads is left the way it reads it. */
+  virtual void Repeat() {}
+
   virtual u32 GetWidth() const = 0;
   virtual u32 GetHeight() const = 0;
   virtual AbstractTextureFormat GetFormat() const = 0;