(Vulkan) Present on a queue of its own where the family has one

vkQueuePresentKHR shared the graphics queue and so its lock, and on
some drivers the exclusive-fullscreen present waits for the display
inside the call - holding the lock a hardware core's submissions go
through. The frame fence wait before the present kept that from
stopping the frontend, at the cost of a wait on the video thread.

The present now goes to a second queue of the graphics family when the
device offers one: requested on the default path, and taken from a
core that creates the device through the negotiation interface when
its presentation_queue is a different queue of the same family (a
different family would need an image ownership transfer around every
frame, and is still refused). On that queue the present holds no lock,
the core's lock_queue never waits behind the display, and the fence
wait before the present is not needed and not done. A device with one
queue in the family keeps the shared queue, its lock and the wait.
Presentation on the second queue is synchronised with the render on the
first by the swapchain semaphore that already ordered them.

gfx/common/vulkan_common.c

@@ -81,6 +81,10 @@ static dylib_t                       vulkan_library;
 static VkInstance                    cached_instance_vk;
 static VkDevice                      cached_device_vk;
 static retro_vulkan_destroy_device_t cached_destroy_device_vk;
+/* Index in the graphics family of the present queue the cached device
+ * was created with; 0 means it shares the graphics queue. A queue is
+ * only a handle, so the index is what survives a cached reuse. */
+static uint32_t                      cached_present_queue_index_vk;
 
 #ifdef __APPLE__
 /* On Apple platforms the Vulkan implementation is provided by MoltenVK
@@ -765,7 +769,8 @@ static bool vulkan_context_init_device(gfx_ctx_vulkan_data_t *vk)
    const char *enabled_device_extensions[8];
    VkDeviceCreateInfo device_info;
    VkDeviceQueueCreateInfo queue_info;
-   static const float one                  = 1.0f;
+   static const float priorities[2]        = { 1.0f, 1.0f };
+   uint32_t present_queue_index            = 0;
    bool found_queue                        = false;
 #ifdef VULKAN_HDR_SWAPCHAIN
    bool hdr_metadata_enabled               = false;
@@ -869,15 +874,28 @@ static bool vulkan_context_init_device(gfx_ctx_vulkan_data_t *vk)
          vk->context.destroy_device       = iface->destroy_device;
 
          vk->context.device               = context.device;
-         vk->context.queue                = context.queue;
          vk->context.gpu                  = context.gpu;
          vk->context.graphics_queue_index = context.queue_family_index;
          vk->context.queue                = context.queue;
+         vk->context.present_queue        = context.queue;
 
+         /* A separate presentation queue is taken when it is in the
+          * graphics family: the swapchain images then need no
+          * ownership transfer between the queue that renders them and
+          * the one that presents them. Another family would need one
+          * around every frame, and is not supported. */
          if (context.presentation_queue != context.queue)
          {
-            RARCH_ERR("[Vulkan] Present queue != graphics queue. This is currently not supported.\n");
-            return false;
+            if (context.presentation_queue_family_index
+                  != context.queue_family_index)
+            {
+               RARCH_ERR("[Vulkan] Present queue is in queue family %u, graphics queue in %u. This is not supported.\n",
+                     context.presentation_queue_family_index,
+                     context.queue_family_index);
+               return false;
+            }
+            vk->context.present_queue = context.presentation_queue;
+            RARCH_LOG("[Vulkan] Core provided a separate presentation queue.\n");
          }
       }
       else
@@ -963,6 +981,10 @@ static bool vulkan_context_init_device(gfx_ctx_vulkan_data_t *vk)
             vk->context.graphics_queue_index = i;
             RARCH_LOG("[Vulkan] Queue family %u supports %u sub-queues.\n",
                   i, queue_properties[i].queueCount);
+            /* A second queue of the family, when there is one, takes
+             * the presents off the graphics queue and its lock. */
+            if (queue_properties[i].queueCount >= 2)
+               present_queue_index = 1;
             found_queue = true;
             break;
          }
@@ -1019,8 +1041,8 @@ static bool vulkan_context_init_device(gfx_ctx_vulkan_data_t *vk)
 #endif
 
       queue_info.queueFamilyIndex         = vk->context.graphics_queue_index;
-      queue_info.queueCount               = 1;
-      queue_info.pQueuePriorities         = &one;
+      queue_info.queueCount               = present_queue_index ? 2 : 1;
+      queue_info.pQueuePriorities         = priorities;
 
       device_info.queueCreateInfoCount    = 1;
       device_info.pQueueCreateInfos       = &queue_info;
@@ -1030,8 +1052,10 @@ static bool vulkan_context_init_device(gfx_ctx_vulkan_data_t *vk)
 
       if (cached_device_vk)
       {
-         vk->context.device = cached_device_vk;
-         cached_device_vk   = NULL;
+         vk->context.device  = cached_device_vk;
+         cached_device_vk    = NULL;
+         /* The device was created once, with the queues it has. */
+         present_queue_index = cached_present_queue_index_vk;
 
          if (cached_destroy_device_vk)
          {
@@ -1087,7 +1111,19 @@ static bool vulkan_context_init_device(gfx_ctx_vulkan_data_t *vk)
    {
       vkGetDeviceQueue(vk->context.device,
             vk->context.graphics_queue_index, 0, &vk->context.queue);
+      vk->context.present_queue = vk->context.queue;
+      if (present_queue_index)
+      {
+         vkGetDeviceQueue(vk->context.device,
+               vk->context.graphics_queue_index, present_queue_index,
+               &vk->context.present_queue);
+         RARCH_LOG("[Vulkan] Presenting on queue %u of family %u.\n",
+               present_queue_index, vk->context.graphics_queue_index);
+      }
    }
+   else if (vk->context.present_queue == VK_NULL_HANDLE)
+      vk->context.present_queue = vk->context.queue;
+   cached_present_queue_index_vk = present_queue_index;
 
 #ifdef HAVE_THREADS
    vk->context.queue_lock = slock_new();
@@ -3333,6 +3369,7 @@ retro_time_t vulkan_last_present_time(gfx_ctx_vulkan_data_t *vk)
 
 void vulkan_present(gfx_ctx_vulkan_data_t *vk, unsigned index)
 {
+   VkQueue present_queue;
    VkPresentInfoKHR present;
    VkPresentTimesInfoGOOGLE times;
    VkPresentTimeGOOGLE ptime;
@@ -3360,11 +3397,18 @@ void vulkan_present(gfx_ctx_vulkan_data_t *vk, unsigned index)
       present.pNext              = ×
    }
 
-   /* Better hope QueuePresent doesn't block D: */
+   /* On its own queue the present holds no lock, and a blocking
+    * present (the exclusive-fullscreen path on some drivers waits for
+    * the display inside the call) holds nothing a hardware core's
+    * submissions need. Sharing the graphics queue it takes queue_lock,
+    * as any use of that queue must. */
+   present_queue = vk->context.present_queue
+      ? vk->context.present_queue : vk->context.queue;
 #ifdef HAVE_THREADS
-   slock_lock(vk->context.queue_lock);
+   if (present_queue == vk->context.queue)
+      slock_lock(vk->context.queue_lock);
 #endif
-   err = vkQueuePresentKHR(vk->context.queue, &present);
+   err = vkQueuePresentKHR(present_queue, &present);
 
    /* VK_SUBOPTIMAL_KHR can be returned on
     * Android 10 when prerotate is not dealt with.
@@ -3389,6 +3433,7 @@ void vulkan_present(gfx_ctx_vulkan_data_t *vk, unsigned index)
    }
 
 #ifdef HAVE_THREADS
-   slock_unlock(vk->context.queue_lock);
+   if (present_queue == vk->context.queue)
+      slock_unlock(vk->context.queue_lock);
 #endif
 }

gfx/common/vulkan_common.h

@@ -214,6 +214,15 @@ typedef struct vulkan_context
    VkPhysicalDevice gpu;
    VkDevice device;
    VkQueue queue;
+   /* The queue presents go to. Distinct from queue where the device
+    * offers a second queue in the graphics family - requested on the
+    * default path, and accepted as presentation_queue from a core that
+    * creates the device - so vkQueuePresentKHR needs no lock at all:
+    * only the frame thread touches it, and a hardware core submitting
+    * through lock_queue is never held behind a present that is waiting
+    * on the display. Equal to queue when the family has one queue, in
+    * which case the present shares queue_lock as before. */
+   VkQueue present_queue;
 
    VkPhysicalDeviceProperties gpu_properties;
    VkPhysicalDeviceMemoryProperties memory_properties;

gfx/drivers/vulkan.c

@@ -7204,13 +7204,16 @@ static void vulkan_retain_backbuffer(vk_t *vk, struct vk_image *backbuffer)
  * So for a hardware core under the wrapper the frame's fence is waited
  * out first, with the lock free: the core submits, the queue drains,
  * the fence signals, and the present under the lock has nothing left
- * to wait for. A software core, or a hardware core presenting on its
- * own thread, has no other submitter to wait on and keeps the
- * asynchronous present; those paths are exactly as they were. */
+ * to wait for. Only where the present shares the graphics queue: on a
+ * present queue of its own it holds no lock, and needs no wait. A
+ * software core, or a hardware core presenting on its own thread, has
+ * no other submitter to wait on and keeps the asynchronous present;
+ * those paths are exactly as they were. */
 static void vulkan_await_frame_before_present(vk_t *vk, unsigned frame_index)
 {
    if (     (vk->flags & VK_FLAG_HW_ENABLE)
          && video_driver_thread_wrapper_active()
+         && vk->context->present_queue == vk->context->queue
          && vk->context->swapchain_fences_signalled[frame_index]
          && vk->context->swapchain_fences[frame_index] != VK_NULL_HANDLE)
       vkWaitForFences(vk->context->device, 1,