(Vulkan) A swapchain rebuild waits on its own frames, not the device

vulkan_create_swapchain() and vulkan_destroy_swapchain() drained the
whole device, without the queue lock, on every resize, fullscreen
toggle and out-of-date acquire: a threaded hardware core's queued work
was waited for by a window event, unsynchronised with its submissions.
The only work that references the swapchain images from this side is
the context's own frames, each with a fence, so a rebuild or teardown
now waits on those and nothing else; the old swapchain retires through
oldSwapchain as before, and the video driver rebuilds its framebuffers
after waiting on the same fences. The one drain left in the context is
the teardown before the device itself is destroyed.

gfx/common/vulkan_common.c

@@ -1550,6 +1550,30 @@ end:
    return ret;
 }
 
+/* Waits, by fence, for every frame this context submitted - the only
+ * work that references the swapchain images from this side. This is
+ * what a swapchain teardown or rebuild has to outlive; the rest of the
+ * device's work is a hardware core's, has nothing to do with the
+ * swapchain, and is not drained for it. Needs no queue lock. */
+static void vulkan_context_wait_frames(gfx_ctx_vulkan_data_t *vk)
+{
+   VkFence fences[VULKAN_MAX_SWAPCHAIN_IMAGES];
+   unsigned count = 0;
+   unsigned i;
+
+   if (vk->context.device == VK_NULL_HANDLE)
+      return;
+   for (i = 0; i < VULKAN_MAX_SWAPCHAIN_IMAGES; i++)
+   {
+      if (     vk->context.swapchain_fences_signalled[i]
+            && vk->context.swapchain_fences[i] != VK_NULL_HANDLE)
+         fences[count++] = vk->context.swapchain_fences[i];
+   }
+   if (count)
+      vkWaitForFences(vk->context.device, count, fences, VK_TRUE,
+            UINT64_MAX);
+}
+
 static void vulkan_destroy_swapchain(gfx_ctx_vulkan_data_t *vk)
 {
    unsigned i;
@@ -1566,7 +1590,11 @@ static void vulkan_destroy_swapchain(gfx_ctx_vulkan_data_t *vk)
    vulkan_emulated_mailbox_deinit(&vk->mailbox);
    if (vk->swapchain != VK_NULL_HANDLE)
    {
-      vkDeviceWaitIdle(vk->context.device);
+      /* Our frames are the only submissions that touch its images.
+       * The context teardown drains the device before it gets here;
+       * a rebuild after an out-of-date acquire, or a resize, does
+       * not, and a threaded hardware core's work goes on. */
+      vulkan_context_wait_frames(vk);
       vkDestroySwapchainKHR(vk->context.device, vk->swapchain, NULL);
       memset(vk->context.swapchain_images, 0, sizeof(vk->context.swapchain_images));
       vk->swapchain                      = VK_NULL_HANDLE;
@@ -2318,7 +2346,14 @@ bool vulkan_create_swapchain(gfx_ctx_vulkan_data_t *vk,
    format.format                           = VK_FORMAT_UNDEFINED;
    format.colorSpace                       = VK_COLOR_SPACE_SRGB_NONLINEAR_KHR;
 
-   vkDeviceWaitIdle(vk->context.device);
+   /* Every frame that could still render into the old swapchain's
+    * images has completed, by its own fence: the new swapchain retires
+    * the old one and the old one is destroyed below, and the video
+    * driver rebuilds its framebuffers after waiting on the same
+    * fences. Nothing else on the device is waited for - a resize or a
+    * fullscreen toggle used to drain a threaded hardware core's whole
+    * queue here, unsynchronised with its submissions. */
+   vulkan_context_wait_frames(vk);
    vulkan_acquire_clear_fences(vk);
 
    vkGetPhysicalDeviceSurfaceCapabilitiesKHR(vk->context.gpu,

tools/vulkan_idle_wait_allow.list

@@ -5,11 +5,9 @@
 # fails too. Do not add to this list to make a wait go away: wait on
 # the driver's own fences instead (vulkan_wait_own_submissions).
 #
-# Swapchain lifetime: the presentation engine's use of the images has
-# no fence, so creating over or destroying a swapchain drains the
-# device. Rare, and on no per-frame path.
-gfx/common/vulkan_common.c|vulkan_create_swapchain|1
-gfx/common/vulkan_common.c|vulkan_destroy_swapchain|1
+# Context teardown: the one place the whole device is drained, before
+# the device itself goes. A swapchain rebuild or teardown waits on the
+# context's own frame fences instead.
 gfx/common/vulkan_common.c|vulkan_context_destroy|1
 # An acquire semaphore the frame never consumed goes on the stale list
 # for the next submission to wait on; the drain here is the fallback