Vulkan: cull mode and depth state per draw, one pipeline for the rest

Where the device has VK_EXT_extended_dynamic_state, the cull mode and
the depth test, write and compare function of a GX or ubershader
pipeline are set per draw, and pipelines that differ only there are one
VkPipeline: a pipeline asked for whose shaders, vertex layout, blending,
topology and framebuffer match one already built takes that one at once
instead of being compiled. Fewer pipelines are compiled, in the
background and on the spot alike.

The shared pipelines sit in a fixed open-addressed table every thread
building pipelines reads and adds to without a lock, counted by the
pipelines using them; a shader is told apart by a number given it when
it is made, not by its address. The state tracker sets only what
differs from what the command buffer has set already, and sets it again
after a pipeline with the state built in.

The colour blend attachments of a pipeline are kept on the stack.

Source/Core/VideoBackends/Vulkan/ObjectCache.cpp

@@ -3,6 +3,8 @@
 
 #include "VideoBackends/Vulkan/ObjectCache.h"
 
+#include "VideoBackends/Vulkan/VKPipeline.h"
+
 #include <algorithm>
 #include <array>
 #include <type_traits>
@@ -35,6 +37,8 @@ ObjectCache::~ObjectCache()
 
 bool ObjectCache::Initialize()
 {
+  VKPipeline::ResetSharedPipelines();
+
   if (!CreateDescriptorSetLayouts())
     return false;
 

Source/Core/VideoBackends/Vulkan/StateTracker.cpp

@@ -369,6 +369,34 @@ void StateTracker::SetScissor(const VkRect2D& scissor)
   m_dirty_flags |= DIRTY_FLAG_SCISSOR;
 }
 
+/* Sets what differs from what this command buffer has set already. */
+void StateTracker::BindDynamicState(VkCommandBuffer command_buffer, u32 state)
+{
+  const u64 fence = g_command_buffer_mgr->GetCurrentFenceCounter();
+  u32 changed = state ^ m_dynamic_state;
+  if (fence != m_dynamic_fence)
+    changed = ~0u;
+  if (!changed)
+    return;
+  if (changed & VK_DYNAMIC_CULL_MASK)
+    vkCmdSetCullModeEXT(command_buffer, state & VK_DYNAMIC_CULL_MASK);
+  if (changed & VK_DYNAMIC_DEPTH_TEST)
+    vkCmdSetDepthTestEnableEXT(command_buffer, (state & VK_DYNAMIC_DEPTH_TEST) ? VK_TRUE : VK_FALSE);
+  if (changed & VK_DYNAMIC_DEPTH_WRITE)
+  {
+    vkCmdSetDepthWriteEnableEXT(command_buffer,
+                                (state & VK_DYNAMIC_DEPTH_WRITE) ? VK_TRUE : VK_FALSE);
+  }
+  if (changed & VK_DYNAMIC_DEPTH_OP_MASK)
+  {
+    vkCmdSetDepthCompareOpEXT(command_buffer, static_cast<VkCompareOp>(
+                                                  (state & VK_DYNAMIC_DEPTH_OP_MASK) >>
+                                                  VK_DYNAMIC_DEPTH_OP_SHIFT));
+  }
+  m_dynamic_state = state;
+  m_dynamic_fence = fence;
+}
+
 bool StateTracker::Bind()
 {
   // Must have a pipeline.
@@ -400,7 +428,15 @@ bool StateTracker::Bind()
     vkCmdBindIndexBuffer(command_buffer, m_index_buffer, m_index_buffer_offset, m_index_type);
 
   if (m_dirty_flags & DIRTY_FLAG_PIPELINE)
+  {
     vkCmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, m_pipeline->GetVkPipeline());
+    /* a pipeline with the state built in leaves it undefined for the next */
+    if (!m_pipeline->GetDynamicState())
+      m_dynamic_fence = 0;
+  }
+
+  if (m_pipeline->GetDynamicState())
+    BindDynamicState(command_buffer, m_pipeline->GetDynamicState());
 
   if (m_dirty_flags & DIRTY_FLAG_VIEWPORT)
     vkCmdSetViewport(command_buffer, 0, 1, &m_viewport);

Source/Core/VideoBackends/Vulkan/StateTracker.h

@@ -78,6 +78,7 @@ public:
   bool IsWithinRenderArea(s32 x, s32 y, u32 width, u32 height) const;
 
 private:
+  void BindDynamicState(VkCommandBuffer command_buffer, u32 state);
   // Number of descriptor sets for game draws.
   enum
   {
@@ -131,6 +132,11 @@ private:
   VkDeviceSize m_index_buffer_offset = 0;
   VkIndexType m_index_type = VK_INDEX_TYPE_UINT16;
 
+  // cull mode and depth state set per draw (VK_DYNAMIC_*), and the command buffer they were set
+  // in (its fence counter): 0 when they are not set in the current one
+  u64 m_dynamic_fence = 0;
+  u32 m_dynamic_state = 0;
+
   // pipeline state
   const VKPipeline* m_pipeline = nullptr;
   const VKShader* m_compute_shader = nullptr;

Source/Core/VideoBackends/Vulkan/VKPipeline.cpp

@@ -5,6 +5,8 @@
 
 #include <array>
 
+#include <retro_atomic.h>
+
 #include "Common/Assert.h"
 #include "Common/EnumMap.h"
 #include "Common/MsgHandler.h"
@@ -17,15 +19,156 @@
 
 namespace Vulkan
 {
+/* Pipelines that differ only in what is set per draw are one VkPipeline,
+ * found in an open-addressed table every thread building pipelines reads
+ * and adds to without a lock. A slot goes from free to building to ready,
+ * and to dead when the last pipeline using it goes: it is never free
+ * again until the backend comes up again, so a search ends at the first
+ * free slot. Its key is written once, before it is ready. */
+#define SHARED_SLOTS 16384u
+#define SHARED_PROBES 64u
+
+enum
+{
+  SHARED_FREE = 0,
+  SHARED_BUILDING,
+  SHARED_READY,
+  SHARED_DEAD
+};
+
+struct VKSharedKey
+{
+  const NativeVertexFormat* vertex_format;
+  u32 vertex_shader;
+  u32 geometry_shader;
+  u32 pixel_shader;
+  u32 rasterization;
+  u32 blending;
+  u32 framebuffer;
+  u32 usage;
+};
+
+struct VKSharedPipeline
+{
+  VkPipeline pipeline;
+  VKSharedKey key;
+  u32 hash;
+  retro_atomic_int_t state;
+  retro_atomic_int_t refs;
+};
+
+static VKSharedPipeline s_shared[SHARED_SLOTS];
+
+static u32 ShaderUid(const AbstractShader* shader)
+{
+  return shader ? static_cast<const VKShader*>(shader)->GetUid() : 0u;
+}
+
+static u32 HashMix(u32 h, u32 v)
+{
+  h ^= v;
+  h *= 0x01000193u;
+  return h ^ (h >> 15);
+}
+
+static u32 HashSharedKey(const VKSharedKey* key)
+{
+  const uintptr_t vf = reinterpret_cast<uintptr_t>(key->vertex_format);
+  u32 h = 0x811c9dc5u;
+  h = HashMix(h, static_cast<u32>(vf));
+  h = HashMix(h, static_cast<u32>(static_cast<u64>(vf) >> 32));
+  h = HashMix(h, key->vertex_shader);
+  h = HashMix(h, key->geometry_shader);
+  h = HashMix(h, key->pixel_shader);
+  h = HashMix(h, key->rasterization);
+  h = HashMix(h, key->blending);
+  h = HashMix(h, key->framebuffer);
+  return HashMix(h, key->usage);
+}
+
+static bool SharedKeyEqual(const VKSharedKey* a, const VKSharedKey* b)
+{
+  return a->vertex_format == b->vertex_format && a->vertex_shader == b->vertex_shader &&
+         a->geometry_shader == b->geometry_shader && a->pixel_shader == b->pixel_shader &&
+         a->rasterization == b->rasterization && a->blending == b->blending &&
+         a->framebuffer == b->framebuffer && a->usage == b->usage;
+}
+
+/* A ready slot with the key, its count raised: one whose last user is
+ * going is passed over, as its count is never raised from zero. */
+static VKSharedPipeline* SharedFind(const VKSharedKey* key, u32 hash)
+{
+  u32 i;
+  for (i = 0; i < SHARED_PROBES; i++)
+  {
+    VKSharedPipeline* s = &s_shared[(hash + i) & (SHARED_SLOTS - 1)];
+    const int state = retro_atomic_load_acquire_int(&s->state);
+    if (state == SHARED_FREE)
+      return nullptr;
+    if (state != SHARED_READY || s->hash != hash || !SharedKeyEqual(&s->key, key))
+      continue;
+    for (;;)
+    {
+      const int refs = retro_atomic_load_acquire_int(&s->refs);
+      if (refs <= 0)
+        break;
+      if (retro_atomic_cas_int(&s->refs, refs, refs + 1))
+        return s;
+    }
+  }
+  return nullptr;
+}
+
+/* The first free slot on the key's chain, or none: the pipeline is then
+ * its own. */
+static VKSharedPipeline* SharedInsert(const VKSharedKey* key, u32 hash, VkPipeline pipeline)
+{
+  u32 i;
+  for (i = 0; i < SHARED_PROBES; i++)
+  {
+    VKSharedPipeline* s = &s_shared[(hash + i) & (SHARED_SLOTS - 1)];
+    if (retro_atomic_load_acquire_int(&s->state) != SHARED_FREE ||
+        !retro_atomic_cas_int(&s->state, SHARED_FREE, SHARED_BUILDING))
+    {
+      continue;
+    }
+    s->key = *key;
+    s->hash = hash;
+    s->pipeline = pipeline;
+    retro_atomic_store_relaxed_int(&s->refs, 1);
+    retro_atomic_store_release_int(&s->state, SHARED_READY);
+    return s;
+  }
+  return nullptr;
+}
+
+void VKPipeline::ResetSharedPipelines()
+{
+  u32 i;
+  for (i = 0; i < SHARED_SLOTS; i++)
+  {
+    retro_atomic_store_relaxed_int(&s_shared[i].refs, 0);
+    retro_atomic_store_relaxed_int(&s_shared[i].state, SHARED_FREE);
+  }
+  retro_atomic_thread_fence_release();
+}
+
 VKPipeline::VKPipeline(const AbstractPipelineConfig& config, VkPipeline pipeline,
-                       VkPipelineLayout pipeline_layout, AbstractPipelineUsage usage)
+                       VkPipelineLayout pipeline_layout, AbstractPipelineUsage usage,
+                       u32 dynamic_state, VKSharedPipeline* shared)
     : AbstractPipeline(config), m_pipeline(pipeline), m_pipeline_layout(pipeline_layout),
-      m_usage(usage)
+      m_shared(shared), m_usage(usage), m_dynamic_state(dynamic_state)
 {
 }
 
 VKPipeline::~VKPipeline()
 {
+  if (m_shared)
+  {
+    if (retro_atomic_fetch_sub_int(&m_shared->refs, 1) != 1)
+      return;
+    retro_atomic_store_release_int(&m_shared->state, SHARED_DEAD);
+  }
   vkDestroyPipeline(g_vulkan_context->GetDevice(), m_pipeline, nullptr);
 }
 
@@ -37,13 +180,12 @@ static bool IsStripPrimitiveTopology(VkPrimitiveTopology topology)
          topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY;
 }
 
+static const VkCullModeFlags s_cull_modes[4] = {VK_CULL_MODE_NONE, VK_CULL_MODE_BACK_BIT,
+                                                VK_CULL_MODE_FRONT_BIT, VK_CULL_MODE_FRONT_AND_BACK};
+
 static VkPipelineRasterizationStateCreateInfo
 GetVulkanRasterizationState(const RasterizationState& state)
 {
-  static constexpr std::array<VkCullModeFlags, 4> cull_modes = {
-      {VK_CULL_MODE_NONE, VK_CULL_MODE_BACK_BIT, VK_CULL_MODE_FRONT_BIT,
-       VK_CULL_MODE_FRONT_AND_BACK}};
-
   bool depth_clamp = g_backend_info.bSupportsDepthClamp;
 
   return {
@@ -53,7 +195,7 @@ GetVulkanRasterizationState(const RasterizationState& state)
       depth_clamp,           // VkBool32                                  depthClampEnable
       VK_FALSE,              // VkBool32                                  rasterizerDiscardEnable
       VK_POLYGON_MODE_FILL,  // VkPolygonMode                             polygonMode
-      cull_modes[u32(state.cull_mode.Value())],  // VkCullModeFlags        cullMode
+      s_cull_modes[u32(state.cull_mode.Value())],  // VkCullModeFlags     cullMode
       VK_FRONT_FACE_CLOCKWISE,                   // VkFrontFace            frontFace
       VK_FALSE,  // VkBool32                                              depthBiasEnable
       0.0f,      // float                                                 depthBiasConstantFactor
@@ -79,42 +221,37 @@ static VkPipelineMultisampleStateCreateInfo GetVulkanMultisampleState(const Fram
   };
 }
 
-static VkPipelineDepthStencilStateCreateInfo GetVulkanDepthStencilState(const DepthState& state)
+static VkCompareOp GetVulkanCompareOp(CompareMode func)
 {
   // Less/greater are swapped due to inverted depth.
-  VkCompareOp compare_op;
-  bool inverted_depth = !g_backend_info.bSupportsReversedDepthRange;
-  switch (state.func)
+  const bool inverted_depth = !g_backend_info.bSupportsReversedDepthRange;
+  switch (func)
   {
   case CompareMode::Never:
-    compare_op = VK_COMPARE_OP_NEVER;
-    break;
+    return VK_COMPARE_OP_NEVER;
   case CompareMode::Less:
-    compare_op = inverted_depth ? VK_COMPARE_OP_GREATER : VK_COMPARE_OP_LESS;
-    break;
+    return inverted_depth ? VK_COMPARE_OP_GREATER : VK_COMPARE_OP_LESS;
   case CompareMode::Equal:
-    compare_op = VK_COMPARE_OP_EQUAL;
-    break;
+    return VK_COMPARE_OP_EQUAL;
   case CompareMode::LEqual:
-    compare_op = inverted_depth ? VK_COMPARE_OP_GREATER_OR_EQUAL : VK_COMPARE_OP_LESS_OR_EQUAL;
-    break;
+    return inverted_depth ? VK_COMPARE_OP_GREATER_OR_EQUAL : VK_COMPARE_OP_LESS_OR_EQUAL;
   case CompareMode::Greater:
-    compare_op = inverted_depth ? VK_COMPARE_OP_LESS : VK_COMPARE_OP_GREATER;
-    break;
+    return inverted_depth ? VK_COMPARE_OP_LESS : VK_COMPARE_OP_GREATER;
   case CompareMode::NEqual:
-    compare_op = VK_COMPARE_OP_NOT_EQUAL;
-    break;
+    return VK_COMPARE_OP_NOT_EQUAL;
   case CompareMode::GEqual:
-    compare_op = inverted_depth ? VK_COMPARE_OP_LESS_OR_EQUAL : VK_COMPARE_OP_GREATER_OR_EQUAL;
-    break;
+    return inverted_depth ? VK_COMPARE_OP_LESS_OR_EQUAL : VK_COMPARE_OP_GREATER_OR_EQUAL;
   case CompareMode::Always:
-    compare_op = VK_COMPARE_OP_ALWAYS;
-    break;
+    return VK_COMPARE_OP_ALWAYS;
   default:
-    PanicAlertFmt("Invalid compare mode {}", state.func);
-    compare_op = VK_COMPARE_OP_ALWAYS;
-    break;
+    PanicAlertFmt("Invalid compare mode {}", func);
+    return VK_COMPARE_OP_ALWAYS;
   }
+}
+
+static VkPipelineDepthStencilStateCreateInfo GetVulkanDepthStencilState(const DepthState& state)
+{
+  const VkCompareOp compare_op = GetVulkanCompareOp(state.func);
 
   return {
       VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,  // VkStructureType sType
@@ -273,6 +410,40 @@ std::unique_ptr<VKPipeline> VKPipeline::Create(const AbstractPipelineConfig& con
     return nullptr;
   }
 
+  /* Cull mode and depth state set per draw: the pipeline is built, and
+   * shared, with them left out of its key. */
+  AbstractPipelineConfig build = config;
+  u32 dynamic_bits = 0;
+  VKSharedKey key;
+  u32 hash = 0;
+  if (config.usage != AbstractPipelineUsage::Utility &&
+      g_vulkan_context->SupportsExtendedDynamicState())
+  {
+    VKSharedPipeline* shared;
+    dynamic_bits = VK_DYNAMIC_USED | s_cull_modes[u32(config.rasterization_state.cull_mode.Value())] |
+                    (config.depth_state.test_enable ? VK_DYNAMIC_DEPTH_TEST : 0u) |
+                    (config.depth_state.update_enable ? VK_DYNAMIC_DEPTH_WRITE : 0u) |
+                    (static_cast<u32>(GetVulkanCompareOp(config.depth_state.func))
+                     << VK_DYNAMIC_DEPTH_OP_SHIFT);
+    build.rasterization_state.cull_mode = CullMode::None;
+    build.depth_state.hex = 0;
+    key.vertex_format = build.vertex_format;
+    key.vertex_shader = ShaderUid(build.vertex_shader);
+    key.geometry_shader = ShaderUid(build.geometry_shader);
+    key.pixel_shader = ShaderUid(build.pixel_shader);
+    key.rasterization = build.rasterization_state.hex;
+    key.blending = build.blending_state.hex;
+    key.framebuffer = build.framebuffer_state.hex;
+    key.usage = static_cast<u32>(build.usage);
+    hash = HashSharedKey(&key);
+    shared = SharedFind(&key, hash);
+    if (shared)
+    {
+      return std::make_unique<VKPipeline>(config, shared->pipeline, pipeline_layout, config.usage,
+                                          dynamic_bits, shared);
+    }
+  }
+
   // Declare descriptors for empty vertex buffers/attributes
   static const VkPipelineVertexInputStateCreateInfo empty_vertex_input_state = {
       VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,  // VkStructureType sType
@@ -296,7 +467,7 @@ std::unique_ptr<VKPipeline> VKPipeline::Create(const AbstractPipelineConfig& con
        VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP}};
   VkPipelineInputAssemblyStateCreateInfo input_assembly_state = {
       VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, nullptr, 0,
-      vk_primitive_topologies[static_cast<u32>(config.rasterization_state.primitive.Value())],
+      vk_primitive_topologies[static_cast<u32>(build.rasterization_state.primitive.Value())],
       VK_FALSE};
 
   // See Vulkan spec, section 19:
@@ -346,25 +517,22 @@ std::unique_ptr<VKPipeline> VKPipeline::Create(const AbstractPipelineConfig& con
 
   // Fill in Vulkan descriptor structs from our state structures.
   VkPipelineRasterizationStateCreateInfo rasterization_state =
-      GetVulkanRasterizationState(config.rasterization_state);
+      GetVulkanRasterizationState(build.rasterization_state);
   VkPipelineMultisampleStateCreateInfo multisample_state =
       GetVulkanMultisampleState(config.framebuffer_state);
   VkPipelineDepthStencilStateCreateInfo depth_stencil_state =
-      GetVulkanDepthStencilState(config.depth_state);
+      GetVulkanDepthStencilState(build.depth_state);
   VkPipelineColorBlendAttachmentState blend_attachment_state =
       GetVulkanAttachmentBlendState(config.blending_state, config.usage);
 
-  std::vector<VkPipelineColorBlendAttachmentState> blend_attachment_states;
-  blend_attachment_states.push_back(blend_attachment_state);
   // Right now all our attachments have the same state
-  for (u8 i = 0; i < static_cast<u8>(config.framebuffer_state.additional_color_attachment_count);
-       i++)
-  {
-    blend_attachment_states.push_back(blend_attachment_state);
-  }
+  VkPipelineColorBlendAttachmentState blend_attachment_states[8];
+  const u32 num_attachments =
+      1 + static_cast<u32>(config.framebuffer_state.additional_color_attachment_count);
+  for (u32 i = 0; i < num_attachments; i++)
+    blend_attachment_states[i] = blend_attachment_state;
   VkPipelineColorBlendStateCreateInfo blend_state =
-      GetVulkanColorBlendState(config.blending_state, blend_attachment_states.data(),
-                               static_cast<uint32_t>(blend_attachment_states.size()));
+      GetVulkanColorBlendState(config.blending_state, blend_attachment_states, num_attachments);
 
   static const VkDepthClampRangeEXT clamp_range = {0.0f, MAX_EFB_DEPTH};
   static const VkPipelineViewportDepthClampControlCreateInfoEXT depth_clamp_state = {
@@ -386,16 +554,18 @@ std::unique_ptr<VKPipeline> VKPipeline::Create(const AbstractPipelineConfig& con
       &scissor    // const VkRect2D*                       pScissors
   };
 
-  // Set viewport and scissor dynamic state so we can change it elsewhere.
-  static const std::array<VkDynamicState, 2> dynamic_states{
-      VK_DYNAMIC_STATE_VIEWPORT,
-      VK_DYNAMIC_STATE_SCISSOR,
+  // Viewport and scissor are set per draw, and cull mode and depth state with them where the
+  // device allows it.
+  static const VkDynamicState dynamic_states[6] = {
+      VK_DYNAMIC_STATE_VIEWPORT,           VK_DYNAMIC_STATE_SCISSOR,
+      VK_DYNAMIC_STATE_CULL_MODE_EXT,      VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE_EXT,
+      VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE_EXT, VK_DYNAMIC_STATE_DEPTH_COMPARE_OP_EXT,
   };
-  static const VkPipelineDynamicStateCreateInfo dynamic_state = {
+  const VkPipelineDynamicStateCreateInfo dynamic_state = {
       VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, nullptr,
-      0,                                        // VkPipelineDynamicStateCreateFlags    flags
-      static_cast<u32>(dynamic_states.size()),  // uint32_t dynamicStateCount
-      dynamic_states.data()  // const VkDynamicState*                pDynamicStates
+      0,                         // VkPipelineDynamicStateCreateFlags    flags
+      dynamic_bits ? 6u : 2u,    // uint32_t dynamicStateCount
+      dynamic_states             // const VkDynamicState*                pDynamicStates
   };
 
   // Combine to full pipeline info structure.
@@ -431,6 +601,8 @@ std::unique_ptr<VKPipeline> VKPipeline::Create(const AbstractPipelineConfig& con
     return VK_NULL_HANDLE;
   }
 
-  return std::make_unique<VKPipeline>(config, pipeline, pipeline_layout, config.usage);
+  return std::make_unique<VKPipeline>(config, pipeline, pipeline_layout, config.usage,
+                                      dynamic_bits,
+                                      dynamic_bits ? SharedInsert(&key, hash, pipeline) : nullptr);
 }
 }  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/VKPipeline.h

@@ -10,22 +10,42 @@
 
 namespace Vulkan
 {
+/* A pipeline's cull mode and depth state, set per draw when the device
+ * can (VKPipeline::GetDynamicState): the VkCullModeFlags, depth test,
+ * depth write and VkCompareOp, and a bit saying they are set this way. */
+#define VK_DYNAMIC_CULL_MASK 0x3u
+#define VK_DYNAMIC_DEPTH_TEST (1u << 2)
+#define VK_DYNAMIC_DEPTH_WRITE (1u << 3)
+#define VK_DYNAMIC_DEPTH_OP_SHIFT 4
+#define VK_DYNAMIC_DEPTH_OP_MASK (0x7u << VK_DYNAMIC_DEPTH_OP_SHIFT)
+#define VK_DYNAMIC_USED (1u << 31)
+
+struct VKSharedPipeline;
+
 class VKPipeline final : public AbstractPipeline
 {
 public:
   explicit VKPipeline(const AbstractPipelineConfig& config, VkPipeline pipeline,
-                      VkPipelineLayout pipeline_layout, AbstractPipelineUsage usage);
+                      VkPipelineLayout pipeline_layout, AbstractPipelineUsage usage,
+                      u32 dynamic_state, VKSharedPipeline* shared);
   ~VKPipeline() override;
 
   VkPipeline GetVkPipeline() const { return m_pipeline; }
   VkPipelineLayout GetVkPipelineLayout() const { return m_pipeline_layout; }
   AbstractPipelineUsage GetUsage() const { return m_usage; }
+  /* 0, or the VK_DYNAMIC_* state to set before drawing with it */
+  u32 GetDynamicState() const { return m_dynamic_state; }
   static std::unique_ptr<VKPipeline> Create(const AbstractPipelineConfig& config);
+  /* Forgets every shared pipeline; only with none alive, as the backend
+   * comes up. */
+  static void ResetSharedPipelines();
 
 private:
   VkPipeline m_pipeline;
   VkPipelineLayout m_pipeline_layout;
+  VKSharedPipeline* m_shared;
   AbstractPipelineUsage m_usage;
+  u32 m_dynamic_state;
 };
 
 }  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/VKShader.cpp

@@ -3,6 +3,8 @@
 
 #include "VideoBackends/Vulkan/VKShader.h"
 
+#include <retro_atomic.h>
+
 #include "Common/Align.h"
 
 #include "VideoBackends/Vulkan/ObjectCache.h"
@@ -13,10 +15,17 @@
 
 namespace Vulkan
 {
+static retro_atomic_int_t s_next_uid;
+
+static u32 NextUid()
+{
+  return (u32)retro_atomic_fetch_add_int(&s_next_uid, 1) + 1u;
+}
+
 VKShader::VKShader(ShaderStage stage, std::vector<u32> spv, VkShaderModule mod,
                    std::string_view name)
     : AbstractShader(stage), m_spv(std::move(spv)), m_module(mod),
-      m_compute_pipeline(VK_NULL_HANDLE), m_name(name)
+      m_compute_pipeline(VK_NULL_HANDLE), m_name(name), m_uid(NextUid())
 {
   if (!m_name.empty() && g_backend_info.bSupportsSettingObjectNames)
   {
@@ -31,7 +40,7 @@ VKShader::VKShader(ShaderStage stage, std::vector<u32> spv, VkShaderModule mod,
 
 VKShader::VKShader(std::vector<u32> spv, VkPipeline compute_pipeline, std::string_view name)
     : AbstractShader(ShaderStage::Compute), m_spv(std::move(spv)), m_module(VK_NULL_HANDLE),
-      m_compute_pipeline(compute_pipeline), m_name(name)
+      m_compute_pipeline(compute_pipeline), m_name(name), m_uid(NextUid())
 {
   if (!m_name.empty() && g_backend_info.bSupportsSettingObjectNames)
   {

Source/Core/VideoBackends/Vulkan/VKShader.h

@@ -24,6 +24,9 @@ public:
 
   VkShaderModule GetShaderModule() const { return m_module; }
   VkPipeline GetComputePipeline() const { return m_compute_pipeline; }
+  /* unique among the shaders made since the backend came up: a shader
+   * made where a destroyed one was is another shader */
+  u32 GetUid() const { return m_uid; }
   BinaryData GetBinary() const override;
 
   static std::unique_ptr<VKShader> CreateFromSource(ShaderStage stage, std::string_view source,
@@ -36,6 +39,7 @@ private:
   VkShaderModule m_module;
   VkPipeline m_compute_pipeline;
   std::string m_name;
+  u32 m_uid;
 };
 
 }  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/VulkanContext.cpp

@@ -662,6 +662,23 @@ bool VulkanContext::SelectDeviceExtensions(bool enable_surface)
   AddExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME, false);
   AddExtension("VK_KHR_portability_subset", false);
 
+  /* the extension is enabled only with its feature there to enable */
+  m_extended_dynamic_state = false;
+  if (m_device_info.apiVersion >= VK_API_VERSION_1_1 && vkGetPhysicalDeviceFeatures2 &&
+      Common::Contains(available_extension_list,
+                       std::string_view{VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME},
+                       &VkExtensionProperties::extensionName))
+  {
+    VkPhysicalDeviceExtendedDynamicStateFeaturesEXT eds = {};
+    VkPhysicalDeviceFeatures2 features2 = {};
+    eds.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT;
+    features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
+    features2.pNext = &eds;
+    vkGetPhysicalDeviceFeatures2(m_physical_device, &features2);
+    if (eds.extendedDynamicState)
+      m_extended_dynamic_state = AddExtension(VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME, false);
+  }
+
   if (!DriverDetails::HasBug(DriverDetails::BUG_BROKEN_DEPTH_CLAMP_CONTROL))
   {
     // Unrestricted depth range is one of the few extensions that changes the behavior
@@ -804,6 +821,15 @@ bool VulkanContext::CreateDevice(VkSurfaceKHR surface, bool enable_validation_la
   VkPhysicalDeviceFeatures device_features = m_device_info.features();
   device_info.pEnabledFeatures = &device_features;
 
+  VkPhysicalDeviceExtendedDynamicStateFeaturesEXT eds_features = {};
+  if (m_extended_dynamic_state)
+  {
+    eds_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT;
+    eds_features.extendedDynamicState = VK_TRUE;
+    eds_features.pNext = const_cast<void*>(device_info.pNext);
+    device_info.pNext = &eds_features;
+  }
+
   // Enable debug layer on debug builds
   if (enable_validation_layer)
   {
@@ -821,6 +847,11 @@ bool VulkanContext::CreateDevice(VkSurfaceKHR surface, bool enable_validation_la
   // With the device created, we can fill the remaining entry points.
   if (!LoadVulkanDeviceFunctions(m_device))
     return false;
+  m_extended_dynamic_state = m_extended_dynamic_state && vkCmdSetCullModeEXT &&
+                             vkCmdSetDepthTestEnableEXT && vkCmdSetDepthWriteEnableEXT &&
+                             vkCmdSetDepthCompareOpEXT;
+  INFO_LOG_FMT(VIDEO, "Vulkan: cull mode and depth state {}", m_extended_dynamic_state ?
+                          "set per draw" : "in each pipeline");
 
   // Grab the graphics and present queues.
   vkGetDeviceQueue(m_device, m_graphics_queue_family_index, 0, &m_graphics_queue);

Source/Core/VideoBackends/Vulkan/VulkanContext.h

@@ -111,6 +111,10 @@ public:
   bool SupportsPreciseOcclusionQueries() const { return m_device_info.occlusionQueryPrecise; }
   u32 GetShaderSubgroupSize() const { return m_device_info.subgroupSize; }
   bool SupportsShaderSubgroupOperations() const { return m_device_info.shaderSubgroupOperations; }
+  /* cull mode and depth test, write and compare set per draw
+   * (VK_EXT_extended_dynamic_state): pipelines differing only there are
+   * one pipeline */
+  bool SupportsExtendedDynamicState() const { return m_extended_dynamic_state; }
 
   // Helpers for getting constants
   VkDeviceSize GetUniformBufferAlignment() const
@@ -151,6 +155,7 @@ private:
   PhysicalDeviceInfo m_device_info;
 
   std::vector<std::string> m_device_extensions;
+  bool m_extended_dynamic_state = false;
 #ifdef __LIBRETRO__
   VmaVulkanFunctions m_vulkan_functions = {};
 #endif

Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl

@@ -68,6 +68,7 @@ VULKAN_INSTANCE_ENTRY_POINT(vkQueueInsertDebugUtilsLabelEXT, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkSetDebugUtilsObjectTagEXT, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkSubmitDebugUtilsMessageEXT, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceProperties2, false)
+VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceFeatures2, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceSurfaceCapabilities2KHR, false)
 VULKAN_INSTANCE_ENTRY_POINT(vkSetDebugUtilsObjectNameEXT, false)
 
@@ -204,6 +205,10 @@ VULKAN_DEVICE_ENTRY_POINT(vkGetBufferMemoryRequirements2, false)
 VULKAN_DEVICE_ENTRY_POINT(vkGetImageMemoryRequirements2, false)
 VULKAN_DEVICE_ENTRY_POINT(vkBindBufferMemory2, false)
 VULKAN_DEVICE_ENTRY_POINT(vkBindImageMemory2, false)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetCullModeEXT, false)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetDepthTestEnableEXT, false)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetDepthWriteEnableEXT, false)
+VULKAN_DEVICE_ENTRY_POINT(vkCmdSetDepthCompareOpEXT, false)
 
 #ifdef SUPPORTS_VULKAN_EXCLUSIVE_FULLSCREEN
 VULKAN_DEVICE_ENTRY_POINT(vkAcquireFullScreenExclusiveModeEXT, false)