Vulkan: Fix pipeline cache clearing.

Extracted from #16759 and bugfixed. Fixes a leak of Vulkan pipelines.

I guess another way would be to queue the variants for destruction at
the same time as we queue the callback, but I like this better.

Common/GPU/Vulkan/VulkanContext.cpp

@@ -301,7 +301,7 @@ void VulkanContext::DestroyInstance() {
 void VulkanContext::BeginFrame(VkCommandBuffer firstCommandBuffer) {
 	FrameData *frame = &frame_[curFrame_];
 	// Process pending deletes.
-	frame->deleteList.PerformDeletes(device_, allocator_);
+	frame->deleteList.PerformDeletes(this, allocator_);
 	// VK_NULL_HANDLE when profiler is disabled.
 	if (firstCommandBuffer) {
 		frame->profiler.BeginFrame(this, firstCommandBuffer);
@@ -1223,9 +1223,9 @@ VkFence VulkanContext::CreateFence(bool presignalled) {
 
 void VulkanContext::PerformPendingDeletes() {
 	for (int i = 0; i < ARRAY_SIZE(frame_); i++) {
-		frame_[i].deleteList.PerformDeletes(device_, allocator_);
+		frame_[i].deleteList.PerformDeletes(this, allocator_);
 	}
-	Delete().PerformDeletes(device_, allocator_);
+	Delete().PerformDeletes(this, allocator_);
 }
 
 void VulkanContext::DestroyDevice() {
@@ -1446,11 +1446,13 @@ void VulkanDeleteList::Take(VulkanDeleteList &del) {
 	del.callbacks_.clear();
 }
 
-void VulkanDeleteList::PerformDeletes(VkDevice device, VmaAllocator allocator) {
+void VulkanDeleteList::PerformDeletes(VulkanContext *vulkan, VmaAllocator allocator) {
 	for (auto &callback : callbacks_) {
-		callback.func(callback.userdata);
+		callback.func(vulkan, callback.userdata);
 	}
 	callbacks_.clear();
+
+	VkDevice device = vulkan->GetDevice();
 	for (auto &cmdPool : cmdPools_) {
 		vkDestroyCommandPool(device, cmdPool, nullptr);
 	}

Common/GPU/Vulkan/VulkanContext.h

@@ -74,6 +74,7 @@ struct VulkanPhysicalDeviceInfo {
 };
 
 class VulkanProfiler;
+class VulkanContext;
 
 // Extremely rough split of capabilities.
 enum class PerfClass {
@@ -93,11 +94,11 @@ class VulkanDeleteList {
 	};
 
 	struct Callback {
-		explicit Callback(void(*f)(void *userdata), void *u)
+		explicit Callback(void(*f)(VulkanContext *vulkan, void *userdata), void *u)
 			: func(f), userdata(u) {
 		}
 
-		void(*func)(void *userdata);
+		void (*func)(VulkanContext *vulkan, void *userdata);
 		void *userdata;
 	};
 
@@ -118,7 +119,7 @@ public:
 	void QueueDeletePipelineLayout(VkPipelineLayout &pipelineLayout) { _dbg_assert_(pipelineLayout != VK_NULL_HANDLE); pipelineLayouts_.push_back(pipelineLayout); pipelineLayout = VK_NULL_HANDLE; }
 	void QueueDeleteDescriptorSetLayout(VkDescriptorSetLayout &descSetLayout) { _dbg_assert_(descSetLayout != VK_NULL_HANDLE); descSetLayouts_.push_back(descSetLayout); descSetLayout = VK_NULL_HANDLE; }
 	void QueueDeleteQueryPool(VkQueryPool &queryPool) { _dbg_assert_(queryPool != VK_NULL_HANDLE); queryPools_.push_back(queryPool); queryPool = VK_NULL_HANDLE; }
-	void QueueCallback(void(*func)(void *userdata), void *userdata) { callbacks_.push_back(Callback(func, userdata)); }
+	void QueueCallback(void (*func)(VulkanContext *vulkan, void *userdata), void *userdata) { callbacks_.push_back(Callback(func, userdata)); }
 
 	void QueueDeleteBufferAllocation(VkBuffer &buffer, VmaAllocation &alloc) { 
 		_dbg_assert_(buffer != VK_NULL_HANDLE); 
@@ -134,7 +135,7 @@ public:
 	}
 
 	void Take(VulkanDeleteList &del);
-	void PerformDeletes(VkDevice device, VmaAllocator allocator);
+	void PerformDeletes(VulkanContext *vulkan, VmaAllocator allocator);
 
 private:
 	std::vector<VkCommandPool> cmdPools_;

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -169,10 +169,31 @@ void VKRGraphicsPipeline::DestroyVariants(VulkanContext *vulkan, bool msaaOnly)
 	sampleCount_ = VK_SAMPLE_COUNT_FLAG_BITS_MAX_ENUM;
 }
 
+void VKRGraphicsPipeline::DestroyVariantsInstant(VkDevice device) {
+	for (size_t i = 0; i < (size_t)RenderPassType::TYPE_COUNT; i++) {
+		if (pipeline[i]) {
+			vkDestroyPipeline(device, pipeline[i]->BlockUntilReady(), nullptr);
+			delete pipeline[i];
+			pipeline[i] = nullptr;
+		}
+	}
+}
+
+VKRGraphicsPipeline::~VKRGraphicsPipeline() {
+	// This is called from the callbacked queued in QueueForDeletion.
+	// Here we are free to directly delete stuff, don't need to queue.
+	for (size_t i = 0; i < (size_t)RenderPassType::TYPE_COUNT; i++) {
+		_assert_(!pipeline[i]);
+	}
+	if (desc)
+		desc->Release();
+}
+
 void VKRGraphicsPipeline::QueueForDeletion(VulkanContext *vulkan) {
-	DestroyVariants(vulkan, false);
-	vulkan->Delete().QueueCallback([](void *p) {
+	// Can't destroy variants here, the pipeline still lives for a while.
+	vulkan->Delete().QueueCallback([](VulkanContext *vulkan, void *p) {
 		VKRGraphicsPipeline *pipeline = (VKRGraphicsPipeline *)p;
+		pipeline->DestroyVariantsInstant(vulkan->GetDevice());
 		delete pipeline;
 	}, this);
 }

Common/GPU/Vulkan/VulkanRenderManager.h

@@ -118,13 +118,7 @@ struct VKRComputePipelineDesc {
 // Wrapped pipeline. Doesn't own desc.
 struct VKRGraphicsPipeline {
 	VKRGraphicsPipeline(PipelineFlags flags, const char *tag) : flags_(flags), tag_(tag) {}
-	~VKRGraphicsPipeline() {
-		for (size_t i = 0; i < (size_t)RenderPassType::TYPE_COUNT; i++) {
-			delete pipeline[i];
-		}
-		if (desc)
-			desc->Release();
-	}
+	~VKRGraphicsPipeline();
 
 	bool Create(VulkanContext *vulkan, VkRenderPass compatibleRenderPass, RenderPassType rpType, VkSampleCountFlagBits sampleCount);
 
@@ -142,6 +136,8 @@ struct VKRGraphicsPipeline {
 
 	VkSampleCountFlagBits SampleCount() const { return sampleCount_; }
 private:
+	void DestroyVariantsInstant(VkDevice device);
+
 	std::string tag_;
 	PipelineFlags flags_;
 	VkSampleCountFlagBits sampleCount_ = VK_SAMPLE_COUNT_FLAG_BITS_MAX_ENUM;

Common/GPU/Vulkan/thin3d_vulkan.cpp

@@ -201,7 +201,7 @@ public:
 			DEBUG_LOG(G3D, "Queueing %s (shmodule %p) for release", tag_.c_str(), module_);
 			VkShaderModule shaderModule = module_->BlockUntilReady();
 			vulkan_->Delete().QueueDeleteShaderModule(shaderModule);
-			vulkan_->Delete().QueueCallback([](void *m) {
+			vulkan_->Delete().QueueCallback([](VulkanContext *context, void *m) {
 				auto module = (Promise<VkShaderModule> *)m;
 				delete module;
 			}, module_);
@@ -1579,7 +1579,7 @@ public:
 	}
 	~VKFramebuffer() {
 		_assert_msg_(buf_, "Null buf_ in VKFramebuffer - double delete?");
-		buf_->Vulkan()->Delete().QueueCallback([](void *fb) {
+		buf_->Vulkan()->Delete().QueueCallback([](VulkanContext *vulkan, void *fb) {
 			VKRFramebuffer *vfb = static_cast<VKRFramebuffer *>(fb);
 			delete vfb;
 		}, buf_);

GPU/Vulkan/GPU_Vulkan.cpp

@@ -312,6 +312,7 @@ void GPU_Vulkan::BeginHostFrame() {
 		// This most likely means that saw equal depth changed.
 		WARN_LOG(G3D, "Shader use flags changed, clearing all shaders");
 		shaderManagerVulkan_->ClearShaders();
+		pipelineManager_->Clear();
 		gstate_c.useFlagsChanged = false;
 	}
 

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -122,7 +122,7 @@ VulkanFragmentShader::~VulkanFragmentShader() {
 		if (shaderModule) {
 			vulkan_->Delete().QueueDeleteShaderModule(shaderModule);
 		}
-		vulkan_->Delete().QueueCallback([](void *m) {
+		vulkan_->Delete().QueueCallback([](VulkanContext *vulkan, void *m) {
 			auto module = (Promise<VkShaderModule> *)m;
 			delete module;
 		}, module_);
@@ -157,7 +157,7 @@ VulkanVertexShader::~VulkanVertexShader() {
 		if (shaderModule) {
 			vulkan_->Delete().QueueDeleteShaderModule(shaderModule);
 		}
-		vulkan_->Delete().QueueCallback([](void *m) {
+		vulkan_->Delete().QueueCallback([](VulkanContext *vulkan, void *m) {
 			auto module = (Promise<VkShaderModule> *)m;
 			delete module;
 		}, module_);
@@ -192,7 +192,7 @@ VulkanGeometryShader::~VulkanGeometryShader() {
 		if (shaderModule) {
 			vulkan_->Delete().QueueDeleteShaderModule(shaderModule);
 		}
-		vulkan_->Delete().QueueCallback([](void *m) {
+		vulkan_->Delete().QueueCallback([](VulkanContext *vulkan, void *m) {
 			auto module = (Promise<VkShaderModule> *)m;
 			delete module;
 		}, module_);