Merge pull request #16098 from hrydgard/dont-compile-incompatible-shader-variants

Vulkan: Don't compile shader variants for incompatible render pass types

Common/GPU/Vulkan/VulkanQueueRunner.cpp

@@ -10,16 +10,6 @@ using namespace PPSSPP_VK;
 
 // Debug help: adb logcat -s DEBUG PPSSPPNativeActivity PPSSPP NativeGLView NativeRenderer NativeSurfaceView PowerSaveModeReceiver InputDeviceState
 
-static bool RenderPassTypeHasDepth(RenderPassType rpType) {
-	switch (rpType) {
-	case RP_TYPE_BACKBUFFER:
-	case RP_TYPE_COLOR_DEPTH:
-	case RP_TYPE_COLOR_DEPTH_INPUT:
-		return true;
-	}
-	return false;
-}
-
 static void MergeRenderAreaRectInto(VkRect2D *dest, VkRect2D &src) {
 	if (dest->offset.x > src.offset.x) {
 		dest->extent.width += (dest->offset.x - src.offset.x);

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -43,9 +43,9 @@ enum class VKRRenderCommand : uint8_t {
 
 enum class PipelineFlags {
 	NONE = 0,
-	USES_BLEND_CONSTANT = (1 << 3),
-	USES_DEPTH_STENCIL = (1 << 4),  // Reads or writes the depth or stencil buffers.
-	USES_INPUT_ATTACHMENT = (1 << 5),
+	USES_BLEND_CONSTANT = (1 << 1),
+	USES_DEPTH_STENCIL = (1 << 2),  // Reads or writes the depth or stencil buffers.
+	USES_INPUT_ATTACHMENT = (1 << 3),
 };
 ENUM_CLASS_BITOPS(PipelineFlags);
 
@@ -65,6 +65,14 @@ enum RenderPassType {
 	RP_TYPE_COUNT,
 };
 
+inline bool RenderPassTypeHasDepth(RenderPassType type) {
+	return type == RP_TYPE_BACKBUFFER || type == RP_TYPE_COLOR_DEPTH || type == RP_TYPE_COLOR_DEPTH_INPUT;
+}
+
+inline bool RenderPassTypeHasInput(RenderPassType type) {
+	return type == RP_TYPE_COLOR_INPUT || type == RP_TYPE_COLOR_DEPTH_INPUT;
+}
+
 struct VkRenderData {
 	VKRRenderCommand cmd;
 	union {

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -589,7 +589,7 @@ VkCommandBuffer VulkanRenderManager::GetInitCmd() {
 	return frameData_[curFrame].GetInitCmd(vulkan_);
 }
 
-VKRGraphicsPipeline *VulkanRenderManager::CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc, uint32_t variantBitmask, const char *tag) {
+VKRGraphicsPipeline *VulkanRenderManager::CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc, PipelineFlags pipelineFlags, uint32_t variantBitmask, const char *tag) {
 	VKRGraphicsPipeline *pipeline = new VKRGraphicsPipeline();
 	_dbg_assert_(desc->vertexShader);
 	_dbg_assert_(desc->fragmentShader);
@@ -615,6 +615,21 @@ VKRGraphicsPipeline *VulkanRenderManager::CreateGraphicsPipeline(VKRGraphicsPipe
 			if (!(variantBitmask & (1 << i)))
 				continue;
 			RenderPassType rpType = (RenderPassType)i;
+
+			// Sanity check - don't compile incompatible types (could be caused by corrupt caches, changes in data structures, etc).
+			if (pipelineFlags & PipelineFlags::USES_DEPTH_STENCIL) {
+				if (!RenderPassTypeHasDepth(rpType)) {
+					WARN_LOG(G3D, "Not compiling pipeline that requires depth, for non depth renderpass type");
+					continue;
+				}
+			}
+			if (pipelineFlags & PipelineFlags::USES_INPUT_ATTACHMENT) {
+				if (!RenderPassTypeHasInput(rpType)) {
+					WARN_LOG(G3D, "Not compiling pipeline that requires input attachment, for non input renderpass type");
+					continue;
+				}
+			}
+
 			pipeline->pipeline[rpType] = Promise<VkPipeline>::CreateEmpty();
 			compileQueue_.push_back(CompileQueueEntry(pipeline, compatibleRenderPass->Get(vulkan_, rpType), rpType));
 			needsCompile = true;

Common/GPU/Vulkan/VulkanRenderManager.h

@@ -241,7 +241,7 @@ public:
 	// We delay creating pipelines until the end of the current render pass, so we can create the right type immediately.
 	// Unless a variantBitmask is passed in, in which case we can just go ahead.
 	// WARNING: desc must stick around during the lifetime of the pipeline! It's not enough to build it on the stack and drop it.
-	VKRGraphicsPipeline *CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc, uint32_t variantBitmask, const char *tag);
+	VKRGraphicsPipeline *CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc, PipelineFlags pipelineFlags, uint32_t variantBitmask, const char *tag);
 	VKRComputePipeline *CreateComputePipeline(VKRComputePipelineDesc *desc);
 
 	void NudgeCompilerThread() {

Common/GPU/Vulkan/thin3d_vulkan.cpp

@@ -1134,7 +1134,7 @@ Pipeline *VKContext::CreateGraphicsPipeline(const PipelineDesc &desc, const char
 	VkPipelineRasterizationStateCreateInfo rs{ VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO };
 	raster->ToVulkan(&gDesc.rs);
 
-	pipeline->pipeline = renderManager_.CreateGraphicsPipeline(&gDesc, 1 << RP_TYPE_BACKBUFFER, tag ? tag : "thin3d");
+	pipeline->pipeline = renderManager_.CreateGraphicsPipeline(&gDesc, pipelineFlags, 1 << RP_TYPE_BACKBUFFER, tag ? tag : "thin3d");
 
 	if (desc.uniformDesc) {
 		pipeline->dynamicUniformSize = (int)desc.uniformDesc->uniformBufferSize;

GPU/Vulkan/PipelineManagerVulkan.cpp

@@ -296,7 +296,7 @@ static VulkanPipeline *CreateVulkanPipeline(VulkanRenderManager *renderManager,
 
 	desc->pipelineLayout = layout;
 
-	VKRGraphicsPipeline *pipeline = renderManager->CreateGraphicsPipeline(desc, variantBitmask, "game");
+	VKRGraphicsPipeline *pipeline = renderManager->CreateGraphicsPipeline(desc, pipelineFlags, variantBitmask, "game");
 
 	vulkanPipeline->pipeline = pipeline;
 	if (useBlendConstant) {

GPU/Vulkan/ShaderManagerVulkan.cpp

@@ -377,7 +377,7 @@ VulkanFragmentShader *ShaderManagerVulkan::GetFragmentShaderFromModule(VkShaderM
 // instantaneous.
 
 #define CACHE_HEADER_MAGIC 0xff51f420 
-#define CACHE_VERSION 26
+#define CACHE_VERSION 27
 struct VulkanCacheHeader {
 	uint32_t magic;
 	uint32_t version;