Merge pull request #12108 from hrydgard/vulkan-async-pipeline-creation

Vulkan async pipeline creation
2024-11-26 23:10:38 +00:00 · 2021-11-21 22:33:43 +01:00 · 2021-11-21 22:33:43 +01:00 · 712b87ae57
commit 712b87ae57
parent b3bd068051 81fba1f4a4
13 changed files with 375 additions and 102 deletions
--- a/Common/GPU/Vulkan/VulkanContext.cpp
+++ b/Common/GPU/Vulkan/VulkanContext.cpp
@ -1215,6 +1215,8 @@ bool GLSLtoSPV(const VkShaderStageFlagBits shader_type, const char *sourceCode,
 		return false; // something didn't work
 	}

+	// TODO: Propagate warnings into errorMessages even if we succeeded here.
+
 	// Note that program does not take ownership of &shader, so this is fine.
 	program.addShader(&shader);

--- a/Common/GPU/Vulkan/VulkanQueueRunner.cpp
+++ b/Common/GPU/Vulkan/VulkanQueueRunner.cpp
@ -841,10 +841,15 @@ void VulkanQueueRunner::LogRenderPass(const VKRStep &pass, bool verbose) {
 			case VKRRenderCommand::REMOVED:
 				INFO_LOG(G3D, "  (Removed)");
 				break;
-
 			case VKRRenderCommand::BIND_PIPELINE:
 				INFO_LOG(G3D, "  BindPipeline(%x)", (int)(intptr_t)cmd.pipeline.pipeline);
 				break;
+			case VKRRenderCommand::BIND_GRAPHICS_PIPELINE:
+				INFO_LOG(G3D, "  BindGraphicsPipeline(%x)", (int)(intptr_t)cmd.graphics_pipeline.pipeline);
+				break;
+			case VKRRenderCommand::BIND_COMPUTE_PIPELINE:
+				INFO_LOG(G3D, "  BindComputePipeline(%x)", (int)(intptr_t)cmd.compute_pipeline.pipeline);
+				break;
 			case VKRRenderCommand::BLEND:
 				INFO_LOG(G3D, "  BlendColor(%08x)", cmd.blendColor.color);
 				break;
@ -1236,6 +1241,9 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c

 	VKRFramebuffer *fb = step.render.framebuffer;

+	VkPipeline lastGraphicsPipeline = VK_NULL_HANDLE;
+	VkPipeline lastComputePipeline = VK_NULL_HANDLE;
+
 	auto &commands = step.commands;

 	// We can do a little bit of state tracking here to eliminate some calls into the driver.
@ -1251,16 +1259,58 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 		case VKRRenderCommand::REMOVED:
 			break;

+		// Still here to support binding of non-async pipelines.
 		case VKRRenderCommand::BIND_PIPELINE:
-			if (c.pipeline.pipeline != lastPipeline) {
-				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, c.pipeline.pipeline);
-				lastPipeline = c.pipeline.pipeline;
+		{
+			VkPipeline pipeline = c.pipeline.pipeline;
+			if (pipeline != lastGraphicsPipeline) {
+				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+				lastGraphicsPipeline = pipeline;
 				// Reset dynamic state so it gets refreshed with the new pipeline.
 				lastStencilWriteMask = -1;
 				lastStencilCompareMask = -1;
 				lastStencilReference = -1;
 			}
 			break;
+		}
+
+		case VKRRenderCommand::BIND_GRAPHICS_PIPELINE:
+		{
+			VKRGraphicsPipeline *pipeline = c.graphics_pipeline.pipeline;
+			if (!pipeline->pipeline) {
+				// Stall processing, waiting for the compile queue to catch up.
+				std::unique_lock<std::mutex> lock(compileDoneMutex_);
+				while (!pipeline->pipeline) {
+					compileDone_.wait(lock);
+				}
+			}
+			if (pipeline->pipeline != lastGraphicsPipeline) {
+				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
+				lastGraphicsPipeline = pipeline->pipeline;
+				// Reset dynamic state so it gets refreshed with the new pipeline.
+				lastStencilWriteMask = -1;
+				lastStencilCompareMask = -1;
+				lastStencilReference = -1;
+			}
+			break;
+		}
+
+		case VKRRenderCommand::BIND_COMPUTE_PIPELINE:
+		{
+			VKRComputePipeline *pipeline = c.compute_pipeline.pipeline;
+			if (!pipeline->pipeline) {
+				// Stall processing, waiting for the compile queue to catch up.
+				std::unique_lock<std::mutex> lock(compileDoneMutex_);
+				while (!pipeline->pipeline) {
+					compileDone_.wait(lock);
+				}
+			}
+			if (pipeline->pipeline != lastComputePipeline) {
+				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
+				lastComputePipeline = pipeline->pipeline;
+			}
+			break;
+		}

 		case VKRRenderCommand::VIEWPORT:
 			if (fb != nullptr) {
--- a/Common/GPU/Vulkan/VulkanQueueRunner.h
+++ b/Common/GPU/Vulkan/VulkanQueueRunner.h
@ -1,6 +1,9 @@
 #pragma once

 #include <cstdint>
+#include <mutex>
+#include <condition_variable>
+

 #include "Common/Data/Collections/Hashmaps.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
@ -9,6 +12,8 @@
 #include "Common/GPU/DataFormat.h"

 class VKRFramebuffer;
+struct VKRGraphicsPipeline;
+struct VKRComputePipeline;
 struct VKRImage;

 enum {
@ -20,7 +25,9 @@ enum {

 enum class VKRRenderCommand : uint8_t {
 	REMOVED,
-	BIND_PIPELINE,
+	BIND_PIPELINE,  // raw pipeline
+	BIND_GRAPHICS_PIPELINE,  // async
+	BIND_COMPUTE_PIPELINE,  // async
 	STENCIL,
 	BLEND,
 	VIEWPORT,
@ -45,6 +52,12 @@ struct VkRenderData {
 		struct {
 			VkPipeline pipeline;
 		} pipeline;
+		struct {
+			VKRGraphicsPipeline *pipeline;
+		} graphics_pipeline;
+		struct {
+			VKRComputePipeline *pipeline;
+		} compute_pipeline;
 		struct {
 			VkPipelineLayout pipelineLayout;
 			VkDescriptorSet ds;
@ -244,6 +257,15 @@ public:
 		hacksEnabled_ = hacks;
 	}

+	void NotifyCompileDone() {
+		compileDone_.notify_all();
+	}
+
+	void WaitForCompileNotification() {
+		std::unique_lock<std::mutex> lock(compileDoneMutex_);
+		compileDone_.wait(lock);
+	}
+
 private:
 	void InitBackbufferRenderPass();

@ -292,4 +314,8 @@ private:

 	// TODO: Enable based on compat.ini.
 	uint32_t hacksEnabled_ = 0;
+
+	// Compile done notifications.
+	std::mutex compileDoneMutex_;
+	std::condition_variable compileDone_;
 };
--- a/Common/GPU/Vulkan/VulkanRenderManager.cpp
+++ b/Common/GPU/Vulkan/VulkanRenderManager.cpp
@ -22,6 +22,50 @@

 using namespace PPSSPP_VK;

+bool VKRGraphicsPipeline::Create(VulkanContext *vulkan) {
+	if (!desc) {
+		// Already failed to create this one.
+		return false;
+	}
+	VkPipeline pipeline;
+	VkResult result = vkCreateGraphicsPipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &pipeline);
+	this->pipeline = pipeline;
+	delete desc;
+	desc = nullptr;
+	if (result == VK_INCOMPLETE) {
+		// Bad (disallowed by spec) return value seen on Adreno in Burnout :(  Try to ignore?
+		// Would really like to log more here, we could probably attach more info to desc.
+		//
+		// At least create a null placeholder to avoid creating over and over if something is broken.
+		pipeline = VK_NULL_HANDLE;
+		return true;
+	} else if (result != VK_SUCCESS) {
+		pipeline = VK_NULL_HANDLE;
+		ERROR_LOG(G3D, "Failed creating graphics pipeline! result='%s'", VulkanResultToString(result));
+		return false;
+	} else {
+		return true;
+	}
+}
+
+bool VKRComputePipeline::Create(VulkanContext *vulkan) {
+	if (!desc) {
+		// Already failed to create this one.
+		return false;
+	}
+	VkPipeline pipeline;
+	VkResult result = vkCreateComputePipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &pipeline);
+	this->pipeline = pipeline;
+	delete desc;
+	desc = nullptr;
+	if (result != VK_SUCCESS) {
+		pipeline = VK_NULL_HANDLE;
+		ERROR_LOG(G3D, "Failed creating compute pipeline! result='%s'", VulkanResultToString(result));
+		return false;
+	}
+	return true;
+}
+
 VKRFramebuffer::VKRFramebuffer(VulkanContext *vk, VkCommandBuffer initCmd, VkRenderPass renderPass, int _width, int _height, const char *tag) : vulkan_(vk) {
 	width = _width;
 	height = _height;
@ -289,6 +333,8 @@ bool VulkanRenderManager::CreateBackbuffers() {
 		threadInitFrame_ = vulkan_->GetCurFrame();
 		INFO_LOG(G3D, "Starting Vulkan submission thread (threadInitFrame_ = %d)", vulkan_->GetCurFrame());
 		thread_ = std::thread(&VulkanRenderManager::ThreadFunc, this);
+		INFO_LOG(G3D, "Starting Vulkan compiler thread");
+		compileThread_ = std::thread(&VulkanRenderManager::CompileThreadFunc, this);
 	}
 	return true;
 }
@ -312,6 +358,9 @@ void VulkanRenderManager::StopThread() {
 		}
 		thread_.join();
 		INFO_LOG(G3D, "Vulkan submission thread joined. Frame=%d", vulkan_->GetCurFrame());
+		compileCond_.notify_all();
+		compileThread_.join();
+		INFO_LOG(G3D, "Vulkan compiler thread joined.");

 		// Eat whatever has been queued up for this frame if anything.
 		Wipe();
@ -378,6 +427,7 @@ VulkanRenderManager::~VulkanRenderManager() {
 	StopThread();
 	vulkan_->WaitUntilQueueIdle();

+	DrainCompileQueue();
 	VkDevice device = vulkan_->GetDevice();
 	vkDestroySemaphore(device, acquireSemaphore_, nullptr);
 	vkDestroySemaphore(device, renderingCompleteSemaphore_, nullptr);
@ -393,6 +443,42 @@ VulkanRenderManager::~VulkanRenderManager() {
 	queueRunner_.DestroyDeviceObjects();
 }

+void VulkanRenderManager::CompileThreadFunc() {
+	SetCurrentThreadName("ShaderCompile");
+	while (true) {
+		std::vector<CompileQueueEntry> toCompile;
+		{
+			std::unique_lock<std::mutex> lock(compileMutex_);
+			if (compileQueue_.empty()) {
+				compileCond_.wait(lock);
+			}
+			toCompile = std::move(compileQueue_);
+			compileQueue_.clear();
+		}
+		if (!run_) {
+			break;
+		}
+		for (auto entry : toCompile) {
+			switch (entry.type) {
+			case CompileQueueEntry::Type::GRAPHICS:
+				entry.graphics->Create(vulkan_);
+				break;
+			case CompileQueueEntry::Type::COMPUTE:
+				entry.compute->Create(vulkan_);
+				break;
+			}
+		}
+		queueRunner_.NotifyCompileDone();
+	}
+}
+
+void VulkanRenderManager::DrainCompileQueue() {
+	std::unique_lock<std::mutex> lock(compileMutex_);
+	while (!compileQueue_.empty()) {
+		queueRunner_.WaitForCompileNotification();
+	}
+}
+
 void VulkanRenderManager::ThreadFunc() {
 	SetCurrentThreadName("RenderMan");
 	int threadFrame = threadInitFrame_;
--- a/Common/GPU/Vulkan/VulkanRenderManager.h
+++ b/Common/GPU/Vulkan/VulkanRenderManager.h
@ -10,6 +10,7 @@
 #include <cstdint>
 #include <mutex>
 #include <thread>
+#include <queue>

 #include "Common/System/Display.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
@ -109,12 +110,72 @@ struct BoundingRect {
 	}
 };

+// All the data needed to create a graphics pipeline.
+struct VKRGraphicsPipelineDesc {
+	VkPipelineCache pipelineCache = VK_NULL_HANDLE;
+	VkPipelineColorBlendStateCreateInfo cbs{ VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO };
+	VkPipelineColorBlendAttachmentState blend0{};
+	VkPipelineDepthStencilStateCreateInfo dss{ VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO };
+	VkDynamicState dynamicStates[6]{};
+	VkPipelineDynamicStateCreateInfo ds{ VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO };
+	VkPipelineRasterizationStateCreateInfo rs{ VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO };
+	VkPipelineMultisampleStateCreateInfo ms{ VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO };
+	VkPipelineShaderStageCreateInfo shaderStageInfo[2]{};
+	VkPipelineInputAssemblyStateCreateInfo inputAssembly{ VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO };
+	VkVertexInputAttributeDescription attrs[8]{};
+	VkVertexInputBindingDescription ibd{};
+	VkPipelineVertexInputStateCreateInfo vis{ VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO };
+	VkPipelineViewportStateCreateInfo views{ VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO };
+	VkGraphicsPipelineCreateInfo pipe{ VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO };
+};
+
+// All the data needed to create a compute pipeline.
+struct VKRComputePipelineDesc {
+	VkPipelineCache pipelineCache;
+	VkComputePipelineCreateInfo pipe{ VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO };
+};
+
+// Wrapped pipeline, which will later allow for background compilation while emulating the rest of the frame.
+struct VKRGraphicsPipeline {
+	VKRGraphicsPipeline() {
+		pipeline = VK_NULL_HANDLE;
+	}
+	VKRGraphicsPipelineDesc *desc = nullptr;  // While non-zero, is pending and pipeline isn't valid.
+	std::atomic<VkPipeline> pipeline;
+
+	bool Create(VulkanContext *vulkan);
+};
+
+struct VKRComputePipeline {
+	VKRComputePipeline() {
+		pipeline = VK_NULL_HANDLE;
+	}
+	VKRComputePipelineDesc *desc = nullptr;
+	std::atomic<VkPipeline> pipeline;
+
+	bool Create(VulkanContext *vulkan);
+};
+
+struct CompileQueueEntry {
+	CompileQueueEntry(VKRGraphicsPipeline *p) : type(Type::GRAPHICS), graphics(p) {}
+	CompileQueueEntry(VKRComputePipeline *p) : type(Type::COMPUTE), compute(p) {}
+	enum class Type {
+		GRAPHICS,
+		COMPUTE,
+	};
+	Type type;
+	VKRGraphicsPipeline *graphics = nullptr;
+	VKRComputePipeline *compute = nullptr;
+};
+
 class VulkanRenderManager {
 public:
 	VulkanRenderManager(VulkanContext *vulkan);
 	~VulkanRenderManager();

 	void ThreadFunc();
+	void CompileThreadFunc();
+	void DrainCompileQueue();

 	// Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again.
 	void BeginFrame(bool enableProfiling);
@ -152,6 +213,28 @@ public:
 	void CopyFramebuffer(VKRFramebuffer *src, VkRect2D srcRect, VKRFramebuffer *dst, VkOffset2D dstPos, VkImageAspectFlags aspectMask, const char *tag);
 	void BlitFramebuffer(VKRFramebuffer *src, VkRect2D srcRect, VKRFramebuffer *dst, VkRect2D dstRect, VkImageAspectFlags aspectMask, VkFilter filter, const char *tag);

+	// Deferred creation, like in GL. Unlike GL though, the purpose is to allow background creation and avoiding
+	// stalling the emulation thread as much as possible.
+	VKRGraphicsPipeline *CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc) {
+		VKRGraphicsPipeline *pipeline = new VKRGraphicsPipeline();
+		pipeline->desc = desc;
+		compileMutex_.lock();
+		compileQueue_.push_back(CompileQueueEntry(pipeline));
+		compileCond_.notify_one();
+		compileMutex_.unlock();
+		return pipeline;
+	}
+
+	VKRComputePipeline *CreateComputePipeline(VKRComputePipelineDesc *desc) {
+		VKRComputePipeline *pipeline = new VKRComputePipeline();
+		pipeline->desc = desc;
+		compileMutex_.lock();
+		compileQueue_.push_back(CompileQueueEntry(pipeline));
+		compileCond_.notify_one();
+		compileMutex_.unlock();
+		return pipeline;
+	}
+
 	void BindPipeline(VkPipeline pipeline, PipelineFlags flags) {
 		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
 		_dbg_assert_(pipeline != VK_NULL_HANDLE);
@ -161,6 +244,24 @@ public:
 		curRenderStep_->commands.push_back(data);
 	}

+	void BindPipeline(VKRGraphicsPipeline *pipeline, PipelineFlags flags) {
+		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
+		_dbg_assert_(pipeline != nullptr);
+		VkRenderData data{ VKRRenderCommand::BIND_GRAPHICS_PIPELINE };
+		data.graphics_pipeline.pipeline = pipeline;
+		curPipelineFlags_ |= flags;
+		curRenderStep_->commands.push_back(data);
+	}
+
+	void BindPipeline(VKRComputePipeline *pipeline, PipelineFlags flags) {
+		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
+		_dbg_assert_(pipeline != nullptr);
+		VkRenderData data{ VKRRenderCommand::BIND_COMPUTE_PIPELINE };
+		data.compute_pipeline.pipeline = pipeline;
+		curPipelineFlags_ |= flags;
+		curRenderStep_->commands.push_back(data);
+	}
+
 	void SetViewport(const VkViewport &vp) {
 		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
 		_dbg_assert_((int)vp.width >= 0);
@ -421,6 +522,14 @@ private:
 	int threadInitFrame_ = 0;
 	VulkanQueueRunner queueRunner_;

+	// Shader compilation thread to compile while emulating the rest of the frame.
+	// Only one right now but we could use more.
+	std::thread compileThread_;
+	// Sync
+	std::condition_variable compileCond_;
+	std::mutex compileMutex_;
+	std::vector<CompileQueueEntry> compileQueue_;
+
 	// Swap chain management
 	struct SwapchainImageData {
 		VkImage image;
--- a/Core/Config.cpp
+++ b/Core/Config.cpp
@ -914,6 +914,8 @@ static ConfigSetting graphicsSettings[] = {
 	ConfigSetting("InflightFrames", &g_Config.iInflightFrames, 3, true, false),
 	ConfigSetting("RenderDuplicateFrames", &g_Config.bRenderDuplicateFrames, false, true, true),

+	ConfigSetting("ShaderCache", &g_Config.bShaderCache, true, false, false),  // Doesn't save. Ini-only.
+
 	ConfigSetting(false),
 };

--- a/Core/Config.h
+++ b/Core/Config.h
@ -224,6 +224,7 @@ public:
 	bool bFragmentTestCache;
 	int iSplineBezierQuality; // 0 = low , 1 = Intermediate , 2 = High
 	bool bHardwareTessellation;
+	bool bShaderCache;  // Hidden ini-only setting, useful for debugging shader compile times.

 	std::vector<std::string> vPostShaderNames; // Off for chain end (only Off for no shader)
 	std::map<std::string, float> mPostShaderSetting;
--- a/GPU/GLES/GPU_GLES.cpp
+++ b/GPU/GLES/GPU_GLES.cpp
@ -103,10 +103,14 @@ GPU_GLES::GPU_GLES(GraphicsContext *gfxCtx, Draw::DrawContext *draw)
 	// Load shader cache.
 	std::string discID = g_paramSFO.GetDiscID();
 	if (discID.size()) {
-		File::CreateFullPath(GetSysDirectory(DIRECTORY_APP_CACHE));
-		shaderCachePath_ = GetSysDirectory(DIRECTORY_APP_CACHE) / (discID + ".glshadercache");
-		// Actually precompiled by IsReady() since we're single-threaded.
-		shaderManagerGL_->Load(shaderCachePath_);
+		if (g_Config.bShaderCache) {
+			File::CreateFullPath(GetSysDirectory(DIRECTORY_APP_CACHE));
+			shaderCachePath_ = GetSysDirectory(DIRECTORY_APP_CACHE) / (discID + ".glshadercache");
+			// Actually precompiled by IsReady() since we're single-threaded.
+			shaderManagerGL_->Load(shaderCachePath_);
+		} else {
+			INFO_LOG(G3D, "Shader cache disabled. Not loading.");
+		}
 	}

 	if (g_Config.bHardwareTessellation) {
@ -129,7 +133,11 @@ GPU_GLES::~GPU_GLES() {
 	// everything should already be cleared since DeviceLost has been run.

 	if (shaderCachePath_.Valid() && draw_) {
-		shaderManagerGL_->Save(shaderCachePath_);
+		if (g_Config.bShaderCache) {
+			shaderManagerGL_->Save(shaderCachePath_);
+		} else {
+			INFO_LOG(G3D, "Shader cache disabled. Not saving.");
+		}
 	}

 	framebufferManagerGL_->DestroyAllFBOs();
--- a/GPU/Vulkan/DrawEngineVulkan.cpp
+++ b/GPU/Vulkan/DrawEngineVulkan.cpp
@ -826,7 +826,7 @@ void DrawEngineVulkan::DoFlush() {

 			Draw::NativeObject object = framebufferManager_->UseBufferedRendering() ? Draw::NativeObject::FRAMEBUFFER_RENDERPASS : Draw::NativeObject::BACKBUFFER_RENDERPASS;
 			VkRenderPass renderPass = (VkRenderPass)draw_->GetNativeObject(object);
-			VulkanPipeline *pipeline = pipelineManager_->GetOrCreatePipeline(pipelineLayout_, renderPass, pipelineKey_, &dec_->decFmt, vshader, fshader, true);
+			VulkanPipeline *pipeline = pipelineManager_->GetOrCreatePipeline(renderManager, pipelineLayout_, renderPass, pipelineKey_, &dec_->decFmt, vshader, fshader, true);
 			if (!pipeline || !pipeline->pipeline) {
 				// Already logged, let's bail out.
 				return;
@ -957,7 +957,7 @@ void DrawEngineVulkan::DoFlush() {
 				}
 				Draw::NativeObject object = framebufferManager_->UseBufferedRendering() ? Draw::NativeObject::FRAMEBUFFER_RENDERPASS : Draw::NativeObject::BACKBUFFER_RENDERPASS;
 				VkRenderPass renderPass = (VkRenderPass)draw_->GetNativeObject(object);
-				VulkanPipeline *pipeline = pipelineManager_->GetOrCreatePipeline(pipelineLayout_, renderPass, pipelineKey_, &dec_->decFmt, vshader, fshader, false);
+				VulkanPipeline *pipeline = pipelineManager_->GetOrCreatePipeline(renderManager, pipelineLayout_, renderPass, pipelineKey_, &dec_->decFmt, vshader, fshader, false);
 				if (!pipeline || !pipeline->pipeline) {
 					// Already logged, let's bail out.
 					return;
--- a/GPU/Vulkan/GPU_Vulkan.cpp
+++ b/GPU/Vulkan/GPU_Vulkan.cpp
@ -123,6 +123,11 @@ void GPU_Vulkan::CancelReady() {
 }

 void GPU_Vulkan::LoadCache(const Path &filename) {
+	if (!g_Config.bShaderCache) {
+		INFO_LOG(G3D, "Shader cache disabled. Not loading.");
+		return;
+	}
+
 	PSP_SetLoading("Loading shader cache...");
 	// Actually precompiled by IsReady() since we're single-threaded.
 	FILE *f = File::OpenCFile(filename, "rb");
@ -148,6 +153,11 @@ void GPU_Vulkan::LoadCache(const Path &filename) {
 }

 void GPU_Vulkan::SaveCache(const Path &filename) {
+	if (!g_Config.bShaderCache) {
+		INFO_LOG(G3D, "Shader cache disabled. Not saving.");
+		return;
+	}
+
 	if (!draw_) {
 		// Already got the lost message, we're in shutdown.
 		WARN_LOG(G3D, "Not saving shaders - shutting down from in-game.");
--- a/GPU/Vulkan/PipelineManagerVulkan.cpp
+++ b/GPU/Vulkan/PipelineManagerVulkan.cpp
@ -34,8 +34,14 @@ void PipelineManagerVulkan::Clear() {
 	// store the keys.

 	pipelines_.Iterate([&](const VulkanPipelineKey &key, VulkanPipeline *value) {
-		if (value->pipeline)
-			vulkan_->Delete().QueueDeletePipeline(value->pipeline);
+		if (value->pipeline) {
+			VkPipeline pipeline = value->pipeline->pipeline;
+			vulkan_->Delete().QueueDeletePipeline(pipeline);
+			delete value->pipeline;
+		} else {
+			// Something went wrong.
+			ERROR_LOG(G3D, "Null pipeline found in PipelineManagerVulkan::Clear - didn't wait for asyncs?");
+		}
 		delete value;
 	});

@ -163,13 +169,16 @@ static std::string CutFromMain(std::string str) {
 	return rebuilt;
 }

-static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pipelineCache, 
+static VulkanPipeline *CreateVulkanPipeline(VulkanRenderManager *renderManager, VkPipelineCache pipelineCache,
 		VkPipelineLayout layout, VkRenderPass renderPass, const VulkanPipelineRasterStateKey &key,
 		const DecVtxFormat *decFmt, VulkanVertexShader *vs, VulkanFragmentShader *fs, bool useHwTransform) {
+	VKRGraphicsPipelineDesc *desc = new VKRGraphicsPipelineDesc();
+	desc->pipelineCache = pipelineCache;
+
 	PROFILE_THIS_SCOPE("pipelinebuild");
 	bool useBlendConstant = false;

-	VkPipelineColorBlendAttachmentState blend0{};
+	VkPipelineColorBlendAttachmentState &blend0 = desc->blend0;
 	blend0.blendEnable = key.blendEnable;
 	if (key.blendEnable) {
 		blend0.colorBlendOp = (VkBlendOp)key.blendOpColor;
@ -181,7 +190,7 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 	}
 	blend0.colorWriteMask = key.colorWriteMask;

-	VkPipelineColorBlendStateCreateInfo cbs{ VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO };
+	VkPipelineColorBlendStateCreateInfo &cbs = desc->cbs;
 	cbs.flags = 0;
 	cbs.pAttachments = &blend0;
 	cbs.attachmentCount = 1;
@ -191,7 +200,7 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 	else
 		cbs.logicOp = VK_LOGIC_OP_COPY;

-	VkPipelineDepthStencilStateCreateInfo dss{ VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO };
+	VkPipelineDepthStencilStateCreateInfo &dss = desc->dss;
 	dss.depthBoundsTestEnable = false;
 	dss.stencilTestEnable = key.stencilTestEnable;
 	if (key.stencilTestEnable) {
@ -208,7 +217,7 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 		dss.depthWriteEnable = key.depthWriteEnable;
 	}

-	VkDynamicState dynamicStates[8]{};
+	VkDynamicState *dynamicStates = &desc->dynamicStates[0];
 	int numDyn = 0;
 	if (key.blendEnable &&
 		  (UsesBlendConstant(key.srcAlpha) || UsesBlendConstant(key.srcColor) || UsesBlendConstant(key.destAlpha) || UsesBlendConstant(key.destColor))) {
@ -223,12 +232,12 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 		dynamicStates[numDyn++] = VK_DYNAMIC_STATE_STENCIL_REFERENCE;
 	}
 	
-	VkPipelineDynamicStateCreateInfo ds{ VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO };
+	VkPipelineDynamicStateCreateInfo &ds = desc->ds;
 	ds.flags = 0;
 	ds.pDynamicStates = dynamicStates;
 	ds.dynamicStateCount = numDyn;
 	
-	VkPipelineRasterizationStateCreateInfo rs{ VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO };
+	VkPipelineRasterizationStateCreateInfo &rs = desc->rs;
 	rs.flags = 0;
 	rs.depthBiasEnable = false;
 	rs.cullMode = key.cullMode;
@ -238,11 +247,11 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 	rs.polygonMode = VK_POLYGON_MODE_FILL;
 	rs.depthClampEnable = key.depthClampEnable;

-	VkPipelineMultisampleStateCreateInfo ms{ VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO };
+	VkPipelineMultisampleStateCreateInfo &ms = desc->ms;
 	ms.pSampleMask = nullptr;
 	ms.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT;

-	VkPipelineShaderStageCreateInfo ss[2]{};
+	VkPipelineShaderStageCreateInfo *ss = &desc->shaderStageInfo[0];
 	ss[0].sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
 	ss[0].stage = VK_SHADER_STAGE_VERTEX_BIT;
 	ss[0].pSpecializationInfo = nullptr;
@ -265,13 +274,14 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 		return nullPipeline;
 	}

-	VkPipelineInputAssemblyStateCreateInfo inputAssembly{ VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO };
+	VkPipelineInputAssemblyStateCreateInfo &inputAssembly = desc->inputAssembly;
 	inputAssembly.flags = 0;
 	inputAssembly.topology = (VkPrimitiveTopology)key.topology;
 	inputAssembly.primitiveRestartEnable = false;
 	int vertexStride = 0;

-	VkVertexInputAttributeDescription attrs[8];
+	VkVertexInputAttributeDescription *attrs = &desc->attrs[0];
+
 	int attributeCount;
 	if (useHwTransform) {
 		attributeCount = SetupVertexAttribs(attrs, *decFmt);
@ -284,66 +294,49 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 		vertexStride = (int)sizeof(TransformedVertex);
 	}

-	VkVertexInputBindingDescription ibd{};
+	VkVertexInputBindingDescription &ibd = desc->ibd;
 	ibd.binding = 0;
 	ibd.inputRate = VK_VERTEX_INPUT_RATE_VERTEX;
 	ibd.stride = vertexStride;

-	VkPipelineVertexInputStateCreateInfo vis{ VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO };
+	VkPipelineVertexInputStateCreateInfo &vis = desc->vis;
 	vis.flags = 0;
 	vis.vertexBindingDescriptionCount = 1;
-	vis.pVertexBindingDescriptions = &ibd;
+	vis.pVertexBindingDescriptions = &desc->ibd;
 	vis.vertexAttributeDescriptionCount = attributeCount;
 	vis.pVertexAttributeDescriptions = attrs;

-	VkPipelineViewportStateCreateInfo views{ VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO };
+	VkPipelineViewportStateCreateInfo &views = desc->views;
 	views.flags = 0;
 	views.viewportCount = 1;
 	views.scissorCount = 1;
 	views.pViewports = nullptr;  // dynamic
 	views.pScissors = nullptr;  // dynamic

-	VkGraphicsPipelineCreateInfo pipe{ VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO };
+	VkGraphicsPipelineCreateInfo &pipe = desc->pipe;
 	pipe.flags = 0;
 	pipe.stageCount = 2;
 	pipe.pStages = ss;
 	pipe.basePipelineIndex = 0;

-	pipe.pColorBlendState = &cbs;
-	pipe.pDepthStencilState = &dss;
-	pipe.pRasterizationState = &rs;
+	pipe.pColorBlendState = &desc->cbs;
+	pipe.pDepthStencilState = &desc->dss;
+	pipe.pRasterizationState = &desc->rs;

 	// We will use dynamic viewport state.
-	pipe.pVertexInputState = &vis;
-	pipe.pViewportState = &views;
+	pipe.pVertexInputState = &desc->vis;
+	pipe.pViewportState = &desc->views;
 	pipe.pTessellationState = nullptr;
-	pipe.pDynamicState = &ds;
-	pipe.pInputAssemblyState = &inputAssembly;
-	pipe.pMultisampleState = &ms;
+	pipe.pDynamicState = &desc->ds;
+	pipe.pInputAssemblyState = &desc->inputAssembly;
+	pipe.pMultisampleState = &desc->ms;
 	pipe.layout = layout;
 	pipe.basePipelineHandle = VK_NULL_HANDLE;
 	pipe.basePipelineIndex = 0;
 	pipe.renderPass = renderPass;
 	pipe.subpass = 0;

-	VkPipeline pipeline;
-	VkResult result = vkCreateGraphicsPipelines(device, pipelineCache, 1, &pipe, nullptr, &pipeline);
-	if (result != VK_SUCCESS) {
-		ERROR_LOG(G3D, "Failed creating graphics pipeline! result='%s'", VulkanResultToString(result));
-		if (result == VK_INCOMPLETE) {
-			// Typical Adreno return value. It'll usually also log something vague, like "Failed to link shaders".
-			// Let's log some stuff and try to stumble along.
-			ERROR_LOG(G3D, "VS source code:\n%s\n", CutFromMain(vs->GetShaderString(SHADER_STRING_SOURCE_CODE)).c_str());
-			ERROR_LOG(G3D, "FS source code:\n%s\n", CutFromMain(fs->GetShaderString(SHADER_STRING_SOURCE_CODE)).c_str());
-		} else {
-			_dbg_assert_msg_(false, "Failed creating graphics pipeline! result='%s'", VulkanResultToString(result));
-		}
-		// Create a placeholder to avoid creating over and over if something is broken.
-		VulkanPipeline *nullPipeline = new VulkanPipeline();
-		nullPipeline->pipeline = VK_NULL_HANDLE;
-		nullPipeline->flags = 0;
-		return nullPipeline;
-	}
+	VKRGraphicsPipeline *pipeline = renderManager->CreateGraphicsPipeline(desc);

 	VulkanPipeline *vulkanPipeline = new VulkanPipeline();
 	vulkanPipeline->pipeline = pipeline;
@ -358,7 +351,7 @@ static VulkanPipeline *CreateVulkanPipeline(VkDevice device, VkPipelineCache pip
 	return vulkanPipeline;
 }

-VulkanPipeline *PipelineManagerVulkan::GetOrCreatePipeline(VkPipelineLayout layout, VkRenderPass renderPass, const VulkanPipelineRasterStateKey &rasterKey, const DecVtxFormat *decFmt, VulkanVertexShader *vs, VulkanFragmentShader *fs, bool useHwTransform) {
+VulkanPipeline *PipelineManagerVulkan::GetOrCreatePipeline(VulkanRenderManager *renderManager, VkPipelineLayout layout, VkRenderPass renderPass, const VulkanPipelineRasterStateKey &rasterKey, const DecVtxFormat *decFmt, VulkanVertexShader *vs, VulkanFragmentShader *fs, bool useHwTransform) {
 	if (!pipelineCache_) {
 		VkPipelineCacheCreateInfo pc{ VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO };
 		VkResult res = vkCreatePipelineCache(vulkan_->GetDevice(), &pc, nullptr, &pipelineCache_);
@ -380,7 +373,7 @@ VulkanPipeline *PipelineManagerVulkan::GetOrCreatePipeline(VkPipelineLayout layo
 		return iter;

 	VulkanPipeline *pipeline = CreateVulkanPipeline(
-		vulkan_->GetDevice(), pipelineCache_, layout, renderPass, 
+		renderManager, pipelineCache_, layout, renderPass,
 		rasterKey, decFmt, vs, fs, useHwTransform);
 	pipelines_.Insert(key, pipeline);

@ -772,7 +765,7 @@ bool PipelineManagerVulkan::LoadCache(FILE *file, bool loadRawPipelineCache, Sha

 		DecVtxFormat fmt;
 		fmt.InitializeFromID(key.vtxFmtId);
-		VulkanPipeline *pipeline = GetOrCreatePipeline(layout, rp, key.raster,
+		VulkanPipeline *pipeline = GetOrCreatePipeline(rm, layout, rp, key.raster,
 			key.useHWTransform ? &fmt : 0,
 			vs, fs, key.useHWTransform);
 		if (!pipeline) {
--- a/GPU/Vulkan/PipelineManagerVulkan.h
+++ b/GPU/Vulkan/PipelineManagerVulkan.h
@ -25,9 +25,11 @@
 #include "GPU/Common/ShaderCommon.h"
 #include "GPU/Vulkan/VulkanUtil.h"
 #include "GPU/Vulkan/StateMappingVulkan.h"
-
 #include "GPU/Vulkan/VulkanQueueRunner.h"

+struct VKRGraphicsPipeline;
+class VulkanRenderManager;
+
 struct VulkanPipelineKey {
 	VulkanPipelineRasterStateKey raster;  // prim is included here
 	VkRenderPass renderPass;
@ -48,7 +50,7 @@ struct VulkanPipelineKey {

 // Simply wraps a Vulkan pipeline, providing some metadata.
 struct VulkanPipeline {
-	VkPipeline pipeline;
+	VKRGraphicsPipeline *pipeline;
 	int flags;  // PipelineFlags enum above.

 	bool UsesBlendConstant() const { return (flags & PIPELINE_FLAG_USES_BLEND_CONSTANT) != 0; }
@ -67,7 +69,7 @@ public:
 	PipelineManagerVulkan(VulkanContext *ctx);
 	~PipelineManagerVulkan();

-	VulkanPipeline *GetOrCreatePipeline(VkPipelineLayout layout, VkRenderPass renderPass, const VulkanPipelineRasterStateKey &rasterKey, const DecVtxFormat *decFmt, VulkanVertexShader *vs, VulkanFragmentShader *fs, bool useHwTransform);
+	VulkanPipeline *GetOrCreatePipeline(VulkanRenderManager *renderManager, VkPipelineLayout layout, VkRenderPass renderPass, const VulkanPipelineRasterStateKey &rasterKey, const DecVtxFormat *decFmt, VulkanVertexShader *vs, VulkanFragmentShader *fs, bool useHwTransform);
 	int GetNumPipelines() const { return (int)pipelines_.size(); }

 	void Clear();
--- a/GPU/Vulkan/ShaderManagerVulkan.cpp
+++ b/GPU/Vulkan/ShaderManagerVulkan.cpp
@ -42,18 +42,16 @@
 #include "GPU/Vulkan/DrawEngineVulkan.h"
 #include "GPU/Vulkan/FramebufferManagerVulkan.h"

-VulkanFragmentShader::VulkanFragmentShader(VulkanContext *vulkan, FShaderID id, const char *code)
-	: vulkan_(vulkan), id_(id) {
+VkShaderModule CompileShaderModule(VulkanContext *vulkan, VkShaderStageFlagBits stage, const char *code) {
 	PROFILE_THIS_SCOPE("shadercomp");
-	source_ = code;

 	std::string errorMessage;
 	std::vector<uint32_t> spirv;
-#ifdef SHADERLOG
-	OutputDebugStringA(LineNumberString(code).c_str());
-#endif

-	bool success = GLSLtoSPV(VK_SHADER_STAGE_FRAGMENT_BIT, code, GLSLVariant::VULKAN, spirv, &errorMessage);
+	bool success = GLSLtoSPV(stage, code, GLSLVariant::VULKAN, spirv, &errorMessage);
+
+	VkShaderModule shaderModule = VK_NULL_HANDLE;
+
 	if (!errorMessage.empty()) {
 		if (success) {
 			ERROR_LOG(G3D, "Warnings in shader compilation!");
@ -64,18 +62,32 @@ VulkanFragmentShader::VulkanFragmentShader(VulkanContext *vulkan, FShaderID id,
 		ERROR_LOG(G3D, "Shader source:\n%s", code);
 #ifdef SHADERLOG
 		OutputDebugStringA(LineNumberString(code).c_str());
-		OutputDebugStringA("Messages:\n");
+		OutputDebugStringA("Error messages:\n");
 		OutputDebugStringA(errorMessage.c_str());
 #endif
 		Reporting::ReportMessage("Vulkan error in shader compilation: info: %s / code: %s", errorMessage.c_str(), code);
 	} else {
-		success = vulkan_->CreateShaderModule(spirv, &module_);
+#ifdef SHADERLOG
+		OutputDebugStringA(LineNumberString(code).c_str());
+#endif
 #ifdef SHADERLOG
 		OutputDebugStringA("OK\n");
 #endif
 	}

-	if (!success) {
+	if (success) {
+		vulkan->CreateShaderModule(spirv, &shaderModule);
+	}
+
+	return shaderModule;
+}
+
+
+VulkanFragmentShader::VulkanFragmentShader(VulkanContext *vulkan, FShaderID id, const char *code)
+	: vulkan_(vulkan), id_(id) {
+	source_ = code;
+	module_ = CompileShaderModule(vulkan, VK_SHADER_STAGE_FRAGMENT_BIT, source_.c_str());
+	if (!module_) {
 		failed_ = true;
 		return;
 	} else {
@ -102,38 +114,10 @@ std::string VulkanFragmentShader::GetShaderString(DebugShaderStringType type) co

 VulkanVertexShader::VulkanVertexShader(VulkanContext *vulkan, VShaderID id, const char *code, bool useHWTransform)
 	: vulkan_(vulkan), useHWTransform_(useHWTransform), id_(id) {
-	PROFILE_THIS_SCOPE("shadercomp");
 	source_ = code;
-	std::string errorMessage;
-	std::vector<uint32_t> spirv;
-#ifdef SHADERLOG
-	OutputDebugStringA(LineNumberString(code).c_str());
-#endif
-	bool success = GLSLtoSPV(VK_SHADER_STAGE_VERTEX_BIT, code, GLSLVariant::VULKAN, spirv, &errorMessage);
-	if (!errorMessage.empty()) {
-		if (success) {
-			ERROR_LOG(G3D, "Warnings in shader compilation!");
-		} else {
-			ERROR_LOG(G3D, "Error in shader compilation!");
-		}
-		ERROR_LOG(G3D, "Messages: %s", errorMessage.c_str());
-		ERROR_LOG(G3D, "Shader source:\n%s", code);
-#ifdef SHADERLOG
-		OutputDebugStringA(LineNumberString(code).c_str());
-		OutputDebugStringUTF8("Messages:\n");
-		OutputDebugStringUTF8(errorMessage.c_str());
-#endif
-		Reporting::ReportMessage("Vulkan error in shader compilation: info: %s / code: %s", errorMessage.c_str(), code);
-	} else {
-		success = vulkan_->CreateShaderModule(spirv, &module_);
-#ifdef SHADERLOG
-		OutputDebugStringA("OK\n");
-#endif
-	}
-
-	if (!success) {
+	module_ = CompileShaderModule(vulkan, VK_SHADER_STAGE_VERTEX_BIT, source_.c_str());
+	if (!module_) {
 		failed_ = true;
-		module_ = VK_NULL_HANDLE;
 		return;
 	} else {
 		VERBOSE_LOG(G3D, "Compiled vertex shader:\n%s\n", (const char *)code);