diff --git a/Common/Vulkan/VulkanMemory.cpp b/Common/Vulkan/VulkanMemory.cpp
index a8c3b48f1d..103e2188ac 100644
--- a/Common/Vulkan/VulkanMemory.cpp
+++ b/Common/Vulkan/VulkanMemory.cpp
@@ -113,7 +113,13 @@ VulkanDeviceAllocator::~VulkanDeviceAllocator() {
 void VulkanDeviceAllocator::Destroy() {
 	for (Slab &slab : slabs_) {
 		// Did anyone forget to free?
-		assert(slab.allocSizes.empty());
+		for (auto pair : slab.allocSizes) {
+			if (slab.usage[pair.first] != 2) {
+				// If it's not 2 (queued), there's a problem.
+				// If it's zero, it means allocSizes is somehow out of sync.
+				Crash();
+			}
+		}
 
 		vulkan_->Delete().QueueDeleteDeviceMemory(slab.deviceMemory);
 	}
diff --git a/GPU/Vulkan/TextureCacheVulkan.cpp b/GPU/Vulkan/TextureCacheVulkan.cpp
index 07fff5b81b..d552f07dbb 100644
--- a/GPU/Vulkan/TextureCacheVulkan.cpp
+++ b/GPU/Vulkan/TextureCacheVulkan.cpp
@@ -149,6 +149,7 @@ TextureCacheVulkan::TextureCacheVulkan(VulkanContext *vulkan)
 
 TextureCacheVulkan::~TextureCacheVulkan() {
 	Clear(true);
+	allocator_->Destroy();
 }
 
 void TextureCacheVulkan::DownloadFramebufferForClut(u32 clutAddr, u32 bytes) {