diff --git a/GPU/Common/FramebufferCommon.h b/GPU/Common/FramebufferCommon.h
index 93fe5e75c..e2b4d3187 100644
--- a/GPU/Common/FramebufferCommon.h
+++ b/GPU/Common/FramebufferCommon.h
@@ -169,7 +169,7 @@ public:
 	void NotifyBlockTransferAfter(u32 dstBasePtr, int dstStride, int dstX, int dstY, u32 srcBasePtr, int srcStride, int srcX, int srcY, int w, int h, int bpp, u32 skipDrawReason);
 
 	virtual void ReadFramebufferToMemory(VirtualFramebuffer *vfb, bool sync, int x, int y, int w, int h) = 0;
-	virtual void DownloadFramebufferForClut(void *clut, u32 fb_address, u32 loadBytes) = 0;
+	virtual void DownloadFramebufferForClut(u32 fb_address, u32 loadBytes) = 0;
 	virtual void MakePixelTexture(const u8 *srcPixels, GEBufferFormat srcPixelFormat, int srcStride, int width, int height) = 0;
 	virtual void DrawPixels(VirtualFramebuffer *vfb, int dstX, int dstY, const u8 *srcPixels, GEBufferFormat srcPixelFormat, int srcStride, int width, int height) = 0;
 	virtual void DrawFramebufferToOutput(const u8 *srcPixels, GEBufferFormat srcPixelFormat, int srcStride, bool applyPostShader) = 0;
diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp
index e8837d55b..56fcbbd59 100644
--- a/GPU/Common/TextureCacheCommon.cpp
+++ b/GPU/Common/TextureCacheCommon.cpp
@@ -213,13 +213,17 @@ void TextureCacheCommon::LoadClut(u32 clutAddr, u32 loadBytes) {
 		if (Memory::IsVRAMAddress(clutAddr)) {
 			// Clear the uncached bit, etc. to match framebuffers.
 			const u32 clutFramebufAddr = clutAddr & 0x3FFFFFFF;
+			const u32 clutFramebufEnd = clutFramebufAddr + loadBytes;
 
 			for (size_t i = 0, n = fbCache_.size(); i < n; ++i) {
 				auto framebuffer = fbCache_[i];
-				if ((framebuffer->fb_address | 0x04000000) == clutFramebufAddr) {
+				const u32 fb_address = framebuffer->fb_address | 0x04000000;
+				const u32 bpp = framebuffer->drawnFormat == GE_FORMAT_8888 ? 4 : 2;
+				if (fb_address + framebuffer->fb_stride * bpp > clutFramebufAddr && fb_address < clutFramebufEnd) {
 					framebuffer->last_frame_clut = gpuStats.numFlips;
 					framebuffer->usageFlags |= FB_USAGE_CLUT;
 					clutRenderAddress_ = framebuffer->fb_address;
+					clutRenderOffset_ = clutFramebufAddr - fb_address;
 				}
 			}
 		}
@@ -227,7 +231,8 @@ void TextureCacheCommon::LoadClut(u32 clutAddr, u32 loadBytes) {
 		// It's possible for a game to (successfully) access outside valid memory.
 		u32 bytes = Memory::ValidSize(clutAddr, loadBytes);
 		if (clutRenderAddress_ != 0xFFFFFFFF && !g_Config.bDisableSlowFramebufEffects) {
-			DownloadFramebufferForClut(clutAddr, bytes);
+			DownloadFramebufferForClut(clutRenderAddress_, clutRenderOffset_ + bytes);
+			Memory::MemcpyUnchecked(clutBufRaw_, clutAddr, bytes);
 			if (bytes < loadBytes) {
 				memset((u8 *)clutBufRaw_ + bytes, 0x00, loadBytes - bytes);
 			}
diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h
index d369b49d9..6df502572 100644
--- a/GPU/Common/TextureCacheCommon.h
+++ b/GPU/Common/TextureCacheCommon.h
@@ -157,6 +157,7 @@ protected:
 	u32 clutTotalBytes_;
 	u32 clutMaxBytes_;
 	u32 clutRenderAddress_;
+	u32 clutRenderOffset_;
 	int standardScaleFactor_;
 };
 
diff --git a/GPU/Directx9/FramebufferDX9.cpp b/GPU/Directx9/FramebufferDX9.cpp
index 1ce821121..776fec393 100644
--- a/GPU/Directx9/FramebufferDX9.cpp
+++ b/GPU/Directx9/FramebufferDX9.cpp
@@ -859,7 +859,7 @@ namespace DX9 {
 		}
 	}
 
-	void FramebufferManagerDX9::DownloadFramebufferForClut(void *clut, u32 fb_address, u32 loadBytes) {
+	void FramebufferManagerDX9::DownloadFramebufferForClut(u32 fb_address, u32 loadBytes) {
 		VirtualFramebuffer *vfb = GetVFBAt(fb_address);
 		if (vfb && vfb->fb_stride != 0) {
 			const u32 bpp = vfb->drawnFormat == GE_FORMAT_8888 ? 4 : 2;
@@ -889,10 +889,6 @@ namespace DX9 {
 				RebindFramebuffer();
 			}
 		}
-
-		if (Memory::IsValidAddress(fb_address | 0x04000000)) {
-			Memory::MemcpyUnchecked(clut, fb_address | 0x04000000, loadBytes);
-		}
 	}
 
 	bool FramebufferManagerDX9::CreateDownloadTempBuffer(VirtualFramebuffer *nvfb) {
diff --git a/GPU/Directx9/FramebufferDX9.h b/GPU/Directx9/FramebufferDX9.h
index 89b88c73e..4ae60bf15 100644
--- a/GPU/Directx9/FramebufferDX9.h
+++ b/GPU/Directx9/FramebufferDX9.h
@@ -74,7 +74,7 @@ public:
 	void BindFramebufferColor(int stage, VirtualFramebuffer *framebuffer, int flags);
 
 	void ReadFramebufferToMemory(VirtualFramebuffer *vfb, bool sync, int x, int y, int w, int h) override;
-	void DownloadFramebufferForClut(void *clut, u32 fb_address, u32 loadBytes) override;
+	void DownloadFramebufferForClut(u32 fb_address, u32 loadBytes) override;
 
 	std::vector<FramebufferInfo> GetFramebufferList();
 
diff --git a/GPU/Directx9/TextureCacheDX9.cpp b/GPU/Directx9/TextureCacheDX9.cpp
index 0dff2b663..dde7b8a48 100644
--- a/GPU/Directx9/TextureCacheDX9.cpp
+++ b/GPU/Directx9/TextureCacheDX9.cpp
@@ -805,7 +805,7 @@ void TextureCacheDX9::ApplyTexture() {
 }
 
 void TextureCacheDX9::DownloadFramebufferForClut(u32 clutAddr, u32 bytes) {
-	framebufferManager_->DownloadFramebufferForClut(clutBufRaw_, clutAddr, bytes);
+	framebufferManager_->DownloadFramebufferForClut(clutAddr, bytes);
 }
 
 class TextureShaderApplierDX9 {
diff --git a/GPU/GLES/Framebuffer.cpp b/GPU/GLES/Framebuffer.cpp
index f1b762703..ffabb0791 100644
--- a/GPU/GLES/Framebuffer.cpp
+++ b/GPU/GLES/Framebuffer.cpp
@@ -1241,7 +1241,7 @@ void FramebufferManager::ReadFramebufferToMemory(VirtualFramebuffer *vfb, bool s
 	}
 }
 
-void FramebufferManager::DownloadFramebufferForClut(void *clut, u32 fb_address, u32 loadBytes) {
+void FramebufferManager::DownloadFramebufferForClut(u32 fb_address, u32 loadBytes) {
 	PROFILE_THIS_SCOPE("gpu-readback");
 	// Flush async just in case.
 	PackFramebufferAsync_(nullptr);
@@ -1275,10 +1275,6 @@ void FramebufferManager::DownloadFramebufferForClut(void *clut, u32 fb_address,
 			RebindFramebuffer();
 		}
 	}
-
-	if (Memory::IsValidAddress(fb_address | 0x04000000)) {
-		Memory::MemcpyUnchecked(clut, fb_address | 0x04000000, loadBytes);
-	}
 }
 
 bool FramebufferManager::CreateDownloadTempBuffer(VirtualFramebuffer *nvfb) {
diff --git a/GPU/GLES/Framebuffer.h b/GPU/GLES/Framebuffer.h
index aa9eb5de1..8f9ab9647 100644
--- a/GPU/GLES/Framebuffer.h
+++ b/GPU/GLES/Framebuffer.h
@@ -102,7 +102,7 @@ public:
 
 	// Reads a rectangular subregion of a framebuffer to the right position in its backing memory.
 	void ReadFramebufferToMemory(VirtualFramebuffer *vfb, bool sync, int x, int y, int w, int h) override;
-	void DownloadFramebufferForClut(void *clut, u32 fb_address, u32 loadBytes) override;
+	void DownloadFramebufferForClut(u32 fb_address, u32 loadBytes) override;
 
 	std::vector<FramebufferInfo> GetFramebufferList();
 
diff --git a/GPU/GLES/TextureCache.cpp b/GPU/GLES/TextureCache.cpp
index bbe70a5d2..000f7231e 100644
--- a/GPU/GLES/TextureCache.cpp
+++ b/GPU/GLES/TextureCache.cpp
@@ -880,7 +880,7 @@ void TextureCache::ApplyTexture() {
 }
 
 void TextureCache::DownloadFramebufferForClut(u32 clutAddr, u32 bytes) {
-	framebufferManager_->DownloadFramebufferForClut(clutBufRaw_, clutAddr, bytes);
+	framebufferManager_->DownloadFramebufferForClut(clutAddr, bytes);
 }
 
 class TextureShaderApplier {