diff --git a/GPU/Common/FramebufferManagerCommon.cpp b/GPU/Common/FramebufferManagerCommon.cpp
index 2c21be1bc0..65d12b522e 100644
--- a/GPU/Common/FramebufferManagerCommon.cpp
+++ b/GPU/Common/FramebufferManagerCommon.cpp
@@ -453,10 +453,12 @@ VirtualFramebuffer *FramebufferManagerCommon::DoSetRenderFrameBuffer(Framebuffer
 	}
 
 	if (vfb) {
+		bool resized = false;
 		if ((drawing_width != vfb->bufferWidth || drawing_height != vfb->bufferHeight)) {
 			// Even if it's not newly wrong, if this is larger we need to resize up.
 			if (vfb->width > vfb->bufferWidth || vfb->height > vfb->bufferHeight) {
 				ResizeFramebufFBO(vfb, vfb->width, vfb->height);
+				resized = true;
 			} else if (vfb->newWidth != drawing_width || vfb->newHeight != drawing_height) {
 				// If it's newly wrong, or changing every frame, just keep track.
 				vfb->newWidth = drawing_width;
@@ -470,6 +472,7 @@ VirtualFramebuffer *FramebufferManagerCommon::DoSetRenderFrameBuffer(Framebuffer
 				needsRecreate = needsRecreate || vfb->newHeight > vfb->bufferHeight || vfb->newHeight * 2 < vfb->bufferHeight;
 				if (needsRecreate) {
 					ResizeFramebufFBO(vfb, vfb->width, vfb->height, true);
+					resized = true;
 					// Let's discard this information, might be wrong now.
 					vfb->safeWidth = 0;
 					vfb->safeHeight = 0;
@@ -483,6 +486,14 @@ VirtualFramebuffer *FramebufferManagerCommon::DoSetRenderFrameBuffer(Framebuffer
 			// It's not different, let's keep track of that too.
 			vfb->lastFrameNewSize = gpuStats.numFlips;
 		}
+
+		if (!resized && renderScaleFactor_ != 1 && vfb->renderScaleFactor == 1) {
+			// Might be time to change this framebuffer - have we used depth?
+			if (vfb->usageFlags & FB_USAGE_COLOR_MIXED_DEPTH) {
+				ResizeFramebufFBO(vfb, vfb->width, vfb->height, true);
+				_assert_(vfb->renderScaleFactor != 1);
+			}
+		}
 	}
 
 	// None found? Create one.
@@ -692,6 +703,8 @@ void FramebufferManagerCommon::CopyToDepthFromOverlappingFramebuffers(VirtualFra
 			}
 
 			gpuStats.numReinterpretCopies++;
+			src->usageFlags |= FB_USAGE_COLOR_MIXED_DEPTH;
+			dest->usageFlags |= FB_USAGE_COLOR_MIXED_DEPTH;
 
 			// Copying color to depth.
 			BlitUsingRaster(
@@ -1140,6 +1153,8 @@ void FramebufferManagerCommon::DrawPixels(VirtualFramebuffer *vfb, int dstX, int
 	if (channel == RASTER_DEPTH) {
 		_dbg_assert_(srcPixelFormat == GE_FORMAT_DEPTH16);
 		flags = flags | DRAWTEX_DEPTH;
+		if (vfb)
+			vfb->usageFlags |= FB_USAGE_COLOR_MIXED_DEPTH;
 	}
 
 	Draw::Texture *pixelsTex = MakePixelTexture(srcPixels, srcPixelFormat, srcStride, width, height);
@@ -1628,6 +1643,9 @@ void FramebufferManagerCommon::ResizeFramebufFBO(VirtualFramebuffer *vfb, int w,
 		break;
 	}
 
+	if (vfb->usageFlags & FB_USAGE_COLOR_MIXED_DEPTH) {
+		force1x = false;
+	}
 	if (PSP_CoreParameter().compat.flags().Force04154000Download && vfb->fb_address == 0x04154000) {
 		force1x = true;
 	}
@@ -1824,7 +1842,11 @@ bool FramebufferManagerCommon::NotifyFramebufferCopy(u32 src, u32 dst, int size,
 	}
 	if (dstBuffer) {
 		dstBuffer->last_frame_used = gpuStats.numFlips;
+		if (channel == RASTER_DEPTH && !srcBuffer)
+			dstBuffer->usageFlags |= FB_USAGE_COLOR_MIXED_DEPTH;
 	}
+	if (srcBuffer && channel == RASTER_DEPTH && !dstBuffer)
+		srcBuffer->usageFlags |= FB_USAGE_COLOR_MIXED_DEPTH;
 
 	if (dstBuffer && srcBuffer) {
 		if (srcBuffer == dstBuffer) {
diff --git a/GPU/Common/FramebufferManagerCommon.h b/GPU/Common/FramebufferManagerCommon.h
index b900104b7e..4f0212f8cd 100644
--- a/GPU/Common/FramebufferManagerCommon.h
+++ b/GPU/Common/FramebufferManagerCommon.h
@@ -45,6 +45,7 @@ enum {
 	FB_USAGE_BLUE_TO_ALPHA = 64,
 	FB_USAGE_FIRST_FRAME_SAVED = 128,
 	FB_USAGE_RENDER_DEPTH = 256,
+	FB_USAGE_COLOR_MIXED_DEPTH = 512,
 };
 
 enum {
diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp
index ad860e54a2..3f985ae26b 100644
--- a/GPU/Common/TextureCacheCommon.cpp
+++ b/GPU/Common/TextureCacheCommon.cpp
@@ -1100,6 +1100,9 @@ void TextureCacheCommon::SetTextureFramebuffer(const AttachCandidate &candidate)
 		if (gstate_c.curTextureXOffset != 0 || gstate_c.curTextureYOffset != 0) {
 			gstate_c.SetNeedShaderTexclamp(true);
 		}
+		if (channel == RASTER_DEPTH) {
+			framebuffer->usageFlags |= FB_USAGE_COLOR_MIXED_DEPTH;
+		}
 
 		if (channel == RASTER_DEPTH && !gstate_c.Use(GPU_USE_DEPTH_TEXTURE)) {
 			WARN_LOG_ONCE(ndepthtex, G3D, "Depth textures not supported, not binding");