diff --git a/pcsx2/vtlb.cpp b/pcsx2/vtlb.cpp
index 32569ff8b2..4fa67cfcb1 100644
--- a/pcsx2/vtlb.cpp
+++ b/pcsx2/vtlb.cpp
@@ -975,13 +975,13 @@ static void vtlb_RemoveFastmemMappings()
 		if (s_fastmem_virtual_mapping[page] == NO_FASTMEM_MAPPING)
 			continue;
 
+		if (vtlb_IsHostCoalesced(page))
+		{
+			if (!s_fastmem_area->Unmap(s_fastmem_area->PagePointer(vtlb_HostPage(page)), __pagesize))
+				Console.Error("Failed to unmap vaddr %08X", page * __pagesize);
+		}
+
 		s_fastmem_virtual_mapping[page] = NO_FASTMEM_MAPPING;
-
-		if (!vtlb_IsHostAligned(page << VTLB_PAGE_BITS))
-			continue;
-
-		if (!s_fastmem_area->Unmap(s_fastmem_area->PagePointer(vtlb_HostPage(page)), __pagesize))
-			Console.Error("Failed to unmap vaddr %08X", page * __pagesize);
 	}
 
 	s_fastmem_physical_mapping.clear();