From 7e065afb4dedf5b881e661ef2cfc3dbc10570c36 Mon Sep 17 00:00:00 2001
From: cottonvibes <cottonvibes@96395faa-99c1-11dd-bbfe-3dabce05a288>
Date: Fri, 20 Feb 2009 05:37:01 +0000
Subject: [PATCH] tweaked FPU compare opcode clamping. this fixes a bug with
 Digimon rumble arena 2. thanks to Nneeve for figuring out the problem.

git-svn-id: http://pcsx2.googlecode.com/svn/trunk@547 96395faa-99c1-11dd-bbfe-3dabce05a288
---
 pcsx2/x86/iFPU.cpp | 49 +++++++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 20 deletions(-)

diff --git a/pcsx2/x86/iFPU.cpp b/pcsx2/x86/iFPU.cpp
index b6da219f9..b0a202a70 100644
--- a/pcsx2/x86/iFPU.cpp
+++ b/pcsx2/x86/iFPU.cpp
@@ -404,6 +404,16 @@ void fpuFloat2(int regd) { // +NaN -> +fMax, -NaN -> -fMax, +Inf -> +fMax, -Inf
 	}
 }
 
+__forceinline void fpuFloat3(int regd) {
+	// This clamp function used in the recC_xx opcodes
+	// Rule of Rose needs clamping or else it crashes (minss or maxss both fix the crash)
+	// Digimon Rumble Arena 2 needs MAXSS clamping (if you only use minss, it spins on the intro-menus; 
+	// it also doesn't like preserving NaN sign with fpuFloat2, so the only way to make Digimon work
+	// is by calling MAXSS first)
+	SSE_MAXSS_M32_to_XMM(regd, (uptr)&g_minvals[0]);
+	//SSE_MINSS_M32_to_XMM(regd, (uptr)&g_maxvals[0]);
+}
+
 void ClampValues(int regd) { 
 	fpuFloat(regd);
 }
@@ -698,30 +708,30 @@ void recC_EQ_xmm(int info)
 
 	switch(info & (PROCESS_EE_S|PROCESS_EE_T) ) {
 		case PROCESS_EE_S: 
-			SSE_MINSS_M32_to_XMM(EEREC_S, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_S);
 			t0reg = _allocTempXMMreg(XMMT_FPS, -1);
 			if (t0reg >= 0) {
 				SSE_MOVSS_M32_to_XMM(t0reg, (uptr)&fpuRegs.fpr[_Ft_]); 
-				SSE_MINSS_M32_to_XMM(t0reg, (uptr)&g_maxvals[0]);
+				fpuFloat3(t0reg);
 				SSE_UCOMISS_XMM_to_XMM(EEREC_S, t0reg); 
 				_freeXMMreg(t0reg);
 			}
 			else SSE_UCOMISS_M32_to_XMM(EEREC_S, (uptr)&fpuRegs.fpr[_Ft_]); 
 			break;
 		case PROCESS_EE_T: 
-			SSE_MINSS_M32_to_XMM(EEREC_T, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_T);
 			t0reg = _allocTempXMMreg(XMMT_FPS, -1);
 			if (t0reg >= 0) {
 				SSE_MOVSS_M32_to_XMM(t0reg, (uptr)&fpuRegs.fpr[_Fs_]); 
-				SSE_MINSS_M32_to_XMM(t0reg, (uptr)&g_maxvals[0]);
+				fpuFloat3(t0reg);
 				SSE_UCOMISS_XMM_to_XMM(t0reg, EEREC_T); 
 				_freeXMMreg(t0reg);
 			}
 			else SSE_UCOMISS_M32_to_XMM(EEREC_T, (uptr)&fpuRegs.fpr[_Fs_]);
 			break;
 		case (PROCESS_EE_S|PROCESS_EE_T): 
-			SSE_MINSS_M32_to_XMM(EEREC_S, (uptr)&g_maxvals[0]);
-			SSE_MINSS_M32_to_XMM(EEREC_T, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_S);
+			fpuFloat3(EEREC_T);
 			SSE_UCOMISS_XMM_to_XMM(EEREC_S, EEREC_T); 
 			break;
 		default: 
@@ -768,22 +778,22 @@ void recC_LE_xmm(int info )
 
 	switch(info & (PROCESS_EE_S|PROCESS_EE_T) ) {
 		case PROCESS_EE_S: 
-			SSE_MINSS_M32_to_XMM(EEREC_S, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_S);
 			t0reg = _allocTempXMMreg(XMMT_FPS, -1);
 			if (t0reg >= 0) {
 				SSE_MOVSS_M32_to_XMM(t0reg, (uptr)&fpuRegs.fpr[_Ft_]); 
-				SSE_MINSS_M32_to_XMM(t0reg, (uptr)&g_maxvals[0]);
+				fpuFloat3(t0reg);
 				SSE_UCOMISS_XMM_to_XMM(EEREC_S, t0reg); 
 				_freeXMMreg(t0reg);
 			}
 			else SSE_UCOMISS_M32_to_XMM(EEREC_S, (uptr)&fpuRegs.fpr[_Ft_]); 
 			break;
 		case PROCESS_EE_T: 
-			SSE_MINSS_M32_to_XMM(EEREC_T, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_T);
 			t0reg = _allocTempXMMreg(XMMT_FPS, -1);
 			if (t0reg >= 0) {
 				SSE_MOVSS_M32_to_XMM(t0reg, (uptr)&fpuRegs.fpr[_Fs_]); 
-				SSE_MINSS_M32_to_XMM(t0reg, (uptr)&g_maxvals[0]);
+				fpuFloat3(t0reg);
 				SSE_UCOMISS_XMM_to_XMM(t0reg, EEREC_T); 
 				_freeXMMreg(t0reg);
 			}
@@ -800,8 +810,8 @@ void recC_LE_xmm(int info )
 			}
 			break;
 		case (PROCESS_EE_S|PROCESS_EE_T):
-			SSE_MINSS_M32_to_XMM(EEREC_S, (uptr)&g_maxvals[0]);
-			SSE_MINSS_M32_to_XMM(EEREC_T, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_S);
+			fpuFloat3(EEREC_T);
 			SSE_UCOMISS_XMM_to_XMM(EEREC_S, EEREC_T); 
 			break;
 		default: // Untested and incorrect, but this case is never reached AFAIK (cottonvibes)
@@ -842,22 +852,22 @@ void recC_LT_xmm(int info)
 	
 	switch(info & (PROCESS_EE_S|PROCESS_EE_T) ) {
 		case PROCESS_EE_S:
-			SSE_MINSS_M32_to_XMM(EEREC_S, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_S);
 			t0reg = _allocTempXMMreg(XMMT_FPS, -1);
 			if (t0reg >= 0) {
 				SSE_MOVSS_M32_to_XMM(t0reg, (uptr)&fpuRegs.fpr[_Ft_]); 
-				SSE_MINSS_M32_to_XMM(t0reg, (uptr)&g_maxvals[0]);
+				fpuFloat3(t0reg);
 				SSE_UCOMISS_XMM_to_XMM(EEREC_S, t0reg); 
 				_freeXMMreg(t0reg);
 			}
 			else SSE_UCOMISS_M32_to_XMM(EEREC_S, (uptr)&fpuRegs.fpr[_Ft_]);
 			break;
 		case PROCESS_EE_T:
-			SSE_MINSS_M32_to_XMM(EEREC_T, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_T);
 			t0reg = _allocTempXMMreg(XMMT_FPS, -1);
 			if (t0reg >= 0) {
 				SSE_MOVSS_M32_to_XMM(t0reg, (uptr)&fpuRegs.fpr[_Fs_]); 
-				SSE_MINSS_M32_to_XMM(t0reg, (uptr)&g_maxvals[0]);
+				fpuFloat3(t0reg);
 				SSE_UCOMISS_XMM_to_XMM(t0reg, EEREC_T); 
 				_freeXMMreg(t0reg);
 			}
@@ -874,11 +884,10 @@ void recC_LT_xmm(int info)
 			}
 			break;
 		case (PROCESS_EE_S|PROCESS_EE_T):
-			// Makes NaNs and +Infinity be +maximum; -Infinity stays 
-			// the same, but this is okay for a Compare operation.
+			// Clamp NaNs
 			// Note: This fixes a crash in Rule of Rose.
-			SSE_MINSS_M32_to_XMM(EEREC_S, (uptr)&g_maxvals[0]);
-			SSE_MINSS_M32_to_XMM(EEREC_T, (uptr)&g_maxvals[0]);
+			fpuFloat3(EEREC_S);
+			fpuFloat3(EEREC_T);
 			SSE_UCOMISS_XMM_to_XMM(EEREC_S, EEREC_T); 
 			break;
 		default: