GS/HW: Reorganize AA1 setup and fix some performance bugs.

Separate depth feedback setup it is not duplicated in both AA1 and alpha test setup.

Try to optimize out depth feedback in AA1 if triangle interiors are discarded.

Fix barriers not being properly enabled for AA1 triangles (was only activating if depth test was used).

pcsx2/GS/GSState.cpp

@@ -6438,11 +6438,6 @@ void GSState::CalcAlphaMinMax(const int tex_alpha_min, const int tex_alpha_max)
 		min = 128;
 		max = 128;
 	}
-	else if (IsCoverageAlphaSupported())
-	{
-		min = 0;
-		max = 128;
-	}
 	else
 	{
 		const GSDrawingContext* context = m_context;
@@ -6520,7 +6515,34 @@ void GSState::CalcAlphaMinMax(const int tex_alpha_min, const int tex_alpha_max)
 
 	m_vt.m_alpha.min = min;
 	m_vt.m_alpha.max = max;
+	m_vt.m_alpha.depth_min = min;
+	m_vt.m_alpha.depth_max = max;
 	m_vt.m_alpha.valid = true;
+
+	if (IsCoverageAlphaSupported())
+	{
+		// Expand the alpha range depending on what the AA1 can do to the alpha.
+		
+		if (PRIM->ABE)
+		{
+			// ABE==1: Coverage is used for alpha only used when incoming alpha is exactly 128.
+			// If 128 is in the incoming range, expand it down to 0, since edges could be 0-127.
+			// Don't change depth min, since depth isn't written on the edges.
+			if (min <= 128 && 128 <= max)
+			{
+				m_vt.m_alpha.min = std::min(0, min);
+			}
+		}
+		else
+		{
+			// ABE==0: Coverage is always used for alpha, so assume exactly 0-128.
+			// Assume exactly 128 for depth, since depth isn't written on the edges.
+			m_vt.m_alpha.min = 0;
+			m_vt.m_alpha.max = 128;
+			m_vt.m_alpha.depth_min = 128;
+			m_vt.m_alpha.depth_max = 128;
+		}
+	}
 }
 
 void GSState::CorrectATEAlphaMinMax(const u32 atst, const int aref)
@@ -6586,107 +6608,127 @@ bool GSState::TryAlphaTest(u32& fm, u32& zm)
 				return true;
 			break;
 		default:
-			ASSUME(0);
+			pxFailRel("Impossible.");
+			break;
 	}
 
-	bool pass = true;
+	enum AlphaTestResult
+	{
+		UNKNOWN,
+		ALL_PASS,
+		ALL_FAIL,
+	};
+
+	AlphaTestResult result, depth_result;
 
 	if (m_context->TEST.ATST == ATST_NEVER)
 	{
-		pass = false; // Shortcut to avoid GetAlphaMinMax below
+		// Shortcut for NEVER to avoid GetAlphaMinMax below.
+		result = ALL_FAIL;
+		depth_result = ALL_FAIL;
 	}
 	else
 	{
 		const GSVertexTrace::VertexAlpha& aminmax = GetAlphaMinMax();
-		const int amin = aminmax.min;
-		const int amax = aminmax.max;
 
 		const int aref = m_context->TEST.AREF;
 
-		switch (m_context->TEST.ATST)
-		{
-			case ATST_NEVER:
-				pass = false;
-				break;
-			case ATST_ALWAYS:
-				pass = true;
-				break;
-			case ATST_LESS:
-				if (amax < aref)
-					pass = true;
-				else if (amin >= aref)
-					pass = false;
-				else
-					return false;
-				break;
-			case ATST_LEQUAL:
-				if (amax <= aref)
-					pass = true;
-				else if (amin > aref)
-					pass = false;
-				else
-					return false;
-				break;
-			case ATST_EQUAL:
-				if (amin == aref && amax == aref)
-					pass = true;
-				else if (amin > aref || amax < aref)
-					pass = false;
-				else
-					return false;
-				break;
-			case ATST_GEQUAL:
-				if (amin >= aref)
-					pass = true;
-				else if (amax < aref)
-					pass = false;
-				else
-					return false;
-				break;
-			case ATST_GREATER:
-				if (amin > aref)
-					pass = true;
-				else if (amax <= aref)
-					pass = false;
-				else
-					return false;
-				break;
-			case ATST_NOTEQUAL:
-				if (amin == aref && amax == aref)
-					pass = false;
-				else if (amin > aref || amax < aref)
-					pass = true;
-				else
-					return false;
-				break;
-			default:
-				ASSUME(0);
-		}
+		const auto GetResult = [&](int amin, int amax) {
+			switch (m_context->TEST.ATST)
+			{
+				case ATST_NEVER:
+					return ALL_FAIL;
+				case ATST_ALWAYS:
+					return ALL_PASS;
+				case ATST_LESS:
+				{
+					if (amax < aref)
+						return ALL_PASS;
+					else if (amin >= aref)
+						return ALL_FAIL;
+					else
+						return UNKNOWN;
+				}
+				case ATST_LEQUAL:
+				{
+					if (amax <= aref)
+						return ALL_PASS;
+					else if (amin > aref)
+						return ALL_FAIL;
+					else
+						return UNKNOWN;
+				}
+				case ATST_EQUAL:
+				{
+					if (amin == aref && amax == aref)
+						return ALL_PASS;
+					else if (amin > aref || amax < aref)
+						return ALL_FAIL;
+					else
+						return UNKNOWN;
+				}
+				case ATST_GEQUAL:
+				{
+					if (amin >= aref)
+						return ALL_PASS;
+					else if (amax < aref)
+						return ALL_FAIL;
+					else
+						return UNKNOWN;
+				}
+				case ATST_GREATER:
+				{
+					if (amin > aref)
+						return ALL_PASS;
+					else if (amax <= aref)
+						return ALL_FAIL;
+					else
+						return UNKNOWN;
+				}
+				case ATST_NOTEQUAL:
+				{
+					if (amin == aref && amax == aref)
+						return ALL_FAIL;
+					else if (amin > aref || amax < aref)
+						return ALL_PASS;
+					else
+						return UNKNOWN;
+				}
+				default:
+					pxFailRel("Impossible");
+					return UNKNOWN;
+			}
+		};
+
+		result = GetResult(aminmax.min, aminmax.max);
+		depth_result = (result != UNKNOWN) ? result : GetResult(aminmax.depth_min, aminmax.depth_max);
 	}
 
-	if (!pass)
+	const u32 fail_fm = (result == ALL_FAIL) ? 0xffffffff : 0x0;
+	const u32 fail_zm = (depth_result == ALL_FAIL) ? 0xffffffff : 0x0;
+
+	switch (fail_type)
 	{
-		switch (fail_type)
-		{
-			case AFAIL_KEEP:
-				fm = zm = 0xffffffff;
-				break;
-			case AFAIL_FB_ONLY:
-				zm = 0xffffffff;
-				break;
-			case AFAIL_ZB_ONLY:
-				fm = 0xffffffff;
-				break;
-			case AFAIL_RGB_ONLY:
-				fm |= 0xff000000;
-				zm = 0xffffffff;
-				break;
-			default:
-				ASSUME(0);
-		}
+		case AFAIL_KEEP:
+			fm |= fail_fm;
+			zm |= fail_zm;
+			break;
+		case AFAIL_FB_ONLY:
+			zm |= fail_zm;
+			break;
+		case AFAIL_ZB_ONLY:
+			fm |= fail_fm;
+			break;
+		case AFAIL_RGB_ONLY:
+			fm |= (fail_fm & 0xff000000);
+			zm |= fail_zm;
+			break;
+		default:
+			pxFailRel("Impossible.");
+			break;
 	}
 
-	return true;
+	return result != UNKNOWN;
 }
 
 bool GSState::IsFlatShaded()
@@ -6756,6 +6798,7 @@ bool GSState::IsCoverageAlphaFixedOne()
 
 bool GSState::IsCoverageAlphaSupported()
 {
+	pxFailRel("Not implemented");
 	return false;
 }
 

pcsx2/GS/Renderers/Common/GSVertexTrace.h

@@ -30,6 +30,10 @@ public:
 	struct VertexAlpha
 	{
 		int min, max;
+
+		// Separate inference for depth if using AA1 coverage alpha, since edges don't write depth.
+		int depth_min, depth_max;
+
 		bool valid;
 	};
 	bool m_accurate_stq = false;

pcsx2/GS/Renderers/HW/GSRendererHW.cpp

@@ -5428,8 +5428,6 @@ void GSRendererHW::SetupIA(float target_scale, float sx, float sy, bool req_vert
 					m_conf.cb_vs.point_size = GSVector2(16.0f * sx, 16.0f * sy);
 					m_conf.topology = GSHWDrawConfig::Topology::Triangle;
 					m_conf.indices_per_prim = 6;
-					m_conf.ps.aa1 = GSHWDrawConfig::PS_AA1::LINE;
-					m_conf.ps.abe = PRIM->ABE != 0;
 					ExpandLineIndices();
 				}
 				else if (unscale_pt_ln)
@@ -5496,9 +5494,6 @@ void GSRendererHW::SetupIA(float target_scale, float sx, float sy, bool req_vert
 					m_conf.cb_vs.point_size = GSVector2(16.0f * sx, 16.0f * sy);
 					m_conf.topology = GSHWDrawConfig::Topology::Triangle;
 					m_conf.indices_per_prim = 3;
-					m_conf.ps.aa1 = m_cached_ctx.DepthWrite() ? GSHWDrawConfig::PS_AA1::TRIANGLE_SW_Z :
-					                                            GSHWDrawConfig::PS_AA1::TRIANGLE;
-					m_conf.ps.abe = PRIM->ABE != 0;
 				}
 				else
 				{
@@ -5792,7 +5787,7 @@ void GSRendererHW::DetermineAlphaScaling(GSTextureCache::Target* rt, GSTextureCa
 	}
 }
 
-void GSRendererHW::EmulateZbufferAA1()
+void GSRendererHW::EmulateAA1()
 {
 	const GSDevice::FeatureSupport& features = g_gs_device->Features();
 
@@ -5800,29 +5795,34 @@ void GSRendererHW::EmulateZbufferAA1()
 
 	if (IsCoverageAlphaSupported())
 	{
+		m_conf.ps.abe = PRIM->ABE; // ABE flag determines how coverage is used for alpha.
+
 		if (m_vt.m_primclass == GS_LINE_CLASS)
 		{
+			GL_INS("HW: AA1 lines. No depth write.");
+
 			// AA1: Z is not written on lines since coverage is always less than 0x80.
 			m_conf.depth.zwe = false;
+			m_cached_ctx.ZBUF.ZMSK = 1;
+
+			m_conf.ps.aa1 = GSHWDrawConfig::PS_AA1::LINE;
 		}
 		else if (m_vt.m_primclass == GS_TRIANGLE_CLASS)
 		{
 			// Force SW depth so that Z writes can be prevented for edge pixels.
 			if (m_cached_ctx.DepthWrite())
 			{
-				// Z test must be also done in the shader.
-				if (m_conf.depth.ztst == ZTST_GEQUAL || m_conf.depth.ztst == ZTST_GREATER)
-				{
-					m_conf.ps.ztst = m_conf.depth.ztst; // Enable shader Z test.
-					m_conf.depth.ztst = ZTST_ALWAYS; // Disable HW Z test.
+				GL_INS("HW: AA1 triangles with depth feedback.");
 
-					// We need barriers for the feedback
-					if (features.feedback_loops())
-					{
-						m_conf.require_one_barrier |= (m_prim_overlap == PRIM_OVERLAP_NO);
-						m_conf.require_full_barrier |= (m_prim_overlap != PRIM_OVERLAP_NO);
-					}
-				}
+				m_conf.ps.aa1 = GSHWDrawConfig::PS_AA1::TRIANGLE_SW_Z; // Allows discarding depth on edge pixels.
+
+				ConfigureDepthFeedback(); // Enable barriers/SW depth test.
+			}
+			else
+			{
+				GL_INS("HW: AA1 triangles with no depth write.");
+
+				m_conf.ps.aa1 = GSHWDrawConfig::PS_AA1::TRIANGLE; // No special depth handling.
 			}
 		}
 		else
@@ -6950,9 +6950,9 @@ void GSRendererHW::EmulateBlending(int rt_alpha_min, int rt_alpha_max, DATEOptio
 		}
 	}
 
-	if (m_conf.alpha_test == GSHWDrawConfig::AlphaTestMode::FEEDBACK && !features.depth_feedback)
+	if (m_conf.ps.IsFeedbackLoopDepth() && !features.depth_feedback)
 	{
-		// If we are doing feedback alpha test with a second RT we must use SW blending to avoid
+		// If we are doing depth feedback with a second RT we must use SW blending to avoid
 		// mixing dual source blending with multiple render targets.
 		sw_blending = true;
 		color_dest_blend = false;
@@ -8420,12 +8420,12 @@ void GSRendererHW::EmulateAlphaTest(DATEOptions& date_options)
 
 	// Determine whether the feedback methods require a single pass.
 	const bool feedback_one_pass = simple_fb_only || simple_rgb_only || simple_zb_only;
-
-	// If we already have the required barriers for the accurate feedback path and
-	// do not require depth feedback.
+	
+	// If we already have the required barriers for the accurate feedback path.
 	const bool free_barrier_feedback =
 		((m_conf.require_one_barrier && feedback_one_pass) || m_conf.require_full_barrier) &&
-		features.feedback_loops() && !afail_needs_depth;
+		features.feedback_loops() &&
+		(!afail_needs_depth || m_conf.ps.IsFeedbackLoopDepth());
 
 	// Determine if we can use FB-fetch for color only feedback.
 	const bool free_fbfetch_feedback = features.framebuffer_fetch && !afail_needs_depth;
@@ -8460,20 +8460,13 @@ void GSRendererHW::EmulateAlphaTest(DATEOptions& date_options)
 			m_conf.require_full_barrier |= !feedback_one_pass;
 		}
 
-		// Handle SW depth writing and/or testing.
 		if (afail_needs_depth)
 		{
 			pxAssert(zwe);
-			GL_INS("Enable SW depth write for depth feedback");
+
+			// Let the pipeline know we need depth feedback with RGB_ONLY.
 			if (afail == AFAIL_RGB_ONLY)
 				m_conf.ps.afail = PS_AFAIL::RGB_ONLY_SW_Z;
-
-			if (m_cached_ctx.DepthRead())
-			{
-				GL_INS("Enable SW depth testing for depth feedback");
-				m_conf.ps.ztst = m_cached_ctx.TEST.ZTST; // Enable SW Z test.
-				m_conf.depth.ztst = ZTST_ALWAYS; // Disable HW Z test.
-			}
 		}
 		else
 		{
@@ -8481,6 +8474,8 @@ void GSRendererHW::EmulateAlphaTest(DATEOptions& date_options)
 			pxAssert(afail != AFAIL_FB_ONLY);
 		}
 
+		ConfigureDepthFeedback(); // Enable barriers/SW depth test if needed.
+
 		m_conf.alpha_test = GSHWDrawConfig::AlphaTestMode::FEEDBACK;
 	}
 	else if (simple_fb_only)
@@ -8672,6 +8667,30 @@ void GSRendererHW::EmulateAlphaTestSecondPass()
 		m_conf.alpha_test = GSHWDrawConfig::AlphaTestMode::ABORT_DRAW;
 }
 
+// Setup barriers and/or SW depth testing for depth feedback.
+void GSRendererHW::ConfigureDepthFeedback()
+{
+	if (m_conf.ps.IsFeedbackLoopDepth())
+	{
+		const GSDevice::FeatureSupport& features = g_gs_device->Features();
+
+		// We need barriers for the feedback.
+		if (features.feedback_loops())
+		{
+			m_conf.require_one_barrier |= (m_prim_overlap == PRIM_OVERLAP_NO);
+			m_conf.require_full_barrier |= (m_prim_overlap != PRIM_OVERLAP_NO);
+		}
+		
+		// We need to do SW depth testing if it's used.
+		if (m_cached_ctx.DepthRead())
+		{
+			GL_INS("HW: Enable SW depth test and disable HW.");
+			m_conf.ps.ztst = m_cached_ctx.TEST.ZTST; // Enable SW Z test.
+			m_conf.depth.ztst = ZTST_ALWAYS; // Disable HW Z test.
+		}
+	}
+}
+
 void GSRendererHW::CleanupDraw(bool invalidate_temp_src)
 {
 	// Remove any RT source.
@@ -8738,7 +8757,8 @@ __ri void GSRendererHW::DrawPrims(GSTextureCache::Target* rt, GSTextureCache::Ta
 
 	m_prim_overlap = PrimitiveOverlap(false);
 
-	EmulateZbufferAA1();
+	// Do AA1 setup early so we can mask depth if possible.
+	EmulateAA1();
 
 	if (rt)
 	{

pcsx2/GS/Renderers/HW/GSRendererHW.h

@@ -226,10 +226,11 @@ private:
 		const TextureMinMaxResult& tmm);
 
 	void EmulateZbuffer(const GSTextureCache::Target* ds);
-	void EmulateZbufferAA1();
+	void EmulateAA1();
 	static void GetAlphaTestConfigPS(const u32 atst, const u8 aref, const bool invert_test, PS_ATST& ps_atst_out, float& aref_out);
 	void EmulateAlphaTest(DATEOptions& date_options);
 	void EmulateAlphaTestSecondPass();
+	void ConfigureDepthFeedback();
 
 	void CalculateAlphaRange(GSTextureCache::Target* rt, GSTextureCache::Target* ds, DATEOptions& date_options,
 		int& blend_alpha_min, int& blend_alpha_max, int& rt_new_alpha_min, int& rt_new_alpha_max);