BufferUtils: Optimize restart removal with SVE and SVE2.2

rpcs3/Emu/RSX/Common/BufferUtils.cpp

@@ -23,6 +23,9 @@
 #endif
 #undef FORCE_INLINE
 #include "Emu/CPU/sse2neon.h"
+#if defined(__GNUC__) || defined(__clang__)
+#include "BufferUtils_sve.h"
+#endif
 #endif
 
 #if defined(_MSC_VER) || !defined(__SSE2__)
@@ -500,6 +503,20 @@ namespace
 			};
 			return s_avx2;
 		}
+#elif defined(ARCH_ARM64) && (defined(__GNUC__) || defined(__clang__))
+		if (utils::has_sve())
+		{
+			static const upload_untouched_skip_restart_dispatch s_sve =
+			{
+#ifdef BUFFERUTILS_HAS_SVE2P2
+				utils::has_sve2p2() ? upload_swapped_sve2p2_skip_restart : upload_swapped_neon_skip_restart<u16>,
+#else
+				upload_swapped_neon_skip_restart<u16>,
+#endif
+				upload_swapped_sve_skip_restart,
+			};
+			return s_sve;
+		}
 #endif
 #if defined(ARCH_ARM64)
 		static const upload_untouched_skip_restart_dispatch s_neon =

rpcs3/Emu/RSX/Common/BufferUtils_sve.h

@@ -0,0 +1,82 @@
+#pragma once
+
+#include <arm_sve.h>
+
+#if defined(__clang__)
+#define BUFFERUTILS_SVE __attribute__((target("sve")))
+#else
+#define BUFFERUTILS_SVE __attribute__((target("+sve")))
+#endif
+
+// Older compilers do not provide the svcompact_u16 intrinsic.
+// Please remove this nonsense once we upgrade gcc in the CI - Whatcookie
+#if (defined(__clang__) && __clang_major__ >= 22) || (!defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 17)
+#define BUFFERUTILS_HAS_SVE2P2
+#if defined(__clang__)
+#define BUFFERUTILS_SVE2P2 __attribute__((target("sve2p2")))
+#else
+#define BUFFERUTILS_SVE2P2 __attribute__((target("+sve2p2")))
+#endif
+#endif
+
+namespace
+{
+	BUFFERUTILS_SVE std::tuple<u32, u32, u32> upload_swapped_sve_skip_restart(std::span<to_be_t<const u32>> src, std::span<u32> dst, u32 restart_index)
+	{
+		const u32 count = ::size32(src);
+		u32 written = 0;
+		auto min = svdup_n_u32(u32{0xffffffff});
+		auto max = svdup_n_u32(0);
+		const auto all = svptrue_b32();
+
+		for (u64 i = 0; i < count; i += svcntw())
+		{
+			const auto active = svwhilelt_b32(i, u64{count});
+			auto value = svld1_u32(active, reinterpret_cast<const u32*>(src.data()) + i);
+			value = svrevb_u32_x(active, value);
+			const auto keep = svcmpne_n_u32(active, value, restart_index);
+			min = svmin_u32_m(keep, min, value);
+			max = svmax_u32_m(keep, max, value);
+			const auto packed = svcompact_u32(keep, value);
+			const u32 processed = svcntp_b32(all, keep);
+			const auto output = svwhilelt_b32(u64{0}, u64{processed});
+			svst1_u32(output, dst.data() + written, packed);
+			written += processed;
+		}
+
+		return {svminv_u32(all, min), svmaxv_u32(all, max), written};
+	}
+
+#ifdef BUFFERUTILS_HAS_SVE2P2
+	BUFFERUTILS_SVE2P2 std::tuple<u16, u16, u32> upload_swapped_sve2p2_skip_restart(std::span<to_be_t<const u16>> src, std::span<u16> dst, u16 restart_index)
+	{
+		const u32 count = ::size32(src);
+		u32 written = 0;
+		auto min = svdup_n_u16(u16{0xffff});
+		auto max = svdup_n_u16(0);
+		const auto all = svptrue_b16();
+
+		for (u64 i = 0; i < count; i += svcnth())
+		{
+			const auto active = svwhilelt_b16(i, u64{count});
+			auto value = svld1_u16(active, reinterpret_cast<const u16*>(src.data()) + i);
+			value = svrevb_u16_x(active, value);
+			const auto keep = svcmpne_n_u16(active, value, restart_index);
+			min = svmin_u16_m(keep, min, value);
+			max = svmax_u16_m(keep, max, value);
+			const auto packed = svcompact_u16(keep, value);
+			const u32 processed = svcntp_b16(all, keep);
+			const auto output = svwhilelt_b16(u64{0}, u64{processed});
+			svst1_u16(output, dst.data() + written, packed);
+			written += processed;
+		}
+
+		return {svminv_u16(all, min), svmaxv_u16(all, max), written};
+	}
+#endif
+} // namespace
+
+#undef BUFFERUTILS_SVE
+#ifdef BUFFERUTILS_HAS_SVE2P2
+#undef BUFFERUTILS_SVE2P2
+#endif

rpcs3/emucore.vcxproj

@@ -1061,6 +1061,7 @@
     <ClInclude Include="Emu\RSX\Common\BufferUtils.h" />
     <ClInclude Include="Emu\RSX\Common\BufferUtils_avx512.h" />
     <ClInclude Include="Emu\RSX\Common\BufferUtils_neon.h" />
+    <ClInclude Include="Emu\RSX\Common\BufferUtils_sve.h" />
     <ClInclude Include="Emu\RSX\Common\BufferUtils_compact.h" />
     <ClInclude Include="Emu\RSX\Program\FragmentProgramDecompiler.h" />
     <ClInclude Include="Emu\RSX\Common\ring_buffer_helper.h" />

rpcs3/emucore.vcxproj.filters

@@ -1689,6 +1689,9 @@
     <ClInclude Include="Emu\RSX\Common\BufferUtils_neon.h">
       <Filter>Emu\GPU\RSX\Common</Filter>
     </ClInclude>
+    <ClInclude Include="Emu\RSX\Common\BufferUtils_sve.h">
+      <Filter>Emu\GPU\RSX\Common</Filter>
+    </ClInclude>
     <ClInclude Include="Emu\RSX\Common\BufferUtils_compact.h">
       <Filter>Emu\GPU\RSX\Common</Filter>
     </ClInclude>

rpcs3/util/sysinfo.cpp

@@ -501,6 +501,21 @@ bool utils::has_sve2()
 	return g_value;
 }
 
+bool utils::has_sve2p2()
+{
+	static const bool g_value = []() -> bool
+	{
+		// Detection for SVE2.2 is currently only supported on Linux
+#if defined(__linux__)
+		constexpr unsigned long sve2p2 = 1UL << 41;
+		return (getauxval(AT_HWCAP) & (HWCAP_SVE | sve2p2)) == (HWCAP_SVE | sve2p2);
+#else
+		return false;
+#endif
+	}();
+	return g_value;
+}
+
 #if defined(_MSC_VER)
 #define sve_func
 #else
@@ -588,7 +603,7 @@ std::string utils::get_system_info()
 
 	if (has_sve())
 	{
-		fmt::append(result, " | SVE%s-%d", has_sve2() ? "2" : "", sve_length());
+		fmt::append(result, " | SVE%s-%d", has_sve2p2() ? "2.2" : (has_sve2() ? "2" : ""), sve_length());
 	}
 	else
 	{

rpcs3/util/sysinfo.hpp

@@ -76,6 +76,8 @@ namespace utils
 
 	bool has_sve2();
 
+	bool has_sve2p2();
+
 	int sve_length();
 #endif
 	std::string get_cpu_brand();