BufferUtils: Add native NEON upload paths

rpcs3/Emu/RSX/Common/BufferUtils.cpp

@@ -8,6 +8,8 @@
 
 #if defined(ARCH_X64)
 #include "BufferUtils_avx512.h"
+#elif defined(ARCH_ARM64)
+#include "BufferUtils_neon.h"
 #endif
 
 #if !defined(_MSC_VER)
@@ -183,6 +185,9 @@ namespace
 #if defined(ARCH_X64)
 DECLARE(copy_data_swap_u32) = build_function_asm<void(*)(u32*, const u32*, u32), asmjit::simd_builder>("copy_data_swap_u32", &build_copy_data_swap_u32<false>);
 DECLARE(copy_data_swap_u32_cmp) = build_function_asm<bool(*)(u32*, const u32*, u32), asmjit::simd_builder>("copy_data_swap_u32_cmp", &build_copy_data_swap_u32<true>);
+#elif defined(ARCH_ARM64)
+DECLARE(copy_data_swap_u32) = copy_data_swap_u32_neon<false>;
+DECLARE(copy_data_swap_u32_cmp) = copy_data_swap_u32_neon<true>;
 #else
 DECLARE(copy_data_swap_u32) = copy_data_swap_u32_naive<false>;
 DECLARE(copy_data_swap_u32_cmp) = copy_data_swap_u32_naive<true>;
@@ -299,6 +304,8 @@ namespace
 				r = upload_xi16(src.data(), dst.data(), count);
 			else
 				r = upload_xi32(src.data(), dst.data(), count);
+#elif defined(ARCH_ARM64)
+			r = upload_untouched_neon<T, false>(src.data(), dst.data(), count);
 #else
 			r = upload_untouched_naive(src.data(), dst.data(), count);
 #endif
@@ -405,6 +412,8 @@ namespace
 				r = upload_xi16(src.data(), dst.data(), count, restart_index);
 			else
 				r = upload_xi32(src.data(), dst.data(), count, restart_index);
+#elif defined(ARCH_ARM64)
+			r = upload_untouched_neon<T, true>(src.data(), dst.data(), count, restart_index);
 #else
 			r = upload_untouched_naive(src.data(), dst.data(), count, restart_index);
 #endif
@@ -699,10 +708,13 @@ void iota16(u16* dst, u32 count)
 		iota16_avx2(dst, count);
 		return;
 	}
+#elif defined(ARCH_ARM64)
+	iota16_neon(dst, count);
+	return;
 #endif
 
 	u32 i = 0;
-#if defined(ARCH_X64) || defined(ARCH_ARM64)
+#if defined(ARCH_X64)
 	// Force 16-byte alignment
 	const uptr mem_addr = reinterpret_cast<uptr>(dst);
 	const u32 head = std::min<u32>(count, ((0 - mem_addr) & 15) / sizeof(u16));

rpcs3/Emu/RSX/Common/BufferUtils_neon.h

@@ -0,0 +1,143 @@
+#pragma once
+
+#include <arm_neon.h>
+
+namespace
+{
+	template <bool Compare>
+	auto copy_data_swap_u32_neon(u32* dst, const u32* src, u32 count)
+	{
+		uint32x4_t changed = vdupq_n_u32(0);
+		u32 tail_changed = 0;
+		u32 i = 0;
+
+		for (; count - i >= 4; i += 4)
+		{
+			const auto value = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(reinterpret_cast<const u8*>(src + i))));
+
+			if constexpr (Compare)
+			{
+				changed = vorrq_u32(changed, veorq_u32(value, vld1q_u32(dst + i)));
+			}
+
+			vst1q_u32(dst + i, value);
+		}
+
+		for (; i < count; ++i)
+		{
+			const u32 value = stx::se_storage<u32>::swap(src[i]);
+
+			if constexpr (Compare)
+			{
+				tail_changed |= value ^ dst[i];
+			}
+
+			dst[i] = value;
+		}
+
+		if constexpr (Compare)
+		{
+			return (vmaxvq_u32(changed) | tail_changed) != 0;
+		}
+	}
+
+	template <typename T, bool Restart>
+	u64 upload_untouched_neon(const be_t<T>* src, T* dst, u32 count, T restart_index = 0)
+	{
+		T min_index = static_cast<T>(-1);
+		T max_index = 0;
+		u32 i = 0;
+
+		if constexpr (sizeof(T) == 2)
+		{
+			auto min = vdupq_n_u16(-1);
+			auto max = vdupq_n_u16(0);
+			const auto restart = vdupq_n_u16(restart_index);
+
+			for (; count - i >= 8; i += 8)
+			{
+				auto value = vreinterpretq_u16_u8(vrev16q_u8(vld1q_u8(reinterpret_cast<const u8*>(src + i))));
+
+				if constexpr (Restart)
+				{
+					const auto mask = vceqq_u16(value, restart);
+					max = vmaxq_u16(max, vbicq_u16(value, mask));
+					value = vorrq_u16(value, mask);
+				}
+				else
+				{
+					max = vmaxq_u16(max, value);
+				}
+
+				min = vminq_u16(min, value);
+				vst1q_u16(dst + i, value);
+			}
+
+			min_index = vminvq_u16(min);
+			max_index = vmaxvq_u16(max);
+		}
+		else
+		{
+			auto min = vdupq_n_u32(-1);
+			auto max = vdupq_n_u32(0);
+			const auto restart = vdupq_n_u32(restart_index);
+
+			for (; count - i >= 4; i += 4)
+			{
+				auto value = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(reinterpret_cast<const u8*>(src + i))));
+
+				if constexpr (Restart)
+				{
+					const auto mask = vceqq_u32(value, restart);
+					max = vmaxq_u32(max, vbicq_u32(value, mask));
+					value = vorrq_u32(value, mask);
+				}
+				else
+				{
+					max = vmaxq_u32(max, value);
+				}
+
+				min = vminq_u32(min, value);
+				vst1q_u32(dst + i, value);
+			}
+
+			min_index = vminvq_u32(min);
+			max_index = vmaxvq_u32(max);
+		}
+
+		for (; i < count; ++i)
+		{
+			const T value = src[i];
+			if (Restart && value == restart_index)
+			{
+				dst[i] = static_cast<T>(-1);
+			}
+			else
+			{
+				min_index = std::min(min_index, value);
+				max_index = std::max(max_index, value);
+				dst[i] = value;
+			}
+		}
+
+		return (u64{max_index} << 32) | min_index;
+	}
+
+	void iota16_neon(u16* dst, u32 count)
+	{
+		const u16 initial[] = {0, 1, 2, 3, 4, 5, 6, 7};
+		auto value = vld1q_u16(initial);
+		u32 i = 0;
+
+		for (; count - i >= 8; i += 8)
+		{
+			vst1q_u16(dst + i, value);
+			value = vaddq_u16(value, vdupq_n_u16(8));
+		}
+
+		for (; i < count; ++i)
+		{
+			dst[i] = static_cast<u16>(i);
+		}
+	}
+} // namespace

rpcs3/emucore.vcxproj

@@ -1060,6 +1060,7 @@
     <ClInclude Include="Emu\RSX\Program\CgBinaryProgram.h" />
     <ClInclude Include="Emu\RSX\Common\BufferUtils.h" />
     <ClInclude Include="Emu\RSX\Common\BufferUtils_avx512.h" />
+    <ClInclude Include="Emu\RSX\Common\BufferUtils_neon.h" />
     <ClInclude Include="Emu\RSX\Program\FragmentProgramDecompiler.h" />
     <ClInclude Include="Emu\RSX\Common\ring_buffer_helper.h" />
     <ClInclude Include="Emu\RSX\Program\ShaderParam.h" />

rpcs3/emucore.vcxproj.filters

@@ -1686,6 +1686,9 @@
     <ClInclude Include="Emu\RSX\Common\BufferUtils_avx512.h">
       <Filter>Emu\GPU\RSX\Common</Filter>
     </ClInclude>
+    <ClInclude Include="Emu\RSX\Common\BufferUtils_neon.h">
+      <Filter>Emu\GPU\RSX\Common</Filter>
+    </ClInclude>
     <ClInclude Include="util\types.hpp">
       <Filter>Utilities</Filter>
     </ClInclude>