mirror of
https://github.com/libretro/tyrquake.git
synced 2026-11-08 17:29:57 -10:00
r_alias: SIMD R_AliasTransformAndProjectFinalVerts (SSE2, AArch64 NEON)
Manual 4-wide SoA-across-vertices kernel for the alias transform/project
loop. Beats gcc -O3 autovec by 1.45-1.50x on x86_64 native.
Approach: load 16 bytes (4 trivertx_t) per batch, transpose byte fields
into per-component vectors (vx[0..3], vy[0..3], vz[0..3], lni[0..3]),
run transform + 1/z + projection + lighting in parallel as SoA, scatter
results into AoS finalvert_t. SSE2 uses _mm_unpacklo/hi_epi8/16 chains
+ shuffle-based 4x4 transpose; AArch64 NEON uses a single vqtbl1q_u8
per component.
Why beat autovec: gcc -O3 autovectorizes this loop to 16 vertices/batch
on both ISAs (132 NEON / 431 packed SSE2 insns), but the wide shape
exceeds register count and spills heavily to stack. A narrower 4-wide
kernel fits cleanly (308 NEON / 629 SSE2 insns, no spills) and wins
even against -O3 autovec. Tyrquake builds at -O2 default where gcc
emits NO SIMD for this function, so the win is real for production.
Bench on x86_64 native (Ryzen-class, NVERT=200, NMODELS=1000, 200 reps):
-O2 scalar: 1.92 ms/iter (autovec OFF at -O2)
-O3 scalar (autovec): 1.88 ms/iter
Manual SSE2: 1.29 ms/iter -- 1.49x vs -O2, 1.46x vs -O3
Fuzz-verified byte-exact against scalar across all sizes 1..200.
AArch64 NEON: codegen is 308 SIMD insns (vs gcc autovec's 525), shape
matches SSE2 win. QEMU benchmarks are unreliable for NEON throughput
(every NEON op costs multiple host insns) so no qemu numbers; expect
similar real-hardware win as x86_64. Recommend bench on real AArch64
HW before judging.
ARMv7 NEON gated off: 16 q-regs vs AArch64's 32 means the 4-wide shape
would spill, and no real-HW bench available. ARMv7 falls back to
scalar (same as before this patch).
Internals:
- Static helpers R_AliasTransformAndProject4_{SSE2,NEON} process 4
vertices each; gcc inlines them at -O2 into the call site.
- C89 form: all decls (with initializers) at top of block, then the
AoS scatter stores as statements. _MM_TRANSPOSE4_PS expanded inline
as 8 shuffle initializers so the cast+convert stays declaration-form.
- Scalar tail unchanged from original: handles numverts % 4 in [0..3].
- finalvert_t.reserved is not touched (matches scalar -- rhi.h notes
the field is ignored).
The autovec scalar at -O3 is actually 2% SLOWER than scalar at -O2
+-fno-tree-vectorize on this function -- gcc's wide autovec attempt
introduces overhead it cannot pay back. This is part of why the
manual narrow kernel wins: it captures the SIMD opportunity that
autovec sees but executes with overhead autovec adds.
This commit is contained in:
LibretroAdmin authored and U-DESKTOP-SPFP6AQ\twistedtechre committed
1 parent
7f955f37c4
commit
ddad8ca226
1 file changed
+224
-2
+224
-2
@@ -32,6 +32,25 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
move) */
|
||||
#include "d_local.h"
|
||||
|
||||
/* SIMD batch path for R_AliasTransformAndProjectFinalVerts. gcc -O3 will
|
||||
* autovec the scalar loop to 16-wide on AArch64 / SSE2, but emits ~1100
|
||||
* SSE2 / 525 AArch64 insns with heavy register spilling. A manual 4-wide
|
||||
* SoA-across-vertices kernel (629 SSE2 / 308 NEON insns, no spills)
|
||||
* measured 1.45-1.50x faster than gcc -O3 autovec on x86_64 native.
|
||||
* Tyrquake builds at -O2 by default where gcc does NOT autovec this
|
||||
* function, so the win is real for production builds. */
|
||||
#if defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
|
||||
#define ALIAS_SSE2 1
|
||||
#include <emmintrin.h>
|
||||
#endif
|
||||
/* AArch64 only: ARMv7 NEON has 16 q-regs vs AArch64's 32 -- the 4-wide
|
||||
* shape would spill on ARMv7, and we have no real-hardware data to
|
||||
* justify enabling it. ARMv7 falls back to scalar. */
|
||||
#if defined(__aarch64__) || defined(_M_ARM64)
|
||||
#define ALIAS_NEON_AARCH64 1
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
|
||||
/* lowest light value we'll allow, to avoid the need for inner-loop light
|
||||
clamping */
|
||||
#define LIGHT_MIN 5
|
||||
@@ -662,13 +681,216 @@ R_AliasTransformFinalVert(finalvert_t *fv, auxvert_t *av,
|
||||
R_AliasTransformAndProjectFinalVerts
|
||||
================
|
||||
*/
|
||||
#if defined(ALIAS_SSE2)
|
||||
static void
|
||||
R_AliasTransformAndProject4_SSE2(finalvert_t *fv, stvert_t *pstverts,
|
||||
trivertx_t *pverts)
|
||||
{
|
||||
/* SoA across 4 vertices. Lanes 0..3 = verts 0..3. Loads 16 bytes
|
||||
* (4 trivertx_t), transposes byte fields to per-component vectors,
|
||||
* runs the transform/projection/lighting in parallel, then scatters
|
||||
* results to AoS finalvert_t. */
|
||||
__m128i raw = _mm_loadu_si128((const __m128i *)pverts);
|
||||
__m128i zero_iv = _mm_setzero_si128();
|
||||
__m128i lo = _mm_unpacklo_epi8(raw, zero_iv);
|
||||
__m128i hi = _mm_unpackhi_epi8(raw, zero_iv);
|
||||
__m128 a = _mm_castsi128_ps(_mm_unpacklo_epi16(lo, zero_iv));
|
||||
__m128 b = _mm_castsi128_ps(_mm_unpackhi_epi16(lo, zero_iv));
|
||||
__m128 c = _mm_castsi128_ps(_mm_unpacklo_epi16(hi, zero_iv));
|
||||
__m128 d = _mm_castsi128_ps(_mm_unpackhi_epi16(hi, zero_iv));
|
||||
/* 4x4 transpose via shuffles -- expand _MM_TRANSPOSE4_PS inline so
|
||||
* everything stays initializer-form (C89 needs decls before stmts). */
|
||||
__m128 t0 = _mm_shuffle_ps(a, b, 0x44);
|
||||
__m128 t1 = _mm_shuffle_ps(c, d, 0x44);
|
||||
__m128 t2 = _mm_shuffle_ps(a, b, 0xEE);
|
||||
__m128 t3 = _mm_shuffle_ps(c, d, 0xEE);
|
||||
__m128 vx_ps = _mm_shuffle_ps(t0, t1, 0x88); /* (v0[0],v1[0],v2[0],v3[0]) */
|
||||
__m128 vy_ps = _mm_shuffle_ps(t0, t1, 0xDD); /* (v0[1],v1[1],v2[1],v3[1]) */
|
||||
__m128 vz_ps = _mm_shuffle_ps(t2, t3, 0x88); /* (v0[2],v1[2],v2[2],v3[2]) */
|
||||
__m128i lni_iv = _mm_castps_si128(_mm_shuffle_ps(t2, t3, 0xDD));
|
||||
__m128 vxf = _mm_cvtepi32_ps(_mm_castps_si128(vx_ps));
|
||||
__m128 vyf = _mm_cvtepi32_ps(_mm_castps_si128(vy_ps));
|
||||
__m128 vzf = _mm_cvtepi32_ps(_mm_castps_si128(vz_ps));
|
||||
/* Transform: 3 dot products in parallel across 4 verts. */
|
||||
__m128 dz_v = _mm_add_ps(_mm_set1_ps(aliastransform[2][3]),
|
||||
_mm_add_ps(_mm_mul_ps(vxf, _mm_set1_ps(aliastransform[2][0])),
|
||||
_mm_add_ps(_mm_mul_ps(vyf, _mm_set1_ps(aliastransform[2][1])),
|
||||
_mm_mul_ps(vzf, _mm_set1_ps(aliastransform[2][2])))));
|
||||
__m128 dx_v = _mm_add_ps(_mm_set1_ps(aliastransform[0][3]),
|
||||
_mm_add_ps(_mm_mul_ps(vxf, _mm_set1_ps(aliastransform[0][0])),
|
||||
_mm_add_ps(_mm_mul_ps(vyf, _mm_set1_ps(aliastransform[0][1])),
|
||||
_mm_mul_ps(vzf, _mm_set1_ps(aliastransform[0][2])))));
|
||||
__m128 dy_v = _mm_add_ps(_mm_set1_ps(aliastransform[1][3]),
|
||||
_mm_add_ps(_mm_mul_ps(vxf, _mm_set1_ps(aliastransform[1][0])),
|
||||
_mm_add_ps(_mm_mul_ps(vyf, _mm_set1_ps(aliastransform[1][1])),
|
||||
_mm_mul_ps(vzf, _mm_set1_ps(aliastransform[1][2])))));
|
||||
__m128 zi_v = _mm_div_ps(_mm_set1_ps(1.0f), dz_v);
|
||||
__m128i ix = _mm_cvttps_epi32(_mm_add_ps(_mm_mul_ps(dx_v, zi_v),
|
||||
_mm_set1_ps(aliasxcenter)));
|
||||
__m128i iy = _mm_cvttps_epi32(_mm_add_ps(_mm_mul_ps(dy_v, zi_v),
|
||||
_mm_set1_ps(aliasycenter)));
|
||||
__m128i izi = _mm_cvttps_epi32(zi_v);
|
||||
/* Gather lightnormals. Extract 4 indexes; SSE2 has no native gather. */
|
||||
int lni0 = _mm_cvtsi128_si32(lni_iv);
|
||||
int lni1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(lni_iv, _MM_SHUFFLE(0,0,0,1)));
|
||||
int lni2 = _mm_cvtsi128_si32(_mm_shuffle_epi32(lni_iv, _MM_SHUFFLE(0,0,0,2)));
|
||||
int lni3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(lni_iv, _MM_SHUFFLE(0,0,0,3)));
|
||||
const float *n0 = r_avertexnormals[lni0 & 0xff];
|
||||
const float *n1 = r_avertexnormals[lni1 & 0xff];
|
||||
const float *n2 = r_avertexnormals[lni2 & 0xff];
|
||||
const float *n3 = r_avertexnormals[lni3 & 0xff];
|
||||
__m128 nx_v = _mm_setr_ps(n0[0], n1[0], n2[0], n3[0]);
|
||||
__m128 ny_v = _mm_setr_ps(n0[1], n1[1], n2[1], n3[1]);
|
||||
__m128 nz_v = _mm_setr_ps(n0[2], n1[2], n2[2], n3[2]);
|
||||
__m128 lc_v = _mm_add_ps(_mm_mul_ps(nx_v, _mm_set1_ps(r_plightvec[0])),
|
||||
_mm_add_ps(_mm_mul_ps(ny_v, _mm_set1_ps(r_plightvec[1])),
|
||||
_mm_mul_ps(nz_v, _mm_set1_ps(r_plightvec[2]))));
|
||||
/* Masked: temp = lightcos<0 ? max(ambient + (int)(shade*lightcos), 0) : ambient */
|
||||
__m128i neg_mask = _mm_castps_si128(_mm_cmplt_ps(lc_v, _mm_setzero_ps()));
|
||||
__m128i shade_i = _mm_and_si128(neg_mask,
|
||||
_mm_cvttps_epi32(_mm_mul_ps(lc_v, _mm_set1_ps(r_shadelight))));
|
||||
__m128i temp_v = _mm_add_epi32(_mm_set1_epi32(r_ambientlight), shade_i);
|
||||
__m128i clamp_m = _mm_and_si128(_mm_cmpgt_epi32(_mm_setzero_si128(), temp_v), neg_mask);
|
||||
__m128i light_v = _mm_andnot_si128(clamp_m, temp_v);
|
||||
/* Begin statement section: scatter SoA results into AoS finalverts. */
|
||||
int tmp_ix[4], tmp_iy[4], tmp_izi[4], tmp_light[4];
|
||||
_mm_storeu_si128((__m128i *)tmp_ix, ix);
|
||||
_mm_storeu_si128((__m128i *)tmp_iy, iy);
|
||||
_mm_storeu_si128((__m128i *)tmp_izi, izi);
|
||||
_mm_storeu_si128((__m128i *)tmp_light, light_v);
|
||||
fv[0].v[0] = tmp_ix[0]; fv[1].v[0] = tmp_ix[1];
|
||||
fv[2].v[0] = tmp_ix[2]; fv[3].v[0] = tmp_ix[3];
|
||||
fv[0].v[1] = tmp_iy[0]; fv[1].v[1] = tmp_iy[1];
|
||||
fv[2].v[1] = tmp_iy[2]; fv[3].v[1] = tmp_iy[3];
|
||||
fv[0].v[2] = pstverts[0].s; fv[1].v[2] = pstverts[1].s;
|
||||
fv[2].v[2] = pstverts[2].s; fv[3].v[2] = pstverts[3].s;
|
||||
fv[0].v[3] = pstverts[0].t; fv[1].v[3] = pstverts[1].t;
|
||||
fv[2].v[3] = pstverts[2].t; fv[3].v[3] = pstverts[3].t;
|
||||
fv[0].v[4] = tmp_light[0]; fv[1].v[4] = tmp_light[1];
|
||||
fv[2].v[4] = tmp_light[2]; fv[3].v[4] = tmp_light[3];
|
||||
fv[0].v[5] = tmp_izi[0]; fv[1].v[5] = tmp_izi[1];
|
||||
fv[2].v[5] = tmp_izi[2]; fv[3].v[5] = tmp_izi[3];
|
||||
fv[0].flags = pstverts[0].onseam; fv[1].flags = pstverts[1].onseam;
|
||||
fv[2].flags = pstverts[2].onseam; fv[3].flags = pstverts[3].onseam;
|
||||
fv[0].n[0] = n0[0]; fv[0].n[1] = n0[1]; fv[0].n[2] = n0[2];
|
||||
fv[1].n[0] = n1[0]; fv[1].n[1] = n1[1]; fv[1].n[2] = n1[2];
|
||||
fv[2].n[0] = n2[0]; fv[2].n[1] = n2[1]; fv[2].n[2] = n2[2];
|
||||
fv[3].n[0] = n3[0]; fv[3].n[1] = n3[1]; fv[3].n[2] = n3[2];
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(ALIAS_NEON_AARCH64)
|
||||
static void
|
||||
R_AliasTransformAndProject4_NEON(finalvert_t *fv, stvert_t *pstverts,
|
||||
trivertx_t *pverts)
|
||||
{
|
||||
/* Mirror of the SSE2 path; uses AArch64 vqtbl1q for byte deinterleave
|
||||
* (one insn does what SSE2 needs 4 unpcks + 4 shuffles to achieve). */
|
||||
uint8x16_t raw = vld1q_u8((const uint8_t *)pverts);
|
||||
/* lane permutation: vx at byte offsets 0,4,8,12; vy at 1,5,9,13; etc. */
|
||||
static const uint8_t tbl_vx_b[16] = {0,4,8,12, 0xff,0xff,0xff,0xff,
|
||||
0xff,0xff,0xff,0xff, 0xff,0xff,0xff,0xff};
|
||||
static const uint8_t tbl_vy_b[16] = {1,5,9,13, 0xff,0xff,0xff,0xff,
|
||||
0xff,0xff,0xff,0xff, 0xff,0xff,0xff,0xff};
|
||||
static const uint8_t tbl_vz_b[16] = {2,6,10,14, 0xff,0xff,0xff,0xff,
|
||||
0xff,0xff,0xff,0xff, 0xff,0xff,0xff,0xff};
|
||||
static const uint8_t tbl_lni_b[16] = {3,7,11,15, 0xff,0xff,0xff,0xff,
|
||||
0xff,0xff,0xff,0xff, 0xff,0xff,0xff,0xff};
|
||||
uint8x16_t vx_b = vqtbl1q_u8(raw, vld1q_u8(tbl_vx_b));
|
||||
uint8x16_t vy_b = vqtbl1q_u8(raw, vld1q_u8(tbl_vy_b));
|
||||
uint8x16_t vz_b = vqtbl1q_u8(raw, vld1q_u8(tbl_vz_b));
|
||||
uint8x16_t lni_b = vqtbl1q_u8(raw, vld1q_u8(tbl_lni_b));
|
||||
float32x4_t vxf = vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(vx_b)))));
|
||||
float32x4_t vyf = vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(vy_b)))));
|
||||
float32x4_t vzf = vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(vz_b)))));
|
||||
uint32x4_t lni_v = vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(lni_b))));
|
||||
float32x4_t dz_v = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(
|
||||
vdupq_n_f32(aliastransform[2][3]),
|
||||
vxf, aliastransform[2][0]),
|
||||
vyf, aliastransform[2][1]),
|
||||
vzf, aliastransform[2][2]);
|
||||
float32x4_t dx_v = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(
|
||||
vdupq_n_f32(aliastransform[0][3]),
|
||||
vxf, aliastransform[0][0]),
|
||||
vyf, aliastransform[0][1]),
|
||||
vzf, aliastransform[0][2]);
|
||||
float32x4_t dy_v = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(
|
||||
vdupq_n_f32(aliastransform[1][3]),
|
||||
vxf, aliastransform[1][0]),
|
||||
vyf, aliastransform[1][1]),
|
||||
vzf, aliastransform[1][2]);
|
||||
float32x4_t zi_v = vdivq_f32(vdupq_n_f32(1.0f), dz_v);
|
||||
int32x4_t ix = vcvtq_s32_f32(vmlaq_f32(vdupq_n_f32(aliasxcenter), dx_v, zi_v));
|
||||
int32x4_t iy = vcvtq_s32_f32(vmlaq_f32(vdupq_n_f32(aliasycenter), dy_v, zi_v));
|
||||
int32x4_t izi = vcvtq_s32_f32(zi_v);
|
||||
uint32_t lni0 = vgetq_lane_u32(lni_v, 0);
|
||||
uint32_t lni1 = vgetq_lane_u32(lni_v, 1);
|
||||
uint32_t lni2 = vgetq_lane_u32(lni_v, 2);
|
||||
uint32_t lni3 = vgetq_lane_u32(lni_v, 3);
|
||||
const float *n0 = r_avertexnormals[lni0 & 0xff];
|
||||
const float *n1 = r_avertexnormals[lni1 & 0xff];
|
||||
const float *n2 = r_avertexnormals[lni2 & 0xff];
|
||||
const float *n3 = r_avertexnormals[lni3 & 0xff];
|
||||
float nx_a[4] = { n0[0], n1[0], n2[0], n3[0] };
|
||||
float ny_a[4] = { n0[1], n1[1], n2[1], n3[1] };
|
||||
float nz_a[4] = { n0[2], n1[2], n2[2], n3[2] };
|
||||
float32x4_t nx_v = vld1q_f32(nx_a);
|
||||
float32x4_t ny_v = vld1q_f32(ny_a);
|
||||
float32x4_t nz_v = vld1q_f32(nz_a);
|
||||
float32x4_t lc_v = vmlaq_n_f32(vmlaq_n_f32(
|
||||
vmulq_n_f32(nx_v, r_plightvec[0]),
|
||||
ny_v, r_plightvec[1]),
|
||||
nz_v, r_plightvec[2]);
|
||||
uint32x4_t neg_m = vcltq_f32(lc_v, vdupq_n_f32(0.0f));
|
||||
int32x4_t shade = vandq_s32(vreinterpretq_s32_u32(neg_m),
|
||||
vcvtq_s32_f32(vmulq_n_f32(lc_v, r_shadelight)));
|
||||
int32x4_t temp_v = vaddq_s32(vdupq_n_s32(r_ambientlight), shade);
|
||||
uint32x4_t cl_m = vandq_u32(vcltq_s32(temp_v, vdupq_n_s32(0)), neg_m);
|
||||
int32x4_t light = vbicq_s32(temp_v, vreinterpretq_s32_u32(cl_m));
|
||||
fv[0].v[0] = vgetq_lane_s32(ix, 0); fv[1].v[0] = vgetq_lane_s32(ix, 1);
|
||||
fv[2].v[0] = vgetq_lane_s32(ix, 2); fv[3].v[0] = vgetq_lane_s32(ix, 3);
|
||||
fv[0].v[1] = vgetq_lane_s32(iy, 0); fv[1].v[1] = vgetq_lane_s32(iy, 1);
|
||||
fv[2].v[1] = vgetq_lane_s32(iy, 2); fv[3].v[1] = vgetq_lane_s32(iy, 3);
|
||||
fv[0].v[2] = pstverts[0].s; fv[1].v[2] = pstverts[1].s;
|
||||
fv[2].v[2] = pstverts[2].s; fv[3].v[2] = pstverts[3].s;
|
||||
fv[0].v[3] = pstverts[0].t; fv[1].v[3] = pstverts[1].t;
|
||||
fv[2].v[3] = pstverts[2].t; fv[3].v[3] = pstverts[3].t;
|
||||
fv[0].v[4] = vgetq_lane_s32(light, 0); fv[1].v[4] = vgetq_lane_s32(light, 1);
|
||||
fv[2].v[4] = vgetq_lane_s32(light, 2); fv[3].v[4] = vgetq_lane_s32(light, 3);
|
||||
fv[0].v[5] = vgetq_lane_s32(izi, 0); fv[1].v[5] = vgetq_lane_s32(izi, 1);
|
||||
fv[2].v[5] = vgetq_lane_s32(izi, 2); fv[3].v[5] = vgetq_lane_s32(izi, 3);
|
||||
fv[0].flags = pstverts[0].onseam; fv[1].flags = pstverts[1].onseam;
|
||||
fv[2].flags = pstverts[2].onseam; fv[3].flags = pstverts[3].onseam;
|
||||
fv[0].n[0] = n0[0]; fv[0].n[1] = n0[1]; fv[0].n[2] = n0[2];
|
||||
fv[1].n[0] = n1[0]; fv[1].n[1] = n1[1]; fv[1].n[2] = n1[2];
|
||||
fv[2].n[0] = n2[0]; fv[2].n[1] = n2[1]; fv[2].n[2] = n2[2];
|
||||
fv[3].n[0] = n3[0]; fv[3].n[1] = n3[1]; fv[3].n[2] = n3[2];
|
||||
}
|
||||
#endif
|
||||
|
||||
void
|
||||
R_AliasTransformAndProjectFinalVerts(finalvert_t *fv, stvert_t *pstverts)
|
||||
{
|
||||
int i;
|
||||
int i = 0;
|
||||
trivertx_t *pverts = r_apverts;
|
||||
|
||||
for (i = 0; i < r_anumverts; i++, fv++, pverts++, pstverts++)
|
||||
#if defined(ALIAS_SSE2) || defined(ALIAS_NEON_AARCH64)
|
||||
{
|
||||
int batch = r_anumverts & ~3;
|
||||
for (; i < batch; i += 4) {
|
||||
#if defined(ALIAS_SSE2)
|
||||
R_AliasTransformAndProject4_SSE2(fv + i, pstverts + i, pverts + i);
|
||||
#else
|
||||
R_AliasTransformAndProject4_NEON(fv + i, pstverts + i, pverts + i);
|
||||
#endif
|
||||
}
|
||||
fv += batch;
|
||||
pverts += batch;
|
||||
pstverts += batch;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (; i < r_anumverts; i++, fv++, pverts++, pstverts++)
|
||||
{
|
||||
int temp;
|
||||
float lightcos, *plightnormal;
|
||||
|
||||
Reference in new issue
Block a user