2xbr: copy runs of flat pixels a vector at a time

Most of a game frame is pixels 2xBR only copies: a pixel equal to
enough of its four direct neighbours that none of the corner rules can
fire.  The flat-block test already skipped the rest of their
neighbourhood, but still cost five loads, four comparisons and four
scalar stores a pixel, about 6 cycles.  Now that test runs on 4 or 8
pixels at once (SSE2, NEON; 8 or 16 with AVX2), every pixel of the
vector is written doubled on both output rows, and the full rule runs
on just the pixels that failed it, whose own output then replaces the
copy: the same result the per-pixel path gives.

On detail, where no pixel in a vector is copied, the vector work would
be wasted, so after such a vector the next 1, 3, 7 and up to 63 vectors'
worth go one by one, carried from row to row.  The full rule is forced
inline in this path; as a call, the loop around it cost detailed frames
several percent.  Builds with no vector tier keep the previous per-pixel
loop exactly, as the reworked one is slower there.

SSE2 is used where the build has it (all x86-64), NEON on ARMv8 and ARM
builds with NEON, and AVX2 is picked at create from the SIMD mask, gated
like the other filters' tiers.

Output is unchanged byte for byte.  On one thread at 256x224, under GCC
with SSE2 a game frame takes 0.71 (RGB565) and 0.80 (XRGB8888) of the
time it did and a flat one 0.35 and 0.56; with AVX2 0.74, 0.78, 0.33
and 0.49.  Noise and smooth gradients, where nearly nothing is copied,
take the same time as before (within 2%), under Clang too.

Tests: 2xBR moves into samples/gfx/filter_golden, recorded from the
code before this change, where it is also run with the host's SIMD
mask and built without its vector code; samples/gfx/xbr_filter_golden,
which only did the first, is retired.  SOFTFILTER_NO_SIMD now leaves out
the vector code of every filter that has any (2xBR, De-Dither, and SNES
NTSC as well as its own SNES_NTSC_NO_SIMD), and the harness builds each
of them that way too.  NEON was run under qemu on aarch64 and armhf.

.github/workflows/Linux-filter-plugins.yml

@@ -11,10 +11,9 @@ name: CI Linux filter plugins
 # with the plugins and the harness built under ASan, UBSan and
 # LeakSanitizer, and under ThreadSanitizer.  The NTSC, 2xBR, NTSC-CRT,
 # 2xSaI-family, Phosphor2x, Blargg NTSC SNES and De-Dither filters are
-# also held
-# byte for byte to output hashes recorded from their earlier
+# also held byte for byte to output hashes recorded from their earlier
 # implementations
-# (samples/gfx/{ntsc,xbr,crt}_filter_golden and samples/gfx/filter_golden).
+# (samples/gfx/{ntsc,crt}_filter_golden and samples/gfx/filter_golden).
 
 on:
   push:
@@ -26,7 +25,6 @@ on:
       - 'libretro-common/include/libretro_dspfilter.h'
       - 'samples/gfx/filter_plugins/**'
       - 'samples/gfx/ntsc_filter_golden/**'
-      - 'samples/gfx/xbr_filter_golden/**'
       - 'samples/gfx/crt_filter_golden/**'
       - 'samples/gfx/filter_golden/**'
       - '.github/workflows/Linux-filter-plugins.yml'
@@ -37,7 +35,6 @@ on:
       - 'libretro-common/include/libretro_dspfilter.h'
       - 'samples/gfx/filter_plugins/**'
       - 'samples/gfx/ntsc_filter_golden/**'
-      - 'samples/gfx/xbr_filter_golden/**'
       - 'samples/gfx/crt_filter_golden/**'
       - 'samples/gfx/filter_golden/**'
       - '.github/workflows/Linux-filter-plugins.yml'
@@ -92,8 +89,7 @@ jobs:
           UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
         run: |
           set -eu
-          for t in ntsc_filter_golden xbr_filter_golden crt_filter_golden \
-                   filter_golden; do
+          for t in ntsc_filter_golden crt_filter_golden filter_golden; do
              make -C samples/gfx/$t check
              make -C samples/gfx/$t clean
              make -C samples/gfx/$t check SANITIZER=address,undefined

gfx/video_filters/2xbr.c

@@ -38,6 +38,7 @@
 #include <string.h>
 #include <math.h>
 #include <retro_endianness.h>
+#include <retro_inline.h>
 #include <boolean.h>
 
 #ifdef RARCH_INTERNAL
@@ -48,6 +49,74 @@
 
 #define TWOXBR_SCALE 2
 
+/* The full rule is inlined where it is called: as a call, the loop
+ * around it costs detailed frames several percent. */
+#if defined(_MSC_VER)
+#define TWOXBR_FORCE_INLINE __forceinline
+#elif defined(__GNUC__) || defined(__clang__)
+#define TWOXBR_FORCE_INLINE INLINE __attribute__((always_inline))
+#else
+#define TWOXBR_FORCE_INLINE INLINE
+#endif
+
+/* Vector tiers for runs of pixels that are only copied (see TWOXBR_ROWS):
+ * SSE2 where the build has it (all x86-64), AVX2 picked at create from
+ * the SIMD mask, NEON on ARMv8 and ARM builds with NEON. */
+#if !defined(SOFTFILTER_NO_SIMD) && (defined(__SSE2__) || defined(_M_X64) \
+   || (defined(_M_IX86_FP) && _M_IX86_FP >= 2))
+#define TWOXBR_HAVE_SSE2 1
+#include <emmintrin.h>
+
+#if defined(_MSC_VER) && !defined(__clang__)
+#if _MSC_VER >= 1911
+#define TWOXBR_HAVE_AVX2 1
+#define TWOXBR_TARGET_AVX2
+#endif
+#elif defined(__clang__) && (defined(__SCE__) || defined(_MSC_VER))
+/* PlayStation clang and clang-cl declare only the intrinsics of ISAs
+ * the whole build enables, so AVX2 is built only where the baseline
+ * has it. */
+#ifdef __AVX2__
+#define TWOXBR_HAVE_AVX2 1
+#define TWOXBR_TARGET_AVX2
+#endif
+#elif (defined(__clang__) && __clang_major__ >= 4) \
+   || (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 5)
+#define TWOXBR_HAVE_AVX2 1
+#define TWOXBR_TARGET_AVX2 __attribute__((target("avx2")))
+#endif
+
+#ifdef TWOXBR_HAVE_AVX2
+#include <immintrin.h>
+#if defined(_MSC_VER) && !defined(__clang__)
+#define TWOXBR_ZEROUPPER() _mm256_zeroupper()
+#else
+#define TWOXBR_ZEROUPPER()
+#endif
+#endif
+#elif !defined(SOFTFILTER_NO_SIMD) && (defined(__ARM_NEON) \
+   || defined(__ARM_NEON__) || defined(_M_ARM64))
+#define TWOXBR_HAVE_NEON 1
+#include <arm_neon.h>
+#endif
+
+#if defined(TWOXBR_HAVE_SSE2) || defined(TWOXBR_HAVE_NEON)
+#define TWOXBR_VECTOR 1
+#endif
+
+/* One vector of pixels from x, which must end before the row's last
+ * pixel: writes every pixel twice across on both output rows, as if it
+ * were copied out, and returns a bit per pixel (lowest first) for those
+ * that are not, which the caller then runs the full rule on. rm1, r0 and
+ * rp1 are the rows above, of and below the pixels; dst the first of the
+ * two output rows. */
+typedef unsigned (*twoxbr_run_rgb565_t)(const uint16_t *rm1,
+      const uint16_t *r0, const uint16_t *rp1, uint16_t *dst,
+      size_t dst_stride, unsigned x);
+typedef unsigned (*twoxbr_run_xrgb8888_t)(const uint32_t *rm1,
+      const uint32_t *r0, const uint32_t *rp1, uint32_t *dst,
+      size_t dst_stride, unsigned x);
+
 struct softfilter_thread_data
 {
    void *out_data;
@@ -79,6 +148,11 @@ struct filter_data
     * share of one that size. */
    unsigned max_width;
    unsigned max_height;
+   /* The vector runs for this machine, or NULL, and their widths */
+   twoxbr_run_rgb565_t run_rgb565;
+   twoxbr_run_xrgb8888_t run_xrgb8888;
+   unsigned lanes_rgb565;
+   unsigned lanes_xrgb8888;
    uint16_t RGBtoYUV[65536];
    uint16_t tbl_5_to_8[32];
    uint16_t tbl_6_to_8[64];
@@ -237,6 +311,166 @@ static void SetupFormat(void * data)
    }
 }
 
+/* A pixel is copied out when it equals enough of its four direct
+ * neighbours that none of the four corner rules can fire (each needs PE
+ * to differ from both pixels beside that corner):
+ *    (PE == PH || PE == PF) && (PE == PF || PE == PB)
+ * && (PE == PB || PE == PD) && (PE == PD || PE == PH)
+ * The runs test that for a vector of pixels. */
+
+#ifdef TWOXBR_HAVE_SSE2
+#define TWOXBR_COPIED_SSE2(EQ, PE, PB, PD, PF, PH) _mm_and_si128( \
+      _mm_and_si128(_mm_or_si128(EQ(PE, PH), EQ(PE, PF)), \
+                    _mm_or_si128(EQ(PE, PF), EQ(PE, PB))), \
+      _mm_and_si128(_mm_or_si128(EQ(PE, PB), EQ(PE, PD)), \
+                    _mm_or_si128(EQ(PE, PD), EQ(PE, PH))))
+#define TWOXBR_LD(p) _mm_loadu_si128((const __m128i*)(p))
+
+static unsigned twoxbr_run_rgb565_sse2(const uint16_t *rm1,
+      const uint16_t *r0, const uint16_t *rp1, uint16_t *dst,
+      size_t dst_stride, unsigned x)
+{
+   __m128i pe = TWOXBR_LD(r0 + x);
+   __m128i ok = TWOXBR_COPIED_SSE2(_mm_cmpeq_epi16, pe,
+         TWOXBR_LD(rm1 + x), TWOXBR_LD(r0 + x - 1),
+         TWOXBR_LD(r0 + x + 1), TWOXBR_LD(rp1 + x));
+   __m128i lo = _mm_unpacklo_epi16(pe, pe);
+   __m128i hi = _mm_unpackhi_epi16(pe, pe);
+   _mm_storeu_si128((__m128i*)(dst + 2 * x),     lo);
+   _mm_storeu_si128((__m128i*)(dst + 2 * x + 8), hi);
+   _mm_storeu_si128((__m128i*)(dst + dst_stride + 2 * x),     lo);
+   _mm_storeu_si128((__m128i*)(dst + dst_stride + 2 * x + 8), hi);
+   return ~(unsigned)_mm_movemask_epi8(
+         _mm_packs_epi16(ok, _mm_setzero_si128())) & 0xFFu;
+}
+
+static unsigned twoxbr_run_xrgb8888_sse2(const uint32_t *rm1,
+      const uint32_t *r0, const uint32_t *rp1, uint32_t *dst,
+      size_t dst_stride, unsigned x)
+{
+   __m128i pe = TWOXBR_LD(r0 + x);
+   __m128i ok = TWOXBR_COPIED_SSE2(_mm_cmpeq_epi32, pe,
+         TWOXBR_LD(rm1 + x), TWOXBR_LD(r0 + x - 1),
+         TWOXBR_LD(r0 + x + 1), TWOXBR_LD(rp1 + x));
+   __m128i lo = _mm_unpacklo_epi32(pe, pe);
+   __m128i hi = _mm_unpackhi_epi32(pe, pe);
+   _mm_storeu_si128((__m128i*)(dst + 2 * x),     lo);
+   _mm_storeu_si128((__m128i*)(dst + 2 * x + 4), hi);
+   _mm_storeu_si128((__m128i*)(dst + dst_stride + 2 * x),     lo);
+   _mm_storeu_si128((__m128i*)(dst + dst_stride + 2 * x + 4), hi);
+   return ~(unsigned)_mm_movemask_ps(_mm_castsi128_ps(ok)) & 0xFu;
+}
+#endif /* TWOXBR_HAVE_SSE2 */
+
+#ifdef TWOXBR_HAVE_AVX2
+#define TWOXBR_COPIED_AVX2(EQ, PE, PB, PD, PF, PH) _mm256_and_si256( \
+      _mm256_and_si256(_mm256_or_si256(EQ(PE, PH), EQ(PE, PF)), \
+                       _mm256_or_si256(EQ(PE, PF), EQ(PE, PB))), \
+      _mm256_and_si256(_mm256_or_si256(EQ(PE, PB), EQ(PE, PD)), \
+                       _mm256_or_si256(EQ(PE, PD), EQ(PE, PH))))
+#define TWOXBR_LD_256(p) _mm256_loadu_si256((const __m256i*)(p))
+
+/* unpacklo/unpackhi double each 128-bit half; the permutes put the
+ * doubled pixels back in order. */
+#define TWOXBR_STORE_DOUBLED_256(dst, x, lo, hi, n) { \
+      __m256i a_ = _mm256_permute2x128_si256(lo, hi, 0x20); \
+      __m256i b_ = _mm256_permute2x128_si256(lo, hi, 0x31); \
+      _mm256_storeu_si256((__m256i*)((dst) + 2 * (x)),     a_); \
+      _mm256_storeu_si256((__m256i*)((dst) + 2 * (x) + n), b_); \
+      _mm256_storeu_si256((__m256i*)((dst) + dst_stride + 2 * (x)),     a_); \
+      _mm256_storeu_si256((__m256i*)((dst) + dst_stride + 2 * (x) + n), b_); \
+}
+
+TWOXBR_TARGET_AVX2
+static unsigned twoxbr_run_rgb565_avx2(const uint16_t *rm1,
+      const uint16_t *r0, const uint16_t *rp1, uint16_t *dst,
+      size_t dst_stride, unsigned x)
+{
+   __m256i pe = TWOXBR_LD_256(r0 + x);
+   __m256i ok = TWOXBR_COPIED_AVX2(_mm256_cmpeq_epi16, pe,
+         TWOXBR_LD_256(rm1 + x), TWOXBR_LD_256(r0 + x - 1),
+         TWOXBR_LD_256(r0 + x + 1), TWOXBR_LD_256(rp1 + x));
+   __m256i lo = _mm256_unpacklo_epi16(pe, pe);
+   __m256i hi = _mm256_unpackhi_epi16(pe, pe);
+   unsigned m;
+   TWOXBR_STORE_DOUBLED_256(dst, x, lo, hi, 16)
+   m = ~(unsigned)_mm_movemask_epi8(_mm_packs_epi16(
+            _mm256_castsi256_si128(ok), _mm256_extracti128_si256(ok, 1)))
+      & 0xFFFFu;
+   TWOXBR_ZEROUPPER();
+   return m;
+}
+
+TWOXBR_TARGET_AVX2
+static unsigned twoxbr_run_xrgb8888_avx2(const uint32_t *rm1,
+      const uint32_t *r0, const uint32_t *rp1, uint32_t *dst,
+      size_t dst_stride, unsigned x)
+{
+   __m256i pe = TWOXBR_LD_256(r0 + x);
+   __m256i ok = TWOXBR_COPIED_AVX2(_mm256_cmpeq_epi32, pe,
+         TWOXBR_LD_256(rm1 + x), TWOXBR_LD_256(r0 + x - 1),
+         TWOXBR_LD_256(r0 + x + 1), TWOXBR_LD_256(rp1 + x));
+   __m256i lo = _mm256_unpacklo_epi32(pe, pe);
+   __m256i hi = _mm256_unpackhi_epi32(pe, pe);
+   unsigned m;
+   TWOXBR_STORE_DOUBLED_256(dst, x, lo, hi, 8)
+   m = ~(unsigned)_mm256_movemask_ps(_mm256_castsi256_ps(ok)) & 0xFFu;
+   TWOXBR_ZEROUPPER();
+   return m;
+}
+#endif /* TWOXBR_HAVE_AVX2 */
+
+#ifdef TWOXBR_HAVE_NEON
+#define TWOXBR_COPIED_NEON(AND, OR, EQ, PE, PB, PD, PF, PH) AND( \
+      AND(OR(EQ(PE, PH), EQ(PE, PF)), OR(EQ(PE, PF), EQ(PE, PB))), \
+      AND(OR(EQ(PE, PB), EQ(PE, PD)), OR(EQ(PE, PD), EQ(PE, PH))))
+
+/* A bit per byte of an all-ones-or-zero mask of 8 bytes, set where zero */
+static INLINE unsigned twoxbr_zero_bytes_neon(uint8x8_t m)
+{
+   uint64_t v = vget_lane_u64(vreinterpret_u64_u8(m), 0);
+   unsigned bits = 0, j;
+   for (j = 0; j < 8; j++)
+      if (!((v >> (8 * j)) & 0xFF))
+         bits |= 1u << j;
+   return bits;
+}
+
+static unsigned twoxbr_run_rgb565_neon(const uint16_t *rm1,
+      const uint16_t *r0, const uint16_t *rp1, uint16_t *dst,
+      size_t dst_stride, unsigned x)
+{
+   uint16x8_t pe = vld1q_u16(r0 + x);
+   uint16x8_t ok = TWOXBR_COPIED_NEON(vandq_u16, vorrq_u16, vceqq_u16, pe,
+         vld1q_u16(rm1 + x), vld1q_u16(r0 + x - 1),
+         vld1q_u16(r0 + x + 1), vld1q_u16(rp1 + x));
+   uint16x8x2_t d = vzipq_u16(pe, pe);
+   vst1q_u16(dst + 2 * x,     d.val[0]);
+   vst1q_u16(dst + 2 * x + 8, d.val[1]);
+   vst1q_u16(dst + dst_stride + 2 * x,     d.val[0]);
+   vst1q_u16(dst + dst_stride + 2 * x + 8, d.val[1]);
+   return twoxbr_zero_bytes_neon(vmovn_u16(ok));
+}
+
+static unsigned twoxbr_run_xrgb8888_neon(const uint32_t *rm1,
+      const uint32_t *r0, const uint32_t *rp1, uint32_t *dst,
+      size_t dst_stride, unsigned x)
+{
+   uint32x4_t pe = vld1q_u32(r0 + x);
+   uint32x4_t ok = TWOXBR_COPIED_NEON(vandq_u32, vorrq_u32, vceqq_u32, pe,
+         vld1q_u32(rm1 + x), vld1q_u32(r0 + x - 1),
+         vld1q_u32(r0 + x + 1), vld1q_u32(rp1 + x));
+   uint32x4x2_t d = vzipq_u32(pe, pe);
+   vst1q_u32(dst + 2 * x,     d.val[0]);
+   vst1q_u32(dst + 2 * x + 4, d.val[1]);
+   vst1q_u32(dst + dst_stride + 2 * x,     d.val[0]);
+   vst1q_u32(dst + dst_stride + 2 * x + 4, d.val[1]);
+   /* four 32-bit lanes narrowed twice to bytes; the upper four are 0 */
+   return twoxbr_zero_bytes_neon(vmovn_u16(vcombine_u16(vmovn_u32(ok),
+               vdup_n_u16(0xFFFF)))) & 0xFu;
+}
+#endif /* TWOXBR_HAVE_NEON */
+
 static void twoxbr_generic_destroy(void *data);
 
 static void *twoxbr_generic_create(const struct softfilter_config *config,
@@ -280,6 +514,27 @@ static void *twoxbr_generic_create(const struct softfilter_config *config,
 
    SetupFormat(filt);
 
+#if defined(TWOXBR_HAVE_SSE2)
+   filt->run_rgb565     = twoxbr_run_rgb565_sse2;
+   filt->run_xrgb8888   = twoxbr_run_xrgb8888_sse2;
+   filt->lanes_rgb565   = 8;
+   filt->lanes_xrgb8888 = 4;
+#ifdef TWOXBR_HAVE_AVX2
+   if (simd & SOFTFILTER_SIMD_AVX2)
+   {
+      filt->run_rgb565     = twoxbr_run_rgb565_avx2;
+      filt->run_xrgb8888   = twoxbr_run_xrgb8888_avx2;
+      filt->lanes_rgb565   = 16;
+      filt->lanes_xrgb8888 = 8;
+   }
+#endif
+#elif defined(TWOXBR_HAVE_NEON)
+   filt->run_rgb565     = twoxbr_run_rgb565_neon;
+   filt->run_xrgb8888   = twoxbr_run_xrgb8888_neon;
+   filt->lanes_rgb565   = 8;
+   filt->lanes_xrgb8888 = 4;
+#endif
+
    return filt;
 }
 
@@ -565,13 +820,146 @@ static void twoxbr_generic_destroy(void *data)
       } \
    }
 
+#ifdef TWOXBR_VECTOR
+/* The full rule for pixel X of the row: its 3x3 pixels through clamped
+ * indices, the rest of its 5x5 neighbourhood through the padded keys
+ * (km2 the keys two rows up, the others following at width + 4). One
+ * function per format, for both the vector and the one-by-one paths. */
+#define TWOXBR_FULL_FN(NAME, TYPE, FILTRO, MASKS) \
+static TWOXBR_FORCE_INLINE void NAME(const struct filter_data *filt, \
+      const TYPE *rm1, const TYPE *r0, const TYPE *rp1, \
+      const uint16_t *km2, TYPE *dst, unsigned dst_stride, \
+      unsigned width, unsigned xx_) \
+{ \
+   const uint16_t *km1 = km2 + (width + 4); \
+   const uint16_t *k0  = km1 + (width + 4); \
+   const uint16_t *kp1 = k0  + (width + 4); \
+   const uint16_t *kp2 = kp1 + (width + 4); \
+   TYPE *out      = dst + 2 * xx_; \
+   TYPE E[4]; \
+   TYPE ex, e, i, ke, ki, ex2, ex3, px; \
+   unsigned xm1   = xx_ ? xx_ - 1 : 0; \
+   unsigned xp1   = xx_ + 1 < width ? xx_ + 1 : xx_; \
+   int xi         = (int)xx_; \
+   TYPE PB        = rm1[xx_]; \
+   TYPE PD        = r0[xm1]; \
+   TYPE PE        = r0[xx_]; \
+   TYPE PF        = r0[xp1]; \
+   TYPE PH        = rp1[xx_]; \
+   TYPE PA        = rm1[xm1]; \
+   TYPE PC        = rm1[xp1]; \
+   TYPE PG        = rp1[xm1]; \
+   TYPE _PI       = rp1[xp1]; \
+   uint16_t K_A1 = km2[xi - 1], K_B1 = km2[xi],   K_C1 = km2[xi + 1]; \
+   uint16_t K_A0 = km1[xi - 2], K_PA = km1[xi - 1], K_PB = km1[xi]; \
+   uint16_t K_PC = km1[xi + 1], K_C4 = km1[xi + 2]; \
+   uint16_t K_D0 = k0[xi - 2],  K_PD = k0[xi - 1],  K_PE = k0[xi]; \
+   uint16_t K_PF = k0[xi + 1],  K_F4 = k0[xi + 2]; \
+   uint16_t K_G0 = kp1[xi - 2], K_PG = kp1[xi - 1], K_PH = kp1[xi]; \
+   uint16_t K__PI = kp1[xi + 1], K_I4 = kp1[xi + 2]; \
+   uint16_t K_G5 = kp2[xi - 1], K_H5 = kp2[xi],   K_I5 = kp2[xi + 1]; \
+   MASKS \
+   /* \
+    * Map of the pixels:          A1 B1 C1 \
+    *                          A0 PA PB PC C4 \
+    *                          D0 PD PE PF F4 \
+    *                          G0 PG PH _PI I4 \
+    *                             G5 H5 I5 \
+    */ \
+   twoxbr_function(FILTRO, filt); \
+}
+
+#define TWOXBR_FULL(FULL_FN, X) \
+   FULL_FN(filt, rm1, r0, rp1, km2, dst, dst_stride, width, (X));
+
+/* Pixel X, copied out when it equals enough of its four direct
+ * neighbours that none of the four corner rules can fire (each needs
+ * PE to differ from both pixels beside that corner), without loading
+ * the rest of its neighbourhood; the full rule otherwise. */
+#define TWOXBR_PIXEL(TYPE, FULL_FN, X) \
+   { \
+      unsigned xp_ = (X); \
+      unsigned l_  = xp_ ? xp_ - 1 : 0; \
+      unsigned r_  = xp_ + 1 < width ? xp_ + 1 : xp_; \
+      TYPE PB_     = rm1[xp_]; \
+      TYPE PD_     = r0[l_]; \
+      TYPE PE_     = r0[xp_]; \
+      TYPE PF_     = r0[r_]; \
+      TYPE PH_     = rp1[xp_]; \
+      if (     (PE_ == PH_ || PE_ == PF_) && (PE_ == PF_ || PE_ == PB_) \
+            && (PE_ == PB_ || PE_ == PD_) && (PE_ == PD_ || PE_ == PH_)) \
+      { \
+         TYPE *o_ = dst + 2 * xp_; \
+         o_[0] = o_[1] = o_[dst_stride] = o_[dst_stride + 1] = PE_; \
+      } \
+      else \
+         TWOXBR_FULL(FULL_FN, xp_) \
+   }
+
 /* One worker's rows. The 5x5 neighbourhood is clamped to the frame:
+ * the keys through the padded rows above, the 3x3 pixels through
+ * clamped indices. Where there is a vector run (RUN, LANES pixels at a
+ * time), it writes a vector of pixels as copied and the full rule then
+ * runs on those of them that are not, which is what TWOXBR_PIXEL does
+ * one by one; the first pixel and those too near the end of the row for
+ * a vector go one by one. After a vector in which no pixel was copied,
+ * the next 1, 3, 7 and up to 63 vectors' worth go one by one, counting
+ * on from row to row, so detail (where the vector would be wasted work)
+ * costs no more than it did. */
+#define TWOXBR_ROWS(TYPE, FILTRO, FULL_FN, RUN, LANES) \
+   { unsigned misses_ = 0; \
+   for (y = 0; y < height; y++) \
+   { \
+      unsigned abs_y   = (unsigned)first + y; \
+      int up1          = abs_y >= 1 ? -1 : 0; \
+      int dn1          = abs_y + 1 < frame_height ? 1 : 0; \
+      const TYPE *rm1  = src + up1 * (int)src_stride; \
+      const TYPE *r0   = src; \
+      const TYPE *rp1  = src + dn1 * (int)src_stride; \
+      const uint16_t *km2 = keys + (size_t)(y + 0) * (width + 4) + 2; \
+      { \
+         /* where the next vector is tried: never without one, and \
+          * not on the first pixel (its left neighbour is clamped) */ \
+         unsigned try_    = RUN ? 1 + (LANES) * ((1u << misses_) - 1) : width; \
+         x = 0; \
+         for (;;) \
+         { \
+            unsigned m_, j_; \
+            unsigned seg_ = try_ < width ? try_ : width; \
+            for (; x < seg_; x++) \
+               TWOXBR_PIXEL(TYPE, FULL_FN, x) \
+            if (x >= width) \
+               break; \
+            if (x + (LANES) >= width) \
+            { \
+               try_ = width; /* too near the end for a vector */ \
+               continue; \
+            } \
+            m_ = RUN(rm1, r0, rp1, dst, dst_stride, x); \
+            misses_ = m_ != ((1u << (LANES)) - 1) ? 0 \
+               : misses_ < 6 ? misses_ + 1 : 6; \
+            for (j_ = x; m_; m_ >>= 1, j_++) \
+               if (m_ & 1) \
+                  TWOXBR_FULL(FULL_FN, j_) \
+            x   += (LANES); \
+            /* after vectors that copied nothing, the next 1, 3, 7 \
+             * and up to 63 vectors' worth go one by one */ \
+            try_ = x + (LANES) * ((1u << misses_) - 1); \
+         } \
+      } \
+      src += src_stride; \
+      dst += 2 * dst_stride; \
+   } }
+
+#else
+/* One worker's rows, without vector runs (builds with no vector tier).
+ * The 5x5 neighbourhood is clamped to the frame:
  * the keys through the padded rows above, the 3x3 pixels through
  * clamped indices. A pixel equal to enough of its four direct
  * neighbours that none of the four corner rules can fire (each needs
  * PE to differ from both pixels beside that corner) is copied out
  * without loading the rest of its neighbourhood. */
-#define TWOXBR_ROWS(TYPE, FILTRO) \
+#define TWOXBR_ROWS_PLAIN(TYPE, FILTRO) \
    for (y = 0; y < height; y++) \
    { \
       unsigned abs_y   = (unsigned)first + y; \
@@ -632,25 +1020,49 @@ static void twoxbr_generic_destroy(void *data)
       dst += 2 * dst_stride; \
    }
 
+#endif
+
 #define TWOXBR_KEY_565(p)  filt->RGBtoYUV[p]
 #define TWOXBR_KEY_8888(p) filt->RGBtoYUV[XRGB8888_TO_RGB565(p)]
 
+#ifdef TWOXBR_VECTOR
+TWOXBR_FULL_FN(twoxbr_full_xrgb8888, uint32_t, FILTRO_RGB8888,
+      uint32_t pg_red_mask   = RED_MASK8888;
+      uint32_t pg_green_mask = GREEN_MASK8888;
+      uint32_t pg_blue_mask  = BLUE_MASK8888;
+      uint32_t pg_lbmask     = PG_LBMASK8888;
+      uint32_t pg_alpha_mask = ALPHA_MASK8888;
+      (void)pg_alpha_mask;)
+
+TWOXBR_FULL_FN(twoxbr_full_rgb565, uint16_t, FILTRO_RGB565,
+      uint16_t pg_red_mask   = RED_MASK565;
+      uint16_t pg_green_mask = GREEN_MASK565;
+      uint16_t pg_blue_mask  = BLUE_MASK565;
+      uint16_t pg_lbmask     = PG_LBMASK565;)
+#endif
+
 static void twoxbr_generic_xrgb8888(void *data, uint16_t *keys,
       unsigned width, unsigned height, int first, const uint32_t *src,
       unsigned src_stride, uint32_t *dst, unsigned dst_stride)
 {
    unsigned x, y;
+   struct filter_data *filt  = (struct filter_data*)data;
+   unsigned frame_height     = filt->frame_height;
+#ifdef TWOXBR_VECTOR
+   TWOXBR_FILL_KEYS(uint32_t, TWOXBR_KEY_8888)
+   TWOXBR_ROWS(uint32_t, FILTRO_RGB8888, twoxbr_full_xrgb8888,
+         filt->run_xrgb8888, filt->lanes_xrgb8888)
+#else
    uint32_t pg_red_mask      = RED_MASK8888;
    uint32_t pg_green_mask    = GREEN_MASK8888;
    uint32_t pg_blue_mask     = BLUE_MASK8888;
    uint32_t pg_lbmask        = PG_LBMASK8888;
    uint32_t pg_alpha_mask    = ALPHA_MASK8888;
-   struct filter_data *filt  = (struct filter_data*)data;
-   unsigned frame_height     = filt->frame_height;
 
    TWOXBR_FILL_KEYS(uint32_t, TWOXBR_KEY_8888)
-   TWOXBR_ROWS(uint32_t, FILTRO_RGB8888)
+   TWOXBR_ROWS_PLAIN(uint32_t, FILTRO_RGB8888)
    (void)pg_alpha_mask;
+#endif
 }
 
 static void twoxbr_generic_rgb565(void *data, uint16_t *keys,
@@ -659,14 +1071,20 @@ static void twoxbr_generic_rgb565(void *data, uint16_t *keys,
 {
    unsigned x, y;
    struct filter_data *filt = (struct filter_data*)data;
+   unsigned frame_height    = filt->frame_height;
+#ifdef TWOXBR_VECTOR
+   TWOXBR_FILL_KEYS(uint16_t, TWOXBR_KEY_565)
+   TWOXBR_ROWS(uint16_t, FILTRO_RGB565, twoxbr_full_rgb565,
+         filt->run_rgb565, filt->lanes_rgb565)
+#else
    uint16_t pg_red_mask     = RED_MASK565;
    uint16_t pg_green_mask   = GREEN_MASK565;
    uint16_t pg_blue_mask    = BLUE_MASK565;
    uint16_t pg_lbmask       = PG_LBMASK565;
-   unsigned frame_height    = filt->frame_height;
 
    TWOXBR_FILL_KEYS(uint16_t, TWOXBR_KEY_565)
-   TWOXBR_ROWS(uint16_t, FILTRO_RGB565)
+   TWOXBR_ROWS_PLAIN(uint16_t, FILTRO_RGB565)
+#endif
 }
 
 /* Plain 2x pixel doubling, for a frame whose keys could not be

gfx/video_filters/dedither.c

@@ -13,9 +13,10 @@
 /* Vector tiers: SSE2 where the build has it (all x86-64), AVX2 picked at
  * create from the SIMD mask, NEON on ARMv8 and ARM builds with NEON.
  * Each works out every test of a run of pixels at once, branch-free, and
- * gives the plain code's output to the bit. */
-#if defined(__SSE2__) || defined(_M_X64) \
-   || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)
+ * gives the plain code's output to the bit. SOFTFILTER_NO_SIMD leaves
+ * them out. */
+#if !defined(SOFTFILTER_NO_SIMD) && (defined(__SSE2__) || defined(_M_X64) \
+   || (defined(_M_IX86_FP) && _M_IX86_FP >= 2))
 #define DEDITHER_HAVE_SSE2 1
 #include <emmintrin.h>
 
@@ -46,7 +47,8 @@
 #define DEDITHER_ZEROUPPER()
 #endif
 #endif
-#elif defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64)
+#elif !defined(SOFTFILTER_NO_SIMD) && (defined(__ARM_NEON) \
+   || defined(__ARM_NEON__) || defined(_M_ARM64))
 #define DEDITHER_HAVE_NEON 1
 #include <arm_neon.h>
 #endif

gfx/video_filters/snes_ntsc/snes_ntsc.h

@@ -155,7 +155,8 @@ typedef unsigned long snes_ntsc_rgb_t;
  * one vector lane per entry, and half the table where unsigned long is
  * 8 bytes. Elsewhere entries stay snes_ntsc_rgb_t for the scalar
  * blitters. */
-#if !defined(SNES_NTSC_NO_SIMD) && SNES_NTSC_OUT_DEPTH == 16 && \
+#if !defined(SNES_NTSC_NO_SIMD) && !defined(SOFTFILTER_NO_SIMD) && \
+	SNES_NTSC_OUT_DEPTH == 16 && \
 	(defined(__SSE2__) || defined(_M_X64) || \
 	 (defined(_M_IX86_FP) && _M_IX86_FP >= 2) || \
 	 defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM64))

samples/gfx/filter_golden/Makefile

@@ -1,14 +1,15 @@
 # Each listed video filter against its recorded output hashes; see
 # filter_golden_test.c.
-FILTERS := 2xsai super2xsai supereagle phosphor2x blargg_ntsc_snes dedither
+FILTERS := 2xsai super2xsai supereagle phosphor2x blargg_ntsc_snes dedither 2xbr
 # Blargg NTSC SNES again with each other tvtype, which builds a
 # different table (RF also leaves the fields unmerged).
 SNES_TV := rf rgb svideo
-# and with its vector blitters left out, so the plain C ones are held
-# to the same table on machines that have the vector ones.
+# The filters with vector code, which also get a build without it.
+SIMD_FILTERS := blargg_ntsc_snes dedither 2xbr
+C_TARGETS := $(addsuffix _c,$(addprefix golden_,$(SIMD_FILTERS)))
 TARGETS := $(addprefix golden_,$(FILTERS)) \
 	$(addprefix golden_blargg_ntsc_snes_,$(SNES_TV)) \
-	golden_blargg_ntsc_snes_c
+	$(C_TARGETS)
 SNES_NTSC := $(wildcard ../../../gfx/video_filters/snes_ntsc/*)
 
 CFLAGS  += -Wall -std=gnu99 -O2 -I../../../libretro-common/include \
@@ -37,12 +38,13 @@ golden_blargg_ntsc_snes_%: filter_golden_test.c golden_blargg_ntsc_snes_%.h \
 		-DFILTER_NAME='"golden_blargg_ntsc_snes_$*"' \
 		-o $@ filter_golden_test.c $(LDFLAGS)
 
-golden_blargg_ntsc_snes_c: filter_golden_test.c golden_blargg_ntsc_snes.h \
-		../../../gfx/video_filters/blargg_ntsc_snes.c $(SNES_NTSC)
-	$(CC) $(CFLAGS) -DSNES_NTSC_NO_SIMD \
-		-DFILTER_C='"../../../gfx/video_filters/blargg_ntsc_snes.c"' \
-		-DGOLDEN_H='"golden_blargg_ntsc_snes.h"' \
-		-DFILTER_NAME='"golden_blargg_ntsc_snes_c"' \
+# Each filter with vector code, built with it left out and held to the
+# same table.
+$(C_TARGETS): golden_%_c: filter_golden_test.c golden_%.h \
+		../../../gfx/video_filters/%.c $(SNES_NTSC)
+	$(CC) $(CFLAGS) -DSOFTFILTER_NO_SIMD \
+		-DFILTER_C='"../../../gfx/video_filters/$*.c"' \
+		-DGOLDEN_H='"golden_$*.h"' -DFILTER_NAME='"golden_$*_c"' \
 		-o $@ filter_golden_test.c $(LDFLAGS)
 
 check: $(TARGETS)

samples/gfx/filter_golden/golden_2xbr.h

@@ -1,2 +1,2 @@
-{ { 0x017a76f5u, 0xaa27fb35u, 0x92e54655u, 0x87407fe5u, 0xec773125u, 0x1b4bf435u, 0x9c171ae5u, 0xb579356du, 0x554218d5u, 0xe43244adu }, { 0x0b603d85u, 0xd4dca045u, 0x09c0f4a5u, 0x42db0625u, 0x9249026du, 0x0af8b2bdu, 0xc224b035u, 0x4809e7e5u, 0xfc615345u, 0xf0cbfd45u } },
+{ { 0x017a76f5u, 0xaa27fb35u, 0x92e54655u, 0x87407fe5u, 0xec773125u, 0x1b4bf435u, 0x69100b2du, 0x726ac75du, 0xa64b8b85u, 0x1dd6e1fdu }, { 0x0b603d85u, 0xd4dca045u, 0x09c0f4a5u, 0x42db0625u, 0x9249026du, 0x0af8b2bdu, 0x3c263f45u, 0xd4f7d1e5u, 0x24f2cd85u, 0xeaef8ab5u } },
 { { 0xc54b1af5u, 0x2537f175u, 0xefe1d595u, 0x656ca7a5u, 0xc6958e4du, 0xd22678e5u, 0xdb326a45u, 0x66e75cadu, 0x33602dc5u, 0xb08b596du }, { 0xfa37dce5u, 0x650e7f65u, 0xc7720865u, 0x251502c5u, 0xa6374815u, 0x1dca3a9du, 0xe2b5df45u, 0x3a9ccb4du, 0x74a81ee5u, 0x4a09e7e5u } },

samples/gfx/xbr_filter_golden/Makefile

@@ -1,23 +0,0 @@
-TARGET := xbr_filter_golden_test
-
-CFLAGS  += -Wall -std=gnu99 -O2 -I../../../libretro-common/include \
-	-I../../../gfx/video_filters
-LDFLAGS += -lm
-
-ifneq ($(SANITIZER),)
-   CFLAGS  := -fsanitize=$(SANITIZER) -fno-omit-frame-pointer $(CFLAGS)
-   LDFLAGS := -fsanitize=$(SANITIZER) $(LDFLAGS)
-endif
-
-all: $(TARGET)
-
-$(TARGET): xbr_filter_golden_test.c golden.h ../../../gfx/video_filters/2xbr.c
-	$(CC) $(CFLAGS) -o $@ xbr_filter_golden_test.c $(LDFLAGS)
-
-check: $(TARGET)
-	./$(TARGET)
-
-clean:
-	rm -f $(TARGET)
-
-.PHONY: all check clean

samples/gfx/xbr_filter_golden/xbr_filter_golden_test.c

@@ -1,171 +0,0 @@
-/* The 2xBR video filter (gfx/video_filters/2xbr.c) against output hashes
- * recorded from its earlier implementation: game-like frames (flat
- * areas, banded gradients, tiles, outlined sprites, dither) and noise,
- * both pixel formats, widths and heights from 1 up, frames smaller than
- * the size the filter was created for, and one to four workers. Any
- * change to how it computes must keep every byte; run with GOLDEN_PRINT=1
- * in the environment to print the table for a deliberate change. */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-
-#include "../../../gfx/video_filters/2xbr.c"
-
-static int gf(void *u, const char *k, float *v, float d) { (void)u; (void)k; *v = d; return 0; }
-static int gi(void *u, const char *k, int *v, int d) { (void)u; (void)k; *v = d; return 0; }
-static int gh(void *u, const char *k, unsigned *v, unsigned d) { (void)u; (void)k; *v = d; return 0; }
-static int gfa(void *u, const char *k, float **v, unsigned *n, const float *d, unsigned dn)
-{ (void)u; (void)k; (void)d; (void)dn; *v = NULL; *n = 0; return 0; }
-static int gia(void *u, const char *k, int **v, unsigned *n, const int *d, unsigned dn)
-{ (void)u; (void)k; (void)d; (void)dn; *v = NULL; *n = 0; return 0; }
-static int gs(void *u, const char *k, char **v, const char *d) { (void)u; (void)k; (void)d; *v = NULL; return 0; }
-
-static uint32_t fnv1a(const unsigned char *p, size_t n, uint32_t h)
-{
-   while (n--)
-      h = (h ^ *p++) * 16777619u;
-   return h;
-}
-
-static uint32_t rng = 12345u;
-static unsigned rnd(void) { rng = rng * 1103515245u + 12345u; return rng >> 8; }
-
-/* A game-like RGB565 frame. */
-static void game_frame(uint16_t *p, unsigned W, unsigned H)
-{
-   unsigned x, y, s;
-   for (y = 0; y < H; y++)
-      for (x = 0; x < W; x++)
-      {
-         uint16_t c;
-         if (y < H / 2)
-            c = (uint16_t)(((y / 12) & 31) | ((20 + (y / 24)) << 5) | (31 << 11));
-         else
-         {
-            unsigned lx = x % 16, ly = y % 16;
-            c = (((x / 16) + (y / 16)) & 1) ? 0x2c40 : 0x3460;
-            if (lx == 0 || ly == 0)
-               c = 0x1a20;
-            if (((lx * 7 + ly * 3) % 13) == 0)
-               c = 0x4c80;
-         }
-         if (y > H / 3 && y < H / 3 + 20 && ((x + y) & 1) && x > 4 && x < W / 2)
-            c = 0x7bef;
-         p[y * W + x] = c;
-      }
-   for (s = 0; s < 12 && W > 4 && H > 4; s++)
-   {
-      unsigned w  = 2 + rnd() % (W < 26 ? W - 2 : 24);
-      unsigned h  = 2 + rnd() % (H < 26 ? H - 2 : 24);
-      unsigned sx = rnd() % (W - w + 1), sy = rnd() % (H - h + 1);
-      uint16_t c  = (uint16_t)rnd();
-      for (y = sy; y < sy + h; y++)
-         for (x = sx; x < sx + w; x++)
-            p[y * W + x] = (x == sx || x == sx + w - 1 || y == sy
-                  || y == sy + h - 1) ? 0 : c;
-   }
-}
-
-static const unsigned sizes[][2] = {
-   { 1, 1 }, { 2, 1 }, { 1, 3 }, { 3, 3 }, { 5, 2 }, { 7, 9 },
-   { 64, 48 }, { 255, 31 }, { 256, 224 }, { 320, 240 }
-};
-#define NSIZE (sizeof(sizes) / sizeof(sizes[0]))
-
-static const uint32_t golden[2][2][NSIZE] = {
-#include "golden.h"
-};
-
-int main(void)
-{
-   struct softfilter_config cfg = { gf, gi, gh, gfa, gia, gs, free };
-   const struct softfilter_implementation *impl = softfilter_get_implementation(0);
-   uint16_t *src16 = (uint16_t*)malloc(330 * 250 * 2);
-   uint32_t *src32 = (uint32_t*)malloc(330 * 250 * 4);
-   uint8_t  *out   = (uint8_t*)malloc(700 * 500 * 4);
-   unsigned content, f, s, t, bad = 0;
-   int print = getenv("GOLDEN_PRINT") != NULL;
-
-   for (content = 0; content < 2; content++)
-   {
-      if (print) printf("{ ");
-      for (f = 0; f < 2; f++)
-      {
-         if (print) printf("{ ");
-         for (s = 0; s < NSIZE; s++)
-         {
-            unsigned W = sizes[s][0], H = sizes[s][1], i;
-            uint32_t h = 2166136261u;
-            rng = 777u + s;
-            if (content == 0)
-               game_frame(src16, W, H);
-            else
-               for (i = 0; i < W * H; i++)
-                  src16[i] = (uint16_t)rnd();
-            for (i = 0; i < W * H; i++)
-            {
-               uint16_t c = src16[i];
-               /* Low bits set too, so the 8888 path's full-precision
-                * blends and 565-precision decisions both show. */
-               src32[i] = 0xff000000u | ((uint32_t)(c >> 11) << 19)
-                  | ((uint32_t)((c >> 5) & 63) << 10)
-                  | ((uint32_t)(c & 31) << 3) | (rnd() & 0x070307);
-            }
-            for (t = 1; t <= 4; t++)
-            {
-               unsigned fmt = f ? SOFTFILTER_FMT_XRGB8888 : SOFTFILTER_FMT_RGB565;
-               unsigned bpp = f ? 4 : 2, ow, oh, k, max_w, max_h;
-               struct softfilter_work_packet pk[4];
-               size_t is = (W + 3) * bpp, os = (2 * W + 5) * bpp;
-               void *d;
-               /* Created larger than the frame, as RetroArch does for a
-                * core whose size varies; every second run exactly. */
-               max_w = (t & 1) ? W + 13 : W;
-               max_h = (t & 1) ? H + 7  : H;
-               d     = impl->create(&cfg, fmt, fmt, max_w, max_h, t, 0, NULL);
-               impl->query_output_size(d, &ow, &oh, W, H);
-               memset(out, 0x11, 700 * 500 * 4);
-               {
-                  /* Rows at a stride wider than the frame. */
-                  uint8_t *in = (uint8_t*)calloc(H, is);
-                  unsigned y;
-                  for (y = 0; y < H; y++)
-                     memcpy(in + y * is, f ? (void*)(src32 + y * W)
-                           : (void*)(src16 + y * W), W * bpp);
-                  impl->get_work_packets(d, pk, out, os, in, W, H, is);
-                  for (k = 0; k < impl->query_num_threads(d); k++)
-                     pk[k].work(d, pk[k].thread_data);
-                  free(in);
-               }
-               h = fnv1a(out, os * oh, h);
-               impl->destroy(d);
-            }
-            if (print)
-               printf("0x%08xu%s", h, s + 1 < NSIZE ? ", " : " ");
-            else if (h != golden[content][f][s])
-            {
-               printf("FAIL %s %s %ux%u: 0x%08x, recorded 0x%08x\n",
-                     content ? "noise" : "game", f ? "xrgb8888" : "rgb565",
-                     W, H, (unsigned)h, (unsigned)golden[content][f][s]);
-               bad++;
-            }
-         }
-         if (print) printf("}%s", f ? " " : ", ");
-      }
-      if (print) printf("},\n");
-   }
-   free(src16);
-   free(src32);
-   free(out);
-   if (print)
-      return 0;
-   if (bad)
-   {
-      printf("xbr_filter_golden: %u mismatches\n", bad);
-      return 1;
-   }
-   printf("xbr_filter_golden: output matches the recorded hashes\n");
-   return 0;
-}