Texture hashes through libretro-common's xxh64

TextureInfo hashes textures and palettes with libretro-common's
xxh64(), the same value XXH64() gives, so dumped and custom texture
names stay as they are. The pipeline hash helpers nothing calls are
gone, and with them Externals/xxhash.

.gitmodules

@@ -42,10 +42,6 @@
 	path = Externals/lz4/lz4
 	url = https://github.com/lz4/lz4.git
 	shallow = true
-[submodule "Externals/xxhash/xxHash"]
-	path = Externals/xxhash/xxHash
-	url = https://github.com/Cyan4973/xxHash.git
-	shallow = true
 [submodule "hidapi-src"]
 	path = Externals/hidapi/hidapi-src
 	url = https://github.com/libusb/hidapi.git

Externals/libretro-common/encodings/encoding_xxh64.c

@@ -0,0 +1,233 @@
+/* Copyright  (C) 2010-2026 The RetroArch team
+ *
+ * ---------------------------------------------------------------------------------------
+ * The following license statement only applies to this file (encoding_xxh64.c).
+ * ---------------------------------------------------------------------------------------
+ *
+ * Permission is hereby granted, free of charge,
+ * to any person obtaining a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <string.h>
+#if defined(_MSC_VER)
+#include <stdlib.h>
+#endif
+
+#include <retro_inline.h>
+#include <retro_endianness.h>
+#include <encodings/xxh64.h>
+
+#define XXH64_P1 0x9E3779B185EBCA87ULL
+#define XXH64_P2 0xC2B2AE3D27D4EB4FULL
+#define XXH64_P3 0x165667B19E3779F9ULL
+#define XXH64_P4 0x85EBCA77C2B2AE63ULL
+#define XXH64_P5 0x27D4EB2F165667C5ULL
+
+#if defined(_MSC_VER)
+#define XXH64_ROTL(x, r) _rotl64((x), (r))
+#else
+#define XXH64_ROTL(x, r) (((x) << (r)) | ((x) >> (64 - (r))))
+#endif
+
+/* One lane's step: the octets in, multiplied, turned and multiplied */
+#define XXH64_ROUND(acc, in) \
+   ((acc) += (in) * XXH64_P2, (acc) = XXH64_ROTL((acc), 31), \
+    (acc) *= XXH64_P1)
+
+static INLINE uint64_t xxh64_rd64(const uint8_t *p)
+{
+   uint64_t v;
+   memcpy(&v, p, 8);
+   return swap_if_big64(v);
+}
+
+static INLINE uint32_t xxh64_rd32(const uint8_t *p)
+{
+   uint32_t v;
+   memcpy(&v, p, 4);
+   return swap_if_big32(v);
+}
+
+/* The whole stripes of @p up to @end, at least one, through the lanes
+ * @v, which are held in registers for it; returns where the stripes
+ * stop.
+ *
+ * The lanes are chains of multiplies, bound by the multiplier however
+ * the loop is unrolled, so it takes a stripe a turn; what it saves is
+ * the branch a varying length mispredicts: one stripe is taken without
+ * the loop. */
+static INLINE const uint8_t *xxh64_stripes(uint64_t *v, const uint8_t *p,
+      const uint8_t *end)
+{
+   uint64_t       v1   = v[0];
+   uint64_t       v2   = v[1];
+   uint64_t       v3   = v[2];
+   uint64_t       v4   = v[3];
+   size_t         n    = (size_t)(end - p);
+   const uint8_t *stop = p + (n & ~(size_t)31);
+
+   if (n < 64)
+   {
+      XXH64_ROUND(v1, xxh64_rd64(p));
+      XXH64_ROUND(v2, xxh64_rd64(p + 8));
+      XXH64_ROUND(v3, xxh64_rd64(p + 16));
+      XXH64_ROUND(v4, xxh64_rd64(p + 24));
+      p += 32;
+   }
+   else
+      do
+      {
+         XXH64_ROUND(v1, xxh64_rd64(p));
+         XXH64_ROUND(v2, xxh64_rd64(p + 8));
+         XXH64_ROUND(v3, xxh64_rd64(p + 16));
+         XXH64_ROUND(v4, xxh64_rd64(p + 24));
+         p += 32;
+      } while (p != stop);
+   v[0] = v1;
+   v[1] = v2;
+   v[2] = v3;
+   v[3] = v4;
+   return p;
+}
+
+static INLINE uint64_t xxh64_merge(uint64_t h, uint64_t v)
+{
+   uint64_t k = 0;
+   XXH64_ROUND(k, v);
+   h ^= k;
+   return h * XXH64_P1 + XXH64_P4;
+}
+
+/* The four lanes folded into one */
+static INLINE uint64_t xxh64_converge(const uint64_t *v)
+{
+   uint64_t h =  XXH64_ROTL(v[0], 1)  + XXH64_ROTL(v[1], 7)
+               + XXH64_ROTL(v[2], 12) + XXH64_ROTL(v[3], 18);
+   h = xxh64_merge(h, v[0]);
+   h = xxh64_merge(h, v[1]);
+   h = xxh64_merge(h, v[2]);
+   return xxh64_merge(h, v[3]);
+}
+
+#define XXH64_TAKE8(h, p) do { \
+   uint64_t k_ = 0; \
+   XXH64_ROUND(k_, xxh64_rd64(p)); \
+   (h) ^= k_; \
+   (h)  = XXH64_ROTL((h), 27) * XXH64_P1 + XXH64_P4; \
+} while (0)
+
+#define XXH64_TAKE1(h, c) do { \
+   (h) ^= (uint64_t)(c) * XXH64_P5; \
+   (h)  = XXH64_ROTL((h), 11) * XXH64_P1; \
+} while (0)
+
+/* The last @len (under 32) octets at @p, and the avalanche: eights,
+ * then a four, then single octets */
+static INLINE uint64_t xxh64_finish(uint64_t h, const uint8_t *p, size_t len)
+{
+   for (; len >= 8; len -= 8, p += 8)
+      XXH64_TAKE8(h, p);
+   if (len >= 4)
+   {
+      h ^= (uint64_t)xxh64_rd32(p) * XXH64_P1;
+      h  = XXH64_ROTL(h, 23) * XXH64_P2 + XXH64_P3;
+      p   += 4;
+      len -= 4;
+   }
+   for (; len; len--)
+      XXH64_TAKE1(h, *p++);
+   h ^= h >> 33;
+   h *= XXH64_P2;
+   h ^= h >> 29;
+   h *= XXH64_P3;
+   h ^= h >> 32;
+   return h;
+}
+
+static void xxh64_lanes(uint64_t *v, uint64_t seed)
+{
+   v[0] = seed + XXH64_P1 + XXH64_P2;
+   v[1] = seed + XXH64_P2;
+   v[2] = seed;
+   v[3] = seed - XXH64_P1;
+}
+
+uint64_t xxh64(const void *data, size_t len, uint64_t seed)
+{
+   const uint8_t *p = (const uint8_t*)data;
+   uint64_t       h;
+
+   if (len >= 32)
+   {
+      uint64_t v[4];
+      xxh64_lanes(v, seed);
+      p = xxh64_stripes(v, p, p + len);
+      h = xxh64_converge(v);
+   }
+   else
+      h = seed + XXH64_P5;
+   return xxh64_finish(h + (uint64_t)len, p, len & 31);
+}
+
+void xxh64_reset(xxh64_state_t *s, uint64_t seed)
+{
+   xxh64_lanes(s->v, seed);
+   s->total   = 0;
+   s->memsize = 0;
+}
+
+void xxh64_update(xxh64_state_t *s, const void *data, size_t len)
+{
+   const uint8_t *p   = (const uint8_t*)data;
+   const uint8_t *end = p + len;
+
+   if (!len)
+      return;
+   s->total += len;
+
+   /* not yet a stripe */
+   if (s->memsize + len < 32)
+   {
+      memcpy(s->mem + s->memsize, p, len);
+      s->memsize += (uint32_t)len;
+      return;
+   }
+
+   /* the stripe begun before, made whole */
+   if (s->memsize)
+   {
+      size_t fill = 32 - s->memsize;
+      memcpy(s->mem + s->memsize, p, fill);
+      xxh64_stripes(s->v, s->mem, s->mem + 32);
+      p         += fill;
+      s->memsize = 0;
+   }
+
+   if (end - p >= 32)
+      p = xxh64_stripes(s->v, p, end);
+   if (p != end)
+   {
+      memcpy(s->mem, p, (size_t)(end - p));
+      s->memsize = (uint32_t)(end - p);
+   }
+}
+
+uint64_t xxh64_digest(const xxh64_state_t *s)
+{
+   uint64_t h = s->total >= 32
+      ? xxh64_converge(s->v)
+      : s->v[2] + XXH64_P5;
+   return xxh64_finish(h + s->total, s->mem, s->memsize);
+}

Externals/libretro-common/include/encodings/xxh64.h

@@ -0,0 +1,79 @@
+/* Copyright  (C) 2010-2026 The RetroArch team
+ *
+ * ---------------------------------------------------------------------------------------
+ * The following license statement only applies to this file (xxh64.h).
+ * ---------------------------------------------------------------------------------------
+ *
+ * Permission is hereby granted, free of charge,
+ * to any person obtaining a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _LIBRETRO_ENCODINGS_XXH64_H
+#define _LIBRETRO_ENCODINGS_XXH64_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+#include <retro_common_api.h>
+
+RETRO_BEGIN_DECLS
+
+/* XXH64, as xxHash specifies it: the hash Zstandard frames are checked
+ * with, and the same value for the same octets and seed on every
+ * host. */
+
+/* A hash taken over pieces handed to it one after another */
+typedef struct xxh64_state
+{
+   uint64_t v[4];       /* the four lanes */
+   uint64_t total;      /* octets taken */
+   uint8_t  mem[32];    /* a stripe not yet whole */
+   uint32_t memsize;    /* how much of it there is */
+} xxh64_state_t;
+
+/**
+ * xxh64:
+ * @data       : the octets
+ * @len        : how many
+ * @seed       : the seed
+ *
+ * Returns: the XXH64 of @data.
+ */
+uint64_t xxh64(const void *data, size_t len, uint64_t seed);
+
+/**
+ * xxh64_reset:
+ *
+ * Starts @s over with @seed: nothing taken yet.
+ */
+void xxh64_reset(xxh64_state_t *s, uint64_t seed);
+
+/**
+ * xxh64_update:
+ *
+ * Takes @len more octets at @data, which may be any length.
+ */
+void xxh64_update(xxh64_state_t *s, const void *data, size_t len);
+
+/**
+ * xxh64_digest:
+ *
+ * Returns: the XXH64 of what @s has taken, which may go on taking more.
+ */
+uint64_t xxh64_digest(const xxh64_state_t *s);
+
+RETRO_END_DECLS
+
+#endif

Makefile.common

@@ -142,6 +142,7 @@ LIBRETRO_COMMON_SOURCES_C += \
 	$(CORE_DIR)/Externals/libretro-common/compat/compat_strl.c \
 	$(CORE_DIR)/Externals/libretro-common/compat/fopen_utf8.c \
 	$(CORE_DIR)/Externals/libretro-common/encodings/encoding_utf.c \
+	$(CORE_DIR)/Externals/libretro-common/encodings/encoding_xxh64.c \
 	$(CORE_DIR)/Externals/libretro-common/features/features_cpu.c \
 	$(CORE_DIR)/Externals/libretro-common/file/file_path.c \
 	$(CORE_DIR)/Externals/libretro-common/file/file_path_io.c \
@@ -1013,14 +1014,6 @@ TINYGLTF_SOURCES_OBJCXX :=
 TINYGLTF_SOURCES_CXX += \
 	$(CORE_DIR)/Externals/tinygltf/tinygltf/tiny_gltf.cc
 
-# XXHASH
-XXHASH_SOURCES_C :=
-XXHASH_SOURCES_CXX :=
-XXHASH_SOURCES_ASM :=
-XXHASH_SOURCES_OBJCXX :=
-XXHASH_SOURCES_C += \
-	$(CORE_DIR)/Externals/xxhash/xxHash/xxhash.c
-
 # DOLPHIN
 DOLPHIN_SOURCES_C :=
 DOLPHIN_SOURCES_CXX :=
@@ -1961,7 +1954,6 @@ PUGIXML_FLAGS := -w -DPUGIXML_NO_EXCEPTIONS -I$(EXT_DIR)/pugixml/pugixml/src
 TINYGLTF_FLAGS    := -w -DTINYGLTF_NOEXCEPTION -DTINYGLTF_NO_EXTERNAL_IMAGE -DTINYGLTF_USE_CPP14
 TINYGLTF_CXXFLAGS := -std=gnu++20
 
-XXHASH_FLAGS := -w -I$(EXT_DIR)/xxhash/xxHash
 
 CPPIPC_FLAGS := -w -O2 -DUNICODE -D_UNICODE -fexceptions \
 	-I$(EXT_DIR)/cpp-ipc/cpp-ipc/include -I$(EXT_DIR)/cpp-ipc/cpp-ipc/src \
@@ -1988,7 +1980,7 @@ DOLPHIN_FLAGS := \
 	-I$(EXT_DIR)/SFML/SFML/include -I$(EXT_DIR)/bzip2/bzip2 -I$(EXT_DIR)/expr/include \
 	-I$(EXT_DIR)/hidapi/hidapi-src/hidapi -I$(EXT_DIR)/pugixml/pugixml/src \
 	$(MGBA_INC) \
-	-I$(EXT_DIR)/libspng/libspng/spng -I$(EXT_DIR)/xxhash/xxHash \
+	-I$(EXT_DIR)/libspng/libspng/spng \
 	-I$(EXT_DIR)/tinygltf \
 	-I$(EXT_DIR)/curl/curl/include -I$(EXT_DIR)/FatFs -I$(EXT_DIR)/LZO \
 	-I$(EXT_DIR)/lz4/lz4/lib -I$(EXT_DIR)/Bochs_disasm \
@@ -2046,7 +2038,7 @@ endif
 
 # Libraries in link order; the optional ones only where they are built.
 LIB_GROUPS := DOLPHIN LIBRETRO_COMMON CURL MBEDTLS MINIZIP ZSTD ZLIB LZMA BZIP2 LZ4 LZO SPNG \
-	MGBA SFML FMT PUGIXML XXHASH TINYGLTF FATFS
+	MGBA SFML FMT PUGIXML TINYGLTF FATFS
 ifeq ($(ARCH_X86_64),1)
    LIB_GROUPS += BDISASM
 endif

Source/Core/VideoCommon/HiresTextures.cpp

@@ -9,7 +9,6 @@
 #include <string_view>
 #include <unordered_map>
 #include <utility>
-#include <xxhash.h>
 
 #include "Common/CommonPaths.h"
 #include "Common/FileSearch.h"

Source/Core/VideoCommon/PipelineUtils.cpp

@@ -146,25 +146,4 @@ GXPipelineUid ApplyDriverBugs(const GXPipelineUid& in)
 
   return out;
 }
-
-std::size_t PipelineToHash(const GXPipelineUid& in)
-{
-  XXH3_state_t pipeline_hash_state;
-  XXH3_INITSTATE(&pipeline_hash_state);
-  XXH3_64bits_reset_withSeed(&pipeline_hash_state, static_cast<XXH64_hash_t>(1));
-  UpdateHashWithPipeline(in, &pipeline_hash_state);
-  return XXH3_64bits_digest(&pipeline_hash_state);
-}
-
-void UpdateHashWithPipeline(const GXPipelineUid& in, XXH3_state_t* hash_state)
-{
-  XXH3_64bits_update(hash_state, &in.vertex_format->GetVertexDeclaration(),
-                     sizeof(PortableVertexDeclaration));
-  XXH3_64bits_update(hash_state, &in.blending_state, sizeof(BlendingState));
-  XXH3_64bits_update(hash_state, &in.depth_state, sizeof(DepthState));
-  XXH3_64bits_update(hash_state, &in.rasterization_state, sizeof(RasterizationState));
-  XXH3_64bits_update(hash_state, &in.gs_uid, sizeof(GeometryShaderUid));
-  XXH3_64bits_update(hash_state, &in.ps_uid, sizeof(PixelShaderUid));
-  XXH3_64bits_update(hash_state, &in.vs_uid, sizeof(VertexShaderUid));
-}
 }  // namespace VideoCommon

Source/Core/VideoCommon/PipelineUtils.h

@@ -3,20 +3,10 @@
 
 #pragma once
 
-#include <xxh3.h>
-
 #include "VideoCommon/GXPipelineTypes.h"
 
 namespace VideoCommon
 {
 GXPipelineUid ApplyDriverBugs(const GXPipelineUid& in);
 
-// Returns a hash of the pipeline, hashing the
-// vertex declarations instead of the native vertex format
-// object
-std::size_t PipelineToHash(const GXPipelineUid& in);
-
-// Updates an existing hash with the hash of the pipeline
-void UpdateHashWithPipeline(const GXPipelineUid& in, XXH3_state_t* hash_state);
-
 }  // namespace VideoCommon

Source/Core/VideoCommon/TextureInfo.cpp

@@ -6,7 +6,7 @@
 #include <span>
 
 #include <fmt/format.h>
-#include <xxhash.h>
+#include <encodings/xxh64.h>
 
 #include "Common/Align.h"
 #include "Common/Assert.h"
@@ -165,8 +165,8 @@ TextureInfo::NameDetails TextureInfo::CalculateTextureName() const
 
   DEBUG_ASSERT(tlut_size <= m_palette_size.value_or(0));
 
-  const u64 tex_hash = XXH64(m_data.data(), m_texture_size, 0);
-  const u64 tlut_hash = tlut_size ? XXH64(tlut, tlut_size, 0) : 0;
+  const u64 tex_hash = xxh64(m_data.data(), m_texture_size, 0);
+  const u64 tlut_hash = tlut_size ? xxh64(tlut, tlut_size, 0) : 0;
 
   return {.base_name = fmt::format("{}{}x{}{}", format_prefix, m_raw_width, m_raw_height,
                                    m_mipmaps_enabled ? "_m" : ""),