Merge pull request #1764 from Sonicadvance1/fix_xxhash

FEXRootFSFetcher: Update and fix xxhash file hashing

External/xxhash

@@ -1 +1 @@
-Subproject commit 86c1eb957e1722ebbc1761a84caeed03ba5082de
+Subproject commit ba7375d54fbbf7bfd9519b465a146e9a8bf0240f

Scripts/InstallFEX.py

@@ -306,12 +306,12 @@ def GetRootFSPath():
     return _RootFSPath
 
 def CheckRootFSInstallStatus():
-    # Matches what is available on https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt
+    # Matches what is available on https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt
     UbuntuVersionToRootFS = {
         "20.04": "Ubuntu_21_04.sqsh",
         "21.04": "Ubuntu_21_04.sqsh",
         "21.10": "Ubuntu_21_10.sqsh",
-        "22.04": "Ubuntu_21_10.sqsh",
+        "22.04": "Ubuntu_22_04.sqsh",
     }
 
     return os.path.exists(GetRootFSPath() + UbuntuVersionToRootFS[GetDistro()[1]])

Source/Tools/FEXRootFSFetcher/Main.cpp

@@ -346,7 +346,7 @@ namespace WebFileFetcher {
     std::string Hash;
   };
 
-  const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt";
+  const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt";
 
   std::string DownloadToString(const std::string &URL) {
     std::string BigArgs =

Source/Tools/FEXRootFSFetcher/XXFileHash.cpp

@@ -28,32 +28,31 @@ namespace XXFileHash {
     lseek(fd, 0, SEEK_SET);
 
     // Set up XXHash state
-    XXH64_state_t* const State = XXH64_createState();
+    XXH3_state_t* const State = XXH3_createState();
     XXH64_hash_t const Seed = 0;
 
     if (!State) {
       return HadError();
     }
 
-    if (XXH64_reset(State, Seed) == XXH_ERROR) {
+    if (XXH3_64bits_reset_withSeed(State, Seed) == XXH_ERROR) {
       return HadError();
     }
 
     std::vector<char> Data(BLOCK_SIZE);
-    off_t DataRemaining = Size - BLOCK_SIZE;
-    off_t DataTail = Size - DataRemaining;
     off_t CurrentOffset = 0;
     auto Now = std::chrono::high_resolution_clock::now();
 
     // Let the kernel know that we will be reading linearly
     posix_fadvise(fd, 0, Size, POSIX_FADV_SEQUENTIAL);
-    while (CurrentOffset < DataRemaining) {
+    while (CurrentOffset < Size) {
+
       ssize_t Result = pread(fd, Data.data(), BLOCK_SIZE, CurrentOffset);
       if (Result == -1) {
         return HadError();
       }
 
-      if (XXH64_update(State, Data.data(), BLOCK_SIZE) == XXH_ERROR) {
+      if (XXH3_64bits_update(State, Data.data(), Result) == XXH_ERROR) {
         return HadError();
       }
       auto Cur = std::chrono::high_resolution_clock::now();
@@ -62,21 +61,11 @@ namespace XXFileHash {
         fmt::print("{}% hashed\n", (double)CurrentOffset / SizeD);
         Now = Cur;
       }
-      CurrentOffset += BLOCK_SIZE;
+      CurrentOffset += Result;
     }
 
-    // Finish the tail
-    ssize_t Result = pread(fd, Data.data(), DataTail, CurrentOffset);
-    if (Result == -1) {
-      return HadError();
-    }
-
-    if (XXH64_update(State, Data.data(), DataTail) == XXH_ERROR) {
-      return HadError();
-    }
-
-    XXH64_hash_t const Hash = XXH64_digest(State);
-    XXH64_freeState(State);
+    XXH64_hash_t const Hash = XXH3_64bits_digest(State);
+    XXH3_freeState(State);
 
     close(fd);
     return {true, Hash};

docs/ReleaseProcess.md

@@ -80,7 +80,7 @@ Follow the steps in: https://github.com/FEX-Emu/FEX-ppa/blob/main/README_ppa.md
 * Follow the Build_Data file's information for how to generate an image using `build_image.py`
   * This gives a squashfs image for the rootfs
 * Use FEXRootFSFetcher <image.sqsh> to generate the xxhash for the image
-* Update `https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt` with the new rootfs image and hash
+* Update `https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links_XXH3.txt` with the new rootfs image and hash
   * This currently lives in a private FEX-Emu backblaze bucket with cloudflare servicing it.
   * Never publically give the direct backblaze link to the file. Will cause BW costs to skyrocket
   * Always pass through cloudflare
@@ -88,9 +88,9 @@ Follow the steps in: https://github.com/FEX-Emu/FEX-ppa/blob/main/README_ppa.md
 * Upload new image to Backblaze using the b2 upload tool
   * b2 upload-file <bucketname> <image.sqsh> <Image folder name>/<image.sqsh>
 
-* Upload the new RootFS_links.txt
+* Upload the new RootFS_links_XXH3.txt
   * Lives in the root of the bucket
-  * b2 upload-file <bucketname> RootFS_links.txt RootFS_links.txt
+  * b2 upload-file <bucketname> RootFS_links_XXH3.txt RootFS_links_XXH3.txt
 
 * Once uploaded it should propagate immediately
 * Might be worth thinking about the coherency problem of updating the hash versus image independently if overwriting an image