Recompiler: every block let go when the core is

A block holds the blocks that jump to it, so two that jump to each other
hold each other. At the end the block tables were left as they were, and
when they went with the core's statics - the core unloaded with
dlclose(), or the process ending - those blocks and what they held were
never freed: about half a megabyte for a few hundred frames of the test
disc, each time a frontend unloads the core. bm_Term() now cuts the
links between them and empties the tables, nothing running the code by
then.

tools/threads/unload_leaks.sh runs the test discs' frames with the core
built with AddressSanitizer, unloads it, ends the process and fails on
anything LeakSanitizer finds; the sanitizers workflow runs it.
gl_frames.c unloads the core when no frame was drawn too.

.github/workflows/sanitizers.yml

@@ -4,12 +4,16 @@ name: Sanitizers
 # UndefinedBehaviorSanitizer (make DEBUG_UBSAN=1), each running the test
 # disc's frames on Vulkan (lavapipe) on every renderer, at 24-bit and with
 # the HDR output, the recompiler on: any report fails the run
-# (tools/vulkan/vk_sanitize.sh).
+# (tools/vulkan/vk_sanitize.sh). And with AddressSanitizer, the core
+# unloaded after its frames with OpenGL and the process ended, with nothing
+# left over for LeakSanitizer (tools/threads/unload_leaks.sh).
 
 on:
   push:
     paths:
       - 'core/**'
+      - 'tools/threads/unload_leaks.sh'
+      - 'tools/gl/gl_frames.c'
       - 'tools/vulkan/vk_sanitize.sh'
       - 'tools/vulkan/vk_hdr.c'
       - 'tools/hdr/hdr_post_ref.*'
@@ -19,6 +23,8 @@ on:
   pull_request:
     paths:
       - 'core/**'
+      - 'tools/threads/unload_leaks.sh'
+      - 'tools/gl/gl_frames.c'
       - 'tools/vulkan/vk_sanitize.sh'
       - 'tools/vulkan/vk_hdr.c'
       - 'tools/hdr/hdr_post_ref.*'
@@ -40,11 +46,12 @@ jobs:
         sanitizer: [address, undefined]
     steps:
       - uses: actions/checkout@v4
-      - name: Install Vulkan (lavapipe)
+      - name: Install Vulkan (lavapipe) and Mesa's EGL
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends \
-            build-essential libgl1-mesa-dev mesa-vulkan-drivers libvulkan1 python3
+            build-essential libgl1-mesa-dev mesa-vulkan-drivers libvulkan1 python3 \
+            libegl1-mesa-dev libgl1-mesa-dri
       - name: Build the core with the sanitizer
         run: |
           if [ "${{ matrix.sanitizer }}" = address ]; then
@@ -54,3 +61,6 @@ jobs:
           fi
       - name: Frames on every renderer, 24-bit and HDR
         run: VK_DRIVER_FILES=/usr/share/vulkan/icd.d/lvp_icd.json SANITIZE=${{ matrix.sanitizer }} sh tools/vulkan/vk_sanitize.sh
+      - name: The core unloaded, nothing left over (LeakSanitizer)
+        if: matrix.sanitizer == 'address'
+        run: sh tools/threads/unload_leaks.sh

core/hw/sh4/dyna/blockmanager.cpp

@@ -531,6 +531,37 @@ void bm_Init()
 	bm_Reset();
 }
 
+/* Every block let go, and with it what it holds. A block that another
+ * jumps to holds that one (pre_refs), so two that jump to each other hold
+ * each other: dropped from the tables as they are, neither was ever freed.
+ * The tables are the core's statics, and the blocks in them went when the
+ * core was unloaded - unloaded by a frontend, with its memory lost for as
+ * long as the frontend ran - or at the process's exit. Nothing runs
+ * recompiled code now, so the code is not touched: the links are cut and
+ * the blocks freed as their last holder lets go. */
+static void bm_ReleaseBlocks()
+{
+	for (const auto& it : blkmap)
+	{
+		it.second->pre_refs.clear();
+		it.second->pNextBlock = NULL;
+		it.second->pBranchBlock = NULL;
+	}
+	for (const auto& block : del_blocks)
+	{
+		block->pre_refs.clear();
+		block->pNextBlock = NULL;
+		block->pBranchBlock = NULL;
+	}
+	for (const auto& block : all_temp_blocks)
+		block->pre_refs.clear();
+	bm_ForgetVaddrs();
+	all_temp_blocks.clear();
+	blkmap.clear();
+	for (auto& block_list : blocks_per_page)
+		block_list.clear();
+}
+
 void bm_Term()
 {
 #ifdef DYNA_OPROF
@@ -538,6 +569,7 @@ void bm_Term()
 	
 	oprofHandle=0;
 #endif
+	bm_ReleaseBlocks();
 	bm_Reset();
 }
 

tools/gl/gl_frames.c

@@ -317,6 +317,11 @@ int main(int argc, char **argv)
    if (last_w == 0 || last_h == 0)
    {
       fprintf(stderr, "gl_frames: no frame was drawn\n");
+      /* (the core let go all the same, as a frontend does) */
+      if (hw.context_destroy)
+         hw.context_destroy();
+      retro_unload_game();
+      retro_deinit();
       return 1;
    }
    pixels = malloc((size_t)last_w * last_h * 4);

tools/threads/unload_leaks.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+# The core let go whole: the core built with AddressSanitizer (make
+# DEBUG_ASAN=1) runs the test discs' frames with OpenGL (gl_frames.c, Mesa's
+# surfaceless EGL) and the recompiler, is unloaded (retro_unload_game,
+# retro_deinit) and the process ends as any does, its statics destroyed,
+# and LeakSanitizer finds nothing left over. (A frontend that unloads a
+# core with dlclose() destroys them too; tools/vulkan/vk_sanitize.sh's
+# frontend ends with _Exit(), before them.) The recompiled blocks, which
+# hold the blocks that jump to them, were lost there: two that jump to each
+# other held each other.
+#
+#   sh tools/threads/unload_leaks.sh [core.so]
+set -e
+cd "$(dirname "$0")/../.."
+CORE=$(readlink -f "${1:-parallel_dc_libretro.so}")
+WORK=${TMPDIR:-/tmp}/unload_leaks.$$
+mkdir -p "$WORK"
+[ -n "${KEEP:-}" ] || trap 'rm -rf "$WORK"' EXIT
+${CC:-cc} -O1 -g -Wall -o "$WORK/gl_frames" tools/gl/gl_frames.c -lEGL -lGL -ldl
+RUNTIME=$(${CC:-cc} -print-file-name=libasan.so)
+fails=0
+for disc in plain clip; do
+   if [ $disc = plain ]; then python3 tools/threads/live_disc.py "$WORK/$disc.gdi"
+   else python3 tools/threads/live_disc.py "$WORK/$disc.gdi" --$disc; fi > /dev/null
+   for sort in per-triangle per-pixel; do
+      case $sort in
+         per-triangle) o="reicast_alpha_sorting=per-triangle (normal)" ;;
+         per-pixel) o="reicast_alpha_sorting=per-pixel (accurate)" ;;
+      esac
+      name=$disc-$sort
+      echo "$name"
+      rm -rf "$WORK/dir"; mkdir -p "$WORK/dir"
+      # (the recompiler's memory faults are its own: ASan's handler is kept out of them)
+      if ! ASAN_OPTIONS="detect_leaks=1:handle_segv=0:log_path=$WORK/$name.san" LD_PRELOAD="$RUNTIME" \
+            GLF_DIR="$WORK/dir" "$WORK/gl_frames" "$CORE" "$WORK/$disc.gdi" "$WORK/$name.raw" 200 "$o" \
+            > "$WORK/$name.log" 2>&1; then
+         echo "  FAIL: the run failed:"; tail -5 "$WORK/$name.log"
+         cat "$WORK/$name.san".* 2>/dev/null | grep -E "SUMMARY|ERROR" | head -3
+         fails=$((fails + 1)); continue
+      fi
+      if cat "$WORK/$name.san".* > /dev/null 2>&1; then
+         echo "  FAIL: LeakSanitizer:"
+         cat "$WORK/$name.san".* | grep -E "SUMMARY|ERROR" | head -3
+         fails=$((fails + 1))
+      fi
+   done
+done
+if [ $fails != 0 ]; then
+   echo "unload_leaks: $fails failed"
+   exit 1
+fi
+echo "unload_leaks: all let go"