native: retire units whose code is unmapped, place them again on remap

The browser's rule (705dc708, 87176416) on the native host, so both agree:
a page no longer mapped executable marks its unit page patched whatever its
bytes still hash to, the unit's entries there leave the function table
(providers stay), and an ELF object whose code was there is forgotten from
ELF_SEEN, so it is placed again when it is mapped again.

ftab::on_retire clears every entry's probe-cache slot before retiring:
retiring the unit entries first left a cached one behind, and libb loaded
where liba's re-placed unit had been ran liba's unit, which bailed to
tier-1 and was sent straight back (a spin under --fjit).

tests/elf-units-remap: liba and libb at one pinned base with a unit each,
loaded and unloaded twice; output matches native and the units are placed
4 times (the old rule places them twice: the second loads ran on tier-1).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_1896c5b302a54696aaa92c5f4047f20b

shims/native/src/ftab.rs

@@ -1002,17 +1002,27 @@ pub(crate) fn install_bridges(mut cx: impl AsContextMut<Data = HostState>, slots
 /// ranges, and clear their probe cache entries (`retire_translations`).
 /// Their page checks would catch them anyway; this keeps the cache from
 /// sending callers there first.
-pub(crate) fn on_retire(sh: &mut JitShared, memory: &mut [u8], ranges: &[(u64, u64)]) {
+///
+/// `gone` (within `ranges`): pages no longer mapped executable. Units' entries
+/// there leave the table (providers stay): the object that next lands there
+/// clears the patched bits for its own unit, which would otherwise revive
+/// every address of the old one it does not define.
+pub(crate) fn on_retire(sh: &mut JitShared, memory: &mut [u8], ranges: &[(u64, u64)], gone: &[(u64, u64)]) {
     sh.ftab.epoch += 1;
     if let Some(t) = gex_kernel::ftab::translator() {
         t.retire(sh.ftab.process.0, ranges);
     }
     sh.ftab.queued.retain(|&a| !ranges.iter().any(|&(s, e)| a >= s && a < e));
+    let is_gone = |a: u64| gone.iter().any(|&(s, e)| a >= s && a < e);
     for &(start, end) in ranges {
         let hit: Vec<(u64, gex_kernel::ftab::Func)> = sh.ftab.map.range(start, end).collect();
         for (a, f) in hit {
+            // Every entry's probe cache slot, the units' included: a cached
+            // unit entry would outlive its retirement below.
             gex_kernel::ftab::fcache_clear(memory, a);
-            if sh.ftab.map.install(f.install).tier != Tier::Unit {
+            let install = sh.ftab.map.install(f.install);
+            let unit_gone = install.tier == Tier::Unit && !sh.ftab.providers.contains(&f.install) && is_gone(a);
+            if install.tier != Tier::Unit || unit_gone {
                 sh.ftab.map.retire(a);
             }
         }

shims/native/src/lib.rs

@@ -5350,6 +5350,28 @@ fn dispatch_tables(
     Ok(out)
 }
 
+/// The pages of `ranges` no longer mapped executable (an unmapped object,
+/// code made non-executable), merged into ranges, as the browser's
+/// `gone_code`.
+fn gone_code(k: &gex_kernel::Kernel, ranges: &[(u64, u64)]) -> Vec<(u64, u64)> {
+    use gex_kernel::mem::prot;
+    let page = gex_abi::DISPATCH_PAGE;
+    let mut out: Vec<(u64, u64)> = Vec::new();
+    for &(start, end) in ranges {
+        let mut a = start / page * page;
+        while a < end {
+            if !k.mem.vma_at(a).is_some_and(|v| v.prot & prot::EXEC != 0) {
+                match out.last_mut() {
+                    Some(last) if last.1 == a => last.1 = a + page,
+                    _ => out.push((a, a + page)),
+                }
+            }
+            a += page;
+        }
+    }
+    out
+}
+
 /// Forget the translations of code the guest has unmapped. A dispatch slot
 /// answers for an address, not the bytes at it, so after a `dlclose`+`dlopen`
 /// reuses the range the guest would run the old library's code.
@@ -5379,9 +5401,12 @@ fn retire_translations(
         .clone()
         .ok_or_else(|| anyhow!("guest memory unavailable"))?;
 
+    // Pages no longer mapped executable: a unit there is dead whatever its
+    // bytes still are (`nt_cgen::on_retire`).
+    let gone = caller.as_context_mut().data_mut().with_kernel(|k| gone_code(k, ranges));
     // Units' rewritten pages first: a lookup after the probe cache is
     // cleared below must already see them (`ftab::current`).
-    nt_cgen::on_retire(&memory, ranges);
+    nt_cgen::on_retire(&memory, ranges, &gone);
     nt_cgen::provided_rewrites(&memory, ranges);
     let mut sh = shared.lock().unwrap_or_else(|e| e.into_inner());
     let mut pages = 0usize;
@@ -5404,7 +5429,7 @@ fn retire_translations(
             }
         }
     }
-    ftab::on_retire(&mut sh, memory.bytes(), ranges);
+    ftab::on_retire(&mut sh, memory.bytes(), ranges, &gone);
     drop(sh);
     refresh_callouts(caller)?;
     if pages > 0 && std::env::var_os("GEX_VERBOSE_OBJECTS").is_some() {

shims/native/src/nt_cgen.rs

@@ -909,14 +909,26 @@ static UNIT_PAGES: Mutex<Option<HashMap<u64, u64>>> = Mutex::new(None);
 /// Pages that have retired (`retire_translations`): any an installed unit
 /// covers whose bytes are no longer the ones it was built from gets its bit
 /// in `gex_abi::PATCHED_PAGES`, and compiled code stops calling into it.
-pub(crate) fn on_retire(memory: &gex_mem::GuestMemory, ranges: &[(u64, u64)]) {
+///
+/// `gone` (within `ranges`): pages no longer mapped executable. A unit there
+/// is dead whatever the bytes still hash to -- an unmapped object's bytes stay
+/// until the next mapping overwrites them, and that mapping retires nothing
+/// (the browser's `cgen::on_retire`). An ELF object whose code was there is
+/// placed again if it is mapped again.
+pub(crate) fn on_retire(memory: &gex_mem::GuestMemory, ranges: &[(u64, u64)], gone: &[(u64, u64)]) {
+    ELF_SEEN
+        .lock()
+        .unwrap_or_else(|e| e.into_inner())
+        .retain(|(_, _, (lo, hi))| !gone.iter().any(|&(s, e)| s < *hi && *lo < e));
     let memory = memory.bytes();
     let guard = UNIT_PAGES.lock().unwrap_or_else(|e| e.into_inner());
     let Some(pages) = guard.as_ref() else { return };
     for &(start, end) in ranges {
         for page in start / gex_abi::DISPATCH_PAGE..end.div_ceil(gex_abi::DISPATCH_PAGE) {
             let Some(&built) = pages.get(&page) else { continue };
-            if page_hash(memory, page) != built {
+            let at = page * gex_abi::DISPATCH_PAGE;
+            let unmapped = gone.iter().any(|&(s, e)| s <= at && at < e);
+            if unmapped || page_hash(memory, page) != built {
                 let off = (gex_abi::PATCHED_PAGES + page / 8) as usize;
                 if let Some(b) = memory.get_mut(off) {
                     *b |= 1 << (page % 8);
@@ -1535,7 +1547,7 @@ fn elf_wanted(path: &str) -> bool {
 }
 
 /// Objects already looked at in this host process, by path and bias.
-static ELF_SEEN: Mutex<Vec<(String, u64)>> = Mutex::new(Vec::new());
+static ELF_SEEN: Mutex<Vec<(String, u64, (u64, u64))>> = Mutex::new(Vec::new());
 
 /// After a syscall that mapped code: install the unit of each listed ELF
 /// object (`GEX_ELF_UNITS`), before ld.so runs any of it.
@@ -1551,15 +1563,16 @@ pub(crate) fn on_elf_maps(caller: &mut Caller<'_, HostState>, mapped: &[gex_kern
         let Some(base) = elf_bias(&bytes, m.start, m.offset) else {
             continue;
         };
-        {
-            let mut seen = ELF_SEEN.lock().unwrap_or_else(|e| e.into_inner());
-            if seen.iter().any(|(p, b)| *p == m.path && *b == base) {
-                continue;
-            }
-            seen.push((m.path.clone(), base));
+        if ELF_SEEN.lock().unwrap_or_else(|e| e.into_inner()).iter().any(|(p, b, _)| *p == m.path && *b == base) {
+            continue;
         }
         let t0 = std::time::Instant::now();
         let image = elf_image(&bytes, base)?;
+        // The executable span, so a retirement of that code forgets the object (`on_retire`).
+        let span = image.segments.iter().filter(|s| s.executable).fold((u64::MAX, 0u64), |(lo, hi), s| {
+            (lo.min(s.vaddr), hi.max(s.vaddr + s.bytes.len() as u64))
+        });
+        ELF_SEEN.lock().unwrap_or_else(|e| e.into_inner()).push((m.path.clone(), base, span));
         let none = Provided { providers: Vec::new(), exports: HashMap::new(), stubs: Vec::new() };
         let key = unit_key(&image, &none);
         let cwasm_path = cache_root()?.join(&key).join("c.wasm");

tests/elf-units-remap/lib.c

@@ -0,0 +1,3 @@
+/* liba.so (-DK=3 -DC=1) and libb.so (-DK=7 -DC=-2): the same layout, different code. */
+__attribute__((noinline)) static int inner(int x) { return x * K + C; }
+int f(int x) { int s = 0; for (int i = 0; i < x; i++) s += inner(i); return s + inner(x); }

tests/elf-units-remap/main.c

@@ -0,0 +1,26 @@
+#define _GNU_SOURCE
+/* One library unloaded and another loaded at the same address: each must run its own code. */
+#include <dlfcn.h>
+#include <link.h>
+#include <stdio.h>
+#include <stdlib.h>
+static unsigned long base_of(void *h) { struct link_map *m = 0; dlinfo(h, RTLD_DI_LINKMAP, &m); return m ? m->l_addr : 0; }
+int main(int argc, char **argv) {
+	unsigned long first = 0;
+	setvbuf(stdout, NULL, _IOLBF, 0);
+	for (int round = 0; round < 2; round++) {
+		for (const char *n = "a"; n; n = (*n == 'a' ? "b" : 0)) {
+			char path[4096];
+			snprintf(path, sizeof path, "%s/lib%s.so", argc > 1 ? argv[1] : ".", n);
+			void *h = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+			if (!h) { printf("dlopen %s: %s\n", path, dlerror()); return 1; }
+			int (*f)(int) = (int (*)(int))dlsym(h, "f");
+			unsigned long b = base_of(h);
+			if (!first) first = b;
+			printf("lib%s f(5)=%d f(9)=%d %s\n", n, f(5), f(9), b == first ? "same-base" : "other-base");
+			if (getenv("BASES")) fprintf(stderr, "base lib%s %#lx\n", n, b);
+			dlclose(h);
+		}
+	}
+	return 0;
+}

tests/elf-units-remap/run.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# One library unloaded and another loaded at the same address (main.c), with
+# function-tier units for both (GEX_ELF_UNITS): each load must run its own
+# code, and a library mapped again must get its unit again -- a unit whose
+# code was unmapped is retired, not revived by the next object's mapping
+# (nt_cgen::on_retire, ftab::on_retire; the browser's cgen::on_retire).
+#
+#   tests/elf-units-remap/run.sh [gexc]      (needs gcc; host glibc as the guest's)
+set -u
+HERE=$(cd "$(dirname "$0")" && pwd)
+GEXC=${1:-target/x86/release/gexc}
+W=${WORK:-$(mktemp -d)}
+mkdir -p "$W"
+gcc -O1 -shared -fPIC -DK=3 -DC=1 -o "$W/liba.so" "$HERE/lib.c" || exit 1
+gcc -O1 -shared -fPIC -DK=7 -DC=-2 -o "$W/libb.so" "$HERE/lib.c" || exit 1
+gcc -O1 -o "$W/main" "$HERE/main.c" -ldl || exit 1
+# Both at one pinned base, so the units built for it apply.
+BASE=0x30000000
+printf '%s 0x10000 liba.so\n%s 0x10000 libb.so\n' $BASE $BASE > "$W/pins.txt"
+export GEX_CGEN_CACHE="$W/cgen" GEX_ELF_UNITS=liba.so,libb.so GEX_ELF_UNITS_BUILD=0
+for l in liba libb; do
+	"$GEXC" cgen-image --elf "$W/$l.so" --base $BASE --jobs 2 > "$W/$l.cgen.log" 2>&1 || { cat "$W/$l.cgen.log"; exit 1; }
+done
+native=$("$W/main" "$W")
+fail=0
+for flags in "" "--fjit"; do
+	gex=$(timeout 60 "$GEXC" run $flags --pin "$W/pins.txt" "$W/main" "$W" 2> "$W/err.txt")
+	placed=$(grep -c 'function tier for' "$W/err.txt")
+	if [ "$native" = "$gex" ] && [ "$placed" = 4 ]; then
+		echo "ok   elf-units-remap ${flags:-t1}"
+	else
+		echo "FAIL elf-units-remap ${flags:-t1} (units placed $placed times, want 4)"
+		printf 'native:\n%s\ngex:\n%s\n' "$native" "$gex"; tail -20 "$W/err.txt"; fail=1
+	fi
+done
+exit $fail