The function JIT leaves a replaced library's code alone (Muck --fjit hang)

Under --fjit, Muck on .NET Mono spun forever at Begin MonoManager
ReloadAssembly: the function JIT followed UnityPlayer's PLT (folded GOT
loads) into libmonobdwgc-2.0.so and published translations of Boehm's own
code over the stateful provider's bindings, and GC_exclude_static_roots
looped calling the provider's GC_next_exclusion (3 million lookups of one
address). libprov now records each replaced library's text as mapped, and
ftab::job leaves those ranges out of what the function JIT may translate.

Muck --fjit, batch mode: Completed reload in 6.9 s (17-22 s without --fjit),
on to the menu scene's scripts. tests/unity-dotnet/embed gains a --fjit leg
(it called the stub library's mono_domain_assembly_open before the fix).
GEX_FTAB_TRACE lines also name the guest return address.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01PeRutwFgZ66XpxjasKrPn1

shims/native/src/ftab.rs

@@ -559,7 +559,12 @@ pub(crate) fn lookup(caller: &mut Caller<'_, HostState>, addr: u64) -> Result<i3
                     drop(s);
                     let t = caller.data().jit.as_ref().and_then(|j| j.ftab.as_ref()).map(|st| st.table);
                     let ty = t.and_then(|t| t.get(&mut *caller, f.idx as u64)).and_then(|r| r.as_func().flatten().map(|f| format!("{:?}", f.ty(&*caller))));
-                    eprintln!("[ftab] lookup {addr:#x} -> index {} (install {}, module {}, base {}, {:?}) holds {ty:?}", f.idx, f.install, inst.module, inst.base, inst.tier);
+                    // x86-64: the guest's return address, at [rsp] (register 4) on a call.
+                    let ctx_addr = caller.data().thread.ctx_addr as usize;
+                    let rd = |a: usize| memory.bytes().get(a..a + 8).map_or(0, |b| u64::from_le_bytes(b.try_into().unwrap()));
+                    let rsp = rd(ctx_addr + gex_abi::ctx::X + 4 * 8);
+                    let ret = rd(rsp as usize);
+                    eprintln!("[ftab] lookup {addr:#x} -> index {} (install {}, module {}, base {}, {:?}) ret {ret:#x} holds {ty:?}", f.idx, f.install, inst.module, inst.base, inst.tier);
                     gex_kernel::ftab::fcache_write(memory.bytes(), addr, f.idx);
                     return Ok(f.idx as i32);
                 }
@@ -635,7 +640,8 @@ fn job(caller: &mut Caller<'_, HostState>, sh: &Arc<Mutex<JitShared>>, memory: &
     let (exec, ro, wx): (Vec<(u64, u64)>, Vec<(u64, u64)>, Vec<(u64, u64)>) = caller.data_mut().with_kernel(|k| {
         let v = k.mem.vmas();
         (
-            v.iter().filter(|v| v.prot & prot::EXEC != 0).map(|v| (v.start, v.end)).collect(),
+            // A library a stateful provider replaced is not code to translate (`libprov::replaced`).
+            v.iter().filter(|v| v.prot & prot::EXEC != 0 && !crate::libprov::replaced(v.start, v.end)).map(|v| (v.start, v.end)).collect(),
             v.iter().filter(|v| v.prot & prot::WRITE == 0 && v.prot & prot::READ != 0).map(|v| (v.start, v.end)).collect(),
             v.iter().filter(|v| v.prot & prot::EXEC != 0 && v.prot & prot::WRITE != 0).map(|v| (v.start, v.end)).collect(),
         )

shims/native/src/libprov.rs

@@ -294,6 +294,19 @@ impl std::error::Error for Throw {}
 /// [`install`]): musl's `clone` sets the child's PC to it.
 static THREAD_ENTRY: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
 
+/// The text of every library a stateful provider replaced, as mapped: code
+/// that never runs, so no tier translates it ([`replaced`]).
+static REPLACED: Mutex<Vec<(u64, u64)>> = Mutex::new(Vec::new());
+
+fn replaced_add(lo: u64, hi: u64) {
+    REPLACED.lock().unwrap_or_else(|e| e.into_inner()).push((lo, hi));
+}
+
+/// Whether `[start, end)` overlaps a replaced library's text.
+pub(crate) fn replaced(start: u64, end: u64) -> bool {
+    REPLACED.lock().unwrap_or_else(|e| e.into_inner()).iter().any(|&(lo, hi)| start < hi && end > lo)
+}
+
 /// The host functions stateful providers import.
 pub(crate) fn register(linker: &mut Linker<HostState>) -> Result<()> {
     // A wasm64 provider's pointers and table indices are i64 (docs/unity-
@@ -520,6 +533,11 @@ pub(crate) fn on_linux_maps(caller: &mut Caller<'_, HostState>, mapped: &[gex_ke
             (Arc::clone(&jit.shared), jit.engine.clone())
         };
         let memory = caller.data().memory.clone().ok_or_else(|| anyhow!("guest memory unavailable"))?;
+        // None of the replaced library's code may run, so the function JIT must not translate it
+        // either: following a PLT's folded GOT load into it would publish the library's own code
+        // over the provider's bindings (Muck under --fjit then ran Boehm's GC_exclude_static_roots
+        // forever).
+        replaced_add(lo.wrapping_add(bias), hi.wrapping_add(bias));
         for lp in libs {
             CTORS.lock().unwrap_or_else(|e| e.into_inner()).extend(ctors.iter().filter(|&&v| v >= lo && v < hi).map(|&v| v.wrapping_add(bias)));
             let n = install(lp, caller.data_mut(), &shared, &engine, &memory, &exports, bias, 0)?;

tests/unity-dotnet/embed/run.sh

@@ -23,6 +23,11 @@ cd "$T"
 GEX_WASM_DLLS=libmonobdwgc-2.0.so="$OUT/provider/mono.wasm" timeout 300 "$GEXC" run --mem64 \
 	--env GEXMONO_ASSEMBLIES="$T/lib" ./embed ./libmonobdwgc-2.0.so "$T/asm/t.dll" | tee out.txt
 if ! { grep -q "^Compute(50) = 50002$" out.txt && grep -q "^Hello = hello gex from t ptr=8$" out.txt && grep -q "^Guest = twice 42; concat guest strings; mix 323.25; scale 2,4,6; big 11,25,52,3; caught from the guest$" out.txt && grep -q "^Back = callback 806.50; unmanaged 1015$" out.txt && grep -q "^held across collections: 64 of 64 intact$" out.txt && grep -q "^worker thread: 0 bad of 640$" out.txt && grep -q "^gc ok$" out.txt; }; then echo FAIL; exit 1; fi
+# The same under the function JIT, which must leave the replaced library's own code alone: it once
+# followed the PLT into the stub ("x86 Mono stub ... called"), as it followed Muck's into Boehm's GC.
+GEX_WASM_DLLS=libmonobdwgc-2.0.so="$OUT/provider/mono.wasm" timeout 300 "$GEXC" run --mem64 --fjit \
+	--env GEXMONO_ASSEMBLIES="$T/lib" ./embed ./libmonobdwgc-2.0.so "$T/asm/t.dll" > fjit.txt 2>&1 || true
+if ! grep -q "^gc ok$" fjit.txt; then echo "FAIL (--fjit)"; tail -3 fjit.txt; exit 1; fi
 # The control: without scanning the guest's frames, objects held only there die.
 GEX_WASM_DLLS=libmonobdwgc-2.0.so="$OUT/provider/mono.wasm" timeout 300 "$GEXC" run --mem64 --env GEXP_NO_GUEST_ROOTS=1 \
 	--env GEXMONO_ASSEMBLIES="$T/lib" ./embed ./libmonobdwgc-2.0.so "$T/asm/t.dll" > control.txt 2>&1 || true