browser cgen: a unit on code that is no longer mapped is dead, whatever its bytes

on_retire marked a unit's page patched only when the bytes differed from the
build's. An unmapped library's bytes stay until the next mapping overwrites
them, and that mapping retires nothing, so the unit stayed live. HK with
system-library units (hp19, hp22): libdbus-1 is unmapped and libxcb lands at
the same base (0x1a78e000), both with units, and the game exits with status 255
right after Mono loads (~16 s). Pages of a retired range that are not mapped
executable (gone_code) now get the patched bit unconditionally, and their
function-tier entries (not providers) leave the table, so the next object's
unit clearing the bits cannot revive them.

hp23 (UnityPlayer + 15 system-library units): menu, first frame 43 s, 100
presents 73 s, 40.1 presents/s at 212 s; hp24 (same shim, UnityPlayer only):
53 s, 83 s, 41.1 presents/s. The native nt_cgen::on_retire has the same hash-only test.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_1896c5b302a54696aaa92c5f4047f20b

shims/browser/src/cgen.rs

@@ -1720,11 +1720,27 @@ pub(crate) fn leaves() -> u64 {
 /// Pages that retired: any an installed unit covers whose bytes are no longer
 /// the ones it was built from gets its bit in `gex_abi::PATCHED_PAGES`, and
 /// compiled code stops calling into it (the native `on_retire`).
-pub(crate) fn on_retire(ranges: &[(u64, u64)]) {
-    on_retire_pages(ranges);
+///
+/// `gone` (within `ranges`): pages no longer mapped executable. A unit there
+/// is dead whatever the bytes still hash to -- an unmapped object's bytes
+/// stay until the next mapping overwrites them, and that mapping retires
+/// nothing. Its entries leave the table now: otherwise the object that next
+/// lands there (HK: libxcb where libdbus had been, both with units) clears
+/// the bits for its own unit and every address it does not define runs the
+/// old object's functions.
+pub(crate) fn on_retire(ranges: &[(u64, u64)], gone: &[(u64, u64)]) {
+    on_retire_pages(ranges, gone);
     // After the bits: a lookup once the probe cache is cleared sees them.
     let memory = crate::guest_bytes();
     ftab(|m| {
+        for &(start, end) in gone {
+            for (a, f) in m.range(start, end).collect::<Vec<_>>() {
+                let i = m.install(f.install);
+                if i.tier == Tier::Unit && i.module < PROVIDER {
+                    m.retire(a);
+                }
+            }
+        }
         for &(start, end) in ranges {
             for (a, _) in m.range(start, end).collect::<Vec<_>>() {
                 gex_kernel::ftab::fcache_clear(memory, a);
@@ -1733,14 +1749,16 @@ pub(crate) fn on_retire(ranges: &[(u64, u64)]) {
     });
 }
 
-fn on_retire_pages(ranges: &[(u64, u64)]) {
+fn on_retire_pages(ranges: &[(u64, u64)], gone: &[(u64, u64)]) {
     let guard = lock(&UNIT_PAGES);
     let Some(pages) = guard.as_ref() else { return };
     let memory = crate::guest_bytes();
     for &(start, end) in ranges {
         for page in start / gex_abi::DISPATCH_PAGE..end.div_ceil(gex_abi::DISPATCH_PAGE) {
             let Some(&built) = pages.get(&page) else { continue };
-            if page_hash(memory, page) != built {
+            let at = page * gex_abi::DISPATCH_PAGE;
+            let unmapped = gone.iter().any(|&(s, e)| s <= at && at < e);
+            if unmapped || page_hash(memory, page) != built {
                 let off = (gex_abi::PATCHED_PAGES + page / 8) as usize;
                 if let Some(b) = memory.get_mut(off) {
                     *b |= 1 << (page % 8);

shims/browser/src/lib.rs

@@ -11944,13 +11944,36 @@ fn retire_unmapped_code() {
     with_host(|h| {
         let retired = std::mem::take(&mut h.kernel.retired_code);
         if !retired.is_empty() {
-            crate::cgen::on_retire(&retired);
+            crate::cgen::on_retire(&retired, &gone_code(h, &retired));
             crate::fjit::on_retire(&retired);
             retire_code(h, &retired);
         }
     });
 }
 
+/// The pages of `ranges` no longer mapped executable (an unmapped object,
+/// code made non-executable), merged into ranges: `cgen::on_retire` drops
+/// the units there whatever the bytes are.
+pub(crate) fn gone_code(h: &Host, ranges: &[(u64, u64)]) -> Vec<(u64, u64)> {
+    use gex_kernel::mem::prot;
+    let page = gex_abi::DISPATCH_PAGE;
+    let mut out: Vec<(u64, u64)> = Vec::new();
+    for &(start, end) in ranges {
+        let mut a = start / page * page;
+        while a < end {
+            let exec = h.kernel.mem.vma_at(a).is_some_and(|v| v.prot & prot::EXEC != 0);
+            if !exec {
+                match out.last_mut() {
+                    Some(last) if last.1 == a => last.1 = a + page,
+                    _ => out.push((a, a + page)),
+                }
+            }
+            a += page;
+        }
+    }
+    out
+}
+
 /// Forget every translation in `ranges` (see `retire_unmapped_code`). The
 /// old slots are left behind, unreachable.
 pub(crate) fn retire_code(h: &mut Host, ranges: &[(u64, u64)]) {

shims/browser/src/nt.rs

@@ -1095,7 +1095,7 @@ mod imp {
             h.kernel.new_code.clear();
             if !retired.is_empty() {
                 note_retire(&retired);
-                crate::cgen::on_retire(&retired);
+                crate::cgen::on_retire(&retired, &crate::gone_code(h, &retired));
                 if !NO_RETIRE.load(Ordering::Relaxed) {
                     retire(h, &retired);
                 }