shim: execve reuses the dead arena below the old top

A program exec'd from a fork of a large parent placed every mapping above
the parent's high-water mark: node's child_process children (a 1.3 GiB
node parent) ended at 2027 MiB with ~20k maps in the page and ran 61 s
instead of ~30 s, and under run-node the same child hit the 2 GiB arena
ceiling.

do_exec now donates the old address space to the new map: the old map's
arena spans plus, in a fork child, everything inherited below ADOPTED_TOP,
minus every region the runtime owns (memmap::guest_only; nothing when the
region table has overflowed). MemoryMap::donate_arena marks it dirty, so
an anonymous mmap zeroes only the pages that are not already zero.

Since the new program's code can now land on the old program's addresses,
everything keyed by a code address is forgotten at exec (cgen::on_exec):
FuncMap::forget_all retires every address and marks every install
DEAD_MODULE, and the page's table catch-up skips those (a thread of the
new program otherwise instantiated the old program's replaced-library
provider with the wrong stack type: LinkError on __stack_pointer); the
providers' tables in libprov and elfprov are cleared, and fjit retires the
old code ranges.

run-node (gooner, correctness): node spawnSync node and execSync via sh
then node both ok; children 134 maps, top 896 MiB and 1088 MiB, with
replaced libz working in the child. Before this change the same test
stalled at the arena ceiling. gex-kernel 189 tests, item3 PASS,
browser-output-order a64 ok.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_781bad2ec05143db9067c86b4b2678e2

crates/gex-kernel/src/ftab.rs

@@ -34,6 +34,10 @@ pub enum Tier {
     Region,
 }
 
+/// An install's module once its program is gone ([`FuncMap::forget_all`]):
+/// a table catching up skips it.
+pub const DEAD_MODULE: u32 = u32::MAX;
+
 /// One module's functions in the table: `len` slots from `base`, filled by
 /// the module's element segment when a thread instantiates it. `module` is
 /// the host's handle for it (an index into its own list of modules).
@@ -195,6 +199,27 @@ impl FuncMap {
         Some(f.idx)
     }
 
+    /// `execve`: nothing installed so far belongs to the new program. Every
+    /// address is retired (returned, for the caller's caches) and every
+    /// install is marked [`DEAD_MODULE`], so a table that catches up later --
+    /// a Worker for a thread the new program starts -- skips it rather than
+    /// instantiate the old program's modules (a replaced library's provider
+    /// would be linked as something else, its data where the new program's
+    /// now is).
+    pub fn forget_all(&mut self) -> Vec<(u64, u32)> {
+        let gone: Vec<u64> = self.by_addr.keys().copied().collect();
+        let mut out = Vec::with_capacity(gone.len());
+        for a in gone {
+            if let Some(i) = self.retire(a) {
+                out.push((a, i));
+            }
+        }
+        for i in &mut self.installs {
+            i.module = DEAD_MODULE;
+        }
+        out
+    }
+
     /// Retire every function whose entry lies in `[start, end)`.
     pub fn retire_range(&mut self, start: u64, end: u64) -> Vec<(u64, u32)> {
         let gone: Vec<u64> = self.by_addr.range(start..end).map(|(a, _)| *a).collect();
@@ -389,6 +414,21 @@ fn leb(b: &[u8]) -> Option<(u32, usize)> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn forget_all_retires_every_address_and_kills_every_install() {
+        let mut m = FuncMap::new();
+        m.publish(3, 7, Tier::Unit, &[(0x1000, 0), (0x2000, 2)]);
+        m.publish(1, 0x8000_0001, Tier::Unit, &[(0xffff_f000_0000, 0)]);
+        let gone = m.forget_all();
+        assert_eq!(gone.len(), 3);
+        assert!(m.get(0x1000).is_none() && m.get(0xffff_f000_0000).is_none());
+        assert!(m.installs().iter().all(|i| i.module == DEAD_MODULE));
+        // The next program's installs are live, past the old indices.
+        let n = m.publish(2, 9, Tier::Unit, &[(0x1000, 1)]);
+        assert_eq!(m.install(n).module, 9);
+        assert_eq!(m.get(0x1000).unwrap().idx, m.install(n).base + 1);
+    }
+
     #[test]
     fn publish_maps_and_retires() {
         let mut m = FuncMap::new();

crates/gex-kernel/src/mem.rs

@@ -316,6 +316,44 @@ impl MemoryMap {
         self.arena.clear();
     }
 
+    /// The arena spans this map has been handed (sorted, disjoint).
+    pub fn arena_spans(&self) -> &[(u64, u64)] {
+        &self.arena
+    }
+
+    /// Give this map arena that is backed already and holds nothing anyone
+    /// uses: an `execve`'s dead address space (the old image's mappings, and a
+    /// forked child's inherited copy of its parent's). Without it the new
+    /// program's mappings all go above the old ones, and a program exec'd from
+    /// a fork of a large parent (node's child_process: a 1.3 GiB parent) ran
+    /// against the 2 GiB ceiling at twice the time. Nothing here is known to be
+    /// zero, so an anonymous `mmap` zeroes what it hands out of these, page by
+    /// page, as it would any reused hole.
+    pub fn donate_arena(&mut self, spans: &[(u64, u64)]) {
+        for &(lo, hi) in spans {
+            let (lo, hi) = (page_align_up(lo), page_align_down(hi));
+            if lo >= hi {
+                continue;
+            }
+            self.mark_dirty(lo, hi);
+            let at = self.arena.partition_point(|&(s, _)| s < lo);
+            // Disjoint from what is there already, or not given at all.
+            if at > 0 && self.arena[at - 1].1 > lo {
+                continue;
+            }
+            if at < self.arena.len() && self.arena[at].0 < hi {
+                continue;
+            }
+            self.arena.insert(at, (lo, hi));
+            if at + 1 < self.arena.len() && self.arena[at + 1].0 == hi {
+                self.arena[at].1 = self.arena.remove(at + 1).1;
+            }
+            if at > 0 && self.arena[at - 1].1 == lo {
+                self.arena[at - 1].1 = self.arena.remove(at).1;
+            }
+        }
+    }
+
     /// The address one past the highest byte in use, which is how much memory
     /// a process actually needs.
     pub fn high_water(&self) -> u64 {
@@ -1039,6 +1077,39 @@ mod tests {
         }
     }
 
+    /// An exec's dead address space, donated back, is used before the arena
+    /// grows again, and is never taken for zero.
+    #[test]
+    fn donated_arena_is_used_before_growing() {
+        static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0x5000_0000);
+        fn take(len: u64) -> u64 {
+            NEXT.fetch_add(len, std::sync::atomic::Ordering::SeqCst)
+        }
+        fn back(end: u64) -> Option<u64> {
+            Some(NEXT.fetch_max(end, std::sync::atomic::Ordering::SeqCst))
+        }
+        NEXT.store(0x5000_0000, std::sync::atomic::Ordering::SeqCst);
+
+        let mut m = MemoryMap::new(0x40_1000);
+        m.grows_on_demand(take, back);
+        m.mark_clean(0x1000_0000, 0x1800_0000);
+        // Two touching spans merge; an overlapping one and an unaligned
+        // sliver are ignored.
+        m.donate_arena(&[
+            (0x1000_0000, 0x1400_0000),
+            (0x1400_0000, 0x1800_0000),
+            (0x1200_0000, 0x1300_0000),
+            (0x2000_0100, 0x2000_0200),
+        ]);
+        assert_eq!(m.arena_spans(), &[(0x1000_0000, 0x1800_0000)]);
+        // Donated memory held something: none of it is known zero.
+        assert!(m.clean_ranges().iter().all(|&(s, e)| e <= 0x1000_0000 || s >= 0x1800_0000));
+
+        let got = m.mmap(0, 64 << 20, 0, 0, "heap").unwrap() as u64;
+        assert!((0x1000_0000..0x1800_0000).contains(&got), "mapped at {got:#x}");
+        assert_eq!(NEXT.load(std::sync::atomic::Ordering::SeqCst), 0x5000_0000);
+    }
+
     /// `MAP_FIXED_NOREPLACE` refuses any existing mapping, reservations
     /// included, and succeeds on free space.
     #[test]

shims/browser/js/gex.mjs

@@ -3263,7 +3263,11 @@ export class Gex {
     while (st.have < len) {
       const ev = BigInt.asUintN(64, BigInt(e.gex_ftab_event(st.have)));
       const kind = Number(ev >> 32n), v = Number(ev & 0xffffffffn);
-      if (kind === 1) this.#cgenInstall(e.gex_ftab_install_unit(v) >>> 0, e.gex_ftab_install_base(v) >>> 0);
+      // A module of a program an execve replaced (`FuncMap::forget_all`) is skipped.
+      if (kind === 1) {
+        const u = e.gex_ftab_install_unit(v) >>> 0;
+        if (u !== 0xffffffff) this.#cgenInstall(u, e.gex_ftab_install_base(v) >>> 0);
+      }
       else if (kind === 2) st.ftab.set(tIdx(v), st.resolve);
       st.have++;
       e.gex_cgen_have(st.have);

shims/browser/src/cgen.rs

@@ -1764,6 +1764,24 @@ pub(crate) fn on_retire(ranges: &[(u64, u64)], gone: &[(u64, u64)]) {
     });
 }
 
+/// `execve`: the old program's code is all gone, whatever reuses its
+/// addresses next. Before exec reused the dead arena (`do_exec`), nothing new
+/// could land there and stale entries were unreachable; now every table keyed
+/// by a code address forgets them -- the function table (every address, every
+/// install: see `FuncMap::forget_all`), the providers' own bookkeeping, and
+/// the unit pages.
+pub(crate) fn on_exec(old_code: &[(u64, u64)]) {
+    on_retire_pages(old_code, old_code);
+    libprov::on_exec(old_code);
+    elfprov::on_exec();
+    let memory = crate::guest_bytes();
+    ftab(|m| {
+        for (a, _) in m.forget_all() {
+            gex_kernel::ftab::fcache_clear(memory, a);
+        }
+    });
+}
+
 fn on_retire_pages(ranges: &[(u64, u64)], gone: &[(u64, u64)]) {
     let guard = lock(&UNIT_PAGES);
     let Some(pages) = guard.as_ref() else { return };

shims/browser/src/cgen/elfprov.rs

@@ -199,6 +199,16 @@ fn unbind_overlapped(m: &gex_kernel::CodeMap, trace: bool) {
     }
 }
 
+/// `execve`: the old program's bindings go with it (see `cgen::on_exec`).
+/// Left, a miss at one of their addresses -- which the new program may now
+/// hold other code at -- was re-bridged to the old provider.
+pub(crate) fn on_exec() {
+    lock(&BOUND).clear();
+    lock(&QUEUE).clear();
+    lock(&REBRIDGE).clear();
+    lock(&REBRIDGED).clear();
+}
+
 /// The next provider-only unit for JS to serve (`#cgenProviders`), or -1.
 /// Asked before a Worker's guest first runs, so a forked child binds what
 /// it inherited.

shims/browser/src/cgen/libprov.rs

@@ -915,6 +915,19 @@ static ELF_UNITS: Mutex<Vec<String>> = Mutex::new(Vec::new());
 /// executable segments).
 static ELF_PLACED: Mutex<Vec<(String, u64, (u64, u64))>> = Mutex::new(Vec::new());
 
+/// `execve`: every replaced library of the old program is gone, and its
+/// addresses may hold the new program's code (see `cgen::on_exec`).
+pub(crate) fn on_exec(old_code: &[(u64, u64)]) {
+    lock(&LIBS).clear();
+    lock(&REPLACED).clear();
+    THREAD_ENTRY.store(0, Ordering::Release);
+    if let Some(m) = lock(&MISSING).as_mut() {
+        m.clear();
+    }
+    lock(&RESERVES).clear();
+    forget_gone(old_code);
+}
+
 /// Code no longer mapped executable (`cgen::on_retire`'s `gone`): an object
 /// whose code was there is placed again if it is mapped again (ALSA's
 /// dlopen/dlclose cycles reload libasound at the same base), since its unit

shims/browser/src/lib.rs

@@ -7895,10 +7895,39 @@ fn do_exec(h: &mut Host, path: &str, argv: &[String], envp: &[String]) -> u64 {
     // libraries at the same addresses -- finding the old program's *relocated*
     // copy, whose `.got.plt` sends a call through the PLT into unmapped memory.
     let old: Vec<(u64, u64)> = h.kernel.mem.vmas().iter().map(|v| (v.start, v.end)).collect();
+    let old_code: Vec<(u64, u64)> = h
+        .kernel
+        .mem
+        .vmas()
+        .iter()
+        .filter(|v| v.prot & gex_kernel::mem::prot::EXEC != 0)
+        .map(|v| (v.start, v.end))
+        .collect();
+    // The dead arena the new program may reuse: the old map's own spans, and
+    // in a forked child everything it inherited below `ADOPTED_TOP` (its shim
+    // never allocates there; see `arena_take`) -- less anything the runtime
+    // owns, by the memory map's own record.
+    let mut dead: Vec<(u64, u64)> = h.kernel.mem.arena_spans().to_vec();
+    let inherited = ADOPTED_TOP.load(std::sync::atomic::Ordering::Acquire);
+    if inherited > gex_abi::ARENA_BOTTOM {
+        dead.push((gex_abi::ARENA_BOTTOM, inherited));
+    }
+    let dead: Option<Vec<(u64, u64)>> = dead
+        .iter()
+        .map(|&(lo, hi)| memmap::guest_only(lo.max(gex_abi::ARENA_BOTTOM), hi.min(gex_abi::MMAP_TOP)))
+        .collect::<Option<Vec<_>>>()
+        .map(|v| v.into_iter().flatten().collect());
     h.kernel.mem =
         gex_kernel::mem::MemoryMap::new(program.as_ref().map(|p| p.exe.brk).unwrap_or(0));
     // A fresh map for a fresh image, and it grows the same way.
     h.kernel.mem.grows_on_demand(arena_grow, arena_back);
+    if let Some(dead) = dead {
+        h.kernel.mem.donate_arena(&dead);
+    }
+    // Everything keyed by a code address of the old program forgets it, now
+    // that the new one may map code there.
+    cgen::on_exec(&old_code);
+    fjit::on_retire(&old_code);
     {
         // The live memory size, not `guest_bytes().len()` (which is the whole
         // 4 GiB nameable range): the fill must stay inside what actually exists.

shims/browser/src/memmap.rs

@@ -223,6 +223,39 @@ pub fn owner_of(addr: u64) -> String {
     "nobody".into()
 }
 
+/// The parts of `[start, end)` that only the guest owns (no shim heap,
+/// stack, thread-local block, foreign growth or fixed runtime region overlaps
+/// them), in order. `None` when the table has overflowed and so cannot say.
+pub fn guest_only(start: u64, end: u64) -> Option<Vec<(u64, u64)>> {
+    let _h = Held::take();
+    if T.full_said.load(Ordering::Relaxed) != 0 {
+        return None;
+    }
+    let n = T.len.load(Ordering::Relaxed) as usize;
+    let mut out = vec![(start, end)];
+    for i in 0..n {
+        if Kind::from(T.tag[i].load(Ordering::Relaxed) >> 32) == Kind::Guest {
+            continue;
+        }
+        let (s, e) = (T.start[i].load(Ordering::Relaxed), T.end[i].load(Ordering::Relaxed));
+        let mut next = Vec::with_capacity(out.len() + 1);
+        for (a, b) in out {
+            if e <= a || s >= b {
+                next.push((a, b));
+                continue;
+            }
+            if a < s {
+                next.push((a, s));
+            }
+            if e < b {
+                next.push((e, b));
+            }
+        }
+        out = next;
+    }
+    Some(out)
+}
+
 static FIXED: AtomicU32 = AtomicU32::new(0);
 
 /// The fixed layout, once per process (every Worker calls it).