translation pipeline: no page-table clones per miss, slab module store, word-wise content key

Three per-translation costs from node's cold start-up profile (run-node on
edger, 39.8 s main thread):
- the page table (a BTreeMap of every numbered page) was cloned five times
  per dispatch miss and dropped as often (about 1 s); it now moves through
  translate_runtime, the module builder and back;
- the kernel's module store allocated each module on its own, and dlmalloc
  grew the kernel's shared memory every few modules (1.9 s in memory.grow);
  module bytes now go into 16 MiB chunks;
- moduleContentKey hashed every installed module a byte at a time in JS
  (0.65 s per Worker); it and the shim's content_key now hash a word at a
  time (same function on both sides, checked against each other).
Static output is byte-identical (xz, python3.13, nvim).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_781bad2ec05143db9067c86b4b2678e2

crates/gex-emit/src/module.rs

@@ -1245,7 +1245,7 @@ impl ModuleBuilder {
             table_slots,
             dir_entries,
             covered,
-            page_table: self.page_table.clone(),
+            page_table: std::mem::take(&mut self.page_table),
             module_id,
             n_blocks,
             exported_blocks,

crates/gex-xlate/src/lib.rs

@@ -1029,7 +1029,7 @@ fn emit_modules(
             text_base,
             0,
             Linkage::Shared,
-            pages.clone(),
+            std::mem::take(pages),
             shared_memory,
         );
         builder.set_demand_fill(demand_fill);

shims/browser/js/gex.mjs

@@ -188,12 +188,26 @@ function storeAppend(sab, bytes) {
 }
 function moduleContentKey(bytes) {
   if (!(bytes instanceof Uint8Array)) return null;
+  // A little-endian word at a time, then the tail bytes (the shim's
+  // `content_key` is the same function): a byte at a time was 0.65 s of a
+  // node start-up per Worker.
   let a = 0x811c9dc5;
   let b = 0x9e3779b9;
-  for (let i = 0; i < bytes.length; i++) {
+  const n = bytes.length >>> 2;
+  const words = (bytes.byteOffset & 3) === 0 && !(bytes.buffer instanceof SharedArrayBuffer)
+    ? new Uint32Array(bytes.buffer, bytes.byteOffset, n)
+    : null;
+  for (let i = 0; i < n; i++) {
+    const x = words ? words[i]
+      : (bytes[4 * i] | (bytes[4 * i + 1] << 8) | (bytes[4 * i + 2] << 16) | (bytes[4 * i + 3] << 24)) >>> 0;
+    a = Math.imul(a ^ x, 0x01000193);
+    b = Math.imul(b ^ ((x + i) | 0), 0x85ebca6b);
+    b ^= b >>> 13;
+  }
+  for (let i = 4 * n; i < bytes.length; i++) {
     const x = bytes[i];
     a = Math.imul(a ^ x, 0x01000193);
-    b = Math.imul(b ^ (x + i), 0x85ebca6b);
+    b = Math.imul(b ^ (x + (n + i - 4 * n)), 0x85ebca6b);
     b ^= b >>> 13;
   }
   return `${bytes.length}:${a >>> 0}:${b >>> 0}`;

shims/browser/src/lib.rs

@@ -11771,12 +11771,15 @@ pub extern "C" fn gex_dispatch_miss(_ctx: i32, pc: i64) -> i32 {
         );
         }
         phase(11);
-    let (unit, stats) = gex_xlate::translate_runtime(
+    // The page table moves in and back out: cloning it (a BTreeMap of every
+    // page the process has numbered) on each miss, twice, was a second of a
+    // node start-up's 7,400 misses.
+    let (mut unit, stats) = gex_xlate::translate_runtime(
             guest_bytes(),
             pc,
             &|a| exec.iter().any(|(s, e)| a >= *s && a + gex_abi::INST_ALIGN <= *e),
             &|a| known.contains(&a),
-            h.jit.page_table.clone(),
+            std::mem::take(&mut h.jit.page_table),
             Default::default(),
             // A cap keeps one miss from translating a whole library at once;
             // the next miss picks up where this stopped.
@@ -11788,6 +11791,7 @@ pub extern "C" fn gex_dispatch_miss(_ctx: i32, pc: i64) -> i32 {
             &h.jit.intrinsics,
         );
         h.jit.translated = known;
+        h.jit.page_table = std::mem::take(&mut unit.page_table);
         if pc == 0x10f6d24 {
             let ok = exec.iter().any(|(s, e)| pc >= *s && pc + gex_abi::INST_ALIGN <= *e);
             let has_block = unit.covered.iter().any(|(p, _)| *p == 0x10f6);
@@ -11802,7 +11806,6 @@ pub extern "C" fn gex_dispatch_miss(_ctx: i32, pc: i64) -> i32 {
         xlate_add(0, stats.instructions as u64);
         xlate_add(1, stats.wasm_bytes as u64);
         xlate_add(2, 1);
-        h.jit.page_table = unit.page_table.clone();
         // Into the system-wide store, so a sibling installs it rather than
         // re-lifting. Published by what the unit *covers*, not what it allocated
         // (handed the allocation list, nine of ten units published nothing); the
@@ -13540,12 +13543,22 @@ static EXTERNAL_BYTES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU
 /// `moduleContentKey` in gex.mjs, as one u64 (`a` high, `b` low): what a
 /// module's store slot is checked against.
 fn content_key(bytes: &[u8]) -> u64 {
+    // A word at a time (little-endian), then the tail bytes: the same as
+    // gex.mjs `moduleContentKey`, which hashes every module a Worker installs
+    // and spent 0.65 s of a node start-up doing it a byte at a time.
     let mut a: u32 = 0x811c_9dc5;
     let mut b: u32 = 0x9e37_79b9;
-    for (i, &x) in bytes.iter().enumerate() {
-        a = (a ^ x as u32).wrapping_mul(0x0100_0193);
-        b = (b ^ (x as u32).wrapping_add(i as u32)).wrapping_mul(0x85eb_ca6b);
+    let mut step = |i: usize, x: u32| {
+        a = (a ^ x).wrapping_mul(0x0100_0193);
+        b = (b ^ x.wrapping_add(i as u32)).wrapping_mul(0x85eb_ca6b);
         b ^= b >> 13;
+    };
+    let words = bytes.len() / 4;
+    for i in 0..words {
+        step(i, u32::from_le_bytes(bytes[4 * i..4 * i + 4].try_into().unwrap()));
+    }
+    for (i, &x) in bytes[4 * words..].iter().enumerate() {
+        step(words + i, x as u32);
     }
     ((a as u64) << 32) | b as u64
 }

shims/kernel/src/lib.rs

@@ -624,7 +624,8 @@ pub extern "C" fn k_slot_count() -> u32 {
 /// compiled code (that is a JS object): the Worker feeds them to the same
 /// `pending` queue a fresh translation uses.
 struct SharedModule {
-    wasm: Vec<u8>,
+    /// Where its bytes are in [`ModStore::slab`]: chunk, offset, length.
+    at: (u32, u32, u32),
     /// How large the dispatch table must be. A fact about the *producing*
     /// process; the installer's own high-water mark understates it, and the
     /// install then throws.
@@ -655,6 +656,37 @@ struct ModStore {
     translated: BTreeSet<u64>,
     /// What has been stored, against [`MOD_BUDGET`].
     total: usize,
+    /// The modules' bytes, appended into chunks of [`SLAB_CHUNK`] (a larger
+    /// module gets a chunk of its own). One allocation per module grew the
+    /// kernel's shared memory every few modules -- dlmalloc asks `memory.grow`
+    /// for little more than each request -- and growing a shared memory is
+    /// slow: 1.9 s of a 40 s node start-up in the page's runtime.
+    slab: Vec<Vec<u8>>,
+}
+
+const SLAB_CHUNK: usize = 16 << 20;
+
+impl ModStore {
+    fn bytes(&self, sm: &SharedModule) -> &[u8] {
+        let (c, off, len) = sm.at;
+        self.slab
+            .get(c as usize)
+            .and_then(|v| v.get(off as usize..off as usize + len as usize))
+            .unwrap_or(&[])
+    }
+
+    /// Room for `len` bytes: the chunk and offset they go at (written by the caller).
+    fn reserve(&mut self, len: usize) -> (u32, u32) {
+        let fits = self.slab.last().is_some_and(|v| v.capacity() - v.len() >= len);
+        if !fits {
+            self.slab.push(Vec::with_capacity(len.max(SLAB_CHUNK)));
+        }
+        let c = self.slab.len() - 1;
+        let v = &mut self.slab[c];
+        let off = v.len();
+        v.resize(off + len, 0);
+        (c as u32, off as u32)
+    }
 }
 
 /// How much translated wasm to keep for sharing. The kernel memory is capped at
@@ -669,7 +701,7 @@ fn modules() -> &'static Mutex<ModStore> {
     MODS.get_or_init(|| {
         Mutex::new(ModStore {
             mods: vec![SharedModule {
-                wasm: Vec::new(),
+                at: (u32::MAX, 0, 0),
                 slots: 0,
                 dirs: Vec::new(),
             }],
@@ -678,6 +710,7 @@ fn modules() -> &'static Mutex<ModStore> {
             translating: BTreeMap::new(),
             translated: BTreeSet::new(),
             total: 0,
+            slab: Vec::new(),
         })
     })
 }
@@ -699,8 +732,9 @@ pub unsafe extern "C" fn k_mod_store(
     if len == 0 || m.total + len as usize > MOD_BUDGET {
         return 0;
     }
-    let mut wasm = vec![0u8; len as usize];
-    unsafe { gex_kmem::pull_checked(wasm.as_mut_ptr(), ptr, len) };
+    let (c, off) = m.reserve(len as usize);
+    let dst = m.slab[c as usize][off as usize..].as_mut_ptr();
+    unsafe { gex_kmem::pull_checked(dst, ptr, len) };
     let mut raw = vec![0u8; dirs_len as usize];
     if dirs_len > 0 {
         unsafe { gex_kmem::pull_checked(raw.as_mut_ptr(), dirs_ptr, dirs_len) };
@@ -712,7 +746,7 @@ pub unsafe extern "C" fn k_mod_store(
         dirs.push((page, base));
     }
     m.total += len as usize;
-    m.mods.push(SharedModule { wasm, slots, dirs });
+    m.mods.push(SharedModule { at: (c, off, len), slots, dirs });
     // One-based: zero is how every caller says "no module".
     m.mods.len() as u32
 }
@@ -835,7 +869,7 @@ pub extern "C" fn k_mod_len(id: u32) -> u32 {
         .unwrap_or_else(|e| e.into_inner())
         .mods
         .get((id as usize).wrapping_sub(1))
-        .map(|m| m.wasm.len() as u32)
+        .map(|m| m.at.2)
         .unwrap_or(0)
 }
 
@@ -860,11 +894,12 @@ pub unsafe extern "C" fn k_mod_read(id: u32, out: gex_kmem::PAddr, cap: u32) ->
     let Some(sm) = m.mods.get((id as usize).wrapping_sub(1)) else {
         return 0;
     };
-    if sm.wasm.len() > cap as usize {
+    let b = m.bytes(sm);
+    if b.len() > cap as usize {
         return 0;
     }
-    unsafe { gex_kmem::push_checked(out, sm.wasm.as_ptr(), sm.wasm.len() as u32) };
-    sm.wasm.len() as u32
+    unsafe { gex_kmem::push_checked(out, b.as_ptr(), b.len() as u32) };
+    b.len() as u32
 }
 
 /// And the `dirs`, in the same twelve-byte encoding they arrived in.