kernel, browser: lazy files (TmpFs::add_lazy), read from the page's server a megabyte at a time

A lazy file has a size and a URL; its bytes arrive when a guest reads them,
in 1 MiB chunks, each run of missing chunks one synchronous ranged request
(env.lazy_fetch: XMLHttpRequest with Range and Cache-Control: no-store, in the
Worker whose guest reads). Writes and truncation materialise the file first.
Exports k_fs_add_lazy (kernel) and gex_add_lazy (shim); gex.mjs takes
opts.lazy = [[path, size, url]...] after the bundle's files, skipping paths the
bundle has; nt-page's ?lazy=MANIFEST ({base, files: [[guestPath, size, rel]]}).
Every kernel instantiation passes lazy_fetch (the node tests a stub).

hg2 (Hollow Knight in the page, the 4.9 GiB install served lazily, 1,754
files): no failed reads, through the menu, New Game and the intro into the
opening sequence. The first try (hg1, no Cache-Control) failed every read past
a file's first chunk in Chromium (curl against vite: 206 for each range).
web/test kernel-shared, fs-shared, pipe-shared, procs-shared: ok.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_1896c5b302a54696aaa92c5f4047f20b

crates/gex-kernel/src/vfs/tmpfs.rs

@@ -30,6 +30,9 @@ fn note_capacity(before: usize, after: usize) {
 #[derive(Debug, Default)]
 pub struct FileNode {
     data: Mutex<Vec<u8>>,
+    /// Contents fetched on demand (`TmpFs::add_lazy`); `None` once the file
+    /// is an ordinary one (never, or since a write made it one).
+    lazy: Mutex<Option<Lazy>>,
     ino: u64,
     /// Modification time, in seconds. On the node, not the name, so it follows
     /// a rename or a link and an open descriptor's `fstat` sees it -- `gzip`
@@ -41,6 +44,7 @@ impl FileNode {
     fn new(data: Vec<u8>, ino: u64) -> FileNode {
         FileNode {
             data: Mutex::new(data),
+            lazy: Mutex::new(None),
             ino,
             mtime: AtomicI64::new(now()),
         }
@@ -48,6 +52,102 @@ impl FileNode {
     fn touch(&self) {
         self.mtime.store(now(), Ordering::Relaxed);
     }
+
+    /// The size, lazy or not.
+    fn size(&self) -> u64 {
+        if let Some(l) = self.lazy.lock().unwrap().as_ref() {
+            return l.size;
+        }
+        self.data.lock().unwrap().len() as u64
+    }
+
+    /// Make a lazy file an ordinary one (all of it fetched), before a write
+    /// or a truncation changes it.
+    fn materialize(&self) -> Result<(), Errno> {
+        let mut lazy = self.lazy.lock().unwrap();
+        let Some(l) = lazy.as_mut() else { return Ok(()) };
+        let mut all = vec![0u8; l.size as usize];
+        l.read(0, &mut all)?;
+        let mut data = self.data.lock().unwrap();
+        let was = data.capacity();
+        *data = all;
+        note_capacity(was, data.capacity());
+        TMPFS_BYTES.fetch_sub(l.held, core::sync::atomic::Ordering::Relaxed);
+        *lazy = None;
+        Ok(())
+    }
+}
+
+/// How a lazy file's bytes are fetched: `(url, offset, into)`, true when
+/// all of `into` was filled. Installed by the host (the browser's global
+/// kernel: a synchronous ranged request from the Worker that reads).
+pub type LazyFetch = fn(&str, u64, &mut [u8]) -> bool;
+static LAZY_FETCH: std::sync::OnceLock<LazyFetch> = std::sync::OnceLock::new();
+
+/// Set how lazy files are fetched (the first call wins).
+pub fn set_lazy_fetch(f: LazyFetch) {
+    let _ = LAZY_FETCH.set(f);
+}
+
+/// Lazy files are fetched and kept in pieces this big.
+const LAZY_CHUNK: u64 = 1 << 20;
+
+/// A file whose bytes live elsewhere (a URL) and arrive a chunk at a time,
+/// as they are read: a game's gigabytes of data served to a page.
+#[derive(Debug)]
+struct Lazy {
+    url: String,
+    size: u64,
+    chunks: std::collections::HashMap<u64, Box<[u8]>>,
+    held: u64,
+}
+
+impl Lazy {
+    fn read(&mut self, offset: u64, buf: &mut [u8]) -> Result<u64, Errno> {
+        if offset >= self.size || buf.is_empty() {
+            return Ok(0);
+        }
+        let end = (offset + buf.len() as u64).min(self.size);
+        let (first, last) = (offset / LAZY_CHUNK, (end - 1) / LAZY_CHUNK);
+        // Missing chunks in runs, one request a run.
+        let mut c = first;
+        while c <= last {
+            if self.chunks.contains_key(&c) {
+                c += 1;
+                continue;
+            }
+            let mut stop = c;
+            while stop < last && !self.chunks.contains_key(&(stop + 1)) {
+                stop += 1;
+            }
+            let lo = c * LAZY_CHUNK;
+            let hi = ((stop + 1) * LAZY_CHUNK).min(self.size);
+            let mut run = vec![0u8; (hi - lo) as usize];
+            let fetch = LAZY_FETCH.get().ok_or(Errno::EIO)?;
+            if !fetch(&self.url, lo, &mut run) {
+                return Err(Errno::EIO);
+            }
+            for k in c..=stop {
+                let a = ((k - c) * LAZY_CHUNK) as usize;
+                let b = (a + LAZY_CHUNK as usize).min(run.len());
+                self.chunks.insert(k, run[a..b].to_vec().into_boxed_slice());
+                self.held += (b - a) as u64;
+                TMPFS_BYTES.fetch_add((b - a) as u64, core::sync::atomic::Ordering::Relaxed);
+            }
+            c = stop + 1;
+        }
+        let mut at = offset;
+        while at < end {
+            let k = at / LAZY_CHUNK;
+            let chunk = &self.chunks[&k];
+            let from = (at - k * LAZY_CHUNK) as usize;
+            let n = ((end - at) as usize).min(chunk.len() - from);
+            let o = (at - offset) as usize;
+            buf[o..o + n].copy_from_slice(&chunk[from..from + n]);
+            at += n as u64;
+        }
+        Ok(end - offset)
+    }
 }
 
 /// Where a tmpfs gets the time for a file it writes. A host without
@@ -123,6 +223,24 @@ impl TmpFs {
         );
     }
 
+    /// A file of `size` bytes whose contents are fetched from `url` as they
+    /// are read (`set_lazy_fetch`). A path already present is left alone:
+    /// a bundled copy wins.
+    pub fn add_lazy(&self, path: &str, size: u64, url: &str) {
+        let path = path.trim_start_matches('/').to_string();
+        let mut inner = self.inner.lock().unwrap();
+        if inner.nodes.contains_key(&path) {
+            return;
+        }
+        Self::ensure_parents(&mut inner, &path);
+        let ino = inner.next_ino;
+        inner.next_ino += 1;
+        inner.inos.insert(path.clone(), ino);
+        let f = FileNode::new(Vec::new(), ino);
+        *f.lazy.lock().unwrap() = Some(Lazy { url: url.to_string(), size, chunks: Default::default(), held: 0 });
+        inner.nodes.insert(path, Node::File(Arc::new(f)));
+    }
+
     pub fn add_dir(&self, path: &str) {
         let path = path.trim_start_matches('/').to_string();
         let mut inner = self.inner.lock().unwrap();
@@ -225,6 +343,9 @@ impl FileHandle for TmpHandle {
         let Some(file) = &self.file else {
             return Err(Errno::EISDIR);
         };
+        if let Some(l) = file.lazy.lock().unwrap().as_mut() {
+            return l.read(offset, buf);
+        }
         let data = file.data.lock().unwrap();
         let start = (offset as usize).min(data.len());
         let n = buf.len().min(data.len() - start);
@@ -236,6 +357,7 @@ impl FileHandle for TmpHandle {
         let Some(file) = &self.file else {
             return Err(Errno::EISDIR);
         };
+        file.materialize()?;
         let mut data = file.data.lock().unwrap();
         let start = offset as usize;
         if start + buf.len() > data.len() {
@@ -254,7 +376,7 @@ impl FileHandle for TmpHandle {
         if let Some(file) = &self.file {
             return Ok(Metadata {
                 file_type: FileType::Regular,
-                size: file.data.lock().unwrap().len() as u64,
+                size: file.size(),
                 mode: 0o644,
                 ino: file.ino,
                 mtime: file.mtime.load(Ordering::Relaxed),
@@ -268,7 +390,7 @@ impl FileHandle for TmpHandle {
         match inner.nodes.get(&self.path) {
             Some(Node::File(f)) => Ok(Metadata {
                 file_type: FileType::Regular,
-                size: f.data.lock().unwrap().len() as u64,
+                size: f.size(),
                 mode: inner.modes.get(&self.path).copied().unwrap_or(0o644),
                 ino,
                 mtime: f.mtime.load(Ordering::Relaxed),
@@ -366,6 +488,7 @@ impl FileHandle for TmpHandle {
         let Some(file) = &self.file else {
             return Err(Errno::EINVAL);
         };
+        file.materialize()?;
         {
             let mut data = file.data.lock().unwrap();
             let was = data.capacity();
@@ -389,6 +512,9 @@ impl FileSystem for TmpFs {
                     }
                     if flags & oflags::O_TRUNC != 0 {
                         if let Some(Node::File(f)) = inner.nodes.get(&path) {
+                            if let Some(l) = f.lazy.lock().unwrap().take() {
+                                TMPFS_BYTES.fetch_sub(l.held, core::sync::atomic::Ordering::Relaxed);
+                            }
                             {
                                 let mut data = f.data.lock().unwrap();
                                 let was = data.capacity();
@@ -757,3 +883,44 @@ mod tests {
         assert_eq!(fs.read_link("real").err(), Some(Errno::EINVAL));
     }
 }
+
+#[cfg(test)]
+mod lazy_tests {
+    use super::*;
+
+    /// Byte `i` of every lazy file here is `i % 251`.
+    fn fake(url: &str, offset: u64, into: &mut [u8]) -> bool {
+        assert_eq!(url, "/data/f");
+        for (k, b) in into.iter_mut().enumerate() {
+            *b = ((offset + k as u64) % 251) as u8;
+        }
+        true
+    }
+
+    #[test]
+    fn lazy_reads_fetch_what_they_touch() {
+        set_lazy_fetch(fake);
+        let fs = TmpFs::new();
+        let size = 3 * LAZY_CHUNK + 123;
+        fs.add_lazy("/d/f", size, "/data/f");
+        let h = fs.open("d/f", 0, 0).unwrap();
+        assert_eq!(h.metadata().unwrap().size, size);
+        let mut buf = vec![0u8; 5000];
+        let at = LAZY_CHUNK - 2000;
+        assert_eq!(h.read_at(at, &mut buf).unwrap(), 5000);
+        assert!(buf.iter().enumerate().all(|(k, &b)| b == ((at + k as u64) % 251) as u8));
+        let mut tail = vec![0u8; 1000];
+        assert_eq!(h.read_at(size - 100, &mut tail).unwrap(), 100);
+        assert_eq!(tail[99], ((size - 1) % 251) as u8);
+        // A write makes it an ordinary file with all its bytes.
+        h.write_at(0, b"xy").unwrap();
+        let mut two = [0u8; 4];
+        h.read_at(0, &mut two).unwrap();
+        assert_eq!(&two, &[b'x', b'y', 2, 3]);
+        assert_eq!(h.metadata().unwrap().size, size);
+        // A bundled file of the same name wins.
+        fs.add_file("/e/g", b"abc".to_vec());
+        fs.add_lazy("/e/g", 99, "/data/f");
+        assert_eq!(fs.open("e/g", 0, 0).unwrap().metadata().unwrap().size, 3);
+    }
+}

shims/browser/js/gex.mjs

@@ -11,6 +11,8 @@ const GUEST_BYTES = 0xffff0000;
 // memory64 of `gex_abi::LIMIT64` (16 GiB). The shim's own addresses (its
 // stack, data and the low runtime words) stay below 4 GiB either way; only
 // the memory's type and the i64 globals of a wasm64 module differ.
+import { lazyFetch } from "./lazy-fetch.mjs";
+
 // Read when used, not at import: a Worker learns it from its first message.
 const m64 = () => !!globalThis.GEX_MEM64;
 const LIMIT64_BYTES = 16 * 2 ** 30;
@@ -1240,6 +1242,21 @@ export class Gex {
       free?.(dp, d.length);
     }
     if (partBuf) free?.(partBuf, PART);
+    // Files fetched as they are read (`gex_add_lazy`): [path, size, url],
+    // after the bundle's, which win.
+    for (const [path, size, url] of opts.lazy ?? []) {
+      if (!this.#shim.exports.gex_add_lazy) break;
+      if (files[path] !== undefined) continue;
+      const p = enc.encode(path);
+      const u = enc.encode(url);
+      const pp = this.#shim.exports.gex_alloc(p.length) >>> 0;
+      const up = this.#shim.exports.gex_alloc(u.length) >>> 0;
+      this.#bytes().set(p, pp);
+      this.#bytes().set(u, up);
+      this.#shim.exports.gex_add_lazy(pp, p.length, up, u.length, BigInt(size));
+      free?.(pp, p.length);
+      free?.(up, u.length);
+    }
     // Hard links (`web/src/tar.js`), after their targets: one inode with two
     // names, as the host had it. A shim that cannot link gets a copy.
     for (const [path, target] of Object.entries(opts.hard ?? {})) {
@@ -1687,7 +1704,7 @@ export class Gex {
       : await WebAssembly.compile(module);
     _k('instantiate');
     const inst = await WebAssembly.instantiate(mod, {
-      env: { memory, process: this.#memory },
+      env: { memory, process: this.#memory, lazy_fetch: lazyFetch(memory) },
     });
     _k('instantiated');
     // The stack, before anything is called. Each Worker's `__stack_pointer` is

shims/browser/js/lazy-fetch.mjs

@@ -0,0 +1,40 @@
+// The global kernel's `env.lazy_fetch` (shims/kernel: `TmpFs::add_lazy`):
+// a lazy file's bytes, fetched when a guest reads them. A synchronous ranged
+// request, made from the Worker whose guest is reading (the read blocks
+// there anyway); it fills `len` bytes at `dst` in the kernel's memory.
+// Where there is no XMLHttpRequest (Node), lazy files cannot be read.
+
+/** `lazy_fetch(url, urlLen, offset, dst, len)` against the kernel memory `memory`. */
+export function lazyFetch(memory) {
+  return (url, urlLen, offset, dst, len) => {
+    if (typeof XMLHttpRequest !== 'function') return -1;
+    url >>>= 0; urlLen >>>= 0; dst >>>= 0; len >>>= 0;
+    const from = Number(offset);
+    const href = new TextDecoder().decode(new Uint8Array(memory.buffer, url, urlLen).slice());
+    let last = 'no response';
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        const x = new XMLHttpRequest();
+        x.open('GET', href, false);
+        x.responseType = 'arraybuffer';
+        x.setRequestHeader('Range', `bytes=${from}-${from + len - 1}`);
+        // The HTTP cache can answer a range from a partial entry badly (a
+        // later range of a file whose first chunk it holds); the guest keeps
+        // its own copy of each chunk, so the cache buys nothing here.
+        x.setRequestHeader('Cache-Control', 'no-store');
+        x.send();
+        const got = new Uint8Array(x.response ?? new ArrayBuffer(0));
+        last = `status ${x.status}, ${got.length} bytes`;
+        // 206 is the range; a server without ranges sends the whole file (200).
+        const at = x.status === 206 ? 0 : x.status === 200 ? from : -1;
+        if (at < 0 || got.length < at + len) continue;
+        new Uint8Array(memory.buffer, dst, len).set(got.subarray(at, at + len));
+        return len;
+      } catch (e) {
+        last = String(e);
+      }
+    }
+    console.error(`[lazy] ${href} @${from}+${len}: failed (${last})`);
+    return -1;
+  };
+}

shims/browser/js/services.mjs

@@ -6,6 +6,7 @@
 //
 // It holds a process memory of its own only because the kernel imports one and
 // `k_pipe_write` reads bytes from there; it is a staging buffer, nothing more.
+import { lazyFetch } from './lazy-fetch.mjs';
 let k = null;
 let stage = null; // Uint8Array over this Worker's dummy process memory
 const STAGE_AT = 0x1000;
@@ -27,7 +28,7 @@ self.onmessage = async (e) => {
       shared: true,
     });
     const inst = await WebAssembly.instantiate(mod, {
-      env: { memory: msg.kernel.memory, process },
+      env: { memory: msg.kernel.memory, process, lazy_fetch: lazyFetch(msg.kernel.memory) },
     });
     k = inst.exports;
     // A stack of this Worker's own: each instance's `__stack_pointer` starts at

shims/browser/js/tty.mjs

@@ -12,6 +12,7 @@
 //
 // Without a `SharedArrayBuffer` there is no global kernel: `open` answers null
 // and the caller keeps its own line discipline.
+import { lazyFetch } from './lazy-fetch.mjs';
 
 /// Where the page writes bytes for the kernel to copy out of. The kernel
 /// imports the calling process's memory and addresses it with `memory.copy`;
@@ -43,7 +44,7 @@ export class PageTty {
     let inst;
     try {
       inst = await WebAssembly.instantiate(mod, {
-        env: { memory: spec.memory, process: scratch },
+        env: { memory: spec.memory, process: scratch, lazy_fetch: lazyFetch(spec.memory) },
       });
     } catch (e) {
       // A module that will not take a plain process memory is the shared build,

shims/browser/src/lib.rs

@@ -276,6 +276,7 @@ extern "C" {
     fn k_fs_link(from: gex_kernel::PAddr, from_len: u32, to: gex_kernel::PAddr, to_len: u32) -> i32;
     fn k_fs_mksocket(path: gex_kernel::PAddr, path_len: u32) -> i32;
     fn k_fs_add(path: gex_kernel::PAddr, path_len: u32, at: gex_kernel::PAddr, len: u32) -> i32;
+    fn k_fs_add_lazy(path: gex_kernel::PAddr, path_len: u32, url: gex_kernel::PAddr, url_len: u32, size: u64) -> i32;
     /// The system-wide map from a page of a mapped object to the dispatch slots
     /// its code was translated into; zero means nobody has. In the kernel so
     /// the second process to run `ls` finds what the first one translated.
@@ -4130,6 +4131,22 @@ pub extern "C" fn gex_add_file(path: u32, path_len: u32, data: u32, data_len: u3
     FILES.0.borrow_mut().push((name.to_string(), bytes));
 }
 
+/// A file the guest reads from `url` (same origin) a megabyte at a time, as
+/// it reads (`TmpFs::add_lazy` in the global kernel): data too big to
+/// bundle. Only with a global kernel; a path the bundle has is left alone.
+#[no_mangle]
+pub extern "C" fn gex_add_lazy(path: u32, path_len: u32, url: u32, url_len: u32, size: u64) {
+    // SAFETY: scalars naming ranges of this module's memory, which the
+    // kernel copies from.
+    if unsafe { k_present() } != 0 {
+        if unsafe { k_fs_populated() } == 0 {
+            unsafe { k_fs_add_lazy(gex_kernel::PAddr::from(path), path_len, gex_kernel::PAddr::from(url), url_len, size) };
+        }
+        return;
+    }
+    warn_line("gex_add_lazy: no global kernel; a lazy file is left out\n");
+}
+
 /// A bundled hard link: `path` becomes another name for the already-added
 /// `target`, one inode as in the bundle. glibc's loader knows a loaded
 /// library by device and inode, so two copies where the host had two names

shims/kernel/src/lib.rs

@@ -1898,11 +1898,27 @@ pub extern "C" fn k_checksum(at: gex_kmem::PAddr, len: u32) -> u32 {
 
 use gex_kernel::vfs::{remote, FileHandle, FileSystem};
 
+#[link(wasm_import_module = "env")]
+extern "C" {
+    /// Fill `len` bytes at `dst` (this module's memory) with the bytes of
+    /// the URL at `url` from `offset`: the host's synchronous ranged
+    /// request, in the Worker that reads. Returns the count, or negative.
+    fn lazy_fetch(url: u32, url_len: u32, offset: u64, dst: u32, len: u32) -> i32;
+}
+
+fn fetch_lazy(url: &str, offset: u64, into: &mut [u8]) -> bool {
+    // SAFETY: both ranges are in this module's memory and stay borrowed for
+    // the call only.
+    let n = unsafe { lazy_fetch(url.as_ptr() as u32, url.len() as u32, offset, into.as_mut_ptr() as u32, into.len() as u32) };
+    n >= 0 && n as usize == into.len()
+}
+
 /// The root every process sees.
 fn rootfs() -> &'static std::sync::Arc<gex_kernel::vfs::TmpFs> {
     note_panics();
     static ROOT: OnceLock<std::sync::Arc<gex_kernel::vfs::TmpFs>> = OnceLock::new();
     ROOT.get_or_init(|| {
+        gex_kernel::vfs::tmpfs::set_lazy_fetch(fetch_lazy);
         let fs = gex_kernel::vfs::TmpFs::new();
         // Directories a guest expects before anything is bundled: `open(O_CREAT)`
         // under a missing directory is `ENOENT`, not a directory being made.
@@ -2191,6 +2207,17 @@ pub extern "C" fn k_fs_add(path: gex_kmem::PAddr, path_len: u32, at: gex_kmem::P
     0
 }
 
+/// A file in the shared root whose bytes are fetched from `url` as they are
+/// read (`TmpFs::add_lazy`): a game's data served to a page without
+/// bundling it. A path the bundle already has is left alone.
+#[no_mangle]
+pub extern "C" fn k_fs_add_lazy(path: gex_kmem::PAddr, path_len: u32, url: gex_kmem::PAddr, url_len: u32, size: u64) -> i32 {
+    let p = path_of(path, path_len);
+    let u = path_of(url, url_len);
+    rootfs().add_lazy(&p, size, &format!("/{u}"));
+    0
+}
+
 /// Whether the shared root is filled yet, so only the first Worker unpacks the
 /// rootfs.
 #[no_mangle]

web/src/hostx.js

@@ -27,6 +27,7 @@
 // turn left to do it in.
 
 import { WlProxy, outputSize, REC, EV, SLOTS } from '../../shims/browser/js/wl-bridge.mjs';
+import { lazyFetch } from '../../shims/browser/js/lazy-fetch.mjs';
 
 /** Events taken from the page in one turn. */
 const MAX_INPUT = 64;
@@ -179,7 +180,7 @@ export class HostX {
         ? msg.kernel.module
         : await WebAssembly.compile(msg.kernel.module);
       const inst = await WebAssembly.instantiate(mod, {
-        env: { memory: msg.kernel.memory, process: this.#x.wasmMemory },
+        env: { memory: msg.kernel.memory, process: this.#x.wasmMemory, lazy_fetch: lazyFetch(msg.kernel.memory) },
       });
       // Every Worker in the kernel needs its own stack slice: wasm-ld gives
       // each instance the same `__stack_pointer`, so two Workers sharing one

web/src/nt-page.js

@@ -190,6 +190,15 @@ async function main() {
   // libraries with two sets of GL state. Symlinks are not followed here: a
   // bundle is made with -h.
   const hard = Object.fromEntries(Object.entries(tree.hard ?? {}).filter(([, t]) => files[t]));
+  // `lazy=URL`: a manifest, {"base": URL prefix, "files": [[guest path, size,
+  // path under base], ...]}, of files the guest reads from the server as it
+  // reads them instead of from the bundle (a game's data).
+  let lazy = [];
+  if (q.get('lazy')) {
+    const m = await (await fetch(q.get('lazy'))).json();
+    lazy = m.files.map(([path, size, rel]) => [path, size, m.base + rel.split('/').map(encodeURIComponent).join('/')]);
+    mark(`lazy: ${lazy.length} files, ${(lazy.reduce((a, f) => a + f[1], 0) / 1048576).toFixed(0)} MiB`);
+  }
   const exe = q.get('exe') || '/t/window.exe';
   const exeBytes = files[exe] ?? files[hard[exe]];
   if (!exeBytes) throw new Error(`${exe} is not in the bundle`);
@@ -219,6 +228,7 @@ async function main() {
     dirs: tree.dirs,
     modes: tree.modes,
     hard,
+    lazy,
     env,
     args,
     cwd: q.get('cwd') || exe.replace(/\/[^/]*$/, '') || '/',

web/src/persist-worker.js

@@ -8,6 +8,7 @@
 // Message out: { synced: id, wrote, removed, files, bytes } or { error }.
 
 import { openDb, txDone } from './persist-db.js';
+import { lazyFetch } from '../../shims/browser/js/lazy-fetch.mjs';
 
 const SCRATCH = 0x1000; // where paths and buffers go in this worker's process memory
 const CHUNK = 1 << 20;
@@ -190,7 +191,7 @@ self.onmessage = (e) => {
       for (const shared of [true, false]) {
         mem = new WebAssembly.Memory({ initial: 1, maximum: 65535, shared });
         try {
-          inst = await WebAssembly.instantiate(m.spec.module, { env: { memory: m.spec.memory, process: mem } });
+          inst = await WebAssembly.instantiate(m.spec.module, { env: { memory: m.spec.memory, process: mem, lazy_fetch: lazyFetch(m.spec.memory) } });
           break;
         } catch (e) {
           if (!shared) throw e;

web/src/runtime.js

@@ -374,6 +374,8 @@ export async function runProgram(
     // Hard links, path to target (`untar`'s `hard`): made as links, so the
     // two names are one inode as on the host.
     hard = {},
+    // Files fetched as they are read, [path, size, url] (`?lazy=`).
+    lazy = [],
     /// `KEY=VALUE` strings for pid 1. Empty takes the shim's default.
     env = [],
     args,
@@ -801,7 +803,7 @@ export async function runProgram(
       // out an address inside a pinned object and the image lands somewhere
       // its own translated code does not expect.
       opts: {
-        cwd, tty, uid, recompileObjects, heartbeat, link, dirs, modes, hard, env, pins: pinsFrom(closure?.pins, globalThis.GEX_PINS),
+        cwd, tty, uid, recompileObjects, heartbeat, link, dirs, modes, hard, lazy, env, pins: pinsFrom(closure?.pins, globalThis.GEX_PINS),
         // `?strace=1`, and every child inherits it: see `#opts` in gex.mjs.
         strace: globalThis.GEX_STRACE ? 1 : undefined,
         // `?stracepid=N`: narrate that one process, which is affordable.

web/test/fork-worker.mjs

@@ -47,6 +47,7 @@ const kernel = { module: kernelModule, memory: kernelMemory };
 // file makes the memory, so it can instantiate the module once and ask.
 const probe = (await WebAssembly.instantiate(kernelModule, {
   env: {
+    lazy_fetch: () => -1, // the kernel's lazy-file reads (none here)
     memory: kernelMemory,
     process: new WebAssembly.Memory({ initial: 0xffff0000 / 65536, maximum: 0xffff0000 / 65536 }),
   },

web/test/fs-shared.mjs

@@ -29,6 +29,7 @@ const probe = (await WebAssembly.instantiate(
   {
     env: {
       memory: kernel,
+      lazy_fetch: () => -1,
       process: new WebAssembly.Memory({
         initial: GUEST_PAGES, maximum: GUEST_PAGES, shared: true,
       }),
@@ -48,7 +49,7 @@ const WORKER = `
   });
   const k = (await WebAssembly.instantiate(
     await WebAssembly.compile(workerData.bytes),
-    { env: { memory: workerData.kernel, process: proc } },
+    { env: { memory: workerData.kernel, process: proc, lazy_fetch: () => -1 } },
   )).exports;
   // This Worker's own stack. A wasm global belongs to an instance, but wasm-ld
   // starts every one at the same shared address.

web/test/hostx.mjs

@@ -79,7 +79,7 @@ let probe = null;
 if (args.includes('--probe')) {
   const mem = new WebAssembly.Memory({ initial: 2, maximum: 4, shared: true });
   const inst = await WebAssembly.instantiate(kernel.module, {
-    env: { memory: kernel.memory, process: mem },
+    env: { memory: kernel.memory, process: mem, lazy_fetch: () => -1 },
   });
   const k = inst.exports;
   const bytes = new Uint8Array(mem.buffer);

web/test/kernel-shared.mjs

@@ -48,7 +48,7 @@ const WORKER = `
   });
   const mod = await WebAssembly.compile(workerData.bytes);
   const k = (await WebAssembly.instantiate(mod, {
-    env: { memory: workerData.kernel, process: proc },
+    env: { memory: workerData.kernel, process: proc, lazy_fetch: () => -1 },
   })).exports;
   const u8 = new Uint8Array(proc.buffer);
 

web/test/pipe-shared.mjs

@@ -34,6 +34,7 @@ const probe = (await WebAssembly.instantiate(
   {
     env: {
       memory: kernel,
+      lazy_fetch: () => -1,
       process: new WebAssembly.Memory({
         initial: GUEST_PAGES, maximum: GUEST_PAGES, shared: true,
       }),
@@ -53,7 +54,7 @@ const WORKER = `
   });
   const k = (await WebAssembly.instantiate(
     await WebAssembly.compile(workerData.bytes),
-    { env: { memory: workerData.kernel, process: proc } },
+    { env: { memory: workerData.kernel, process: proc, lazy_fetch: () => -1 } },
   )).exports;
   // This Worker's own stack. A wasm global belongs to an instance, but wasm-ld
   // starts every one at the same shared address.

web/test/procs-shared.mjs

@@ -40,6 +40,7 @@ const probe = (await WebAssembly.instantiate(
   {
     env: {
       memory: kernel,
+      lazy_fetch: () => -1,
       process: new WebAssembly.Memory({
         initial: GUEST_PAGES, maximum: GUEST_PAGES, shared: true,
       }),
@@ -59,7 +60,7 @@ const WORKER = `
   });
   const k = (await WebAssembly.instantiate(
     await WebAssembly.compile(workerData.bytes),
-    { env: { memory: workerData.kernel, process: proc } },
+    { env: { memory: workerData.kernel, process: proc, lazy_fetch: () => -1 } },
   )).exports;
   // This Worker's own stack. A wasm global belongs to an instance, but wasm-ld
   // starts every one at the same shared address.