mirror of
https://github.com/HeyPuter/gex.git
synced 2026-11-13 20:12:43 -10:00
page: ELF units (B4), compositor readout to PNG, fjit nesting bound
- libprov: `gex_cgen_elf_units` places an ELF unit (`?elfunits=SO`) at its pinned base when the library maps, as the native shim does. - compositor: `readout()`/`readoutBase64()`; browser-linux.mjs writes the composited output as cNNN.png with each shot, plus --query. - gex-fjit: in the browser, a function whose structured emission would nest deeper than 200 levels (`nest_bound`) is laid out flat; the 1500-block cut alone still let Muck's UnityPlayer overflow a Worker's stack in `tree`/`within`/`node`. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_d2453757b8a84fdbb4ee7c9e093141df
This commit is contained in:
velzieandClaude Opus 5.5 committed
1 parent
f8f80c4275
commit
ff0456a20b
10 files changed
+203
-7
No files matched your search
@@ -140,6 +140,40 @@ fn flags() -> RegSet {
|
||||
/// Blocks above which a function is laid out flat in the browser (see `emit`).
|
||||
const FLAT_ABOVE: usize = 1500;
|
||||
|
||||
/// Nesting (`nest_bound`) above which a function is laid out flat in the
|
||||
/// browser: each level is a `tree`/`within`/`node` frame on the Worker's
|
||||
/// stack, a few hundred bytes to a few KB each.
|
||||
const NEST_MAX: usize = 200;
|
||||
|
||||
/// An upper bound on the structured emission's recursion depth: `tree(x)`
|
||||
/// nests one `within` per merge child, and `x`'s code goes on into its
|
||||
/// dominator-tree children (inline or after a merge's block).
|
||||
fn nest_bound(s: &cfg::Shape) -> usize {
|
||||
let n = s.idom.len();
|
||||
let mut d = vec![0usize; n];
|
||||
// The deepest non-merge dominator-tree child (emitted inline from `node`).
|
||||
let mut inline = vec![0usize; n];
|
||||
let mut top = 0;
|
||||
for &x in s.rpo.iter().rev() {
|
||||
let ys = &s.merge_children[x];
|
||||
let k = ys.len();
|
||||
// `within` nests once per merge child; `tree(ys[i])` runs at
|
||||
// `k - 1 - i` of them, `node(x)` under all `k`.
|
||||
let mut v = k + 1 + inline[x];
|
||||
for (i, &y) in ys.iter().enumerate() {
|
||||
v = v.max(k - i + d[y]);
|
||||
}
|
||||
d[x] = 1 + v;
|
||||
let p = s.idom[x];
|
||||
if p == x || p >= n {
|
||||
top = top.max(d[x]);
|
||||
} else if !s.merge[x] {
|
||||
inline[p] = inline[p].max(d[x]);
|
||||
}
|
||||
}
|
||||
top.max(d.iter().copied().max().unwrap_or(0))
|
||||
}
|
||||
|
||||
pub fn emit(m: &ModuleCx<'_>, f: &Function, loops: &[Loop], stats: &mut Stats) -> Option<Emitted> {
|
||||
let n = f.blocks.len();
|
||||
// The body's graph: the blocks, then one handover node per outlined
|
||||
@@ -159,8 +193,12 @@ pub fn emit(m: &ModuleCx<'_>, f: &Function, loops: &[Loop], stats: &mut Stats) -
|
||||
// of thousands of blocks (UnityPlayer has them) overflowed it
|
||||
// ("Maximum call stack size exceeded" in `tree`/`within`/`node`). Such a
|
||||
// function is laid out flat there.
|
||||
// The block count alone missed some (mp9: Muck's UnityPlayer still
|
||||
// overflowed), so the nesting is bounded too (`nest_bound`).
|
||||
let too_deep = cfg!(target_family = "wasm") && succs.len() > FLAT_ABOVE;
|
||||
let mut layout = match (!too_deep).then(|| cfg::make_reducible(graph)).flatten() {
|
||||
let fixed = (!too_deep).then(|| cfg::make_reducible(graph)).flatten();
|
||||
let fixed = fixed.filter(|fx| !cfg!(target_family = "wasm") || nest_bound(&fx.shape) <= NEST_MAX);
|
||||
let mut layout = match fixed {
|
||||
Some(fx) => {
|
||||
if !fx.dispatch.is_empty() {
|
||||
stats.irreducible += 1;
|
||||
|
||||
@@ -804,6 +804,8 @@ export class Gex {
|
||||
static watchDir = -1;
|
||||
/** `soname=path,...` (the page's `?wasmdlls=`): libraries replaced by wasm64 providers. */
|
||||
static wasmDlls = '';
|
||||
/** `name,...` (the page's `?elfunits=`): shared objects that get function-tier units. */
|
||||
static elfUnits = '';
|
||||
static #whySlot = 0;
|
||||
|
||||
#bytes() {
|
||||
@@ -2532,6 +2534,13 @@ export class Gex {
|
||||
* providers (cgen/libprov.rs). */
|
||||
#libprovConfig() {
|
||||
const e = this.#shim.exports;
|
||||
if (Gex.elfUnits && e.gex_cgen_elf_units) {
|
||||
const b = new TextEncoder().encode(Gex.elfUnits);
|
||||
const p = e.gex_alloc(Math.max(8, b.length)) >>> 0;
|
||||
this.#bytes().set(b, p);
|
||||
e.gex_cgen_elf_units(p, b.length);
|
||||
e.gex_free?.(p, Math.max(8, b.length));
|
||||
}
|
||||
if (!Gex.wasmDlls || !e.gex_libprov_config) return;
|
||||
const b = new TextEncoder().encode(Gex.wasmDlls);
|
||||
const p = e.gex_alloc(Math.max(8, b.length)) >>> 0;
|
||||
|
||||
@@ -415,6 +415,7 @@ async function handle(msg) {
|
||||
// A 64-bit guest memory (docs/mem64.md step 6): the wasm64 shim's process.
|
||||
if (msg.mem64) globalThis.GEX_MEM64 = 1;
|
||||
if (msg.wasmDlls) Gex.wasmDlls = msg.wasmDlls;
|
||||
if (msg.elfUnits) Gex.elfUnits = msg.elfUnits;
|
||||
if (msg.calloutTrace) globalThis.GEX_CALLOUT_TRACE = 1;
|
||||
if (msg.lazyThread) globalThis.GEX_LAZY_THREAD = 1;
|
||||
// `kernelStack` is this Worker's slice of the kernel's stack array. Absent for
|
||||
|
||||
@@ -84,8 +84,10 @@ pub(crate) fn maps(mapped: &[gex_kernel::CodeMap]) -> bool {
|
||||
if let Some(u) = super::libprov::maps(mapped) {
|
||||
lock(&QUEUE).push_back(u);
|
||||
}
|
||||
// Function-tier units for listed shared objects (libprov.rs, B4).
|
||||
let unit = super::libprov::elf_unit_maps(mapped);
|
||||
if !on() {
|
||||
return !lock(&QUEUE).is_empty();
|
||||
return unit || !lock(&QUEUE).is_empty();
|
||||
}
|
||||
if PID.load(Ordering::Acquire) == 0 {
|
||||
PID.store(crate::with_host(|h| h.kernel.pid), Ordering::Release);
|
||||
@@ -153,7 +155,7 @@ pub(crate) fn maps(mapped: &[gex_kernel::CodeMap]) -> bool {
|
||||
};
|
||||
lock(&QUEUE).push_back(u);
|
||||
}
|
||||
!lock(&QUEUE).is_empty()
|
||||
unit || !lock(&QUEUE).is_empty()
|
||||
}
|
||||
|
||||
/// Whether install `i` is an export provider's module.
|
||||
|
||||
@@ -614,3 +614,87 @@ pub extern "C" fn gex_libprov_start() {
|
||||
STOPPING.store(false, Ordering::SeqCst);
|
||||
bump();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Function-tier units for Linux shared objects in the page (P5 B4; the native
|
||||
// host's `nt_cgen::on_elf_maps`): an object named in `?elfunits=` gets the
|
||||
// unit built natively for it at the bias it maps at (`gexc cgen-image --elf
|
||||
// SO --base HEX --mem64`), found in the unit cache by the same key.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The objects that get units (file names, or `all`).
|
||||
static ELF_UNITS: Mutex<Vec<String>> = Mutex::new(Vec::new());
|
||||
/// Objects placed already in this process: (path, bias).
|
||||
static ELF_PLACED: Mutex<Vec<(String, u64)>> = Mutex::new(Vec::new());
|
||||
|
||||
/// `name,name,...` (the page's `?elfunits=`), before the program starts.
|
||||
#[no_mangle]
|
||||
pub extern "C" fn gex_cgen_elf_units(ptr: u32, len: u32) {
|
||||
let spec = String::from_utf8_lossy(&crate::guest_bytes()[ptr as usize..ptr as usize + len as usize]).into_owned();
|
||||
let mut v = lock(&ELF_UNITS);
|
||||
for n in spec.split(',').map(str::trim).filter(|n| !n.is_empty()) {
|
||||
if !v.iter().any(|x| x == n) {
|
||||
v.push(n.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The load bias of an ELF object one of whose executable `PT_LOAD`s was
|
||||
/// mapped at `start` from file offset `offset`.
|
||||
fn elf_bias(bytes: &[u8], start: u64, offset: u64) -> Option<u64> {
|
||||
let rd = |o: usize, n: usize| -> Option<u64> {
|
||||
let b = bytes.get(o..o + n)?;
|
||||
let mut v = [0u8; 8];
|
||||
v[..n].copy_from_slice(b);
|
||||
Some(u64::from_le_bytes(v))
|
||||
};
|
||||
if bytes.get(0..4)? != b"\x7fELF" || *bytes.get(4)? != 2 {
|
||||
return None;
|
||||
}
|
||||
let (phoff, phentsize, phnum) = (rd(0x20, 8)? as usize, rd(0x36, 2)? as usize, rd(0x38, 2)? as usize);
|
||||
for i in 0..phnum {
|
||||
let ph = phoff + i * phentsize;
|
||||
let (ty, flags, off, vaddr) = (rd(ph, 4)?, rd(ph + 4, 4)?, rd(ph + 8, 8)?, rd(ph + 16, 8)?);
|
||||
if ty == 1 && flags & 1 != 0 && off & !0xfff == offset {
|
||||
return Some(start.wrapping_sub(vaddr & !0xfff));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// After a syscall that mapped code: want the unit of each listed object.
|
||||
/// Returns whether one is now wanted.
|
||||
pub(crate) fn elf_unit_maps(mapped: &[gex_kernel::CodeMap]) -> bool {
|
||||
let list = lock(&ELF_UNITS).clone();
|
||||
if list.is_empty() || !super::on() {
|
||||
return false;
|
||||
}
|
||||
let mut any = false;
|
||||
for m in mapped {
|
||||
let file = m.path.rsplit('/').next().unwrap_or(&m.path);
|
||||
if !list.iter().any(|n| n == "all" || n == file) {
|
||||
continue;
|
||||
}
|
||||
let Some(bytes) = crate::with_host(|h| h.kernel.read_file(&m.path)) else { continue };
|
||||
let Some(base) = elf_bias(&bytes, m.start, m.offset) else { continue };
|
||||
{
|
||||
let mut placed = lock(&ELF_PLACED);
|
||||
if placed.iter().any(|(p, b)| *p == m.path && *b == base) {
|
||||
continue;
|
||||
}
|
||||
placed.push((m.path.clone(), base));
|
||||
}
|
||||
let mut image = match gex_ld::elf::load(&bytes, base) {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
crate::warn_line(&format!("gex-cgen: {}: {e:?}; no unit\n", m.path));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
gex_ld::elf::apply_own_relative(&mut image);
|
||||
crate::warn_line(&format!("gex-cgen: {} at {base:#x}: wanting its unit\n", m.path));
|
||||
super::place(&image, &m.path, base, true, None);
|
||||
any = true;
|
||||
}
|
||||
any
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
// node tests/games/browser-linux.mjs --name NAME --dir GAME_DIR --exe REL
|
||||
// [--cwd REL] [--args ARGS] [--libs DIR:DIR] [--files LIST]
|
||||
// [--input SCRIPT.wl] [--size WxH] [--minutes N] [--shot DIR]
|
||||
// [--gameplay N] [--mem64] [--fjit] [--wasmdlls SO=PATH] [--env K=V,K=V] [--end-on-exit]
|
||||
// [--gameplay N] [--mem64] [--fjit] [--wasmdlls SO=PATH] [--query K=V&...] [--env K=V,K=V] [--end-on-exit]
|
||||
//
|
||||
// --files lists host files the game opens, one per line (collected from a
|
||||
// native `gexc run --strace`; tests/games/retrocycles-linux-files.txt).
|
||||
@@ -26,8 +26,24 @@
|
||||
// Prints `ok` lines (first frame, the input script's start, gameplay) and the
|
||||
// frame rate over the last minute, with a screenshot every 30 s.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { deflateSync, crc32 } from 'node:zlib';
|
||||
|
||||
/** RGBA rows (top first) as a PNG. */
|
||||
function encodePng(w, h, px) {
|
||||
const chunk = (type, data) => {
|
||||
const len = Buffer.alloc(4); len.writeUInt32BE(data.length);
|
||||
const td = Buffer.concat([Buffer.from(type, 'latin1'), data]);
|
||||
const crc = Buffer.alloc(4); crc.writeUInt32BE(crc32(td) >>> 0);
|
||||
return Buffer.concat([len, td, crc]);
|
||||
};
|
||||
const ihdr = Buffer.alloc(13);
|
||||
ihdr.writeUInt32BE(w, 0); ihdr.writeUInt32BE(h, 4); ihdr[8] = 8; ihdr[9] = 6;
|
||||
const raw = Buffer.alloc((w * 4 + 1) * h);
|
||||
for (let y = 0; y < h; y++) px.copy(raw, y * (w * 4 + 1) + 1, y * w * 4, (y + 1) * w * 4);
|
||||
return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk('IHDR', ihdr), chunk('IDAT', deflateSync(raw)), chunk('IEND', Buffer.alloc(0))]);
|
||||
}
|
||||
import { dirname, join, basename } from 'node:path';
|
||||
// Ports of its own (the harness reads these when it loads), so a games run
|
||||
// and a gate's browser legs on the defaults (5199, 9339) do not collide.
|
||||
@@ -129,6 +145,7 @@ async function main() {
|
||||
`tar=/gex/nt/${basename(tar)}&exe=/game/${exe}&cwd=/game/${cwd}&gl=1&glbackend=wgpu&hostx=1`
|
||||
+ (argv.includes('--mem64') ? '&mem64=1' : '') + (argv.includes('--fjit') ? '&fjit=1' : '')
|
||||
+ (opt('--wasmdlls', null) ? `&wasmdlls=${encodeURIComponent(opt('--wasmdlls'))}` : '')
|
||||
+ (opt('--query', null) ? `&${opt('--query')}` : '')
|
||||
+ `&size=${width}x${height}&trapstack=1&env=${encodeURIComponent(genv)}&args=${encodeURIComponent(opt('--args', ''))}`,
|
||||
{ width: width + 20, height: height + 100, shotDir },
|
||||
);
|
||||
@@ -182,6 +199,13 @@ async function main() {
|
||||
if (Date.now() >= nextShot) {
|
||||
nextShot += 30000;
|
||||
await shot(`t${String(++shots).padStart(3, '0')}`);
|
||||
// The compositor's own output, read back (the page screenshot can miss the canvas).
|
||||
if (shotDir) {
|
||||
const r = await evaluate('globalThis.__gexCompositor?.readoutBase64()', true);
|
||||
if (r?.b64) {
|
||||
writeFileSync(join(shotDir, `c${String(shots).padStart(3, '0')}.png`), encodePng(r.w, r.h, Buffer.from(r.b64, 'base64')));
|
||||
}
|
||||
}
|
||||
const rate = window.length > 1 ? (n - window[0][1]) / ((Date.now() - window[0][0]) / 1000) : 0;
|
||||
log(`${((Date.now() - t0) / 1000).toFixed(0)} s: presents ${n}, ${rate.toFixed(1)}/s over the last minute; ${(s?.out ?? '').slice(-160).replace(/\n/g, ' | ')}`);
|
||||
// The compositor's own account: its windows, its notes (GL attachment
|
||||
|
||||
@@ -1237,6 +1237,15 @@ class Compositor {
|
||||
win.surface);
|
||||
}
|
||||
|
||||
/** The output as it was last composited: `{ w, h, px }` (RGBA, top row first). */
|
||||
async readout() {
|
||||
try {
|
||||
return await this.#painter.read();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What the harness and the demo UI read.
|
||||
*
|
||||
@@ -1466,6 +1475,9 @@ self.onmessage = ({ data: m }) => {
|
||||
// The shader translation cache, for a training run to record.
|
||||
const blob = glDevice?.wgslCache?.() ?? null;
|
||||
postMessage({ kind: 'wgsl-cache', id: m.id, blob, stats: glDevice?.wgslStats?.() ?? null }, blob ? [blob] : []);
|
||||
} else if (m.kind === 'readout') {
|
||||
// The composited output, read back (RGBA rows, top first): what a test saves as a picture.
|
||||
Promise.resolve(comp?.readout?.() ?? null).then((r) => postMessage({ kind: 'readout', id: m.id, state: r }, r ? [r.px.buffer] : []));
|
||||
} else if (m.kind === 'describe') {
|
||||
Promise.resolve(comp?.describe() ?? null).then((state) => postMessage({ kind: 'describe', id: m.id, state }));
|
||||
} else if (m.kind === 'stop') {
|
||||
|
||||
+24
-2
@@ -118,8 +118,8 @@ export class Compositor {
|
||||
} else if (m.kind === 'cursor') {
|
||||
if (this.#direct) this.#canvas.style.cursor = m.css;
|
||||
this.#cursorCss = m.css;
|
||||
} else if (m.kind === 'describe' || m.kind === 'wgsl-cache') {
|
||||
this.#pending.get(m.id)?.(m.kind === 'describe' ? m.state : { blob: m.blob, stats: m.stats });
|
||||
} else if (m.kind === 'describe' || m.kind === 'wgsl-cache' || m.kind === 'readout') {
|
||||
this.#pending.get(m.id)?.(m.kind === 'wgsl-cache' ? { blob: m.blob, stats: m.stats } : m.state);
|
||||
this.#pending.delete(m.id);
|
||||
} else if (m.kind === 'error') {
|
||||
console.error('[compositor]', m.error);
|
||||
@@ -294,6 +294,28 @@ export class Compositor {
|
||||
});
|
||||
}
|
||||
|
||||
/** The composited output read back: `{ w, h, px }` (RGBA, top row first), or null. */
|
||||
readout() {
|
||||
const id = this.#next++;
|
||||
return new Promise((resolve) => {
|
||||
this.#pending.set(id, resolve);
|
||||
this.#worker.postMessage({ kind: 'readout', id });
|
||||
setTimeout(() => {
|
||||
if (this.#pending.delete(id)) resolve(null);
|
||||
}, 5000);
|
||||
});
|
||||
}
|
||||
|
||||
/** The output read back as base64 RGBA, for a harness: `{ w, h, b64 }` or null. */
|
||||
async readoutBase64() {
|
||||
const r = await this.readout();
|
||||
if (!r) return null;
|
||||
let s = '';
|
||||
const px = r.px;
|
||||
for (let i = 0; i < px.length; i += 0x8000) s += String.fromCharCode.apply(null, px.subarray(i, i + 0x8000));
|
||||
return { w: r.w, h: r.h, b64: btoa(s) };
|
||||
}
|
||||
|
||||
describe() {
|
||||
const id = this.#next++;
|
||||
return new Promise((resolve) => {
|
||||
|
||||
@@ -117,6 +117,8 @@ async function main() {
|
||||
if (q.get('fjit')) globalThis.GEX_FJIT = Number(q.get('fjit')) || 1;
|
||||
// `?wasmdlls=soname=path,...`: shared objects replaced by wasm64 providers (cgen/libprov.rs).
|
||||
if (q.get('wasmdlls')) globalThis.GEX_WASM_DLLS = q.get('wasmdlls');
|
||||
// `?elfunits=name,...`: shared objects that get function-tier units (with `?cgen=1`).
|
||||
if (q.get('elfunits')) globalThis.GEX_ELF_UNITS = q.get('elfunits');
|
||||
if (q.get('dense') === '0') globalThis.GEX_DENSE = 0;
|
||||
if (q.get('dense') === '2') globalThis.GEX_DENSE = 2;
|
||||
// cgen=1|2 (prefer|require units), in the unguarded flavor of i386 units
|
||||
|
||||
@@ -673,6 +673,8 @@ export async function runProgram(
|
||||
fjit: globalThis.GEX_FJIT || undefined,
|
||||
// `?wasmdlls=soname=path,...`: libraries replaced by wasm64 providers (cgen/libprov.rs).
|
||||
wasmDlls: globalThis.GEX_WASM_DLLS || undefined,
|
||||
// `?elfunits=name,...`: shared objects that get function-tier units (cgen/libprov.rs).
|
||||
elfUnits: globalThis.GEX_ELF_UNITS || undefined,
|
||||
traceMisses: !!globalThis.GEX_TRACE_MISSES,
|
||||
// `?sharetrace=1`: say how compiled modules are shared between a process's Workers.
|
||||
shareTrace: flag('sharetrace') ? 1 : undefined,
|
||||
|
||||
Reference in new issue
Block a user