HeyPuter/gex
1
0 Fork 0
mirror of https://github.com/HeyPuter/gex.git synced 2026-11-13 20:12:43 -10:00
Code Issues Packages Projects Releases Wiki Activity

Merge master (9e5ac0ff) into merge/unity-seed

Browse Source 
Conflicts: web/src/runtime.js (master's symlinks next to lazy, pinsFrom kept),
shims/browser/js/tty.mjs (master's shared-or-plain scratch memory, with lazy_fetch).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_1896c5b302a54696aaa92c5f4047f20b
This commit is contained in:
velzieandClaude Opus 5.5 committed 2026-10-09 09:16:14 -04:00
2 parents 83d4423626 9e5ac0ff6f
commit 114b6b6984
117 files changed
+7077 -179
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/gex-abi/src/lib.rs
+16
View File
@@ -310,6 +310,16 @@ pub mod ctx {
/// Program counter. Only meaningful at block boundaries and safepoints.
pub const PC: usize = 32 * 8;
/// The stack pointer of a process on the i386 ABI (`compat`): `esp`,
/// `x(4)`, in every build. On x86-64 that is `SP` itself; an AArch64
/// build has i386-ABI processes too -- native wasm32 programs, whose
/// musl port reads and writes `x(4)` -- and their stack pointer is not
/// its `SP`. Everything else on the i386 ABI (the number in `x(0)`, the
/// arguments, `PC`) is at the same offsets in both layouts.
pub const fn sp_slot(compat: bool) -> usize {
if compat && !super::PPC { x(4) } else { SP }
}
/// Condition flags: one **byte** each, inside a single 32-bit slot. One
/// flag stores as one `i32.store8`, all four as one `i32.store`; a byte
/// (not a bit) each keeps the single-flag store off a read-modify-write.
@@ -435,6 +445,12 @@ pub mod ctx {
/// `poll` becomes `ppoll` with a timespec that has to live somewhere).
pub const SCRATCH: usize = X87_TMP + 16;
/// Scratch for the i386 ABI's rewritten calls (`gex-kernel`'s `i386`):
/// `SCRATCH` where the context has one; an AArch64 context ends at
/// `v31`, and its only i386-ABI processes are native wasm32 programs,
/// which never use the vector registers, so their first 64 bytes.
pub const COMPAT_SCRATCH: usize = if super::X86 || super::PPC { SCRATCH } else { V };
/// PowerPC only: the area after `v31` (where x86-64 keeps its x87
/// stack). The 32 condition-register bits, one byte each: bit `4f + k` is
/// field `f`'s LT, GT, EQ, SO for `k` = 0..3.
crates/gex-cache/src/lib.rs
+29
View File
@@ -344,6 +344,35 @@ pub fn compile(engine: &Engine, wasm: &[u8]) -> Result<Module> {
compile_here(engine, wasm)
}
/// [`compile`], keeping what it compiles: a native wasm32 program (a
/// gexcc build) is a whole module compiled at every exec, seconds for a
/// large one, so its compiled form goes in the cache by content
/// (`cache_dir()/wasm32`). A cached module from another engine
/// configuration or wasmtime version fails to deserialize and is
/// recompiled over.
pub fn compile_cached(engine: &Engine, wasm: &[u8]) -> Result<Module> {
if precompiled_dir().is_some() {
return compile(engine, wasm);
}
let Some(dir) = cache_dir().map(|d| d.join("wasm32")) else {
return compile(engine, wasm);
};
let path = dir.join(content_name(wasm));
if let Ok(bytes) = std::fs::read(&path) {
// SAFETY: written below by this runtime with an engine of the same
// kind; `deserialize` checks the engine configuration and version.
if let Ok(m) = unsafe { Module::deserialize(engine, &bytes) } {
return Ok(m);
}
}
let m = compile_here(engine, wasm)?;
#[cfg(feature = "compile")]
if let Ok(bytes) = m.serialize() {
store(&dir, &path, &bytes);
}
Ok(m)
}
#[cfg(feature = "compile")]
fn compile_here(engine: &Engine, wasm: &[u8]) -> Result<Module> {
Module::new(engine, wasm).map_err(anyhow::Error::from)
crates/gex-kernel/src/eventfd.rs
+1
View File
@@ -98,6 +98,7 @@ impl FileHandle for EventFd {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
crates/gex-kernel/src/file.rs
+35 -5
View File
@@ -166,6 +166,9 @@ impl OpenFile {
}
pub fn seek(&self, offset: i64, whence: u32) -> SysResult {
if let Some(r) = self.handle.seek(offset, whence) {
return r;
}
if self.handle.is_stream() {
return Err(Errno::ESPIPE);
}
@@ -218,8 +221,15 @@ pub struct FdTable {
/// Bumped by every change, so `/proc/self/fd` can skip rebuilding its
/// listing when nothing changed -- as `MemoryMap::generation` does.
generation: u64,
/// `RLIMIT_NOFILE`'s soft limit: no descriptor at or above it.
limit: usize,
}
/// `RLIMIT_NOFILE`, as Linux starts a process: 1024 soft, and the hard
/// limit is what the table can hold.
pub const NOFILE_SOFT: usize = 1024;
pub const NOFILE_HARD: usize = 4096;
impl Default for FdTable {
fn default() -> Self {
Self::new()
@@ -232,9 +242,19 @@ impl FdTable {
entries: Vec::new(),
cloexec: Vec::new(),
generation: 0,
limit: NOFILE_SOFT,
}
}
/// `RLIMIT_NOFILE`'s soft limit (descriptors already above it stay).
pub fn set_limit(&mut self, limit: usize) {
self.limit = limit.min(NOFILE_HARD);
}
pub fn limit(&self) -> usize {
self.limit
}
pub fn get(&self, fd: i32) -> Result<Arc<OpenFile>, Errno> {
if fd < 0 {
return Err(Errno::EBADF);
@@ -248,7 +268,11 @@ impl FdTable {
/// Install a description at the lowest free descriptor at or above `min`.
pub fn insert_at(&mut self, file: Arc<OpenFile>, min: i32, cloexec: bool) -> SysResult {
self.generation += 1;
let min = min.max(0) as usize;
// F_DUPFD's argument: negative or past the limit is EINVAL
if min < 0 || min as usize >= self.limit {
return Err(Errno::EINVAL);
}
let min = min as usize;
while self.entries.len() <= min {
self.entries.push(None);
self.cloexec.push(false);
@@ -258,9 +282,10 @@ impl FdTable {
.position(|e| e.is_none())
.map(|p| p + min);
let idx = match slot {
Some(i) => i,
Some(i) if i < self.limit => i,
Some(_) => return Err(Errno::EMFILE),
None => {
if self.entries.len() >= 4096 {
if self.entries.len() >= self.limit {
return Err(Errno::EMFILE);
}
self.entries.push(None);
@@ -280,7 +305,8 @@ impl FdTable {
/// Force a description into a specific descriptor, closing what was there.
pub fn install(&mut self, fd: i32, file: Arc<OpenFile>, cloexec: bool) -> SysResult {
self.generation += 1;
if fd < 0 {
// dup2/dup3 to a descriptor past RLIMIT_NOFILE: EBADF, as Linux
if fd < 0 || fd as usize >= self.limit {
return Err(Errno::EBADF);
}
let fd = fd as usize;
@@ -403,6 +429,7 @@ impl FdTable {
entries: self.entries.clone(),
cloexec: self.cloexec.clone(),
generation: self.generation,
limit: self.limit,
}
}
}
@@ -417,8 +444,10 @@ pub struct PipeEnd {
impl FileHandle for PipeEnd {
/// An empty pipe is not readable -- the trait default "yes" is right for a
/// file and wrong here, and a `select` on it would spin ready/read-nothing.
/// A write end is never readable (select would report it ready for
/// reading when asked about both directions; Linux does not).
fn poll_readable(&self) -> bool {
self.write || self.pipe.readable()
!self.write && self.pipe.readable()
}
fn available(&self) -> Option<u64> {
@@ -461,6 +490,7 @@ impl FileHandle for PipeEnd {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
crates/gex-kernel/src/fs.rs
+23 -8
View File
@@ -52,9 +52,20 @@ pub struct Stat {
pub blksize: i32,
pub blocks: i64,
pub mtime: i64,
pub mtime_ns: u32,
pub atime: i64,
pub atime_ns: u32,
pub ctime: i64,
pub ctime_ns: u32,
}
impl Stat {
/// (seconds, nanoseconds) of atime, mtime, ctime, in `struct stat`'s
/// order.
pub fn times(&self) -> [(i64, u32); 3] {
[(self.atime, self.atime_ns), (self.mtime, self.mtime_ns), (self.ctime, self.ctime_ns)]
}
/// PowerPC's 32-bit `struct stat64` (104 bytes, big-endian): no padding
/// after `st_dev`, a 64-bit `st_ino` second, 32-bit times.
fn encode_ppc(&self) -> Vec<u8> {
@@ -70,8 +81,9 @@ impl Stat {
b[48..56].copy_from_slice(&self.size.to_ge_bytes());
b[56..60].copy_from_slice(&self.blksize.to_ge_bytes());
b[64..72].copy_from_slice(&self.blocks.to_ge_bytes());
for off in [72usize, 80, 88] {
b[off..off + 4].copy_from_slice(&(self.mtime as u32).to_ge_bytes());
for (off, (t, ns)) in [72usize, 80, 88].into_iter().zip(self.times()) {
b[off..off + 4].copy_from_slice(&(t as u32).to_ge_bytes());
b[off + 4..off + 8].copy_from_slice(&ns.to_ge_bytes());
}
b
}
@@ -101,8 +113,9 @@ impl Stat {
b[44..52].copy_from_slice(&self.size.to_le_bytes());
b[52..56].copy_from_slice(&(self.blksize as u32).to_le_bytes());
b[56..64].copy_from_slice(&(self.blocks as u64).to_le_bytes());
for off in [64usize, 72, 80] {
b[off..off + 4].copy_from_slice(&(self.mtime as u32).to_le_bytes());
for (off, (t, ns)) in [64usize, 72, 80].into_iter().zip(self.times()) {
b[off..off + 4].copy_from_slice(&(t as u32).to_le_bytes());
b[off + 4..off + 8].copy_from_slice(&ns.to_le_bytes());
}
b[88..96].copy_from_slice(&self.ino.to_le_bytes());
return b;
@@ -119,8 +132,9 @@ impl Stat {
b[48..56].copy_from_slice(&self.size.to_le_bytes());
b[56..64].copy_from_slice(&(self.blksize as i64).to_le_bytes());
b[64..72].copy_from_slice(&self.blocks.to_le_bytes());
for off in [72usize, 88, 104] {
b[off..off + 8].copy_from_slice(&self.mtime.to_le_bytes());
for (off, (t, ns)) in [72usize, 88, 104].into_iter().zip(self.times()) {
b[off..off + 8].copy_from_slice(&t.to_le_bytes());
b[off + 8..off + 16].copy_from_slice(&(ns as u64).to_le_bytes());
}
return b;
}
@@ -135,8 +149,9 @@ impl Stat {
b[48..56].copy_from_slice(&self.size.to_le_bytes());
b[56..60].copy_from_slice(&self.blksize.to_le_bytes());
b[64..72].copy_from_slice(&self.blocks.to_le_bytes());
for off in [72usize, 88, 104] {
b[off..off + 8].copy_from_slice(&self.mtime.to_le_bytes());
for (off, (t, ns)) in [72usize, 88, 104].into_iter().zip(self.times()) {
b[off..off + 8].copy_from_slice(&t.to_le_bytes());
b[off + 8..off + 16].copy_from_slice(&(ns as u64).to_le_bytes());
}
b
}
crates/gex-kernel/src/i386.rs
+7 -3
View File
@@ -865,9 +865,13 @@ pub fn translate(
*a = [0, a[0], 0, scratch, 0, 0];
nr::PRLIMIT64
}
x::GETITIMER | x::GETRUSAGE | x::TIMES => {
// Their 32-bit result layouts are not written yet. ENOSYS is
// loud where a silently wrong struct would not be.
// getrusage(who, &rusage32) and times(&tms32): the kernel writes
// their 32-bit layouts itself for an i386 process.
x::GETRUSAGE => nr::GETRUSAGE,
x::TIMES => nr::TIMES,
x::GETITIMER => {
// Its 32-bit result layout is not written yet. ENOSYS is loud
// where a silently wrong struct would not be.
UNKNOWN
}
x::PRLIMIT64 => {
crates/gex-kernel/src/mem.rs
+28
View File
@@ -91,6 +91,9 @@ pub struct MemoryMap {
brk_cur: u64,
/// Next address the top-down allocator will try.
mmap_next: u64,
/// Where the top-down allocator starts over when the cursor runs out:
/// the holes `munmap` left above it (MMAP_TOP, or `set_mmap_top`).
mmap_ceiling: u64,
/// How far up the arena has been claimed, and who to ask for more.
///
/// A guest in a browser lives in a `WebAssembly.Memory` allocated up to its
@@ -161,6 +164,7 @@ impl MemoryMap {
brk_start: page_align_up(brk_start),
brk_cur: page_align_up(brk_start),
mmap_next: MMAP_TOP,
mmap_ceiling: MMAP_TOP,
arena: Vec::new(),
take: None,
back: None,
@@ -542,6 +546,7 @@ impl MemoryMap {
/// KUSER_SHARED_DATA and the top of user space for itself).
pub fn set_mmap_top(&mut self, top: u64) {
self.mmap_next = self.mmap_next.min(page_align_down(top));
self.mmap_ceiling = self.mmap_ceiling.min(page_align_down(top));
}
/// Find and reserve a free range of `len` bytes. Top-down from [`MMAP_TOP`],
@@ -669,7 +674,17 @@ impl MemoryMap {
}
let mut candidate = self.mmap_next;
// The cursor only moves down; once it runs out, one more pass from
// the top finds what `munmap` freed above it (Linux searches the whole
// gap list each time: a program that maps and frees 1 GiB buffers in
// turn, as expat's tests do, must get the second one).
let mut wrapped = self.mmap_next >= self.mmap_ceiling;
loop {
if !wrapped && (candidate < len || candidate - len < MMAP_BOTTOM) {
wrapped = true;
candidate = self.mmap_ceiling;
continue;
}
if candidate < len || candidate - len < MMAP_BOTTOM {
if self.high.is_none() {
note_address_space_short(len);
@@ -1116,6 +1131,19 @@ mod tests {
assert_eq!(a % PAGE_SIZE, 0);
}
#[test]
fn mmap_reuses_a_hole_above_the_cursor() {
// more than half the arena, mapped, a small mapping below it, then
// freed: the next big request fits only in the hole above the cursor
let mut m = MemoryMap::new(0x40_1000);
let big = page_align_down((MMAP_TOP - MMAP_BOTTOM) / 5 * 3);
let a = m.mmap(0, big, prot::READ, flags::ANONYMOUS, "a").unwrap();
m.mmap(0, 0x2000, prot::READ, flags::ANONYMOUS, "b").unwrap();
m.unmap(a, big).unwrap();
let c = m.mmap(0, big, prot::READ, flags::ANONYMOUS, "c").unwrap();
assert_eq!(c, a);
}
#[test]
fn vma_at_finds_the_mapping_holding_an_address() {
let mut m = MemoryMap::new(0x40_1000);
crates/gex-kernel/src/net.rs
+159 -2
View File
@@ -56,6 +56,24 @@ pub trait Transport: Send + Sync + std::fmt::Debug {
usize::from(self.has_data())
}
fn close(&self);
/// `shutdown(how)`: 0 reads, 1 writes, 2 both. A transport that cannot
/// half-close accepts it and does nothing.
fn shutdown(&self, how: u64) -> Result<(), Errno> {
let _ = how;
Ok(())
}
/// A write once `status` says Closed: `EPIPE`, unless the transport
/// knows better (only the reading side went away, as after the peer's
/// `shutdown(SHUT_WR)`).
fn send_closed(&self, data: &[u8]) -> Result<usize, Errno> {
let _ = data;
Err(Errno::EPIPE)
}
/// `recv(MSG_PEEK)`: what `recv` would take, left queued.
fn peek(&self, buf: &mut [u8]) -> Result<usize, Errno> {
let _ = buf;
Err(Errno::EOPNOTSUPP)
}
}
/// Where a host's connections come from.
@@ -63,6 +81,18 @@ pub trait NetStack: Send + Sync + std::fmt::Debug {
fn tcp(&self) -> Result<Arc<dyn Transport>, Errno>;
}
/// No network behind the kernel (the browser without a Wisp relay): inet
/// sockets still work over loopback -- bind, listen, connect and the
/// kernel's own DNS answers -- and a connection out is unreachable.
#[derive(Debug)]
pub struct NoNet;
impl NetStack for NoNet {
fn tcp(&self) -> Result<Arc<dyn Transport>, Errno> {
Err(Errno::ENETUNREACH)
}
}
// -- names ------------------------------------------------------------------
/// The private range the synthetic answers come from. 240.0.0.0/4 is reserved
@@ -173,8 +203,11 @@ struct LoopBuf {
data: VecDeque<u8>,
/// The writing end closed: once drained, the reader sees end of file.
writer_gone: bool,
/// The reading end closed: a write is `EPIPE`.
/// The reading end closed: a write is `EPIPE` (after the first, which
/// Linux takes before the peer's reset arrives).
reader_gone: bool,
/// A write after `reader_gone` was taken: the next is `EPIPE`.
reset: bool,
}
/// One end of a TCP connection between two guest sockets on this machine
@@ -211,6 +244,14 @@ impl Transport for LoopEnd {
fn send(&self, data: &[u8]) -> Result<usize, Errno> {
let mut tx = self.tx.lock().unwrap();
if tx.reader_gone {
if !tx.reset {
tx.reset = true;
return Ok(data.len());
}
return Err(Errno::EPIPE);
}
if tx.writer_gone {
// this end shut down its writing
return Err(Errno::EPIPE);
}
let n = data.len().min(LOOP_BUF_MAX.saturating_sub(tx.data.len()));
@@ -242,6 +283,28 @@ impl Transport for LoopEnd {
self.tx.lock().unwrap().writer_gone = true;
self.rx.lock().unwrap().reader_gone = true;
}
fn shutdown(&self, how: u64) -> Result<(), Errno> {
// SHUT_WR (or RDWR): the peer reads to end of file. SHUT_RD only
// changes this end's reads, which Linux does not tell the peer.
if how == 1 || how == 2 {
self.tx.lock().unwrap().writer_gone = true;
}
Ok(())
}
fn send_closed(&self, data: &[u8]) -> Result<usize, Errno> {
self.send(data)
}
fn peek(&self, buf: &mut [u8]) -> Result<usize, Errno> {
let rx = self.rx.lock().unwrap();
if rx.data.is_empty() {
return if rx.writer_gone { Ok(0) } else { Err(Errno::EAGAIN) };
}
let n = buf.len().min(rx.data.len());
for (d, s) in buf.iter_mut().zip(rx.data.iter()) {
*d = *s;
}
Ok(n)
}
}
/// One end of a TCP connection between guests in different processes of a
@@ -286,6 +349,14 @@ impl Transport for RemoteEnd {
fn close(&self) {
(self.remote.close)(self.id);
}
/// The global kernel's shutdown is SHUT_WR's: the far end reads to its
/// end. SHUT_RD only changes this end, which the far end is not told.
fn shutdown(&self, how: u64) -> Result<(), Errno> {
if how == 1 || how == 2 {
(self.remote.shutdown)(self.id);
}
Ok(())
}
}
/// The global kernel's name for a TCP port (abstract, so no file appears).
@@ -496,6 +567,16 @@ impl Socket {
if port != 0 && self.kind == Kind::Tcp && self.remote.is_none() && self.ports.listening(port) {
return Err(Errno::EADDRINUSE);
}
// Port 0 is a port of the kernel's choosing, chosen now, as Linux
// does: a program reads it back with getsockname before listening
// (Wine's server does, for every bind), and connects to it.
let mut port = port;
if port == 0 {
port = self.ports.ephemeral();
while self.remote_taken(port) {
port = self.ports.ephemeral();
}
}
*self.local.lock().unwrap() = Some((addr, port));
Ok(())
}
@@ -664,6 +745,17 @@ impl Socket {
Err(Errno::EINPROGRESS)
}
/// `shutdown` on a connected stream.
pub fn shutdown(&self, how: u64) -> Result<u64, Errno> {
if how > 2 {
return Err(Errno::EINVAL);
}
match &*self.transport.lock().unwrap() {
Some(t) => t.shutdown(how).map(|_| 0),
None => Err(Errno::ENOTCONN),
}
}
pub fn status(&self) -> Status {
match &*self.transport.lock().unwrap() {
Some(t) => t.status(),
@@ -798,6 +890,39 @@ impl Socket {
}
}
impl Socket {
/// `recv` with `MSG_PEEK`: the bytes (or the next datagram) left queued
/// for the next read. Wine's server peeks a byte to tell a socket that
/// has data from one that was closed.
pub fn peek(&self, buf: &mut [u8]) -> SysResult {
if self.kind == Kind::Udp {
self.udp_pump();
return match self.dgrams.lock().unwrap().front() {
Some((d, from)) => {
let n = buf.len().min(d.len());
buf[..n].copy_from_slice(&d[..n]);
*self.last_from.lock().unwrap() = Some(*from);
Ok(n as u64)
}
None => Err(Errno::EAGAIN),
};
}
let t = self.transport.lock().unwrap();
let Some(t) = t.as_ref() else {
return Err(Errno::ENOTCONN);
};
match t.status() {
Status::Refused => Err(Errno::ECONNREFUSED),
Status::Closed => match t.peek(buf) {
Ok(n) => Ok(n as u64),
Err(Errno::EAGAIN) => Ok(0),
Err(e) => Err(e),
},
_ => t.peek(buf).map(|n| n as u64),
}
}
}
impl FileHandle for Socket {
fn read_at(&self, _offset: u64, buf: &mut [u8]) -> SysResult {
if self.kind == Kind::Udp {
@@ -833,7 +958,7 @@ impl FileHandle for Socket {
};
match t.status() {
Status::Refused => Err(Errno::ECONNREFUSED),
Status::Closed => Err(Errno::EPIPE),
Status::Closed => t.send_closed(buf).map(|n| n as u64),
// Still connecting: the bytes cannot go yet, and a caller that
// waited for the connection would not be here.
Status::Connecting => Err(Errno::EAGAIN),
@@ -850,6 +975,7 @@ impl FileHandle for Socket {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
@@ -1034,6 +1160,37 @@ mod tests {
assert!(answer_dns(&q, &r).is_none());
}
/// No network: loopback between guests only.
#[derive(Debug)]
struct NoNet;
impl NetStack for NoNet {
fn tcp(&self) -> Result<Arc<dyn Transport>, Errno> {
Err(Errno::ENETUNREACH)
}
}
#[test]
fn a_tcp_bind_to_port_zero_gets_its_port_at_once() {
let (net, res, ports) = (Arc::new(NoNet) as Arc<dyn NetStack>, Arc::new(Resolver::new()), Arc::new(Ports::new()));
let l = Arc::new(Socket::new(Kind::Tcp, Arc::clone(&net), Arc::clone(&res), Arc::clone(&ports), None));
l.set_self();
l.bind(0x7f00_0001, 0).unwrap();
let port = l.local().1;
assert_ne!(port, 0, "getsockname after bind has the port");
l.listen().unwrap();
assert_eq!(l.local().1, port, "listen keeps it");
let c = Socket::new(Kind::Tcp, net, res, ports, None);
c.connect(0x7f00_0001, port).unwrap();
let s = l.accept().expect("a connection to it arrives");
assert_eq!(c.write_at(0, b"hi").unwrap(), 2);
let mut b = [0u8; 8];
assert_eq!(s.peek(&mut b[..1]).unwrap(), 1, "a peek sees the first byte");
assert_eq!(s.read_at(0, &mut b).unwrap(), 2, "and leaves it for the read");
assert_eq!(&b[..2], b"hi");
c.shutdown(1).unwrap();
assert_eq!(s.status(), Status::Closed, "and its shutdown reaches the far end");
}
#[test]
fn sockaddr_round_trips() {
let bytes = build_sockaddr(0xf0000001, 80);
crates/gex-kernel/src/pipe.rs
+3 -1
View File
@@ -410,6 +410,7 @@ impl crate::vfs::FileHandle for RemotePipeEnd {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
@@ -419,8 +420,9 @@ impl crate::vfs::FileHandle for RemotePipeEnd {
/// A pipe with nothing in it is not readable, whatever the default says.
/// The default is "yes", which is right for a file and wrong here.
/// (A write end is never readable, as Linux.)
fn poll_readable(&self) -> bool {
self.write || (self.remote.readable)(self.id) != 0
!self.write && (self.remote.readable)(self.id) != 0
}
fn poll_writable(&self) -> bool {
crates/gex-kernel/src/pty.rs
+1
View File
@@ -307,6 +307,7 @@ impl crate::vfs::FileHandle for RemotePty {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
crates/gex-kernel/src/signal.rs
+9 -6
View File
@@ -554,7 +554,7 @@ pub fn setup_frame(
if gex_abi::PPC {
return setup_frame_ppc(signals, m, ctx_addr, signal, fault_address);
}
if gex_abi::X86 && compat {
if compat {
return setup_frame_i386(signals, m, ctx_addr, signal, fault_address);
}
if gex_abi::X86 {
@@ -840,7 +840,7 @@ fn setup_frame_i386(
fault_address: u64,
) -> Result<u64, Errno> {
let action = signals.action(signal);
let sp = m.read_u64(ctx_addr + ctx::SP as u64)? & 0xffff_ffff;
let sp = m.read_u64(ctx_addr + ctx::sp_slot(true) as u64)? & 0xffff_ffff;
let rt = action.flags & sa::SIGINFO != 0;
signals.legacy_frame = !rt;
@@ -917,7 +917,10 @@ fn setup_frame_i386(
m.write_u32(sc + iframe::SC_CR2, fault_address as u32)?;
// The FP image: x87 control word, MXCSR and the eight xmm registers.
m.write_u32(fpstate, m.read_u32(ctx_addr + ctx::X87_CW as u64)? & 0xffff)?;
// (an AArch64 build's i386-ABI processes are wasm32 programs: no x87,
// and the context has no x87 area)
let cw = if gex_abi::X86 { m.read_u32(ctx_addr + ctx::X87_CW as u64)? & 0xffff } else { 0x37f };
m.write_u32(fpstate, cw)?;
m.write_u32(fpstate + iframe::FP_MXCSR, m.read_u32(ctx_addr + ctx::MXCSR as u64)?)?;
for n in 0..8u8 {
for half in 0..2u64 {
@@ -944,7 +947,7 @@ fn setup_frame_i386(
);
}
// esp = frame; eax = signo; for an rt frame edx = &info and ecx = &uc.
m.write_u64(ctx_addr + ctx::SP as u64, frame_addr)?;
m.write_u64(ctx_addr + ctx::sp_slot(true) as u64, frame_addr)?;
m.write_u64(ctx_addr + ctx::x(0) as u64, signal as u64)?;
if rt {
m.write_u64(ctx_addr + ctx::x(2) as u64, frame_addr + iframe::RT_INFO)?;
@@ -966,7 +969,7 @@ fn sigreturn_i386(
ctx_addr: u64,
legacy: bool,
) -> Result<u64, Errno> {
let esp = m.read_u64(ctx_addr + ctx::SP as u64)? & 0xffff_ffff;
let esp = m.read_u64(ctx_addr + ctx::sp_slot(true) as u64)? & 0xffff_ffff;
let (sc, mask) = if legacy {
let frame_addr = esp - 8;
let sc = frame_addr + iframe::SF_SC;
@@ -1064,7 +1067,7 @@ pub fn sigreturn(
if gex_abi::PPC {
return sigreturn_ppc(signals, m, ctx_addr);
}
if gex_abi::X86 && compat {
if compat {
// The frame's own kind decides; the entry number is a cross-check.
let _ = legacy;
let kind = signals.legacy_frame;
crates/gex-kernel/src/stdio.rs
+57 -1
View File
@@ -31,11 +31,37 @@ pub struct StdStream {
pub stream: Stream,
/// Whether `isatty` should say yes, which decides libc's buffering.
pub tty: bool,
/// The host's descriptor is a regular file (`prog < file`, `> file`).
/// Its offset stays the host's: reads and writes go through the host
/// descriptor, and `lseek` and `fstat` reach it too, so the next host
/// reader or writer continues where the guest stopped (`(prog; head) <
/// file`, `pcre2grep -m`) and concurrent writers to one log file share
/// the offset as on Linux.
host_file: bool,
}
/// The host's own descriptor for a stream: a duplicate, which shares the
/// open-file description and so its offset (made once, kept open).
#[cfg(not(target_family = "wasm"))]
fn host(stream: Stream) -> Option<&'static std::fs::File> {
use std::os::fd::AsFd;
use std::sync::OnceLock;
static FDS: [OnceLock<Option<std::fs::File>>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()];
let (k, fd) = match stream {
Stream::In => (0, std::io::stdin().as_fd().try_clone_to_owned()),
Stream::Out => (1, std::io::stdout().as_fd().try_clone_to_owned()),
Stream::Err => (2, std::io::stderr().as_fd().try_clone_to_owned()),
};
FDS[k].get_or_init(|| fd.ok().map(std::fs::File::from)).as_ref()
}
impl StdStream {
pub fn new(stream: Stream, tty: bool) -> StdStream {
StdStream { stream, tty }
#[cfg(not(target_family = "wasm"))]
let host_file = !tty && host(stream).is_some_and(|f| f.metadata().is_ok_and(|m| m.is_file()));
#[cfg(target_family = "wasm")]
let host_file = false;
StdStream { stream, tty, host_file }
}
}
@@ -48,6 +74,12 @@ impl FileHandle for StdStream {
if self.stream != Stream::In {
return Err(Errno::EBADF);
}
// Unbuffered, so the guest takes no more than it asked for: what it
// leaves stays for whoever reads the host's stdin next.
#[cfg(not(target_family = "wasm"))]
if let Some(mut f) = host(self.stream) {
return f.read(buf).map(|n| n as u64).map_err(|e| Errno::from_io(&e));
}
std::io::stdin()
.read(buf)
.map(|n| n as u64)
@@ -76,7 +108,30 @@ impl FileHandle for StdStream {
Ok(n as u64)
}
fn seek(&self, offset: i64, whence: u32) -> Option<SysResult> {
#[cfg(not(target_family = "wasm"))]
if self.host_file {
use std::io::{Seek, SeekFrom};
let to = match whence {
0 if offset >= 0 => SeekFrom::Start(offset as u64),
1 => SeekFrom::Current(offset),
2 => SeekFrom::End(offset),
_ => return Some(Err(Errno::EINVAL)),
};
let mut f = host(self.stream)?;
return Some(f.seek(to).map_err(|e| Errno::from_io(&e)));
}
let _ = (offset, whence);
None
}
fn metadata(&self) -> Result<Metadata, Errno> {
#[cfg(not(target_family = "wasm"))]
if self.host_file {
if let Some(f) = host(self.stream) {
return f.metadata().map(|m| crate::vfs::hostfs::convert(&m)).map_err(|e| Errno::from_io(&e));
}
}
Ok(Metadata {
file_type: FileType::CharDevice,
size: 0,
@@ -85,6 +140,7 @@ impl FileHandle for StdStream {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
crates/gex-kernel/src/syscalls.rs
+445 -78
View File
@@ -626,6 +626,9 @@ pub struct Kernel {
/// The pc of the syscall being handled (docs/mem64.md: a mapping the
/// dynamic loader asks for stays below 4 GiB, since it will hold code).
pub syscall_pc: u64,
/// The canonical number of the syscall being handled (for the host's
/// `GEX_SYS_WAITS`, which sees only the guest's own numbering).
pub syscall_nr: u64,
/// Where each pinned object actually landed, for checking afterwards.
///
/// A pin that silently does not take is the worst outcome available: the
@@ -658,6 +661,9 @@ pub struct Kernel {
/// it replaced, and a build system that does not get one back concludes
/// the mask was zero and re-runs itself.
pub umask: u32,
/// `getrlimit`/`setrlimit` (prlimit64): soft and hard per resource.
/// RLIMIT_NOFILE's soft limit is the descriptor table's (`FdTable`).
pub rlimits: [[u64; 2]; RLIM_NLIMITS],
/// The terminal: flags, size, foreground process group, and the line
/// discipline behind them. See [`crate::tty`].
///
@@ -684,6 +690,30 @@ pub struct Itimer {
}
/// The three interval timers, in the order `which` names them.
/// Resource limits (prlimit64): Linux's numbering and its defaults for a
/// login process. RLIM_INFINITY is all ones.
pub const RLIM_NLIMITS: usize = 16;
pub const RLIMIT_STACK: usize = 3;
pub const RLIMIT_CORE: usize = 4;
pub const RLIMIT_NOFILE: usize = 7;
pub const RLIMIT_NPROC: usize = 6;
pub const RLIMIT_MEMLOCK: usize = 8;
pub const RLIMIT_SIGPENDING: usize = 11;
pub const RLIMIT_MSGQUEUE: usize = 12;
pub const RLIM_INFINITY: u64 = u64::MAX;
pub fn default_rlimits() -> [[u64; 2]; RLIM_NLIMITS] {
let mut l = [[RLIM_INFINITY; 2]; RLIM_NLIMITS];
l[RLIMIT_STACK] = [8 << 20, RLIM_INFINITY];
l[RLIMIT_CORE] = [0, RLIM_INFINITY];
l[RLIMIT_NOFILE] = [crate::file::NOFILE_SOFT as u64, crate::file::NOFILE_HARD as u64];
l[RLIMIT_NPROC] = [63_000, 63_000];
l[RLIMIT_MEMLOCK] = [8 << 20, 8 << 20];
l[RLIMIT_SIGPENDING] = [63_000, 63_000];
l[RLIMIT_MSGQUEUE] = [819_200, 819_200];
l
}
pub const ITIMER_REAL: usize = 0;
pub const ITIMER_VIRTUAL: usize = 1;
pub const ITIMER_PROF: usize = 2;
@@ -854,6 +884,9 @@ pub enum PollReport {
max: u32,
/// The opaque word to hand back for each descriptor.
data: Vec<(i32, u64)>,
/// `struct epoll_event` packed in 12 bytes (x86-64's, and the i386
/// ABI's in any build), rather than 16.
packed: bool,
},
/// `select`'s three `fd_set`s. A zero address means the caller did not
/// pass that set.
@@ -870,6 +903,15 @@ pub fn poll_revents(p: &PollFd) -> u16 {
const POLLIN: u16 = 0x001;
const POLLOUT: u16 = 0x004;
const POLLNVAL: u16 = 0x020;
// A negative descriptor is a slot the caller is not using: its events
// are ignored and it reports nothing, as on Linux. musl's resolver keeps
// one such slot per query for a TCP retry; reporting POLLNVAL there made
// every `ppoll` return at once with nothing to read, and the resolver
// spun until its 5 s timeout -- on every `getaddrinfo` (Wine's
// `wineboot` resolving the host name was 5 s of each session boot).
if p.fd < 0 {
return 0;
}
let Some(h) = &p.handle else {
return POLLNVAL;
};
@@ -917,7 +959,7 @@ pub fn poll_finish(fds: &[PollFd], report: &PollReport, m: &mut GuestMem<'_>) ->
}
Ok(ready)
}
PollReport::Epoll { out, max, data } => {
PollReport::Epoll { out, max, data, packed } => {
// Only the ready ones, each with the caller's own token, which is
// the whole difference between this and `poll`.
let mut n = 0u64;
@@ -938,7 +980,7 @@ pub fn poll_finish(fds: &[PollFd], report: &PollReport, m: &mut GuestMem<'_>) ->
// on x86-64 (the token at offset four), 16 on aarch64. Xvfb
// read its `data.ptr` from the wrong offset after every wait
// that had to block, and dereferenced the mask.
if gex_abi::X86 {
if *packed {
let at = out + n * 12;
m.write_u32(at, r as u32)?;
m.write_u64(at + 4, token)?;
@@ -1256,6 +1298,7 @@ impl Kernel {
place: None,
armed_place: None,
syscall_pc: 0,
syscall_nr: 0,
thread_signals: std::collections::HashMap::new(),
thread_calls: std::collections::HashMap::new(),
sig_tid: 0,
@@ -1271,6 +1314,7 @@ impl Kernel {
retired_code: Vec::new(),
next_tmpfile: 0,
umask: 0o022,
rlimits: default_rlimits(),
net: None,
resolver: Arc::new(net::Resolver::new()),
ports: Arc::new(net::Ports::new()),
@@ -1412,13 +1456,21 @@ impl Kernel {
) -> Result<(String, Vec<String>, Vec<String>), Errno> {
let bytes_to_string = |b: Vec<u8>| String::from_utf8(b).map_err(|_| Errno::EINVAL);
let path = bytes_to_string(m.read_cstr(a[0], 4096)?)?;
if path.is_empty() {
return Err(Errno::ENOENT);
}
// A 32-bit process hands over vectors of 32-bit pointers.
let compat = self.compat;
let read_vec = |mut p: u64| -> Result<Vec<String>, Errno> {
let mut out = Vec::new();
// A bound, because the vector is the guest's and its terminator
// may not be there.
for _ in 0..4096 {
// Linux's limits: one string up to MAX_ARG_STRLEN (32 pages),
// all of them together a quarter of the 8 MiB stack limit;
// past either, E2BIG. (A make recipe of several KiB, passed to
// sh -c, is ordinary: automake's test-suite.log rule.)
const MAX_ARG_STRLEN: usize = 32 * 4096;
const ARG_MAX: usize = 2 << 20;
let mut total = 0usize;
loop {
if p == 0 {
break;
}
@@ -1426,7 +1478,12 @@ impl Kernel {
if ptr == 0 {
break;
}
out.push(bytes_to_string(m.read_cstr(ptr, 4096)?)?);
let b = m.read_cstr(ptr, MAX_ARG_STRLEN).map_err(|e| if e == Errno::ENAMETOOLONG { Errno::E2BIG } else { e })?;
total += b.len() + 1 + if compat { 4 } else { 8 };
if total > ARG_MAX {
return Err(Errno::E2BIG);
}
out.push(bytes_to_string(b)?);
p += if compat { 4 } else { 8 };
}
Ok(out)
@@ -1489,6 +1546,7 @@ impl Kernel {
place: self.place.clone(),
armed_place: None,
syscall_pc: 0,
syscall_nr: 0,
thread_signals: std::collections::HashMap::new(),
thread_calls: std::collections::HashMap::new(),
sig_tid: 0,
@@ -1504,6 +1562,7 @@ impl Kernel {
retired_code: Vec::new(),
next_tmpfile: self.next_tmpfile,
umask: self.umask,
rlimits: self.rlimits,
net: self.net.clone(),
resolver: Arc::clone(&self.resolver),
ports: Arc::clone(&self.ports),
@@ -1564,7 +1623,7 @@ impl Kernel {
m.write_u64(child + off as u64, v)?;
}
set_result(m, child, 0)?;
m.write_u64(child + ctx::SP as u64, child_stack)?;
m.write_u64(child + ctx::sp_slot(self.compat) as u64, child_stack)?;
m.write_u64(child + ctx::MON_SIZE as u64, 0)?;
m.write_u64(child + ctx::TID as u64, tid as u64)?;
if flags & clone_flags::SETTLS != 0 {
@@ -1636,8 +1695,11 @@ impl Kernel {
fn syscall_inner(&mut self, m: &mut GuestMem<'_>, pc: u64) -> SyscallOutcome {
self.syscall_pc = pc;
// An i386-ABI process has its number in eax (`x(0)`) in every build:
// on an AArch64 build that is a native wasm32 program.
let nr_slot = if self.compat && !gex_abi::PPC { ctx::x(0) } else { ctx::SYSCALL_NR };
let raw = m
.read_u64(self.ctx_addr + ctx::SYSCALL_NR as u64)
.read_u64(self.ctx_addr + nr_slot as u64)
.unwrap_or(!0);
let mut a: [u64; 6] = if self.compat && !gex_abi::PPC {
let mut a = [0u64; 6];
@@ -1675,12 +1737,12 @@ impl Kernel {
post = p;
ppc_post = pp;
n
} else if gex_abi::X86 && self.compat {
} else if self.compat {
let (n, p) = crate::i386::translate(
raw & 0xffff_ffff,
&mut a,
m,
self.ctx_addr + ctx::SCRATCH as u64,
self.ctx_addr + ctx::COMPAT_SCRATCH as u64,
self.ctx_addr,
);
post = p;
@@ -1699,6 +1761,7 @@ impl Kernel {
};
self.fd_activity |= wakes_fd_waiters(n);
self.syscall_nr = n;
// Returning from a handler restores a whole context, so it cannot go
// through the normal "write x0 and continue" path.
@@ -1925,7 +1988,7 @@ impl Kernel {
}
// An x86-64 number with no mapping reaches `dispatch` as `UNKNOWN`;
// record the guest's own number, which is the one worth looking up.
let outcome = if (gex_abi::X86 || gex_abi::PPC) && n == crate::x86::UNKNOWN {
let outcome = if (gex_abi::X86 || gex_abi::PPC || self.compat) && n == crate::x86::UNKNOWN {
let before = self.unimplemented.len();
let r = self.dispatch(m, n, a);
if self.unimplemented.len() > before {
@@ -2088,6 +2151,7 @@ impl Kernel {
out: a[1],
max: a[2] as u32,
data: set.watches().iter().map(|w| (w.fd, w.data)).collect(),
packed: gex_abi::X86 || self.compat,
},
})
}
@@ -2100,7 +2164,7 @@ impl Kernel {
)
} else {
(
self.collect_selectfds(m, a[0] as u32, a[1], a[2], a[3])?,
self.collect_selectfds(m, a[0] as u32, a[1], a[2], a[3]).ok()?,
PollReport::Select {
nfds: a[0] as u32,
readfds: a[1],
@@ -2438,11 +2502,11 @@ impl Kernel {
// -- x86-64 legacy forms (see `crate::x86`) ----------------------------
// Only `x86::translate` produces these numbers; an AArch64 guest
// issuing one raw falls through to ENOSYS below.
nr::DUP2 if gex_abi::X86 || gex_abi::PPC => self.sys_dup2(a[0] as i32, a[1] as i32),
nr::TIME if gex_abi::X86 || gex_abi::PPC => self.sys_time(m, a[0]),
nr::ALARM if gex_abi::X86 || gex_abi::PPC => self.sys_alarm(m, a[0]),
nr::DUP2 if gex_abi::X86 || gex_abi::PPC || self.compat => self.sys_dup2(a[0] as i32, a[1] as i32),
nr::TIME if gex_abi::X86 || gex_abi::PPC || self.compat => self.sys_time(m, a[0]),
nr::ALARM if gex_abi::X86 || gex_abi::PPC || self.compat => self.sys_alarm(m, a[0]),
nr::ARCH_PRCTL if gex_abi::X86 => self.sys_arch_prctl(m, a[0], a[1]),
nr::NOOP_OK if gex_abi::X86 || gex_abi::PPC => Ok(0),
nr::NOOP_OK if gex_abi::X86 || gex_abi::PPC || self.compat => Ok(0),
// -- I/O ---------------------------------------------------------
nr::WRITE => self.sys_write(m, a[0] as i32, a[1], a[2]),
@@ -2538,7 +2602,22 @@ impl Kernel {
// Ownership is not modelled: there is one user and the guest is
// it. Refusing would be worse than accepting -- tar treats a
// failed chown while unpacking as a failure to unpack.
nr::FCHOWN | nr::FCHOWNAT => Ok(0),
// Ownership is not kept, but the file must be there.
nr::FCHOWN => self.fds.get(a[0] as i32).map(|_| 0),
nr::FCHOWNAT => {
let empty = m.read_cstr(a[1], 4096).map(|r| r.is_empty()).unwrap_or(false);
if empty && a[4] & AT_EMPTY_PATH != 0 {
self.fds.get(a[0] as i32).map(|_| 0)
} else {
self.at_path(m, a[0] as i32, a[1]).and_then(|p| {
if a[4] & AT_SYMLINK_NOFOLLOW != 0 {
self.vfs.metadata_nofollow(&p).map(|_| 0)
} else {
self.vfs.metadata(&p).map(|_| 0)
}
})
}
}
// -- memory ------------------------------------------------------
nr::BRK => {
@@ -2618,7 +2697,7 @@ impl Kernel {
nr::UMOUNT2 => self.sys_umount(m, a[0]),
// -- sockets -----------------------------------------------------
nr::SOCKET => self.sys_socket(a[0], a[1]),
nr::SOCKET => self.sys_socket(a[0], a[1], a[2]),
// A unix socket and an internet socket are different concrete
// types behind the same descriptor, so which `connect` this is
// depends on what the descriptor turned out to be.
@@ -2627,7 +2706,7 @@ impl Kernel {
other => other,
},
nr::SENDTO => self.sys_sendto(m, a[0] as i32, a[1], a[2], a[4]),
nr::RECVFROM => self.sys_recvfrom(m, a[0] as i32, a[1], a[2], a[4], a[5]),
nr::RECVFROM => self.sys_recvfrom(m, a[0] as i32, a[1], a[2], a[3], a[4], a[5]),
nr::SENDMSG => self.sys_sendmsg(m, a[0] as i32, a[1]),
nr::RECVMSG => self.sys_recvmsg(m, a[0] as i32, a[1], a[2]),
nr::SENDMMSG => self.sys_sendmmsg(m, a[0] as i32, a[1], a[2]),
@@ -2642,13 +2721,17 @@ impl Kernel {
// Nothing here has an option worth setting -- no timeouts, no
// buffer sizes that mean anything -- and refusing makes libraries
// treat the socket as broken.
nr::SETSOCKOPT => Ok(0),
// agreed to without effect, on a descriptor that exists
nr::SETSOCKOPT => self.fds.get(a[0] as i32).map(|_| 0),
nr::GETSOCKOPT => self.sys_getsockopt(m, a[0] as i32, a[2], a[3]),
// A unix socket's directions close as asked; any other socket
// (the proxied network, the global kernel's) just agrees.
nr::SHUTDOWN => match self.unix_of(a[0] as i32) {
Ok(sock) => sock.shutdown_how(a[1]),
Err(_) => self.fds.get(a[0] as i32).map(|_| 0),
Err(_) => match self.socket_of(a[0] as i32) {
Ok(s) => s.shutdown(a[1]),
Err(_) => self.fds.get(a[0] as i32).map(|_| 0),
},
},
// A local address means nothing when there is one interface and
// it is a proxy, but a resolver binds its socket before it sends
@@ -2735,7 +2818,7 @@ impl Kernel {
nr::TIMES => self.sys_times(m, a[0]),
nr::GETRUSAGE => self.sys_getrusage(m, a[1]),
nr::SYSINFO => self.sys_sysinfo(m, a[0]),
nr::PRLIMIT64 => self.sys_prlimit(m, a[2], a[3]),
nr::PRLIMIT64 => self.sys_prlimit(m, a[1], a[2], a[3]),
// -- signals -----------------------------------------------------
nr::RT_SIGACTION => self.sys_rt_sigaction(m, a[0] as u32, a[1], a[2]),
@@ -3164,12 +3247,86 @@ impl Kernel {
fn at_path(&self, m: &GuestMem<'_>, dirfd: i32, path: u64) -> Result<String, Errno> {
let raw = m.read_cstr(path, 4096)?;
let p = String::from_utf8(raw).map_err(|_| Errno::EINVAL)?;
if p.starts_with('/') || dirfd == AT_FDCWD {
return Ok(self.vfs.absolute(&p));
self.at_path_str(dirfd, &p)
}
/// [`Self::at_path`] for a path already read.
fn at_path_str(&self, dirfd: i32, p: &str) -> Result<String, Errno> {
let p = p.to_string();
// An empty path names nothing (the AT_EMPTY_PATH forms check for it
// before they get here), as on Linux.
if p.is_empty() {
return Err(Errno::ENOENT);
}
// Relative to an open directory: its recorded path is the base.
let base = self.fds.get(dirfd)?.path.clone();
Ok(crate::vfs::normalize(&p, &base))
let abs = if p.starts_with('/') || dirfd == AT_FDCWD {
self.vfs.absolute(&p)
} else {
self.at_dir_path(dirfd, &p)?
};
// A trailing slash names a directory: "file/" is ENOTDIR (the
// normalised path has lost the slash, so look now).
if p.len() > 1 && p.ends_with('/') {
let followed = self.vfs.metadata(&abs);
if let Ok(md) = followed.as_ref() {
if md.file_type != crate::vfs::FileType::Directory {
return Err(Errno::ENOTDIR);
}
// "link/" names the directory the link points to, even for
// lstat and the other calls that do not follow a last link
let mut cur = abs;
for _ in 0..40 {
match self.vfs.read_link(&cur) {
Ok(t) if t.starts_with('/') => cur = crate::vfs::normalize(&t, "/"),
Ok(t) => cur = crate::vfs::normalize(&t, &crate::vfs::parent_of(&cur)),
Err(_) => break,
}
}
return Ok(cur);
}
// a dangling or looping link with a slash names nothing: the
// error following it gives (ENOENT, ELOOP)
if self.vfs.metadata_nofollow(&abs).is_ok() {
return Err(followed.err().unwrap_or(Errno::ENOENT));
}
}
Ok(abs)
}
/// How the path at `ptr` ends, which normalising loses: with a slash
/// (`name/`), and whether its last component is `.` (1) or `..` (2).
fn path_tail(m: &GuestMem<'_>, ptr: u64) -> Result<(bool, u8), Errno> {
let raw = m.read_cstr(ptr, 4096)?;
let slash = raw.len() > 1 && raw.ends_with(b"/");
let last = raw.split(|&b| b == b'/').filter(|c| !c.is_empty()).next_back();
let dot = match last {
Some(b".") => 1,
Some(b"..") => 2,
_ => 0,
};
Ok((slash, dot))
}
/// A new non-directory name (link, symlink) that ends in a slash or in
/// `.`/`..` names a directory: EEXIST when something is there, else
/// ENOENT, as Linux answers.
fn new_name_check(&self, m: &GuestMem<'_>, ptr: u64, p: &str) -> Result<(), Errno> {
let (slash, dot) = Self::path_tail(m, ptr)?;
if slash || dot != 0 {
return Err(if self.vfs.metadata_nofollow(p).is_ok() { Errno::EEXIST } else { Errno::ENOENT });
}
Ok(())
}
/// `p` relative to the directory open at `dirfd`.
fn at_dir_path(&self, dirfd: i32, p: &str) -> Result<String, Errno> {
// Relative to an open directory: its recorded path is the base. A
// descriptor that is not a directory is ENOTDIR.
let f = self.fds.get(dirfd)?;
if f.metadata().is_ok_and(|md| md.file_type != crate::vfs::FileType::Directory) {
return Err(Errno::ENOTDIR);
}
let base = f.path.clone();
Ok(crate::vfs::normalize(p, &base))
}
// -- I/O -----------------------------------------------------------------
@@ -3186,8 +3343,11 @@ impl Kernel {
fn sys_read(&mut self, m: &mut GuestMem<'_>, fd: i32, buf: u64, len: u64) -> SysResult {
// Straight into the guest's memory, no intermediate buffer: the guest
// picks the length and a copy would allocate that much on every call.
// Capped at 16 MiB.
let len = len.min(1 << 24) as usize;
// Capped as Linux caps it (MAX_RW_COUNT): a regular file gives every
// byte asked for below that, and programs rely on it (Wine maps a PE
// image's sections with one pread each; a 16 MiB cap left the rest of
// a 24 MiB .text zero).
let len = rw_len(m, buf, len);
let dst = m.read_mut(buf, len)?;
self.fds.get(fd)?.read(dst)
}
@@ -3201,7 +3361,7 @@ impl Kernel {
off: u64,
) -> SysResult {
// As `sys_read`: the guest's buffer is the destination already.
let len = len.min(1 << 24) as usize;
let len = rw_len(m, buf, len);
let dst = m.read_mut(buf, len)?;
self.fds.get(fd)?.pread(off, dst)
}
@@ -3328,7 +3488,7 @@ impl Kernel {
nr::SENDFILE => self.sys_sendfile(m, a[0] as i32, a[1] as i32, a[2], a[3]),
// A blocking `recvfrom`/`recv` that waited for a datagram or
// bytes (the classifier below).
nr::RECVFROM => self.sys_recvfrom(m, a[0] as i32, a[1], a[2], a[4], a[5]),
nr::RECVFROM => self.sys_recvfrom(m, a[0] as i32, a[1], a[2], a[3], a[4], a[5]),
nr::COPY_FILE_RANGE => self.sys_copy_file_range(m, a[0] as i32, a[1], a[2] as i32, a[3], a[4], a[5] as u32),
_ => self.sys_splice(m, a[0] as i32, a[1], a[2] as i32, a[3], a[4]),
}
@@ -3553,6 +3713,24 @@ impl Kernel {
if let Some(base) = self.pin_for(&p) {
self.armed_pin = Some((p.clone(), base));
}
// O_CREAT of "name/" that is not there: a slash names a
// directory, which open does not create (EISDIR, as Linux).
if flags & oflags::O_CREAT != 0 && Self::path_tail(m, path)?.0 && self.vfs.metadata(&p).is_err() {
return Err(Errno::EISDIR);
}
// O_NOFOLLOW on a symlink: ELOOP.
if flags & oflags::O_NOFOLLOW != 0
&& self.vfs.metadata_nofollow(&p).is_ok_and(|md| md.file_type == FileType::Symlink)
{
return Err(Errno::ELOOP);
}
// A directory opens for reading only, whatever the filesystem.
let acc = flags & oflags::ACCMODE;
if (acc == oflags::O_WRONLY || acc == oflags::O_RDWR)
&& self.vfs.metadata(&p).is_ok_and(|md| md.file_type == FileType::Directory)
{
return Err(Errno::EISDIR);
}
let h = self.vfs.open(&p, flags, mode)?;
// And arm a placement, for an object nobody pinned. Only for a
// shared object: the executable's own base is fixed by the
@@ -3655,6 +3833,7 @@ impl Kernel {
from
};
let to = self.at_path(m, a[2] as i32, a[3])?;
self.new_name_check(m, a[3], &to)?;
self.vfs.link(&from, &to).map(|_| 0)
}
@@ -3852,6 +4031,12 @@ impl Kernel {
blksize: 4096,
blocks: (md.size as i64 + 511) / 512,
mtime: md.mtime,
// A filesystem that keeps one time reports it for all three.
mtime_ns: md.times.mtime_ns,
atime: if md.times == Default::default() { md.mtime } else { md.times.atime },
atime_ns: md.times.atime_ns,
ctime: if md.times == Default::default() { md.mtime } else { md.times.ctime },
ctime_ns: md.times.ctime_ns,
// What the node says, or for a character device that does not,
// a pseudo-terminal's number (136:0).
rdev: if md.rdev != 0 {
@@ -3912,10 +4097,16 @@ impl Kernel {
put64(&mut out, 32, st.ino);
put64(&mut out, 40, st.size as u64);
put64(&mut out, 48, st.blocks as u64);
// atime, btime, ctime, mtime: 16 bytes each from offset 64. Only the
// seconds are known, and all four get the same one.
for at in [64usize, 80, 96, 112] {
put64(&mut out, at, st.mtime as u64);
// atime, btime, ctime, mtime: 16 bytes each from offset 64, seconds
// then nanoseconds; btime is reported as mtime.
for (at, (t, ns)) in [64usize, 80, 96, 112].into_iter().zip([
(st.atime, st.atime_ns),
(st.mtime, st.mtime_ns),
(st.ctime, st.ctime_ns),
(st.mtime, st.mtime_ns),
]) {
put64(&mut out, at, t as u64);
put32(&mut out, at + 8, ns);
}
put32(&mut out, 128, (st.rdev >> 8) as u32);
put32(&mut out, 132, (st.rdev & 0xff) as u32);
@@ -3966,6 +4157,8 @@ impl Kernel {
const F_ADD_SEALS: u64 = 1033;
const F_GET_SEALS: u64 = 1034;
const F_DUPFD_CLOEXEC: u64 = 1030;
// every command is on an open descriptor
self.fds.get(fd)?;
match cmd {
F_DUPFD | F_DUPFD_CLOEXEC => {
let file = self.fds.get(fd)?;
@@ -4508,6 +4701,9 @@ impl Kernel {
fn sys_chdir(&mut self, m: &GuestMem<'_>, path: u64) -> SysResult {
let raw = m.read_cstr(path, 4096)?;
let p = String::from_utf8(raw).map_err(|_| Errno::EINVAL)?;
if p.is_empty() {
return Err(Errno::ENOENT);
}
self.vfs.set_cwd(&p).map(|_| 0)
}
@@ -4529,13 +4725,33 @@ impl Kernel {
}
fn sys_mkdirat(&mut self, m: &GuestMem<'_>, dirfd: i32, path: u64, mode: u32) -> SysResult {
// "link/" where link dangles: the name is taken (EEXIST); resolving
// the slash would say ENOENT
if Self::path_tail(m, path)?.0 {
let raw = String::from_utf8(m.read_cstr(path, 4096)?).map_err(|_| Errno::EINVAL)?;
let bare = self.at_path_str(dirfd, raw.trim_end_matches('/'))?;
if self.vfs.metadata_nofollow(&bare).is_ok() {
return Err(Errno::EEXIST);
}
}
let p = self.at_path(m, dirfd, path)?;
// "dir/." and "dir/.." exist if dir does, else ENOENT
if Self::path_tail(m, path)?.1 != 0 {
return Err(if self.vfs.metadata(&p).is_ok() { Errno::EEXIST } else { Errno::ENOENT });
}
self.vfs.mkdir(&p, mode & !self.umask).map(|_| 0)
}
fn sys_unlinkat(&mut self, m: &GuestMem<'_>, dirfd: i32, path: u64, flags: u64) -> SysResult {
let tail = Self::path_tail(m, path)?;
let p = self.at_path(m, dirfd, path)?;
if flags & AT_REMOVEDIR != 0 {
// rmdir("dir/.") is EINVAL, rmdir("dir/..") ENOTEMPTY
match tail.1 {
1 => return Err(Errno::EINVAL),
2 => return Err(Errno::ENOTEMPTY),
_ => {}
}
self.vfs.rmdir(&p).map(|_| 0)
} else {
self.vfs.unlink(&p).map(|_| 0)
@@ -4550,9 +4766,7 @@ impl Kernel {
new_fd: i32,
new: u64,
) -> SysResult {
let from = self.at_path(m, old_fd, old)?;
let to = self.at_path(m, new_fd, new)?;
self.vfs.rename(&from, &to).map(|_| 0)
self.sys_renameat2(m, old_fd, old, new_fd, new, 0)
}
/// `renameat2`: `renameat` plus a flags word. The flags matter -- programs
@@ -4578,8 +4792,32 @@ impl Kernel {
) -> SysResult {
const RENAME_NOREPLACE: u32 = 1;
const RENAME_EXCHANGE: u32 = 2;
// "." or ".." as the last component of either name: EINVAL (the
// normalised paths would name the directory itself)
for ptr in [old, new] {
let raw = m.read_cstr(ptr, 4096)?;
let last = raw.split(|&b| b == b'/').filter(|c| !c.is_empty()).next_back();
if matches!(last, Some(b".") | Some(b"..")) {
return Err(Errno::EINVAL);
}
}
let from = self.at_path(m, old_fd, old)?;
// "x" to "link/" where link is a symlink: the slash asks to follow
// it, and rename never replaces through a link (ENOTDIR, as GNU)
if Self::path_tail(m, new)?.0 {
let raw = String::from_utf8(m.read_cstr(new, 4096)?).map_err(|_| Errno::EINVAL)?;
let bare = self.at_path_str(new_fd, raw.trim_end_matches('/'))?;
if self.vfs.metadata_nofollow(&bare).is_ok_and(|md| md.file_type == FileType::Symlink) {
return Err(Errno::ENOTDIR);
}
}
let to = self.at_path(m, new_fd, new)?;
// "file" to "name/": a slash names a directory, which a file is not
if Self::path_tail(m, new)?.0
&& self.vfs.metadata_nofollow(&from).is_ok_and(|md| md.file_type != FileType::Directory)
{
return Err(Errno::ENOTDIR);
}
if flags & !RENAME_NOREPLACE != 0 {
// EXCHANGE, WHITEOUT, or any unknown bit: unsupported here.
let _ = RENAME_EXCHANGE;
@@ -4604,25 +4842,29 @@ impl Kernel {
const AT_SYMLINK_NOFOLLOW: u32 = 0x100;
const UTIME_NOW: u64 = (1 << 30) - 1;
const UTIME_OMIT: u64 = (1 << 30) - 2;
// The modification time to set, if any.
let mtime = if times == 0 {
Some((self.clock)().0 as i64)
// The access and modification times to set (`None`: leave it).
let now = {
let (s, ns) = (self.clock)();
(s as i64, ns as u32)
};
let (atime, mtime) = if times == 0 {
(Some(now), Some(now))
} else {
for t in [times, times + 16] {
let mut v = [None, None];
for (k, t) in [times, times + 16].into_iter().enumerate() {
let ns = m.read_u64(t + 8)?;
if ns >= 1_000_000_000 && ns != UTIME_NOW && ns != UTIME_OMIT {
return Err(Errno::EINVAL);
}
}
match m.read_u64(times + 24)? {
UTIME_OMIT => None,
UTIME_NOW => Some((self.clock)().0 as i64),
_ => Some(m.read_u64(times + 16)? as i64),
v[k] = match ns {
UTIME_OMIT => None,
UTIME_NOW => Some(now),
_ if ns >= 1_000_000_000 => return Err(Errno::EINVAL),
_ => Some((m.read_u64(t)? as i64, ns as u32)),
};
}
(v[0], v[1])
};
if path == 0 {
// futimens: the "path" is really the open descriptor in `dirfd`.
self.fds.get(dirfd)?;
self.fds.get(dirfd)?.handle.set_times(atime, mtime)?;
return Ok(0);
}
let p = self.at_path(m, dirfd, path)?;
@@ -4632,8 +4874,8 @@ impl Kernel {
} else {
self.vfs.metadata_nofollow(&p)?;
}
if let Some(t) = mtime {
self.vfs.set_mtime(&p, t, follow)?;
if atime.is_some() || mtime.is_some() {
self.vfs.set_times(&p, atime, mtime, follow)?;
}
Ok(0)
}
@@ -4650,6 +4892,7 @@ impl Kernel {
) -> SysResult {
let t = String::from_utf8(m.read_cstr(target, 4096)?).map_err(|_| Errno::EINVAL)?;
let p = self.at_path(m, newdirfd, path)?;
self.new_name_check(m, path, &p)?;
self.vfs.symlink(&t, &p).map(|_| 0)
}
@@ -4686,17 +4929,21 @@ impl Kernel {
readfds: u64,
writefds: u64,
exceptfds: u64,
) -> Option<Vec<PollFd>> {
) -> Result<Vec<PollFd>, Errno> {
const POLLIN: u16 = 0x001;
const POLLOUT: u16 = 0x004;
if (nfds as i32) < 0 {
return Err(Errno::EINVAL);
}
let nfds = nfds.min(1024);
let mut want: std::collections::BTreeMap<i32, u16> = Default::default();
for (set, ev) in [(readfds, POLLIN), (writefds, POLLOUT)] {
// (exceptfds: watched for nothing, but its descriptors must be open)
for (set, ev) in [(readfds, POLLIN), (writefds, POLLOUT), (exceptfds, 0)] {
if set == 0 {
continue;
}
for w in 0..(nfds as usize).div_ceil(64) {
let word = m.read_bits64(set + (w as u64) * 8).ok()?;
let word = m.read_bits64(set + (w as u64) * 8).map_err(|_| Errno::EFAULT)?;
if word == 0 {
continue;
}
@@ -4712,9 +4959,12 @@ impl Kernel {
}
}
// An exceptional condition never happens here, so a descriptor named
// only in that set is watched for nothing and simply never fires.
let _ = exceptfds;
Some(
// only in that set is watched for nothing and simply never fires. A
// descriptor in any set that is not open is EBADF, as on Linux.
if want.keys().any(|&fd| self.fds.get(fd).is_err()) {
return Err(Errno::EBADF);
}
Ok(
want.into_iter()
.map(|(fd, events)| {
let file = self.fds.get(fd).ok();
@@ -4962,9 +5212,7 @@ impl Kernel {
/// `pselect6`, for the case where something is ready already. The waiting
/// case is a [`BlockOp::Poll`], decided before this is reached.
fn sys_pselect(&mut self, m: &mut GuestMem<'_>, a: [u64; 6]) -> SysResult {
let set = self
.collect_selectfds(m, a[0] as u32, a[1], a[2], a[3])
.ok_or(Errno::EFAULT)?;
let set = self.collect_selectfds(m, a[0] as u32, a[1], a[2], a[3])?;
poll_finish(
&set,
&PollReport::Select {
@@ -5348,7 +5596,7 @@ impl Kernel {
/// `socket`. AF_UNIX, an inert AF_NETLINK (below), and AF_INET with only the two kinds that matter: a stream,
/// which the host carries, and a datagram, which only ever goes to the
/// resolver this kernel is.
fn sys_socket(&mut self, domain: u64, ty: u64) -> SysResult {
fn sys_socket(&mut self, domain: u64, ty: u64, protocol: u64) -> SysResult {
const AF_UNIX: u64 = 1;
const AF_INET: u64 = 2;
const SOCK_STREAM: u64 = 1;
@@ -5375,16 +5623,25 @@ impl Kernel {
// its video subsystem too, and Unity 6 then saw no display). There
// are no devices to announce, so the socket binds, takes its options
// and never has a message: an empty eventfd behind the descriptor.
// Only NETLINK_KOBJECT_UEVENT: glibc's getaddrinfo (AI_ADDRCONFIG)
// asks NETLINK_ROUTE for the addresses, and an eventfd answers its
// request with the 8-byte counter its own send wrote, which glibc
// aborts on (apt's http method). Without a route socket it assumes
// IPv4 and IPv6 are there.
const AF_NETLINK: u64 = 16;
const NETLINK_KOBJECT_UEVENT: u64 = 15;
if domain == AF_NETLINK && protocol != NETLINK_KOBJECT_UEVENT {
return Err(Errno::EAFNOSUPPORT);
}
if domain == AF_NETLINK {
let handle: Arc<dyn crate::vfs::FileHandle> = Arc::new(crate::eventfd::EventFd::new(0, false));
let flags = if ty & SOCK_NONBLOCK != 0 { crate::fs::oflags::O_NONBLOCK } else { 0 };
let file = OpenFile::new(handle, "socket:[netlink]".to_string(), flags, true, true);
return self.fds.insert(file, ty & SOCK_CLOEXEC != 0);
}
let Some(net) = self.net.clone() else {
return Err(Errno::EAFNOSUPPORT);
};
// Without a network stack (the browser with no relay), loopback still
// works: Wine's in-process server and Winsock use it.
let net = self.net.clone().unwrap_or_else(|| Arc::new(net::NoNet) as Arc<dyn net::NetStack>);
if domain != AF_INET {
return Err(Errno::EAFNOSUPPORT);
}
@@ -5591,7 +5848,7 @@ impl Kernel {
// `EPOLL_CTL_DEL` is allowed a null event pointer.
let (events, data) = if event == 0 {
(0, 0)
} else if gex_abi::X86 {
} else if gex_abi::X86 || self.compat {
(m.read_u32(event)?, m.read_u64(event + 4)?)
} else {
(m.read_u32(event)?, m.read_u64(event + 8)?)
@@ -5692,7 +5949,7 @@ impl Kernel {
) -> SysResult {
let n = ready.len().min(max as usize);
for (i, (events, data)) in ready.iter().take(n).enumerate() {
if gex_abi::X86 {
if gex_abi::X86 || self.compat {
// Packed: 12 bytes, the token at offset four.
let at = out + i as u64 * 12;
m.write_u32(at, *events)?;
@@ -5966,12 +6223,16 @@ impl Kernel {
fd: i32,
buf: u64,
len: u64,
flags: u64,
addr: u64,
addrlen: u64,
) -> SysResult {
let len = (len as usize).min(1 << 22);
let mut tmp = vec![0u8; len];
let n = self.fds.get(fd)?.read(&mut tmp)?;
let n = match self.socket_of(fd) {
Ok(sock) if flags & Self::MSG_PEEK != 0 => sock.peek(&mut tmp)?,
_ => self.fds.get(fd)?.read(&mut tmp)?,
};
m.write(buf, &tmp[..n as usize])?;
if addr != 0 {
// Where the datagram came from (a guest's port, the resolver),
@@ -6284,6 +6545,10 @@ impl Kernel {
let mut tmp = vec![0u8; len.min(1 << 22)];
let n = match &unix_sock {
Some(sock) => sock.recv(&mut tmp, room, peek)?,
None if peek => match self.socket_of(fd) {
Ok(sock) => sock.peek(&mut tmp)?,
Err(_) => self.fds.get(fd)?.read(&mut tmp)?,
},
None => self.fds.get(fd)?.read(&mut tmp)?,
};
m.write(base, &tmp[..n as usize])?;
@@ -6294,9 +6559,23 @@ impl Kernel {
break;
}
}
let is_unix = unix_sock.is_some();
let received = unix_sock.map(|s| s.take_received()).unwrap_or_default();
let written = self.write_scm_rights(m, control, controllen, received)?;
self.msg_wr(m, msg, 5, written)?;
// Where the datagram came from, as `recvfrom` says it. musl's
// resolver (1.2.4 on) reads its answers with `recvmsg` and drops any
// whose source is not a nameserver it asked: with the name left
// unwritten every answer was dropped, and each `getaddrinfo` waited
// out the resolver's 5 s timeout and failed.
let name = self.msg_rd(m, msg, 0)?;
if name != 0 && !is_unix {
if let Ok(sock) = self.socket_of(fd) {
let (ip, port) = sock.last_from().or_else(|| sock.peer()).unwrap_or((0, 0));
let namelen = msg + if self.compat { 4 } else { 8 };
self.sys_write_sockaddr(m, ip, port, name, namelen)?;
}
}
Ok(total)
}
@@ -6324,7 +6603,7 @@ impl Kernel {
/// previous timer had left. Both itimervals live in the thread's scratch
/// area, since the call has none of its own.
fn sys_alarm(&mut self, m: &mut GuestMem<'_>, secs: u64) -> SysResult {
let scratch = self.ctx_addr + ctx::SCRATCH as u64;
let scratch = self.ctx_addr + ctx::COMPAT_SCRATCH as u64;
for off in 0..4 {
m.write_u64(scratch + off * 8, 0)?;
}
@@ -6364,7 +6643,10 @@ impl Kernel {
"gex",
"6.1.0-gex",
"#1 SMP gex",
if self.compat && !gex_abi::PPC { "i686" } else { gex_abi::ARCH_NAME },
// An i386 process on the x86 build; a wasm32 program on the
// AArch64 build is part of an aarch64 system (its dpkg and
// config.guess should agree with the arm64 programs beside it).
if self.compat && gex_abi::X86 { "i686" } else { gex_abi::ARCH_NAME },
"(none)",
];
let mut out = vec![0u8; 65 * 6];
@@ -6462,18 +6744,28 @@ impl Kernel {
/// on Linux, and a return value on the same scale.
fn sys_times(&mut self, m: &mut GuestMem<'_>, buf: u64) -> SysResult {
let ticks = (self.cpu_nanos() / 10_000_000) as u64;
if buf != 0 {
if buf != 0 && self.compat {
// i386: four 32-bit clock_t
m.zero(buf, 16)?;
m.write_u32(buf, ticks as u32)?;
} else if buf != 0 {
m.zero(buf, 32)?;
m.write_u64(buf, ticks)?;
}
Ok(ticks)
Ok(if self.compat { ticks as u32 as u64 } else { ticks })
}
/// `getrusage`. Only the two times are filled in; the page-fault and
/// context-switch counters stay zero, which is what a program that asks
/// for them under emulation should expect to see.
fn sys_getrusage(&mut self, m: &mut GuestMem<'_>, buf: u64) -> SysResult {
if buf != 0 {
if buf != 0 && self.compat {
// i386: 32-bit timevals and longs (72 bytes)
let ns = self.cpu_nanos();
m.zero(buf, 72)?;
m.write_u32(buf, (ns / 1_000_000_000) as u32)?;
m.write_u32(buf + 4, (ns % 1_000_000_000 / 1000) as u32)?;
} else if buf != 0 {
m.zero(buf, 144)?;
write_timeval(m, buf, self.cpu_nanos())?;
}
@@ -6519,12 +6811,41 @@ impl Kernel {
Ok(len as u64)
}
fn sys_prlimit(&mut self, m: &mut GuestMem<'_>, new: u64, old: u64) -> SysResult {
/// `prlimit64(pid, resource, new, old)`, for the calling process (gex's
/// processes set only their own). Limits are recorded and reported;
/// RLIMIT_NOFILE is enforced by the descriptor table, the rest are not.
fn sys_prlimit(&mut self, m: &mut GuestMem<'_>, resource: u64, new: u64, old: u64) -> SysResult {
let r = resource as usize;
if r >= RLIM_NLIMITS {
return Err(Errno::EINVAL);
}
if r == RLIMIT_NOFILE {
self.rlimits[r][0] = self.fds.limit() as u64;
}
let cur = self.rlimits[r];
let want = if new != 0 { Some([m.read_u64(new)?, m.read_u64(new + 8)?]) } else { None };
if let Some([soft, hard]) = want {
if soft > hard {
return Err(Errno::EINVAL);
}
// raising the hard limit takes privilege; NOFILE cannot pass
// what the table holds
if (hard > cur[1] && self.uid != 0)
|| (r == RLIMIT_NOFILE && hard > crate::file::NOFILE_HARD as u64)
{
return Err(Errno::EPERM);
}
}
if old != 0 {
m.write_u64(old, 8 * 1024 * 1024)?;
m.write_u64(old + 8, u64::MAX)?;
m.write_u64(old, cur[0])?;
m.write_u64(old + 8, cur[1])?;
}
if let Some(lim) = want {
self.rlimits[r] = lim;
if r == RLIMIT_NOFILE {
self.fds.set_limit(lim[0] as usize);
}
}
let _ = new;
Ok(0)
}
}
@@ -6917,11 +7238,32 @@ pub fn syscall_name(n: u64) -> &'static str {
}
}
/// Linux's per-call limit on read and write sizes (`INT_MAX & PAGE_MASK`).
const MAX_RW_COUNT: u64 = 0x7fff_f000;
/// A read's length: at most MAX_RW_COUNT, and no further than the guest's
/// memory goes from `buf` (Linux faults only on a page it touches, so a
/// length past the end with a short file still reads; a `buf` outside the
/// memory keeps its length and faults).
fn rw_len(m: &GuestMem<'_>, buf: u64, len: u64) -> usize {
let room = (m.bytes.len() as u64).saturating_sub(buf);
(if room == 0 { len.min(MAX_RW_COUNT) } else { len.min(MAX_RW_COUNT).min(room) }) as usize
}
#[cfg(test)]
mod tests {
use super::*;
use crate::vfs::TmpFs;
#[test]
fn poll_skips_negative_descriptors() {
let slot = |fd| PollFd { addr: 0, fd, events: 1, handle: None, writable: false };
// An unused slot reports nothing; a closed descriptor is POLLNVAL.
assert_eq!(poll_revents(&slot(-1)), 0);
assert_eq!(poll_revents(&slot(7)), 0x020);
assert!(!poll_ready(&[slot(-1), slot(-1)]));
}
fn setup() -> (Vec<u8>, Kernel) {
let mem = vec![0u8; 0x40000];
let mut k = Kernel::new(0x1000, 0x10000, "/prog".into());
@@ -7227,6 +7569,31 @@ mod tests {
assert_eq!(m.read(0x22000, 7).unwrap(), b"payload");
}
/// A regular file gives a read every byte asked for, past 16 MiB too
/// (Wine reads a PE section with one pread and trusts the count).
#[test]
fn a_large_pread_of_a_file_is_not_short() {
let mut buf = vec![0u8; 0x40000 + (40 << 20)];
let mut k = Kernel::new(0x1000, 0x10000, "/prog".into());
k.vfs.mount("/", Arc::new(TmpFs::new()));
let mut m = GuestMem::new(&mut buf);
put_str(&mut m, 0x20000, "/big");
set_args(&mut m, k.ctx_addr, nr::OPENAT, &[(-100i64) as u64, 0x20000, oflags::O_CREAT | oflags::O_RDWR, 0o644]);
k.syscall(&mut m, 0);
let fd = ret(&m, k.ctx_addr) as u64;
let n: u64 = 20 << 20;
let data: Vec<u8> = (0..n).map(|i| (i % 251) as u8 + 1).collect();
m.write(0x40000, &data).unwrap();
set_args(&mut m, k.ctx_addr, nr::WRITE, &[fd, 0x40000, n]);
k.syscall(&mut m, 0);
assert_eq!(ret(&m, k.ctx_addr), n as i64);
let dst = 0x40000 + (20 << 20);
set_args(&mut m, k.ctx_addr, nr::PREAD64, &[fd, dst, n, 0]);
k.syscall(&mut m, 0);
assert_eq!(ret(&m, k.ctx_addr), n as i64);
assert!(m.read(dst, n as usize).unwrap() == &data[..]);
}
#[test]
fn dup_shares_the_file_offset() {
let (mut buf, mut k) = setup();
crates/gex-kernel/src/timerfd.rs
+1
View File
@@ -157,6 +157,7 @@ impl FileHandle for TimerFd {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
crates/gex-kernel/src/unix.rs
+4
View File
@@ -372,6 +372,7 @@ impl FileHandle for UnixSocket {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
@@ -826,6 +827,7 @@ impl FileHandle for MemFd {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
}
@@ -1048,6 +1050,7 @@ impl FileHandle for EpollSet {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
}
@@ -1197,6 +1200,7 @@ impl crate::vfs::FileHandle for RemoteUnix {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
crates/gex-kernel/src/vfs/devfs.rs
+28 -2
View File
@@ -75,6 +75,7 @@ impl FileHandle for DevHandle {
mtime: 0,
nlink: 1,
rdev: rdev_of(self.device),
times: Default::default(),
})
}
@@ -91,11 +92,14 @@ impl FileHandle for DevHandle {
pub struct DevFs {
/// Nodes the host added, by name without a leading slash.
extra: Vec<(String, Arc<dyn FileHandle>)>,
/// Nodes the host added whose every open is a handle of its own (an
/// audio device: each open is a stream with its own format and buffer).
per_open: Vec<(String, Arc<dyn Fn() -> Arc<dyn FileHandle> + Send + Sync>)>,
}
impl DevFs {
pub fn new() -> DevFs {
DevFs { extra: Vec::new() }
DevFs { extra: Vec::new(), per_open: Vec::new() }
}
/// Add a device the host provides. The handle is shared, not cloned per
@@ -105,6 +109,19 @@ impl DevFs {
self.extra.push((name.to_string(), handle));
}
/// Add a device whose every open gets a new handle from `open`.
pub fn add_per_open(&mut self, name: &str, open: Arc<dyn Fn() -> Arc<dyn FileHandle> + Send + Sync>) {
self.per_open.retain(|(n, _)| n != name);
self.per_open.push((name.to_string(), open));
}
/// The OSS audio devices, `dsp` and `mixer` (`oss`: a null output).
pub fn add_oss(&mut self) {
use super::oss::{OssHandle, OssNode};
self.add_per_open("dsp", Arc::new(|| Arc::new(OssHandle::new(OssNode::Dsp)) as Arc<dyn FileHandle>));
self.add_per_open("mixer", Arc::new(|| Arc::new(OssHandle::new(OssNode::Mixer)) as Arc<dyn FileHandle>));
}
fn device_of(path: &str) -> Option<Device> {
Some(match path {
"null" => Device::Null,
@@ -131,9 +148,12 @@ impl FileSystem for DevFs {
if let Some((_, handle)) = self.extra.iter().find(|(n, _)| n == path) {
return Ok(Arc::clone(handle));
}
if let Some((_, open)) = self.per_open.iter().find(|(n, _)| n == path) {
return Ok(open());
}
if path.is_empty() {
return Ok(Arc::new(DevDir {
extra: self.extra.iter().map(|(n, _)| n.clone()).collect(),
extra: self.extra.iter().map(|(n, _)| n.clone()).chain(self.per_open.iter().map(|(n, _)| n.clone())).collect(),
}));
}
Err(Errno::ENOENT)
@@ -143,6 +163,9 @@ impl FileSystem for DevFs {
if path.is_empty() {
return Ok(Metadata::dir());
}
if let Some((_, open)) = self.per_open.iter().find(|(n, _)| n == path) {
return open().metadata();
}
if let Some(device) = Self::device_of(path) {
return Ok(Metadata {
file_type: FileType::CharDevice,
@@ -152,6 +175,7 @@ impl FileSystem for DevFs {
mtime: 0,
nlink: 1,
rdev: rdev_of(device),
times: Default::default(),
});
}
if matches!(path, "stdin" | "stdout" | "stderr" | "fd") {
@@ -163,6 +187,7 @@ impl FileSystem for DevFs {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
});
}
Err(Errno::ENOENT)
@@ -322,6 +347,7 @@ mod tests {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
fn ioctl(
crates/gex-kernel/src/vfs/hostfs.rs
+94 -8
View File
@@ -72,7 +72,7 @@ fn file_type_of(md: &std::fs::Metadata) -> FileType {
}
}
fn convert(md: &std::fs::Metadata) -> Metadata {
pub(crate) fn convert(md: &std::fs::Metadata) -> Metadata {
Metadata {
file_type: file_type_of(md),
size: md.len(),
@@ -81,6 +81,13 @@ fn convert(md: &std::fs::Metadata) -> Metadata {
mtime: md.mtime(),
nlink: md.nlink() as u32,
rdev: 0,
times: super::Times {
mtime_ns: md.mtime_nsec() as u32,
atime: md.atime(),
atime_ns: md.atime_nsec() as u32,
ctime: md.ctime(),
ctime_ns: md.ctime_nsec() as u32,
},
}
}
@@ -88,9 +95,33 @@ fn convert(md: &std::fs::Metadata) -> Metadata {
#[derive(Debug)]
struct HostHandle {
file: Mutex<File>,
/// Not a regular file (a FIFO, a socket, a device): no offset.
stream: bool,
}
/// `(seconds, nanoseconds)` since the epoch as a host time.
fn host_time((secs, ns): (i64, u32)) -> std::time::SystemTime {
let d = std::time::Duration::new(secs.unsigned_abs(), 0);
let t = if secs >= 0 { std::time::UNIX_EPOCH + d } else { std::time::UNIX_EPOCH - d };
t + std::time::Duration::from_nanos(ns as u64)
}
fn file_times(atime: Option<(i64, u32)>, mtime: Option<(i64, u32)>) -> std::fs::FileTimes {
let mut t = std::fs::FileTimes::new();
if let Some(a) = atime {
t = t.set_accessed(host_time(a));
}
if let Some(m) = mtime {
t = t.set_modified(host_time(m));
}
t
}
impl FileHandle for HostHandle {
fn set_times(&self, atime: Option<(i64, u32)>, mtime: Option<(i64, u32)>) -> Result<(), Errno> {
self.file.lock().unwrap().set_times(file_times(atime, mtime)).map_err(|e| Errno::from_io(&e))
}
/// `ftruncate`. The trait default answers `EINVAL`; apt truncates a file to
/// 0 before hashing a download and reported that as "Unable to truncate".
fn truncate(&self, len: u64) -> Result<(), Errno> {
@@ -103,8 +134,10 @@ impl FileHandle for HostHandle {
fn read_at(&self, offset: u64, buf: &mut [u8]) -> SysResult {
let mut f = self.file.lock().unwrap();
f.seek(SeekFrom::Start(offset))
.map_err(|e| Errno::from_io(&e))?;
if !self.stream {
f.seek(SeekFrom::Start(offset))
.map_err(|e| Errno::from_io(&e))?;
}
f.read(buf)
.map(|n| n as u64)
.map_err(|e| Errno::from_io(&e))
@@ -112,13 +145,19 @@ impl FileHandle for HostHandle {
fn write_at(&self, offset: u64, buf: &[u8]) -> SysResult {
let mut f = self.file.lock().unwrap();
f.seek(SeekFrom::Start(offset))
.map_err(|e| Errno::from_io(&e))?;
if !self.stream {
f.seek(SeekFrom::Start(offset))
.map_err(|e| Errno::from_io(&e))?;
}
f.write(buf)
.map(|n| n as u64)
.map_err(|e| Errno::from_io(&e))
}
fn is_stream(&self) -> bool {
self.stream
}
fn metadata(&self) -> Result<Metadata, Errno> {
let f = self.file.lock().unwrap();
f.metadata()
@@ -152,6 +191,11 @@ impl FileSystem for HostFs {
let host = self.host_path(path)?;
let md = std::fs::metadata(&host);
if let Ok(md) = &md {
// a directory opens for reading only
let acc = flags & oflags::ACCMODE;
if md.is_dir() && (acc == oflags::O_WRONLY || acc == oflags::O_RDWR) {
return Err(Errno::EISDIR);
}
if md.is_dir() {
let mut entries = vec![
DirEntry {
@@ -189,24 +233,51 @@ impl FileSystem for HostFs {
let mut opts = std::fs::OpenOptions::new();
opts.read(acc == oflags::O_RDONLY || acc == oflags::O_RDWR)
.write(want_write);
// Linux's O_CREAT, O_EXCL, O_TRUNC and O_NONBLOCK on the host.
let mut custom = 0;
if flags & oflags::O_CREAT != 0 {
// The guest's mode but never without owner rw: gex does not enforce
// permissions, but the host process is not root, and a file it
// creates mode 0 (as dpkg does for `.dpkg-new`) it cannot reopen.
opts.create(true).mode(mode | 0o600);
opts.mode(mode | 0o600);
if want_write {
opts.create(true);
} else {
// std refuses to create without write access; Linux creates
// with O_RDONLY (gnulib's open tests, `touch`-like opens)
custom |= 0o100;
}
}
if flags & oflags::O_EXCL != 0 {
opts.create_new(true);
if want_write {
opts.create_new(true);
} else if flags & oflags::O_CREAT != 0 {
custom |= 0o200;
}
}
if flags & oflags::O_TRUNC != 0 {
opts.truncate(true);
if want_write {
opts.truncate(true);
} else {
custom |= 0o1000;
}
}
if flags & oflags::O_APPEND != 0 {
opts.append(true);
}
// O_NONBLOCK reaches the host: opening a host FIFO without a writer
// must not block (grep -D skip opens one so, to fstat and skip it).
// Linux's O_NONBLOCK on the host.
if flags & oflags::O_NONBLOCK != 0 {
custom |= 0o4000;
}
opts.custom_flags(custom);
let file = opts.open(&host).map_err(|e| Errno::from_io(&e))?;
// A FIFO, socket or device has no offset: read and write it as is.
let stream = file.metadata().is_ok_and(|m| !m.is_file());
Ok(Arc::new(HostHandle {
file: Mutex::new(file),
stream,
}))
}
@@ -293,6 +364,21 @@ impl FileSystem for HostFs {
.map_err(|e| Errno::from_io(&e))
}
fn set_times(&self, path: &str, atime: Option<(i64, u32)>, mtime: Option<(i64, u32)>) -> Result<(), Errno> {
if self.read_only {
return Err(Errno::EROFS);
}
let host = self.host_path(path)?;
// A symlink's own times (`utimensat` with AT_SYMLINK_NOFOLLOW: dpkg
// stamps every link it unpacks, before its target exists): std opens
// only what a link names, so this leaves the link's times as they are.
if std::fs::symlink_metadata(&host).is_ok_and(|m| m.file_type().is_symlink()) {
return Ok(());
}
let f = std::fs::File::open(host).map_err(|e| Errno::from_io(&e))?;
f.set_times(file_times(atime, mtime)).map_err(|e| Errno::from_io(&e))
}
fn set_mtime(&self, path: &str, secs: i64) -> Result<(), Errno> {
if self.read_only {
return Err(Errno::EROFS);
crates/gex-kernel/src/vfs/mod.rs
+125 -3
View File
@@ -14,6 +14,7 @@ pub mod devfs;
// instead.
#[cfg(unix)]
pub mod hostfs;
pub mod oss;
pub mod procfs;
/// The filesystem when it is not in this process's memory.
pub mod remote;
@@ -86,6 +87,19 @@ pub struct Metadata {
/// Linux's `makedev` encoding. Zero where the node does not say; `stat`
/// then reports a character device as a pseudo-terminal (136:0).
pub rdev: u64,
/// The other times and the nanoseconds, where the filesystem keeps
/// them (the host's). All zero: every time is `mtime`, whole seconds.
pub times: Times,
}
/// `st_atim`, `st_ctim` and the nanoseconds of `st_mtim`.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Times {
pub mtime_ns: u32,
pub atime: i64,
pub atime_ns: u32,
pub ctime: i64,
pub ctime_ns: u32,
}
impl Metadata {
@@ -98,6 +112,7 @@ impl Metadata {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
}
}
pub fn dir() -> Metadata {
@@ -109,6 +124,7 @@ impl Metadata {
mtime: 0,
nlink: 2,
rdev: 0,
times: Default::default(),
}
}
}
@@ -149,6 +165,14 @@ pub trait FileHandle: Send + Sync + std::fmt::Debug {
fn set_mtime(&self, _secs: i64) -> Result<(), Errno> {
Ok(())
}
/// `futimens`: [`FileSystem::set_times`] on an open file.
fn set_times(&self, atime: Option<(i64, u32)>, mtime: Option<(i64, u32)>) -> Result<(), Errno> {
let _ = atime;
match mtime {
Some((secs, _)) => self.set_mtime(secs),
None => Ok(()),
}
}
fn truncate(&self, len: u64) -> Result<(), Errno> {
let _ = len;
Err(Errno::EINVAL)
@@ -158,6 +182,12 @@ pub trait FileHandle: Send + Sync + std::fmt::Debug {
fn is_stream(&self) -> bool {
false
}
/// `lseek` on a handle that keeps its own offset (a standard stream
/// redirected from a host file: the offset is the host descriptor's).
/// `None`: the open-file description's offset, or `ESPIPE` for a stream.
fn seek(&self, _offset: i64, _whence: u32) -> Option<SysResult> {
None
}
/// True when the file should behave like a terminal for `ioctl`.
fn is_tty(&self) -> bool {
false
@@ -325,6 +355,16 @@ pub trait FileSystem: Send + Sync {
let _ = (path, secs);
Ok(())
}
/// `utimensat` with both times and nanoseconds (`None`: leave it). A
/// filesystem that keeps only whole-second modification times takes
/// what it can.
fn set_times(&self, path: &str, atime: Option<(i64, u32)>, mtime: Option<(i64, u32)>) -> Result<(), Errno> {
let _ = atime;
match mtime {
Some((secs, _)) => self.set_mtime(path, secs),
None => Ok(()),
}
}
/// A second name for the same file. `EPERM` is what Linux says for a
/// filesystem that has no hard links, which is the honest default.
@@ -386,7 +426,58 @@ impl Vfs {
/// Longest prefix wins, so a `/proc` mount shadows the root mount for
/// everything beneath it.
fn resolve<'a>(&'a self, path: &str) -> Result<(&'a Arc<dyn FileSystem>, String), Errno> {
let abs = normalize(path, &self.cwd);
let abs = self.through_dir_links(normalize(path, &self.cwd))?;
self.mount_of(abs)
}
/// `abs` with every symlink among its directories followed (not the
/// final component: each caller decides about that). A filesystem here
/// is a map of whole paths, so `/bin/sh` under a merged /usr (`/bin ->
/// usr/bin`) names nothing until the `/bin` link is followed. Only the
/// root filesystem's links: /proc and /dev resolve their own names.
fn through_dir_links(&self, mut abs: String) -> Result<String, Errno> {
'again: for _ in 0..40 {
let comps: Vec<&str> = abs.split('/').filter(|c| !c.is_empty()).collect();
let mut prefix = String::new();
for (i, c) in comps.iter().enumerate().take(comps.len().saturating_sub(1)) {
prefix.push('/');
prefix.push_str(c);
let (at, fs) = self.mount_at(&prefix)?;
if at != "/" {
break;
}
if let Ok(target) = fs.read_link(prefix.trim_start_matches('/')) {
let base = if target.starts_with('/') {
normalize(&target, "/")
} else {
normalize(&target, &parent_of(&prefix))
};
abs = normalize(&comps[i + 1..].join("/"), &base);
continue 'again;
}
}
return Ok(abs);
}
Err(Errno::ELOOP)
}
/// The mount holding absolute path `abs`, and where it is mounted.
fn mount_at<'a>(&'a self, abs: &str) -> Result<(&'a str, &'a Arc<dyn FileSystem>), Errno> {
let mut best: Option<(&String, &Arc<dyn FileSystem>)> = None;
for (at, fs) in &self.mounts {
let matches = if at == "/" {
true
} else {
abs == *at || abs.starts_with(&format!("{at}/"))
};
if matches && best.map(|(b, _)| at.len() > b.len()).unwrap_or(true) {
best = Some((at, fs));
}
}
best.map(|(at, fs)| (at.as_str(), fs)).ok_or(Errno::ENOENT)
}
fn mount_of<'a>(&'a self, abs: String) -> Result<(&'a Arc<dyn FileSystem>, String), Errno> {
let mut best: Option<(&String, &Arc<dyn FileSystem>)> = None;
for (at, fs) in &self.mounts {
let matches = if at == "/" {
@@ -478,6 +569,7 @@ impl Vfs {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
});
}
fs.metadata(&rel)
@@ -521,6 +613,12 @@ impl Vfs {
/// Set the modification time of `path`, following a final symlink when
/// `follow` (as `utimensat` does without `AT_SYMLINK_NOFOLLOW`).
pub fn set_mtime(&self, path: &str, secs: i64, follow: bool) -> Result<(), Errno> {
self.set_times(path, None, Some((secs, 0)), follow)
}
/// Set the access and modification times of `path` (`None`: leave it),
/// following a final symlink when `follow`.
pub fn set_times(&self, path: &str, atime: Option<(i64, u32)>, mtime: Option<(i64, u32)>, follow: bool) -> Result<(), Errno> {
let mut current = normalize(path, &self.cwd);
for _ in 0..40 {
let (fs, rel) = self.resolve(&current)?;
@@ -532,9 +630,13 @@ impl Vfs {
normalize(&target, &parent_of(&current))
};
}
_ => {
// The link itself (AT_SYMLINK_NOFOLLOW), whether or not
// what it names exists yet: dpkg stamps each link it unpacks
// as `NAME.dpkg-new`, before its target has that name.
Ok(_) => return fs.set_times(&rel, atime, mtime),
Err(_) => {
fs.metadata(&rel)?;
return fs.set_mtime(&rel, secs);
return fs.set_times(&rel, atime, mtime);
}
}
}
@@ -694,6 +796,26 @@ mod tests {
assert_eq!(rel, "procession");
}
#[test]
fn directory_symlinks_are_followed() {
// A merged /usr: /bin -> usr/bin, /lib -> /usr/lib (absolute).
let mut vfs = Vfs::new();
let root = Arc::new(TmpFs::new());
root.add_file("usr/bin/dash", b"sh".to_vec());
root.add_file("usr/lib/ld.so", b"ld".to_vec());
vfs.mount("/", root);
vfs.symlink("usr/bin", "/bin").unwrap();
vfs.symlink("/usr/lib", "/lib").unwrap();
vfs.symlink("dash", "/usr/bin/sh").unwrap();
assert_eq!(vfs.resolve("/bin/dash").unwrap().1, "usr/bin/dash");
assert_eq!(vfs.resolve("/lib/ld.so").unwrap().1, "usr/lib/ld.so");
// The final component is left to the caller.
assert_eq!(vfs.resolve("/bin").unwrap().1, "bin");
assert_eq!(vfs.read_link("/bin/sh").unwrap(), "dash");
assert!(vfs.metadata("/bin/sh").is_ok());
assert!(vfs.metadata("/bin/../lib/ld.so").is_ok());
}
#[test]
fn dirents_encode_and_stop_at_the_buffer() {
let entries = vec![
crates/gex-kernel/src/vfs/oss.rs
+326
View File
@@ -0,0 +1,326 @@
//! OSS audio devices, `/dev/dsp` and `/dev/mixer`: a null output for now.
//!
//! Native Wine's audio (W3.5, docs/wine-native.md) is mmdevapi over
//! wineoss.drv, the OSSv4 driver, and this is the device it talks to: one
//! playback engine that takes 16-, 24- and 32-bit and float PCM at any rate,
//! and plays it into nothing at the rate it was set to. Its buffer drains
//! with the wall clock, so a program that paces itself by the device (as
//! mmdevapi's clients do, by the padding it reports) runs at the right
//! speed. No capture. A device that plays (the browser's WebAudio, the
//! host's) replaces the drain, not the interface.
//!
//! The requests are Linux's OSSv4 numbers (`_IOR`/`_IOWR` of Linux's
//! `sys/ioctl.h`), and the structs `oss_sysinfo` (1248 bytes),
//! `oss_audioinfo` (1180) and `audio_buf_info` (16) are laid out as OSSv4's
//! `sys/soundcard.h` has them: ints and char arrays only, the same on every
//! ABI gex runs.
use super::{FileHandle, FileType, Metadata};
use crate::errno::{Errno, SysResult};
use std::sync::Mutex;
use std::time::Instant;
/// Which node a handle was opened as.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum OssNode {
Dsp,
Mixer,
}
// Requests (Linux encoding).
const SNDCTL_SYSINFO: u32 = 0x84e0_5801;
const SNDCTL_AUDIOINFO: u32 = 0xc49c_5807;
const SNDCTL_ENGINEINFO: u32 = 0xc49c_580c;
const SNDCTL_AUDIOINFO_EX: u32 = 0xc49c_580d;
const SNDCTL_DSP_RESET: u32 = 0x0000_5000;
const SNDCTL_DSP_SYNC: u32 = 0x0000_5001;
const SNDCTL_DSP_SPEED: u32 = 0xc004_5002;
const SNDCTL_DSP_SETFMT: u32 = 0xc004_5005;
const SNDCTL_DSP_CHANNELS: u32 = 0xc004_5006;
const SNDCTL_DSP_SETFRAGMENT: u32 = 0xc004_500a;
const SNDCTL_DSP_GETFMTS: u32 = 0x8004_500b;
const SNDCTL_DSP_GETOSPACE: u32 = 0x8010_500c;
const SNDCTL_DSP_GETISPACE: u32 = 0x8010_500d;
const SNDCTL_DSP_GETCAPS: u32 = 0x8004_500f;
const SNDCTL_DSP_GETODELAY: u32 = 0x8004_5017;
/// `SOUND_MIXER_READ_*` and `_WRITE_*`: `_IOR('M', n, int)`, `_IOWR('M', n, int)`.
const MIXER_READ: u32 = 0x8004_4d00;
const MIXER_WRITE: u32 = 0xc004_4d00;
// Formats.
const AFMT_U8: u32 = 0x0000_0008;
const AFMT_S16_LE: u32 = 0x0000_0010;
const AFMT_S32_LE: u32 = 0x0000_1000;
const AFMT_S24_LE: u32 = 0x0001_0000;
const AFMT_S24_PACKED: u32 = 0x0004_0000;
const AFMT_F32_LE: u32 = 0x1000_0000;
const FORMATS: u32 = AFMT_U8 | AFMT_S16_LE | AFMT_S24_LE | AFMT_S24_PACKED | AFMT_S32_LE | AFMT_F32_LE;
const PCM_CAP_OUTPUT: u32 = 0x0002_0000;
const PCM_CAP_TRIGGER: u32 = 0x0000_1000;
/// The playback buffer: 16 fragments of 2 KiB (about 190 ms of 44.1 kHz
/// stereo 16-bit).
const FRAGSIZE: u64 = 2048;
const FRAGMENTS: u64 = 16;
const BUFSIZE: u64 = FRAGSIZE * FRAGMENTS;
#[derive(Debug)]
struct Stream {
format: u32,
rate: u32,
channels: u32,
/// Bytes written and not yet played.
queued: u64,
/// When `queued` was last brought up to date.
at: Instant,
}
impl Stream {
fn bytes_per_second(&self) -> u64 {
let sample = match self.format {
AFMT_U8 => 1,
AFMT_S16_LE => 2,
AFMT_S24_PACKED => 3,
_ => 4,
};
self.rate as u64 * self.channels as u64 * sample
}
/// Play what the time since the last look has played.
fn drain(&mut self) {
let now = Instant::now();
let ns = now.duration_since(self.at).as_nanos() as u64;
let played = (ns as u128 * self.bytes_per_second() as u128 / 1_000_000_000) as u64;
if played > 0 || self.queued == 0 {
self.queued = self.queued.saturating_sub(played);
self.at = now;
}
}
}
/// An open `/dev/dsp` or `/dev/mixer`.
#[derive(Debug)]
pub struct OssHandle {
node: OssNode,
stream: Mutex<Stream>,
}
impl OssHandle {
pub fn new(node: OssNode) -> OssHandle {
OssHandle {
node,
stream: Mutex::new(Stream { format: AFMT_U8, rate: 8000, channels: 1, queued: 0, at: Instant::now() }),
}
}
}
fn put_str(buf: &mut [u8], at: usize, s: &str) {
buf[at..at + s.len()].copy_from_slice(s.as_bytes());
}
fn put_i32(buf: &mut [u8], at: usize, v: i32) {
buf[at..at + 4].copy_from_slice(&v.to_le_bytes());
}
/// `oss_sysinfo`: one card, one mixer, one playback engine.
fn sysinfo() -> Vec<u8> {
let mut b = vec![0u8; 1248];
put_str(&mut b, 0, "gex");
put_str(&mut b, 32, "4.2");
put_i32(&mut b, 64, 0x040200); // versionnum
put_i32(&mut b, 196, 1); // numaudios
put_i32(&mut b, 244, 1); // nummixers
put_i32(&mut b, 280, 1); // numcards
put_i32(&mut b, 284, 1); // numaudioengines
put_str(&mut b, 288, "BSD");
b
}
/// `oss_audioinfo` of the one engine.
fn audioinfo() -> Vec<u8> {
let mut b = vec![0u8; 1180];
put_i32(&mut b, 0, 0); // dev
put_str(&mut b, 4, "gex null output");
put_i32(&mut b, 76, (PCM_CAP_OUTPUT | PCM_CAP_TRIGGER) as i32); // caps
put_i32(&mut b, 84, FORMATS as i32); // oformats
put_i32(&mut b, 172, 1); // enabled
put_i32(&mut b, 180, 8000); // min_rate
put_i32(&mut b, 184, 192000); // max_rate
put_i32(&mut b, 188, 1); // min_channels
put_i32(&mut b, 192, 2); // max_channels: the mix format is stereo
put_i32(&mut b, 400, -1); // latency: unknown
put_str(&mut b, 404, "/dev/dsp"); // devnode
b
}
impl FileHandle for OssHandle {
fn read_at(&self, _offset: u64, _buf: &mut [u8]) -> SysResult {
// No capture: a reader waits for nothing.
Err(Errno::EAGAIN)
}
fn write_at(&self, _offset: u64, buf: &[u8]) -> SysResult {
if self.node != OssNode::Dsp {
return Err(Errno::EINVAL);
}
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
s.drain();
let room = BUFSIZE - s.queued.min(BUFSIZE);
if room == 0 {
return Err(Errno::EAGAIN);
}
let n = (buf.len() as u64).min(room);
s.queued += n;
Ok(n)
}
fn metadata(&self) -> Result<Metadata, Errno> {
Ok(Metadata {
file_type: FileType::CharDevice,
size: 0,
mode: 0o666,
ino: 1,
mtime: 0,
nlink: 1,
// 14:3 and 14:0, Linux's sound major.
rdev: if self.node == OssNode::Dsp { 0xe03 } else { 0xe00 },
times: Default::default(),
})
}
fn is_stream(&self) -> bool {
true
}
fn poll_writable(&self) -> bool {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
s.drain();
s.queued < BUFSIZE
}
fn poll_readable(&self) -> bool {
false
}
fn ioctl(&self, m: &mut crate::GuestMem<'_>, request: u64, arg: u64) -> Option<SysResult> {
let request = request as u32;
// An int argument read and written back.
let mut int = |m: &mut crate::GuestMem<'_>, f: &mut dyn FnMut(i32) -> i32| -> SysResult {
let v = m.read_u32(arg)? as i32;
m.write_u32(arg, f(v) as u32)?;
Ok(0)
};
let r = match request {
SNDCTL_SYSINFO => m.write(arg, &sysinfo()).map(|_| 0),
SNDCTL_AUDIOINFO | SNDCTL_AUDIOINFO_EX | SNDCTL_ENGINEINFO => {
// Device -1 is "this one" (on /dev/dsp) or the default.
match m.read_u32(arg) {
Ok(d) if d == 0 || d == u32::MAX => m.write(arg, &audioinfo()).map(|_| 0),
Ok(_) => Err(Errno::ENXIO),
Err(e) => Err(e),
}
}
_ if self.node == OssNode::Mixer => match request & 0xffff_ff00 {
// Volumes: full, both channels; what is written is accepted.
MIXER_READ => m.write_u32(arg, 0x6464).map(|_| 0),
MIXER_WRITE => Ok(0),
_ => return None,
},
SNDCTL_DSP_SETFMT => {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
int(m, &mut |v| {
let v = v as u32;
if v != 0 {
s.drain();
s.format = if v.count_ones() == 1 && FORMATS & v != 0 { v } else { AFMT_S16_LE };
}
s.format as i32
})
}
SNDCTL_DSP_SPEED => {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
int(m, &mut |v| {
s.drain();
s.rate = (v.max(8000) as u32).min(192000);
s.rate as i32
})
}
SNDCTL_DSP_CHANNELS => {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
int(m, &mut |v| {
s.drain();
s.channels = v.clamp(1, 8) as u32;
s.channels as i32
})
}
SNDCTL_DSP_GETFMTS => m.write_u32(arg, FORMATS).map(|_| 0),
SNDCTL_DSP_GETCAPS => m.write_u32(arg, PCM_CAP_OUTPUT | PCM_CAP_TRIGGER).map(|_| 0),
// The fragment layout is fixed; the request is answered as made.
SNDCTL_DSP_SETFRAGMENT => Ok(0),
SNDCTL_DSP_GETOSPACE => {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
s.drain();
let free = BUFSIZE - s.queued.min(BUFSIZE);
let mut b = [0u8; 16];
put_i32(&mut b, 0, (free / FRAGSIZE) as i32);
put_i32(&mut b, 4, FRAGMENTS as i32);
put_i32(&mut b, 8, FRAGSIZE as i32);
put_i32(&mut b, 12, free as i32);
m.write(arg, &b).map(|_| 0)
}
SNDCTL_DSP_GETISPACE => {
let mut b = [0u8; 16];
put_i32(&mut b, 4, FRAGMENTS as i32);
put_i32(&mut b, 8, FRAGSIZE as i32);
m.write(arg, &b).map(|_| 0)
}
SNDCTL_DSP_GETODELAY => {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
s.drain();
m.write_u32(arg, s.queued as u32).map(|_| 0)
}
SNDCTL_DSP_RESET | SNDCTL_DSP_SYNC => {
let mut s = self.stream.lock().unwrap_or_else(|e| e.into_inner());
s.queued = 0;
s.at = Instant::now();
Ok(0)
}
_ => return None,
};
Some(r)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_buffer_drains_with_the_clock() {
let h = OssHandle::new(OssNode::Dsp);
{
let mut s = h.stream.lock().unwrap();
s.format = AFMT_S16_LE;
s.rate = 48000;
s.channels = 2;
}
// 192000 bytes a second: the 32 KiB buffer fills, then drains in
// about 171 ms.
assert_eq!(h.write_at(0, &vec![0u8; 100_000]).unwrap(), BUFSIZE);
assert_eq!(h.write_at(0, &[0u8; 4]), Err(Errno::EAGAIN));
std::thread::sleep(std::time::Duration::from_millis(60));
let n = h.write_at(0, &vec![0u8; 100_000]).unwrap();
// 60 ms played at least 11520 bytes, and less than the whole buffer.
assert!(n >= 11_520 && n < BUFSIZE, "{n}");
}
#[test]
fn the_engine_describes_itself_as_osv4_output() {
let b = audioinfo();
assert_eq!(i32::from_le_bytes(b[76..80].try_into().unwrap()) as u32 & PCM_CAP_OUTPUT, PCM_CAP_OUTPUT);
assert_eq!(&b[404..412], b"/dev/dsp");
let s = sysinfo();
assert_eq!(s[32], b'4');
assert_eq!(i32::from_le_bytes(s[196..200].try_into().unwrap()), 1);
}
}
crates/gex-kernel/src/vfs/procfs.rs
+3
View File
@@ -52,6 +52,7 @@ impl FileHandle for Generated {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
}
@@ -202,6 +203,7 @@ impl FileSystem for ProcFs {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
});
}
match path.as_str() {
@@ -214,6 +216,7 @@ impl FileSystem for ProcFs {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
}),
_ => Err(Errno::ENOENT),
}
crates/gex-kernel/src/vfs/remote.rs
+1
View File
@@ -92,6 +92,7 @@ pub fn decode_meta(b: &[u8]) -> Metadata {
mtime: u64at(24) as i64,
nlink: u32at(32).max(1),
rdev: 0,
times: Default::default(),
}
}
crates/gex-kernel/src/vfs/tmpfs.rs
+4
View File
@@ -382,6 +382,7 @@ impl FileHandle for TmpHandle {
mtime: file.mtime.load(Ordering::Relaxed),
nlink: 1,
rdev: 0,
times: Default::default(),
});
}
let inner = self.inner.lock().unwrap();
@@ -397,6 +398,7 @@ impl FileHandle for TmpHandle {
// Every name that maps to this inode is a hard link to it.
nlink: inner.inos.values().filter(|&&v| v == ino).count() as u32,
rdev: 0,
times: Default::default(),
}),
Some(Node::Dir) => {
let d = Metadata::dir();
@@ -416,6 +418,7 @@ impl FileHandle for TmpHandle {
mtime,
nlink: 1,
rdev: 0,
times: Default::default(),
}),
Some(Node::Socket) => Ok(Metadata {
file_type: FileType::Socket,
@@ -425,6 +428,7 @@ impl FileHandle for TmpHandle {
mtime,
nlink: 1,
rdev: 0,
times: Default::default(),
}),
None => Err(Errno::ENOENT),
}
crates/gex-ld/src/elf.rs
+27 -4
View File
@@ -236,7 +236,24 @@ impl DynSym {
}
}
/// The header a native wasm32 program's container carries 0x40 bytes below
/// its entry (`tools/wine-wasm/wasm-elf.py`; shims' `wasmprog`).
pub const WASM32_MAGIC: &[u8; 8] = b"GEXWASM\0";
impl Image {
/// Whether this is a native wasm32 program's container: an i386 image
/// with [`WASM32_MAGIC`] 0x40 bytes below the entry.
pub fn is_wasm32(&self) -> bool {
let at = self.entry.wrapping_sub(0x40);
self.class32
&& self.segments.iter().any(|s| {
at >= s.vaddr && at + 8 <= s.vaddr + s.bytes.len() as u64 && {
let o = (at - s.vaddr) as usize;
&s.bytes[o..o + 8] == WASM32_MAGIC
}
})
}
/// The defined, non-TLS dynamic symbols in `[lo, hi)` (unbiased) by every
/// name a binding may use: `name@@VER`, `name@VER`, and the bare `name`
/// for the default version (or an unversioned object), each with its
@@ -437,12 +454,14 @@ pub fn load(data: &[u8], pie_base: u64) -> Result<Image, LoadError> {
if r.u8(5)? != order {
return Err(LoadError::NotElf64Le);
}
// A 32-bit i386 image is accepted on the x86-64 guest only: it runs in
// the same process model as everything else, in compatibility mode. The
// PowerPC guest is 32-bit only.
// A 32-bit i386 image runs in the same process model as everything
// else, on the i386 ABI: on the x86-64 guest any i386 program, in
// compatibility mode; on the AArch64 guest only a native wasm32 program
// (an i386 container around a wasm module, checked below), which has no
// i386 code to translate. The PowerPC guest is 32-bit only.
let class32 = match r.u8(4)? {
ELFCLASS64 if !gex_abi::PPC => false,
ELFCLASS32 if gex_abi::X86 || gex_abi::PPC => true,
ELFCLASS32 => true,
_ => return Err(LoadError::NotElf64Le),
};
let e_type = r.u16(16)?;
@@ -609,6 +628,10 @@ pub fn load(data: &[u8], pie_base: u64) -> Result<Image, LoadError> {
runpath: Vec::new(),
};
if class32 && !gex_abi::X86 && !gex_abi::PPC && !image.is_wasm32() {
return Err(LoadError::NotAarch64);
}
if let (Some((dyn_vaddr, dyn_size)), false) = (dynamic, class32) {
apply_relative_relocations(&mut image, dyn_vaddr, dyn_size);
image.named_relocs = read_named_relocs(&image, dyn_vaddr, dyn_size);
crates/gex-wayland/src/lib.rs
+26 -3
View File
@@ -1236,9 +1236,18 @@ impl Client {
}
}
// A client may not draw into a buffer again until it is released,
// and most will not draw a second frame without this.
let w = Writer::new(buffer_id, 0);
self.send(w);
// and most will not draw a second frame without this. Once per
// attach: a commit that attaches nothing new (a subsurface
// position, a viewport, an input region) shows the committed
// buffer again but does not release it again. Wine's
// winewayland counts releases, and fourteen for one attach
// freed a buffer still in its queue, which it then destroyed a
// second time (Katana ZERO and Undertale, W3: libwayland's
// `proxy->refcount > 0` assertion at the first swap-chain resize).
if state.pending_buffer.is_some() {
let w = Writer::new(buffer_id, 0);
self.send(w);
}
}
if let Some(frame) = produced {
// The host that composites from copies has one window, and it is
@@ -2755,6 +2764,20 @@ mod tests {
);
}
#[test]
fn a_commit_with_nothing_attached_does_not_release_the_buffer_again() {
let px = vec![0u8; 16];
let (mut c, _) = draw(px.clone(), 2, 2, 8);
let _ = c.take_outgoing();
// wl_surface.commit (6) on surface 6, with no attach since the last.
c.receive(&Writer::new(6, 6).finish(), &[], &Pool(px));
let msgs = messages(&c.take_outgoing());
assert!(
!msgs.iter().any(|(o, op, _)| *o == 10 && *op == 0),
"no second wl_buffer.release for one attach: {msgs:?}"
);
}
#[test]
fn a_toplevel_is_configured_before_it_may_draw() {
let px = vec![0u8; 16];
crates/gexc/src/main.rs
+51 -1
View File
@@ -199,6 +199,7 @@ run options:
--strace trace every syscall on stderr
--verbose report translation statistics
--env K=V add an environment variable (repeatable)
--host-env start from gexc's own environment (default: PATH, HOME)
--sysroot DIR present DIR as the guest's root filesystem
--bind H=G mount host file or directory H at guest path G (repeatable)
--busybox FILE present FILE (a wasm32 busybox) as /bin/sh and the common
@@ -308,6 +309,7 @@ fn parse_run_args(args: &[String]) -> Result<RunArgs> {
let mut mem64 = None;
let mut argv0: Option<String> = None;
let mut wine_prefix: Option<String> = None;
let mut host_env = false;
let mut i = 0;
while i < args.len() {
let a = &args[i];
@@ -326,7 +328,10 @@ fn parse_run_args(args: &[String]) -> Result<RunArgs> {
"--profile" => opts.profile = true,
"--wayland" => {
i += 1;
opts.wayland = Some(args.get(i).context("--wayland needs a path")?.clone());
// Lexically normal: the compositor's name is bound as given,
// and a client's connect resolves `.` and `..` first, so
// `dir/../wl` would never meet its bound name.
opts.wayland = Some(normal_path(args.get(i).context("--wayland needs a path")?));
}
"--wayland-size" => {
i += 1;
@@ -424,12 +429,27 @@ fn parse_run_args(args: &[String]) -> Result<RunArgs> {
let v = args.get(i).context("--env needs an argument")?;
opts.env.push(v.clone());
}
"--host-env" => host_env = true,
other if other.starts_with('-') => bail!("unknown option `{other}`"),
other => path = Some(other.to_string()),
}
i += 1;
}
let path = path.context("expected a program to run")?;
// `--host-env`: the guest starts with gexc's own environment (a gexcc
// program's host launcher passes it on: tools/gexcc/hoststub.c), then
// the `--env` settings over it.
if host_env {
let set: std::collections::HashSet<String> =
opts.env.iter().filter_map(|e| e.split_once('=').map(|(k, _)| k.to_string())).collect();
let mut env: Vec<String> = std::env::vars_os()
.filter_map(|(k, v)| Some((k.into_string().ok()?, v.into_string().ok()?)))
.filter(|(k, _)| !set.contains(k))
.map(|(k, v)| format!("{k}={v}"))
.collect();
env.append(&mut opts.env);
opts.env = env;
}
// `GEX_PROFILE=1` is `--record $GEX_PROFILE_OUT`.
if opts.record.is_none()
&& std::env::var("GEX_PROFILE").is_ok_and(|v| !v.is_empty() && v != "0")
@@ -2472,3 +2492,33 @@ fn stdin_ready() -> bool {
let n = unsafe { libc::poll(&mut p, 1, 0) };
n != 0 && (n < 0 || p.revents != 0)
}
/// `path` with `.` components, repeated slashes and `name/..` pairs removed,
/// without touching the file system (`..` above the root stays at the root).
fn normal_path(path: &str) -> String {
let mut parts: Vec<&str> = Vec::new();
for c in path.split('/') {
match c {
"" | "." => {}
".." if parts.last().is_some_and(|p| *p != "..") => {
parts.pop();
}
".." if path.starts_with('/') => {}
c => parts.push(c),
}
}
let joined = parts.join("/");
if path.starts_with('/') { format!("/{joined}") } else if joined.is_empty() { ".".into() } else { joined }
}
#[cfg(test)]
mod normal_path_tests {
#[test]
fn dot_dot_and_dots_go() {
assert_eq!(super::normal_path("/a/b/../c/./wl"), "/a/c/wl");
assert_eq!(super::normal_path("/a//b/"), "/a/b");
assert_eq!(super::normal_path("/../x"), "/x");
assert_eq!(super::normal_path("a/../../wl"), "../wl");
assert_eq!(super::normal_path("wl"), "wl");
}
}
crates/gexc/src/prof.rs
+18 -8
View File
@@ -11,10 +11,11 @@
//! (tools/prof-report.py).
//!
//! Each sample also records the thread it hit, and every thread with at
//! least 5% of the samples gets a profile of its own beside the file,
//! `<stem>-<thread name>-<tid>.<ext>` (and its `.maps`), in the same format:
//! what the main thread, the critical path of a game's frame, spends its
//! time on.
//! least `GEX_PROF_MIN_PCT` (default 5) percent of the samples gets a
//! profile of its own beside the file, `<stem>-<thread name>-<tid>.<ext>`
//! (and its `.maps`), in the same format: what the main thread, the
//! critical path of a game's frame, spends its time on. Every thread's
//! total is in `<stem>.threads`.
use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering};
@@ -113,19 +114,28 @@ fn dump(path: &std::path::Path) {
*by_tid.entry(t.load(Ordering::Relaxed)).or_default().entry(pc).or_default() += 1;
}
write_one(path, &hist, maps.as_deref());
// Threads with at least 5% of the samples, each on its own.
// Threads with at least GEX_PROF_MIN_PCT (default 5) percent of the
// samples, each on its own; every thread's total in `<stem>.threads`
// (`tid comm samples`, most first), so the time spread over many small
// threads (compile workers, guest threads) is still attributable.
let stem = path.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
let ext = path.extension().map(|s| s.to_string_lossy().into_owned()).unwrap_or_else(|| "prof".into());
let min_pct = std::env::var("GEX_PROF_MIN_PCT").ok().and_then(|v| v.parse::<u64>().ok()).unwrap_or(5);
let mut totals = Vec::new();
for (tid, h) in by_tid {
let k: u64 = h.values().sum();
if tid == 0 || k * 20 < n as u64 {
continue;
}
let comm = std::fs::read_to_string(format!("/proc/self/task/{tid}/comm"))
.map(|c| c.trim().replace(['/', ' ', ':'], "_"))
.unwrap_or_else(|_| "gone".into());
totals.push((k, tid, comm.clone()));
if tid == 0 || k * 100 < n as u64 * min_pct {
continue;
}
write_one(&path.with_file_name(format!("{stem}-{comm}-{tid}.{ext}")), &h, maps.as_deref());
}
totals.sort_by(|a, b| b.cmp(a));
let list: String = totals.iter().map(|(k, t, c)| format!("{t} {c} {k}\n")).collect();
let _ = std::fs::write(path.with_extension("threads"), list);
}
/// One histogram as a profile file (`pid`, `samples`, `pc count`), with
docs/gexcc.md
+778
View File
@@ -0,0 +1,778 @@
# gexcc: building Unix programs for gex
Stream C (2026-10-07). Goal (merge-plan §10, §10.1): source compatibility with
any Linux codebase. Ordinary C and C++ projects cross-build to gex's native
wasm32 target with their own build systems, and their own test suites pass
under gexc. Later, glibc itself.
The target is stream W's (docs/wine-native.md §3.10–§3.12):
- clang `--target=wasm32-unknown-linux-musl`;
- musl with gex's `arch/wasm32` (syscalls on the i386 ABI through
`gex.syscall`) plus libgex;
- wasm-ld PIE modules that import gex's memory and function table, in an ELF32
container that gexc runs as an ordinary process.
gexcc packages that target as a cross toolchain.
## 1. Using it
### Install
```sh
BUSYBOX_TAR=.../busybox-1.37.0.tar.bz2 LLVM_SRC=.../llvm-project-22.1.8.src \
tools/gexcc/install.sh PREFIX [MUSL_SRC] # MUSL_SRC: musl-wasm, gex-wasm32-slim or later
export PATH=PREFIX/bin:$PATH
```
Install options (environment variables):
| variable | effect |
|---|---|
| `GEXC` | the gexc programs run under (default: this tree's `target/release/gexc`) |
| `GEX_BINARYEN` | an existing binaryen (otherwise downloaded) |
| `BUSYBOX_TAR` | builds busybox (defconfig, 400 applets) as the guest's native `/bin/sh` and coreutils (§1.4) |
| `LLVM_SRC` | an LLVM tree of clang's version (22.1.8); builds libc++, libc++abi and libunwind (`build-cxx.sh`). It needs `runtimes cmake llvm/cmake llvm/utils libcxx libcxxabi libunwind libc third-party` |
| `JOBS` | parallel jobs |
What it installs:
| path | what |
|---|---|
| `bin/gexcc`, `bin/gexc++` | the compiler driver (gcc/clang interface) |
| `bin/wasm32-unknown-linux-musl-{cc,gcc,clang,c++,g++,clang++}` | the driver under the names autotools looks for |
| `bin/wasm32-unknown-linux-musl-{ar,ranlib,nm,objdump,ld,...}`, `gexcc-{ar,ranlib,nm}` | LLVM's tools; `ld` is wasm-ld |
| `bin/…-strip`, `gexcc-strip` | does nothing (a program is a container around a wasm module whose names gex binds by) |
| `bin/…-pkg-config`, `gexcc-pkg-config` | pkg-config over the sysroot only, `--static` |
| `bin/gexcc-exec` | the runner programs' launchers exec: gexc with gexcc's defaults (busybox) |
| `bin/gexcc-run PROG ARGS` | `gexcc-exec run --host-env PROG ARGS` |
| `lib/gexcc/` | flags.sh, wasm-elf.py, forkreach.py, hoststub.bin, GexPoll.so, main3.o, empty.o, busybox |
| `sysroot/` | musl (include, libc.a, crt1.o), libgex.a, libgexcc.a, compiler-rt builtins, libc++ (include/c++/v1, lib), Linux UAPI headers (i386), binaryen |
| `sysroot/usr/` | where cross-built libraries install: `--prefix=/usr`, `DESTDIR=$(gexcc --print-sysroot)` |
| `share/gexcc/toolchain.cmake`, `meson-cross.ini`, `config.sub`, `config.guess` | build-system files |
### Build systems
```sh
# autotools
./configure --host=wasm32-unknown-linux-musl --prefix=/usr && make && make check
make install DESTDIR=$(gexcc --print-sysroot) # a library others use
# CMake (try_run and ctest work)
cmake -DCMAKE_TOOLCHAIN_FILE=PREFIX/share/gexcc/toolchain.cmake ...
# Meson (cc.run() and meson test work)
meson setup build --cross-file PREFIX/share/gexcc/meson-cross.ini
# plain Makefiles
make CC=gexcc AR=gexcc-ar RANLIB=gexcc-ranlib
# a build system that configures as native (perl's Configure, Python's
# configure without --host): CC=gexcc and pkg-config/library paths into the
# sysroot (see tests/gexcc/corpus/recipes.sh)
```
### Running programs
A gexcc program runs like any binary:
- **Under gex:** `gexc run ./prog`. Inside gex, an exec of another gexcc
program stays inside gex.
- **On the build machine:** `./prog`, which needs an x86 Linux host with
IA-32 emulation. The container's entry holds a small i386 launcher
(`tools/gexcc/hoststub.c`) that re-execs the file as `RUNNER run --host-env
--argv0 ARGV0 FILE ARGS`.
- The runner is `$GEXCC_RUNNER` if set, else the one baked in at install
(`PREFIX/bin/gexcc-exec`), else `gexc` on `$PATH`.
- gex never runs the launcher: it binds the module at the entry.
- Because of the launcher, configure sees a native build ("cross
compiling… no"), and `AC_RUN_IFELSE`, `make check`, `try_run` and
`ctest` all run the real target.
The guest sees:
- the host filesystem, the working directory and the environment
(`--host-env`);
- standard streams redirected from files, sharing the host's offset;
- busybox's applets as `/bin/*` and `/usr/bin/*` (gexcc-exec passes
`--busybox` with the whole applet list).
### Inside gex: busybox
A gexcc program that runs `system()`, `popen` or `sh`, or execs `sleep`,
`kill`, `cat`, …, would otherwise exec the host's x86 binaries, translated:
seconds per command on a cold cache. gexcc-exec presents a native busybox
in their place.
- `GEXCC_BUSYBOX=` (empty) turns it off; `GEXCC_BUSYBOX=path` picks another.
- `GEXCC_GEXC` picks another gexc.
### Libraries
Everything links statically. A project that builds a shared library still
works: `-shared` writes an **archive** under the `.so` name. Its objects go
in, and a `gexcc.deps.o` member records the `-l`/`-L` it was linked with.
Linking `-lfoo` or `libfoo.so` takes the archive and adds its recorded
dependencies. libtool, CMake `SHARED` and Meson `library()` need no options.
`dlopen` of real side modules is a gap (§3).
### Driver options and environment
| | |
|---|---|
| `GEXCC_POLL=0`, `-fno-gex-poll` | no loop safepoints (§4); signals then reach only code that makes system calls |
| `GEXCC_VERBOSE=1` | print every command |
| `GEXCC_KEEP_WASM=1` | keep `OUT.wasm` beside a program |
| `GEXCC_SYSROOT` | another sysroot |
| `--print-sysroot`, `--print-prefix`, `-dumpmachine`, `-print-prog-name=ld`, `-print-file-name=` | queries build systems use |
C++: exceptions (wasm EH), RTTI, iostreams and threads work
(`tests/gexcc/cxx.cpp`).
## 2. Status
Natively, on gooner: gexc built from this branch, the corpus built and its
own test suites run with `tests/gexcc/corpus/run.sh`. "pass" counts are the
suite's own units:
- automake tests (including gnulib's unit tests);
- the suite's own cases (gawk, make, tar's autotest);
- one unit for a single all-in-one script (lua all.lua, the sqlite and dash
sessions).
Skips are the suite's own: locales, root, missing tools.
| program | version | build system | build | tests (pass/total) | notes |
|---|---|---|---|---|---|
| zlib | 1.3.1 | own configure | ok | 3/3 | static, "shared" and 64-bit examples |
| xz | 5.6.3 | autotools+libtool | ok | 19/19 | |
| bzip2 | 1.0.8 | Makefile | ok | 6/6 | |
| lua | 5.4.7 | Makefile | ok | 1/1 (all.lua, `_port`) | locale probe patched: musl accepts any locale name |
| sqlite | 3.46.1 | autotools | ok | 1/1 | autoconf tarball has no suite: a session diffed against the host's sqlite3 |
| jq | 1.7.1 | autotools | ok | 29/29 | bundled oniguruma |
| pcre2 | 10.44 | autotools | ok | 1/3 | RunTest test 8: compiled sizes are 64-bit expectations (ILP32); RunGrepTest 150: `setlocale("badlocale")` succeeds on musl |
| expat | 2.6.4 | autotools | ok | 2/2 | |
| libpng | 1.6.44 | autotools | ok | 32/32 | |
| mbedtls | 3.6.2 | CMake (ctest) | ok | 129/130 | psa_crypto passes alone (1958/1958); failed once under `ctest -j3` |
| gawk | 5.3.1 | autotools | ok | 654/654 ("ALL TESTS PASSED") | |
| tar | 1.35 | autotools (autotest) | ok | 204/204, 40 skip | |
| grep | 3.11 | autotools+gnulib | ok | 568/606, 100 skip | remaining: signal timing (exec latency), SIGSEGV catching, multibyte locales |
| sed | 4.9 | autotools+gnulib | ok | 206/214, 46 skip | |
| diffutils | 3.10 | autotools+gnulib | ok | 386/410, 70 skip | before the fixes; rerun pending |
| findutils | 4.10.0 | autotools | ok | 2/2 | the find/xargs suites need DejaGnu (not on gooner) |
| dash | 0.5.12 | autotools | ok | 1/1 | a script session against the host's sh |
| less | 668 | autotools | ok | 1/1 | as a filter (no terminal) |
| ncurses | 6.5 | autotools | ok | – | installed for less, bash, vim, tmux |
| make | 4.4.1 | autotools | ok | rerun pending | first run used the translated host sh inside gex (60 s timeouts) |
| bash | 5.2.37 | autotools | ok | 10/83 before fpcast; rerun pending | needs `-fgex-fpcast-emu` (function pointer casts in unwind-protect); sigsetjmp fixed |
| fmt | 11.0.2 | CMake (C++) | rerun pending | | libc++ 22 strictness: `-include algorithm -Wno-invalid-specialization` |
| curl | 8.10.1 | autotools + mbedtls | ok | rerun pending | |
| perl | 5.40.0 | Configure | rerun pending | | needs `-D_GNU_SOURCE` (musl hides eaccess) |
| coreutils | 9.5 | autotools+gnulib | pending | | |
Recurring gnulib failures not fixed yet:
- `test-select`/`test-pselect`: a write to a loopback TCP socket whose peer
closed is EPIPE at once (Linux takes the first write), and the SIGPIPE then
leaves the process hanging.
- `test-sigsegv-*`, `test-c-stack`: a wasm trap is not a SIGSEGV a handler
can catch.
- Locale tests (musl).
Toolchain checks:
- `tests/gexcc/run.sh` (host against gex): alarmloop, cxx, emptypath,
fdsemantics, fifononblock, mmapreuse, pathcorners, stdinseek,
timestamps.
- `tests/wasm32/run.sh` (W's): dlopen, fork, signals, sjlj, threads.
- A CMake project (`find_package(ZLIB)`, `check_c_source_runs`, a `SHARED`
library, ctest) and a Meson one (`dependency('zlib')`, `cc.run`,
`library()`, `meson test`) both build and pass.
**Browser** (shims/browser under Node, `tests/gexcc/run-browser.sh`,
`GEX_ARCH=aarch64` for the AArch64 build's shim):
- 2026-10-08, both builds alike: 11 of 15 pass; longargs, pathcorners,
tcphalfclose and timestamps fail in the browser runtime only (they were
written after the last browser check; the browser filesystem and network
lack the fixes made natively). To fix.
- hello, cxx (exceptions, threads), alarmloop (polls deliver SIGALRM, the
default action kills the child), and three-argument main all match their
native runs.
- The browser arena's ceiling is 2 GiB (0x80000000): mapping and freeing
1 GiB in turn works, but a 1 GiB `realloc` beside it fails, where
natively it succeeds.
Sources and their sha256: `tests/gexcc/corpus/sources.txt`.
### Test runs and the host (2026-10-07, after gooner's crash)
Every gexcc program started on the host is a whole gexc (wasmtime, large
reservations, about 27 threads). Configure try-runs and a host-side
`make check` start thousands, and several suites in parallel took gooner
down (the system log showed about 45,000 processes a minute). Now:
- **Suites run inside one gexc** (`tests/gexcc/corpus/recipes.sh`):
- configure runs in cross mode (`--build=x86_64-pc-linux-gnu`), so it runs
nothing it builds;
- the host builds the suite (`make check TESTS=`) without running it;
- `guest make -k check SHELL=/bin/sh` runs the suite in the guest, with the
gexcc-built GNU make (`$GEXCC_WORK/guest-bin`) and busybox's sh and tools.
Every test process is a guest process.
- **Measured (xz 5.6.3, 19/19, in the guest):**
- 6 host execs: gexcc-exec's shell, its helpers and the one gexc;
- 92,037 host clones from that gexc's tree, about 120 a second over 13
minutes. These are threads, one per guest process: a guest fork plus
exec of `/bin/true` costs one host thread (50 execs: 77 clones against
27 for none).
- zlib's `make test`: 6 host execs.
- /proc/stat's machine-wide counter also counts those threads and other
users' processes (about 46 a second on an idle gooner).
- **In-guest results so far** (corpus3, one suite at a time; "host execs"
counts the run's host programs, "host clones" the gexc tree's threads):
| suite | tests | host execs | host clones |
|---|---|---|---|
| zlib | 3/3 | 6 | — |
| xz | 19/19 | 6 | 92,037 |
| bzip2 | 6/6 | 6 | 43 |
| jq | 28/29 (shtest: busybox `script` has no `-e`) | 6 | 8,203 |
| pcre2 | 1/3 (as before: ILP32 sizes, musl locales) | 6 | 11,027 |
| expat | 2/2 | 6 | 22,438 |
| libpng | 32/32 | 6 | 4,539 |
| lua, sqlite | 1/1 each | one host gexc each | — |
| gawk | 654/654 ("ALL TESTS PASSED") | 6 | 8,090 |
| sed | 206/214 (gnulib counted right; first in-guest run, before the path and select fixes) | 6 | 176,577 |
After that run (strace counting is now opt-in, `COUNT=1`, because it
slowed tar's suite):
- grep and diffutils stopped compiling in cross mode (gnulib's sigsegv
guessed a sigaltstack): fixed by `share/gexcc/config.site`, the answers
the same checks gave natively on gex.
- The gnulib failures in sed's guest run were gex bugs: trailing slashes
and `.`/`..` in creating calls (test-mkdir, test-rename, test-link,
test-symlink, test-rmdir, test-fopen), select's EBADF/EINVAL and a pipe
write end reported readable (test-select, test-pselect). Fixed, each
with a test (pathcorners, selectcorners); also O_NOFOLLOW and chown on
missing files (fscorners).
- findutils: its suites need DejaGnu (`runtest`), absent on gooner.
- **Slots** (`gexcc-exec`): at most `GEXCC_SLOTS` (4) host-started gexcc
programs run at once per user. The rest wait on an flock. Guest execs take
none. Tested by `tests/gexcc/slots/run.sh`: at most 2 at once with 2
slots, all 6 overlap without.
- **Every run** is wrapped in `systemd-run --user --scope -p TasksMax=256
-p MemoryMax=32G nice -n 19`, one suite at a time, `-j2`.
- **Converted:** automake suites, gawk, tar (autotest), bash's run-* loop,
CTest suites (`ctest-guest.py` writes the suite as one guest script).
**Not yet** (refused without `HOSTRUN=1`): make and curl (perl test
runners: need a gexcc perl in the guest), perl, python3.
## 3. Gaps found, and what fixed them
| gap | where it showed | fix | test |
|---|---|---|---|
| stdin read through std's 8 KiB buffer: input the guest never asked for was consumed | pcre2grep -m tests 132/133 | gex-kernel `StdStream` reads the host fd unbuffered | stdinseek |
| a stream redirected from a host file was a char device: no lseek, no size, no shared offset | pcre2grep -m | the host descriptor's offset, lseek and fstat (`FileHandle::seek`) | stdinseek |
| top-down mmap never reused a hole above its cursor: a 1 GiB buffer freed and asked for again got ENOMEM | expat test_buffer_can_grow_to_max, libpng pngvalid/pngimage | one more pass from the top when the cursor runs out | mmapreuse, gex-kernel unit test |
| `""` resolved to the working directory | Lua `package.searchpath` (`fopen("")` succeeded) | ENOENT for every empty path | emptypath |
| signals reached only code making system calls | findutils configure hung (alarm-guarded check); any busy loop | GexPoll: loop safepoints (§4) | alarmloop (native and browser) |
| no C++ runtime; with one, every `catch` was dropped | — | libc++ over wasm EH; `-mllvm -wasm-enable-eh` (clang enables it only for emscripten) | cxx |
| `main(argc, argv, envp)` did not link | GNU make | rt/main3.c adapter, linked when needed | make |
| prlimit answered 8 MiB/∞ for everything; no RLIMIT_NOFILE | gnulib test-dup2, test-getdtablesize, test-fcntl | per-process limits, NOFILE enforced by the fd table | fdsemantics |
| O_CREAT/O_TRUNC with read-only access failed (std refuses them); a directory opened for writing; *at() on a non-directory fd; fcntl/setsockopt on a bad fd | gnulib test-open, test-fdopendir, test-fopen, test-openat-safer, test-setsockopt | gex-kernel open/at_path/fcntl | fdsemantics |
| host FIFO opened O_NONBLOCK blocked; reads on it seeked (ESPIPE) | grep -D skip | O_NONBLOCK to the host; non-regular host files are streams | fifononblock |
| `file/` resolved; rename to `dir/.` | gnulib test-canonicalize-lgpl, test-rename | ENOTDIR for a trailing slash on a non-directory; EINVAL for . / .. | pathcorners |
| whole-second mtime only; futimens did nothing | gnulib test-stat-time; touch | atime, ctime and nanoseconds from the host; utimensat/futimens set them | timestamps |
| every exec of a wasm32 program recompiled it (2.2–2.7 s for busybox) | background `sleep 1; kill` arrived too late (test-sigprocmask) | compiled modules cached by content (`~/.cache/gex/modules/wasm32`): 0.17 s | — |
| a gexcc program's `sh`/`sleep`/`kill` ran the translated host tools | test-sigprocmask, dash script | busybox through gexcc-exec | — |
| `-Wl,-rpath -Wl,DIR`; Meson's `-Wl,--version` | libtool relinks; Meson | driver | xz, Meson demo |
| sigsetjmp/_setjmp not lowered (only `setjmp` is) | bash error recovery | include/setjmp.h wrapper + rt/sigsetjmp.c | sigjmp |
| calls through mismatched function pointer types trap | bash unwind-protect | `-fgex-fpcast-emu` (binaryen fpcast-emu, entries restored by elemfix.py) | fpcast |
| TCP shutdown did nothing; EPIPE on the open direction | gnulib test-select | Transport::shutdown, send_closed | tcphalfclose |
| "link/" with lstat, dangling/looping links with a slash | gnulib test-lstat | at_path | pathcorners |
| libtool made no shared library (ld -v not GNU-like) | xz | ld wrapper | xz |
| execve arguments capped at 4096 bytes | automake's test-suite.log rule run in the guest | Linux's limits (128 KiB a string, 2 MiB in all, E2BIG) | longargs |
| musl setjmp on try_table broke asyncify | coreutils timeout | GEX_WASM_LEGACY_EH libc | coreutils |
Expected gaps not reached yet (merge-plan §10):
- dlopen of real side modules (W's registry is link-time);
- `sigsetjmp`/`siglongjmp` across handlers;
- SIGSEGV from wasm traps;
- locale and iconv (musl's);
- syscall cost.
Platform differences, not gex bugs, recorded so they are not chased:
- musl accepts any locale name (Lua, pcre2grep test 150);
- ILP32 (wasm32) structure sizes differ from 64-bit expectations (pcre2
test 8);
- clang 22 rejects implicit function declarations, as gcc 14 does (old code
such as bash's termcap).
## 4. How it works (for changing it)
**The driver** (`tools/gexcc/gexcc`):
- **Compile steps** go to clang with W's flags (`tools/wine-wasm/flags.sh`;
setjmp lowered onto the legacy EH instructions, which asyncify can
instrument), plus:
- `-mllvm -wasm-enable-eh` (all languages) and `-fwasm-exceptions` (C++);
- `-fpass-plugin=GexPoll.so`;
- `-isystem` for `include/c++/v1`, `usr/local/include` and `usr/include`.
- **Links:**
- wasm-ld with `--experimental-pic -pie --shared-memory --import-memory`,
then crt1, the objects, main3.o when needed, the libraries,
libc++/libc++abi/libunwind for C++, libgexcc, libgex, libc and the
builtins;
- binaryen: asyncify if the program can fork, and always
`--translate-to-exnref`;
- `wasm-elf.py --host-stub hoststub.bin --runner PREFIX/bin/gexcc-exec`
writes the container.
- **Linker options** with no meaning for a static wasm module (rpath,
soname, version scripts, `-z`, as-needed, …) are dropped, including
arguments split across `-Wl`.
- **`-l` lookup:** the driver resolves `-lNAME` itself (a `libNAME.so`
archive, then `libNAME.a`) so that it can follow recorded dependencies.
libc's own names (`-lm -lpthread -ldl -lrt …`) are no-ops.
**The container's host launcher:**
- `wasm-elf.py --host-stub` puts a `jmp` at the entry and the stub at
entry+0x20.
- It leaves out `PT_GNU_STACK`, so an x86-64 host maps the read-only header
page executable (IA-32 `READ_IMPLIES_EXEC`).
- gex goes by `PF_X` alone, binds `__gex_start` at the entry and never runs
it.
- Without `--host-stub`, the container is byte-for-byte what W's wcc writes.
**Loop safepoints** (`tools/gexcc/poll/GexPoll.cpp`, `rt/poll.{c,s}`):
- gex delivers a signal to native code at a system call's return. The pass
puts `if (--__gex_poll_counter < 0) __gex_poll();` at every loop header,
at the end of the optimisation pipeline.
- The counter is a wasm global, so it is per thread.
- Every 65536 back-edges, `__gex_poll` re-arms it and makes one `getpid`,
where pending signals are delivered and interval timers checked.
- This works natively and in the browser. Its cost on the browser has not
been measured yet (edger).
**GexPoll's cost** (edger, browser runtime under node, lua 5.4.7 with and
without polls, ABBA, 6 runs each): median total 1.91 s with polls against
1.84 s without, about 4% slower, inside the run-to-run spread (1.52–2.25 s).
**C++** (`build-cxx.sh`):
- libc++/libc++abi/libunwind 22.1.8, static, with
`-D_LIBUNWIND_HIDE_SYMBOLS` (wasm is not ELF, so libunwind would otherwise
use `__declspec`).
- The `__cpp_exception` tag comes from `cpp_exception.s`, added to the
three archives.
**gex-kernel and gexc changes:**
- **stdio:** `StdStream` duplicates host fds 0–2. A regular file reads,
writes, seeks and stats through the duplicate. stdin is read unbuffered.
The browser (`target_family = "wasm"`) is unchanged.
- **mmap:** `MemoryMap::mmap` makes a second pass from `mmap_ceiling`.
- **Paths:** `at_path` rejects empty paths, a trailing slash on a
non-directory, and a dirfd that is not a directory. rename refuses `.`
and `..`.
- **Limits:** `Kernel::rlimits` holds the limits; `FdTable::limit` holds
NOFILE's soft limit.
- **Times:** `Metadata::times` and `Stat`'s atime, ctime and nanoseconds.
`FileSystem::set_times` and `FileHandle::set_times` serve utimensat and
futimens.
- **HostFs:** O_CREAT, O_EXCL, O_TRUNC and O_NONBLOCK go to the host as
flags. Non-regular files are streams.
- **gexc:** `run --host-env`. `gex_cache::compile_cached` caches compiled
wasm32 programs.
## 5. Resume (stream C)
- Worktree `~/gex-sandbox/wt/gexcc`, branch `merge/gexcc` (off master
5a066f67). Scratch: `~/gex-sandbox/scratch/CC`.
- `musl/`: clone of the musl fork, branch `gex-wasm32-cc` (= mw
gex-wasm32-slim; no changes yet).
- `tc/`: the installed toolchain.
- `src/`: tarballs (busybox, llvm-project-22.1.8 too).
- `llvm/`: the extracted LLVM runtimes tree.
- `bb/`: busybox experiments.
- `corpus/`: build trees, `logs/`, `results.txt`.
- `tmp/`: anything else.
- Reinstall the toolchain (about 15 s without busybox and libc++):
```sh
GEX_BINARYEN=~/gex-sandbox/scratch/W/sr-slim/sysroot/binaryen \
BUSYBOX_TAR=~/gex-sandbox/scratch/CC/src/busybox-1.37.0.tar.bz2 \
LLVM_SRC=~/gex-sandbox/scratch/CC/llvm \
tools/gexcc/install.sh ~/gex-sandbox/scratch/CC/tc ~/gex-sandbox/scratch/CC/musl
```
After editing only the driver: `cp tools/gexcc/gexcc ~/gex-sandbox/scratch/CC/tc/bin/`.
- Build gexc: `cargo build --release -p gexc --features gexc/x86`, giving
`target/release/gexc`, which gexcc-exec runs.
- Corpus:
```sh
export GEXCC_PREFIX=~/gex-sandbox/scratch/CC/tc GEXCC_SRC=~/gex-sandbox/scratch/CC/src GEXCC_WORK=~/gex-sandbox/scratch/CC/corpus
tests/gexcc/corpus/fetch.sh # pins sha256 on first fetch
tests/gexcc/corpus/run.sh NAME... # TESTONLY=1: rerun the suite only
```
- Don't edit `run.sh` while a run is going: bash reads scripts as it goes.
Long runs use a frozen copy (`scratch/CC/run-frozen.sh`).
- Programs built before a runner change keep the old runner baked in;
rebuild to pick up gexcc-exec.
- Checks before a gate:
- `cargo test -p gex-kernel`;
- `tests/wasm32/run.sh $PWD/target/release/gexc` (the gexc path must be
absolute; `GEX_WASM_SYSROOT=~/gex-sandbox/scratch/CC/sr/sysroot`);
- `tests/gexcc/run.sh`;
- shims build (`GEX_ARCH=x86 shims/kernel/build.sh`, then
`GEX_SHARED=1 GEX_ARCH=x86 shims/browser/build.sh`) and
`tests/gexcc/run-browser.sh`;
- the corpus.
- Ownership:
- The musl fork, Wine and the `$W` sysroots are stream W's. musl changes
go on `gex-wasm32-cc` in our clone; ask the coordinator before pushing
to mw.
- Don't edit crates/gex-cgen without asking.
- No game runs on gooner; the corpus is fine here. No timing on gooner.
- **Demo (§6), state 2026-10-08:**
- Design: §6.1a (arm64 native from Debian, `musl-linux-wasm32` ours),
approved. Item 1 done (gated as integ/2026-10-08f). Item 2 (aarch64
ELF providers) ready on branch `merge/gexcc-a64prov` (ecf2b251), no
x86 re-key; the coordinator schedules its landing with U. Item 3
(LP64-layout libraries lowered to wasm32, no memory64) not started:
measure the lowering's cost in V8 on edger first.
- AArch64-only gexc for testing: `cargo build --release -p gexc
--target-dir target/a64`. shims/browser/build.sh rebuilds
`target/release/gexc` without x86; rebuild it with `--features
gexc/x86` afterwards (gexcc-exec runs it).
- Debian builder: `tools/gexcc/debian/` (§6.3). Toolchain
`~/gex-sandbox/scratch/CC/dtc`; work area `scratch/CC/debian`
(`src/`, `stage/`, `logs/`, `built.txt`, `pool/`); Debian downloads
cached in `scratch/CC/deb`. Whole thing: `web/build-native-debian.sh`
with `GEXCC_PREFIX=~/gex-sandbox/scratch/CC/dtc`, in the build scope.
- Tier 1 rootfs: 44 Debian arm64 packages + 27 of ours, 91.5 MiB
unpacked, 20.4 MB as tar.zst.
- Works (gooner, correctness only): natively (`gexc run --uid 0:0
--sysroot TREE`) and in the browser runtime under Node
(`run-node.mjs --tar`, `GEX_UID=0`, `GEX_NETWORK=1`): bash, the tools,
dpkg (`--audit` clean), `apt-get update` against deb.debian.org
(signatures checked), `apt-get install jq` (arm64 jq, libjq1,
libonig5 translated) and jq runs; the pins keep ours over
`coreutils:arm64`. Test trees in `scratch/CC/rootfs` (`tree` fresh,
`native-cache.tar`: lists and jq's .debs, for `apt-get install
--no-download` without the network).
- Unverified: update and install in one browser-runtime session went
past 900 s once on gooner (each alone passes).
- Page startup, edger (bench.sh, headless Chromium, 8 threads,
`web/test/site.mjs native-debian`): browser launch to the
`root@gex:~#` prompt 5.4-5.6 s (5578, 5408, 5395 ms; 20.4 MB rootfs
from localhost). In the guest (`native-debian-cmd`, each figure
includes two `date` forks, about 0.5 s): `ls /` 1054 then 1041 ms,
`perl -e 1` 3234 then 2204 ms, `dpkg -l` 2658 then 2684 ms,
`apt-cache policy coreutils` 11504 then 7415 ms.
- Next: tier 2; GUI; item 3.
- **Next (corpus):**
1. Rerun the corpus clean.
2. coreutils, bash, make.
3. perl, python3 (recipes written), curl+mbedtls, libevent+tmux, vim,
fmt (C++), openssh, git.
4. The loopback TCP EPIPE/hang.
5. Measure GexPoll's cost on edger.
## 6. Demo: a native Debian rootfs (plan, 2026-10-08)
Goal (user, 2026-10-08): a slim but usable Debian root filesystem whose
userland is built from Debian's sources with gexcc into native wasm32
programs (not translated x86 binaries), plus developer tools and GUI apps,
booting in the browser page. Native gexc is for debugging; numbers come
from edger (bench.sh).
### 6.1 What "Debian" means here
- **Release:** trixie (13.7 on 2026-09-12). `InRelease` verified with the
host's `debian-archive-keyring`, `Sources.xz` checked against it, every
`.dsc` against `Sources`, every tarball against its `.dsc`
(`tools/gexcc/debian/debsrc.py`; binaries: `debfetch.py`, same chain
through `Packages.xz`).
- **Sources:** Debian source packages as Debian ships them:
`dpkg-source -x` unpacks the orig tarball and applies `debian/patches`.
Our changes on top are small patch files per package
(`tools/gexcc/debian/patches/PKG/*.patch`), applied after Debian's.
- **Not `dpkg-buildpackage`:** `debian/rules` cross-builds through
`dpkg-architecture`, debhelper and build-profiles, none of which know a
wasm32 musl architecture. Each package gets a recipe
(`tools/gexcc/debian/recipes.sh`) that runs its own build system with
gexcc and the configure options `debian/rules` uses, minus what we lack
(selinux, audit, systemd; nls where it costs size).
- **Binary packages:** each recipe installs into a staging tree; the staged
files are split into Debian's binary package names and written as real
`.deb`s with `dpkg-deb --build`. Control fields come from `debian/control`
and `debian/changelog`.
### 6.1a Two architectures: arm64 from Debian, wasm32 from us (user, 2026-10-08)
Requirements: apt is in the rootfs and works (`apt update`, `apt install X`
against real Debian mirrors, through gex's network path); what apt
downloads are Debian's **arm64** builds, run translated; our gexcc packages
live in a separate repository that apt never overwrites.
**Why arm64, measured** (edger, browser runtime under Node, Debian trixie's
own arm64 and amd64 builds of the same versions, whole-process wall time,
median of 3, ABBA; `scratch/CC/archbench`, 2026-10-08):
| workload | fjit arm64 | fjit amd64 | tier-1 arm64 | tier-1 amd64 |
|---|---|---|---|---|
| `xz --version` (start-up) | 3.48 s | 9.28 s | 1.62 s | 1.77 s |
| `xz -6` of 5.3 MB | 9.70 s | 16.10 s | 9.61 s | 12.15 s |
| `gzip -6` of 5.3 MB | 3.00 s | 10.76 s | 2.42 s | 3.39 s |
| python3.13, dict/sort/fib | 17.16 s | crash | 10.39 s | 13.84 s |
| sqlite3, 100k rows | 12.52 s | 24.58 s | 10.84 s | 12.61 s |
| lua5.4, fib/sort | 5.11 s | 10.69 s | 5.83 s | 7.82 s |
- arm64 is faster than amd64 in every case: on tier-1 by 9-29% (e.g.
python 10.39 s against 13.84 s), with fjit by 1.6-3.6 times (gzip 3.00 s
against 10.76 s). The user's claim holds.
- amd64 python3.13 under fjit crashes: one translated function (CPython's
eval loop) is 8,298,855 bytes, over V8's 7,654,321-byte function limit.
An fjit bug (sent to the coordinator).
- For short command-line runs fjit is slower than tier-1 on both
architectures (arm64 `xz --version` 3.48 s against 1.62 s, python
17.16 s against 10.39 s): its compile cost is not repaid in a few
seconds. The shell demo runs translated programs on tier-1 unless a
program is known to be long-running.
**dpkg's architectures.**
- Native architecture **arm64**: apt fetches Debian's arm64 index and
packages as native, its dependency resolution unmodified. `dpkg
--print-architecture` says arm64. Until our own dpkg and apt are built,
dpkg, apt, perl-base, debconf and sqv are Debian's arm64 builds,
translated.
- Foreign architecture **`musl-linux-wasm32`** (`dpkg --add-architecture`;
the name dpkg's tables give a musl wasm32 tuple) for every package we
build. Tested with the host's dpkg: accepted, and an arm64 package's
`Depends: coreutils (>= 9.7)` is satisfied by
`coreutils:musl-linux-wasm32`.
**How arm64 packages' dependencies are satisfied.**
- **Tools** (bash, coreutils, dash, grep, sed, tar, gzip, findutils,
diffutils, debianutils, mawk, hostname, ncurses-bin, util-linux,
bsdutils, bsdextrautils, sysvinit-utils, base-files, base-passwd, and
later perl, python3, git...) are ours, `Multi-Arch: foreign`, so they
satisfy any architecture's dependency by name. Version = Debian's +
`+gex1` (above that Debian revision and its `+debNuM` point updates).
- **Libraries** (`Multi-Arch: same`: libc6, libzstd1, libtinfo6...): an
arm64 program's ld.so loads arm64 ELF objects, so these are Debian's
arm64 packages, installed by apt as usual. The dpkg database stays true.
Native code reaches them through gex's loader, not through dpkg (below).
- **Essential:** `apt dist-upgrade` installs missing Essential packages of
the native architecture. Our repository carries every Essential tool by
name and the pins below stop the arm64 copies; simulated against the
real trixie arm64 index, `dist-upgrade` then pulls only libraries and the
arm64 system pieces the rootfs ships anyway (dpkg, apt, perl-base,
debconf, sqv, libc-bin, ca-certificates), and `apt install python3 git
vim` (128 packages) touches none of ours.
**Pinning** (`/etc/apt/preferences.d/gex-native`):
```
Package: bash:arm64 coreutils:arm64 ... (every package name we ship)
Pin: release *
Pin-Priority: -1
Package: *
Pin: release o=gex
Pin-Priority: 1001
```
Libraries are deliberately not pinned (pinning libc6:arm64 makes every
arm64 program uninstallable).
**Repositories** (`/etc/apt/sources.list.d/`):
- `debian.sources`: `Types: deb`, `URIs: http://deb.debian.org/debian`
(https once the guest's TLS is checked), `Suites: trixie
trixie-updates`, plus `trixie-security`, `Architectures: arm64`,
`Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg`.
- `gex.sources`: our repository, `Architectures: musl-linux-wasm32`,
Origin `gex`, Suite `trixie`, signed by a gex archive key
(`/usr/share/keyrings/gex-archive-keyring.gpg`; `trusted=yes` only for
local test builds). Layout: `dists/trixie/{InRelease,Release}`,
`dists/trixie/main/binary-musl-linux-wasm32/Packages.xz`,
`pool/main/<initial>/<srcpkg>/*.deb`. In the rootfs it is a `file:`
repository at `/var/lib/gex-apt`; the page serves the same tree beside
the rootfs, so packages not preinstalled (tier 2 extras) can be fetched
on `apt install` once the page exposes its own origin to the guest
(a host name the page's network layer answers with `fetch()` instead of
Wisp).
**The loader path for translated programs.**
- An arm64 program execs as today: gex maps it and Debian's
`ld-linux-aarch64.so.1` (translated), which maps Debian's arm64
`libc.so.6`, `libz.so.1`, ...
- When an object maps, gex's ELF-provider hook (docs/elf-providers.md)
binds native code over its exports, by soname and versioned symbol:
first glibc's mem*/str*, printf family and libm, as the x86-64 providers
do now; later whole libraries (zlib, zstd, lzma, sqlite, pcre2, expat:
pure-C ABIs) built by gexcc for an LP64 layout; finally the whole libc
(merge-plan §10.1). Unbound exports keep running translated, so this
degrades to correct.
- The providers ship as a package of ours (`gex-providers`, files under
`/usr/lib/gex/providers/`), so they are versioned with the rootfs.
- Our native programs link musl statically and need no loader. A
translated program that execs `/bin/sh` gets our dash; ours that exec an
arm64 program get it translated.
**gex work this needs (expected):**
1. **wasm32 programs in the aarch64 build: done** (0366efa6). gex-ld
accepts a wasm32 container (ELF32 i386 with the `GEXWASM` header) on
the AArch64 guest; the kernel keys the i386 ABI on the process
(`compat`) instead of the build; `ctx::sp_slot(compat)` (esp is `x(4)`
in every build) and `ctx::COMPAT_SCRATCH` (the vector registers of an
AArch64 context, which wasm32 programs never use) cover the two layout
differences. Checked on an AArch64-only gexc (tests/wasm32 5/5,
tests/gexcc 15/15, a wasm32 bash running Debian's arm64 xz, python3.13
and sqlite3, python's `os.system` running the wasm32 dash) and on the
aarch64 browser shim (tests/wasm32 5/5; tests/gexcc as on x86).
2. **AAPCS64 ELF providers:** the existing ones are SysV x86-64 only. An
aarch64 class (x0-x7, v0-v7, `TPIDR_EL0` TLS for errno) in gex-cgen's
`provider.rs` re-keys every unit: one commit, landed when the
coordinator schedules it.
3. **arm64-ABI native libraries in the guest's own wasm32 memory** (user,
2026-10-08: no memory64 at run time). Whole-library providers for
arm64 programs need aarch64's LP64 data layout. Approach to evaluate:
- Compile with an LP64 layout matching AAPCS64: clang's wasm64 target
for the layout, plus `-funsigned-char`. A layout test compares
`sizeof`/`offsetof`/alignment of every exported struct against
`aarch64-linux-gnu`: `long double` (16-byte quad), `wchar_t`,
`__int128` alignment, bitfields.
- Lower to wasm32 before shipping (binaryen `--memory64-lowering`, or
the same lowering in gex-cgen): addresses are i64 in the IR and become
i32 at each access. Guest addresses stay under 4 GB, so this is safe
today.
- **The address step is one hook** (one helper or one pass), not an
assumption the library code relies on. If memory ever runs short, the
plan is the dual heap (a guard on loads and stores of high pointers
that routes them to a second 4 GB heap), never memory64, and that
guard replaces the hook without changing the approach.
- The real work is glibc's aarch64 ABI over musl: opaque type sizes
(`pthread_mutex_t`, `sigset_t`), `struct stat`/`dirent`, `jmp_buf`,
`FILE` and the `stdin`/`stdout` symbols, `__errno_location`, symbol
versions (`memcpy@GLIBC_2.17`), `va_list` adapters (aarch64's is a
struct), and callbacks into translated code through the provider
guest-call path.
- First, measure what the lowering's i64 address arithmetic costs in V8
on a small benchmark (edger, browser runtime).
4. **fjit:** split or refuse functions over V8's size limit; a hotness
threshold so short processes stay on tier-1.
**Network.** Wisp, already in gex: the page's `WispNetwork`
(web/src/net.js), the kernel's socket shapes and its own DNS answers
(crates/gex-kernel/src/net.rs: a private address per name, mapped back to
the name when the guest connects, so apt's getaddrinfo works over a
TCP-only proxy). The demo's default endpoint is `wss://anura.pro/`
(user's), overridable with `?wisp=`. https goes through the guest's own TLS
(apt's OpenSSL). Test loops never use the public endpoint: natively gexc
has real sockets, and repeated runs go to a local HTTP mirror of the
cached trixie subset (`scratch/CC/deb`) instead of deb.debian.org.
### 6.2 Packages, in tiers
Tier 1, the shell (first milestone):
bash, dash, coreutils (single binary, symlinks), mawk (Debian's Essential awk), sysvinit-utils,
util-linux (the basics:
`more`, `column`, `kill`, `getopt`, `script`, `cal`, `hexdump`, `lsblk`
not), grep, sed, gawk, findutils, diffutils, tar, gzip, xz-utils, bzip2,
less, ncurses (libs + `tput`, `clear`, `reset`), procps (`ps`, `top`,
`free`, `uptime`, `pgrep`, `kill`), debianutils (`which`, `run-parts`,
`tempfile`), hostname, nano, vim (vim-tiny-sized `vim` with the Debian
runtime trimmed), file (+ libmagic), base-files, base-passwd,
sensible-utils, ncurses-base, tzdata (trimmed). Libraries: zlib, libbz2,
liblzma, libzstd, libmd, readline, pcre2, libacl/libattr where needed.
Help: `--help` everywhere; `man` (man-db + groff + libpipeline + gdbm) is
tier 2 since groff is C++ and large.
Tier 2, developer tools:
git (local clone/commit/log/diff; `git-remote-http` needs curl), python3.13
(stdlib trimmed: no test/, idlelib, tkinter, ensurepip), perl 5.40
(perl-base + perl-modules), make, sqlite3, lua5.4, jq, tree, patch, bc, ed,
strace-free: `gdb` not. tmux (libevent, ptys), htop (needs /proc), man-db,
dpkg itself (in the guest: `dpkg -l`, `dpkg -S`), apt if its dependency set
(OpenSSL, xxhash, lz4, zstd, gcrypt-free in 3.0) builds; curl + OpenSSL or
mbedtls (network over Wisp in the page). A C compiler in the guest: tcc
targeting i386 (its output then runs translated) is the cheap option;
clang/wasm-ld as wasm32 programs (gexcc in the guest) the impressive one,
about 80-100 MB of wasm, decided after measuring.
Tier 3, GUI (gex's Wayland compositor; Xwayland for X11, never Xvfb):
- Wayland: foot (wayland, libxkbcommon, pixman, fcft, freetype, fontconfig,
tllist, utf8proc), wayland-utils; fonts-dejavu-core.
- SDL2 (Wayland backend; software rendering, GLES through gex-gl's EGL):
chocolate-doom with freedoom (SDL2_mixer, SDL2_net), maybe another small
game.
- X11 under Xwayland (`hostx=1` in the page): libX11, libxcb, Xt, Xaw, Xmu,
Xext, Xrender; xterm, x11-apps (xeyes, xclock), xcalc.
- GTK 3 (glib, cairo, pango, harfbuzz, fribidi, gdk-pixbuf, epoxy, atk)
only if tiers 1-3 above are done: it is the biggest dependency graph.
Not attempted: ripgrep (Rust has no wasm32 Linux musl target; would need
`-Zbuild-std` and a custom target), systemd, anything needing kernel
modules or real devices.
### 6.3 Build layout
- `tools/gexcc/debian/` (state 2026-10-08):
- `debsrc.py`: verified source fetch, `dpkg-source -x`, our patches
(`patches/SRCPKG/`). `debfetch.py`: verified binaries and their
closure.
- `build.sh TIER|SRCPKG...` with `recipes.sh` (`build_SRCPKG`, installing
into `$GEXCC_DEB/stage/SRCPKG`; libraries also into the toolchain's
sysroot); `packages.txt` pins the versions and tiers.
- `mkdebs.py`: staging tree to binary packages (`binpkgs.txt`: the split
and `depends` lines), `Architecture: musl-linux-wasm32`, version
`+gex1`, Provides/Conflicts/Breaks/Replaces from `debian/control`.
- `mkrootfs.py`: Debian's arm64 closure (apt, dpkg, perl-base, debconf,
ca-certificates, base-files, ...) plus our packages standing in for
Debian's tools by name; every .deb unpacked under a merged /usr, no
maintainer scripts; dpkg's status/info (multiarch layout) written from
the packages; what the scripts would do (accounts, /etc/shells,
alternatives, CA bundle); apt sources, our `file:` repository at
/var/lib/gex-apt, the pins; Debian's minimized-image path excludes
(written to /etc/dpkg/dpkg.cfg.d/excludes too).
- `web/build-native-debian.sh [TIER...]`: all of the above, then
`web/public/rootfs/native-debian.tar.zst`. One command, cached.
- Builds run in `systemd-run --user --scope -p TasksMax=4096 -p
MemoryMax=32G nice -n 19`, configure in cross mode (§2: runs nothing it
builds).
- Reproducible: Debian versions pinned in `packages.txt`, our patches in
the tree.
### 6.4 The page
Demo `native-debian` ("Debian, built from source to wasm") on the aarch64
page (web/src/demos.js): fetches `/rootfs/native-debian.tar.zst`
(`?rootfs=` overrides; decoded with the vendored zstddec), bundles it with
its symbolic links as links (`gex_add_symlink`), and boots `/bin/bash -l`
(`-l -i`) as pid 1 with uid 0, `HOME=/root`, `TERM`, `PATH`. Network: Wisp,
default `wss://anura.pro/`, `?wisp=URL` overrides (`?wisp=` empty: none).
Translated programs run on tier 1 (`?fjit=1` turns the function JIT on).
`?cmd=` runs one command for tests. Under Node the same runtime runs the
tar with `run-node.mjs --tar FILE` (`GEX_UID=0`, `GEX_NETWORK=1` for real
sockets instead of Wisp); there apt updates from Debian and installs
arm64 packages (§5).
Size budget (zstd, transferred): tier 1 at most 15 MB, tier 1+2 at most
45 MB, everything at most 80 MB. Tier 1 today: 20.4 MB (Debian's arm64
system pieces are most of it: libc6, perl-base, libssl, apt, dpkg).
If the whole-tar load is too slow, split into a base tar and per-package
tars fetched when first opened (the kernel's demand-fill hook).
Startup (edger, bench.sh): page open to bash prompt; the first exec of
each program pays a WebAssembly compile; measured, then cut with a
compiled-module cache shared across Workers (exists) and, if needed,
ahead-of-page compilation of the hot set while the rootfs downloads.
### 6.5 Expected gex gaps
- See §6.1a for the arm64 items (wasm32 in the aarch64 build, AAPCS64
providers, LP64 libraries, fjit).
- Browser: exec/fork churn of a login shell (bash's startup forks for
`$(...)` in Debian's `/etc/bash.bashrc` and profile.d); module compile
per distinct program; the 2 GiB arena per process (python, git fine).
- ptys and job control for tmux, htop, vim, nano (`/dev/ptmx`, `TIOCSCTTY`,
`SIGWINCH` from the page's terminal size).
- `/proc` coverage for procps/htop (`/proc/PID/stat`, `statm`, `status`,
`/proc/meminfo`, `/proc/stat`, `/proc/uptime`, `/proc/loadavg`).
- `getpwnam` etc. through musl reading `/etc/passwd` (fine);
locales: musl's C.UTF-8 only.
- dlopen: python extension modules and perl XS are linked in statically
(Setup.local / static_ext); gexcc's shared-library archives make the
rest link.
- GUI: wl_shm + xdg-shell clients against gex-wayland; fcft/fontconfig
file access; SDL2's Wayland backend; X11 clients' unix socket to the
page's Xwayland.
- Program sizes: every program links libc statically; coreutils as one
binary keeps that once.
docs/wine-native.md
+125
View File
@@ -1834,6 +1834,131 @@ Steps:
## 7. State and how to resume (stream W)
**RESUME HERE (2026-10-09, early).** merge/w3-audio has master 41917e73
merged (bc6c6e2c) and three commits on top: 39bdabd8 `GEX_GL_WAITS`,
4d3bd189 the compositor read-buffer fix, dbf61bc7 `GEX_PROF` per-thread
totals and `GEX_SYS_WAITS`. Suite: `$W/check-w3-gate.sh` (W3 legs through
`gate-w3boot/run.sh` with `W3_GATE_DIST=$W/w3dist-g`, the audio leg on
w3dist-g too); 39bdabd8 clean in `$W/w3chk5.out` (the same failure list
as integ-e2), dbf61bc7 clean in `$W/w3chk6.out` (W3 null 40 of 53,
Wayland+GL 38 of 49, install as children and audio ok; x86/childloop
passed this time, it is load-dependent).
- **Install as children passes again.** Wine's DefaultInstall runs
`control.exe appwiz.cpl install_mono`, which waits for an answer; it
used to return at once only because control.exe was missing from the
corrupted prefix copy. install-as-children.sh passes
`WINEDLLOVERRIDES=${INSTALL_OVERRIDES:-mscoree,mshtml=}`. One dist from
one build: `$W/w3dist-g` = `$W/w3dist`'s suite dll dir (no wineboot)
with w3dist-n's loader.
- **Where the games' time went: the compositor.** The native host pumps
the compositor at every syscall of every guest thread, and each pump
allocated a zeroed 64 KiB buffer per client (calloc's memset, glibc
growing and trimming its heap: `syscall` and `_dl_mcount_wrapper`, i.e.
libc's memset/memcpy, were over half the process's CPU). 4d3bd189 keeps
the buffer (native and browser). On edger, against the same loader and
prefix: Retrocycles gameplay 3.0-3.5 fps -> 38-46 fps (the game's own
counter 37), Katana ZERO menu 4.4 -> 34 fps, Undertale plays its intro
at 30 fps (its own cap) once loaded (before: still loading at 420 s).
- **Undertale's "white window" was the frame dump, not the game.** Its
d3d9 output is a 640x480 gex-gl subsurface drawn on the host; the PPM
frames came from the decorated toplevel, which is white. Run it with
`GEX_PPM_GL_ONLY=1` (gamekit `ut9.sh`, frames in edger's
`gameout/undertale`). Its load is still long: the first GL present at
107 s, the next 24 by 350 s, then 30 fps.
Runs: gamekit `prof2-rc.sh` (before), `prof3-rc.sh` and `run8.sh`
(after); `run-game-g.sh` takes `GEXC=`.
- Tools: `GEX_GL_WAITS=1` (wall time in synchronous GL submits, by the
batch's last opcode: 1-3% in all three games, so GL round trips are not
the cost; Undertale's 3303 glGetInternalformativ at wined3d init are
89 ms in all), `GEX_SYS_WAITS=1` (wall time per thread and syscall),
`GEX_PROF` writes `<stem>.threads`, `GEX_PROF_MIN_PCT`.
- Next costs seen: every Wine child process runs `usd_clock_thread`
(wine fork, virtual.c), a 1 ms nanosleep loop that updates its
USER_SHARED_DATA clocks: six such threads in Undertale, about 900
wakeups a second each. Undertale's 350 s load: its main thread is
mostly parked (GEX_SYS_WAITS shows a futex wait of about 10 s in every
10 s window), so the next step is which thread it waits for.
**RESUME HERE (2026-10-08 evening, after the reboot).** Branch
**merge/w3-audio** (wt/w3-boot; master 7a4a62c8 merged): W3.5's OSS null
audio device (/dev/dsp, /dev/mixer), inet socket fixes (port at bind,
SHUT_WR, MSG_PEEK), tests/nt attachwin. Wine fork `wasm32-w3` 3710abbd462
(mw): **explorer's desktop is its own process again** (child processes see
the session through requests now); `WINEGEX_DESKTOP_THREAD=1` keeps the old
desktop thread. Why: a DllMain that makes a window holds the loader lock,
and the desktop thread needed that lock to start (Retrocycles and Katana
ZERO hung, "loader_section wait timed out"). Fix in the same commit: the
desktop process (explorer /desktop) does not start another explorer when its
lazily loaded null driver asks for the desktop window (it did: a chain of
seven explorers).
- Dists: `$W/w3dist-a` (the current loader + every W3 program skeleton,
wineboot included: games), `$W/w3dist-s` (the current loader over the
suite's dll dir without wineboot.exe, so a test doesn't boot the prefix:
`W3_DIST=$W/w3dist-s $W/check-w3-gate.sh wt/w3-boot`, output
`$W/w3chk4.out`). `$W/w3dist` stays the merge/w3-boot gate's.
- Games: `$W/gamekit` (gexc, dist -> w3dist-a, the installed prefix,
inputs), `gamekit/games.sh` runs Retrocycles, Undertale and Katana ZERO
on edger through bench.sh; results in edger's `$W/gameout/<slug>`
(summary, log, frames/, frames.txt: frame number and time, for fps).
- **Pitfall: never stage a W3 prefix with symbolic links into a shared
prefix.** stage-native.sh links the DLLs into ~/.cache/gex/wine-11.15;
under W3, wineboot writes system32 with CREATE_ALWAYS, which follows the
link and overwrites the shared file, and wineserver deletes on close by
the realpath() of the name it opened (server/fd.c), so a DeleteFile on a
linked file deletes the shared target (2026-10-08: 4 DLLs overwritten and
22 programs deleted in gooner's default prefix; the coordinator restored
them from edger's). `$W/deref-pfx.sh PREFIX` replaces those links with
copies; stage-w3.sh calls it. Any symlinked prefix is unsafe under
native Wine. (gooner's and edger's default prefixes have no
drive_c/windows/Fonts though the manifest lists 56 fonts: never staged,
not from this.)
- **Gate artifacts for merge/w3-boot** (frozen, nothing shared with the
other runs): `$W/gate-w3boot/run.sh TREE GEXC [stage]` stages its own
prefixes (pfx-w3 null driver, pfx-w3wl Graphics=wayland) and dists
(w3dist-null, w3dist-wl: copies of `$W/w3dist`, the 03:37 loader),
generates nt-run-x86.sh from TREE's tests/nt/run.sh, and runs the null,
Wayland+GL and install-as-children legs (its own install prefix, dist and
log under gate-w3boot). The Wayland leg needs no compositor running:
`gexc run --wayland PATH` serves its own at that guest path. On
integ/2026-10-08e's gexc: null 37 of 51, Wayland+GL 36 of 47 (w3chk3:
37 of 50, 34 of 46). x86/childloop times out at 120 s under load
(gooner's load average 21; 3 runs: 1 pass, 2 timeouts, the same with
merge/w3-audio's gexc): `gate-w3boot/childloop.sh`.
The install-as-children leg no longer passes on today's artifacts with
any gexc: integ-e2's, merge/w3-audio's and 36dd674f's own
(`wt/gate-36dd`) all write the registry (16 crypto provider keys) and
then stop after DefaultInstall's rundll32, so wineboot never exits
(`gate-w3boot/inst-{i,b}`). The 03:37 loader's PE side was relinked since
(`$W/wine-w32-w3`, the W3.5 work), so the 03:37 loader no longer has a
matching dll dir; with the current loader (w3dist-a) explorer's window
dies with "indirect call type mismatch" there (`inst-a`). Next: one
dist whose loader and dll dir come from the same build.
- **Guest GL (2026-10-08 night):** the loaders linked `$W/gl/gexgl.o` from
10-06 (wt/w3's gexgl.c); `$W/gl/build-gl.sh` now compiles
`${GL_TREE:-wt/w3-boot}`'s. 7ef0f203 adds glNormalPointer (the host had
GL_NORMAL_ARRAY enabled with no pointer: Retrocycles' host SIGSEGV at
gameplay start, core analysed on gooner from edger's,
`$W/core/`; the GL capture decoder is `$W/core/capdec`). Loader with it:
`$W/w3dist-n` (gamekit/dist points there).
- Undertale and Katana ZERO (both GameMaker, d3d9 over wined3d) called
address 0 in wined3d's caps probe: match_allows_spec_alpha calls
glSecondaryColorPointerEXT, which wined3d assumes on GL 1.4, and the guest
library had no secondary colour (or fog coordinate) array, so the
extension was withheld and wglGetProcAddress returned NULL. 47f830a4
adds both (loader `$W/w3dist-n`, 20:22). Found with `gamekit/dbg-ut6.sh`
(WINEDEBUG trace+d3d) and `gamekit/glprobe` (glprobe2.exe lists the
extension functions wglGetProcAddress cannot resolve).
- Retrocycles on w3dist-n (edger, 2026-10-08 20:0x): menus, then gameplay
for the rest of the 420 s run, no crash; composited 10-12 fps in the
menus and 1-6 fps in gameplay (the game's own counter: 1). Frames:
`$W/shots/r7`.
- With those, Undertale and Katana ZERO got past wined3d (two more
fixes: 0a525987, program option extensions claimed only with their
base; 552cca51, gex-wayland releases a buffer once per attach, which
Wine's buffer queue counts) and then hit the release bug at the first
swap-chain resize. gexc 22:25 has the fix; run `gamekit/run7.sh`.
**RESUME HERE (2026-10-06, after a Claude Code restart).** `$W` =
`~/gex-sandbox/scratch/W`. Run `. ~/gex-sandbox/scratch/env.sh` first.
`gexc run` does not pass the host environment: `--env HOME=...`.
guest/gl/gexgl.c
+47 -6
View File
@@ -1303,8 +1303,21 @@ static void gexgl_record_attrib(unsigned index, int size, unsigned type,
/* The fixed-function arrays (glVertexPointer and friends), kept like the
generic attributes above: a pointer with no GL_ARRAY_BUFFER bound is client
memory, uploaded just before each draw. Index 0 is the vertex array, 1 the
colour array, 2 + n texture unit n's coordinates. */
#define GEXGL_FF_ARRAYS 6
colour array, 2 + n texture unit n's coordinates, then the normal array
(glNormalPointer: the compat profile's planned opcode, which the host
decodes like the generated commands). Before the normal array was kept,
GL_NORMAL_ARRAY reached the host enabled with no pointer and the host's GL
read normals from address 0 (Retrocycles' display lists: SIGSEGV in the
host driver). */
#define GEXGL_FF_TEX_UNITS 4
#define GEXGL_FF_NORMAL (2 + GEXGL_FF_TEX_UNITS)
#define GEXGL_FF_SECONDARY (GEXGL_FF_NORMAL + 1)
#define GEXGL_FF_FOG (GEXGL_FF_SECONDARY + 1)
#define GEXGL_FF_ARRAYS (GEXGL_FF_FOG + 1)
/* compat_plans.rs: the planned opcodes, as the hosts decode them. */
#define GEXGL_NORMAL_POINTER_PLAN 10051 /* "v w w P" */
#define GEXGL_SECONDARY_COLOR_POINTER_PLAN 10560 /* "v w w w P" */
#define GEXGL_FOG_COORD_POINTER_PLAN 8915 /* "v w w P" */
struct ff_array_state {
int size;
unsigned type;
@@ -1367,7 +1380,17 @@ static void gexgl_upload_client_arrays(size_t vertices, int instances) {
static void ff_pointer_command(unsigned slot, const struct ff_array_state *a, uint64_t offset) {
static const uint32_t ops[3] = { GEXGL_VERTEX_POINTER, GEXGL_COLOR_POINTER, GEXGL_TEX_COORD_POINTER };
uint32_t *p = begin(ops[slot < 2 ? slot : 2], 5, 0);
/* The normal and fog coordinate arrays have no size on the wire. */
if (slot == GEXGL_FF_NORMAL || slot == GEXGL_FF_FOG) {
uint32_t *p = begin(slot == GEXGL_FF_NORMAL ? GEXGL_NORMAL_POINTER_PLAN
: GEXGL_FOG_COORD_POINTER_PLAN, 4, 0);
if (!p) return;
p[0] = a->type; p[1] = (uint32_t)a->stride;
p[2] = (uint32_t)offset; p[3] = (uint32_t)(offset >> 32);
return;
}
uint32_t *p = begin(slot == GEXGL_FF_SECONDARY ? GEXGL_SECONDARY_COLOR_POINTER_PLAN
: ops[slot < 2 ? slot : 2], 5, 0);
if (!p) return;
p[0] = (uint32_t)a->size; p[1] = a->type; p[2] = (uint32_t)a->stride;
p[3] = (uint32_t)offset; p[4] = (uint32_t)(offset >> 32);
@@ -1393,7 +1416,7 @@ static void gexgl_upload_ff_arrays(size_t vertices) {
}
glBindBuffer(0x8892u, a->scratch);
glBufferData(0x8892u, (long)bytes, a->pointer, 0x88E8u /* DYNAMIC_DRAW */);
if (i >= 2) CMD1(GEXGL_CLIENT_ACTIVE_TEXTURE, 0x84C0u + (i - 2));
if (i >= 2 && i < GEXGL_FF_NORMAL) CMD1(GEXGL_CLIENT_ACTIVE_TEXTURE, 0x84C0u + (i - 2));
ff_pointer_command(i, a, 0);
}
CMD1(GEXGL_CLIENT_ACTIVE_TEXTURE, 0x84C0u + unit);
@@ -1404,8 +1427,11 @@ static struct ff_array_state *ff_slot_for(unsigned array) {
switch (array) {
case 0x8074u: return &ff_arrays[0]; /* GL_VERTEX_ARRAY */
case 0x8076u: return &ff_arrays[1]; /* GL_COLOR_ARRAY */
case 0x8075u: return &ff_arrays[GEXGL_FF_NORMAL]; /* GL_NORMAL_ARRAY */
case 0x845Eu: return &ff_arrays[GEXGL_FF_SECONDARY]; /* GL_SECONDARY_COLOR_ARRAY */
case 0x8457u: return &ff_arrays[GEXGL_FF_FOG]; /* GL_FOG_COORD_ARRAY */
case 0x8078u: /* GL_TEXTURE_COORD_ARRAY */
return client_active_texture < GEXGL_FF_ARRAYS - 2 ? &ff_arrays[2 + client_active_texture] : NULL;
return client_active_texture < GEXGL_FF_TEX_UNITS ? &ff_arrays[2 + client_active_texture] : NULL;
default: return NULL;
}
}
@@ -1437,9 +1463,24 @@ void glColorPointer(int size, unsigned type, int stride, const void *pointer) {
ff_record(1, size, type, stride, pointer);
}
void glTexCoordPointer(int size, unsigned type, int stride, const void *pointer) {
if (client_active_texture < GEXGL_FF_ARRAYS - 2)
if (client_active_texture < GEXGL_FF_TEX_UNITS)
ff_record(2 + client_active_texture, size, type, stride, pointer);
}
/* Three components always; `size` is kept for the upload's element size. */
void glNormalPointer(unsigned type, int stride, const void *pointer) {
ff_record(GEXGL_FF_NORMAL, 3, type, stride, pointer);
}
/* The secondary colour and fog coordinate arrays likewise (GL 1.4; the EXT
spellings are gl-aliases-generated.inc's). Missing, they kept
GL_EXT_secondary_color unclaimed, and wined3d, which assumes it on GL 1.4,
called a null glSecondaryColorPointerEXT in its caps probe (Undertale,
Katana ZERO). */
void glSecondaryColorPointer(int size, unsigned type, int stride, const void *pointer) {
ff_record(GEXGL_FF_SECONDARY, size, type, stride, pointer);
}
void glFogCoordPointer(unsigned type, int stride, const void *pointer) {
ff_record(GEXGL_FF_FOG, 1, type, stride, pointer);
}
void glClientActiveTexture(unsigned unit) {
client_active_texture = unit >= 0x84C0u ? unit - 0x84C0u : 0;
CMD1(GEXGL_CLIENT_ACTIVE_TEXTURE, unit);
guest/gl/gl-aliases-generated.inc
+3 -1
View File
@@ -4,7 +4,7 @@
* a different name. An extension requires the suffixed spelling even
* when the function has been core, unsuffixed, for twenty years, and
* an extension whose entry points are missing cannot honestly be
* claimed -- so here are the 594 of them whose target this
* claimed -- so here are the 596 of them whose target this
* library already exports. Included last, where every target is
* declared, and typed from the target so the two cannot disagree.
*/
@@ -154,6 +154,7 @@ extern __typeof(glEndTransformFeedback) glEndTransformFeedbackNV __attribute__((
extern __typeof(glFenceSync) glFenceSyncAPPLE __attribute__((alias("glFenceSync")));
extern __typeof(glFlushMappedBufferRange) glFlushMappedBufferRangeAPPLE __attribute__((alias("glFlushMappedBufferRange")));
extern __typeof(glFlushMappedBufferRange) glFlushMappedBufferRangeEXT __attribute__((alias("glFlushMappedBufferRange")));
extern __typeof(glFogCoordPointer) glFogCoordPointerEXT __attribute__((alias("glFogCoordPointer")));
extern __typeof(glFogCoordd) glFogCoorddEXT __attribute__((alias("glFogCoordd")));
extern __typeof(glFogCoorddv) glFogCoorddvEXT __attribute__((alias("glFogCoorddv")));
extern __typeof(glFogCoordf) glFogCoordfEXT __attribute__((alias("glFogCoordf")));
@@ -403,6 +404,7 @@ extern __typeof(glSecondaryColor3ui) glSecondaryColor3uiEXT __attribute__((alias
extern __typeof(glSecondaryColor3uiv) glSecondaryColor3uivEXT __attribute__((alias("glSecondaryColor3uiv")));
extern __typeof(glSecondaryColor3us) glSecondaryColor3usEXT __attribute__((alias("glSecondaryColor3us")));
extern __typeof(glSecondaryColor3usv) glSecondaryColor3usvEXT __attribute__((alias("glSecondaryColor3usv")));
extern __typeof(glSecondaryColorPointer) glSecondaryColorPointerEXT __attribute__((alias("glSecondaryColorPointer")));
extern __typeof(glShaderSource) glShaderSourceARB __attribute__((alias("glShaderSource")));
extern __typeof(glSpecializeShader) glSpecializeShaderARB __attribute__((alias("glSpecializeShader")));
extern __typeof(glStencilOpSeparate) glStencilOpSeparateATI __attribute__((alias("glStencilOpSeparate")));
guest/gl/gl-extensions-generated.inc
+3 -8
View File
@@ -2,7 +2,7 @@
*
* Extensions the guest may repeat from the host: every command the
* Khronos registry says they require is exported by this library.
* 715 claimable, 145 withheld for missing entry
* 710 claimable, 150 withheld for missing entry
* points, out of the registry's whole extension set. A host
* extension absent from this table is not advertised to the guest,
* which is why adding a driver capability can need a rebuild here.
@@ -106,7 +106,6 @@ static const char *const gexgl_claimable_extensions[] = {
"GL_ARB_explicit_uniform_location",
"GL_ARB_fragment_coord_conventions",
"GL_ARB_fragment_layer_viewport",
"GL_ARB_fragment_program_shadow",
"GL_ARB_fragment_shader",
"GL_ARB_fragment_shader_interlock",
"GL_ARB_framebuffer_no_attachments",
@@ -288,6 +287,7 @@ static const char *const gexgl_claimable_extensions[] = {
"GL_EXT_draw_range_elements",
"GL_EXT_draw_transform_feedback",
"GL_EXT_float_blend",
"GL_EXT_fog_coord",
"GL_EXT_fragment_shading_rate",
"GL_EXT_framebuffer_blit",
"GL_EXT_framebuffer_blit_layers",
@@ -338,6 +338,7 @@ static const char *const gexgl_claimable_extensions[] = {
"GL_EXT_rescale_normal",
"GL_EXT_sRGB",
"GL_EXT_sRGB_write_control",
"GL_EXT_secondary_color",
"GL_EXT_semaphore_fd",
"GL_EXT_separate_depth_stencil",
"GL_EXT_separate_specular_color",
@@ -497,9 +498,7 @@ static const char *const gexgl_claimable_extensions[] = {
"GL_NV_fog_distance",
"GL_NV_fragment_coverage_to_color",
"GL_NV_fragment_program",
"GL_NV_fragment_program2",
"GL_NV_fragment_program4",
"GL_NV_fragment_program_option",
"GL_NV_fragment_shader_barycentric",
"GL_NV_fragment_shader_interlock",
"GL_NV_framebuffer_blit",
@@ -581,10 +580,6 @@ static const char *const gexgl_claimable_extensions[] = {
"GL_NV_uniform_buffer_unified_memory",
"GL_NV_vertex_array_range2",
"GL_NV_vertex_attrib_integer_64bit",
"GL_NV_vertex_program1_1",
"GL_NV_vertex_program2",
"GL_NV_vertex_program2_option",
"GL_NV_vertex_program3",
"GL_NV_vertex_program4",
"GL_NV_viewport_array2",
"GL_NV_viewport_swizzle",
shims/browser/js/gex.mjs
+18 -1
View File
@@ -1103,7 +1103,7 @@ export class Gex {
// Each of these is called through `?.` at its use site so a shim that
// predates it still loads -- which is how a stale runtime turns into a
// silent wrong answer (no `gex_chmod` leaves every file 0644).
'gex_chmod', 'gex_add_dir', 'gex_add_env',
'gex_chmod', 'gex_add_dir', 'gex_add_env', 'gex_add_symlink',
'gex_paused_for_input', 'gex_paused_for_net', 'gex_paused_for_host',
'gex_park', 'gex_resume_pc',
'gex_resume_ctx', 'gex_exit_code', 'gex_jit_pending_len',
@@ -1278,6 +1278,23 @@ export class Gex {
free?.(pp, p.length);
free?.(tp, t.length);
}
// Symbolic links (`web/src/tar.js`'s `links`), after files and hard links:
// path -> target text. Only with the global kernel (a page or run-node):
// a process that is the whole system has no filesystem yet to link in,
// so a caller without one passes copies as files instead.
for (const [path, target] of Object.entries(opts.symlinks ?? {})) {
const p = enc.encode(path);
const t = enc.encode(target);
const pp = this.#shim.exports.gex_alloc(p.length) >>> 0;
const tp = this.#shim.exports.gex_alloc(t.length) >>> 0;
this.#bytes().set(p, pp);
this.#bytes().set(t, tp);
if (!this.#shim.exports.gex_add_symlink(tp, t.length, pp, p.length)) {
console.warn(`gex: symlink ${path} -> ${target} not made (no global kernel)`);
}
free?.(pp, p.length);
free?.(tp, t.length);
}
// The bundle's file times (seconds since the epoch), as the host had them:
// a program that records one -- gzip's header, make's rebuild test --
// sees what it would natively rather than 1970.
shims/browser/js/run-node.mjs
+20 -2
View File
@@ -7,6 +7,9 @@
// --file <guest>=<host> bundle a host file into the guest's filesystem
// --dir <guest>=<host> bundle a whole directory tree
// --extract <guest>=<host> write a file back out when the program ends
// --tar <host.tar> a root filesystem tar (web/src/tar.js, as the
// page unpacks one): files, modes, empty
// directories, hard and symbolic links
// --busybox <host> a wasm32 busybox as /bin/busybox, hard-linked
// as /bin/<applet> and /usr/bin/<applet> for
// tools/wine-wasm/busybox.applets
@@ -63,6 +66,8 @@ let cgenMode = 0;
let noJit = false;
let aotRun = false;
const dirs = [];
// Symbolic links, path -> target text (`--tar`).
const symlinks = {};
const rest = [];
{
const av = process.argv.slice(2);
@@ -135,6 +140,15 @@ const rest = [];
const eq = spec.indexOf('=');
if (eq <= 0) { console.error('--extract wants <guest>=<host>'); process.exit(2); }
extract.push([spec.slice(0, eq), spec.slice(eq + 1)]);
} else if (av[i] === '--tar') {
const { untar } = await import('../../../web/src/tar.js');
const buf = readFileSync(av[++i] ?? '');
const tree = untar(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
Object.assign(files, tree.files);
Object.assign(modes, tree.modes ?? {});
Object.assign(hard, tree.hard ?? {});
Object.assign(symlinks, tree.links ?? {});
dirs.push(...(tree.dirs ?? []));
} else if (av[i] === '--busybox') {
const host = av[++i] ?? '';
files['/bin/busybox'] = readFileSync(host);
@@ -476,6 +490,8 @@ try {
args,
opts: {
strace,
// GEX_STRACE_PID=N: narrate that one process (a forked child too).
stracePid: process.env.GEX_STRACE_PID ? Number(process.env.GEX_STRACE_PID) : undefined,
mtimes,
// The debugging switches, in the run's own options as well as the boot
// message: `opts` is what a fork record carries to a child's Worker.
@@ -487,13 +503,15 @@ try {
intrinsicMask: process.env.GEX_INTRINSIC_MASK ? Number(process.env.GEX_INTRINSIC_MASK) : undefined,
// The suite compares this run against QEMU on the same machine, and QEMU
// reports the real user.
uid: process.getuid?.(),
gid: process.getgid?.(),
// GEX_UID=0 runs the guest as root (a rootfs's dpkg wants it).
uid: process.env.GEX_UID !== undefined ? Number(process.env.GEX_UID) : process.getuid?.(),
gid: process.env.GEX_UID !== undefined ? Number(process.env.GEX_UID) : process.getgid?.(),
stdin: readStdin(),
cwd,
env: env.length ? env : undefined,
dirs,
hard,
symlinks,
modes,
// GEX_PINS names a layout written by `gexc build --closure`. Every
// object in it is placed exactly where the build assumed, which is
shims/browser/js/tty.mjs
+21 -4
View File
@@ -40,12 +40,29 @@ export class PageTty {
// A maximum, because a memory import's limits are part of its type: one
// with no maximum does not satisfy an import that declares one. 65535 is
// what `tools/add-memory.mjs` writes for the process import.
const scratch = new WebAssembly.Memory({ initial: 1, maximum: 65535 });
//
// Shared or plain to match the build: the page loads the kernel whose
// process import is shared when its processes' memories are (every
// multi-threaded shim), and a memory import's `shared` flag is part of
// its type. Only the plain one was offered, so the page's terminal never
// opened under the shared shim -- harmless for a translated shell, which
// gives its stack back and is fed by the fallback below, and fatal for a
// wasm32 one, which waits inside its own frames for keys that only the
// kernel's terminal can hand it ("can't even type").
let inst;
let scratch;
try {
inst = await WebAssembly.instantiate(mod, {
env: { memory: spec.memory, process: scratch, lazy_fetch: lazyFetch(spec.memory) },
});
try {
scratch = new WebAssembly.Memory({ initial: 1, maximum: 65535, shared: true });
inst = await WebAssembly.instantiate(mod, {
env: { memory: spec.memory, process: scratch, lazy_fetch: lazyFetch(spec.memory) },
});
} catch {
scratch = new WebAssembly.Memory({ initial: 1, maximum: 65535 });
inst = await WebAssembly.instantiate(mod, {
env: { memory: spec.memory, process: scratch, lazy_fetch: lazyFetch(spec.memory) },
});
}
} catch (e) {
// A module that will not take a plain process memory is the shared build,
// not what a page loads. Warn rather than fail silently: the fallback
shims/browser/src/gl.rs
+1
View File
@@ -135,6 +135,7 @@ impl FileHandle for GlDevice {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
shims/browser/src/lib.rs
+56 -5
View File
@@ -565,7 +565,9 @@ extern "C" fn r_has_children(ppid: i32) -> i32 {
fn signal_note(what: &str, caller: i32, target: i32, signo: u32) {
use std::sync::atomic::{AtomicU32, Ordering};
static SAID: AtomicU32 = AtomicU32::new(0);
if signo == 0 || SAID.fetch_add(1, Ordering::Relaxed) >= 24 {
// SIGWINCH (28 on every guest) changes nothing: bash -l sends it to its
// own group on start.
if signo == 0 || signo == 28 || SAID.fetch_add(1, Ordering::Relaxed) >= 24 {
return;
}
warn_line(&format!("[sig] pid {caller} sends signal {signo} to {what} {target}\n"));
@@ -1229,6 +1231,7 @@ impl FileHandle for StdinQueue {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
@@ -1368,6 +1371,7 @@ impl FileHandle for Tty {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
@@ -1429,6 +1433,7 @@ impl FileHandle for Console {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
@@ -3910,7 +3915,8 @@ pub extern "C" fn gex_load(ptr: u32, len: u32, argv: u32, argv_len: u32) -> u64
for i in 0..31u8 {
write_u64(data, c + ctx::x(i), 0);
}
write_u64(data, c + ctx::SP, layout.sp);
// (an i386-ABI process's is `esp` in every build: `ctx::sp_slot`)
write_u64(data, c + ctx::sp_slot(program.compat()), layout.sp);
write_u64(data, c + ctx::PC, program.entry());
data[c + ctx::SAFEPOINT_COUNTER..c + ctx::SAFEPOINT_COUNTER + 4]
.copy_from_slice(&1024i32.to_le_bytes());
@@ -4165,6 +4171,22 @@ pub extern "C" fn gex_add_hardlink(target: u32, target_len: u32, path: u32, path
0
}
/// A bundled symbolic link: `path` names `target` (text, resolved when the
/// guest follows it), as in the bundle. A rootfs whose links became copies
/// carries a single-binary coreutils once per command name. Returns 1 when
/// made; 0 tells the caller to add a copy instead (no global kernel).
#[no_mangle]
pub extern "C" fn gex_add_symlink(target: u32, target_len: u32, path: u32, path_len: u32) -> i32 {
// SAFETY: scalars naming ranges of this module's memory, no borrow held.
if unsafe { k_present() } != 0 {
if unsafe { k_fs_populated() } == 0 {
return (unsafe { k_fs_symlink(gex_kernel::PAddr::from(target), target_len, gex_kernel::PAddr::from(path), path_len) } == 0) as i32;
}
return 1;
}
0
}
/// A bundled file's modification time, as the host's filesystem had it, so a
/// program that records one (`gzip`, `make`, `tar`) sees what it would
/// natively. Like `gex_chmod`.
@@ -6860,7 +6882,7 @@ fn wasm32_wait(pc: u64) -> i32 {
// A wait on the page itself (its keyboard, the Worker's own
// network) needs this Worker's event loop, which a wasm32
// program's frames hold: it is polled here instead.
warn_line("gex: a wasm32 program waits on the page (stdin or the network): polled\n");
trace_line("gex: a wasm32 program waits on the page (stdin or the network): polled\n");
said = true;
}
if gex_park(20) < 0 {
@@ -8018,7 +8040,8 @@ fn do_exec(h: &mut Host, path: &str, argv: &[String], envp: &[String]) -> u64 {
// `sp` is the stack just built, not the recorded one: the two are the
// same address by construction -- see `BLOB_RESERVE` -- and the argv on
// it is this process's.
write_u64(data, c + ctx::SP, layout.sp);
let compat = program.as_ref().is_some_and(|p| p.compat());
write_u64(data, c + ctx::sp_slot(compat), layout.sp);
let entry = match (&program, &restored) {
(Some(p), _) => p.entry(),
(None, Some(l)) => l.entry,
@@ -10099,6 +10122,21 @@ pub extern "C" fn gex_compat() -> i32 {
#[no_mangle]
pub extern "C" fn gex_set_compat(on: i32) {
with_host(|h| h.kernel.compat = on != 0);
// An adopted child's stack (`gex_adopt_child`, before this) went to
// `ctx::SP`; an i386-ABI process's is `esp` (`ctx::sp_slot`), which on
// an AArch64 build is another slot (its i386-ABI processes are wasm32
// programs: posix_spawn's child starts on that stack).
if on != 0 && !gex_abi::X86 && !gex_abi::PPC {
with_host(|h| {
let mut gm = GuestMem::new(guest_bytes());
let c = h.kernel.ctx_addr;
let sp = gm.read_u64(c + ctx::SP as u64).unwrap_or(0);
if sp != 0 {
let _ = gm.write_u64(c + ctx::sp_slot(true) as u64, sp);
let _ = gm.write_u64(c + ctx::SP as u64, 0);
}
});
}
gex_xlate::set_compat(on != 0);
}
@@ -10401,8 +10439,18 @@ pub extern "C" fn gex_cache_op(_ctx: i32, _pc: i64, op: i32, addr: i64) {
// A block failed its page check (gex-emit `emit_page_check`):
// translate its address again if its module still provides it.
let start = _pc as u64;
// The same stale block asking twice in a row: the catch-up below did
// not move this Worker's directory off it, so the newer translation
// it waits for is never coming here (seen in an exec'd aarch64
// program's `_dl_fini`, which re-entered the block from JavaScript
// for ever). Translating it again is always correct, only slower.
let repeat = STALE_LAST.with(|c| {
let same = c.get() == (start, addr as u64);
c.set((start, addr as u64));
same
});
let again = with_host(|h| {
let again = h.jit.provider.get(&start).map_or(true, |&m| m == addr as u64);
let again = repeat || h.jit.provider.get(&start).map_or(true, |&m| m == addr as u64);
if again {
h.jit.translated.remove(&start);
h.jit.provider.remove(&start);
@@ -10740,6 +10788,9 @@ thread_local! {
/// next dispatch miss's answer: that miss must not answer
/// `DISPATCH_MISS_LEAVE` (the caller would dispatch again, for ever).
static NO_LEAVE: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
/// The last `PAGE_STALE` (block, module), to tell a stale block that
/// keeps coming back from one met once. See `gex_cache_op`.
static STALE_LAST: std::cell::Cell<(u64, u64)> = const { std::cell::Cell::new((0, 0)) };
}
/// Dispatch misses by how `gex_dispatch_miss` dealt with them (the NT
shims/browser/src/wl.rs
+9 -1
View File
@@ -281,6 +281,9 @@ struct Session {
/// Event bytes the socket had no room for yet, and how many have gone.
out: Vec<u8>,
sent: usize,
/// Where requests are read into, kept rather than a fresh zeroed 64 KiB
/// each pump (as the native host's `wayland.rs`).
rbuf: Vec<u8>,
}
#[derive(Default)]
@@ -356,7 +359,10 @@ impl WaylandDevice {
let mut changed = false;
sessions.retain_mut(|s| {
let mut gone = false;
let mut buf = vec![0u8; 64 * 1024];
let mut buf = std::mem::take(&mut s.rbuf);
if buf.is_empty() {
buf = vec![0u8; 64 * 1024];
}
loop {
let n = match s.socket.recv(&mut buf, 32, false) {
Ok(0) => {
@@ -379,6 +385,7 @@ impl WaylandDevice {
break;
}
}
s.rbuf = buf;
// Deliver this turn's events now that any `get_pointer`/
// `get_keyboard` from this turn's bytes has been processed.
if s.client.wants_input() {
@@ -707,6 +714,7 @@ impl HostListener for WaylandDevice {
fds: Vec::new(),
out: Vec::new(),
sent: 0,
rbuf: Vec::new(),
});
}
}
shims/hostwl/src/lib.rs
+1
View File
@@ -96,6 +96,7 @@ impl FileHandle for Pipe {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
shims/native/src/gl.rs
+59
View File
@@ -56,6 +56,8 @@ pub struct GlDevice {
/// The guest's memory (`set_memory`): a synchronous batch is read where
/// it is in it.
memory: Mutex<Option<gex_mem::GuestMemory>>,
/// The last synchronous batch's last opcode, for `GEX_GL_WAITS`.
first_op: std::sync::atomic::AtomicU32,
}
/// `GEX_GL_SYNC=1`: every batch runs before its submit returns.
@@ -64,6 +66,45 @@ fn sync_only() -> bool {
*ON.get_or_init(|| std::env::var("GEX_GL_SYNC").as_deref() == Ok("1"))
}
/// `GEX_GL_WAITS=1`: about once a second, how many synchronous submits the
/// guest made and how long it waited in them (wall time inside `submit`,
/// which is all a guest thread does there): whether a slow program is waiting
/// on GL round trips or on something else.
fn wait_meter_on() -> bool {
static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ON.get_or_init(|| std::env::var_os("GEX_GL_WAITS").is_some_and(|v| v != "0"))
}
fn wait_meter(waited: std::time::Duration, last_op: u32) {
use std::sync::Mutex;
use std::time::Instant;
#[allow(clippy::type_complexity)]
static M: Mutex<Option<(Instant, u64, f64, f64, std::collections::HashMap<u32, (u64, f64)>)>> = Mutex::new(None);
if !wait_meter_on() {
return;
}
let mut g = M.lock().unwrap();
let m = g.get_or_insert_with(|| (Instant::now(), 0, 0.0, 0.0, Default::default()));
let ms = waited.as_secs_f64() * 1e3;
m.1 += 1;
m.2 += ms;
m.3 = m.3.max(ms);
let e = m.4.entry(last_op).or_default();
e.0 += 1;
e.1 += ms;
let dt = m.0.elapsed().as_secs_f64();
if dt >= 1.0 {
let mut top: Vec<_> = m.4.iter().map(|(o, (n, t))| (*o, *n, *t)).collect();
top.sort_by(|a, b| b.2.total_cmp(&a.2));
let top: Vec<String> = top.iter().take(4).map(|(o, n, t)| format!("op {o} x{n} {t:.0} ms")).collect();
eprintln!(
"[gex-gl] waits: {} submits in {:.2} s, {:.0} ms waited ({:.0}%), max {:.1} ms; by last op: {}",
m.1, dt, m.2, m.2 / (dt * 10.0), m.3, top.join(", ")
);
*g = None;
}
}
/// One batch for the GL thread; `done` takes the result back to a
/// synchronous submit: (reply, status, index of the failed command).
struct Job {
@@ -131,6 +172,7 @@ impl GlDevice {
me: me.clone(),
threaded: std::sync::atomic::AtomicBool::new(false),
memory: Mutex::new(None),
first_op: std::sync::atomic::AtomicU32::new(0),
})
}
@@ -293,7 +335,23 @@ impl GlDevice {
/// Run one batch, reading it out of guest memory and writing the reply back.
fn submit(&self, m: &mut GuestMem<'_>, ring: u64) -> SysResult {
let t0 = std::time::Instant::now();
let r = self.submit_inner(m, ring);
wait_meter(t0.elapsed(), self.first_op.load(std::sync::atomic::Ordering::Relaxed));
r
}
fn submit_inner(&self, m: &mut GuestMem<'_>, ring: u64) -> SysResult {
let (header, batch) = self.ring_batch(m, ring)?;
if wait_meter_on() {
// The last command: the one whose reply the guest waits for.
let mut c = gex_gl::proto::Commands::new(batch);
let mut op = 0;
while let Some(Ok((o, _))) = c.next() {
op = o;
}
self.first_op.store(op, std::sync::atomic::Ordering::Relaxed);
}
let (bytes, status, executed) = if self.threaded() {
// In place when the ring is in the memory the device knows (the
// process's own; a forked child's is copied as before).
@@ -380,6 +438,7 @@ impl FileHandle for GlDevice {
mtime: 0,
nlink: 1,
rdev: 0,
times: Default::default(),
})
}
shims/native/src/lib.rs
+47 -4
View File
@@ -25,6 +25,7 @@ pub mod nt_cgen;
pub mod wasmprog;
pub mod wasmdll;
mod ready;
mod sys_waits;
pub(crate) mod ftab;
pub mod wayland;
@@ -522,12 +523,19 @@ impl std::error::Error for StopExecution {}
/// placed segments rather than the file, because that is what the translator
/// reads.
fn cache_key(image: &Image, xlate: &Options, build: &str) -> String {
// The program file goes in as bytes, not through `Debug`: formatting a
// multi-MiB file as a list of decimal numbers cost more than all the
// rest of an exec (a gexcc busybox: about 12 MB of text, twice, at every
// exec in a guest shell's loop).
let mut plain = xlate.clone();
let elf = plain.image_elf.take();
let mut fields: Vec<Vec<u8>> = vec![
image.bias.to_le_bytes().into(),
image.entry.to_le_bytes().into(),
image.text_range.0.to_le_bytes().into(),
image.text_range.1.to_le_bytes().into(),
format!("{xlate:?}").into_bytes(),
format!("{plain:?}").into_bytes(),
elf.as_deref().map(|e| e.to_vec()).unwrap_or_default(),
// The same bytes mean different instructions in 32-bit mode.
vec![image.class32 as u8],
// Symbols steer discovery, so two images with the same code but
@@ -1953,7 +1961,7 @@ fn fork_process(store: &impl wasmtime::AsContext<Data = HostState>, req: ForkReq
let mut gm = GuestMem::new(dst);
let _ = gex_kernel::syscalls::set_result(&mut gm, ctx_addr, 0);
if req.child_stack != 0 {
let _ = gm.write_u64(ctx_addr + ctx::SP as u64, req.child_stack);
let _ = gm.write_u64(ctx_addr + ctx::sp_slot(child_kernel.compat) as u64, req.child_stack);
}
if let Some(tls) = req.tls {
// PowerPC's thread pointer is r2.
@@ -2409,6 +2417,9 @@ fn mount_filesystems(
let _ = crate::gl::DEVICE.set(Arc::clone(&dev));
devfs.add("gex-gl", dev);
}
// Audio (W3.5): OSS's /dev/dsp and /dev/mixer, a null output, for
// native Wine's wineoss.drv (and any OSS program).
devfs.add_oss();
kernel.vfs.mount("/dev", Arc::new(devfs));
// POSIX shared memory (shm_open) lives in /dev/shm: a guest's own,
// in memory (Chromium's, in a CEF game, refuses to start without it).
@@ -2882,9 +2893,32 @@ impl gex_kernel::net::Transport for HostStream {
sock.peek(&mut scratch).unwrap_or(0)
}
fn peek(&self, buf: &mut [u8]) -> Result<usize, gex_kernel::Errno> {
let guard = self.sock.lock().unwrap_or_else(|e| e.into_inner());
let Some(sock) = guard.as_ref() else { return Err(gex_kernel::Errno::ENOTCONN) };
match sock.peek(buf) {
Ok(n) => Ok(n),
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => Err(gex_kernel::Errno::EAGAIN),
Err(_) => Err(gex_kernel::Errno::ECONNRESET),
}
}
fn close(&self) {
let _ = self.sock.lock().unwrap_or_else(|e| e.into_inner()).take();
}
fn shutdown(&self, how: u64) -> Result<(), gex_kernel::Errno> {
use std::net::Shutdown;
let guard = self.sock.lock().unwrap_or_else(|e| e.into_inner());
let Some(sock) = guard.as_ref() else {
return Err(gex_kernel::Errno::ENOTCONN);
};
let h = match how {
0 => Shutdown::Read,
1 => Shutdown::Write,
_ => Shutdown::Both,
};
sock.shutdown(h).map_err(|e| gex_kernel::Errno::from_io(&e))
}
}
/// The host process's real user and group ids.
@@ -2983,7 +3017,8 @@ fn place_image(
for i in 0..31u8 {
write_u64(data, ctx_addr + ctx::x(i), 0);
}
write_u64(data, ctx_addr + ctx::SP, layout.sp);
// (an i386-ABI process's is `esp` in every build: `ctx::sp_slot`)
write_u64(data, ctx_addr + ctx::sp_slot(program.compat()), layout.sp);
write_u64(data, ctx_addr + ctx::PC, image.entry);
data[ctx_addr + ctx::SAFEPOINT_COUNTER..ctx_addr + ctx::SAFEPOINT_COUNTER + 4]
.copy_from_slice(&SAFEPOINT_INTERVAL.to_le_bytes());
@@ -3111,6 +3146,9 @@ fn register_imports(linker: &mut Linker<HostState>) -> Result<()> {
let nr = gm
.read_u64(state.thread.ctx_addr + ctx::x(8) as u64)
.unwrap_or(!0);
// `GEX_SYS_WAITS`: wall time per (thread, syscall), blocking included.
let sys_t0 = sys_waits::on().then(std::time::Instant::now);
let mut sys_nr = !0u64;
watch_check(&gm, pc as u64, state.thread.tid);
lockstep::log_entry(&gm, state.thread.ctx_addr, state.thread.tid, pc as u64);
if lockstep::fake_syscall(&mut gm, state.thread.ctx_addr) {
@@ -3167,7 +3205,9 @@ fn register_imports(linker: &mut Linker<HostState>) -> Result<()> {
if !gex_kernel::skips_proc_snapshot(nr) {
k.refresh_proc(&proc_state);
}
k.syscall(&mut gm, pc as u64)
let o = k.syscall(&mut gm, pc as u64);
sys_nr = k.syscall_nr;
o
});
// A blocking syscall is run here, with the kernel unlocked, and
// then finished under the lock again.
@@ -3269,6 +3309,9 @@ fn register_imports(linker: &mut Linker<HostState>) -> Result<()> {
let (ctx_addr, tid) = (caller.data().thread.ctx_addr, caller.data().thread.tid);
lockstep::log_result(&GuestMem::new(memory.bytes()), ctx_addr, tid);
}
if let Some(t0) = sys_t0 {
sys_waits::record(caller.data().thread.tid, sys_nr, t0.elapsed());
}
check_i386_high(&caller, "a syscall");
let state = caller.data_mut();
match outcome {
shims/native/src/sys_waits.rs
+51
View File
@@ -0,0 +1,51 @@
//! `GEX_SYS_WAITS=1`: where guest threads spend their wall time in system
//! calls, blocking included -- the half of a slow program that a CPU
//! profile (`GEX_PROF`) cannot see. Every 10 s, on stderr, the (thread,
//! call) pairs that took the most wall time in that window:
//! `[gex-sys] 10.0 s: tid 1 futex x812 7210 ms, tid 1 read x40 1800 ms, ...`.
//! Off unless set; when off, the cost is one load per syscall.
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, Instant};
pub fn on() -> bool {
static ON: OnceLock<bool> = OnceLock::new();
*ON.get_or_init(|| std::env::var_os("GEX_SYS_WAITS").is_some_and(|v| v != "0"))
}
struct Window {
start: Instant,
by: HashMap<(i32, u64), (u64, Duration)>,
}
static W: Mutex<Option<Window>> = Mutex::new(None);
pub fn record(tid: i32, nr: u64, took: Duration) {
let mut g = W.lock().unwrap_or_else(|e| e.into_inner());
let w = g.get_or_insert_with(|| Window {
start: Instant::now(),
by: HashMap::new(),
});
let e = w.by.entry((tid, nr)).or_default();
e.0 += 1;
e.1 += took;
let dt = w.start.elapsed();
if dt >= Duration::from_secs(10) {
let mut top: Vec<_> = w.by.iter().map(|(&(t, n), &(c, d))| (d, t, n, c)).collect();
top.sort_by(|a, b| b.0.cmp(&a.0));
let line: Vec<String> = top
.iter()
.take(12)
.map(|(d, t, n, c)| {
format!(
"tid {t} {} x{c} {} ms",
gex_kernel::syscalls::syscall_name(*n),
d.as_millis()
)
})
.collect();
eprintln!("[gex-sys] {:.1} s: {}", dt.as_secs_f64(), line.join(", "));
*g = None;
}
}
shims/native/src/wasmprog.rs
+1 -1
View File
@@ -97,7 +97,7 @@ pub(crate) fn install(store: &mut Store<HostState>, memory: &gex_mem::GuestMemor
let pairs = [(entry, slot("__gex_start")?), (thread_entry, slot("__gex_thread_start")?)];
let engine = store.engine().clone();
let t0 = std::time::Instant::now();
let module = gex_cache::compile(&engine, &wasm).context("compiling the wasm32 program")?;
let module = gex_cache::compile_cached(&engine, &wasm).context("compiling the wasm32 program")?;
let shared = store
.data()
.jit
shims/native/src/wayland.rs
+12 -1
View File
@@ -28,6 +28,12 @@ struct Session {
/// -- it desyncs the stream and leaves the client waiting forever.
out: Vec<u8>,
sent: usize,
/// Where requests are read into, kept: `pump` runs at every syscall of
/// every thread, and a fresh 64 KiB zeroed buffer each time (calloc's
/// memset, and glibc growing and trimming its heap around it) was most
/// of what a W3 game's threads spent their CPU on. Keeping it took
/// Retrocycles (edger, gameplay) from 3.0-3.5 fps to 38-46 fps.
rbuf: Vec<u8>,
}
/// The compositor, as the guest sees it: a name that accepts connections.
@@ -348,7 +354,10 @@ impl Wayland {
sessions.retain_mut(|s| {
let mut gone = false;
// A big read: a toolkit's startup is a few KB of requests at once.
let mut buf = vec![0u8; 64 * 1024];
let mut buf = std::mem::take(&mut s.rbuf);
if buf.is_empty() {
buf = vec![0u8; 64 * 1024];
}
loop {
let n = match s.socket.recv(&mut buf, 32, false) {
Ok(0) => {
@@ -372,6 +381,7 @@ impl Wayland {
break;
}
}
s.rbuf = buf;
if !due.is_empty() {
eprintln!("[gex] input: {}", s.client.describe_input());
}
@@ -437,6 +447,7 @@ impl HostListener for Wayland {
fds: Vec::new(),
out: Vec::new(),
sent: 0,
rbuf: Vec::new(),
});
}
}
tests/gexcc/alarmloop.c
+41
View File
@@ -0,0 +1,41 @@
/* Signals reach code that never makes a system call (GexPoll's loop
* safepoints): alarm() with a handler stops a busy loop, and so does the
* default action (SIGALRM terminates). */
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>
static volatile sig_atomic_t fired;
static void on_alarm(int s) { (void)s; fired = 1; }
static unsigned long spin(volatile unsigned long *x)
{
unsigned long n = 0;
while (!fired)
n += (*x)++ & 1;
return n;
}
int main(void)
{
volatile unsigned long x = 0;
signal(SIGALRM, on_alarm);
alarm(1);
spin(&x);
printf("handler ran: %d\n", (int)fired);
fflush(stdout);
/* default action: the child dies of SIGALRM in its loop */
pid_t p = fork();
if (p == 0) {
signal(SIGALRM, SIG_DFL);
alarm(1);
for (;;)
x++;
}
int st;
waitpid(p, &st, 0);
printf("child: signaled %d sig %d\n", WIFSIGNALED(st), WIFSIGNALED(st) ? WTERMSIG(st) : 0);
return 0;
}
tests/gexcc/corpus/config-site.sh
Executable
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Regenerate tools/gexcc/config.site: the cache variables whose answers a
# native-mode configure (its checks run under gexc) and a cross-mode one
# (guesses) disagree on, with the native answers.
# config-site.sh NATIVE_TREES_DIR CROSS_TREES_DIR > config.site
# Each dir holds configured trees (*/config.log). Running configure natively
# starts one host gexc per check: do it one tree at a time, inside
# systemd-run --user --scope -p TasksMax=256 nice -n 19
set -eu
ext() { awk '/^## Cache variables. ##/{f=1;next} f && /^[A-Za-z_0-9]+=/ {print; seen=1; next} f && seen && /^## /{exit}' "$@"; }
t=$(mktemp -d)
ext "$1"/*/config.log | sort -t= -k1,1 -u > "$t/n"
ext "$2"/*/config.log | sort -t= -k1,1 -u > "$t/c"
sed -n '1,9p' "$(dirname "$0")/../../../tools/gexcc/config.site"
join -t= "$t/n" "$t/c" | awk -F= '$2 != $3' |
grep -v '^ac_cv_build\|^ac_cv_host\|^ac_cv_env\|^ac_cv_path\|^ac_cv_prog\|^ac_cv_objext\|^am_cv\|^ac_cv_exeext\|^gl_cv_compiler_check_decl_option' |
while IFS= read -r l; do k=${l%%=*}; v=$(grep "^$k=" "$t/n" | head -1); echo "$k=\${$k=${v#*=}}"; done
rm -rf "$t"
tests/gexcc/corpus/ctest-guest.py
Executable
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""A CTest suite as one shell script for the guest (recipes.sh: ctest_guest).
ctest-guest.py BUILD_DIR > script.sh
Reads `ctest --show-only=json-v1` and writes a busybox-sh script that runs
each test in its working directory and prints `PASS: name` or `FAIL: name`,
so the whole suite runs inside one gexc instead of one host gexc per test.
The CMAKE_CROSSCOMPILING_EMULATOR (gexcc-run) prefix is dropped: inside the
guest a gexcc program runs directly. Honoured properties: WORKING_DIRECTORY,
WILL_FAIL, PASS_REGULAR_EXPRESSION, FAIL_REGULAR_EXPRESSION, ENVIRONMENT,
DISABLED, TIMEOUT."""
import json, shlex, subprocess, sys
b = sys.argv[1]
info = json.loads(subprocess.run(['ctest', '--show-only=json-v1'], cwd=b, capture_output=True, text=True, check=True).stdout)
out = ['#!/bin/sh', '# generated by ctest-guest.py', 'run() {']
out.append(' n=$1; shift; wd=$1; shift; mode=$1; shift; pat=$1; shift; fpat=$1; shift')
out.append(' o=$( cd "$wd" && "$@" 2>&1 ); r=$?')
out.append(' ok=0; [ $r -eq 0 ] && ok=1')
out.append(' [ -n "$pat" ] && { printf "%s" "$o" | grep -Eq -- "$pat" && ok=1 || ok=0; }')
out.append(' [ -n "$fpat" ] && printf "%s" "$o" | grep -Eq -- "$fpat" && ok=0')
out.append(' [ "$mode" = willfail ] && ok=$((1 - ok))')
out.append(' if [ $ok = 1 ]; then echo "PASS: $n"; else echo "FAIL: $n"; printf "%s\\n" "$o" | tail -20; fi')
out.append('}')
for t in info.get('tests', []):
cmd = t.get('command')
if not cmd:
continue
if cmd[0].endswith('gexcc-run'):
cmd = cmd[1:]
props = {p['name']: p['value'] for p in t.get('properties', [])}
if props.get('DISABLED'):
out.append('echo %s' % shlex.quote('SKIP: ' + t['name']))
continue
wd = props.get('WORKING_DIRECTORY', b)
mode = 'willfail' if props.get('WILL_FAIL') else 'normal'
pat = '|'.join(props.get('PASS_REGULAR_EXPRESSION', []) or [])
fpat = '|'.join(props.get('FAIL_REGULAR_EXPRESSION', []) or [])
env = ' '.join(shlex.quote(e) for e in props.get('ENVIRONMENT', []) or [])
tmo = int(float(props.get('TIMEOUT', 1500)))
pre = ('env %s ' % env if env else '') + 'timeout %d ' % tmo
out.append('run %s %s %s %s %s %s%s' % (shlex.quote(t['name']), shlex.quote(wd), mode, shlex.quote(pat),
shlex.quote(fpat), pre, ' '.join(shlex.quote(c) for c in cmd)))
print('\n'.join(out))
tests/gexcc/corpus/fetch.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
# Fetch the corpus sources into SRC (default $GEXCC_SRC) and check their
# pinned sha256 (sources.txt). A "-" pin is filled in on first fetch.
# tests/gexcc/corpus/fetch.sh [NAME...]
set -eu
HERE=$(cd "$(dirname "$0")" && pwd)
SRC=${GEXCC_SRC:?set GEXCC_SRC (where tarballs go)}
mkdir -p "$SRC"
grep -v '^#' "$HERE/sources.txt" | while read -r name ver url sum; do
[ -n "$name" ] || continue
if [ $# -gt 0 ]; then case " $* " in *" $name "*) ;; *) continue ;; esac; fi
f=$SRC/$(basename "$url")
if [ ! -s "$f" ]; then
if timeout 600 curl -fsSL --connect-timeout 30 -o "$f.part" "$url"; then mv "$f.part" "$f"
else rm -f "$f.part"; echo "FAIL fetch $name"; continue; fi
fi
got=$(sha256sum "$f" | cut -d' ' -f1)
if [ "$sum" = - ]; then
sed -i "s|^$name $ver $url -\$|$name $ver $url $got|" "$HERE/sources.txt"
echo "pinned $name $got"
elif [ "$sum" != "$got" ]; then
echo "FAIL sha256 $name: $got, want $sum"; exit 1
fi
done
tests/gexcc/corpus/recipes.sh
+357
View File
@@ -0,0 +1,357 @@
# Corpus recipes (run.sh). build_NAME runs in the unpacked tree with `set -e`;
# test_NAME prints `RESULT pass=N fail=N skip=N`. Libraries other programs
# use install into the sysroot (--prefix=/usr, DESTDIR=$SYSROOT).
#
# Test suites run INSIDE one gexc (guest), never as one host gexc per test
# program (2026-10-07: thousands of host gexc took gooner down):
# - configure in cross mode (--build given): no test programs run on the host;
# - the host builds the suite's programs without running them;
# - `guest CMD` runs CMD under one gexc: busybox's sh and tools, the gexcc
# GNU make in $GUEST_BIN; every test process is a guest process.
# suites safe to run: in the guest, or a handful of host gexc at most (lua,
# sqlite, dash, less: one or two). The rest still start a host gexc per test
# and are refused unless HOSTRUN=1 (then one at a time, in a resource scope).
GUEST_SUITES="zlib xz bzip2 jq pcre2 expat libpng coreutils grep sed diffutils findutils libevent lua sqlite dash less gawk tar mbedtls fmt bash"
# autotools, cross mode: configure runs nothing it builds
# (config.site: what the run-time checks found on gex itself)
at_configure() { CONFIG_SITE=$GEXCC_PREFIX/share/gexcc/config.site ./configure --host=$T --build=x86_64-pc-linux-gnu --prefix=/usr "$@"; }
# a shell command line in the guest, in this directory
# With COUNT=1, `GUEST host-execs N`: host programs the guest run started (execve in the
# gexc's process tree, counted by strace; gexc itself is 1), `GUEST
# host-clones N`: threads and processes that tree made (clone, clone3), and
# `GUEST machine-forks N`: processes created on the whole machine meanwhile
# (/proc/stat; includes other users').
guest() {
local p0 p1 r out
out=$(mktemp)
p0=$(awk '/^processes/{print $2}' /proc/stat)
# (strace only with COUNT=1: it slows a long suite down)
${COUNT:+strace -f -qq --seccomp-bpf -e trace=execve,clone,clone3,fork,vfork -e signal=none -c -o "$out"} \
timeout ${TMO:-3600} gexcc-exec run --host-env --env PATH=$GUEST_BIN:/bin:/usr/bin --env SHELL=/bin/sh \
--cwd "$PWD" /bin/sh -c "$*"
r=$?
p1=$(awk '/^processes/{print $2}' /proc/stat)
[ -n "${COUNT:-}" ] && echo "GUEST host-execs $(awk '$NF == "execve" {print $4}' "$out")" >&2
[ -n "${COUNT:-}" ] && echo "GUEST host-clones $(awk '$NF ~ /^(clone|clone3|fork|vfork)$/ {n += $4} END {print n + 0}' "$out")" >&2
echo "GUEST machine-forks $((p1 - p0))" >&2
rm -f "$out"
return $r
}
# automake's test-suite summary (# PASS: n ...) summed over every log
am_result() {
# the per-test lines (a directory whose test-suite.log rule failed prints
# no summary)
awk '/^PASS: /{p++} /^XFAIL: /{p++} /^FAIL: /{f++} /^ERROR: /{f++} /^XPASS: /{f++} /^SKIP: /{s++}
END{printf "RESULT pass=%d fail=%d skip=%d\n", p, f, s}' "$@"
}
# automake: the host builds the suite (TESTS empty runs nothing), the guest runs it
am_check() {
make -j$JOBS check TESTS= XFAIL_TESTS= > check-build.out 2>&1 || { tail -20 check-build.out; echo "RESULT pass=0 fail=1 skip=0"; return; }
guest make -k check SHELL=/bin/sh "$@" > check.out 2>&1; cat check.out; am_result check.out
}
build_zlib() {
CC=gexcc AR=gexcc-ar RANLIB=gexcc-ranlib CHOST=$T ./configure --prefix=/usr
make -j$JOBS
make install DESTDIR=$SYSROOT
}
test_zlib() {
guest make test SHELL=/bin/sh 2>&1 | tee t.out
n=$(grep -c 'test OK' t.out); echo "RESULT pass=$n fail=$((3 - n)) skip=0"
}
build_xz() { at_configure --disable-nls; make -j$JOBS; make install DESTDIR=$SYSROOT; }
test_xz() { am_check; }
# plain Makefile
build_bzip2() {
make -j$JOBS CC=gexcc AR=gexcc-ar RANLIB=gexcc-ranlib bzip2 bzip2recover
}
test_bzip2() {
guest make check SHELL=/bin/sh CC=true 2>&1 | tee t.out
# six round trips (three compress, three decompress), each a cmp
f=$(grep -c 'differ\|EOF on\|Error\|rror' t.out)
echo "RESULT pass=$((6 - f)) fail=$f skip=0"
}
build_lua() {
make -j$JOBS posix CC=gexcc AR="gexcc-ar rcu" RANLIB=gexcc-ranlib MYCFLAGS=-DLUA_USE_POSIX
}
test_lua() {
rm -rf lt && mkdir lt && tar xf "$GEXCC_SRC/lua-5.4.7-tests.tar.gz" -C lt --strip-components=1
cd lt
# musl's setlocale accepts any name (as C.UTF-8), so the suite's
# "pt_BR exists" probe is wrong there: skip the decimal-comma checks
sed -i 's/^if os.setlocale("pt_BR") or os.setlocale("ptb") then/if false then/' literals.lua
# the portable suite (_port: no OS-specific checks); one RESULT for all.lua
if timeout ${TMO:-3600} ../src/lua -e"_U=true _port=true" all.lua > all.out 2>&1; then
tail -5 all.out; echo "RESULT pass=1 fail=0 skip=0"
else
tail -20 all.out; echo "RESULT pass=0 fail=1 skip=0"
fi
}
build_sqlite() { at_configure --disable-shared; make -j$JOBS; make install DESTDIR=$SYSROOT; }
test_sqlite() {
# the autoconf tarball has no test suite: a scripted session against the
# host's sqlite3 when present
cat > t.sql <<'SQL'
create table t(a integer primary key, b text, c real);
insert into t(b,c) values ('x',1.5),('y',2.5),('z',-3);
with recursive n(i) as (select 1 union all select i+1 from n where i<1000) insert into t(b,c) select 'r'||i, i*0.5 from n;
select count(*), sum(c), group_concat(b) filter (where a<4) from t;
create index ti on t(b); select a from t where b='r500';
select printf('%.3f', avg(c)), max(length(b)) from t;
select json_object('a',1,'b',json_array(1,2,3));
pragma integrity_check;
SQL
./sqlite3 db.gex < t.sql > gex.out 2>&1; rm -f db.gex
if command -v sqlite3 >/dev/null; then sqlite3 db.host < t.sql > host.out 2>&1; rm -f db.host; else cp gex.out host.out; fi
cat gex.out
if cmp -s gex.out host.out; then echo "RESULT pass=1 fail=0 skip=0"; else diff host.out gex.out; echo "RESULT pass=0 fail=1 skip=0"; fi
}
build_jq() { at_configure --with-oniguruma=builtin --disable-docs --disable-maintainer-mode; make -j$JOBS; }
# jq's src/version.h depends on .remake-version-h, which only exists after
# the version changed, so every make regenerates it and recompiles main.c;
# the guest has no compiler, so the marker is made, older than version.h
test_jq() {
make -j$JOBS check TESTS= > check-build.out 2>&1 || { tail -20 check-build.out; echo "RESULT pass=0 fail=1 skip=0"; return; }
touch -d 2000-01-01 .remake-version-h
guest make -k check SHELL=/bin/sh > check.out 2>&1; cat check.out; am_result check.out
}
build_pcre2() { at_configure --enable-pcre2-16 --enable-pcre2-32; make -j$JOBS; make install DESTDIR=$SYSROOT; }
test_pcre2() { am_check; }
build_expat() { at_configure --without-docbook --without-examples; make -j$JOBS; make install DESTDIR=$SYSROOT; }
test_expat() { am_check; }
build_libpng() { at_configure; make -j$JOBS; make install DESTDIR=$SYSROOT; }
test_libpng() { am_check; }
# --- core userland (GNU: automake test suites) ---
build_coreutils() { FORCE_UNSAFE_CONFIGURE=1 at_configure --disable-nls --without-selinux --without-libgmp; make -j$JOBS; }
test_coreutils() { am_check; }
build_grep() { at_configure --disable-nls; make -j$JOBS; }
test_grep() { am_check; }
build_sed() { at_configure --disable-nls; make -j$JOBS; }
test_sed() { am_check; }
build_gawk() { at_configure --disable-nls --without-readline; make -j$JOBS; }
test_gawk() {
# gawk's own runner: "ALL TESTS PASSED" or a list of failures
guest make -k check SHELL=/bin/sh > check.out 2>&1; cat check.out
if grep -q 'ALL TESTS PASSED' check.out; then
n=$(ls test/*.ok 2>/dev/null | wc -l); echo "RESULT pass=$n fail=0 skip=0"
else
f=$(sed -n 's/^\([0-9][0-9]*\) TESTS* FAILED.*/\1/p' check.out | tail -1)
n=$(ls test/*.ok 2>/dev/null | wc -l); echo "RESULT pass=$((n - ${f:-0})) fail=${f:-0} skip=0"
fi
}
build_diffutils() { at_configure --disable-nls; make -j$JOBS; }
test_diffutils() { am_check; }
build_findutils() { at_configure --disable-nls; make -j$JOBS; }
test_findutils() { am_check; }
build_tar() { FORCE_UNSAFE_CONFIGURE=1 at_configure --disable-nls; make -j$JOBS; }
test_tar() {
# autotest: "N tests were successful" / "ERROR: N tests were run, M failed"
make -j$JOBS check TESTSUITEFLAGS=--version > check-build.out 2>&1
guest make -k check SHELL=/bin/sh TESTSUITEFLAGS=-j$JOBS > check.out 2>&1; tail -30 check.out
ok=$(grep -o '^[0-9]* tests* were successful' check.out | awk '{print $1}')
if [ -n "$ok" ]; then sk=$(grep -o '[0-9]* tests* were skipped' check.out | awk '{print $1}'); echo "RESULT pass=$ok fail=0 skip=${sk:-0}"
else
l=$(grep -E '^ERROR: [0-9]+ tests* were run' check.out | tail -1)
run=$(echo "$l" | awk '{print $2}'); f=$(echo "$l" | sed -n 's/.*run, *\([0-9]*\) failed.*/\1/p')
sk=$(grep -o '[0-9]* tests* were skipped' check.out | awk '{print $1}')
echo "RESULT pass=$((run - ${f:-0})) fail=${f:-0} skip=${sk:-0}"
fi
}
# GNU make for the guest (the in-guest test runs need it): $GUEST_BIN/make
build_make() { at_configure --disable-nls; make -j$JOBS; mkdir -p "$GUEST_BIN"; cp make "$GUEST_BIN/make"; }
test_make_host() {
timeout ${TMO:-3600} make -k check > check.out 2>&1; tail -20 check.out
l=$(grep -E '[0-9]+ Tests? in [0-9]+ Categor' check.out | tail -1)
# "N Tests in M Categories Complete ... No Failures" or "x Tests Failed"
t=$(echo "$l" | awk '{print $1}')
f=$(grep -oE '[0-9]+ Tests? Failed' check.out | tail -1 | awk '{print $1}')
echo "RESULT pass=$(( ${t:-0} - ${f:-0} )) fail=${f:-0} skip=0"
}
build_bash() { export GEXCC_FPCAST=1; at_configure --without-bash-malloc --disable-nls --with-curses CFLAGS="-O2 -Wno-implicit-function-declaration"; make -j$JOBS; }
test_bash() {
# bash's suite prints diffs; a run-* script that prints nothing passed.
# The loop runs in the guest: every bash it starts is a guest process.
cat > tests/.gexcc-bash.sh <<'SH'
cd tests
for t in run-*; do
[ "$t" = run-all ] && continue
[ "$t" = run-minimal ] && continue
o=$(THIS_SH=$PWD/../bash PATH=$PWD:$PATH BUILD_DIR=$PWD/.. BASH_TSTOUT=$PWD/../tstout.$t timeout 300 $PWD/../bash ./$t 2>&1)
if [ -z "$(printf '%s' "$o" | grep -v '^warning:')" ]; then echo "PASS: $t"; else echo "FAIL: $t"; printf '%s\n' "$o" | head -20; fi
done
SH
guest sh tests/.gexcc-bash.sh > check.out 2>&1; cat check.out; am_result check.out
}
build_dash() { at_configure; make -j$JOBS; }
test_dash() {
# no suite of its own: a script against the host's dash/sh
cat > t.sh <<'SH'
set -e
x=$(echo a b | tr ' ' '\n' | sort -r | head -1); echo "$x"
for i in 1 2 3; do echo $((i * i)); done
f() { local v=$1; echo "f:$v"; return 3; }; f q || echo "rc $?"
case foo.c in *.c) echo C;; *) echo other;; esac
( exit 4 ) || echo "sub $?"
trap 'echo trapped' EXIT
echo ${HOME:+set} ${UNSET_VAR:-dflt} ${#x}
cat <<EOF2 | wc -l
1
2
EOF2
read a b <<EOF3
one two
EOF3
echo "$b $a"
printf '%05d|%-3s|%x\n' 42 ab 255
kill -0 $$ && echo alive
wait
SH
./src/dash t.sh > gex.out 2>&1; s1=$?
sh t.sh > host.out 2>&1; s2=$?
cat gex.out
if cmp -s gex.out host.out && [ $s1 = $s2 ]; then echo "RESULT pass=1 fail=0 skip=0"; else diff host.out gex.out; echo "RESULT pass=0 fail=1 skip=0"; fi
}
build_ncurses() {
at_configure --without-cxx-binding --without-ada --without-manpages --without-tests --enable-widec \
--with-termlib --disable-stripping --without-shared --with-fallbacks=xterm,vt100,linux,screen,xterm-256color \
--with-default-terminfo-dir=/usr/share/terminfo --enable-pc-files --with-pkg-config-libdir=/usr/lib/pkgconfig
make -j$JOBS
make install DESTDIR=$SYSROOT
# the names configure scripts probe for (-ltinfo, -lncurses, -lcurses)
for l in tinfo ncurses curses; do ln -sf lib${l}w.a $SYSROOT/usr/lib/lib$l.a 2>/dev/null || true; done
ln -sf libncursesw.a $SYSROOT/usr/lib/libcurses.a
[ -e $SYSROOT/usr/include/curses.h ] || ln -sf ncursesw/curses.h $SYSROOT/usr/include/curses.h
[ -e $SYSROOT/usr/include/term.h ] || ln -sf ncursesw/term.h $SYSROOT/usr/include/term.h
[ -e $SYSROOT/usr/include/ncurses.h ] || ln -sf ncursesw/ncurses.h $SYSROOT/usr/include/ncurses.h
}
build_less() {
CPPFLAGS="-I$SYSROOT/usr/include/ncursesw" at_configure --with-regex=posix
make -j$JOBS
}
test_less() {
# non-interactive: less as a filter (not a terminal) copies its input
seq 1 2000 > in.txt
./less -f in.txt | cmp - in.txt && ./less -V | head -1 && echo "RESULT pass=1 fail=0 skip=0" || echo "RESULT pass=0 fail=1 skip=0"
}
# --- bigger programs ---
build_mbedtls() {
cmake -S . -B b -G Ninja -DCMAKE_TOOLCHAIN_FILE=$GEXCC_PREFIX/share/gexcc/toolchain.cmake \
-DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=ON -DENABLE_PROGRAMS=ON -DMBEDTLS_FATAL_WARNINGS=OFF
ninja -C b -j$JOBS
cmake --install b
}
test_mbedtls() { ctest_guest b; }
# a CTest suite in the guest (ctest-guest.py writes it as one script)
ctest_guest() {
python3 "$HERE/ctest-guest.py" "$1" > .gexcc-ctest.sh || { echo "RESULT pass=0 fail=1 skip=0"; return; }
guest sh .gexcc-ctest.sh > ct.out 2>&1; grep -E '^(PASS|FAIL|SKIP):' ct.out | tail -40; am_result ct.out
}
# ctest's "N% tests passed, F tests failed out of T"
ctest_result() {
l=$(grep -E 'tests passed, [0-9]+ tests failed out of' "$1" | tail -1)
f=$(echo "$l" | sed -n 's/.* \([0-9]*\) tests failed out of \([0-9]*\).*/\1/p')
t=$(echo "$l" | sed -n 's/.* tests failed out of \([0-9]*\).*/\1/p')
echo "RESULT pass=$(( ${t:-0} - ${f:-0} )) fail=${f:-0} skip=0"
}
build_curl() {
at_configure --with-mbedtls=$SYSROOT/usr --without-libpsl --disable-ldap --without-brotli --without-zstd \
--without-libidn2 --without-nghttp2 --with-zlib=$SYSROOT/usr --disable-shared
make -j$JOBS
}
test_curl() {
# the suite's runner is perl (the host's): test programs and curl run under gexc
timeout ${TMO:-5400} make test-full TFLAGS="-j$JOBS -a -p" > t.out 2>&1
tail -15 t.out
l=$(grep -E '^TESTDONE: [0-9]+ tests out of [0-9]+ reported OK' t.out | tail -1)
ok=$(echo "$l" | awk '{print $2}'); tot=$(echo "$l" | awk '{print $6}')
echo "RESULT pass=${ok:-0} fail=$(( ${tot:-0} - ${ok:-0} )) skip=0"
}
build_libevent() {
at_configure --disable-openssl --disable-shared --disable-samples
make -j$JOBS; make install DESTDIR=$SYSROOT
}
test_libevent() { am_check; }
build_tmux() {
PKG_CONFIG=$T-pkg-config at_configure CPPFLAGS="-I$SYSROOT/usr/include/ncursesw"
make -j$JOBS
}
test_tmux() {
# tmux's regress/ needs a terminal; a detached server, a session, a command, its output
S=$PWD/tsock; rm -f "$S"
./tmux -S "$S" -f /dev/null new-session -d -x 80 -y 24 'printf "hello from tmux\n"; sleep 30'
sleep 2
out=$(./tmux -S "$S" capture-pane -p | head -1)
./tmux -S "$S" kill-server
echo "pane: $out"
[ "$out" = "hello from tmux" ] && echo "RESULT pass=1 fail=0 skip=0" || echo "RESULT pass=0 fail=1 skip=0"
}
build_vim() {
cd src
vim_cv_toupper_broken=no vim_cv_terminfo=yes vim_cv_tgetent=zero vim_cv_getcwd_broken=no \
vim_cv_stat_ignores_slash=no vim_cv_memmove_handles_overlap=yes \
./configure --host=$T --prefix=/usr --with-features=normal --enable-gui=no --without-x \
--with-tlib=ncursesw --disable-nls --disable-libsodium CPPFLAGS="-I$SYSROOT/usr/include/ncursesw"
make -j$JOBS
}
test_vim() {
# the old-style tests (testdir) need a terminal; run vim's script tests in Ex mode
cd src/testdir
timeout ${TMO:-3600} make -k -j1 VIMPROG=../vim test_vim9_expr.res test_functions.res test_listdict.res test_regexp_utf8.res \
test_string.res test_sort.res test_eval_stuff.res test_let.res > ../../t.out 2>&1; cd ../..
tail -20 t.out
p=$(grep -c '^Executed' t.out); f=$(grep -cE '^(Found errors|FAILED)' t.out)
echo "RESULT pass=$p fail=$f skip=0"
}
build_fmt() {
# fmt 11.0.2's color.h uses std::copy without <algorithm>, which libc++ 22
# no longer brings in transitively (fixed upstream later); its tests
# specialize std::is_floating_point, which libc++ 22 rejects
cmake -S . -B b -G Ninja -DCMAKE_TOOLCHAIN_FILE=$GEXCC_PREFIX/share/gexcc/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DFMT_TEST=ON \
-DCMAKE_CXX_FLAGS="-include algorithm -Wno-invalid-specialization"
ninja -C b -j$JOBS
}
test_fmt() { ctest_guest b; }
build_perl() {
# Configure as a native build: its try programs run (under gexc). Library
# and header searches only in the sysroot, so host libraries (gdbm, db)
# are not picked up. Extensions static (no dlopen of side modules).
sh Configure -des -Dcc=gexcc -Dld=gexcc -Dar=gexcc-ar -Dranlib=gexcc-ranlib -Dnm=gexcc-nm -Uusenm \
-Dprefix=/usr -Uusedl -Dlibpth="$SYSROOT/lib $SYSROOT/usr/lib" -Dloclibpth= -Dlocincpth= -Dglibpth= \
-Dusrinc="$SYSROOT/include" -Doptimize=-O2 -Accflags=-D_GNU_SOURCE -Dman1dir=none -Dman3dir=none
make -j$JOBS
}
test_perl() {
TEST_JOBS=$JOBS timeout ${TMO:-7200} make test_harness > t.out 2>&1; tail -40 t.out
files=$(sed -n 's/^Files=\([0-9]*\),.*/\1/p' t.out | tail -1)
f=$(sed -n 's/^Failed \([0-9]*\)\/[0-9]* test programs.*/\1/p' t.out | tail -1)
echo "RESULT pass=$(( ${files:-0} - ${f:-0} )) fail=${f:-0} skip=0"
}
build_python3() {
# a native-style configure (its checks run under gexc); extension modules
# static, as for wasm; libraries from the sysroot only
MODULE_BUILDTYPE=static PKG_CONFIG=$T-pkg-config CC=gexcc CXX=gexc++ AR=gexcc-ar RANLIB=gexcc-ranlib \
./configure --prefix=/usr --disable-shared --without-ensurepip --disable-test-modules \
--with-pkg-config=yes ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no
make -j$JOBS
}
test_python3() {
timeout ${TMO:-7200} ./python -m test -j$JOBS --timeout 600 > t.out 2>&1; tail -60 t.out
ok=$(sed -n 's/^\([0-9]*\) tests* OK\./\1/p' t.out | tail -1)
f=$(sed -n 's/^\([0-9]*\) tests* failed:/\1/p' t.out | tail -1)
e=$(sed -n 's/^\([0-9]*\) tests* failed again:/\1/p' t.out | tail -1)
sk=$(sed -n 's/^\([0-9]*\) tests* skipped:/\1/p' t.out | tail -1)
echo "RESULT pass=${ok:-0} fail=${f:-0} skip=${sk:-0}"
}
tests/gexcc/corpus/run.sh
Executable
+66
View File
@@ -0,0 +1,66 @@
#!/bin/bash
# Build corpus programs with gexcc and run their own test suites (the
# programs run under gexc through their containers' host launcher).
#
# tests/gexcc/corpus/run.sh NAME... (or `all`)
#
# GEXCC_PREFIX: an installed toolchain (tools/gexcc/install.sh)
# GEXCC_SRC: the tarballs (fetch.sh)
# GEXCC_WORK: build trees, logs/, results.txt
# JOBS: make -j (default 6)
# KEEP=1: reuse an unpacked tree (no fresh unpack)
# TESTONLY=1: reuse the built tree, run the tests only
#
# Each recipe (recipes.sh) builds in the unpacked tree and its test step
# prints `RESULT pass=N fail=N skip=N`; results.txt keeps one line per
# program: name version build tests.
set -u
HERE=$(cd "$(dirname "$0")" && pwd)
: "${GEXCC_PREFIX:?set GEXCC_PREFIX}" "${GEXCC_SRC:?set GEXCC_SRC}" "${GEXCC_WORK:?set GEXCC_WORK}"
export PATH=$GEXCC_PREFIX/bin:$PATH
export SYSROOT=$GEXCC_PREFIX/sysroot T=wasm32-unknown-linux-musl JOBS=${JOBS:-2}
export GUEST_BIN=$GEXCC_WORK/guest-bin
mkdir -p "$GEXCC_WORK/logs"
. "$HERE/recipes.sh"
names=("$@")
[ "${names[0]:-}" = all ] && names=($(grep -v '^#' "$HERE/sources.txt" | awk '{print $1}' | grep -v -- '-tests$'))
for name in "${names[@]}"; do
line=$(grep "^$name " "$HERE/sources.txt") || { echo "no source for $name"; continue; }
read -r _ ver url _ <<< "$line"
tarball=$GEXCC_SRC/$(basename "$url")
dir=$GEXCC_WORK/$name
if [ -z "${KEEP:-}${TESTONLY:-}" ] || [ ! -d "$dir" ]; then
rm -rf "$dir"; mkdir -p "$dir"
case $tarball in
*.zip) (cd "$dir" && unzip -q "$tarball" && d=$(ls) && mv "$d"/* "$d"/.[!.]* . 2>/dev/null; rmdir "$d") ;;
*) tar xf "$tarball" -C "$dir" --strip-components=1 ;;
esac || { echo "FAIL unpack $name"; continue; }
fi
log=$GEXCC_WORK/logs/$name
build=FAIL tests=-
if [ -n "${TESTONLY:-}" ] || (cd "$dir" && set -e && "build_$name") > "$log.build.log" 2>&1; then
build=ok
guest_suite=no; case " $GUEST_SUITES " in *" $name "*) guest_suite=yes ;; esac
if declare -F "test_$name" > /dev/null && [ $guest_suite = no ] && [ -z "${HOSTRUN:-}" ]; then
tests="not run (host-run suite; HOSTRUN=1)"
elif declare -F "test_$name" > /dev/null; then
[ -x "$GUEST_BIN/make" ] || [ -n "${HOSTRUN:-}" ] || { echo "no $GUEST_BIN/make: run.sh make first"; exit 2; }
(cd "$dir" && "test_$name") > "$log.test.log" 2>&1
hx=$(awk '/^GUEST host-execs/{n+=$3} END{print n+0}' "$log.test.log")
hf=$(awk '/^GUEST machine-forks/{n+=$3} END{print n+0}' "$log.test.log")
hc=$(awk '/^GUEST host-clones/{n+=$3} END{print n+0}' "$log.test.log")
r=$(grep -a '^RESULT ' "$log.test.log" | tail -1)
if [ -n "$r" ]; then
p=$(sed -n 's/.*pass=\([0-9]*\).*/\1/p' <<< "$r"); f=$(sed -n 's/.*fail=\([0-9]*\).*/\1/p' <<< "$r")
s=$(sed -n 's/.*skip=\([0-9]*\).*/\1/p' <<< "$r")
tests="$p/$((p + f)) pass, $s skip, ${COUNT:+host execs $hx, host clones $hc, }machine forks $hf"
else
tests="no result"
fi
fi
fi
echo "$name $ver build=$build tests=$tests"
touch "$GEXCC_WORK/results.txt"
sed -i "/^$name /d" "$GEXCC_WORK/results.txt"
echo "$name $ver build=$build tests=$tests" >> "$GEXCC_WORK/results.txt"
done
tests/gexcc/corpus/sources.txt
+35
View File
@@ -0,0 +1,35 @@
# name version url sha256 (pinned: fetch.sh checks it; "-" records it on first fetch)
zlib 1.3.1 https://zlib.net/fossils/zlib-1.3.1.tar.gz 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
xz 5.6.3 https://github.com/tukaani-project/xz/releases/download/v5.6.3/xz-5.6.3.tar.gz b1d45295d3f71f25a4c9101bd7c8d16cb56348bbef3bbc738da0351e17c73317
bzip2 1.0.8 https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
lua 5.4.7 https://www.lua.org/ftp/lua-5.4.7.tar.gz 9fbf5e28ef86c69858f6d3d34eccc32e911c1a28b4120ff3e84aaa70cfbf1e30
lua-tests 5.4.7 https://www.lua.org/tests/lua-5.4.7-tests.tar.gz 8a4898ffe4c7613c8009327a0722db7a41ef861d526c77c5b46114e59ebf811e
sqlite 3.46.1 https://www.sqlite.org/2024/sqlite-autoconf-3460100.tar.gz 67d3fe6d268e6eaddcae3727fce58fcc8e9c53869bdd07a0c61e38ddf2965071
jq 1.7.1 https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-1.7.1.tar.gz 478c9ca129fd2e3443fe27314b455e211e0d8c60bc8ff7df703873deeee580c2
pcre2 10.44 https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.44/pcre2-10.44.tar.bz2 d34f02e113cf7193a1ebf2770d3ac527088d485d4e047ed10e5d217c6ef5de96
expat 2.6.4 https://github.com/libexpat/libexpat/releases/download/R_2_6_4/expat-2.6.4.tar.xz a695629dae047055b37d50a0ff4776d1d45d0a4c842cf4ccee158441f55ff7ee
libpng 1.6.44 https://download.sourceforge.net/libpng/libpng-1.6.44.tar.xz 60c4da1d5b7f0aa8d158da48e8f8afa9773c1c8baa5d21974df61f1886b8ce8e
coreutils 9.5 https://mirrors.kernel.org/gnu/coreutils/coreutils-9.5.tar.xz cd328edeac92f6a665de9f323c93b712af1858bc2e0d88f3f7100469470a1b8a
grep 3.11 https://mirrors.kernel.org/gnu/grep/grep-3.11.tar.xz 1db2aedde89d0dea42b16d9528f894c8d15dae4e190b59aecc78f5a951276eab
sed 4.9 https://mirrors.kernel.org/gnu/sed/sed-4.9.tar.xz 6e226b732e1cd739464ad6862bd1a1aba42d7982922da7a53519631d24975181
gawk 5.3.1 https://mirrors.kernel.org/gnu/gawk/gawk-5.3.1.tar.xz 694db764812a6236423d4ff40ceb7b6c4c441301b72ad502bb5c27e00cd56f78
diffutils 3.10 https://mirrors.kernel.org/gnu/diffutils/diffutils-3.10.tar.xz 90e5e93cc724e4ebe12ede80df1634063c7a855692685919bfe60b556c9bd09e
findutils 4.10.0 https://mirrors.kernel.org/gnu/findutils/findutils-4.10.0.tar.xz 1387e0b67ff247d2abde998f90dfbf70c1491391a59ddfecb8ae698789f0a4f5
tar 1.35 https://mirrors.kernel.org/gnu/tar/tar-1.35.tar.xz 4d62ff37342ec7aed748535323930c7cf94acf71c3591882b26a7ea50f3edc16
make 4.4.1 https://mirrors.kernel.org/gnu/make/make-4.4.1.tar.gz dd16fb1d67bfab79a72f5e8390735c49e3e8e70b4945a15ab1f81ddb78658fb3
bash 5.2.37 https://mirrors.kernel.org/gnu/bash/bash-5.2.37.tar.gz 9599b22ecd1d5787ad7d3b7bf0c59f312b3396d1e281175dd1f8a4014da621ff
dash 0.5.12 http://gondor.apana.org.au/~herbert/dash/files/dash-0.5.12.tar.gz 6a474ac46e8b0b32916c4c60df694c82058d3297d8b385b74508030ca4a8f28a
ncurses 6.5 https://mirrors.kernel.org/gnu/ncurses/ncurses-6.5.tar.gz 136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6
less 668 https://www.greenwoodsoftware.com/less/less-668.tar.gz 2819f55564d86d542abbecafd82ff61e819a3eec967faa36cd3e68f1596a44b8
perl 5.40.0 https://www.cpan.org/src/5.0/perl-5.40.0.tar.gz c740348f357396327a9795d3e8323bafd0fe8a5c7835fc1cbaba0cc8dfe7161f
python3 3.12.7 https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tar.xz 24887b92e2afd4a2ac602419ad4b596372f67ac9b077190f459aba390faf5550
git 2.47.0 https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.47.0.tar.xz 1ce114da88704271b43e027c51e04d9399f8c88e9ef7542dae7aebae7d87bc4e
mbedtls 3.6.2 https://github.com/Mbed-TLS/mbedtls/releases/download/mbedtls-3.6.2/mbedtls-3.6.2.tar.bz2 8b54fb9bcf4d5a7078028e0520acddefb7900b3e66fec7f7175bb5b7d85ccdca
curl 8.10.1 https://curl.se/download/curl-8.10.1.tar.xz 73a4b0e99596a09fa5924a4fb7e4b995a85fda0d18a2c02ab9cf134bebce04ee
openssh 9.9p1 https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-9.9p1.tar.gz b343fbcdbff87f15b1986e6e15d6d4fc9a7d36066be6b7fb507087ba8f966c02
vim 9.1.0800 https://github.com/vim/vim/archive/refs/tags/v9.1.0800.tar.gz 3bc15301f35addac9acde1da64da0976dbeafe1264e904c25a3cdc831e347303
libevent 2.1.12 https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz 92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb
tmux 3.5a https://github.com/tmux/tmux/releases/download/3.5a/tmux-3.5a.tar.gz 16216bd0877170dfcc64157085ba9013610b12b082548c7c9542cc0103198951
fmt 11.0.2 https://github.com/fmtlib/fmt/releases/download/11.0.2/fmt-11.0.2.zip 40fc58bebcf38c759e11a7bd8fdc163507d2423ef5058bba7f26280c5b9c5465
nginx 1.26.2 https://nginx.org/download/nginx-1.26.2.tar.gz 627fe086209bba80a2853a0add9d958d7ebbdffa1a8467a5784c9a6b4f03d738
redis 7.4.1 https://download.redis.io/releases/redis-7.4.1.tar.gz bc34b878eb89421bbfca6fa78752343bf37af312a09eb0fae47c9575977dfaa2
tests/gexcc/cxx.cpp
+43
View File
@@ -0,0 +1,43 @@
// C++ on gexcc: exceptions (through frames, rethrow, std types), RTTI,
// iostreams, containers, std::thread, static constructors.
#include <iostream>
#include <map>
#include <memory>
#include <mutex>
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <typeinfo>
#include <vector>
struct Base { virtual ~Base() = default; virtual std::string name() const { return "base"; } };
struct Derived : Base { std::string name() const override { return "derived"; } };
struct Noisy { std::string n; explicit Noisy(std::string s) : n(std::move(s)) {} ~Noisy() { std::cout << "~" << n << "\n"; } };
static std::vector<int> ctor_order;
struct Init { Init() { ctor_order.push_back(1); } } init_obj;
static int depth(int n) {
Noisy guard("frame" + std::to_string(n));
if (n == 0) throw std::runtime_error("bottom");
return depth(n - 1) + 1;
}
int main() {
try { depth(3); } catch (const std::exception &e) { std::cout << "caught " << e.what() << "\n"; }
try { try { throw 42; } catch (int v) { std::cout << "int " << v << "\n"; throw; } }
catch (...) { std::cout << "rethrown\n"; }
try { std::vector<int> v; v.at(5); } catch (const std::out_of_range &) { std::cout << "out_of_range\n"; }
std::unique_ptr<Base> b = std::make_unique<Derived>();
std::cout << b->name() << " " << (dynamic_cast<Derived *>(b.get()) != nullptr) << " " << (typeid(*b) == typeid(Derived)) << "\n";
std::map<std::string, int> m{{"a", 1}, {"b", 2}};
std::ostringstream os; for (auto &[k, v] : m) os << k << "=" << v << ";";
std::cout << os.str() << " ctors " << ctor_order.size() << "\n";
std::mutex mu; long total = 0; std::vector<std::thread> ts;
for (int i = 0; i < 4; i++) ts.emplace_back([&, i] { for (int j = 0; j < 1000; j++) { std::lock_guard<std::mutex> l(mu); total += i; } });
for (auto &t : ts) t.join();
std::cout << "threads " << total << "\n";
try { std::thread([] {}).join(); throw std::logic_error("after thread"); } catch (const std::logic_error &e) { std::cout << e.what() << "\n"; }
return 0;
}
tests/gexcc/emptypath.c
+33
View File
@@ -0,0 +1,33 @@
/* An empty path names nothing: every path call fails with ENOENT. The
* AT_EMPTY_PATH forms take the descriptor instead. */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
static void show(const char *what, int r)
{
printf("%s: %d %s\n", what, r < 0 ? -1 : 0, r < 0 ? strerror(errno) : "");
errno = 0;
}
int main(void)
{
struct stat st;
FILE *f;
show("open", open("", O_RDONLY));
show("stat", stat("", &st));
show("lstat", lstat("", &st));
show("access", access("", F_OK));
show("fstatat", fstatat(AT_FDCWD, "", &st, 0));
show("fstatat AT_EMPTY_PATH", fstatat(0, "", &st, AT_EMPTY_PATH));
show("chdir", chdir(""));
show("mkdir", mkdir("", 0755));
show("unlink", unlink(""));
f = fopen("", "r");
printf("fopen: %s\n", f ? "opened" : strerror(errno));
return 0;
}
tests/gexcc/fdsemantics.c
+44
View File
@@ -0,0 +1,44 @@
/* Descriptor and open semantics gnulib's tests check (grep, sed, coreutils
* all carry them): RLIMIT_NOFILE bounds dup2/F_DUPFD, bad descriptors are
* EBADF, O_CREAT works read-only, a directory does not open for writing,
* and *at() relative to a non-directory is ENOTDIR. */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <unistd.h>
#define T(e) printf("%-44s %s\n", #e, (e) ? "ok" : strerror(errno))
int main(void)
{
struct rlimit rl;
int v = 1;
T(getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_cur > 2 && rl.rlim_cur <= rl.rlim_max);
int lim = (int)rl.rlim_cur;
T(getdtablesize() == lim);
T(dup2(0, lim) == -1 && errno == EBADF);
T(fcntl(0, F_DUPFD, lim) == -1 && errno == EINVAL);
T(fcntl(0, F_DUPFD_CLOEXEC, -1) == -1 && errno == EINVAL);
T(fcntl(-1, F_GETFD) == -1 && errno == EBADF);
T(setsockopt(-1, SOL_SOCKET, SO_REUSEADDR, &v, sizeof v) == -1 && errno == EBADF);
rl.rlim_cur = 64;
T(setrlimit(RLIMIT_NOFILE, &rl) == 0);
T(dup2(0, 64) == -1 && errno == EBADF);
T(dup2(0, 63) == 63);
rl.rlim_cur = rl.rlim_max + 1;
T(setrlimit(RLIMIT_NOFILE, &rl) == -1 && errno == EINVAL);
T(getrlimit(RLIMIT_STACK, &rl) == 0 && rl.rlim_cur == 8 << 20);
unlink("fdsem.tmp");
int fd = open("fdsem.tmp", O_CREAT | O_RDONLY, 0600);
T(fd >= 0);
T(open("fdsem.tmp", O_CREAT | O_EXCL | O_RDONLY, 0600) == -1 && errno == EEXIST);
T(open(".", O_WRONLY) == -1 && errno == EISDIR);
T(openat(fd, ".", O_RDONLY) == -1 && errno == ENOTDIR);
T(fopen(".", "w") == NULL);
unlink("fdsem.tmp");
return 0;
}
tests/gexcc/fifononblock.c
+19
View File
@@ -0,0 +1,19 @@
/* A host FIFO with no writer, opened O_NONBLOCK: the open returns at once
* (grep -D skip opens devices so, to fstat and skip them). */
#include <fcntl.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
int main(int argc, char **argv)
{
struct stat st;
int fd = open(argc > 1 ? argv[1] : "fifo", O_RDONLY | O_NONBLOCK);
if (fd < 0 || fstat(fd, &st) < 0) {
perror("open");
return 1;
}
char b[8];
printf("fifo %d, read %ld\n", S_ISFIFO(st.st_mode), (long)read(fd, b, sizeof b));
return 0;
}
tests/gexcc/fifononblock.sh
+3
View File
@@ -0,0 +1,3 @@
rm -f fifononblock.p; mkfifo fifononblock.p
timeout 10 "$1" fifononblock.p; echo "status $?"
rm -f fifononblock.p
tests/gexcc/fpcast.c
+25
View File
@@ -0,0 +1,25 @@
/* A call through a function pointer of another type (old C: callbacks
* declared as int (*)() and called with arguments, cleanups returning int
* stored as void (*)(void *)). wasm traps on it; with -fgex-fpcast-emu
* (GEXCC_FPCAST=1) gexcc links it with binaryen's cast emulation and it
* behaves as on x86. run.sh builds files whose name starts with fpcast this way. */
#include <stdio.h>
static int counted;
static int returns_int(void *p) { counted += *(int *)p; return 42; }
static void takes_two(int a, int b) { counted += a + b; }
typedef void (*cleanup_t)(void *);
typedef void (*thunk_t)(int);
int main(void)
{
int v = 5;
cleanup_t c = (cleanup_t)returns_int;
c(&v);
thunk_t t = (thunk_t)takes_two;
counted = 5;
t(3);
printf("survived, first call counted %d\n", counted != 0);
return 0;
}
tests/gexcc/fscorners.c
+41
View File
@@ -0,0 +1,41 @@
/* More file corners: O_NOFOLLOW on a symlink is ELOOP; chown of a missing
* file fails (ownership is not kept, but the file must exist); nanosleep
* rejects bad times; getgroups' size check; sigaltstack. */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#define T(e) printf("%-56s %s\n", #e, (e) ? "ok" : strerror(errno))
int main(void)
{
static char stk[65536];
gid_t g[256];
unlink("fc.l");
T(symlink("fscorners.tmp", "fc.l") == 0);
close(open("fscorners.tmp", O_CREAT | O_WRONLY, 0644));
T(open("fc.l", O_RDONLY | O_NOFOLLOW) == -1 && errno == ELOOP);
T(open("fc.l", O_RDONLY) >= 0);
T(chown("fc.missing", getuid(), getgid()) == -1 && errno == ENOENT);
T(lchown("fc.l", getuid(), getgid()) == 0);
T(fchownat(AT_FDCWD, "", getuid(), getgid(), 0) == -1 && errno == ENOENT);
struct timespec ts = {-1, 0};
T(nanosleep(&ts, 0) == -1 && errno == EINVAL);
ts.tv_sec = 0;
ts.tv_nsec = 1000000000;
T(nanosleep(&ts, 0) == -1 && errno == EINVAL);
int n = getgroups(0, 0);
T(n >= 0 && getgroups(256, g) == n);
stack_t ss = {0};
ss.ss_sp = stk;
ss.ss_size = sizeof stk;
T(sigaltstack(&ss, 0) == 0);
unlink("fc.l");
unlink("fscorners.tmp");
return 0;
}
tests/gexcc/longargs.c
+43
View File
@@ -0,0 +1,43 @@
/* execve with long arguments: a 20 KiB argument and 2000 arguments reach
* the new program intact (a make recipe of several KiB passed to sh -c,
* automake's test-suite.log rule); one past MAX_ARG_STRLEN is E2BIG. */
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/wait.h>
#include <unistd.h>
int main(int argc, char **argv)
{
if (argc > 1 && strcmp(argv[1], "child") == 0) {
size_t total = 0;
for (int i = 2; i < argc; i++)
total += strlen(argv[i]);
printf("child: %d args, %zu bytes\n", argc, total);
return 0;
}
static char big[20000], huge[140000];
memset(big, 'x', sizeof big - 1);
memset(huge, 'y', sizeof huge - 1);
char *v1[] = {argv[0], "child", big, 0};
char **v2 = calloc(2003, sizeof *v2);
v2[0] = argv[0];
v2[1] = "child";
for (int i = 2; i < 2002; i++)
v2[i] = "abcdefgh";
char *v3[] = {argv[0], "child", huge, 0};
char **vs[] = {v1, v2};
for (int k = 0; k < 2; k++) {
fflush(stdout);
pid_t p = fork();
if (p == 0) {
execv("/proc/self/exe", vs[k]);
_exit(127);
}
waitpid(p, 0, 0);
}
execv("/proc/self/exe", v3);
printf("huge: %s\n", errno == E2BIG ? "E2BIG" : strerror(errno));
return 0;
}
tests/gexcc/mmapreuse.c
+21
View File
@@ -0,0 +1,21 @@
/* Big buffers mapped and freed in turn: each one reuses the hole the last
* left (expat's test_buffer_can_grow_to_max, libpng's pngvalid). */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(void)
{
for (int i = 0; i < 8; i++) {
char *p = malloc(1u << 30);
char *small = malloc(64 << 10); /* kept: lands below the big one */
printf("%d: big %s small %s\n", i, p ? "ok" : "NULL", small ? "ok" : "NULL");
if (p)
memset(p + (1u << 29), 1, 4096);
free(p);
}
char *a = malloc(10);
char *b = realloc(a, (1u << 30) + 5);
printf("realloc to 1 GiB: %s\n", b ? "ok" : "NULL");
return 0;
}
tests/gexcc/pathcorners.c
+44
View File
@@ -0,0 +1,44 @@
/* Path corners gnulib checks: a trailing slash on a non-directory is
* ENOTDIR, and rename to or from "dir/." or "dir/.." is refused. */
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#define T(e) printf("%-48s %s\n", #e, (e) ? "ok" : strerror(errno))
int main(void)
{
struct stat st;
system("rm -rf pc.d pc.f pc.l pc.dangling pc.loop; mkdir pc.d; touch pc.f");
T(stat("pc.f/", &st) == -1 && errno == ENOTDIR);
T(open("pc.f/", O_RDONLY) == -1 && errno == ENOTDIR);
T(stat("pc.d/", &st) == 0);
T(symlink("pc.d", "pc.l") == 0);
T(lstat("pc.l", &st) == 0 && S_ISLNK(st.st_mode));
T(lstat("pc.l/", &st) == 0 && S_ISDIR(st.st_mode));
T(symlink("pc.nosuch", "pc.dangling") == 0);
T(lstat("pc.dangling/", &st) == -1 && errno == ENOENT);
T(symlink("pc.loop", "pc.loop") == 0);
T(lstat("pc.loop/", &st) == -1 && errno == ELOOP);
T(rename("pc.f", "pc.d/.") == -1);
T(rename("pc.d/..", "pc.x") == -1);
T(realpath("pc.f/", NULL) == NULL && errno == ENOTDIR);
T(mkdir("pc.d/.", 0755) == -1 && errno == EEXIST);
T(mkdir("pc.none/.", 0755) == -1 && errno == ENOENT);
T(mkdir("pc.dangling/", 0755) == -1 && errno == EEXIST);
T(rename("pc.d", "pc.dangling/") == -1 && errno == ENOTDIR);
T(rmdir("pc.d/.") == -1 && errno == EINVAL);
T(link("pc.f", "pc.d/.") == -1 && errno == EEXIST);
T(link("pc.f", "pc.new/") == -1 && errno == ENOENT);
T(symlink("nowhere", "pc.new/") == -1 && errno == ENOENT);
T(rename("pc.f", "pc.new/") == -1 && errno == ENOTDIR);
T(open("pc.new/", O_CREAT | O_WRONLY, 0644) == -1 && errno == EISDIR);
T(access("pc.new", F_OK) == -1);
T(rename("pc.f", "pc.d/f") == 0);
system("rm -rf pc.d pc.f pc.l pc.dangling pc.loop");
return 0;
}
tests/gexcc/run-browser.sh
Executable
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
# tests/gexcc programs in the browser runtime (shims/browser/js/run-node.mjs,
# the page's shim and glue under Node), against their native runs, as
# tests/wasm32/run-browser.sh does for stream W's. Programs with a NAME.sh
# driver (host FIFOs, host stdin) are native-only. BUSYBOX: a gexcc busybox
# for the guest's /bin/sh (system() in pathcorners), else that one is skipped.
#
# tests/gexcc/run-browser.sh (gexcc on PATH; web/public/gex has the x86
# shared shim and seed.x86.shared.wasm)
set -u
HERE=$(cd "$(dirname "$0")" && pwd)
ROOT=$(cd "$HERE/../.." && pwd)
# GEX_ARCH=aarch64: the AArch64 build's shim and seed (native wasm32
# programs run on both guests' builds); default x86
ARCH=${GEX_ARCH:-x86}
case $ARCH in x86) INFIX=.x86 ;; *) INFIX= ;; esac
SEED=$ROOT/web/public/gex/seed$INFIX.shared.wasm
[ -f "$SEED" ] || { echo "no $SEED: GEX_ARCH=x86 GEX_SHARED=1 shims/browser/build.sh, gexc seed -o $SEED" >&2; exit 2; }
WORK=${WORK:-$(mktemp -d)}
fail=0
run() {
GEX_ARCH=$ARCH GEX_SHARED=1 timeout "${TMO:-300}" node "$ROOT/shims/browser/js/run-node.mjs" "$@" 2>/dev/null | grep -v '^\[host\]\|^\[gex\] \|^\[worker\] \|^\[gex-console'
}
for src in "$HERE"/*.c "$HERE"/*.cpp; do
case $src in *.cpp) name=$(basename "$src" .cpp); hcc=c++; gcc_=gexc++ ;; *) name=$(basename "$src" .c); hcc=cc; gcc_=gexcc ;; esac
[ -f "$HERE/$name.sh" ] && { echo "skip $name (native only)"; continue; }
bb=
if grep -q 'system(' "$src"; then
[ -n "${BUSYBOX:-}" ] || { echo "skip $name (set BUSYBOX)"; continue; }
bb="--busybox $BUSYBOX"
fi
$hcc -O1 -w -o "$WORK/$name.native" "$src" -lpthread || { fail=1; continue; }
$gcc_ -O2 -w -o "$WORK/$name" "$src" || { fail=1; continue; }
native=$(cd "$WORK" && "./$name.native" 2>&1); ns=$?
web=$(run $bb --file "/t/$name=$WORK/$name" --cwd /t --argv0 "./$name" "$SEED" "$WORK/$name");
ws=$(run $bb --file "/t/$name=$WORK/$name" --cwd /t --argv0 "./$name" "$SEED" "$WORK/$name" >/dev/null; echo $?)
if [ "$native" = "$web" ] && [ "$ns" = "$ws" ]; then
echo "ok $name (browser)"
else
echo "FAIL $name (browser): native exit $ns, browser exit $ws"
printf '%s\n' "$native" > "$WORK/$name.out-native"; printf '%s\n' "$web" > "$WORK/$name.out-web"
diff "$WORK/$name.out-native" "$WORK/$name.out-web" | head -20
fail=1
fi
done
exit $fail
tests/gexcc/run.sh
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
# gexcc programs checked against the host: each NAME.c (NAME.cpp) is built
# with cc (c++) and with gexcc (gexc++), run (directly, or by NAME.sh given the binary), and the output
# and exit status must match. The gexcc build runs itself under gexc (the
# container's host launcher).
#
# tests/gexcc/run.sh (gexcc on PATH: tools/gexcc/install.sh)
# GEXCC_RUNNER=GEXC tests/gexcc/run.sh to pick the gexc
set -u
HERE=$(cd "$(dirname "$0")" && pwd)
WORK=${WORK:-$(mktemp -d)}
command -v gexcc > /dev/null || { echo "gexcc not on PATH (tools/gexcc/install.sh)"; exit 2; }
fail=0
for src in "$HERE"/*.c "$HERE"/*.cpp; do
case $src in
*.cpp) name=$(basename "$src" .cpp); hcc=c++; gcc_=gexc++ ;;
*) name=$(basename "$src" .c); hcc=cc; gcc_=gexcc ;;
esac
$hcc -O1 -w -o "$WORK/$name.native" "$src" -lpthread || { echo "FAIL $name: $hcc"; fail=1; continue; }
extra=; case $name in fpcast*) extra=-fgex-fpcast-emu ;; esac
$gcc_ -O2 -w $extra -o "$WORK/$name" "$src" || { echo "FAIL $name: $gcc_"; fail=1; continue; }
if [ -f "$HERE/$name.sh" ]; then
native=$(cd "$WORK" && sh "$HERE/$name.sh" "./$name.native" 2>&1); ns=$?
gex=$(cd "$WORK" && sh "$HERE/$name.sh" "./$name" 2>&1); gs=$?
else
native=$(cd "$WORK" && "./$name.native" 2>&1); ns=$?
gex=$(cd "$WORK" && "./$name" 2>&1); gs=$?
fi
if [ "$native" = "$gex" ] && [ $ns = $gs ]; then
echo "ok $name"
else
echo "FAIL $name: native exit $ns, gex exit $gs"
printf '%s\n' "$native" > "$WORK/$name.out-native"
printf '%s\n' "$gex" > "$WORK/$name.out-gex"
diff "$WORK/$name.out-native" "$WORK/$name.out-gex" | head -20
fail=1
fi
done
exit $fail
tests/gexcc/selectcorners.c
+35
View File
@@ -0,0 +1,35 @@
/* select corners (gnulib test-select): a pipe's write end is ready for
* writing only, not reading; a closed descriptor in any set is EBADF;
* negative nfds is EINVAL. */
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <sys/select.h>
#include <unistd.h>
int main(void)
{
int p[2];
fd_set r, w, x;
struct timeval tv = {0, 0};
pipe(p);
FD_ZERO(&r); FD_ZERO(&w); FD_ZERO(&x);
FD_SET(p[1], &r); FD_SET(p[1], &w); FD_SET(p[1], &x);
int n = select(p[1] + 1, &r, &w, &x, &tv);
printf("write end: %d ready, read %d write %d\n", n, FD_ISSET(p[1], &r) != 0, FD_ISSET(p[1], &w) != 0);
int bad = dup(0);
close(bad);
const char *names[] = {"rfds", "wfds", "xfds"};
for (int k = 0; k < 3; k++) {
FD_ZERO(&r); FD_ZERO(&w); FD_ZERO(&x);
FD_SET(bad, k == 0 ? &r : k == 1 ? &w : &x);
tv.tv_sec = 0;
errno = 0;
n = select(bad + 1, &r, &w, &x, &tv);
printf("closed fd in %s: %d %s\n", names[k], n, n < 0 ? strerror(errno) : "");
}
errno = 0;
n = select(-1, 0, 0, 0, &tv);
printf("nfds -1: %d %s\n", n, n < 0 ? strerror(errno) : "");
return 0;
}
tests/gexcc/sigjmp.c
+46
View File
@@ -0,0 +1,46 @@
/* sigsetjmp/siglongjmp (bash's error recovery): out of nested calls and
* out of a signal handler, with the mask restored when asked; _setjmp. */
#include <setjmp.h>
#include <signal.h>
#include <stdio.h>
#include <unistd.h>
static sigjmp_buf top;
static jmp_buf plain;
static void deep(int n)
{
if (n == 0)
siglongjmp(top, 7);
deep(n - 1);
}
static void on_usr1(int s)
{
(void)s;
siglongjmp(top, 9);
}
int main(void)
{
volatile int round = 0;
sigset_t cur;
int r = sigsetjmp(top, 1);
printf("sigsetjmp -> %d\n", r);
if (round == 0) {
round = 1;
deep(5);
}
if (round == 1) {
round = 2;
signal(SIGUSR1, on_usr1);
kill(getpid(), SIGUSR1);
printf("not reached\n");
}
sigprocmask(SIG_BLOCK, 0, &cur);
printf("SIGUSR1 blocked after the handler's jump: %d\n", sigismember(&cur, SIGUSR1));
if (_setjmp(plain) == 0)
_longjmp(plain, 3);
printf("_setjmp done\n");
return 0;
}
tests/gexcc/slots/hold.c
+31
View File
@@ -0,0 +1,31 @@
/* Records when it ran (CLOCK_REALTIME, ns) to the log named in argv[1],
* holding for argv[2] milliseconds: tests/gexcc/slots/run.sh counts how
* many overlapped. */
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
static long long now(void)
{
struct timespec t;
clock_gettime(CLOCK_REALTIME, &t);
return (long long)t.tv_sec * 1000000000 + t.tv_nsec;
}
int main(int argc, char **argv)
{
long long a = now();
long ms = atol(argv[2]);
struct timespec d = {ms / 1000, (ms % 1000) * 1000000L};
nanosleep(&d, 0);
long long b = now();
char line[64];
int n = snprintf(line, sizeof line, "%lld %lld\n", a, b);
int fd = open(argv[1], O_WRONLY | O_APPEND | O_CREAT, 0644);
write(fd, line, n);
close(fd);
return 0;
}
tests/gexcc/slots/run.sh
Executable
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
# gexcc-exec's machine-wide slots: 6 gexcc programs started at once from
# the host (6 gexc fit a 256-task scope), each holding for 3 s, may not
# run more than GEXCC_SLOTS (here
# 2, in a private slot directory) at a time; with the limit off (0), more
# than 2 overlap, which shows the measurement sees overlaps. Run inside a
# resource scope:
# systemd-run --user --scope -p TasksMax=256 nice -n 19 tests/gexcc/slots/run.sh
set -u
HERE=$(cd "$(dirname "$0")" && pwd)
WORK=${WORK:-$(mktemp -d)}
command -v gexcc > /dev/null || { echo "gexcc not on PATH"; exit 2; }
gexcc -O2 -o "$WORK/hold" "$HERE/hold.c" || exit 1
# most intervals open at once: +1 at each start, -1 at each end
overlap() { awk '{print $1, 1; print $2, -1}' "$1" | sort -n -k1,1 -k2,2n | awk '{c += $2; if (c > m) m = c} END {print m}'; }
burst() {
rm -f "$WORK/log.$1"
i=0
while [ $i -lt 6 ]; do GEXCC_SLOTS=$1 GEXCC_SLOT_DIR=$WORK/slots "$WORK/hold" "$WORK/log.$1" 3000 & i=$((i + 1)); done
wait
}
burst 2; burst 0
on=$(overlap "$WORK/log.2"); off=$(overlap "$WORK/log.0")
runs=$(cat "$WORK/log.2" | wc -l)
if [ "$runs" -eq 6 ] && [ "$on" -le 2 ] && [ "$off" -gt 2 ]; then
echo "ok slots (6 runs: at most $on at once with 2 slots, $off without)"
else
echo "FAIL slots: $runs runs, $on at once with 2 slots (want <= 2), $off without (want > 2)"; exit 1
fi
tests/gexcc/stdinseek.c
+18
View File
@@ -0,0 +1,18 @@
/* Standard streams redirected from a host file: the guest's offset is the
* host descriptor's (stdinseek.sh reads on after the program). */
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
int main(void)
{
char b[4];
struct stat st;
int r = fstat(0, &st);
long n = read(0, b, 4);
long cur = (long)lseek(0, 0, SEEK_CUR);
long set = (long)lseek(0, 6, SEEK_SET);
printf("fstat %d regular %d size %lld read %ld cur %ld set %ld\n", r, S_ISREG(st.st_mode),
(long long)st.st_size, n, cur, set);
return 0;
}
tests/gexcc/stdinseek.sh
+7
View File
@@ -0,0 +1,7 @@
# run as: sh stdinseek.sh PROG
printf 'abcdefghij\n' > stdinseek.in
(exec 3< stdinseek.in; "$1" <&3; echo "next: $(head -c 4 <&3)")
# a pipe: the program takes only what it reads (unbuffered)
printf 'abcdefghij\n' | { "$1" > /dev/null; echo "after: $(cat)"; }
printf 'line\n' > stdinseek.out; "$1" < stdinseek.in >> stdinseek.out; cat stdinseek.out
rm -f stdinseek.in stdinseek.out
tests/gexcc/tcphalfclose.c
+44
View File
@@ -0,0 +1,44 @@
/* Loopback TCP between two sockets of one process: shutdown(SHUT_WR) gives
* the peer end of file while the other direction still works; after a full
* close the first write is taken and the next is EPIPE (SIGPIPE ignored),
* as on Linux. */
#include <arpa/inet.h>
#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
int main(void)
{
struct sockaddr_in a;
socklen_t al = sizeof a;
char b[16];
int one = 1;
signal(SIGPIPE, SIG_IGN);
memset(&a, 0, sizeof a);
a.sin_family = AF_INET;
inet_pton(AF_INET, "127.0.0.1", &a.sin_addr);
int s = socket(AF_INET, SOCK_STREAM, 0);
setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof one);
bind(s, (void *)&a, sizeof a);
listen(s, 1);
getsockname(s, (void *)&a, &al);
int c = socket(AF_INET, SOCK_STREAM, 0);
printf("connect %d\n", connect(c, (void *)&a, sizeof a));
int d = accept(s, 0, 0);
printf("write %ld\n", (long)write(c, "ping", 4));
printf("shutdown %d\n", shutdown(c, SHUT_WR));
long n = read(d, b, sizeof b);
printf("read %ld then eof %ld\n", n, (long)read(d, b, sizeof b));
printf("reply %ld\n", (long)write(d, "pong", 4));
printf("got %ld\n", (long)read(c, b, sizeof b));
close(c);
usleep(50000);
printf("first write after close %ld\n", (long)write(d, "x", 1));
usleep(50000);
long w = write(d, "x", 1);
printf("second %ld %s\n", w, w < 0 && errno == EPIPE ? "EPIPE" : "?");
return 0;
}
tests/gexcc/timestamps.c
+30
View File
@@ -0,0 +1,30 @@
/* Timestamps to the nanosecond: utimensat and futimens set both times,
* stat reports atime, mtime and ctime with their nanoseconds (make, tar,
* cp -p and gnulib's stat-time rely on them). */
#define _GNU_SOURCE
#include <fcntl.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
int main(void)
{
struct stat st;
struct timespec ts[2] = {{1000000000, 123456789}, {1500000000, 987654321}};
unlink("ts.tmp");
int fd = open("ts.tmp", O_CREAT | O_WRONLY, 0644);
printf("utimensat %d\n", utimensat(AT_FDCWD, "ts.tmp", ts, 0));
stat("ts.tmp", &st);
printf("a %ld.%09ld m %ld.%09ld\n", (long)st.st_atim.tv_sec, st.st_atim.tv_nsec, (long)st.st_mtim.tv_sec, st.st_mtim.tv_nsec);
ts[0].tv_nsec = UTIME_OMIT;
ts[1].tv_sec = 1600000000; ts[1].tv_nsec = 5;
printf("futimens %d\n", futimens(fd, ts));
fstat(fd, &st);
printf("a %ld.%09ld m %ld.%09ld\n", (long)st.st_atim.tv_sec, st.st_atim.tv_nsec, (long)st.st_mtim.tv_sec, st.st_mtim.tv_nsec);
printf("ctime after mtime set: %d\n", st.st_ctim.tv_sec > 1600000000);
ts[1].tv_nsec = 1000000000;
printf("bad nsec %d\n", utimensat(AT_FDCWD, "ts.tmp", ts, 0));
close(fd);
unlink("ts.tmp");
return 0;
}
tests/nt/bin/x64/attachwin.exe
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x64/attachwin_dll.dll
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x64/audio.exe
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x64/sockets.exe
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x86/attachwin.exe
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x86/attachwin_dll.dll
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x86/audio.exe
Executable
BIN
View File
Binary file not shown.
tests/nt/bin/x86/sockets.exe
Executable
BIN
View File
Binary file not shown.
tests/nt/run.sh
+10
View File
@@ -111,6 +111,12 @@ for arch in x64 x86; do
# A loaded DLL mapped again as an image section: the loaded one keeps
# working (ntdll's Unix calls after the view).
check $arch/imageview 0 "ok ntdll mapped twice as an image view" "$b/imageview.exe"
# Audio out through winmm and mmdevapi: native Wine (W3) only, whose
# OSS driver plays into gex's /dev/dsp. GEX_NT_W3_INSTALLED=1 runs it:
# the prefix needs wineboot's install (mmdevapi's COM classes).
[ -n "${GEX_NT_W3_INSTALLED:-}" ] && check $arch/audio 0 "${allok}" "$b/audio.exe"
# Winsock over loopback (TCP, UDP, select, WSAEventSelect): W3 only.
[ -n "${GEX_NT_W3:-}" ] && check $arch/sockets 0 "${allok}" "$b/sockets.exe"
# The same as a child process (CreateProcess, waited for): native Wine
# (W3) only, where a child reads the session through requests; the NT
# layer has no child processes. GEX_NT_CHILDREN=1 runs it.
@@ -121,6 +127,10 @@ for arch in x64 x86; do
[ -n "${GEX_NT_CHILDREN:-}" ] && check $arch/childloop 0 "0 of 8 children failed" "$b/childloop.exe"
# A child that dies in native code releases its parent.
[ -n "${GEX_NT_CHILDREN:-}" ] && check $arch/childtrap 0 "child gone" "$b/childtrap.exe"
# A window made in a DllMain (the loader lock held) before the process
# has a desktop: under W3 the desktop process starts meanwhile. W3
# only (the NT layer lacks NtAreMappedFilesTheSame; it is legacy).
[ -n "${GEX_NT_W3:-}" ] && check $arch/attachwin 0 "${allok}" "$b/attachwin.exe"
check $arch/gdi 0 "$allok" "$b/gdi.exe"
check $arch/input 0 "$allok" "$b/input.exe"
check $arch/filemap 0 "$allok" "$b/filemap.exe"
tests/nt/src/attachwin.c
+16
View File
@@ -0,0 +1,16 @@
/* A window made in a DllMain, the loader lock held, before the process has
* a desktop: under native Wine on gex (W3) the desktop is a thread of the
* process, which needed the lock its creator held: explorer is a process
* of its own again (docs/wine-native.md).
* gex-nt-cc: clang --target=ARCH-w64-mingw32 -fuse-ld=lld */
#include <windows.h>
#include <stdio.h>
int main(void) {
HMODULE m = LoadLibraryA("attachwin_dll.dll");
HWND *h = m ? (HWND *)GetProcAddress(m, "attachwin_hwnd") : 0;
int made = h && *h && IsWindow(*h);
printf("%s the DLL loads\n", m ? "ok" : "FAIL");
printf("%s its DllMain made a window\n", made ? "ok" : "FAIL");
printf("%d failures\n", !m + !made);
return !made;
}
tests/nt/src/attachwin_dll.c
+16
View File
@@ -0,0 +1,16 @@
/* attachwin's DLL: its DllMain makes a window (which needs the desktop)
* while the loader lock is held, as some games' DLLs and drivers do.
* gex-nt-dll: 0x1a000000 */
#include <windows.h>
__declspec(dllexport) HWND attachwin_hwnd;
static LRESULT CALLBACK proc(HWND h, UINT m, WPARAM w, LPARAM l) { return DefWindowProcA(h, m, w, l); }
BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, void *reserved) {
if (reason == DLL_PROCESS_ATTACH) {
WNDCLASSA c = {0};
c.lpfnWndProc = proc; c.hInstance = inst; c.lpszClassName = "attachwin";
RegisterClassA(&c);
if (GetDesktopWindow())
attachwin_hwnd = CreateWindowA("attachwin", "attachwin", WS_OVERLAPPEDWINDOW, 0, 0, 100, 100, 0, 0, inst, 0);
}
return TRUE;
}
tests/nt/src/audio.c
+36
View File
@@ -0,0 +1,36 @@
/* Audio output through winmm (waveOut over mmdevapi): a device exists, a
* stream opens, and a buffer is played at its own rate (0.4 s of 44.1 kHz
* stereo takes at least 0.3 s to come back done). Native Wine (W3) only,
* where mmdevapi's OSS driver plays into gex's /dev/dsp (a null output):
* GEX_NT_W3=1 runs it.
* gex-nt-cc: clang --target=ARCH-w64-mingw32 -fuse-ld=lld */
#include <windows.h>
#include <mmsystem.h>
#include <stdio.h>
static int fails;
static void ok(int c, const char *what) { printf("%s %s\n", c ? "ok" : "FAIL", what); if (!c) fails++; }
int main(void) {
WAVEFORMATEX f = {WAVE_FORMAT_PCM, 2, 44100, 44100 * 4, 4, 16, 0};
HWAVEOUT wo = 0;
HANDLE done = CreateEventA(0, FALSE, FALSE, 0);
static short pcm[44100 * 2 * 4 / 10];
WAVEHDR h = {(LPSTR)pcm, sizeof pcm};
DWORD t0, waited;
int i;
for (i = 0; i < (int)(sizeof pcm / sizeof *pcm); i++) pcm[i] = (short)((i % 100) * 300 - 15000);
ok(waveOutGetNumDevs() >= 1, "an output device");
ok(waveOutOpen(&wo, WAVE_MAPPER, &f, (DWORD_PTR)done, 0, CALLBACK_EVENT) == MMSYSERR_NOERROR, "waveOutOpen 44.1 kHz stereo");
if (!wo) { printf("%d failures\n", fails); return 1; }
WaitForSingleObject(done, 1000); /* WOM_OPEN */
ok(waveOutPrepareHeader(wo, &h, sizeof h) == MMSYSERR_NOERROR, "waveOutPrepareHeader");
t0 = GetTickCount();
ok(waveOutWrite(wo, &h, sizeof h) == MMSYSERR_NOERROR, "waveOutWrite");
while (!(h.dwFlags & WHDR_DONE) && GetTickCount() - t0 < 10000) WaitForSingleObject(done, 100);
waited = GetTickCount() - t0;
ok(h.dwFlags & WHDR_DONE, "the buffer comes back done");
ok(waited >= 300, "played at its rate, not at once");
waveOutUnprepareHeader(wo, &h, sizeof h);
ok(waveOutClose(wo) == MMSYSERR_NOERROR, "waveOutClose");
printf("%d failures\n", fails);
return fails != 0;
}
tests/nt/src/sockets.c
+80
View File
@@ -0,0 +1,80 @@
/* Winsock over loopback: TCP (listen, connect, accept, send, recv, select,
* a non-blocking connect, shutdown), UDP (sendto, recvfrom), name lookup
* of localhost, and WSAEventSelect. What a game's networking (and a
* launcher's single-instance port) starts with.
* gex-nt-cc: clang --target=ARCH-w64-mingw32 -fuse-ld=lld */
#include <winsock2.h>
#include <ws2tcpip.h>
#include <stdio.h>
#include <string.h>
static int fails;
static void ok(int c, const char *what) { printf("%s %s\n", c ? "ok" : "FAIL", what); if (!c) { fails++; printf(" (WSAGetLastError %d)\n", WSAGetLastError()); } }
int main(void) {
WSADATA wsa;
struct sockaddr_in a = {0}, b = {0};
int len = sizeof a, n;
SOCKET l, c, s, u1, u2;
char buf[64];
u_long nb = 1;
fd_set w, r;
struct timeval tv = {5, 0};
struct addrinfo *ai = 0, hints = {0};
WSAEVENT ev;
WSANETWORKEVENTS ne;
setvbuf(stdout, 0, _IONBF, 0);
ok(!WSAStartup(MAKEWORD(2, 2), &wsa), "WSAStartup 2.2");
hints.ai_family = AF_INET;
ok(!getaddrinfo("localhost", "80", &hints, &ai) && ai, "getaddrinfo localhost");
if (ai) { ok(((struct sockaddr_in *)ai->ai_addr)->sin_addr.s_addr == htonl(INADDR_LOOPBACK), "localhost is 127.0.0.1"); freeaddrinfo(ai); }
l = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
ok(l != INVALID_SOCKET, "a TCP socket");
a.sin_family = AF_INET; a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
ok(!bind(l, (struct sockaddr *)&a, sizeof a), "bind 127.0.0.1:0");
ok(!getsockname(l, (struct sockaddr *)&a, &len) && a.sin_port, "getsockname gives a port");
ok(!listen(l, 4), "listen");
c = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
ok(!ioctlsocket(c, FIONBIO, &nb), "FIONBIO");
n = connect(c, (struct sockaddr *)&a, sizeof a);
ok(n == 0 || WSAGetLastError() == WSAEWOULDBLOCK, "a non-blocking connect starts");
FD_ZERO(&w); FD_SET(c, &w);
ok(select(0, 0, &w, 0, &tv) == 1, "select: the connect completes");
len = sizeof b;
s = accept(l, (struct sockaddr *)&b, &len);
ok(s != INVALID_SOCKET && b.sin_addr.s_addr == htonl(INADDR_LOOPBACK), "accept, from 127.0.0.1");
nb = 0; ioctlsocket(c, FIONBIO, &nb);
ok(send(c, "hello", 5, 0) == 5, "send");
FD_ZERO(&r); FD_SET(s, &r);
ok(select(0, &r, 0, 0, &tv) == 1, "select: readable");
memset(buf, 0, sizeof buf);
n = recv(s, buf, sizeof buf, 0);
if (n != 5) printf(" (recv returned %d: %.8s)\n", n, buf);
ok(n == 5 && !memcmp(buf, "hello", 5), "recv what was sent");
ev = WSACreateEvent();
ok(!WSAEventSelect(s, ev, FD_READ | FD_CLOSE), "WSAEventSelect");
ok(send(c, "x", 1, 0) == 1, "send again");
ok(WSAWaitForMultipleEvents(1, &ev, FALSE, 5000, FALSE) == WSA_WAIT_EVENT_0, "the event fires");
ok(!WSAEnumNetworkEvents(s, ev, &ne) && (ne.lNetworkEvents & FD_READ), "FD_READ");
nb = 0; WSAEventSelect(s, 0, 0); ioctlsocket(s, FIONBIO, &nb);
ok(recv(s, buf, sizeof buf, 0) == 1, "recv the byte");
ok(!shutdown(c, SD_SEND), "shutdown");
ok(recv(s, buf, sizeof buf, 0) == 0, "recv sees the end");
closesocket(c); closesocket(s); closesocket(l); WSACloseEvent(ev);
u1 = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
u2 = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
a.sin_port = 0; len = sizeof a;
ok(!bind(u1, (struct sockaddr *)&a, sizeof a) && !getsockname(u1, (struct sockaddr *)&a, &len), "a UDP socket bound");
ok(sendto(u2, "dgram", 5, 0, (struct sockaddr *)&a, sizeof a) == 5, "sendto");
len = sizeof b; memset(buf, 0, sizeof buf);
FD_ZERO(&r); FD_SET(u1, &r);
ok(select(0, &r, 0, 0, &tv) == 1, "select: a datagram");
ok(recvfrom(u1, buf, sizeof buf, 0, (struct sockaddr *)&b, &len) == 5 && !memcmp(buf, "dgram", 5), "recvfrom");
closesocket(u1); closesocket(u2);
WSACleanup();
printf("%d failures\n", fails);
return fails != 0;
}
tests/wasm32/inet.c
+79
View File
@@ -0,0 +1,79 @@
/* Loopback TCP and UDP the way Wine's server uses them: bind to port 0,
* getsockname for the port, listen, a non-blocking connect completed by
* poll, accept, send and recv, shutdown; a UDP datagram. */
#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <poll.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
static int fails;
static void ok(int c, const char *what) { printf("%s %s\n", c ? "ok" : "FAIL", what); if (!c) fails++; }
int main(void)
{
struct sockaddr_in a = {0}, b = {0};
socklen_t len = sizeof a;
char buf[16] = {0};
int l, c, s, u1, u2, r, err = -1;
struct pollfd p;
setvbuf(stdout, 0, _IONBF, 0);
a.sin_family = AF_INET;
a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
l = socket(AF_INET, SOCK_STREAM, 0);
ok(bind(l, (struct sockaddr *)&a, sizeof a) == 0, "bind 127.0.0.1:0");
ok(getsockname(l, (struct sockaddr *)&a, &len) == 0 && a.sin_port != 0, "getsockname gives the port");
ok(listen(l, 4) == 0, "listen");
c = socket(AF_INET, SOCK_STREAM, 0);
fcntl(c, F_SETFL, O_NONBLOCK);
r = connect(c, (struct sockaddr *)&a, sizeof a);
ok(r == 0 || errno == EINPROGRESS, "a non-blocking connect starts");
p.fd = c; p.events = POLLOUT;
ok(poll(&p, 1, 5000) == 1, "poll: the connect completes");
len = sizeof err;
ok(getsockopt(c, SOL_SOCKET, SO_ERROR, &err, &len) == 0 && err == 0, "SO_ERROR 0");
len = sizeof b;
s = accept(l, (struct sockaddr *)&b, &len);
ok(s >= 0 && b.sin_addr.s_addr == htonl(INADDR_LOOPBACK), "accept, from 127.0.0.1");
len = sizeof b;
ok(getpeername(c, (struct sockaddr *)&b, &len) == 0 && b.sin_port == a.sin_port, "getpeername is the listener");
ok(send(c, "hello", 5, 0) == 5, "send");
p.fd = s; p.events = POLLIN;
ok(poll(&p, 1, 5000) == 1, "poll: readable");
ok(recv(s, buf, sizeof buf, 0) == 5 && !memcmp(buf, "hello", 5), "recv");
ok(shutdown(c, SHUT_WR) == 0, "shutdown");
ok(recv(s, buf, sizeof buf, 0) == 0, "recv sees the end");
close(c); close(s); close(l);
u1 = socket(AF_INET, SOCK_DGRAM, 0);
u2 = socket(AF_INET, SOCK_DGRAM, 0);
a.sin_port = 0; len = sizeof a;
ok(bind(u1, (struct sockaddr *)&a, sizeof a) == 0 && getsockname(u1, (struct sockaddr *)&a, &len) == 0 && a.sin_port, "UDP bound, with its port");
ok(sendto(u2, "dgram", 5, 0, (struct sockaddr *)&a, sizeof a) == 5, "sendto");
p.fd = u1; p.events = POLLIN;
ok(poll(&p, 1, 5000) == 1, "poll: a datagram");
len = sizeof b;
ok(recvfrom(u1, buf, sizeof buf, 0, (struct sockaddr *)&b, &len) == 5 && !memcmp(buf, "dgram", 5), "recvfrom");
{
/* What musl's resolver does: poll with unused (negative) slots,
* which report nothing, and recvmsg, which names the sender. */
struct pollfd ps[3] = {{.fd = -1}, {.fd = -1}, {.fd = u1, .events = POLLIN}};
struct sockaddr_in from = {0}, me = {0};
struct iovec io = {buf, sizeof buf};
struct msghdr mh = {.msg_name = &from, .msg_namelen = sizeof from, .msg_iov = &io, .msg_iovlen = 1};
socklen_t ml = sizeof me;
ok(sendto(u2, "again", 5, 0, (struct sockaddr *)&a, sizeof a) == 5, "sendto again");
ok(poll(ps, 3, 5000) == 1 && ps[0].revents == 0 && ps[2].revents == POLLIN, "poll: negative slots report nothing");
ok(recvmsg(u1, &mh, 0) == 5 && !memcmp(buf, "again", 5), "recvmsg");
ok(getsockname(u2, (struct sockaddr *)&me, &ml) == 0 && mh.msg_namelen == sizeof from
&& from.sin_family == AF_INET && from.sin_port == me.sin_port, "recvmsg names the sender");
}
close(u1); close(u2);
printf("%d failures\n", fails);
return fails != 0;
}
tests/wasm32/run-browser.sh
+6 -2
View File
@@ -11,12 +11,16 @@ set -u
HERE=$(cd "$(dirname "$0")" && pwd)
ROOT=$(cd "$HERE/../.." && pwd)
: "${GEX_WASM_SYSROOT:?set GEX_WASM_SYSROOT (tools/wine-wasm/build-sysroot.sh)}"
SEED=$ROOT/web/public/gex/seed.x86.shared.wasm
# GEX_ARCH=aarch64: the AArch64 build's shim and seed (native wasm32
# programs run on both guests' builds); default x86
ARCH=${GEX_ARCH:-x86}
case $ARCH in x86) INFIX=.x86 ;; *) INFIX= ;; esac
SEED=$ROOT/web/public/gex/seed$INFIX.shared.wasm
[ -f "$SEED" ] || { echo "no $SEED; build with GEX_ARCH=x86 GEX_SHARED=1 shims/browser/build.sh" >&2; exit 2; }
WORK=${WORK:-$(mktemp -d)}
fail=0
run() {
GEX_ARCH=x86 GEX_SHARED=1 timeout "${TMO:-300}" node "$ROOT/shims/browser/js/run-node.mjs" "$@" 2>/dev/null | grep -v '^\[host\]\|^\[gex\] \|^\[worker\] '
GEX_ARCH=$ARCH GEX_SHARED=1 timeout "${TMO:-300}" node "$ROOT/shims/browser/js/run-node.mjs" "$@" 2>/dev/null | grep -v '^\[host\]\|^\[gex\] \|^\[worker\] '
}
for src in "$HERE"/*.c; do
name=$(basename "$src" .c)
tools/gen_gl_extensions.py
+24 -2
View File
@@ -109,6 +109,26 @@ CURATED_ALIASES = {
}
# Extensions that add no commands of their own but are options on another
# extension's programs, which the registry does not record: with nothing
# required they were always "claimable", while the extension they extend was
# withheld for its missing commands (glGenProgramsARB and the rest). An
# application that checks only the option then calls a null pointer. wined3d
# does exactly that: its caps probe tests a vertex program with OPTION
# NV_vertex_program2 when GL_NV_vertex_program2_option is listed, and on
# NVIDIA's host (which lists it) Undertale and Katana ZERO died in
# glGenProgramsARB at address 0. Each is claimable only with its base.
BUILDS_ON = {
"GL_ARB_fragment_program_shadow": "GL_ARB_fragment_program",
"GL_NV_fragment_program_option": "GL_ARB_fragment_program",
"GL_NV_fragment_program2": "GL_ARB_fragment_program",
"GL_NV_vertex_program2_option": "GL_ARB_vertex_program",
"GL_NV_vertex_program3": "GL_ARB_vertex_program",
"GL_NV_vertex_program1_1": "GL_NV_vertex_program",
"GL_NV_vertex_program2": "GL_NV_vertex_program",
}
def command_aliases(registry: pathlib.Path) -> dict[str, str]:
"""Commands the registry declares to be another command under a new name."""
root = ElementTree.parse(registry).getroot()
@@ -161,8 +181,10 @@ def generate(registry: pathlib.Path, library: pathlib.Path) -> tuple[str, str]:
alias_lines.append("")
available = defined | set(emit)
claimable = sorted(name for name, required in commands.items()
if required <= available)
claimable = {name for name, required in commands.items()
if required <= available}
claimable -= {name for name, base in BUILDS_ON.items() if base not in claimable}
claimable = sorted(claimable)
withheld = len(commands) - len(claimable)
lines = [
"/* Generated by tools/gen_gl_extensions.py; do not edit.",
tools/gexcc/build-cxx.sh
Executable
+41
View File
@@ -0,0 +1,41 @@
#!/bin/sh
# libc++, libc++abi and libunwind for a gexcc install (C++ with wasm
# exception handling), from an LLVM source tree of clang's version.
#
# tools/gexcc/build-cxx.sh PREFIX LLVM_SRC [BUILD_DIR]
#
# LLVM_SRC needs runtimes/, cmake/, llvm/cmake, llvm/utils, libcxx/,
# libcxxabi/, libunwind/, libc/ (libc++ uses libc's shared headers) and
# third-party/. Installs into PREFIX/sysroot: include/c++/v1, lib/libc++.a,
# libc++abi.a, libunwind.a (gexc++ links them).
set -eu
PREFIX=$(realpath "$1")
SRC=$(realpath "$2")
B=$(realpath -m "${3:-$PREFIX/build/cxx}")
SR=$PREFIX/sysroot
JOBS=${JOBS:-6}
export PATH=$PREFIX/bin:$PATH
rm -rf "$B"
# wasm is not ELF, so libunwind would mark its exports __declspec(dllexport)
# unless told the symbols are hidden (one static module: they need not be
# exported at all).
F="-fwasm-exceptions -D_LIBUNWIND_HIDE_SYMBOLS"
cmake -G Ninja -S "$SRC/runtimes" -B "$B" \
-DCMAKE_TOOLCHAIN_FILE="$PREFIX/share/gexcc/toolchain.cmake" \
-DCMAKE_STAGING_PREFIX="$SR" -DCMAKE_INSTALL_PREFIX="$SR" -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_WORKS=ON -DCMAKE_CXX_COMPILER_WORKS=ON \
-DCMAKE_C_FLAGS="$F" -DCMAKE_CXX_FLAGS="$F" \
-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
-DLIBCXX_ENABLE_SHARED=OFF -DLIBCXXABI_ENABLE_SHARED=OFF -DLIBUNWIND_ENABLE_SHARED=OFF \
-DLIBCXX_HAS_MUSL_LIBC=ON -DLIBCXX_CXX_ABI=libcxxabi -DLIBCXXABI_USE_LLVM_UNWINDER=ON \
-DLIBCXXABI_ENABLE_STATIC_UNWINDER=ON -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
-DLIBCXX_ENABLE_EXCEPTIONS=ON -DLIBCXXABI_ENABLE_EXCEPTIONS=ON \
-DLIBCXX_USE_COMPILER_RT=ON -DLIBCXXABI_USE_COMPILER_RT=ON -DLIBUNWIND_USE_COMPILER_RT=ON \
-DLIBCXX_INCLUDE_BENCHMARKS=OFF -DLIBCXX_INCLUDE_TESTS=OFF -DLIBCXXABI_INCLUDE_TESTS=OFF \
-DLIBUNWIND_INCLUDE_TESTS=OFF > "$B.cmake.log" 2>&1 || { tail -20 "$B.cmake.log"; exit 1; }
ninja -C "$B" -j"$JOBS" install > "$B.build.log" 2>&1 || { grep -E 'error' "$B.build.log" | head -20; exit 1; }
# the C++ exception tag (cpp_exception.s), next to Unwind-wasm.c's users
HERE=$(dirname "$(realpath "$0")")
clang --target=wasm32-unknown-linux-musl -mexception-handling -c "$HERE/cpp_exception.s" -o "$B/cpp_exception.o"
for a in libc++.a libc++abi.a libunwind.a; do llvm-ar rs "$SR/lib/$a" "$B/cpp_exception.o"; done
echo "libc++: $SR/lib/libc++.a"
tools/gexcc/config.site
+138
View File
@@ -0,0 +1,138 @@
# config.site for gexcc (CONFIG_SITE): configure in cross mode (--build
# given) runs none of the programs it builds, so it guesses what they would
# have found. These are the answers gnulib's and autoconf's run-time checks
# gave on gex itself (measured 2026-10-07, natively, gex at merge/gexcc
# 24be3714, musl gex-wasm32-slim). A "no" makes gnulib use its own
# replacement, which is always safe; several are gex gaps being fixed
# (trailing slashes, select's EBADF, getgroups, sigaltstack), see
# docs/gexcc.md. Regenerate: tests/gexcc/corpus/config-site.sh.
ac_cv_func_chown_works=${ac_cv_func_chown_works=yes}
ac_cv_func_getgroups_works=${ac_cv_func_getgroups_works=yes}
ac_cv_func_malloc_0_nonnull=${ac_cv_func_malloc_0_nonnull=yes}
ac_cv_func_realloc_0_nonnull=${ac_cv_func_realloc_0_nonnull=yes}
ac_cv_func_strcoll_works=${ac_cv_func_strcoll_works=yes}
ac_cv_func_working_mktime=${ac_cv_func_working_mktime=yes}
gl_cv_cc_long_double_expbit0=${gl_cv_cc_long_double_expbit0='word 3 bit 16'}
gl_cv_double_slash_root=${gl_cv_double_slash_root=no}
gl_cv_func_access_slash_works=${gl_cv_func_access_slash_works=yes}
gl_cv_func_btowc_consistent=${gl_cv_func_btowc_consistent=yes}
gl_cv_func_btowc_nul=${gl_cv_func_btowc_nul=yes}
gl_cv_func_c32rtomb_retval=${gl_cv_func_c32rtomb_retval=yes}
gl_cv_func_chmod_works=${gl_cv_func_chmod_works=yes}
gl_cv_func_chown_ctime_works=${gl_cv_func_chown_ctime_works=no}
gl_cv_func_chown_follows_symlink=${gl_cv_func_chown_follows_symlink=no}
gl_cv_func_chown_slash_works=${gl_cv_func_chown_slash_works=no}
gl_cv_func_dup2_works=${gl_cv_func_dup2_works=yes}
gl_cv_func_dup_works=${gl_cv_func_dup_works=yes}
gl_cv_func_fchmodat_works=${gl_cv_func_fchmodat_works=no}
gl_cv_func_fchownat_empty_filename_works=${gl_cv_func_fchownat_empty_filename_works=no}
gl_cv_func_fchownat_nofollow_works=${gl_cv_func_fchownat_nofollow_works=yes}
gl_cv_func_fcntl_f_dupfd_cloexec=${gl_cv_func_fcntl_f_dupfd_cloexec='needs runtime check'}
gl_cv_func_fcntl_f_dupfd_works=${gl_cv_func_fcntl_f_dupfd_works=yes}
gl_cv_func_fdopendir_works=${gl_cv_func_fdopendir_works=yes}
gl_cv_func_fdopen_works=${gl_cv_func_fdopen_works=yes}
gl_cv_func_fflush_stdin=${gl_cv_func_fflush_stdin=yes}
gl_cv_func_fnmatch_gnu=${gl_cv_func_fnmatch_gnu=no}
gl_cv_func_fnmatch_posix=${gl_cv_func_fnmatch_posix=yes}
gl_cv_func_fopen_mode_e=${gl_cv_func_fopen_mode_e=yes}
gl_cv_func_fopen_mode_x=${gl_cv_func_fopen_mode_x=yes}
gl_cv_func_fopen_slash=${gl_cv_func_fopen_slash=no}
gl_cv_func_fpurge_works=${gl_cv_func_fpurge_works=no}
gl_cv_func_freopen_works_on_closed=${gl_cv_func_freopen_works_on_closed=yes}
gl_cv_func_fstatat_zero_flag=${gl_cv_func_fstatat_zero_flag=yes}
gl_cv_func_ftello_works=${gl_cv_func_ftello_works=yes}
gl_cv_func_futimens_works=${gl_cv_func_futimens_works=yes}
gl_cv_func_getcwd_null=${gl_cv_func_getcwd_null=yes}
gl_cv_func_getcwd_path_max=${gl_cv_func_getcwd_path_max=yes}
gl_cv_func_getcwd_succeeds_beyond_4k=${gl_cv_func_getcwd_succeeds_beyond_4k=yes}
gl_cv_func_getdtablesize_works=${gl_cv_func_getdtablesize_works=yes}
gl_cv_func_getgroups_works=${gl_cv_func_getgroups_works=no}
gl_cv_func_getopt_posix=${gl_cv_func_getopt_posix=no}
gl_cv_func_isnanf_works=${gl_cv_func_isnanf_works=yes}
gl_cv_func_isnanl_works=${gl_cv_func_isnanl_works=yes}
gl_cv_func_iswcntrl_works=${gl_cv_func_iswcntrl_works=yes}
gl_cv_func_iswpunct_works=${gl_cv_func_iswpunct_works=yes}
gl_cv_func_itold_works=${gl_cv_func_itold_works=yes}
gl_cv_func_linkat_nofollow=${gl_cv_func_linkat_nofollow=yes}
gl_cv_func_linkat_slash=${gl_cv_func_linkat_slash=no}
gl_cv_func_link_follows_symlink=${gl_cv_func_link_follows_symlink=no}
gl_cv_func_link_works=${gl_cv_func_link_works=no}
gl_cv_func_localeconv_works=${gl_cv_func_localeconv_works=yes}
gl_cv_func_localtime_works=${gl_cv_func_localtime_works=yes}
gl_cv_func_lstat_dereferences_slashed_symlink=${gl_cv_func_lstat_dereferences_slashed_symlink=yes}
gl_cv_func_mbrtoc32_C_locale_sans_EILSEQ=${gl_cv_func_mbrtoc32_C_locale_sans_EILSEQ=yes}
gl_cv_func_mbrtoc32_empty_input=${gl_cv_func_mbrtoc32_empty_input=yes}
gl_cv_func_mbrtowc_C_locale_sans_EILSEQ=${gl_cv_func_mbrtowc_C_locale_sans_EILSEQ=yes}
gl_cv_func_memchr_works=${gl_cv_func_memchr_works=yes}
gl_cv_func_mkdir_trailing_dot_works=${gl_cv_func_mkdir_trailing_dot_works=no}
gl_cv_func_mkdir_trailing_slash_works=${gl_cv_func_mkdir_trailing_slash_works=yes}
gl_cv_func_mkfifoat_works=${gl_cv_func_mkfifoat_works=yes}
gl_cv_func_mkfifo_works=${gl_cv_func_mkfifo_works=yes}
gl_cv_func_mknod_works=${gl_cv_func_mknod_works=no}
gl_cv_func_nanosleep=${gl_cv_func_nanosleep='no (mishandles large arguments)'}
gl_cv_func_nl_langinfo_yesexpr_works=${gl_cv_func_nl_langinfo_yesexpr_works=yes}
gl_cv_func_open_slash=${gl_cv_func_open_slash=no}
gl_cv_func_perror_works=${gl_cv_func_perror_works=yes}
gl_cv_func_printf_positions=${gl_cv_func_printf_positions=yes}
gl_cv_func_pselect_detects_ebadf=${gl_cv_func_pselect_detects_ebadf=no}
gl_cv_func_pthread_sigmask_in_libc_works=${gl_cv_func_pthread_sigmask_in_libc_works=yes}
gl_cv_func_pthread_sigmask_return_works=${gl_cv_func_pthread_sigmask_return_works=yes}
gl_cv_func_readlink_trailing_slash=${gl_cv_func_readlink_trailing_slash=no}
gl_cv_func_readlink_truncate=${gl_cv_func_readlink_truncate=yes}
gl_cv_func_realpath_works=${gl_cv_func_realpath_works=no}
gl_cv_func_re_compile_pattern_working=${gl_cv_func_re_compile_pattern_working=no}
gl_cv_func_rename_dest_works=${gl_cv_func_rename_dest_works=yes}
gl_cv_func_rename_link_works=${gl_cv_func_rename_link_works=yes}
gl_cv_func_rename_slash_dst_works=${gl_cv_func_rename_slash_dst_works=no}
gl_cv_func_rename_slash_src_works=${gl_cv_func_rename_slash_src_works=yes}
gl_cv_func_rmdir_works=${gl_cv_func_rmdir_works=no}
gl_cv_func_select_detects_ebadf=${gl_cv_func_select_detects_ebadf=no}
gl_cv_func_select_supports0=${gl_cv_func_select_supports0=yes}
gl_cv_func_setenv_works=${gl_cv_func_setenv_works=yes}
gl_cv_func_setlocale_works=${gl_cv_func_setlocale_works=yes}
gl_cv_func_signbit=${gl_cv_func_signbit=yes}
gl_cv_func_signbit_builtins=${gl_cv_func_signbit_builtins=yes}
gl_cv_func_sleep_works=${gl_cv_func_sleep_works=yes}
gl_cv_func_snprintf_retval_c99=${gl_cv_func_snprintf_retval_c99=yes}
gl_cv_func_snprintf_size1=${gl_cv_func_snprintf_size1=yes}
gl_cv_func_snprintf_truncation_c99=${gl_cv_func_snprintf_truncation_c99=yes}
gl_cv_func_stat_file_slash=${gl_cv_func_stat_file_slash=yes}
gl_cv_func_strchrnul_works=${gl_cv_func_strchrnul_works=yes}
gl_cv_func_strerror_0_works=${gl_cv_func_strerror_0_works=yes}
gl_cv_func_strerror_r_works=${gl_cv_func_strerror_r_works=yes}
gl_cv_func_strndup_works=${gl_cv_func_strndup_works=yes}
gl_cv_func_strtoimax=${gl_cv_func_strtoimax=yes}
gl_cv_func_strtoll_works=${gl_cv_func_strtoll_works=no}
gl_cv_func_strtol_works=${gl_cv_func_strtol_works=no}
gl_cv_func_strtoull_works=${gl_cv_func_strtoull_works=no}
gl_cv_func_strtoul_works=${gl_cv_func_strtoul_works=no}
gl_cv_func_svid_putenv=${gl_cv_func_svid_putenv=yes}
gl_cv_func_symlinkat_works=${gl_cv_func_symlinkat_works=no}
gl_cv_func_symlink_works=${gl_cv_func_symlink_works=no}
gl_cv_func_tmpfile_works=${gl_cv_func_tmpfile_works=yes}
gl_cv_func_ungetc_works=${gl_cv_func_ungetc_works=yes}
gl_cv_func_unlink_honors_slashes=${gl_cv_func_unlink_honors_slashes=yes}
gl_cv_func_unsetenv_works=${gl_cv_func_unsetenv_works=yes}
gl_cv_func_usleep_works=${gl_cv_func_usleep_works=yes}
gl_cv_func_utime_file_slash=${gl_cv_func_utime_file_slash=yes}
gl_cv_func_utimensat_works=${gl_cv_func_utimensat_works=yes}
gl_cv_func_wcrtomb_works=${gl_cv_func_wcrtomb_works=yes}
gl_cv_func_wctype_works=${gl_cv_func_wctype_works=yes}
gl_cv_func_wcwidth_works=${gl_cv_func_wcwidth_works=yes}
gl_cv_func_working_getdelim=${gl_cv_func_working_getdelim=yes}
gl_cv_func_working_mkstemp=${gl_cv_func_working_mkstemp=yes}
gl_cv_func_working_mktime=${gl_cv_func_working_mktime=yes}
gl_cv_func_working_strerror=${gl_cv_func_working_strerror=yes}
gl_cv_func_working_utimes=${gl_cv_func_working_utimes=yes}
gl_cv_have_weak=${gl_cv_have_weak=no}
gl_cv_header_working_fcntl_h=${gl_cv_header_working_fcntl_h='no (bad O_NOFOLLOW)'}
gl_cv_pthread_rwlock_rdlock_prefer_writer=${gl_cv_pthread_rwlock_rdlock_prefer_writer=no}
gl_cv_struct_dirent_d_ino=${gl_cv_struct_dirent_d_ino=yes}
gt_cv_func_uselocale_works=${gt_cv_func_uselocale_works=yes}
gt_cv_locale_fake=${gt_cv_locale_fake=no}
make_cv_synchronous_posix_spawn=${make_cv_synchronous_posix_spawn=yes}
sed_cv_fopen_rt=${sed_cv_fopen_rt=yes}
sed_cv_libcp_needed=${sed_cv_libcp_needed=no}
sv_cv_sigaltstack=${sv_cv_sigaltstack=no}
sv_cv_stack_direction=${sv_cv_stack_direction=-1}
sv_cv_stack_direction_msg=${sv_cv_stack_direction_msg='grows down'}
tools/gexcc/cpp_exception.s
+7
View File
@@ -0,0 +1,7 @@
# The wasm exception tag C++ throws and catches with (LLVM's C++ EH lowering
# and libunwind's Unwind-wasm.c refer to it; the runtime defines it, as
# emscripten's libcxxabi/src/__cpp_exception.S does). wasm32: the payload is
# a pointer, an i32.
.globl __cpp_exception
.tagtype __cpp_exception i32
__cpp_exception:
tools/gexcc/debian/binpkgs.txt
+22
View File
@@ -0,0 +1,22 @@
# SRCPKG BINPKG GLOB... : how mkdebs.py splits a staging tree into Debian's
# binary packages. A file goes to the first matching line; BINPKG `-` drops
# it. A source package with no lines becomes one package of its own name.
# `depends BINPKG DEPENDS...` adds a Depends field (Debian's data packages
# ours use: architecture-independent content, Multi-Arch: foreign).
zlib - *
libmd - *
readline - *
pcre2 - *
bzip2 bzip2 *
xz-utils xz-utils *
libzstd zstd *
ncurses ncurses-bin usr/bin/* usr/share/man/*
ncurses - *
util-linux bsdutils usr/bin/logger usr/bin/renice usr/bin/script usr/bin/scriptlive usr/bin/scriptreplay usr/bin/wall usr/share/man/*/logger.* usr/share/man/*/renice.* usr/share/man/*/script.* usr/share/man/*/scriptlive.* usr/share/man/*/scriptreplay.* usr/share/man/*/wall.*
util-linux bsdextrautils usr/bin/col usr/bin/colcrt usr/bin/colrm usr/bin/column usr/bin/hexdump usr/bin/hd usr/bin/look usr/bin/ul usr/bin/write usr/share/man/*/col.* usr/share/man/*/colcrt.* usr/share/man/*/colrm.* usr/share/man/*/column.* usr/share/man/*/hexdump.* usr/share/man/*/look.* usr/share/man/*/ul.*
util-linux uuid-runtime usr/bin/uuidgen usr/bin/uuidparse usr/share/man/*/uuidgen.* usr/share/man/*/uuidparse.*
util-linux util-linux *
sysvinit sysvinit-utils *
file file *
depends file libmagic-mgc
depends vim vim-common
tools/gexcc/debian/build.sh
Executable
+60
View File
@@ -0,0 +1,60 @@
#!/bin/bash
# Build Debian source packages with gexcc into native wasm32 programs.
#
# tools/gexcc/debian/build.sh SRCPKG... (or a tier: tier1, tier2, ...)
#
# Environment:
# GEXCC_PREFIX an installed toolchain (tools/gexcc/install.sh); its sysroot
# receives the libraries later packages link
# GEXCC_DEB work area (default ~/gex-sandbox/scratch/CC/debian):
# src/ unpacked sources, stage/SRCPKG/ install trees,
# logs/, built.txt
# JOBS make -j (default 6)
# KEEP=1 reuse an unpacked source tree
#
# Versions are pinned in packages.txt; debsrc.py fetches and verifies them
# and applies Debian's patches, then ours (patches/SRCPKG/). Each recipe
# (recipes.sh) builds in the source tree and installs into $DEST. Run builds
# in a resource scope (docs/gexcc.md §2): systemd-run --user --scope -p
# TasksMax=4096 -p MemoryMax=32G nice -n 19.
set -u
HERE=$(cd "$(dirname "$0")" && pwd)
: "${GEXCC_PREFIX:?set GEXCC_PREFIX}"
export GEXCC_DEB=${GEXCC_DEB:-$HOME/gex-sandbox/scratch/CC/debian}
export PATH=$GEXCC_PREFIX/bin:$PATH
export SYSROOT=$GEXCC_PREFIX/sysroot T=wasm32-unknown-linux-musl JOBS=${JOBS:-6}
# for every configure, including the reruns make starts when Debian's patches
# touched configure.ac or m4/ (config.status --recheck keeps the arguments,
# not the environment)
export CONFIG_SITE=$GEXCC_PREFIX/share/gexcc/config.site
# no Texinfo on the host, and the rootfs carries no info pages
export MAKEINFO=true
mkdir -p "$GEXCC_DEB"/{src,stage,logs}
. "$HERE/recipes.sh"
pin() { awk -v p="$1" '$1 == p {print $2}' "$HERE/packages.txt"; }
names=()
for n in "$@"; do
case $n in
tier*) names+=($(awk -v t="$n" '$3 == t {print $1}' "$HERE/packages.txt")) ;;
*) names+=("$n") ;;
esac
done
for name in "${names[@]}"; do
ver=$(pin "$name")
[ -n "$ver" ] || { echo "$name: not in packages.txt"; continue; }
upver=${ver#*:}
dir=$GEXCC_DEB/src/$name-$upver
[ -z "${KEEP:-}" ] && rm -rf "$dir"
python3 -I "$HERE/debsrc.py" --into "$GEXCC_DEB/src" "$name=$ver" > /dev/null || { echo "FAIL fetch $name"; continue; }
export DEST=$GEXCC_DEB/stage/$name
rm -rf "$DEST"; mkdir -p "$DEST"
log=$GEXCC_DEB/logs/$name.log
# (set -e is ignored inside a function called from an && list or an if
# condition, even in a subshell: run it as a plain command)
(cd "$dir" || exit 1; set -e; "build_${name//[-.+]/_}") > "$log" 2>&1
if [ $? = 0 ]; then r=ok; else r=FAIL; fi
echo "$name $ver build=$r"
touch "$GEXCC_DEB/built.txt"; sed -i "/^$name /d" "$GEXCC_DEB/built.txt"
echo "$name $ver build=$r" >> "$GEXCC_DEB/built.txt"
done
tools/gexcc/debian/debfetch.py
Executable
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Fetch Debian binary packages and their dependency closure, verified.
debfetch.py --arch ARCH [--suite trixie] [--cache DIR] [--extract ROOT]
[--no-deps] [--exclude PKG,...] PKG...
The chain of trust: dists/SUITE/InRelease is checked with gpgv against the
host's debian-archive-keyring, binary-ARCH/Packages.xz against InRelease's
SHA256, every .deb against Packages' SHA256. Downloads are cached by file
name under CACHE/pool (default ~/gex-sandbox/scratch/CC/deb), so repeated
runs touch the mirror only for the index (and not even that within a day).
Dependency closure: Pre-Depends and Depends; for "a | b" the first
alternative already in the set wins, else the first that exists. Virtual
packages resolve to their first provider. Prints the chosen packages
(name version arch file), one per line, and with --extract unpacks them
into ROOT (dpkg-deb -x; no maintainer scripts).
"""
import argparse, hashlib, lzma, os, subprocess, sys, time, urllib.request
MIRROR = os.environ.get('GEX_DEBIAN_MIRROR', 'http://deb.debian.org/debian')
KEYRING = '/usr/share/keyrings/debian-archive-keyring.gpg'
def get(url, dest):
tmp = dest + '.part'
with urllib.request.urlopen(url, timeout=120) as r, open(tmp, 'wb') as f:
while True:
b = r.read(1 << 20)
if not b:
break
f.write(b)
os.replace(tmp, dest)
def sha256(path):
h = hashlib.sha256()
with open(path, 'rb') as f:
for b in iter(lambda: f.read(1 << 20), b''):
h.update(b)
return h.hexdigest()
def fresh(path, max_age=86400):
return os.path.exists(path) and time.time() - os.path.getmtime(path) < max_age
def release(cache, suite):
d = f'{cache}/dists/{suite}'
os.makedirs(d, exist_ok=True)
inrel = f'{d}/InRelease'
if not fresh(inrel):
get(f'{MIRROR}/dists/{suite}/InRelease', inrel)
subprocess.run(['gpgv', '--keyring', KEYRING, inrel], check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
sums, on = {}, False
for line in open(inrel, encoding='utf-8'):
if line.startswith('SHA256:'):
on = True
continue
if on:
if not line.startswith(' '):
on = False
continue
h, size, name = line.split()
sums[name] = h
return d, sums
def stanzas(text):
cur = {}
last = None
for line in text.splitlines():
if not line.strip():
if cur:
yield cur
cur, last = {}, None
elif line[0] in ' \t':
if last:
cur[last] += '\n' + line
else:
k, _, v = line.partition(':')
cur[k] = v.strip()
last = k
if cur:
yield cur
def index(cache, suite, kind):
"""kind: 'binary-ARCH' or 'source'."""
d, sums = release(cache, suite)
leaf = 'Sources' if kind == 'source' else 'Packages'
name = f'main/{kind}/{leaf}.xz'
path = f'{d}/{kind}-{leaf}.xz'
if not (os.path.exists(path) and sha256(path) == sums[name]):
get(f'{MIRROR}/dists/{suite}/{name}', path)
if sha256(path) != sums[name]:
sys.exit(f'{name}: sha256 does not match InRelease')
return lzma.open(path, 'rt', encoding='utf-8', errors='replace').read()
def packages(cache, suite, arch):
pkgs, provides = {}, {}
for kind in (f'binary-{arch}', 'binary-all'):
try:
text = index(cache, suite, kind)
except KeyError:
continue # No-Support-for-Architecture-all: 'all' is in binary-ARCH
for p in stanzas(text):
pkgs.setdefault(p['Package'], p)
for name, p in pkgs.items():
for item in p.get('Provides', '').split(','):
v = item.strip().split(' ')[0] if item.strip() else ''
if v:
provides.setdefault(v, name)
return pkgs, provides
def clauses(p):
out = []
for field in ('Pre-Depends', 'Depends'):
for clause in p.get(field, '').split(','):
alts = [a.strip().split(' ')[0].split(':')[0] for a in clause.split('|') if a.strip()]
if alts:
out.append(alts)
return out
def closure(pkgs, provides, roots, exclude, deps=True):
chosen, order, todo = set(), [], list(roots)
def real(n):
return n if n in pkgs else provides.get(n)
while todo:
n = real(todo.pop(0))
if not n or n in chosen or n in exclude:
continue
chosen.add(n)
order.append(n)
if not deps:
continue
for alts in clauses(pkgs[n]):
if any(real(a) in chosen or a in exclude for a in alts):
continue
pick = next((real(a) for a in alts if real(a)), None)
if pick:
todo.append(pick)
return order
def fetch(cache, p):
pool = f'{cache}/pool'
os.makedirs(pool, exist_ok=True)
path = f'{pool}/{os.path.basename(p["Filename"])}'
if not (os.path.exists(path) and sha256(path) == p['SHA256']):
get(f'{MIRROR}/{p["Filename"]}', path)
if sha256(path) != p['SHA256']:
sys.exit(f'{path}: sha256 does not match Packages')
return path
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--arch', required=True)
ap.add_argument('--suite', default='trixie')
ap.add_argument('--cache', default=os.path.expanduser('~/gex-sandbox/scratch/CC/deb'))
ap.add_argument('--extract')
ap.add_argument('--no-deps', action='store_true')
ap.add_argument('--exclude', default='')
ap.add_argument('roots', nargs='+')
a = ap.parse_args()
pkgs, provides = packages(a.cache, a.suite, a.arch)
missing = [r for r in a.roots if r not in pkgs and r not in provides]
if missing:
sys.exit(f'not in {a.suite}/{a.arch}: {" ".join(missing)}')
exclude = set(x for x in a.exclude.split(',') if x)
for n in closure(pkgs, provides, a.roots, exclude, not a.no_deps):
p = pkgs[n]
path = fetch(a.cache, p)
print(n, p['Version'], p['Architecture'], path)
if a.extract:
os.makedirs(a.extract, exist_ok=True)
subprocess.run(['dpkg-deb', '-x', path, a.extract], check=True)
if __name__ == '__main__':
main()
tools/gexcc/debian/debsrc.py
Executable
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Fetch and unpack Debian source packages, verified, with our patches.
debsrc.py [--suite trixie] [--cache DIR] --into DIR SRCPKG[=VERSION]...
InRelease (gpgv, debian-archive-keyring) -> Sources.xz (SHA256 from
InRelease) -> the .dsc (SHA256 from Sources) -> the tarballs (dpkg-source
checks them against the .dsc). `dpkg-source -x` unpacks and applies
debian/patches; then tools/gexcc/debian/patches/SRCPKG/*.patch are applied
with `patch -p1` in name order. A VERSION pin that no longer matches the
index fails (a Debian point release changes the pin explicitly).
Prints `SRCPKG VERSION DIR` per package. An existing DIR/SRCPKG-VERSION is
left alone (delete it to re-unpack).
"""
import argparse, glob, os, subprocess, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import debfetch # noqa: E402
HERE = os.path.dirname(os.path.abspath(__file__))
def sources(cache, suite):
out = {}
for s in debfetch.stanzas(debfetch.index(cache, suite, 'source')):
# Several stanzas can share a name (rare); the last (newest) wins.
out[s['Package']] = s
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--suite', default='trixie')
ap.add_argument('--cache', default=os.path.expanduser('~/gex-sandbox/scratch/CC/deb'))
ap.add_argument('--into', required=True)
ap.add_argument('pkgs', nargs='+')
a = ap.parse_args()
idx = sources(a.cache, a.suite)
os.makedirs(a.into, exist_ok=True)
for spec in a.pkgs:
name, _, pin = spec.partition('=')
s = idx.get(name)
if not s:
sys.exit(f'{name}: no such source package in {a.suite}')
ver = s['Version']
if pin and pin != ver:
sys.exit(f'{name}: pinned {pin}, {a.suite} has {ver}')
upver = ver.split(':', 1)[-1]
d = os.path.join(a.into, f'{name}-{upver}')
if not os.path.isdir(d):
sums = {}
for line in s.get('Checksums-Sha256', '').splitlines():
parts = line.split()
if len(parts) == 3:
sums[parts[2]] = parts[0]
pool = os.path.join(a.cache, 'src', name)
os.makedirs(pool, exist_ok=True)
dsc = None
for fn, h in sums.items():
path = os.path.join(pool, fn)
if not (os.path.exists(path) and debfetch.sha256(path) == h):
debfetch.get(f'{debfetch.MIRROR}/{s["Directory"]}/{fn}', path)
if debfetch.sha256(path) != h:
sys.exit(f'{fn}: sha256 does not match Sources')
if fn.endswith('.dsc'):
dsc = path
# The .dsc's own signature is not checked (--no-check): its
# hash came from the signed index, which is the stronger chain.
subprocess.run(['dpkg-source', '--no-check', '--require-strong-checksums', '-x', dsc, d],
check=True, stdout=subprocess.DEVNULL)
for p in sorted(glob.glob(os.path.join(HERE, 'patches', name, '*.patch'))):
subprocess.run(['patch', '-p1', '-s', '-i', p], cwd=d, check=True)
print(name, ver, d)
if __name__ == '__main__':
main()
tools/gexcc/debian/mkdebs.py
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""Split a staging tree into Debian binary packages and build the .debs.
mkdebs.py --src SRCDIR --stage STAGE --out POOL SRCPKG VERSION
The split comes from binpkgs.txt (`SRCPKG BINPKG GLOB...`, globs relative to
the staging root, in order; a file goes to the first package whose glob
matches it; a source package with no lines there becomes one binary package
of its own name). Control fields come from the source's debian/control
stanza for that binary package: Section, Priority, Essential, Multi-Arch,
Description; the version is Debian's plus `+gex1`, the architecture
`musl-linux-wasm32`. Tools are `Multi-Arch: foreign` so that they satisfy
arm64 packages' dependencies by name (docs/gexcc.md §6.1a). Programs link
musl statically, so there are no library dependencies to record.
Prints `BINPKG VERSION PATH` per .deb.
"""
import argparse, fnmatch, hashlib, os, shutil, subprocess, sys, tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
ARCH = 'musl-linux-wasm32'
# development files: the rootfs runs programs, it does not build against
# these (static archives and headers stay in the gexcc sysroot)
DROP = ['usr/include/*', 'usr/lib/*.a', 'usr/lib/pkgconfig/*', 'usr/share/pkgconfig/*',
'usr/share/aclocal/*', 'usr/share/info/*', 'usr/lib/*.la', 'usr/share/man/man3/*']
SUFFIX = '+gex1'
def control_stanzas(path):
out, cur, last = [], {}, None
for line in open(path, encoding='utf-8'):
line = line.rstrip('\n')
if line.startswith('#'):
continue
if not line.strip():
if cur:
out.append(cur)
cur, last = {}, None
elif line[0] in ' \t' and last:
cur[last] += '\n' + line
else:
k, _, v = line.partition(':')
cur[k.strip()] = v.strip()
last = k.strip()
if cur:
out.append(cur)
return out
def split_rules(srcpkg):
rules = []
for line in open(os.path.join(HERE, 'binpkgs.txt')):
f = line.split()
if f and not f[0].startswith('#') and f[0] == srcpkg and f[0] != 'depends':
rules.append((f[1], f[2:]))
return rules or [(srcpkg, ['*'])]
def depends(binpkg):
out = []
for line in open(os.path.join(HERE, 'binpkgs.txt')):
f = line.split()
if len(f) > 2 and f[0] == 'depends' and f[1] == binpkg:
out += f[2:]
return ', '.join(out)
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--src', required=True)
ap.add_argument('--stage', required=True)
ap.add_argument('--out', required=True)
ap.add_argument('srcpkg')
ap.add_argument('version')
a = ap.parse_args()
stanzas = control_stanzas(os.path.join(a.src, 'debian', 'control'))
source = stanzas[0]
bins = {s['Package']: s for s in stanzas[1:] if 'Package' in s}
rules = split_rules(a.srcpkg)
owned = {b: [] for b, _ in rules}
for root, dirs, files in os.walk(a.stage):
for n in files + [d for d in dirs if os.path.islink(os.path.join(root, d))]:
rel = os.path.relpath(os.path.join(root, n), a.stage)
if any(fnmatch.fnmatch(rel, g) for g in DROP):
continue
for b, globs in rules:
if any(fnmatch.fnmatch(rel, g) for g in globs):
owned[b].append(rel)
break
version = a.version + SUFFIX
os.makedirs(a.out, exist_ok=True)
for b, rels in owned.items():
if b == '-':
continue
if not rels:
print(f'{b}: no files', file=sys.stderr)
continue
st = bins.get(b, {})
with tempfile.TemporaryDirectory() as t:
for rel in rels:
s, d = os.path.join(a.stage, rel), os.path.join(t, rel)
os.makedirs(os.path.dirname(d), exist_ok=True)
if os.path.islink(s):
os.symlink(os.readlink(s), d)
else:
shutil.copy2(s, d)
size = sum(os.path.getsize(os.path.join(t, r)) for r in rels
if not os.path.islink(os.path.join(t, r))) // 1024
os.makedirs(os.path.join(t, 'DEBIAN'))
md5 = []
for rel in sorted(rels):
p = os.path.join(t, rel)
if not os.path.islink(p):
md5.append(f'{hashlib.md5(open(p, "rb").read()).hexdigest()} {rel}')
open(os.path.join(t, 'DEBIAN', 'md5sums'), 'w').write('\n'.join(md5) + '\n')
ctl = [f'Package: {b}', f'Source: {a.srcpkg}' + (f' ({a.version})' if a.version else ''),
f'Version: {version}', f'Architecture: {ARCH}',
'Maintainer: gex native builds <gex@localhost>',
f'Original-Maintainer: {source.get("Maintainer", "unknown")}',
f'Installed-Size: {size}']
for k in ('Essential', 'Section', 'Priority'):
if k in st or k in source:
ctl.append(f'{k}: {st.get(k, source.get(k))}')
ma = st.get('Multi-Arch', 'foreign')
ctl.append(f'Multi-Arch: {"foreign" if ma in ("foreign", "allowed", "no") else ma}')
# relations from debian/control, without the substitution
# variables dh would have filled (library dependencies, which a
# static program does not have)
for k in ('Provides', 'Conflicts', 'Breaks', 'Replaces'):
v = ', '.join(x.strip() for x in st.get(k, '').replace('\n', ' ').split(',')
if x.strip() and '${' not in x)
if v:
ctl.append(f'{k}: {v}')
if depends(b):
ctl.append(f'Depends: {depends(b)}')
if 'Homepage' in source:
ctl.append(f'Homepage: {source["Homepage"]}')
desc = st.get('Description', f'{b} (native wasm32 build)')
ctl.append(f'Description: {desc}')
open(os.path.join(t, 'DEBIAN', 'control'), 'w').write('\n'.join(ctl) + '\n')
deb = os.path.join(a.out, f'{b}_{version.split(":", 1)[-1]}_{ARCH}.deb')
subprocess.run(['dpkg-deb', '--root-owner-group', '-Zxz', '--build', t, deb],
check=True, stdout=subprocess.DEVNULL)
print(b, version, deb)
if __name__ == '__main__':
main()
tools/gexcc/debian/mkrootfs.py
+400
View File
@@ -0,0 +1,400 @@
#!/usr/bin/env python3
"""Assemble the native Debian rootfs (docs/gexcc.md §6): Debian's arm64
system pieces (dpkg, apt, perl-base, libc6, ...), Debian's architecture-
independent data, and our musl-linux-wasm32 packages in place of Debian's
tools, with a dpkg database that says so and apt set up for both.
mkrootfs.py --pool POOL --out DIR [--tar FILE] [--roots PKG,...]
No root and no maintainer scripts: every .deb is unpacked (dpkg-deb
--fsys-tarfile) into DIR with /bin, /sbin and /lib as /usr's (merged /usr);
/var/lib/dpkg/status, info/*.list, *.md5sums and *.conffiles are written
from the packages; what the scripts would have done for this closure
(accounts, /etc/shells, alternatives, the CA bundle, /root's dot files) is
done here. Debian's packages are verified (debfetch.py: InRelease ->
Packages -> .deb) and cached.
Our packages (POOL, from mkdebs.py) stand in for Debian's by name: a
dependency an arm64 package has on `coreutils` or `awk` is met by ours
(Multi-Arch: foreign), and Debian's copy is not unpacked. apt: Debian's
repository for arm64 only, ours (Origin gex) as a file: repository at
/var/lib/gex-apt for musl-linux-wasm32, pinned so that apt never replaces
one of ours with Debian's (docs/gexcc.md §6.1a).
"""
import argparse, glob, gzip, hashlib, io, lzma, os, shutil, subprocess, sys, tarfile, tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import debfetch # noqa: E402
a_minimal = True
OURS_ARCH = 'musl-linux-wasm32'
NATIVE = 'arm64'
SUITE = 'trixie'
# The arm64 roots: what apt and dpkg need to run, and what the image needs
# to be a Debian system (the rest of Essential comes from our pool).
ROOTS = ['apt', 'dpkg', 'base-files', 'base-passwd', 'debian-archive-keyring', 'ncurses-base',
'libc-bin', 'perl-base', 'debconf', 'ca-certificates', 'usr-is-merged', 'sensible-utils',
'netbase', 'init-system-helpers', 'login.defs']
MIRROR_URI = 'http://deb.debian.org/debian'
SECURITY_URI = 'http://security.debian.org/debian-security'
def control(deb):
out = subprocess.run(['dpkg-deb', '-f', deb], check=True, capture_output=True, text=True).stdout
return next(debfetch.stanzas(out))
def control_member(deb, name):
"""A file of the control archive (md5sums, conffiles), or None."""
with tempfile.TemporaryDirectory() as t:
subprocess.run(['dpkg-deb', '-e', deb, t], check=True)
p = os.path.join(t, name)
return open(p, encoding='utf-8', errors='replace').read() if os.path.exists(p) else None
# As Debian's minimized images (and dpkg's path-exclude, which the image
# carries so apt installs behave the same): no documentation or
# translations; copyright files stay.
EXCLUDE = ['/usr/share/doc/*', '/usr/share/man/*', '/usr/share/info/*', '/usr/share/locale/*',
'/usr/share/lintian/*', '/usr/share/linda/*', '/usr/share/groff/*',
# glibc's iconv modules (20 MB): the Unicode and Western ones stay
# (UTF-8 and ISO-8859-1 are built into libc)
'/usr/lib/*/gconv/*.so']
INCLUDE = ['/usr/share/doc/*/copyright', '/usr/lib/*/gconv/UTF-*.so', '/usr/lib/*/gconv/UNICODE.so',
'/usr/lib/*/gconv/ISO8859-15.so', '/usr/lib/*/gconv/CP1252.so', '/usr/lib/*/gconv/libGB.so',
'/usr/lib/*/gconv/libJIS.so', '/usr/lib/*/gconv/EUC-JP.so', '/usr/lib/*/gconv/SJIS.so']
def excluded(path):
import fnmatch
return any(fnmatch.fnmatch(path, g) for g in EXCLUDE) and not any(fnmatch.fnmatch(path, g) for g in INCLUDE)
def unpack(deb, root):
"""Unpack the data archive; returns the paths it holds (dpkg's .list)."""
data = subprocess.run(['dpkg-deb', '--fsys-tarfile', deb], check=True, capture_output=True).stdout
paths = []
with tarfile.open(fileobj=io.BytesIO(data)) as tf:
for m in tf.getmembers():
rel = os.path.normpath(m.name)
if rel == '.':
paths.append('/.')
continue
if a_minimal and excluded('/' + rel) and not m.isdir():
continue
paths.append('/' + rel)
dest = os.path.join(root, real_rel(root, rel))
if m.isdir():
if not os.path.isdir(dest):
os.makedirs(dest, exist_ok=True)
os.chmod(dest, m.mode | 0o700)
continue
os.makedirs(os.path.dirname(dest), exist_ok=True)
if m.issym() and os.path.islink(dest) and os.readlink(dest) == m.linkname:
continue # base-files' /bin -> usr/bin, made already
if os.path.islink(dest) or (os.path.lexists(dest) and not os.path.isdir(dest)):
os.unlink(dest)
if m.issym():
os.symlink(m.linkname, dest)
elif m.islnk():
os.link(os.path.join(root, real_rel(root, os.path.normpath(m.linkname))), dest)
elif m.isfile():
with tf.extractfile(m) as src, open(dest, 'wb') as f:
shutil.copyfileobj(src, f)
os.chmod(dest, m.mode & 0o7777 | 0o600)
return paths
def real_rel(root, rel):
"""`rel` with merged-/usr aliases followed (bin -> usr/bin ...)."""
top, _, rest = rel.partition('/')
if rest and top in ('bin', 'sbin', 'lib') and os.path.islink(os.path.join(root, top)):
return os.path.join('usr', top, rest)
return rel
def md5(path):
return hashlib.md5(open(path, 'rb').read()).hexdigest()
def status_stanza(ctl, conffiles_md5):
keys = list(ctl)
out = [f'Package: {ctl["Package"]}', 'Status: install ok installed']
for k in keys:
if k in ('Package', 'Filename', 'Size', 'MD5sum', 'SHA256', 'SHA1', 'Description-md5'):
continue
out.append(f'{k}: {ctl[k]}')
if conffiles_md5:
out.append('Conffiles:')
out += [f' {p} {h}' for p, h in conffiles_md5]
return '\n'.join(out) + '\n'
def info_name(ctl):
"""dpkg's info file stem: `pkg:arch` for a Multi-Arch: same package."""
if ctl.get('Multi-Arch') == 'same':
return f'{ctl["Package"]}:{ctl["Architecture"]}'
return ctl['Package']
def write_repo(root, debs):
"""Our packages as an apt repository at /var/lib/gex-apt (Packages,
Release; the .debs themselves are installed already and not copied
unless --repo-debs)."""
repo = os.path.join(root, 'var/lib/gex-apt')
bindir = os.path.join(repo, f'dists/{SUITE}/main/binary-{OURS_ARCH}')
os.makedirs(bindir, exist_ok=True)
stanzas = []
for deb, ctl in debs:
name = ctl['Package']
fn = f'pool/main/{name[0]}/{name}/{os.path.basename(deb)}'
data = open(deb, 'rb').read()
s = [f'{k}: {v}' for k, v in ctl.items()]
s += [f'Filename: {fn}', f'Size: {len(data)}', f'MD5sum: {hashlib.md5(data).hexdigest()}',
f'SHA256: {hashlib.sha256(data).hexdigest()}']
stanzas.append('\n'.join(s) + '\n')
text = '\n'.join(stanzas).encode()
open(os.path.join(bindir, 'Packages'), 'wb').write(text)
open(os.path.join(bindir, 'Packages.xz'), 'wb').write(lzma.compress(text))
import email.utils
rel = [f'Origin: gex', 'Label: gex native', f'Suite: {SUITE}', f'Codename: {SUITE}',
f'Date: {email.utils.formatdate(usegmt=True)}',
f'Architectures: {OURS_ARCH}', 'Components: main',
'Description: Debian tools built from source with gexcc as native wasm32', 'SHA256:']
for leaf in ('Packages', 'Packages.xz'):
p = os.path.join(bindir, leaf)
rel.append(f' {hashlib.sha256(open(p, "rb").read()).hexdigest()} {os.path.getsize(p)} '
f'main/binary-{OURS_ARCH}/{leaf}')
open(os.path.join(repo, f'dists/{SUITE}/Release'), 'w').write('\n'.join(rel) + '\n')
return repo
def configure(root, ours, native_pkgs):
"""What maintainer scripts and debootstrap would have done."""
def w(rel, text, mode=0o644):
p = os.path.join(root, rel)
os.makedirs(os.path.dirname(p), exist_ok=True)
open(p, 'w').write(text)
os.chmod(p, mode)
# base-passwd
pw = os.path.join(root, 'usr/share/base-passwd')
shutil.copy(os.path.join(pw, 'passwd.master'), os.path.join(root, 'etc/passwd'))
shutil.copy(os.path.join(pw, 'group.master'), os.path.join(root, 'etc/group'))
users = [l.split(':')[0] for l in open(os.path.join(root, 'etc/passwd'))]
groups = [l.split(':')[0] for l in open(os.path.join(root, 'etc/group'))]
w('etc/shadow', ''.join(f'{u}:*:20000:0:99999:7:::\n' for u in users), 0o640)
w('etc/gshadow', ''.join(f'{g}:*::\n' for g in groups), 0o640)
# /etc/shells: debianutils' list plus every shells.d file (add-shell)
shells = open(os.path.join(root, 'usr/share/debianutils/shells')).read() \
if os.path.exists(os.path.join(root, 'usr/share/debianutils/shells')) else '# /etc/shells: valid login shells\n/bin/sh\n'
for f in sorted(glob.glob(os.path.join(root, 'usr/share/debianutils/shells.d/*'))):
shells += open(f).read()
seen = []
for l in shells.splitlines():
if l not in seen:
seen.append(l)
w('etc/shells', '\n'.join(seen) + '\n')
# base-files' postinst: /root's dot files, /etc/os-release is shipped
for n in ('profile', 'bashrc'):
src = os.path.join(root, f'usr/share/base-files/dot.{n}')
if os.path.exists(src):
os.makedirs(os.path.join(root, 'root'), exist_ok=True)
shutil.copy(src, os.path.join(root, f'root/.{n}'))
os.chmod(os.path.join(root, 'root'), 0o700)
# and its install_from_default: /etc/profile (PS1, PATH), /etc/motd
for n in ('profile', 'motd'):
src = os.path.join(root, f'usr/share/base-files/{n}')
if os.path.exists(src) and not os.path.lexists(os.path.join(root, f'etc/{n}')):
shutil.copy(src, os.path.join(root, f'etc/{n}'))
for d in ('tmp', 'var/tmp'):
os.makedirs(os.path.join(root, d), exist_ok=True)
os.chmod(os.path.join(root, d), 0o1777)
# ca-certificates: the configuration and the bundle update-ca-certificates writes
crts = sorted(glob.glob(os.path.join(root, 'usr/share/ca-certificates/mozilla/*.crt')))
w('etc/ca-certificates.conf', ''.join(f'mozilla/{os.path.basename(c)}\n' for c in crts))
os.makedirs(os.path.join(root, 'etc/ssl/certs'), exist_ok=True)
w('etc/ssl/certs/ca-certificates.crt', ''.join(open(c).read().rstrip('\n') + '\n' for c in crts))
# alternatives the postinsts register (the highest priority of what is here)
alts = {
'awk': [('/usr/bin/mawk', 5), ('/usr/bin/gawk', 10)],
'editor': [('/usr/bin/nano', 40), ('/usr/bin/vim.basic', 30)],
'vi': [('/usr/bin/vim.basic', 30)], 'vim': [('/usr/bin/vim.basic', 30)],
'view': [('/usr/bin/vim.basic', 30)], 'ex': [('/usr/bin/vim.basic', 30)],
'pager': [('/usr/bin/less', 77), ('/usr/bin/more', 50)],
}
links = {'awk': '/usr/bin/awk', 'editor': '/usr/bin/editor', 'vi': '/usr/bin/vi', 'vim': '/usr/bin/vim',
'view': '/usr/bin/view', 'ex': '/usr/bin/ex', 'pager': '/usr/bin/pager'}
os.makedirs(os.path.join(root, 'etc/alternatives'), exist_ok=True)
os.makedirs(os.path.join(root, 'var/lib/dpkg/alternatives'), exist_ok=True)
for name, choices in alts.items():
have = [(p, pr) for p, pr in choices if os.path.lexists(os.path.join(root, p.lstrip('/')))]
if not have:
continue
best = max(have, key=lambda c: c[1])[0]
alt = os.path.join(root, 'etc/alternatives', name)
link = os.path.join(root, links[name].lstrip('/'))
for p in (alt, link):
if os.path.lexists(p):
os.unlink(p)
os.symlink(best, alt)
os.symlink(f'/etc/alternatives/{name}', link)
state = ['auto', links[name], '']
for p, pr in have:
state += [p, str(pr)]
w(f'var/lib/dpkg/alternatives/{name}', '\n'.join(state) + '\n\n')
# the system's identity and name service
w('etc/hostname', 'gex\n')
w('etc/hosts', '127.0.0.1\tlocalhost\n127.0.1.1\tgex\n::1\t\tlocalhost ip6-localhost ip6-loopback\n')
w('etc/resolv.conf', 'nameserver 1.1.1.1\n')
w('etc/fstab', '# gex: no block devices\n')
if not os.path.exists(os.path.join(root, 'etc/nsswitch.conf')):
w('etc/nsswitch.conf', 'passwd: files\ngroup: files\nshadow: files\nhosts: files dns\n'
'networks: files\nprotocols: files\nservices: files\nethers: files\nrpc: files\n')
# apt: Debian for arm64, ours for musl-linux-wasm32, the pins
w('etc/apt/sources.list.d/debian.sources',
f'Types: deb\nURIs: {MIRROR_URI}\nSuites: {SUITE} {SUITE}-updates\nComponents: main\n'
f'Architectures: {NATIVE}\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n\n'
f'Types: deb\nURIs: {SECURITY_URI}\nSuites: {SUITE}-security\nComponents: main\n'
f'Architectures: {NATIVE}\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n')
w('etc/apt/sources.list.d/gex.sources',
f'Types: deb\nURIs: file:/var/lib/gex-apt\nSuites: {SUITE}\nComponents: main\n'
f'Architectures: {OURS_ARCH}\nTrusted: yes\n')
names = sorted({c['Package'] for _, c in ours})
w('etc/apt/preferences.d/gex-native',
'# Our native tools (musl-linux-wasm32) are never replaced by Debian\'s\n'
'# arm64 builds; libraries are not pinned (docs/gexcc.md §6.1a).\n'
f'Package: {" ".join(n + ":" + NATIVE for n in names)}\nPin: release *\nPin-Priority: -1\n\n'
'Package: *\nPin: release o=gex\nPin-Priority: 1001\n')
w('etc/apt/apt.conf.d/99gex',
'// gex: one user, no sandbox user to drop to\n'
'APT::Sandbox::User "root";\n'
'// no translations: apt-get update fetches the Packages indexes only\n'
'Acquire::Languages "none";\n'
'// the trixie arm64 index and ours, in one mapping (fork copies it: no\n'
'// copy-on-write in the guest, so no bigger than needed)\n'
'APT::Cache-Start "100663296";\n'
'Dpkg::Use-Pty "false";\n'
'APT::Install-Recommends "false";\n'
'// said here so apt does not fork dpkg to ask (two dpkg runs, 3 s of\n'
'// every apt-cache in the page)\n'
f'APT::Architecture "{NATIVE}";\n'
f'APT::Architectures {{ "{NATIVE}"; "{OURS_ARCH}"; }};\n')
for d in ('var/lib/apt/lists/partial', 'var/cache/apt/archives/partial', 'var/lib/dpkg/updates',
'var/lib/dpkg/triggers', 'var/log/apt', 'etc/apt/trusted.gpg.d', 'etc/apt/keyrings',
'etc/dpkg/dpkg.cfg.d', 'var/cache/debconf', 'var/lib/apt/periodic', 'proc', 'dev', 'run', 'sys',
'home', 'mnt', 'srv', 'opt', 'media', 'usr/local/bin'):
os.makedirs(os.path.join(root, d), exist_ok=True)
if a_minimal:
w('etc/dpkg/dpkg.cfg.d/excludes', '# no documentation or translations (mkrootfs.py)\n'
+ ''.join(f'path-exclude={g}\n' for g in EXCLUDE) + ''.join(f'path-include={g}\n' for g in INCLUDE))
w('var/lib/dpkg/arch', f'{NATIVE}\n{OURS_ARCH}\n')
for f in ('diversions', 'statoverride', 'available'):
w(f'var/lib/dpkg/{f}', '')
w('var/lib/dpkg/triggers/File', '')
w('var/lib/dpkg/triggers/Unincorp', '')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--pool', default=os.path.expanduser('~/gex-sandbox/scratch/CC/debian/pool'))
ap.add_argument('--cache', default=os.path.expanduser('~/gex-sandbox/scratch/CC/deb'))
ap.add_argument('--out', required=True)
ap.add_argument('--tar', help='also write the tree as a tar (owner root)')
ap.add_argument('--roots', default=','.join(ROOTS))
ap.add_argument('--extra', default='', help='more arm64 packages (and their closure)')
ap.add_argument('--exclude', default='', help='our packages to leave out of the image (still in the repo)')
ap.add_argument('--full', action='store_true', help='keep documentation and translations')
a = ap.parse_args()
global a_minimal
a_minimal = not a.full
ours = []
for deb in sorted(glob.glob(os.path.join(a.pool, f'*_{OURS_ARCH}.deb'))):
ours.append((deb, control(deb)))
excluded = set(filter(None, a.exclude.split(',')))
image_ours = [(d, c) for d, c in ours if c['Package'] not in excluded]
our_names = {c['Package'] for _, c in ours}
for _, c in ours:
for item in c.get('Provides', '').split(','):
if item.strip():
our_names.add(item.strip().split(' ')[0])
# (a package of ours left out of the image does not satisfy anything)
have_ours = {c['Package'] for _, c in image_ours}
for _, c in image_ours:
for item in c.get('Provides', '').split(','):
if item.strip():
have_ours.add(item.strip().split(' ')[0])
pkgs, provides = debfetch.packages(a.cache, SUITE, NATIVE)
roots = [r for r in a.roots.split(',') + a.extra.split(',') if r]
# what ours depend on from Debian (architecture-independent data)
for _, c in image_ours:
for alts in debfetch.clauses(c):
if not any(x in have_ours for x in alts):
roots.append(alts[0])
order = debfetch.closure(pkgs, provides, roots, exclude=our_names)
# A name ours provides is never Debian's in the image.
order = [n for n in order if n not in our_names]
unmet = []
have = set(order) | have_ours
for n in order:
for item in pkgs[n].get('Provides', '').split(','):
if item.strip():
have.add(item.strip().split(' ')[0])
for p in [pkgs[n] for n in order] + [c for _, c in image_ours]:
for alts in debfetch.clauses(p):
if not any(x in have for x in alts):
unmet.append(f'{p["Package"]} needs {" | ".join(alts)}')
if unmet:
sys.exit('the closure is not self-consistent:\n ' + '\n '.join(unmet))
native = [(debfetch.fetch(a.cache, pkgs[n]), pkgs[n]) for n in order]
print(f'{len(native)} Debian packages, {len(image_ours)} of ours', file=sys.stderr)
root = a.out
if os.path.exists(root):
subprocess.run(['chmod', '-R', 'u+rwX', root], check=False)
shutil.rmtree(root)
os.makedirs(os.path.join(root, 'usr'))
for d in ('bin', 'sbin', 'lib'):
os.makedirs(os.path.join(root, 'usr', d), exist_ok=True)
os.symlink(f'usr/{d}', os.path.join(root, d))
info = os.path.join(root, 'var/lib/dpkg/info')
os.makedirs(info)
# the multiarch database layout (`pkg:arch.*` for Multi-Arch: same)
open(os.path.join(info, 'format'), 'w').write('1\n')
status = []
for deb, idx in native + image_ours:
ctl = control(deb)
paths = unpack(deb, root)
stem = info_name(ctl)
open(os.path.join(info, f'{stem}.list'), 'w').write('\n'.join(paths) + '\n')
sums = control_member(deb, 'md5sums')
if sums:
open(os.path.join(info, f'{stem}.md5sums'), 'w').write(sums)
conf = control_member(deb, 'conffiles')
cmd5 = []
if conf:
open(os.path.join(info, f'{stem}.conffiles'), 'w').write(conf)
for p in conf.split():
f = os.path.join(root, real_rel(root, p.lstrip('/')))
if os.path.isfile(f):
cmd5.append((p, md5(f)))
status.append(status_stanza(ctl, cmd5))
open(os.path.join(root, 'var/lib/dpkg/status'), 'w').write('\n'.join(status))
configure(root, ours, native)
write_repo(root, ours)
size = sum(os.path.getsize(os.path.join(d, f)) for d, _, fs in os.walk(root) for f in fs
if not os.path.islink(os.path.join(d, f)))
print(f'{root}: {size / (1 << 20):.1f} MiB', file=sys.stderr)
if a.tar:
subprocess.run(['tar', '-C', root, '--owner=0', '--group=0', '--numeric-owner', '--sort=name',
'-cf', a.tar, '.'], check=True)
print(f'{a.tar}: {os.path.getsize(a.tar) / (1 << 20):.1f} MiB', file=sys.stderr)
if __name__ == '__main__':
main()
tools/gexcc/debian/packages.txt
+52
View File
@@ -0,0 +1,52 @@
# Debian trixie source packages built with gexcc (tools/gexcc/debian/build.sh).
# source-package version (pinned: a point release changes it here) tier
zlib 1:1.3.dfsg+really1.3.1-1 tier1
bzip2 1.0.8-6 tier1
xz-utils 5.8.1-1+deb13u1 tier1
libzstd 1.5.7+dfsg-1 tier1
libmd 1.1.0-2 tier1
ncurses 6.5+20250216-2 tier1
readline 8.2-6 tier1
pcre2 10.46-1~deb13u2 tier1
bash 5.2.37-2 tier1
dash 0.5.12-12 tier1
coreutils 9.7-3 tier1
grep 3.11-4 tier1
sed 4.9-2+deb13u1 tier1
gawk 1:5.2.1-2 tier1
findutils 4.10.0-3 tier1
diffutils 1:3.10-4 tier1
tar 1.35+dfsg-3.1 tier1
gzip 1.13-1+deb13u1 tier1
less 668-1 tier1
procps 2:4.0.4-9 tier1
debianutils 5.23.2 tier1
hostname 3.25 tier1
nano 8.4-1+deb13u1 tier1
vim 2:9.1.1230-2 tier1
file 1:5.46-5 tier1
util-linux 2.41.5-0+deb13u1 tier1
make-dfsg 4.4.1-2 tier2
sqlite3 3.46.1-7+deb13u2 tier2
lua5.4 5.4.7-1 tier2
jq 1.7.1-6+deb13u3 tier2
tree 2.2.1-1 tier2
patch 2.8-2 tier2
bc 1.07.1-4 tier2
ed 1.21.1-1 tier2
libevent 2.1.12-stable-10 tier2
tmux 3.5a-3 tier2
htop 3.4.1-5 tier2
expat 2.8.3-1~deb13u1 tier2
libffi 3.4.8-2 tier2
mpdecimal 4.0.1-5 tier2
openssl 3.5.7-1~deb13u2 tier2
curl 8.14.1-2+deb13u5 tier2
git 1:2.47.3-0+deb13u1 tier2
python3.13 3.13.5-2+deb13u5 tier2
perl 5.40.1-6+deb13u1 tier2
dpkg 1.22.22 tier2
apt 3.0.3 tier2
lz4 1.10.0-4 tier2
mawk 1.3.4.20250131-1 tier1
sysvinit 3.14-4 tier1
tools/gexcc/debian/recipes.sh
Executable
+271
View File
@@ -0,0 +1,271 @@
# Recipes for tools/gexcc/debian/build.sh. build_SRCPKG (dashes, dots and
# pluses as underscores) runs in the unpacked Debian source tree with `set -e`
# and installs into $DEST (that package's staging tree, a merged-/usr root).
# Libraries also install into $SYSROOT so later packages link them. Options
# follow debian/rules where they mean something here; what we lack (selinux,
# audit, systemd, nls) is turned off.
# autotools in cross mode: configure runs nothing it builds (config.site holds
# what the run-time checks found on gex itself)
at_configure() {
CONFIG_SITE=$GEXCC_PREFIX/share/gexcc/config.site ./configure --host=$T --build=x86_64-pc-linux-gnu \
--prefix=/usr --sysconfdir=/etc --localstatedir=/var --mandir=/usr/share/man --infodir=/usr/share/info \
--disable-dependency-tracking "$@"
}
# install into the staging tree and (for a library) the sysroot
both() { make "$@" DESTDIR=$DEST; make "$@" DESTDIR=$SYSROOT; }
# libtool archives point at /usr/lib: drop them so the driver finds .a/.so archives
nola() { find "$DEST" "$SYSROOT/usr/lib" -name '*.la' -delete 2>/dev/null || true; }
build_zlib() {
CC=gexcc AR=gexcc-ar RANLIB=gexcc-ranlib CHOST=$T ./configure --prefix=/usr
make -j$JOBS
both install
}
build_bzip2() {
make -j$JOBS CC=gexcc AR=gexcc-ar RANLIB=gexcc-ranlib libbz2.a bzip2 bzip2recover
for d in $DEST $SYSROOT; do
mkdir -p $d/usr/bin $d/usr/lib $d/usr/include
cp libbz2.a $d/usr/lib/; cp bzlib.h $d/usr/include/
done
cp bzip2 bzip2recover bzgrep bzmore bzdiff debian/bzexe $DEST/usr/bin/
ln -sf bzip2 $DEST/usr/bin/bunzip2; ln -sf bzip2 $DEST/usr/bin/bzcat
ln -sf bzgrep $DEST/usr/bin/bzegrep; ln -sf bzgrep $DEST/usr/bin/bzfgrep
ln -sf bzmore $DEST/usr/bin/bzless; ln -sf bzdiff $DEST/usr/bin/bzcmp
}
build_xz_utils() {
at_configure --disable-nls --disable-doc --enable-static --disable-shared
make -j$JOBS
both install; nola
}
build_libzstd() {
# Debian builds lib/ and programs/ with plain make
make -j$JOBS -C lib CC=gexcc AR=gexcc-ar libzstd.a
make -j$JOBS -C programs CC=gexcc AR=gexcc-ar HAVE_ZLIB=0 HAVE_LZMA=0 HAVE_LZ4=0 zstd
for d in $DEST $SYSROOT; do
make -C lib install-static install-includes install-pc DESTDIR=$d PREFIX=/usr LIBDIR=/usr/lib
done
mkdir -p $DEST/usr/bin
cp programs/zstd programs/zstdgrep programs/zstdless $DEST/usr/bin/
for n in unzstd zstdcat zstdmt; do ln -sf zstd $DEST/usr/bin/$n; done
}
build_libmd() {
at_configure --enable-static --disable-shared
make -j$JOBS
both install; nola
}
build_ncurses() {
# Debian: wide-character libraries, a separate libtinfo, terminfo from
# ncurses-base/ncurses-term (Architecture: all), so no database here
at_configure --without-cxx-binding --without-ada --without-tests --enable-widec \
--with-termlib --disable-stripping --without-shared --with-normal \
--with-default-terminfo-dir=/usr/share/terminfo \
--with-terminfo-dirs=/etc/terminfo:/lib/terminfo:/usr/share/terminfo \
--enable-pc-files --with-pkg-config-libdir=/usr/lib/pkgconfig \
--with-fallbacks=xterm,xterm-256color,vt100,linux,screen,tmux-256color
make -j$JOBS
both install.libs install.includes
make install.progs DESTDIR=$DEST
# the names configure scripts probe for (-ltinfo, -lncurses, -lcurses)
for d in $DEST $SYSROOT; do
for l in tinfo ncurses curses; do ln -sf lib${l}w.a $d/usr/lib/lib$l.a; done
ln -sf libncursesw.a $d/usr/lib/libcurses.a
# <ncursesw/ncurses.h> as Debian's libncurses-dev has it
ln -sfn . $d/usr/include/ncursesw
done
}
build_readline() {
at_configure --with-curses --disable-shared --enable-static
make -j$JOBS
both install-static install-headers
for d in $DEST $SYSROOT; do mkdir -p $d/usr/lib/pkgconfig; cp readline.pc $d/usr/lib/pkgconfig/; done
}
build_pcre2() {
at_configure --enable-pcre2-16 --enable-pcre2-32 --disable-jit --enable-static --disable-shared
make -j$JOBS
both install; nola
}
build_bash() {
# function pointer casts in unwind-protect (docs/gexcc.md §3); Debian's
# system-wide bashrc and logout files
export GEXCC_FPCAST=1
at_configure --without-bash-malloc --disable-nls --with-curses --enable-largefile \
CFLAGS="-O2 -Wno-implicit-function-declaration -DSYS_BASHRC='\"/etc/bash.bashrc\"' -DSYS_BASH_LOGOUT='\"/etc/bash.bash_logout\"' -DNON_INTERACTIVE_LOGIN_SHELLS"
make -j$JOBS
make install DESTDIR=$DEST
mkdir -p $DEST/etc/skel
cp debian/etc.bash.bashrc $DEST/etc/bash.bashrc
cp debian/skel.bashrc $DEST/etc/skel/.bashrc; cp debian/skel.profile $DEST/etc/skel/.profile
cp debian/skel.bash_logout $DEST/etc/skel/.bash_logout
ln -sf bash $DEST/usr/bin/rbash
mkdir -p $DEST/usr/share/debianutils/shells.d
printf '/bin/bash\n/bin/rbash\n/usr/bin/bash\n/usr/bin/rbash\n' > $DEST/usr/share/debianutils/shells.d/bash
rm -rf $DEST/usr/share/info $DEST/usr/share/doc/bash
}
build_dash() {
at_configure
make -j$JOBS
make install DESTDIR=$DEST
# trixie's dash ships /bin/sh itself
ln -sf dash $DEST/usr/bin/sh
mkdir -p $DEST/usr/share/debianutils/shells.d
printf '/bin/dash\n/usr/bin/dash\n' > $DEST/usr/share/debianutils/shells.d/dash
}
build_coreutils() {
# one binary, the commands as symlinks (each program would otherwise carry
# its own copy of libc and gnulib)
FORCE_UNSAFE_CONFIGURE=1 at_configure --disable-nls --without-selinux --without-libgmp \
--enable-single-binary=symlinks --disable-acl --disable-xattr --without-openssl \
--enable-install-program=arch --enable-no-install-program=kill,uptime,hostname
make -j$JOBS
make install DESTDIR=$DEST
# Debian: chroot in sbin
mkdir -p $DEST/usr/sbin; rm -f $DEST/usr/bin/chroot; ln -s ../bin/coreutils $DEST/usr/sbin/chroot
rm -rf $DEST/usr/share/info
}
build_grep() { at_configure --disable-nls; make -j$JOBS; make install DESTDIR=$DEST; cp debian/rgrep $DEST/usr/bin/rgrep; chmod 755 $DEST/usr/bin/rgrep; rm -rf $DEST/usr/share/info; }
build_sed() { at_configure --disable-nls --without-selinux --disable-acl; make -j$JOBS; make install DESTDIR=$DEST; rm -rf $DEST/usr/share/info; }
build_gawk() {
# Debian's (DFSG) source has no Texinfo manual sources: build without doc/,
# and without extension/ (gawk's dynamic extensions cannot load here)
at_configure --disable-nls --without-mpfr --with-readline
make -j$JOBS SUBDIRS="support ."
make install-exec SUBDIRS="support ." DESTDIR=$DEST
mkdir -p $DEST/usr/share/man/man1; cp doc/gawk.1 $DEST/usr/share/man/man1/
rm -rf $DEST/usr/include $DEST/usr/lib $DEST/usr/libexec
}
build_findutils() { at_configure --disable-nls --without-selinux; make -j$JOBS; make install DESTDIR=$DEST; rm -rf $DEST/usr/share/info; }
build_diffutils() { at_configure --disable-nls; make -j$JOBS; make install DESTDIR=$DEST; rm -rf $DEST/usr/share/info; }
build_tar() {
FORCE_UNSAFE_CONFIGURE=1 at_configure --disable-nls --without-selinux --disable-acl --without-xattrs \
--with-rmt=/usr/sbin/rmt
make -j$JOBS
make install DESTDIR=$DEST
rm -rf $DEST/usr/share/info $DEST/usr/libexec
}
build_gzip() { at_configure; make -j$JOBS; make install DESTDIR=$DEST; rm -rf $DEST/usr/share/info; }
build_less() {
CPPFLAGS="-I$SYSROOT/usr/include/ncursesw" at_configure --with-regex=pcre2 --with-secure=no
make -j$JOBS
make install DESTDIR=$DEST
cp debian/lesspipe $DEST/usr/bin/lesspipe 2>/dev/null || true
ln -sf lesspipe $DEST/usr/bin/lessfile
}
build_debianutils() {
# (po4a: translated man pages, which need po4a on the host; left out)
# dh autoreconfs: the source has configure.ac only
autoreconf -fi
at_configure
make -j$JOBS SUBDIRS=
make install SUBDIRS= DESTDIR=$DEST
}
build_hostname() {
make -j$JOBS CC=gexcc
mkdir -p $DEST/usr/bin
cp hostname $DEST/usr/bin/
for n in dnsdomainname domainname nisdomainname ypdomainname; do ln -sf hostname $DEST/usr/bin/$n; done
}
build_nano() {
# gnulib's strcasecmp check, as it runs on gex (musl's works)
export gl_cv_func_strcasecmp_works=yes
at_configure --disable-nls --enable-utf8 --disable-speller
make -j$JOBS
make install DESTDIR=$DEST
rm -rf $DEST/usr/share/info
mkdir -p $DEST/etc; cp doc/sample.nanorc $DEST/etc/nanorc 2>/dev/null || true
}
build_mawk() {
at_configure
make -j$JOBS
mkdir -p $DEST/usr/bin; cp mawk $DEST/usr/bin/
}
build_sysvinit() {
# sysvinit-utils only (Essential): killall5/pidof and fstab-decode
make -C src -j$JOBS CC=gexcc killall5 fstab-decode
mkdir -p $DEST/usr/bin $DEST/usr/sbin
cp src/killall5 $DEST/usr/sbin/; ln -sf ../sbin/killall5 $DEST/usr/bin/pidof
cp src/fstab-decode $DEST/usr/bin/
}
build_procps() {
# Debian: watch with 8-bit characters, no systemd/elogind
at_configure --disable-nls --without-systemd --without-elogind --disable-shared --enable-static \
--enable-watch8bit --disable-pidwait --disable-modern-top \
CPPFLAGS="-I$SYSROOT/usr/include/ncursesw"
make -j$JOBS
make install DESTDIR=$DEST
rm -rf $DEST/usr/include $DEST/usr/lib/pkgconfig $DEST/usr/lib/*.a $DEST/usr/share/doc
}
build_file() {
# cross-compiling the magic database needs a host `file` of the same
# version
rm -rf ../file-host; cp -a . ../file-host
(cd ../file-host && CONFIG_SITE= CC=cc ./configure --build=x86_64-pc-linux-gnu --disable-shared --disable-libseccomp -q && make -j$JOBS -C src >/dev/null)
at_configure --disable-shared --enable-static --disable-libseccomp --disable-bzlib --disable-xzlib --disable-zstdlib --disable-lzlib
make -j$JOBS FILE_COMPILE=$PWD/../file-host/src/file
# the database where Debian's libmagic-mgc (which ours depends on) has it
make -C src clean; make -j$JOBS -C src MAGIC=/usr/lib/file/magic
make install DESTDIR=$DEST FILE_COMPILE=$PWD/../file-host/src/file
rm -rf $DEST/usr/share/misc $DEST/usr/lib
rm -rf $DEST/usr/include $DEST/usr/lib/*.a $DEST/usr/lib/pkgconfig
rm -rf ../file-host
}
build_vim() {
# Debian's `vim` binary package (vim.basic); vim-common and vim-runtime
# are Architecture: all and come from Debian. Cross configure: the
# answers its run-time checks need (vim's own variables)
export vim_cv_toupper_broken=no vim_cv_terminfo=yes vim_cv_tgetent=zero vim_cv_getcwd_broken=no \
vim_cv_stat_ignores_slash=no vim_cv_memmove_handles_overlap=yes vim_cv_tty_group=world \
vim_cv_tty_mode=0620 vim_cv_bcopy_handles_overlap=yes vim_cv_memcpy_handles_overlap=yes \
ac_cv_small_wchar_t=no ac_cv_sizeof_int=4 ac_cv_sizeof_long=4 ac_cv_sizeof_time_t=8 ac_cv_sizeof_off_t=8
cd src
at_configure --with-features=huge --enable-gui=no --without-x --disable-gpm --disable-selinux \
--disable-nls --disable-canberra --disable-libsodium --with-tlib=tinfo \
--with-compiledby='gex (gexcc)' --with-modified-by='gex' \
--with-vim-name=vim.basic --with-ex-name=ex.vim --with-view-name=view.vim
make -j$JOBS
mkdir -p $DEST/usr/bin
cp vim.basic $DEST/usr/bin/vim.basic
}
build_util_linux() {
# dh autoreconfs (Debian's patches touch Makefile.am); no autopoint: nls is off
AUTOPOINT=true GTKDOCIZE=true autoreconf -fi
# Every program by default (most have no --enable switch, and
# --disable-all-programs turns those off for good), minus what needs
# mounts, logins, block devices or systemd. Debian splits the rest across
# util-linux, bsdutils, bsdextrautils (and uuid-runtime).
at_configure --bindir=/usr/bin --sbindir=/usr/sbin --libdir=/usr/lib \
--disable-nls --without-systemd --without-udev --without-python --without-selinux --without-audit \
--without-libmagic --disable-shared --enable-static --disable-makeinstall-chown --disable-makeinstall-setuid \
--disable-poman --disable-asciidoc --disable-libmount --disable-libfdisk --disable-liblastlog2 --disable-pam-lastlog2 \
--disable-login --disable-su --disable-runuser --disable-sulogin --disable-nologin --disable-chfn-chsh --disable-newgrp \
--disable-vipw --disable-lslogins --disable-utmpdump --disable-last --disable-mount --disable-losetup --disable-fsck \
--disable-swapon --disable-agetty --disable-hwclock --disable-cramfs --disable-bfs --disable-minix --disable-fdisks \
--disable-partx --disable-eject --disable-lsfd --disable-irqtop --disable-lsirq --disable-zramctl --disable-wipefs \
--disable-mkfs --disable-fstrim --disable-switch_root --disable-pivot_root --disable-unshare --disable-nsenter \
--disable-setpriv --disable-lsns --disable-fallocate --disable-uuidd --disable-wdctl \
CPPFLAGS="-I$SYSROOT/usr/include/ncursesw"
make -j$JOBS
make install DESTDIR=$DEST RANLIB=gexcc-ranlib
# kill is procps's in Debian, rename the perl one's; the sbin tools are
# for devices the guest does not have
ln -sf hexdump $DEST/usr/bin/hd
rm -rf $DEST/usr/include $DEST/usr/lib $DEST/usr/share/doc $DEST/usr/sbin $DEST/usr/bin/kill $DEST/usr/bin/rename \
$DEST/usr/share/man/man8 $DEST/usr/share/man/man1/kill.1 $DEST/usr/share/man/man1/rename.1
}
tools/gexcc/elemfix.py
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Point table entries back at the real functions after binaryen's
--fpcast-emu (gexcc FPCAST): every element `byn$fpcast-emu$NAME` for NAME in
the arguments becomes NAME again. gex calls those entries from the host with
their own signatures. Binary rewrite of the element section, by the name
section's function names.
elemfix.py MODULE.wasm NAME...
"""
import sys
path, names = sys.argv[1], sys.argv[2:]
w = bytearray(open(path, 'rb').read())
def leb(b, p):
r = s = 0
while True:
x = b[p]; p += 1
r |= (x & 0x7f) << s; s += 7
if not x & 0x80:
return r, p
def enc(v):
out = bytearray()
while True:
x = v & 0x7f; v >>= 7
if v:
out.append(x | 0x80)
else:
out.append(x)
return bytes(out)
# sections
secs = []
p = 8
while p < len(w):
sid = w[p]
n, q = leb(w, p + 1)
secs.append((sid, p, q, q + n))
p = q + n
# function names (custom "name" section, subsection 1)
fname = {}
for sid, s0, b0, e in secs:
if sid != 0:
continue
ln, q = leb(w, b0)
if bytes(w[q:q + ln]) != b'name':
continue
q += ln
while q < e:
kind = w[q]
sz, q = leb(w, q + 1)
end = q + sz
if kind == 1:
cnt, r = leb(w, q)
for _ in range(cnt):
idx, r = leb(w, r)
nl, r = leb(w, r)
fname[idx] = w[r:r + nl].decode('utf-8', 'replace')
r += nl
q = end
byname = {v: k for k, v in fname.items()}
remap = {}
for n in names:
t = byname.get('byn$fpcast-emu$' + n)
f = byname.get(n)
if t is not None and f is not None:
remap[t] = f
for sid, s0, b0, e in secs:
if sid != 9:
continue
out = bytearray()
cnt, q = leb(w, b0)
out += enc(cnt)
for _ in range(cnt):
flags, q2 = leb(w, q)
if flags != 0:
sys.exit('elemfix: element segment flags %d not handled' % flags)
# offset expression up to its end (0x0b)
r = q2
while w[r] != 0x0b:
r += 1
r += 1
out += w[q:r]
n, r = leb(w, r)
out += enc(n)
for _ in range(n):
fi, r = leb(w, r)
out += enc(remap.get(fi, fi))
q = r
new = w[:s0] + bytes([9]) + enc(len(out)) + out + w[e:]
open(path, 'wb').write(new)
if len(remap) != len(names):
print('elemfix: %d of %d entries restored' % (len(remap), len(names)), file=sys.stderr)
sys.exit(0)
sys.exit('elemfix: no element section')
tools/gexcc/gexcc
Executable
+478
View File
@@ -0,0 +1,478 @@
#!/usr/bin/env python3
"""gexcc / gexc++: a C/C++ compiler for gex's wasm32 Linux target (docs/gexcc.md).
Used like gcc or clang. Compiles with clang --target=wasm32-unknown-linux-musl
against the gexcc sysroot (musl with gex's wasm32 arch, libgex, compiler-rt).
A link writes a program: a position-independent wasm module importing gex's
memory and function table, in the ELF32 container gexc runs
(tools/wine-wasm/wasm-elf.py). The container also carries a host launcher
(hoststub.c): on an x86 Linux build machine the program runs itself under
gexc, so configure checks and test suites run it like a native binary.
Libraries are static. `-shared` writes an archive (with a `gexcc.deps` member
naming the libraries it was linked with), so a project that insists on
building libfoo.so still links: -lfoo then takes the archive, and its
dependencies follow it into the program.
Layout (tools/gexcc/install.sh PREFIX):
PREFIX/bin/gexcc, gexc++, wasm32-unknown-linux-musl-{cc,gcc,c++,g++,ar,...}
PREFIX/lib/gexcc/ flags.sh, wasm-elf.py, forkreach.py, hoststub.bin,
runner (the gexc path baked into containers)
PREFIX/sysroot/ include/, lib/ (musl, libgex, builtins), binaryen/,
usr/ (where cross-built libraries install)
Environment: GEXCC_SYSROOT overrides the sysroot; GEXCC_RUNNER the gexc a
container runs under (also read by the container itself at run time);
GEXCC_KEEP_WASM=1 keeps OUT.wasm beside a linked program; GEXCC_VERBOSE=1
prints every command."""
import atexit, os, re, shutil, subprocess, sys, tempfile
TRIPLE = 'wasm32-unknown-linux-musl'
me = os.path.realpath(__file__)
PREFIX = os.path.dirname(os.path.dirname(me))
LIBDIR = os.path.join(PREFIX, 'lib', 'gexcc')
SYSROOT = os.environ.get('GEXCC_SYSROOT') or os.path.join(PREFIX, 'sysroot')
VERBOSE = bool(os.environ.get('GEXCC_VERBOSE'))
name = os.path.basename(sys.argv[0])
CXX = name.endswith(('++', 'cxx')) or name.endswith('-c++')
def die(msg, code=1):
sys.stderr.write('gexcc: %s\n' % msg)
sys.exit(code)
def run(cmd):
if VERBOSE:
sys.stderr.write('+ %s\n' % ' '.join(cmd))
return subprocess.call(cmd)
if not os.path.isdir(LIBDIR):
die('no %s: run tools/gexcc/install.sh PREFIX and use PREFIX/bin/gexcc' % LIBDIR)
def read_flags():
flags = {}
for line in open(os.path.join(LIBDIR, 'flags.sh')):
if '=' in line and not line.startswith('#'):
k, v = line.strip().split('=', 1)
v = v.strip('"')
for kk, vv in flags.items():
v = v.replace('$' + kk, vv)
flags[k] = v
return flags
FLAGS = read_flags()
# Programs and their libraries lower setjmp onto the legacy exception
# instructions: binaryen's asyncify (fork) can instrument try/catch but not
# try_table. The link translates them to the standard ones (finish()).
BASE = [a.replace('-wasm-use-legacy-eh=false', '-wasm-use-legacy-eh=true') for a in FLAGS['WASM_CFLAGS'].split()]
# C++: exceptions and RTTI are on unless asked off (libc++abi over wasm EH).
# C++ exceptions on wasm EH: clang turns on the backend's lowering
# (-wasm-enable-eh) only for emscripten; without it every catch is dropped.
# C gets it too, for -fexceptions cleanups.
BASE += ['-mllvm', '-wasm-enable-eh']
if CXX:
BASE = [a for a in BASE if a != '-fno-exceptions'] + ['-fwasm-exceptions']
# Where cross-built libraries install (--prefix=/usr DESTDIR=SYSROOT); the
# libc's include/ is clang's own sysroot default.
INCS = []
for d in ('include/gexcc', 'usr/local/include', 'usr/include'):
if os.path.isdir(os.path.join(SYSROOT, d)):
INCS += ['-isystem', os.path.join(SYSROOT, d)]
# libc++'s headers come before the C library's (its <math.h> wraps musl's),
# unless asked not to (libc++'s own build, with its headers)
if CXX and os.path.isdir(os.path.join(SYSROOT, 'include', 'c++', 'v1')) and \
not any(a in ('-nostdinc++', '-nostdinc') for a in sys.argv[1:]):
INCS = ['-isystem', os.path.join(SYSROOT, 'include', 'c++', 'v1')] + INCS
CLANG = ['clang++' if CXX else 'clang'] + BASE + ['--sysroot=' + SYSROOT]
# Loop safepoints (poll/GexPoll.cpp): signals reach code that never makes a
# system call. GEXCC_POLL=0 or -fno-gex-poll leaves them out.
POLL = os.environ.get('GEXCC_POLL', '1') != '0' and '-fno-gex-poll' not in sys.argv[1:]
if POLL and os.path.exists(os.path.join(LIBDIR, 'GexPoll.so')):
CLANG += ['-fpass-plugin=' + os.path.join(LIBDIR, 'GexPoll.so')]
CLANG_TAIL = INCS
def expand_rsp(args):
out = []
for a in args:
if a.startswith('@') and os.path.isfile(a[1:]):
import shlex
out += expand_rsp(shlex.split(open(a[1:]).read()))
else:
out.append(a)
return out
# Function pointer cast emulation (finish below): -fgex-fpcast-emu or
# GEXCC_FPCAST=1, at link time.
FPCAST = os.environ.get('GEXCC_FPCAST', '0') != '0' or '-fgex-fpcast-emu' in sys.argv[1:]
args = [a for a in expand_rsp(sys.argv[1:]) if a not in ('-fno-gex-poll', '-fgex-poll', '-fgex-fpcast-emu')]
# --- queries -------------------------------------------------------------
if args == ['--print-sysroot'] or args == ['-print-sysroot']:
print(SYSROOT); sys.exit(0)
if args == ['--print-prefix']:
print(PREFIX); sys.exit(0)
if args == ['-dumpmachine']:
print(TRIPLE); sys.exit(0)
for a in args:
if a.startswith('-print-file-name='):
f = a.split('=', 1)[1]
for d in (SYSROOT + '/lib', SYSROOT + '/usr/lib'):
if os.path.exists(os.path.join(d, f)):
print(os.path.join(d, f)); sys.exit(0)
print(f); sys.exit(0)
if a == '-print-libgcc-file-name':
print(SYSROOT + '/lib/libclang_rt.builtins-wasm32.a'); sys.exit(0)
if a == '-print-search-dirs':
print('programs: =%s\nlibraries: =%s/lib:%s/usr/lib' % (os.path.dirname(me), SYSROOT, SYSROOT)); sys.exit(0)
if a.startswith('-print-prog-name='):
t = a.split('=', 1)[1]
p = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), TRIPLE + '-' + t)
print(p if os.path.exists(p) else t); sys.exit(0)
if a == '-print-multi-os-directory':
print('.'); sys.exit(0)
# --- compile only -----------------------------------------------------------
NOLINK = ('-c', '-S', '-E', '-M', '-MM', '-fsyntax-only', '--version', '-v', '-###', '--help')
if not args or any(a in NOLINK or a.startswith(('-print-', '-dump')) for a in args):
if args == ['-v'] or args == ['--version'] or not args:
os.execvp(CLANG[0], [CLANG[0], '--target=' + TRIPLE] + args)
os.execvp(CLANG[0], CLANG + args + CLANG_TAIL)
# --- link -----------------------------------------------------------------
LIBC_NAMES = {'c', 'm', 'pthread', 'dl', 'rt', 'util', 'crypt', 'xnet', 'resolv', 'gcc', 'gcc_s', 'gcc_eh',
'atomic', 'ssp', 'ssp_nonshared', 'unwind'}
SRC_EXT = ('.c', '.cc', '.cpp', '.cxx', '.c++', '.C', '.s', '.S', '.i', '.ii')
# Linker options with no meaning for a static wasm module, dropped quietly.
LD_DROP = re.compile(r'^(-rpath|-rpath-link|--rpath|-soname|-h|--version-script|-z|--as-needed|--no-as-needed|'
r'--no-undefined|-O\d|--hash-style|--build-id|--enable-new-dtags|--disable-new-dtags|'
r'-Bsymbolic|-Bsymbolic-functions|--start-group|--end-group|--sort-common|--warn-common|'
r'--sort-section|--dynamic-list|-E|--export-dynamic|--no-copy-dt-needed-entries|'
r'--copy-dt-needed-entries|-Bdynamic|-Bstatic|--eh-frame-hdr|--icf|-x|-X|--discard-all|'
r'--discard-locals|--fatal-warnings|--warn-shared-textrel|--no-relax|--relax|--push-state|'
r'--pop-state|-dynamic-linker|--dynamic-linker|--allow-shlib-undefined|--exclude-libs|'
r'--undefined-version|--no-undefined-version|-nostdlib)(=.*)?$')
LD_ARG = {'-rpath', '-rpath-link', '-soname', '-h', '--version-script', '-z', '--sort-section', '--dynamic-list',
'-dynamic-linker', '--dynamic-linker', '--exclude-libs'}
out = None
srcs, inputs, ld_flags, cflags, libdirs = [], [], [], [], []
shared = relocatable = nostdlib = nostartfiles = nodefaultlibs = False
xlang = None
ld_skip = 0 # a dropped linker option's argument still to come (-Wl,-rpath -Wl,DIR)
i = 0
while i < len(args):
a = args[i]
nxt = args[i + 1] if i + 1 < len(args) else None
if a == '-o':
out = nxt; i += 2; continue
if a.startswith('-o') and len(a) > 2:
out = a[2:]
elif a == '-shared':
shared = True
elif a == '-r':
relocatable = True
elif a == '-nostdlib':
nostdlib = True
elif a == '-nostartfiles':
nostartfiles = True
elif a in ('-nodefaultlibs', '-nolibc'):
nodefaultlibs = True
elif a.startswith('-Wl,') or a == '-Xlinker':
if a == '-Xlinker':
parts = [nxt]; i += 1
else:
parts = [x for x in a[4:].split(',') if x]
k = 0
while k < len(parts):
p = parts[k]
if ld_skip:
ld_skip -= 1; k += 1; continue
if p in LD_ARG:
if k + 1 == len(parts):
ld_skip = 1
k += 2; continue
if LD_DROP.match(p):
k += 1; continue
ld_flags.append(p); k += 1
# -Xlinker -rpath -Xlinker DIR: the second half arrives separately
elif a == '-L':
libdirs.append(nxt); i += 2; continue
elif a.startswith('-L'):
libdirs.append(a[2:])
elif a == '-l':
inputs.append('-l' + nxt); i += 2; continue
elif a.startswith('-l'):
inputs.append(a)
elif a == '-x':
xlang = nxt; cflags += [a, nxt]; i += 2; continue
elif a in ('-I', '-D', '-U', '-include', '-imacros', '-isystem', '-idirafter', '-iquote', '-MF', '-MT', '-MQ',
'-target', '--target', '-arch', '-Xclang', '-mllvm', '-iprefix', '-iwithprefix', '-isysroot'):
cflags += [a, nxt]; i += 2; continue
elif not a.startswith('-') and (a.endswith(SRC_EXT) or (xlang and xlang != 'none')):
srcs.append((a, xlang))
elif not a.startswith('-'):
inputs.append(a)
elif a in ('-pthread', '-pthreads', '-rdynamic', '-static', '-static-pie', '-pie', '-no-pie', '-fPIE', '-fpie',
'-fPIC', '-fpic', '-s', '-static-libgcc', '-shared-libgcc', '-static-libstdc++', '-symbolic',
'-fuse-ld=lld', '-Bstatic', '-Bdynamic', '-export-dynamic', '-Wl'):
pass
else:
cflags.append(a)
i += 1
tmp = tempfile.mkdtemp(prefix='gexcc-')
atexit.register(shutil.rmtree, tmp, True)
objs = []
for k, (s, lang) in enumerate(srcs):
o = os.path.join(tmp, '%d-%s.o' % (k, os.path.basename(s)))
pre = ['-x', lang] if lang else []
# clang's -x applies to what follows: name the language per source
cf = []
j = 0
while j < len(cflags):
if cflags[j] == '-x':
j += 2; continue
cf.append(cflags[j]); j += 1
r = run(CLANG + cf + pre + ['-c', s, '-o', o] + CLANG_TAIL)
if r:
sys.exit(r)
objs.append(o)
if not objs and not inputs:
# build systems ask the linker who it is (Meson: -Wl,--version)
if any(f in ('--version', '-v', '--help') for f in ld_flags):
os.execvp('wasm-ld', ['wasm-ld'] + [f for f in ld_flags if f in ('--version', '-v', '--help')][:1])
die('no input files')
if out is None:
out = 'a.out'
LIB = SYSROOT + '/lib'
SEARCH = libdirs + [SYSROOT + '/usr/local/lib', SYSROOT + '/usr/lib', LIB]
AR_MAGIC = b'!<arch>\n'
def is_archive(p):
try:
with open(p, 'rb') as f:
return f.read(8) == AR_MAGIC
except OSError:
return False
def resolve(lib):
"""-lNAME to a file: an archive libNAME.so (gexcc's -shared) or libNAME.a."""
for d in SEARCH:
for ext in ('.so', '.a'):
p = os.path.join(d, 'lib' + lib + ext)
if os.path.exists(p) and (ext == '.a' or is_archive(p)):
return os.path.abspath(p)
return None
def archive_deps(path):
"""The link inputs recorded in an archive's gexcc.deps.o member."""
try:
data = open(path, 'rb').read()
except OSError:
return []
if not data.startswith(AR_MAGIC):
return []
a = data.find(b'GEXCC-DEPS\n')
if a < 0:
return []
b = data.find(b'\nGEXCC-END', a)
return data[a + 11:b].decode().split('\n') if b > 0 else []
def expand(items, seen):
"""Link inputs with -l resolved and archives' recorded deps appended."""
res, missing = [], []
for it in items:
if not it:
continue
if it.startswith('-l'):
nm = it[2:]
p = resolve(nm)
if p is None:
if nm in LIBC_NAMES or (CXX and nm in ('stdc++', 'c++', 'c++abi', 'supc++')):
continue
missing.append(it)
continue
it = p
elif not it.startswith('-'):
it = os.path.abspath(it)
if it in seen:
continue
seen.add(it)
res.append(it)
if is_archive(it):
sub, miss = expand(archive_deps(it), seen)
res += sub
missing += miss
return res, missing
if relocatable:
ins, miss = expand(inputs, set())
sys.exit(run(['wasm-ld', '-r', '-o', out] + objs + ins + [f for f in ld_flags if not f.startswith('--export')]))
if shared:
# A static library under the shared name (see the module docstring).
deps = []
members = list(objs)
for it in inputs:
if it.startswith('-l'):
deps.append(it)
elif is_archive(it):
# an archive given by path: recorded, not copied (its members
# would collide with the same archive linked again later)
deps.append(os.path.abspath(it))
else:
members.append(it)
# the -L directories, so the program's link finds the -l names
deps = ['-L' + os.path.abspath(d) for d in libdirs] + deps
# the deps go in a custom section of an empty wasm object (wasm-ld
# reads every member; it never pulls this one, which defines nothing)
tmpd = os.path.join(tmp, 'deps')
os.makedirs(tmpd)
txt = os.path.join(tmpd, 'deps.txt')
open(txt, 'w').write('GEXCC-DEPS\n' + '\n'.join(deps) + '\nGEXCC-END')
dfile = os.path.join(tmpd, 'gexcc.deps.o')
r = run(['llvm-objcopy', '--add-section=gexcc.deps=' + txt, os.path.join(LIBDIR, 'empty.o'), dfile])
if r:
sys.exit(r)
if os.path.exists(out):
os.unlink(out)
# members by path: objects in different directories may share a name
r = run(['llvm-ar', 'crsP', out] + members + [dfile])
sys.exit(r)
# A program.
ins, missing = expand(inputs, set())
# -L from archives' deps: they only feed resolve(), already applied
ins = [x for x in ins if not x.startswith('-L')]
if missing:
# wasm-ld says which are missing, as ld would
ins += missing
crt = [] if (nostdlib or nostartfiles) else [LIB + '/crt1.o']
libs = []
if not (nostdlib or nodefaultlibs):
if CXX:
for l in ('c++', 'c++abi', 'unwind'):
if os.path.exists('%s/lib%s.a' % (LIB, l)):
libs.append('%s/lib%s.a' % (LIB, l))
# gex's support library first: its dlopen, __dlsym and vfork replace
# libc's; the setjmp runtime and the signal frames are only there.
libs += [LIB + '/libgex.a', LIB + '/libc.a', LIB + '/libclang_rt.builtins-wasm32.a']
# gexcc's runtime: the loop safepoint (rt/poll.c)
if os.path.exists(LIB + '/libgexcc.a'):
libs.insert(0, LIB + '/libgexcc.a')
# main(argc, argv, envp): see rt/main3.c
if not nostartfiles and not nostdlib:
cand = objs + [x for x in ins if not x.startswith('-') and not is_archive(x)]
if cand:
nm = subprocess.run(['llvm-nm', '-j', '--defined-only', '--extern-only'] + cand,
capture_output=True, text=True).stdout.split()
if 'main' in nm and '__main_argc_argv' not in nm and '__main_void' not in nm:
objs.append(os.path.join(LIBDIR, 'main3.o'))
wasm = os.path.join(tmp, 'out.wasm')
cmd = ['wasm-ld', '--experimental-pic', '--shared-memory', '--import-memory', '--max-memory=4294967296',
'-pie', '--no-entry', '-o', wasm] + crt + objs + ins + ld_flags + libs
r = run(cmd)
if r:
sys.exit(r)
def imports(path):
"""(module, name) of every function import of the module at path."""
w = open(path, 'rb').read()
def leb(p):
r = sh = 0
while True:
x = w[p]; p += 1
r |= (x & 0x7f) << sh; sh += 7
if not x & 0x80:
return r, p
def nm(p):
n, p = leb(p)
return w[p:p + n].decode(), p + n
res, p = [], 8
while p < len(w):
sid = w[p]; n, q = leb(p + 1); end = q + n
if sid == 2:
cnt, q = leb(q)
for _ in range(cnt):
m, q = nm(q); f, q = nm(q)
kind = w[q]; q += 1
if kind == 0:
_, q = leb(q); res.append((m, f))
elif kind == 1:
q += 1; fl, q = leb(q); _, q = leb(q)
if fl & 1: _, q = leb(q)
elif kind == 2:
fl, q = leb(q); _, q = leb(q)
if fl & 1: _, q = leb(q)
elif kind == 3:
q += 2
elif kind == 4:
q += 1; _, q = leb(q)
break
p = end
return res
# Functions above fork's driver (musl's src/process/wasm32/__gex_fork.c): the
# unwind for a fork stops at the driver, so they keep no state.
FORK_ROOTS = ['__gex_start', '__gex_start_c', '__gex_thread_start', '__gex_thread_c',
'__gex_fork_run', '__gex_fork_drive', '__gex_fork_child']
opt = os.path.join(SYSROOT, 'binaryen', 'bin', 'wasm-opt')
feats = ['--enable-threads', '--enable-bulk-memory', '--enable-mutable-globals', '--enable-sign-ext',
'--enable-nontrapping-float-to-int', '--enable-exception-handling', '--enable-reference-types']
if FPCAST:
# Calls through a function pointer of another type trap in wasm, where C
# on a real machine gets away with them (bash's unwind-protect cleanups,
# old K&R code). binaryen's --fpcast-emu makes every indirect call and
# every table entry one uniform signature. gex calls the two entries
# (__gex_start, __gex_thread_start) with their own signatures, so their
# table entries go back to the real functions afterwards.
r = run([opt, wasm, '-g', '--strip-dwarf'] + feats + ['--fpcast-emu', '-o', wasm])
if r:
die('wasm-opt --fpcast-emu failed')
r = run([sys.executable, os.path.join(LIBDIR, 'elemfix.py'), wasm, '__gex_start', '__gex_thread_start'])
if r:
die('elemfix.py failed after --fpcast-emu')
passes = []
if ('gex', 'fork_point') in imports(wasm):
passes += ['--asyncify', '[email protected]_point']
dis = os.path.join(os.path.dirname(opt), 'wasm-dis')
reach = subprocess.run([sys.executable, os.path.join(LIBDIR, 'forkreach.py'), dis, wasm, 'gex.fork_point'] + FORK_ROOTS,
capture_output=True, text=True)
if reach.returncode == 0:
lst = os.path.join(tmp, 'onlylist')
open(lst, 'w').write(reach.stdout.strip().replace(',', '\n') + '\n')
passes += ['--pass-arg=asyncify-onlylist@@' + lst]
else:
passes += ['--pass-arg=asyncify-removelist@' + ','.join(FORK_ROOTS)]
passes += ['-O2']
passes += ['--translate-to-exnref']
# names kept (-g): gex binds the entries by name. (DWARF dropped: binaryen
# cannot rewrite clang's version 5.)
r = run([opt, wasm, '-g', '--strip-dwarf'] + feats + passes + ['-o', wasm])
if r:
die('wasm-opt failed')
cont = ['--host-stub', os.path.join(LIBDIR, 'hoststub.bin')]
rf = os.path.join(LIBDIR, 'runner')
if os.path.exists(rf):
cont += ['--runner', open(rf).read().strip()]
r = run([sys.executable, os.path.join(LIBDIR, 'wasm-elf.py'), wasm, out] + cont)
if r:
sys.exit(r)
if os.environ.get('GEXCC_KEEP_WASM'):
shutil.copy(wasm, out + '.wasm')
tools/gexcc/gexcc-exec.in
+47
View File
@@ -0,0 +1,47 @@
#!/bin/sh
# gexcc-exec: gexc for gexcc programs. A container's host launcher
# (hoststub.c) execs this; so do gexcc-run, CMake's emulator and Meson's
# exe_wrapper. install.sh fills in @GEXC@ and @LIBDIR@.
#
# Machine-wide slots: every program started from the host is a whole gexc
# (wasmtime, large reservations, ~27 threads). A configure script or a
# `make check` on the host starts thousands; many at once took a 24-thread
# machine down (2026-10-07). So at most GEXCC_SLOTS (default 4) run at
# once, per user: each holds an flock on one slot file for its lifetime and
# the rest wait. Programs a gexc's guest execs stay inside that gexc and
# take no slot. GEXCC_SLOTS=0 turns the limit off.
#
# The native busybox is the guest's /bin/sh and coreutils, unless
# GEXCC_BUSYBOX is set (empty: none; a path: that busybox). GEXCC_GEXC
# picks another gexc.
[ "$1" = run ] && shift
G=${GEXCC_GEXC:-@GEXC@}
L=@LIBDIR@
slots=${GEXCC_SLOTS:-4}
if [ "$slots" -gt 0 ] 2>/dev/null && command -v flock >/dev/null; then
d=${GEXCC_SLOT_DIR:-${XDG_RUNTIME_DIR:-/tmp}/gexcc-slots-$(id -u)}
mkdir -p "$d" 2>/dev/null
got=
i=0
while [ $i -lt "$slots" ]; do
exec 9>>"$d/$i"
if flock -n 9; then got=1; break; fi
exec 9>&-
i=$((i + 1))
done
if [ -z "$got" ]; then
# all taken: wait on one (spread by pid); the lock goes with fd 9 into gexc
exec 9>>"$d/$(( $$ % slots ))"
flock 9
fi
fi
bb=${GEXCC_BUSYBOX-$L/busybox}
if [ -n "$bb" ] && [ -f "$bb" ]; then
if [ "$bb" = "$L/busybox" ] && [ -z "${GEX_BUSYBOX_APPLETS:-}" ] && [ -f "$L/busybox.applets" ]; then
GEX_BUSYBOX_APPLETS=$(cat "$L/busybox.applets"); export GEX_BUSYBOX_APPLETS
fi
exec "$G" run --busybox "$bb" "$@"
fi
exec "$G" run "$@"
Loaded 100 of 117 files, more files were not shown because too many files have changed in this diff. Show more