mirror of
https://github.com/HeyPuter/gex.git
synced 2026-11-13 20:12:43 -10:00
page: clock calls without the kernel lock, and no wake after calls that change nothing
Undertale in the page (W3) spent about 41 ms of each 87 ms game-thread frame in KLock::take, most of it in clock_gettime and getrusage: Wine's NtDelayExecution yields with getrusage, sched_yield, getrusage, and every syscall takes the host lock several times against the process's ~25 idle Wine Workers, which every syscall's k_wake had just woken. - gex_kernel::syscalls::clock_call answers clock_gettime, gettimeofday, getrusage and sched_yield from the context alone (i386, x86-64 and canonical numbering), declining what it cannot answer exactly as the kernel would. - gex_syscall serves those calls first, lock-free, with the process's ABI and start time mirrored in atomics by each full call. It stands down while a signal may be pending (k_signal_seq moved since this thread's last full call) and for every 64th call in a row, so signal delivery at a syscall boundary is kept. - The no-wake list for side-effect-free calls (wakes_nobody) now matches the kernel's canonical number. The raw one is i386 numbering in a W3 process, so the list matched nothing useful there. - sched_yield is a pure yield. Test: clock_call_answers_as_the_kernel_does (aarch64 and x86). W3 page echo leg ok (49 s). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_ea9f71488b6e448da93d6c2dabf4879c
This commit is contained in:
velzieandClaude Opus 5.5 committed
1 parent
5dff5591b2
commit
3d9814ec3b
2 files changed
+332
-4
No files matched your search
@@ -812,6 +812,144 @@ fn monotonic_from((secs, nanos): (u64, u32)) -> (u64, u32) {
|
||||
(secs.saturating_sub(base), nanos)
|
||||
}
|
||||
|
||||
/// Answer one of the calls that only read the clock -- `clock_gettime`,
|
||||
/// `gettimeofday`, `getrusage`, `sched_yield` -- straight from the context at
|
||||
/// `ctx_addr`, without a [`Kernel`], and say whether it was one.
|
||||
///
|
||||
/// A host takes its kernel lock for every syscall, and Wine asks the time
|
||||
/// several times per sleep: Undertale's game thread made ~2,000 of these
|
||||
/// calls a second in the page and spent about 25 ms of each 87 ms frame
|
||||
/// queueing for the lock behind the process's other Workers. None of them
|
||||
/// needs the kernel: the answer is the clock, and `started` (the process's
|
||||
/// [`Kernel::started`], which the processor-time clocks count from).
|
||||
///
|
||||
/// Returns false, writing nothing that matters, for any other call, for a
|
||||
/// processor-time question before `started` is known (0), and for a buffer
|
||||
/// the guest cannot be written at: the host then makes the call as usual, so
|
||||
/// the answer and any `EFAULT` are the kernel's. `compat` is
|
||||
/// [`Kernel::compat`] (the i386 numbering, number in `eax`, 32-bit layouts).
|
||||
/// The result goes where [`set_result`] puts it. Signal delivery, which the
|
||||
/// kernel does at every syscall boundary, is the host's to arrange: it falls
|
||||
/// back to the full call whenever a signal may be pending.
|
||||
pub fn clock_call(
|
||||
m: &mut GuestMem<'_>,
|
||||
ctx_addr: u64,
|
||||
compat: bool,
|
||||
clock: ClockFn,
|
||||
started: u128,
|
||||
) -> bool {
|
||||
if gex_abi::PPC {
|
||||
return false;
|
||||
}
|
||||
let (raw, a0, a1) = if compat {
|
||||
let r = |slot: usize| m.read_u64(ctx_addr + slot as u64).unwrap_or(!0) & 0xffff_ffff;
|
||||
(
|
||||
r(ctx::x(0)),
|
||||
r(crate::i386::ARG_SLOTS[0]),
|
||||
r(crate::i386::ARG_SLOTS[1]),
|
||||
)
|
||||
} else {
|
||||
let r = |slot: usize| m.read_u64(ctx_addr + slot as u64).unwrap_or(!0);
|
||||
(r(ctx::SYSCALL_NR), r(ctx::SYSCALL_ARGS[0]), r(ctx::SYSCALL_ARGS[1]))
|
||||
};
|
||||
enum Call {
|
||||
// (clock id, buffer, 32-bit timespec)
|
||||
Clock(u64, u64, bool),
|
||||
// (buffer, 32-bit timeval)
|
||||
Tod(u64, bool),
|
||||
Rusage(u64),
|
||||
Yield,
|
||||
}
|
||||
let call = if compat {
|
||||
use crate::i386::raw as x;
|
||||
match raw {
|
||||
x::CLOCK_GETTIME => Call::Clock(a0, a1, true),
|
||||
x::CLOCK_GETTIME64 => Call::Clock(a0, a1, false),
|
||||
x::GETTIMEOFDAY => Call::Tod(a0, true),
|
||||
x::GETRUSAGE => Call::Rusage(a1),
|
||||
x::SCHED_YIELD => Call::Yield,
|
||||
_ => return false,
|
||||
}
|
||||
} else if gex_abi::X86 {
|
||||
// x86-64's own numbers: these four pass through `x86::translate`
|
||||
// unchanged but for the number.
|
||||
match raw {
|
||||
228 => Call::Clock(a0, a1, false),
|
||||
96 => Call::Tod(a0, false),
|
||||
98 => Call::Rusage(a1),
|
||||
24 => Call::Yield,
|
||||
_ => return false,
|
||||
}
|
||||
} else {
|
||||
match raw {
|
||||
nr::CLOCK_GETTIME => Call::Clock(a0, a1, false),
|
||||
nr::GETTIMEOFDAY => Call::Tod(a0, false),
|
||||
nr::GETRUSAGE => Call::Rusage(a1),
|
||||
nr::SCHED_YIELD => Call::Yield,
|
||||
_ => return false,
|
||||
}
|
||||
};
|
||||
let cpu = || {
|
||||
let (s, n) = clock();
|
||||
(s as u128 * 1_000_000_000 + n as u128).saturating_sub(started)
|
||||
};
|
||||
let pair = |m: &mut GuestMem<'_>, at: u64, hi: u64, lo: u64, w32: bool| {
|
||||
if w32 {
|
||||
m.write_u32(at, hi as u32).and_then(|_| m.write_u32(at + 4, lo as u32))
|
||||
} else {
|
||||
m.write_u64(at, hi).and_then(|_| m.write_u64(at + 8, lo))
|
||||
}
|
||||
};
|
||||
let ok = match call {
|
||||
Call::Clock(clk, buf, w32) => {
|
||||
// As `sys_clock_gettime`, which a null buffer faults in.
|
||||
if buf == 0 || (matches!(clk, 2 | 3) && started == 0) {
|
||||
return false;
|
||||
}
|
||||
let (secs, nanos) = match clk {
|
||||
2 | 3 => {
|
||||
let t = cpu();
|
||||
((t / 1_000_000_000) as u64, (t % 1_000_000_000) as u32)
|
||||
}
|
||||
c if is_monotonic_clock(c) => monotonic_from(clock()),
|
||||
_ => clock(),
|
||||
};
|
||||
pair(m, buf, secs, nanos as u64, w32).is_ok()
|
||||
}
|
||||
Call::Tod(buf, w32) => {
|
||||
if buf == 0 {
|
||||
true
|
||||
} else {
|
||||
let (secs, nanos) = clock();
|
||||
pair(m, buf, secs, (nanos / 1000) as u64, w32).is_ok()
|
||||
}
|
||||
}
|
||||
Call::Rusage(buf) => {
|
||||
if started == 0 {
|
||||
return false;
|
||||
}
|
||||
if buf == 0 {
|
||||
true
|
||||
} else {
|
||||
let ns = cpu();
|
||||
let (s, us) = ((ns / 1_000_000_000) as u64, (ns % 1_000_000_000 / 1000) as u64);
|
||||
let size = if compat { 72 } else { 144 };
|
||||
m.zero(buf, size).is_ok() && pair(m, buf, s, us, compat).is_ok()
|
||||
}
|
||||
}
|
||||
// A pure yield: the caller's Worker is its own OS thread, and the
|
||||
// browser schedules those without being asked.
|
||||
Call::Yield => true,
|
||||
};
|
||||
if !ok {
|
||||
return false;
|
||||
}
|
||||
if gex_abi::X86 {
|
||||
let _ = m.write_u64(ctx_addr + ctx::LAST_SYSCALL as u64, raw);
|
||||
}
|
||||
set_result(m, ctx_addr, 0).is_ok()
|
||||
}
|
||||
|
||||
pub fn system_clock() -> (u64, u32) {
|
||||
#[cfg(not(target_family = "wasm"))]
|
||||
{
|
||||
@@ -7738,6 +7876,80 @@ mod tests {
|
||||
assert_eq!(k.unix_of(pfd).err(), Some(Errno::ENOTSOCK));
|
||||
}
|
||||
|
||||
/// `clock_call`, the lock-free answer, writes what the full call writes,
|
||||
/// in both numberings and layouts, and declines what it cannot answer.
|
||||
#[test]
|
||||
fn clock_call_answers_as_the_kernel_does() {
|
||||
fn fixed() -> (u64, u32) {
|
||||
(1_700_000_000, 123_456_789)
|
||||
}
|
||||
if gex_abi::PPC {
|
||||
return;
|
||||
}
|
||||
let (mut mem, mut k) = setup();
|
||||
k.clock = fixed;
|
||||
let buf = 0x3000u64;
|
||||
let at = k.ctx_addr;
|
||||
// Native numbering: each call made both ways into a cleared buffer.
|
||||
for (n, args) in [
|
||||
(nr::CLOCK_GETTIME, [0u64, buf]),
|
||||
(nr::CLOCK_GETTIME, [1, buf]),
|
||||
(nr::CLOCK_GETTIME, [3, buf]),
|
||||
(nr::GETTIMEOFDAY, [buf, 0]),
|
||||
(nr::GETRUSAGE, [0, buf]),
|
||||
(nr::SCHED_YIELD, [0, 0]),
|
||||
] {
|
||||
let mut m = GuestMem::new(&mut mem);
|
||||
m.zero(buf, 160).unwrap();
|
||||
set_args(&mut m, at, n, &args);
|
||||
k.syscall(&mut m, 0);
|
||||
let want = m.read(buf, 160).unwrap().to_vec();
|
||||
let want_ret = ret(&m, at);
|
||||
m.zero(buf, 160).unwrap();
|
||||
m.write_u64(at + ctx::x(0) as u64, 0x5555).unwrap();
|
||||
set_args(&mut m, at, n, &args);
|
||||
assert!(clock_call(&mut m, at, false, fixed, k.started), "call {n}");
|
||||
assert_eq!(m.read(buf, 160).unwrap(), &want[..], "call {n} {args:?}");
|
||||
assert_eq!(ret(&m, at), want_ret, "call {n}");
|
||||
}
|
||||
// Declined: another call, a null clock buffer, the processor clock
|
||||
// before the process's start is known.
|
||||
let mut m = GuestMem::new(&mut mem);
|
||||
set_args(&mut m, at, nr::GETPID, &[]);
|
||||
assert!(!clock_call(&mut m, at, false, fixed, k.started));
|
||||
set_args(&mut m, at, nr::CLOCK_GETTIME, &[0, 0]);
|
||||
assert!(!clock_call(&mut m, at, false, fixed, k.started));
|
||||
set_args(&mut m, at, nr::GETRUSAGE, &[0, buf]);
|
||||
assert!(!clock_call(&mut m, at, false, fixed, 0));
|
||||
// i386: the number in eax, arguments in ebx/ecx, 32-bit layouts.
|
||||
let set32 = |m: &mut GuestMem<'_>, n: u64, a: [u64; 2]| {
|
||||
m.write_u64(at + ctx::x(0) as u64, n).unwrap();
|
||||
m.write_u64(at + crate::i386::ARG_SLOTS[0] as u64, a[0]).unwrap();
|
||||
m.write_u64(at + crate::i386::ARG_SLOTS[1] as u64, a[1]).unwrap();
|
||||
};
|
||||
let started = 1_699_999_990u128 * 1_000_000_000;
|
||||
set32(&mut m, crate::i386::raw::CLOCK_GETTIME, [0, buf]);
|
||||
assert!(clock_call(&mut m, at, true, fixed, started));
|
||||
assert_eq!(m.read_u32(buf).unwrap(), 1_700_000_000);
|
||||
assert_eq!(m.read_u32(buf + 4).unwrap(), 123_456_789);
|
||||
set32(&mut m, crate::i386::raw::GETTIMEOFDAY, [buf, 0]);
|
||||
assert!(clock_call(&mut m, at, true, fixed, started));
|
||||
assert_eq!(m.read_u32(buf + 4).unwrap(), 123_456);
|
||||
m.write_u64(buf + 64, !0).unwrap();
|
||||
m.write_u64(buf + 72, !0).unwrap();
|
||||
set32(&mut m, crate::i386::raw::GETRUSAGE, [0, buf]);
|
||||
assert!(clock_call(&mut m, at, true, fixed, started));
|
||||
assert_eq!(m.read_u32(buf).unwrap(), 10);
|
||||
assert_eq!(m.read_u32(buf + 4).unwrap(), 123_456);
|
||||
assert_eq!(m.read_u64(buf + 64).unwrap(), 0, "the 72-byte i386 rusage is cleared");
|
||||
assert_eq!(m.read_u64(buf + 72).unwrap(), !0, "and nothing past it");
|
||||
set32(&mut m, crate::i386::raw::SCHED_YIELD, [0, 0]);
|
||||
assert!(clock_call(&mut m, at, true, fixed, started));
|
||||
assert_eq!(m.read_u64(at + ctx::x(0) as u64).unwrap(), 0);
|
||||
set32(&mut m, crate::i386::raw::GETPID, [0, 0]);
|
||||
assert!(!clock_call(&mut m, at, true, fixed, started));
|
||||
}
|
||||
|
||||
/// What `clock_nanosleep` turns into: a sleep of the time left for an
|
||||
/// absolute deadline (none if it has passed), EINVAL for a malformed
|
||||
/// request instead of a sleep.
|
||||
|
||||
+120
-4
@@ -6226,6 +6226,10 @@ pub extern "C" fn gex_syscall(ctx: i32, pc: i64) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The clock calls, answered without the kernel lock (see `clock_fast_path`).
|
||||
if !IN_PROVIDER.with(|c| c.get()) && clock_fast_path(ctx) {
|
||||
return;
|
||||
}
|
||||
fjit::watch_check(pc as u64, "syscall");
|
||||
// A guest thread in a syscall counts as stopped for a provider's
|
||||
// collection (cgen/libprov.rs); it parks on its way back if one runs.
|
||||
@@ -6235,8 +6239,9 @@ pub extern "C" fn gex_syscall(ctx: i32, pc: i64) {
|
||||
cgen::libprov::syscall_enter(tracked_tid);
|
||||
}
|
||||
// Which syscall is running, so the allocator can name who was asking (wasm
|
||||
// has no backtrace). See `HEAP_BY_CALL`.
|
||||
{
|
||||
// has no backtrace). See `HEAP_BY_CALL`. Kept here too: `SYSCALL_NR` is
|
||||
// shared by every Worker of the process.
|
||||
let this_nr = {
|
||||
let gm = GuestMem::new(guest_bytes());
|
||||
let base = with_host(|h| h.threads[cur()].kstate.ctx_addr);
|
||||
let n = gm.read_u64(base + ctx::SYSCALL_NR as u64).unwrap_or(u64::MAX);
|
||||
@@ -6267,7 +6272,8 @@ pub extern "C" fn gex_syscall(ctx: i32, pc: i64) {
|
||||
} else {
|
||||
SELECT_ARGS[0].store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
n
|
||||
};
|
||||
if tracing() {
|
||||
let gm = GuestMem::new(guest_bytes());
|
||||
let base = with_host(|h| h.threads[cur()].kstate.ctx_addr);
|
||||
@@ -6522,7 +6528,18 @@ pub extern "C" fn gex_syscall(ctx: i32, pc: i64) {
|
||||
}
|
||||
}
|
||||
let slot = sys_note(nr, a0, a1);
|
||||
// The signals this call will see, read before it takes them: a fast
|
||||
// clock call after it is safe only while no newer one was sent.
|
||||
let sig_seen = if unsafe { k_present() } != 0 { unsafe { k_signal_seq() } } else { 0 };
|
||||
let outcome = h.kernel.syscall(&mut gm, pc as u64);
|
||||
FAST_ABI.store(
|
||||
if h.kernel.compat { 1 } else { 2 },
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
FAST_STARTED.store(h.kernel.started as u64, std::sync::atomic::Ordering::Relaxed);
|
||||
FAST_SIG.with(|c| c.set(sig_seen));
|
||||
FAST_RUN.with(|c| c.set(0));
|
||||
CANON_NR.with(|c| c.set(h.kernel.syscall_nr));
|
||||
let ret = gm.read_u64(kstate.ctx_addr + ctx::x(0) as u64).unwrap_or(0);
|
||||
sys_ret(slot, ret);
|
||||
// A process raising SIGABRT (`abort()`, a fortify/stack-check trap, a
|
||||
@@ -6969,7 +6986,11 @@ pub extern "C" fn gex_syscall(ctx: i32, pc: i64) {
|
||||
// every syscall, not a hand-picked list: the list is long and not obviously
|
||||
// closed, and a missed entry is a process that sleeps through its wake. The
|
||||
// notify costs nothing when nobody is parked.
|
||||
if unsafe { k_present() } != 0 {
|
||||
// The kernel's own (canonical) number: `this_nr` is the raw one, which
|
||||
// for an i386-ABI process -- W3's Wine -- is in i386 numbering.
|
||||
let _ = this_nr;
|
||||
let canon = CANON_NR.with(|c| c.replace(u64::MAX));
|
||||
if unsafe { k_present() } != 0 && !wakes_nobody(canon) {
|
||||
unsafe { k_wake() };
|
||||
}
|
||||
// Objects the guest just mapped, recompiled in bulk before it can branch in.
|
||||
@@ -7152,6 +7173,101 @@ const LOW_END: u64 = gex_abi::DISPATCH_DIR + gex_abi::DISPATCH_MAX_PAGES * 4;
|
||||
// parent's regions, `wait4`/`kill`/`exit` route through the kernel's table, and
|
||||
// each process is a Worker the engine schedules.
|
||||
|
||||
/// Whether syscall `nr` (canonical) can never be what another Worker waits
|
||||
/// for: it reads a clock or an identity, or changes only the caller's own
|
||||
/// state. The wake after every syscall wakes every Worker parked on the global
|
||||
/// counter -- a W3 game's idle Wine threads, ~25 of them -- and each takes the
|
||||
/// host lock to re-check. A game thread that reads the clock and yields
|
||||
/// thousands of times a second (Wine's NtYieldExecution is getrusage,
|
||||
/// sched_yield, getrusage) kept them all cycling, and the game thread then
|
||||
/// spent ~40 ms of every Undertale frame in KLock::take. A list of what
|
||||
/// cannot wake, not of what can: a call missing here costs a wake, never a
|
||||
/// lost one.
|
||||
fn wakes_nobody(nr: u64) -> bool {
|
||||
use gex_kernel::syscalls::nr;
|
||||
matches!(
|
||||
nr,
|
||||
nr::CLOCK_GETTIME
|
||||
| nr::CLOCK_GETRES
|
||||
| nr::GETTIMEOFDAY
|
||||
| nr::TIMES
|
||||
| nr::GETRUSAGE
|
||||
| nr::GETPID
|
||||
| nr::GETPPID
|
||||
| nr::GETTID
|
||||
| nr::GETUID
|
||||
| nr::GETEUID
|
||||
| nr::GETGID
|
||||
| nr::GETEGID
|
||||
| nr::SCHED_YIELD
|
||||
| nr::SCHED_GETAFFINITY
|
||||
| nr::RT_SIGPROCMASK
|
||||
| nr::SIGALTSTACK
|
||||
| nr::NANOSLEEP
|
||||
| nr::CLOCK_NANOSLEEP
|
||||
)
|
||||
}
|
||||
|
||||
/// What `clock_fast_path` needs to know about this process without taking
|
||||
/// the kernel: its ABI (0 not yet known, 1 i386, 2 native) and when it
|
||||
/// started (`Kernel::started`), both refreshed by every full syscall.
|
||||
static FAST_ABI: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
static FAST_STARTED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
/// Clock calls answered by `clock_fast_path`.
|
||||
static FAST_CALLS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
thread_local! {
|
||||
/// `k_signal_seq` as this thread's last full syscall read it.
|
||||
static FAST_SIG: Cell<i32> = const { Cell::new(i32::MIN) };
|
||||
/// Fast calls since this thread's last full syscall.
|
||||
static FAST_RUN: Cell<u32> = const { Cell::new(0) };
|
||||
/// The canonical number of the syscall this thread just made, for
|
||||
/// `wakes_nobody`; `u64::MAX` when the kernel did not run it.
|
||||
static CANON_NR: Cell<u64> = const { Cell::new(u64::MAX) };
|
||||
}
|
||||
|
||||
/// `clock_gettime`, `gettimeofday`, `getrusage` and `sched_yield` served
|
||||
/// lock-free (`gex_kernel::syscalls::clock_call`), with no proc snapshot and
|
||||
/// no wake. Every other syscall takes the host lock several times
|
||||
/// (`with_host`), and a W3 process's ~25 idle Wine Workers contend for it:
|
||||
/// Undertale's game thread asks the time ~2,000 times a second (Wine's
|
||||
/// `NtDelayExecution` calls `NtYieldExecution`, which is getrusage,
|
||||
/// sched_yield, getrusage) and spent ~25 ms of an 87 ms frame in `KLock::take`
|
||||
/// for these alone.
|
||||
///
|
||||
/// The kernel delivers pending signals at every syscall boundary, which this
|
||||
/// skips; so it stands down -- the full call runs -- while a signal may be
|
||||
/// waiting: whenever `k_signal_seq` has moved since this thread's last full
|
||||
/// call, and on every 64th call in a row (a signal raised inside the process
|
||||
/// does not move the sequence).
|
||||
fn clock_fast_path(ctx: i32) -> bool {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
let abi = FAST_ABI.load(Relaxed);
|
||||
if abi == 0 || tracing() || EXITED.load(Relaxed) || cgen::libprov::any_registered() {
|
||||
return false;
|
||||
}
|
||||
if unsafe { k_present() } != 0 && FAST_SIG.with(|c| c.get()) != unsafe { k_signal_seq() } {
|
||||
return false;
|
||||
}
|
||||
if FAST_RUN.with(|c| c.get()) >= 63 {
|
||||
return false;
|
||||
}
|
||||
let mut gm = GuestMem::new(guest_bytes());
|
||||
let started = FAST_STARTED.load(Relaxed) as u128;
|
||||
if !gex_kernel::syscalls::clock_call(&mut gm, ctx as u32 as u64, abi == 1, browser_clock, started) {
|
||||
return false;
|
||||
}
|
||||
FAST_RUN.with(|c| c.set(c.get() + 1));
|
||||
SYS_CALLS.fetch_add(1, Relaxed);
|
||||
if FAST_CALLS.fetch_add(1, Relaxed) == 0 {
|
||||
warn_line(&format!(
|
||||
"[gex] pid {}: clock calls served without the kernel lock\n",
|
||||
MY_PID.load(Relaxed)
|
||||
));
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// The mark the page's glue puts on a stateful provider's syscalls (as gexrt's
|
||||
/// `NATIVE_SYSCALL` on native Wine's, `nt.rs`).
|
||||
const PROVIDER_SYSCALL: u64 = 1 << 61;
|
||||
|
||||
Reference in new issue
Block a user