mirror of
https://github.com/HeyPuter/gex.git
synced 2026-11-13 20:12:43 -10:00
x86: SSE honours MXCSR's rounding control, FTZ and DAZ in every tier
Wasm rounds only to nearest. An SSE operation MXCSR affects (arithmetic, square root, rounding to integral, the conversions that round) is now Expr::Mx: the operation's own lowering, kept when MXCSR has none of RC, FTZ or DAZ set, and kept as well under FTZ or DAZ alone when no float operand nor the result is a denormal or the smallest normal. Otherwise the pure gex.mx import computes it exactly (gex_f80::sse::op: rounded once into single or double, tininess after rounding as the hardware detects it, SSE's NaN rules), once for a scalar or once per lane. Tier-1, the function JIT, the function tier's C and the page all lower it; x87 rounding and precision control were already exact. tests/x86-programs/fp-modes: SSE under eight MXCSR modes and x87 under eight control words, against the hardware, on tier-1, the function JIT and the page. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_8b3490cccb4848b495dde1c5cdcacba0
This commit is contained in:
velzieandClaude Opus 5.5 committed
1 parent
607dc757ff
commit
702d904f30
22 files changed
+1032
-117
No files matched your search
@@ -940,6 +940,14 @@ pub mod imports {
|
||||
/// `ctx::X87_TMP + 12`.
|
||||
pub const X87: &str = "x87";
|
||||
|
||||
/// An SSE operation under a non-default MXCSR (`gex_ir::Expr::Mx`'s
|
||||
/// slow path: a rounding control other than to nearest, FTZ or DAZ),
|
||||
/// computed exactly (`gex_f80::sse::op`). Pure, as [`X87`] is.
|
||||
/// Signature `(ctx: i32, code: i32, a: i64, b: i64, mxcsr: i32) -> i64`:
|
||||
/// `code` is `gex_f80::sse`'s, the operands and the result raw bits
|
||||
/// (a single's or an `i32`'s zero-extended).
|
||||
pub const MX: &str = "mx";
|
||||
|
||||
/// Safepoint poll taken when the interrupt flag is set.
|
||||
/// Signature `(ctx: i32, pc: i64) -> ()`.
|
||||
pub const SAFEPOINT: &str = "safepoint";
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/* The function tier's slow path of Expr::Mx (an SSE operation under a
|
||||
* MXCSR with a rounding control other than to nearest, FTZ or DAZ): the
|
||||
* gex.mx import, which computes it exactly (gex_f80::sse::op), as tier-1's
|
||||
* `emit_mx` calls it -- once for a scalar, once per lane, lanes past `n`
|
||||
* zero. The fast path is the operation's own C: kept when none of those
|
||||
* bits is set, and under FTZ or DAZ alone when no float operand nor the
|
||||
* result is a denormal or the smallest normal (the gex_mx_tiny tests), as
|
||||
* neither flush then has anything to do.
|
||||
*
|
||||
* Included from gex_abi.h, after the imports. */
|
||||
|
||||
/* A denormal or the smallest normal (|x| - 1 below the smallest normal). */
|
||||
GEX_INLINE int gex_mx_tiny32(uint32_t x) { return ((x & 0x7fffffffu) - 1) >> 23 == 0; }
|
||||
GEX_INLINE int gex_mx_tiny64(uint64_t x) { return ((x & 0x7fffffffffffffffull) - 1) >> 52 == 0; }
|
||||
/* Any lane of `v`, lanes `w` bytes wide. */
|
||||
GEX_INLINE int gex_mx_tiny_v(v128_t v, uint32_t w) {
|
||||
if (w == 4) {
|
||||
v128_t t = wasm_u32x4_shr(wasm_i32x4_sub(wasm_v128_and(v, wasm_i32x4_splat(0x7fffffff)), wasm_i32x4_splat(1)), 23);
|
||||
return wasm_v128_any_true(wasm_i32x4_eq(t, wasm_i32x4_splat(0)));
|
||||
}
|
||||
v128_t t = wasm_u64x2_shr(wasm_i64x2_sub(wasm_v128_and(v, wasm_i64x2_splat(0x7fffffffffffffffll)), wasm_i64x2_splat(1)), 52);
|
||||
return wasm_v128_any_true(wasm_i64x2_eq(t, wasm_i64x2_splat(0)));
|
||||
}
|
||||
|
||||
GEX_INLINE uint64_t gex_mx_scalar(int ctx, int32_t code, uint64_t a, uint64_t b, uint32_t mx) {
|
||||
return (uint64_t)gex_mx(ctx, code, (int64_t)a, (int64_t)b, (int32_t)mx);
|
||||
}
|
||||
|
||||
/* Lane `k` (a constant) of `v`, `w` bytes wide, as the import's operand. */
|
||||
#define GEX_MX_IN(v, k, w) \
|
||||
((w) == 4 ? (uint64_t)(uint32_t)wasm_i32x4_extract_lane(v, k) : (uint64_t)wasm_i64x2_extract_lane(v, (k) & 1))
|
||||
|
||||
GEX_INLINE v128_t gex_mx_lanes(int ctx, int32_t code, v128_t a, v128_t b, uint32_t n, uint32_t in_bytes,
|
||||
uint32_t out_bytes, uint32_t mx) {
|
||||
v128_t r = wasm_i64x2_const(0, 0);
|
||||
#define GEX_MX_LANE(k) \
|
||||
if ((k) < n) { \
|
||||
uint64_t x = gex_mx_scalar(ctx, code, GEX_MX_IN(a, k, in_bytes), GEX_MX_IN(b, k, in_bytes), mx); \
|
||||
r = out_bytes == 4 ? wasm_i32x4_replace_lane(r, k, (int32_t)x) : wasm_i64x2_replace_lane(r, (k) & 1, (int64_t)x); \
|
||||
}
|
||||
GEX_MX_LANE(0)
|
||||
GEX_MX_LANE(1)
|
||||
GEX_MX_LANE(2)
|
||||
GEX_MX_LANE(3)
|
||||
#undef GEX_MX_LANE
|
||||
return r;
|
||||
}
|
||||
+91
-27
@@ -1237,7 +1237,74 @@ impl<'a> Emitter<'a> {
|
||||
*c = is_ctx;
|
||||
}
|
||||
self.note_disp(idx, expr);
|
||||
let rhs: String = match expr {
|
||||
if let Expr::HostCall { func, args, ty } = expr {
|
||||
if *func == HostFunc::Fma && args.len() == 4 {
|
||||
if let Some(dbl) = self.value_const[args[3].0 as usize] {
|
||||
let (a, b, c) = (self.v(args[0]), self.v(args[1]), self.v(args[2]));
|
||||
let e = if dbl != 0 {
|
||||
format!("gex_fma64(ctx, {a}, {b}, {c}, 1)")
|
||||
} else {
|
||||
format!("gex_fmaf({a}, {b}, {c})")
|
||||
};
|
||||
self.line(format!("const {} {d} = ({}){e};", cty(*ty), cty(*ty)));
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let call = self.host_call_expr(*func, args)?;
|
||||
if func.clobbers_guest_state() {
|
||||
self.flush();
|
||||
}
|
||||
self.line(format!("const {} {d} = ({}){call};", cty(*ty), cty(*ty)));
|
||||
if func.clobbers_guest_state() {
|
||||
self.invalidate();
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Expr::Mx { mx, e } = expr {
|
||||
// `e`'s own C, then the exact import when the bits ask for it:
|
||||
// a rounding other than to nearest, or FTZ or DAZ with a float
|
||||
// operand or the result denormal or the smallest normal (see
|
||||
// `runtime/mx.h`).
|
||||
let fast = self.rhs(idx, e)?;
|
||||
let m = self.v(*mx);
|
||||
let plan = MxPlan::of(e);
|
||||
let w = |t: Ty| if t == Ty::I64 { 8 } else { 4 };
|
||||
let (a, b, in_bytes, out_bytes) = match plan {
|
||||
MxPlan::Scalar { a, b, in_ty, .. } => (a, b, w(in_ty), w(ty)),
|
||||
MxPlan::Lanes { a, b, in_bytes, out_bytes, .. } => (a, b, in_bytes, out_bytes),
|
||||
};
|
||||
let tiny = |v: String, bytes: u32| {
|
||||
if ty == Ty::V128 {
|
||||
format!("gex_mx_tiny_v({v}, {bytes})")
|
||||
} else if bytes == 8 {
|
||||
format!("gex_mx_tiny64((uint64_t){v})")
|
||||
} else {
|
||||
format!("gex_mx_tiny32((uint32_t){v})")
|
||||
}
|
||||
};
|
||||
let mut redo = format!("((uint32_t){m} & 0x6000u) != 0");
|
||||
if plan.float_operands() {
|
||||
for v in [Some(a), b].into_iter().flatten() {
|
||||
redo += &format!(" || {}", tiny(self.v(v), in_bytes));
|
||||
}
|
||||
}
|
||||
redo += &format!(" || {}", tiny(d.clone(), out_bytes));
|
||||
let slow = self.mx_slow(*mx, e);
|
||||
self.line(format!("{} {d} = {fast};", cty(ty)));
|
||||
self.line(format!(
|
||||
"if (__builtin_expect(((uint32_t){m} & 0x{:x}u) != 0, 0) && ({redo})) {d} = {slow};",
|
||||
MX_SPECIAL
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
let rhs = self.rhs(idx, expr)?;
|
||||
self.line(format!("const {} {d} = {rhs};", cty(ty)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The C expression of `expr`, defining value `idx`.
|
||||
fn rhs(&mut self, idx: usize, expr: &Expr) -> Result<String> {
|
||||
Ok(match expr {
|
||||
Expr::ConstI32(c) => {
|
||||
self.value_const[idx] = Some(*c as u64);
|
||||
format!("0x{c:x}u")
|
||||
@@ -1422,29 +1489,7 @@ impl<'a> Emitter<'a> {
|
||||
&format!("(uint64_t)({nt}){r}"),
|
||||
)
|
||||
}
|
||||
Expr::HostCall { func, args, ty } => {
|
||||
if *func == HostFunc::Fma && args.len() == 4 {
|
||||
if let Some(dbl) = self.value_const[args[3].0 as usize] {
|
||||
let (a, b, c) = (self.v(args[0]), self.v(args[1]), self.v(args[2]));
|
||||
let e = if dbl != 0 {
|
||||
format!("gex_fma64(ctx, {a}, {b}, {c}, 1)")
|
||||
} else {
|
||||
format!("gex_fmaf({a}, {b}, {c})")
|
||||
};
|
||||
self.line(format!("const {} {d} = ({}){e};", cty(*ty), cty(*ty)));
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let call = self.host_call_expr(*func, args)?;
|
||||
if func.clobbers_guest_state() {
|
||||
self.flush();
|
||||
}
|
||||
self.line(format!("const {} {d} = ({}){call};", cty(*ty), cty(*ty)));
|
||||
if func.clobbers_guest_state() {
|
||||
self.invalidate();
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Expr::HostCall { .. } => unreachable!("a host call is emitted by `def`"),
|
||||
Expr::VLoad {
|
||||
addr,
|
||||
offset,
|
||||
@@ -1654,9 +1699,28 @@ impl<'a> Emitter<'a> {
|
||||
dst_wide,
|
||||
a,
|
||||
} => self.fp_convert(*kind, *src_wide, *dst_wide, *a),
|
||||
};
|
||||
self.line(format!("const {} {d} = {rhs};", cty(ty)));
|
||||
Ok(())
|
||||
Expr::Mx { .. } => unreachable!("def lowers Mx"),
|
||||
})
|
||||
}
|
||||
|
||||
/// [`Expr::Mx`]'s slow path: the `mx` import through `runtime/mx.h`.
|
||||
fn mx_slow(&self, mx: Value, e: &Expr) -> String {
|
||||
let m = self.v(mx);
|
||||
match MxPlan::of(e) {
|
||||
MxPlan::Scalar { code, a, b, .. } => {
|
||||
let b = b.map_or("0".to_string(), |b| format!("(uint64_t){}", self.v(b)));
|
||||
let r = format!("gex_mx_scalar(ctx, {code}, (uint64_t){}, {b}, (uint32_t){m})", self.v(a));
|
||||
if e.ty() == Ty::I32 {
|
||||
format!("(uint32_t){r}")
|
||||
} else {
|
||||
r
|
||||
}
|
||||
}
|
||||
MxPlan::Lanes { code, a, b, n, in_bytes, out_bytes } => {
|
||||
let b = b.map_or("wasm_i64x2_const(0, 0)".to_string(), |b| self.v(b));
|
||||
format!("gex_mx_lanes(ctx, {code}, {}, {b}, {n}, {in_bytes}, {out_bytes}, (uint32_t){m})", self.v(a))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A value's bit pattern as a C float of the given precision.
|
||||
|
||||
@@ -134,6 +134,7 @@ pub fn header() -> String {
|
||||
"gex_x87",
|
||||
"int ctx, int32_t op, int64_t a_sig, int32_t a_se, int64_t b_sig, int32_t b_se, int32_t cw",
|
||||
);
|
||||
decl(w, imports::MX, "int64_t", "gex_mx", "int ctx, int32_t code, int64_t a, int64_t b, int32_t mx");
|
||||
let _ = writeln!(
|
||||
w,
|
||||
"/* Tail-called with ctx.LAST_TARGET holding the target; the splice replaces\n \
|
||||
@@ -157,6 +158,7 @@ pub fn header() -> String {
|
||||
}
|
||||
}
|
||||
w.push_str(X87_HELPERS);
|
||||
w.push_str(MX_HELPERS);
|
||||
w.push_str("#endif\n");
|
||||
h
|
||||
}
|
||||
@@ -164,6 +166,9 @@ pub fn header() -> String {
|
||||
/// The x87 helpers, `Stmt::X87`'s lowering (see `emit.rs`).
|
||||
const X87_HELPERS: &str = include_str!("../runtime/x87.h");
|
||||
|
||||
/// The MXCSR slow path's helpers, `Expr::Mx`'s lowering (see `emit.rs`).
|
||||
const MX_HELPERS: &str = include_str!("../runtime/mx.h");
|
||||
|
||||
const BODY_TYPES: &str = r#"
|
||||
/* Unaligned, alias-everything views of guest memory: a guest access is a
|
||||
* plain C access through one of these (guest VA == linear-memory offset ==
|
||||
|
||||
@@ -651,6 +651,7 @@ struct Imports {
|
||||
safepoint: u32,
|
||||
fma: u32,
|
||||
x87: u32,
|
||||
mx: u32,
|
||||
}
|
||||
|
||||
impl ModuleBuilder {
|
||||
@@ -740,6 +741,11 @@ impl ModuleBuilder {
|
||||
vec![ValType::I64],
|
||||
));
|
||||
|
||||
let t_mx = m.func_type(FuncType::new(
|
||||
vec![p, ValType::I32, ValType::I64, ValType::I64, ValType::I32],
|
||||
vec![ValType::I64],
|
||||
));
|
||||
|
||||
let im = IMPORT_MODULE;
|
||||
let imports = Imports {
|
||||
syscall: m.import_func(im, imports::SYSCALL, t_syscall),
|
||||
@@ -752,6 +758,7 @@ impl ModuleBuilder {
|
||||
safepoint: m.import_func(im, imports::SAFEPOINT, t_syscall),
|
||||
fma: m.import_func(im, imports::FMA, t_fma),
|
||||
x87: m.import_func(im, imports::X87, t_x87),
|
||||
mx: m.import_func(im, imports::MX, t_mx),
|
||||
};
|
||||
|
||||
// The memory is the host's and shared, so every guest thread acts on
|
||||
@@ -977,6 +984,7 @@ impl ModuleBuilder {
|
||||
safepoint: self.imports.safepoint,
|
||||
fma: self.imports.fma,
|
||||
x87: self.imports.x87,
|
||||
mx: self.imports.mx,
|
||||
block_ty: self.block_ty,
|
||||
entry_ty: self.entry_ty,
|
||||
blocks: &self.blocks,
|
||||
|
||||
@@ -91,6 +91,13 @@ mod simd {
|
||||
pub const I32X4_EXTADD_PAIRWISE_I16X8_S: u32 = 0x7e;
|
||||
pub const I32X4_EXTADD_PAIRWISE_I16X8_U: u32 = 0x7f;
|
||||
pub const V128_AND: u32 = 0x4e;
|
||||
pub const V128_ANY_TRUE: u32 = 0x53;
|
||||
pub const I32X4_EQ: u32 = 0x37;
|
||||
pub const I32X4_SHR_U: u32 = 0xad;
|
||||
pub const I32X4_SUB: u32 = 0xb1;
|
||||
pub const I64X2_EQ: u32 = 0xd6;
|
||||
pub const I64X2_SHR_U: u32 = 0xcd;
|
||||
pub const I64X2_SUB: u32 = 0xd1;
|
||||
pub const V128_ANDNOT: u32 = 0x4f;
|
||||
pub const V128_OR: u32 = 0x50;
|
||||
pub const V128_XOR: u32 = 0x51;
|
||||
@@ -171,6 +178,8 @@ pub struct Context<'a> {
|
||||
pub fma: u32,
|
||||
/// Import index of `gex.x87`.
|
||||
pub x87: u32,
|
||||
/// Import index of `gex.mx`.
|
||||
pub mx: u32,
|
||||
/// Import index of `gex.sysreg_write`.
|
||||
pub sysreg_write: u32,
|
||||
/// Import index of `gex.cache_op`.
|
||||
@@ -3064,6 +3073,10 @@ impl<'a> Emitter<'a> {
|
||||
self.f.simd_lane(sub, *lane as u8);
|
||||
}
|
||||
|
||||
Expr::Mx { mx, e } => {
|
||||
self.emit_mx(dst, *mx, e);
|
||||
return;
|
||||
}
|
||||
Expr::FpBin { op, double, a, b } => {
|
||||
self.push_fp(*a, *double);
|
||||
self.push_fp(*b, *double);
|
||||
@@ -3131,6 +3144,160 @@ impl<'a> Emitter<'a> {
|
||||
self.value_local[idx] = Some(local);
|
||||
}
|
||||
|
||||
/// An [`Expr::Mx`]: `e`'s own lowering, kept when `mx` has none of
|
||||
/// [`MX_SPECIAL`]'s bits; and kept as well under FTZ or DAZ alone
|
||||
/// (rounding to nearest) when no float operand or result is a denormal
|
||||
/// or the smallest normal (one a tiny result may have rounded to), as
|
||||
/// then neither flush has anything to do. Otherwise the pure `mx`
|
||||
/// import, once for a scalar and once per lane ([`MxPlan`]). Either
|
||||
/// way the result is left in one integer-typed local.
|
||||
fn emit_mx(&mut self, dst: Value, mx: Value, e: &Expr) {
|
||||
use code::*;
|
||||
let idx = dst.0 as usize;
|
||||
let ty = e.ty();
|
||||
let res = self.def_into.take().unwrap_or_else(|| self.alloc(ty));
|
||||
self.def(dst, e);
|
||||
self.push(dst);
|
||||
self.f.local_set(res);
|
||||
let plan = MxPlan::of(e);
|
||||
self.push(mx);
|
||||
self.f.i32_const(MX_SPECIAL as i32);
|
||||
self.f.byte(I32_AND);
|
||||
self.f.if_void();
|
||||
self.push(mx);
|
||||
self.f.i32_const(0x6000);
|
||||
self.f.byte(I32_AND);
|
||||
let w = |t: Ty| if t == Ty::I64 { 8 } else { 4 };
|
||||
let (a, b, in_bytes, out_bytes) = match plan {
|
||||
MxPlan::Scalar { a, b, in_ty, .. } => (a, b, w(in_ty), w(ty)),
|
||||
MxPlan::Lanes { a, b, in_bytes, out_bytes, .. } => (a, b, in_bytes, out_bytes),
|
||||
};
|
||||
let lanes = ty == Ty::V128;
|
||||
if plan.float_operands() {
|
||||
for v in [Some(a), b].into_iter().flatten() {
|
||||
self.push(v);
|
||||
self.emit_tiny_check(lanes, in_bytes);
|
||||
self.f.byte(I32_OR);
|
||||
}
|
||||
}
|
||||
self.f.local_get(res);
|
||||
self.emit_tiny_check(lanes, out_bytes);
|
||||
self.f.byte(I32_OR);
|
||||
self.f.if_void();
|
||||
// The import's operand: an integer of `in_bytes` (from lane `lane`
|
||||
// of a vector), as an i64.
|
||||
let operand = |this: &mut Self, v: Value, lane: Option<(u32, u32)>, in_ty: Ty| {
|
||||
this.push(v);
|
||||
match lane {
|
||||
Some((k, 4)) => {
|
||||
this.f.simd_lane(simd::I32X4_EXTRACT_LANE, k as u8);
|
||||
this.f.byte(I64_EXTEND_I32_U);
|
||||
}
|
||||
Some((k, _)) => this.f.simd_lane(simd::I64X2_EXTRACT_LANE, k as u8),
|
||||
None if in_ty == Ty::I32 => this.f.byte(I64_EXTEND_I32_U),
|
||||
None => {}
|
||||
}
|
||||
};
|
||||
match plan {
|
||||
MxPlan::Scalar { code, a, b, in_ty } => {
|
||||
self.f.local_get(self.ctx_local);
|
||||
self.f.i32_const(code as i32);
|
||||
operand(self, a, None, in_ty);
|
||||
match b {
|
||||
Some(b) => operand(self, b, None, in_ty),
|
||||
None => self.f.i64_const(0),
|
||||
}
|
||||
self.push(mx);
|
||||
self.f.call(self.cx.mx);
|
||||
if ty == Ty::I32 {
|
||||
self.f.byte(I32_WRAP_I64);
|
||||
}
|
||||
self.f.local_set(res);
|
||||
}
|
||||
MxPlan::Lanes { code, a, b, n, in_bytes, out_bytes } => {
|
||||
self.f.v128_const(&[0; 16]);
|
||||
self.f.local_set(res);
|
||||
for k in 0..n {
|
||||
self.f.local_get(res);
|
||||
self.f.local_get(self.ctx_local);
|
||||
self.f.i32_const(code as i32);
|
||||
operand(self, a, Some((k, in_bytes)), Ty::I64);
|
||||
match b {
|
||||
Some(b) => operand(self, b, Some((k, in_bytes)), Ty::I64),
|
||||
None => self.f.i64_const(0),
|
||||
}
|
||||
self.push(mx);
|
||||
self.f.call(self.cx.mx);
|
||||
if out_bytes == 4 {
|
||||
self.f.byte(I32_WRAP_I64);
|
||||
self.f.simd_lane(simd::I32X4_REPLACE_LANE, k as u8);
|
||||
} else {
|
||||
self.f.simd_lane(simd::I64X2_REPLACE_LANE, k as u8);
|
||||
}
|
||||
self.f.local_set(res);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.f.end();
|
||||
self.f.end();
|
||||
self.value_ty[idx] = ty;
|
||||
if let Some(f) = self.value_fp.get_mut(idx) {
|
||||
*f = None;
|
||||
}
|
||||
self.value_const[idx] = None;
|
||||
self.value_local[idx] = Some(res);
|
||||
}
|
||||
|
||||
/// Replace the float bits on the stack (a scalar of `bytes`, or a vector
|
||||
/// of `bytes`-wide lanes) with an i32, nonzero when it (a lane) is a
|
||||
/// denormal or the smallest normal: `(|x| - 1) >> fraction bits == 0`.
|
||||
fn emit_tiny_check(&mut self, lanes: bool, bytes: u32) {
|
||||
use code::*;
|
||||
let frac = if bytes == 8 { 52 } else { 23 };
|
||||
let mut abs = [0xffu8; 16];
|
||||
let mut one = [0u8; 16];
|
||||
for k in 0..16 / bytes as usize {
|
||||
abs[k * bytes as usize + bytes as usize - 1] = 0x7f;
|
||||
one[k * bytes as usize] = 1;
|
||||
}
|
||||
match (lanes, bytes) {
|
||||
(false, 8) => {
|
||||
self.f.i64_const(i64::MAX);
|
||||
self.f.byte(I64_AND);
|
||||
self.f.i64_const(1);
|
||||
self.f.byte(I64_SUB);
|
||||
self.f.i64_const(frac);
|
||||
self.f.byte(I64_SHR_U);
|
||||
self.f.byte(I64_EQZ);
|
||||
}
|
||||
(false, _) => {
|
||||
self.f.i32_const(i32::MAX);
|
||||
self.f.byte(I32_AND);
|
||||
self.f.i32_const(1);
|
||||
self.f.byte(I32_SUB);
|
||||
self.f.i32_const(frac as i32);
|
||||
self.f.byte(I32_SHR_U);
|
||||
self.f.byte(I32_EQZ);
|
||||
}
|
||||
(true, _) => {
|
||||
let (sub, shr, eq) = if bytes == 8 {
|
||||
(simd::I64X2_SUB, simd::I64X2_SHR_U, simd::I64X2_EQ)
|
||||
} else {
|
||||
(simd::I32X4_SUB, simd::I32X4_SHR_U, simd::I32X4_EQ)
|
||||
};
|
||||
self.f.v128_const(&abs);
|
||||
self.f.simd(simd::V128_AND);
|
||||
self.f.v128_const(&one);
|
||||
self.f.simd(sub);
|
||||
self.f.i32_const(frac as i32);
|
||||
self.f.simd(shr);
|
||||
self.f.v128_const(&[0; 16]);
|
||||
self.f.simd(eq);
|
||||
self.f.simd(simd::V128_ANY_TRUE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reinterpret the integer bit pattern on top of the stack as a float.
|
||||
/// Reverse the bytes of the value on the stack. Wasm
|
||||
/// has no byte swap: 32 bits is two rotates, two masks and an or; 16 bits
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod sse;
|
||||
|
||||
/// Status-word exception flags, in their status-word bit positions (and the
|
||||
/// control word's mask bits, which sit at the same positions).
|
||||
pub const IE: u16 = 1 << 0;
|
||||
@@ -111,6 +113,14 @@ pub struct Env {
|
||||
pub flags: u16,
|
||||
/// `C1`: set when the result was rounded up in magnitude.
|
||||
pub c1: bool,
|
||||
/// The format results are rounded into: extended for the x87, single
|
||||
/// or double for SSE ([`sse`]), whose results have their format's
|
||||
/// exponent range as well as its precision.
|
||||
fmt: Fmt,
|
||||
/// The last rounding's result was tiny (below the format's smallest
|
||||
/// normal after rounding, exponent unbounded), exact or not: what SSE's
|
||||
/// FTZ flushes.
|
||||
tiny: bool,
|
||||
}
|
||||
|
||||
impl Env {
|
||||
@@ -131,12 +141,14 @@ impl Env {
|
||||
},
|
||||
flags: 0,
|
||||
c1: false,
|
||||
fmt: EXT,
|
||||
tiny: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A floating-point format results are rounded into.
|
||||
#[derive(Copy, Clone)]
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
struct Fmt {
|
||||
/// Significand bits, integer bit included.
|
||||
bits: u32,
|
||||
@@ -241,6 +253,7 @@ fn round_pack(env: &mut Env, sign: bool, e: i32, sig: u128, prec: u32, f: Fmt) -
|
||||
// Tininess is detected after rounding: a result that rounds up to the
|
||||
// smallest normal (at `prec` bits, exponent unbounded) is not tiny.
|
||||
let tiny = e < f.emin && !(e == f.emin - 1 && rounds_to_carry(env, sign, sig, prec));
|
||||
env.tiny = tiny;
|
||||
// Bits of `sig` below the kept part.
|
||||
let drop = (127 - (e - lsb)) as u32;
|
||||
let (kept, rem, half) = if drop >= 129 {
|
||||
@@ -312,12 +325,29 @@ fn rounds_to_carry(env: &Env, sign: bool, sig: u128, prec: u32) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Round to an extended-format result at precision `prec`.
|
||||
/// Round to an extended-format result at precision `prec`; under SSE
|
||||
/// ([`Env::fmt`] single or double) to that format, its value carried
|
||||
/// exactly in an extended one.
|
||||
fn round_ext(env: &mut Env, sign: bool, e: i32, sig: u128, prec: u32) -> F80 {
|
||||
let (be, m) = round_pack(env, sign, e, sig, prec, EXT);
|
||||
let f = env.fmt;
|
||||
let (be, m) = round_pack(env, sign, e, sig, prec.min(f.bits), f);
|
||||
let s = if sign { 0x8000 } else { 0 };
|
||||
if f.bits == EXT.bits {
|
||||
return F80 { sig: m, se: be as u16 | s };
|
||||
}
|
||||
if be == (2 * f.emax + 1) as u32 {
|
||||
return F80::inf(sign);
|
||||
}
|
||||
if m == 0 {
|
||||
return F80::zero(sign);
|
||||
}
|
||||
// A denormal of the format: its integer bit's weight is 2^emin.
|
||||
let (e, m) = if be == 0 { (f.emin, m) } else { (be as i32 - f.emax, m) };
|
||||
let lz = m.leading_zeros() as i32;
|
||||
let e = e - (f.bits as i32 - 1) + (63 - lz);
|
||||
F80 {
|
||||
sig: m,
|
||||
se: be as u16 | if sign { 0x8000 } else { 0 },
|
||||
sig: m << lz,
|
||||
se: (e + 16383) as u16 | s,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
//! SSE's scalar floating point as MXCSR controls it: the rounding control
|
||||
//! (bits 13-14, the x87's RC encoding), DAZ (bit 6: a denormal operand is
|
||||
//! read as a zero of its sign) and FTZ (bit 15: a denormal result is
|
||||
//! written as one). Results are single or double, rounded once into their
|
||||
//! format, exponent range included; NaNs follow SSE's rules (the first
|
||||
//! operand's if it is one, else the second's, quieted; an invalid operation
|
||||
//! gives the default NaN, negative and quiet).
|
||||
//!
|
||||
//! One function, [`op`], as `gex_ir::Expr::Mx`'s slow path calls it: the
|
||||
//! operation as a code, the operands and result as raw bits.
|
||||
//! Exception flags are not reported.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// MXCSR's DAZ bit.
|
||||
pub const DAZ: u32 = 1 << 6;
|
||||
/// MXCSR's FTZ bit.
|
||||
pub const FTZ: u32 = 1 << 15;
|
||||
|
||||
/// The operations of [`op`], in its code's low four bits; [`DOUBLE`] says
|
||||
/// the width of the operation (for the conversions, of the result).
|
||||
pub const ADD: u32 = 0;
|
||||
pub const SUB: u32 = 1;
|
||||
pub const MUL: u32 = 2;
|
||||
pub const DIV: u32 = 3;
|
||||
pub const SQRT: u32 = 4;
|
||||
/// Round to an integral value by the rounding control (`roundsd` with
|
||||
/// imm bit 2, and a conversion to integer's rounding).
|
||||
pub const RINT: u32 = 5;
|
||||
/// A signed 32-bit integer to a float.
|
||||
pub const FROM_I32: u32 = 6;
|
||||
/// A signed 64-bit integer to a float.
|
||||
pub const FROM_I64: u32 = 7;
|
||||
/// Double to single (`cvtsd2ss`); [`DOUBLE`] is not set.
|
||||
pub const NARROW: u32 = 8;
|
||||
/// Single to double (`cvtss2sd`); [`DOUBLE`] is set.
|
||||
pub const WIDEN: u32 = 9;
|
||||
/// The operation is on doubles.
|
||||
pub const DOUBLE: u32 = 1 << 4;
|
||||
|
||||
fn fmt_of(double: bool) -> (Fmt, Mem) {
|
||||
if double {
|
||||
(DBL, Mem::F64)
|
||||
} else {
|
||||
(SGL, Mem::F32)
|
||||
}
|
||||
}
|
||||
|
||||
/// The exponent field and fraction of `raw` in `m`'s format, and its sign
|
||||
/// bit's value.
|
||||
fn fields(raw: u64, m: Mem) -> (u64, u64, u64) {
|
||||
match m {
|
||||
Mem::F32 => ((raw >> 23) & 0xff, raw & 0x7f_ffff, raw & 0x8000_0000),
|
||||
_ => ((raw >> 52) & 0x7ff, raw & 0xf_ffff_ffff_ffff, raw & 0x8000_0000_0000_0000),
|
||||
}
|
||||
}
|
||||
|
||||
/// `raw`, a denormal of `m`'s format flushed to a zero of its sign.
|
||||
fn flush(raw: u64, m: Mem) -> u64 {
|
||||
let (e, f, s) = fields(raw, m);
|
||||
if e == 0 && f != 0 {
|
||||
s
|
||||
} else {
|
||||
raw
|
||||
}
|
||||
}
|
||||
|
||||
fn is_nan(raw: u64, m: Mem) -> bool {
|
||||
let (e, f, _) = fields(raw, m);
|
||||
let all = if m == Mem::F32 { 0xff } else { 0x7ff };
|
||||
e == all && f != 0
|
||||
}
|
||||
|
||||
fn quieted(raw: u64, m: Mem) -> u64 {
|
||||
raw | if m == Mem::F32 { 1 << 22 } else { 1 << 51 }
|
||||
}
|
||||
|
||||
/// `code` (one of the operations above, with [`DOUBLE`]) on `a` and `b`
|
||||
/// (raw bits; `b` unused by the one-operand operations, a float operand
|
||||
/// zero-extended, an integer one as it is) under the MXCSR value `mxcsr`.
|
||||
/// The result's raw bits, zero-extended.
|
||||
pub fn op(code: u32, a: u64, b: u64, mxcsr: u32) -> u64 {
|
||||
let double = code & DOUBLE != 0;
|
||||
let mut env = Env::from_cw(0x37f);
|
||||
env.round = match (mxcsr >> 13) & 3 {
|
||||
0 => Round::Nearest,
|
||||
1 => Round::Down,
|
||||
2 => Round::Up,
|
||||
_ => Round::Zero,
|
||||
};
|
||||
let daz = |raw: u64, m: Mem| if mxcsr & DAZ != 0 { flush(raw, m) } else { raw };
|
||||
// FTZ writes a tiny result (after rounding, exponent unbounded: one
|
||||
// that rounds to the smallest normal only in the format's own denormals
|
||||
// is tiny) as a zero of its sign, exact or not. The operation's own
|
||||
// rounding says so, or the store's when the operation passed an
|
||||
// operand through unrounded.
|
||||
let ftz = |env: &mut Env, r: F80, m: Mem| {
|
||||
let t = env.tiny;
|
||||
env.tiny = false;
|
||||
let raw = store(env, r, m, false);
|
||||
if mxcsr & FTZ != 0 && (t || env.tiny) {
|
||||
fields(raw, m).2
|
||||
} else {
|
||||
raw
|
||||
}
|
||||
};
|
||||
match code & 0xf {
|
||||
k @ (ADD | SUB | MUL | DIV | SQRT | RINT) => {
|
||||
let (f, m) = fmt_of(double);
|
||||
let (a, b) = (daz(a, m), daz(b, m));
|
||||
let unary = k >= SQRT;
|
||||
if is_nan(a, m) {
|
||||
return quieted(a, m);
|
||||
}
|
||||
if !unary && is_nan(b, m) {
|
||||
return quieted(b, m);
|
||||
}
|
||||
let fa = load(&mut env, a, m);
|
||||
let fb = load(&mut env, b, m);
|
||||
env.tiny = false;
|
||||
env.fmt = f;
|
||||
env.prec = f.bits;
|
||||
let r = match k {
|
||||
ADD => add(&mut env, fa, fb, false),
|
||||
SUB => add(&mut env, fa, fb, true),
|
||||
MUL => mul(&mut env, fa, fb),
|
||||
DIV => div(&mut env, fa, fb),
|
||||
SQRT => sqrt(&mut env, fa),
|
||||
_ => round_int(&mut env, fa, false),
|
||||
};
|
||||
env.fmt = EXT;
|
||||
ftz(&mut env, r, m)
|
||||
}
|
||||
k @ (FROM_I32 | FROM_I64) => {
|
||||
let v = load(&mut env, a, if k == FROM_I32 { Mem::I32 } else { Mem::I64 });
|
||||
store(&mut env, v, fmt_of(double).1, false)
|
||||
}
|
||||
NARROW => {
|
||||
let a = daz(a, Mem::F64);
|
||||
let v = load(&mut env, a, Mem::F64);
|
||||
env.tiny = false;
|
||||
ftz(&mut env, v, Mem::F32)
|
||||
}
|
||||
WIDEN => {
|
||||
let a = daz(a & 0xffff_ffff, Mem::F32);
|
||||
let v = load(&mut env, a, Mem::F32);
|
||||
store(&mut env, v, Mem::F64, false)
|
||||
}
|
||||
_ => panic!("sse::op: unknown operation {code:#x}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const RN: u32 = 0x1f80;
|
||||
const RD: u32 = 0x3f80;
|
||||
const RU: u32 = 0x5f80;
|
||||
const RZ: u32 = 0x7f80;
|
||||
|
||||
fn d(code: u32, a: f64, b: f64, mx: u32) -> f64 {
|
||||
f64::from_bits(op(code | DOUBLE, a.to_bits(), b.to_bits(), mx))
|
||||
}
|
||||
fn s(code: u32, a: f32, b: f32, mx: u32) -> f32 {
|
||||
f32::from_bits(op(code, a.to_bits() as u64, b.to_bits() as u64, mx) as u32)
|
||||
}
|
||||
|
||||
/// To nearest, every operation is the host's.
|
||||
#[test]
|
||||
fn nearest_is_ieee() {
|
||||
let vals = [1.0, 1.0 / 3.0, -2.0 / 7.0, 1e300, -1e-300, 4.9e-324, 2.2250738585072014e-308, 0.0, -0.0, 1e16 + 3.0];
|
||||
for &a in &vals {
|
||||
for &b in &vals {
|
||||
assert_eq!(d(ADD, a, b, RN).to_bits(), (a + b).to_bits(), "{a} + {b}");
|
||||
assert_eq!(d(SUB, a, b, RN).to_bits(), (a - b).to_bits(), "{a} - {b}");
|
||||
assert_eq!(d(MUL, a, b, RN).to_bits(), (a * b).to_bits(), "{a} * {b}");
|
||||
if !(a == 0.0 && b == 0.0) {
|
||||
assert_eq!(d(DIV, a, b, RN).to_bits(), (a / b).to_bits(), "{a} / {b}");
|
||||
}
|
||||
let (fa, fb) = (a as f32, b as f32);
|
||||
assert_eq!(s(ADD, fa, fb, RN).to_bits(), (fa + fb).to_bits(), "{fa} + {fb}");
|
||||
assert_eq!(s(MUL, fa, fb, RN).to_bits(), (fa * fb).to_bits(), "{fa} * {fb}");
|
||||
if !(fa == 0.0 && fb == 0.0) {
|
||||
assert_eq!(s(DIV, fa, fb, RN).to_bits(), (fa / fb).to_bits(), "{fa} / {fb}");
|
||||
}
|
||||
}
|
||||
if a >= 0.0 {
|
||||
assert_eq!(d(SQRT, a, 0.0, RN).to_bits(), a.sqrt().to_bits(), "sqrt {a}");
|
||||
}
|
||||
assert_eq!(op(NARROW, a.to_bits(), 0, RN) as u32, (a as f32).to_bits(), "narrow {a}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Directed rounding brackets the exact result.
|
||||
#[test]
|
||||
fn directed() {
|
||||
assert_eq!(d(DIV, 1.0, 3.0, RD), 0.3333333333333333);
|
||||
assert_eq!(d(DIV, 1.0, 3.0, RU), 0.33333333333333337);
|
||||
assert_eq!(d(DIV, -1.0, 3.0, RZ), -0.3333333333333333);
|
||||
assert_eq!(d(DIV, -1.0, 3.0, RD), -0.33333333333333337);
|
||||
assert_eq!(d(ADD, 1.7e308, 1.7e308, RZ), f64::MAX);
|
||||
assert_eq!(d(ADD, 1.7e308, 1.7e308, RN), f64::INFINITY);
|
||||
assert_eq!(d(MUL, 1e300, 1e300, RD), f64::MAX);
|
||||
assert_eq!(d(MUL, -1e300, 1e300, RD), f64::NEG_INFINITY);
|
||||
assert_eq!(d(SUB, 1.0, 1.0, RD).to_bits(), (-0.0f64).to_bits());
|
||||
assert_eq!(s(ADD, 1.0, 1e-30, RU), 1.0000001);
|
||||
assert_eq!(op(NARROW, (1.0f64 / 3.0).to_bits(), 0, RZ) as u32, 0.3333333f32.to_bits());
|
||||
assert_eq!(op(NARROW, (1.0f64 / 3.0).to_bits(), 0, RU) as u32, 0.33333334f32.to_bits());
|
||||
assert_eq!(op(FROM_I64 | DOUBLE, 9007199254740993, 0, RU), 9007199254740994.0f64.to_bits());
|
||||
assert_eq!(op(FROM_I64 | DOUBLE, 9007199254740993, 0, RD), 9007199254740992.0f64.to_bits());
|
||||
assert_eq!(op(FROM_I32, 16777217, 0, RU) as u32, 16777218.0f32.to_bits());
|
||||
assert_eq!(d(RINT, 2.5, 0.0, RN), 2.0);
|
||||
assert_eq!(d(RINT, 2.5, 0.0, RU), 3.0);
|
||||
assert_eq!(d(RINT, -2.5, 0.0, RZ), -2.0);
|
||||
// A denormal result, rounded once into the format's own denormals.
|
||||
assert_eq!(d(MUL, 3.0e-310, 0.5, RU), f64::from_bits(3.0e-310f64.to_bits() / 2 + 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ftz_daz() {
|
||||
let den = 3.0e-310;
|
||||
assert_eq!(d(ADD, den, 0.0, RN | DAZ).to_bits(), 0);
|
||||
assert_eq!(d(ADD, -den, 0.0, RN | DAZ).to_bits(), 0);
|
||||
assert_eq!(d(MUL, -den, 1.0, RN | DAZ).to_bits(), (-0.0f64).to_bits());
|
||||
assert_eq!(d(ADD, den, 0.0, RN | FTZ).to_bits(), 0);
|
||||
assert_eq!(d(MUL, 1e-300, 1e-10, RN | FTZ).to_bits(), 0);
|
||||
assert_eq!(d(MUL, -1e-300, 1e-10, RN | FTZ).to_bits(), (-0.0f64).to_bits());
|
||||
assert_eq!(d(ADD, den, den, RN).to_bits(), (den + den).to_bits());
|
||||
assert_eq!(op(NARROW, 1e-39f64.to_bits(), 0, RN | FTZ), 0);
|
||||
assert_eq!(op(WIDEN | DOUBLE, 1e-39f32.to_bits() as u64, 0, RN | DAZ), 0);
|
||||
// Rounds to the smallest normal only in the format's denormals: tiny.
|
||||
let min = f64::MIN_POSITIVE;
|
||||
assert_eq!(d(MUL, f64::from_bits(0x3fef_ffff_ffff_ffff), min, RN | FTZ).to_bits(), 0);
|
||||
assert_eq!(d(MUL, f64::from_bits(0x3fef_ffff_ffff_ffff), min, RN).to_bits(), min.to_bits());
|
||||
// Rounds to it with the exponent unbounded too: not tiny.
|
||||
let below = d(MUL, f64::from_bits(0x3ff0_0000_0000_0001), f64::from_bits(0x000f_ffff_ffff_ffff), RN | FTZ);
|
||||
assert_eq!(below, min);
|
||||
assert_eq!(s(MUL, 0.99999994, f32::MIN_POSITIVE, RN | FTZ).to_bits(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nans() {
|
||||
let qa = f64::from_bits(0x7ff8_0000_0000_0001);
|
||||
let sb = f64::from_bits(0x7ff0_0000_0000_0002);
|
||||
assert_eq!(op(ADD | DOUBLE, qa.to_bits(), sb.to_bits(), RZ), 0x7ff8_0000_0000_0001);
|
||||
assert_eq!(op(ADD | DOUBLE, 1f64.to_bits(), sb.to_bits(), RZ), 0x7ff8_0000_0000_0002);
|
||||
assert_eq!(op(SUB | DOUBLE, f64::INFINITY.to_bits(), f64::INFINITY.to_bits(), RZ), 0xfff8_0000_0000_0000);
|
||||
assert_eq!(op(SQRT, (-1f32).to_bits() as u64, 0, RU), 0xffc0_0000);
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ struct Imports {
|
||||
safepoint: u32,
|
||||
fma: u32,
|
||||
x87: u32,
|
||||
mx: u32,
|
||||
dispatch: u32,
|
||||
block_ty: u32,
|
||||
}
|
||||
@@ -58,6 +59,7 @@ fn host_imports(m: &mut Module) -> Imports {
|
||||
vec![p, ValType::I32, ValType::I64, ValType::I32, ValType::I64, ValType::I32, ValType::I32],
|
||||
vec![ValType::I64],
|
||||
));
|
||||
let t_mx = m.func_type(FuncType::new(vec![p, ValType::I32, ValType::I64, ValType::I64, ValType::I32], vec![ValType::I64]));
|
||||
let im = IMPORT_MODULE;
|
||||
let out = Imports {
|
||||
syscall: m.import_func(im, imports::SYSCALL, t_syscall),
|
||||
@@ -70,6 +72,7 @@ fn host_imports(m: &mut Module) -> Imports {
|
||||
safepoint: m.import_func(im, imports::SAFEPOINT, t_syscall),
|
||||
fma: m.import_func(im, imports::FMA, t_fma),
|
||||
x87: m.import_func(im, imports::X87, t_x87),
|
||||
mx: m.import_func(im, imports::MX, t_mx),
|
||||
dispatch: m.import_func(gex_cgen::splice::DISPATCH_MODULE, gex_cgen::splice::DISPATCH_NAME, block_ty),
|
||||
block_ty,
|
||||
};
|
||||
@@ -124,6 +127,7 @@ pub fn build_module(
|
||||
safepoint: im.safepoint,
|
||||
fma: im.fma,
|
||||
x87: im.x87,
|
||||
mx: im.mx,
|
||||
block_ty: im.block_ty,
|
||||
entry_ty: 0,
|
||||
blocks: &no_blocks,
|
||||
@@ -383,10 +387,10 @@ pub fn build_runtime(
|
||||
let mut funcs: Vec<Function> = funcs.to_vec();
|
||||
loop {
|
||||
let loops: Vec<Vec<Loop>> = funcs.iter().map(|f| if outline { cf::outer_loops(f) } else { Vec::new() }).collect();
|
||||
// Imports: tier-1's ten host functions, in `host_imports`' order.
|
||||
const N_IMPORTS: u32 = 10;
|
||||
let (block_ty, t_syscall, t_miss, t_pc_i32, t_sysreg_read, t_pc_i32_i64, t_fma, t_x87, body_ty, loop_ty) =
|
||||
(0u32, 1, 2, 3, 4, 5, 6, 7, 8, 9);
|
||||
// Imports: tier-1's eleven host functions, in `host_imports`' order.
|
||||
const N_IMPORTS: u32 = 11;
|
||||
let (block_ty, t_syscall, t_miss, t_pc_i32, t_sysreg_read, t_pc_i32_i64, t_fma, t_x87, body_ty, loop_ty, t_mx) =
|
||||
(0u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
|
||||
let mut next = N_IMPORTS;
|
||||
let mut bodies: HashMap<u64, u32> = HashMap::new();
|
||||
let mut lfns: HashMap<(u64, usize), u32> = HashMap::new();
|
||||
@@ -415,6 +419,7 @@ pub fn build_runtime(
|
||||
safepoint: 7,
|
||||
fma: 8,
|
||||
x87: 9,
|
||||
mx: 10,
|
||||
block_ty,
|
||||
entry_ty: 0,
|
||||
blocks: &no_blocks,
|
||||
@@ -467,6 +472,7 @@ pub fn build_runtime(
|
||||
types.ty().function([i32_, i32_, i64_, i32_, i64_, i32_, i32_], [i64_]);
|
||||
types.ty().function([i32_, i32_, i32_, i64_], [i32_]);
|
||||
types.ty().function([i32_, i32_, i32_, i64_, i64_], [i32_]);
|
||||
types.ty().function([i32_, i32_, i64_, i64_, i32_], [i64_]);
|
||||
m.section(&types);
|
||||
let mut imports = ImportSection::new();
|
||||
let im = gex_abi::IMPORT_MODULE;
|
||||
@@ -481,6 +487,7 @@ pub fn build_runtime(
|
||||
(imports::SAFEPOINT, t_syscall),
|
||||
(imports::FMA, t_fma),
|
||||
(imports::X87, t_x87),
|
||||
(imports::MX, t_mx),
|
||||
] {
|
||||
imports.import(im, name, EntityType::Function(ty));
|
||||
}
|
||||
|
||||
+76
-67
@@ -723,74 +723,8 @@ pub fn for_each_def(s: &Stmt, mut f: impl FnMut(Value)) {
|
||||
|
||||
/// Mutable access to every value a statement reads.
|
||||
pub fn for_each_use_mut(s: &mut Stmt, mut f: impl FnMut(&mut Value)) {
|
||||
use crate::Expr::*;
|
||||
match s {
|
||||
Stmt::Def { expr, .. } => match expr {
|
||||
ConstI32(_) | ConstI64(_) | ReadReg(_) | VConst(_) => {}
|
||||
Bin { a, b, .. } | Cmp { a, b, .. } => {
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
Un { a, .. } | Wrap(a) | Extend { a, .. } | ExtendBits { a, .. } => f(a),
|
||||
Select { cond, a, b, .. } => {
|
||||
f(cond);
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
Load { addr, .. } | VLoad { addr, .. } => f(addr),
|
||||
Rmw { addr, val, .. } => {
|
||||
f(addr);
|
||||
f(val);
|
||||
}
|
||||
CmpXchg {
|
||||
addr,
|
||||
expected,
|
||||
replacement,
|
||||
..
|
||||
} => {
|
||||
f(addr);
|
||||
f(expected);
|
||||
f(replacement);
|
||||
}
|
||||
HostCall { args, .. } => args.iter_mut().for_each(f),
|
||||
VSplat { a, .. }
|
||||
| VExtract { a, .. }
|
||||
| VUn { a, .. }
|
||||
| VBitmask { a, .. }
|
||||
| VLaneUn { a, .. } => f(a),
|
||||
VLaneBin { a, b, .. } | VShuffle { a, b, .. } | VBin { a, b, .. } => {
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
VShift { a, amount, .. } => {
|
||||
f(a);
|
||||
f(amount);
|
||||
}
|
||||
VSwizzle { a, idx } => {
|
||||
f(a);
|
||||
f(idx);
|
||||
}
|
||||
VBitSelect { mask, a, b } => {
|
||||
f(mask);
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
VCvt { a, b, .. } => {
|
||||
f(a);
|
||||
if let Some(b) = b {
|
||||
f(b);
|
||||
}
|
||||
}
|
||||
VReplace { a, val, .. } => {
|
||||
f(a);
|
||||
f(val);
|
||||
}
|
||||
FpBin { a, b, .. } | FpCmp { a, b, .. } => {
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
FpUn { a, .. } | FpCvt { a, .. } => f(a),
|
||||
},
|
||||
Stmt::Def { expr, .. } => expr_uses_mut(expr, &mut f),
|
||||
Stmt::WriteReg { val, .. } => f(val),
|
||||
Stmt::Store { addr, val, .. } | Stmt::VStore { addr, val, .. } => {
|
||||
f(addr);
|
||||
@@ -1305,3 +1239,78 @@ impl Summary {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mutable access to every value an expression reads.
|
||||
fn expr_uses_mut(expr: &mut crate::Expr, f: &mut impl FnMut(&mut Value)) {
|
||||
use crate::Expr::*;
|
||||
match expr {
|
||||
ConstI32(_) | ConstI64(_) | ReadReg(_) | VConst(_) => {}
|
||||
Bin { a, b, .. } | Cmp { a, b, .. } => {
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
Un { a, .. } | Wrap(a) | Extend { a, .. } | ExtendBits { a, .. } => f(a),
|
||||
Select { cond, a, b, .. } => {
|
||||
f(cond);
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
Load { addr, .. } | VLoad { addr, .. } => f(addr),
|
||||
Rmw { addr, val, .. } => {
|
||||
f(addr);
|
||||
f(val);
|
||||
}
|
||||
CmpXchg {
|
||||
addr,
|
||||
expected,
|
||||
replacement,
|
||||
..
|
||||
} => {
|
||||
f(addr);
|
||||
f(expected);
|
||||
f(replacement);
|
||||
}
|
||||
HostCall { args, .. } => args.iter_mut().for_each(f),
|
||||
VSplat { a, .. }
|
||||
| VExtract { a, .. }
|
||||
| VUn { a, .. }
|
||||
| VBitmask { a, .. }
|
||||
| VLaneUn { a, .. } => f(a),
|
||||
VLaneBin { a, b, .. } | VShuffle { a, b, .. } | VBin { a, b, .. } => {
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
VShift { a, amount, .. } => {
|
||||
f(a);
|
||||
f(amount);
|
||||
}
|
||||
VSwizzle { a, idx } => {
|
||||
f(a);
|
||||
f(idx);
|
||||
}
|
||||
VBitSelect { mask, a, b } => {
|
||||
f(mask);
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
VCvt { a, b, .. } => {
|
||||
f(a);
|
||||
if let Some(b) = b {
|
||||
f(b);
|
||||
}
|
||||
}
|
||||
VReplace { a, val, .. } => {
|
||||
f(a);
|
||||
f(val);
|
||||
}
|
||||
FpBin { a, b, .. } | FpCmp { a, b, .. } => {
|
||||
f(a);
|
||||
f(b);
|
||||
}
|
||||
FpUn { a, .. } | FpCvt { a, .. } => f(a),
|
||||
Mx { mx, e } => {
|
||||
f(mx);
|
||||
expr_uses_mut(e, f);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -230,6 +230,7 @@ pub fn run_from(f: &Function, seed: u64, mut regs: Vec<u64>, m: &Model) -> Outco
|
||||
| Expr::FpUn { .. }
|
||||
| Expr::FpCmp { .. }
|
||||
| Expr::FpCvt { .. }
|
||||
| Expr::Mx { .. }
|
||||
| Expr::HostCall { .. }
|
||||
| Expr::Rmw { .. }
|
||||
| Expr::CmpXchg { .. }
|
||||
|
||||
@@ -472,6 +472,108 @@ pub enum Expr {
|
||||
dst_wide: bool,
|
||||
a: Value,
|
||||
},
|
||||
/// `e` computed as x86's SSE computes it under the `I32` MXCSR value
|
||||
/// `mx`: rounded by its rounding control (bits 13-14) instead of to
|
||||
/// nearest, a denormal operand read as a zero of its sign under DAZ
|
||||
/// (bit 6), a denormal result written as one under FTZ (bit 15). With
|
||||
/// none of those bits set ([`MX_SPECIAL`]) it is `e` exactly, so a
|
||||
/// backend lowers it as `e` behind that one test.
|
||||
///
|
||||
/// `e` is one of [`Expr::mx_able`]'s: a float add, subtract, multiply,
|
||||
/// divide or square root, a round to nearest (an integer value, the
|
||||
/// rounding of a conversion to integer), a conversion to a float, as
|
||||
/// scalars (`FpBin`, `FpUn`, `FpCvt`) or lanes (`VLaneBin`, `VLaneUn`,
|
||||
/// `VCvt`). Its operands are this expression's.
|
||||
Mx { mx: Value, e: Box<Expr> },
|
||||
}
|
||||
|
||||
/// The MXCSR bits [`Expr::Mx`] honours: rounding control, FTZ, DAZ.
|
||||
pub const MX_SPECIAL: u32 = 0xe040;
|
||||
|
||||
/// How an [`Expr::Mx`]'s operation is computed off its fast path: by
|
||||
/// `gex_f80::sse::op` with `code`, on raw bits, once for a scalar or once
|
||||
/// per lane.
|
||||
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
|
||||
pub enum MxPlan {
|
||||
/// `a` (and `b`), each an integer of `in_ty` holding a float's bits or
|
||||
/// a conversion's integer; the result is the expression's type.
|
||||
Scalar { code: u32, a: Value, b: Option<Value>, in_ty: Ty },
|
||||
/// Lanes `0..n` of `a` (and `b`), `in_bytes` wide, each to lane `k`,
|
||||
/// `out_bytes` wide, of a zero vector.
|
||||
Lanes { code: u32, a: Value, b: Option<Value>, n: u32, in_bytes: u32, out_bytes: u32 },
|
||||
}
|
||||
|
||||
impl MxPlan {
|
||||
/// The plan of [`Expr::mx_able`] `e`.
|
||||
pub fn of(e: &Expr) -> MxPlan {
|
||||
use gex_f80::sse::*;
|
||||
let bin = |op: u32| match op {
|
||||
0 => ADD,
|
||||
1 => SUB,
|
||||
2 => MUL,
|
||||
_ => DIV,
|
||||
};
|
||||
let wide = |d: bool| if d { DOUBLE } else { 0 };
|
||||
let int_ty = |d: bool| if d { Ty::I64 } else { Ty::I32 };
|
||||
match *e {
|
||||
Expr::FpBin { op, double, a, b } => MxPlan::Scalar {
|
||||
code: bin(op as u32) | wide(double),
|
||||
a,
|
||||
b: Some(b),
|
||||
in_ty: int_ty(double),
|
||||
},
|
||||
Expr::FpUn { op, double, a } => MxPlan::Scalar {
|
||||
code: if op == FpUnOp::Sqrt { SQRT } else { RINT } | wide(double),
|
||||
a,
|
||||
b: None,
|
||||
in_ty: int_ty(double),
|
||||
},
|
||||
Expr::FpCvt { kind: FpConvert::IntToFp { .. }, src_wide, dst_wide, a } => MxPlan::Scalar {
|
||||
code: if src_wide { FROM_I64 } else { FROM_I32 } | wide(dst_wide),
|
||||
a,
|
||||
b: None,
|
||||
in_ty: int_ty(src_wide),
|
||||
},
|
||||
Expr::FpCvt { src_wide, a, .. } => MxPlan::Scalar {
|
||||
code: if src_wide { NARROW } else { WIDEN | DOUBLE },
|
||||
a,
|
||||
b: None,
|
||||
in_ty: int_ty(src_wide),
|
||||
},
|
||||
Expr::VLaneBin { op, lane, a, b } => {
|
||||
let d = lane == VLane::F64;
|
||||
let k = match op {
|
||||
VLaneBinOp::Add => ADD,
|
||||
VLaneBinOp::Sub => SUB,
|
||||
VLaneBinOp::Mul => MUL,
|
||||
_ => DIV,
|
||||
};
|
||||
let lb = lane.bytes();
|
||||
MxPlan::Lanes { code: k | wide(d), a, b: Some(b), n: 16 / lb, in_bytes: lb, out_bytes: lb }
|
||||
}
|
||||
Expr::VLaneUn { op, lane, a } => {
|
||||
let d = lane == VLane::F64;
|
||||
let lb = lane.bytes();
|
||||
let k = if op == VLaneUnOp::Sqrt { SQRT } else { RINT };
|
||||
MxPlan::Lanes { code: k | wide(d), a, b: None, n: 16 / lb, in_bytes: lb, out_bytes: lb }
|
||||
}
|
||||
Expr::VCvt { op: VCvtOp::DemoteF64, a, .. } => {
|
||||
MxPlan::Lanes { code: NARROW, a, b: None, n: 2, in_bytes: 8, out_bytes: 4 }
|
||||
}
|
||||
Expr::VCvt { op: VCvtOp::PromoteF32, a, .. } => {
|
||||
MxPlan::Lanes { code: WIDEN | DOUBLE, a, b: None, n: 2, in_bytes: 4, out_bytes: 8 }
|
||||
}
|
||||
Expr::VCvt { a, .. } => MxPlan::Lanes { code: FROM_I32, a, b: None, n: 4, in_bytes: 4, out_bytes: 4 },
|
||||
_ => panic!("MxPlan::of: {e:?} is not an operation MXCSR rounds"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The operands are floats (DAZ reads them), not a conversion's integers.
|
||||
pub fn float_operands(&self) -> bool {
|
||||
use gex_f80::sse::*;
|
||||
let (MxPlan::Scalar { code, .. } | MxPlan::Lanes { code, .. }) = *self;
|
||||
!matches!(code & 0xf, FROM_I32 | FROM_I64)
|
||||
}
|
||||
}
|
||||
|
||||
/// Scalar floating-point operations.
|
||||
@@ -716,6 +818,25 @@ impl Expr {
|
||||
Ty::I32
|
||||
}
|
||||
}
|
||||
Expr::Mx { ref e, .. } => e.ty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `self` may be the operation of an [`Expr::Mx`].
|
||||
pub fn mx_able(&self) -> bool {
|
||||
match self {
|
||||
Expr::FpBin { op, .. } => matches!(op, FpBinOp::Add | FpBinOp::Sub | FpBinOp::Mul | FpBinOp::Div),
|
||||
Expr::FpUn { op, .. } => matches!(op, FpUnOp::Sqrt | FpUnOp::RoundNearest),
|
||||
Expr::FpCvt { kind, .. } => matches!(kind, FpConvert::IntToFp { signed: true } | FpConvert::FpToFp),
|
||||
Expr::VLaneBin { op, lane, .. } => {
|
||||
lane.is_float() && matches!(op, VLaneBinOp::Add | VLaneBinOp::Sub | VLaneBinOp::Mul | VLaneBinOp::Div)
|
||||
}
|
||||
Expr::VLaneUn { op, lane, .. } => lane.is_float() && matches!(op, VLaneUnOp::Sqrt | VLaneUnOp::Nearest),
|
||||
Expr::VCvt { op, from, .. } => matches!(
|
||||
(op, from),
|
||||
(VCvtOp::DemoteF64, VLane::F64) | (VCvtOp::PromoteF32, VLane::F32) | (VCvtOp::IntToFp { signed: true }, VLane::I32)
|
||||
),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -793,6 +914,10 @@ impl Expr {
|
||||
f(*b);
|
||||
}
|
||||
Expr::FpUn { a, .. } | Expr::FpCvt { a, .. } => f(*a),
|
||||
Expr::Mx { mx, e } => {
|
||||
f(*mx);
|
||||
e.for_each_use(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,6 +180,7 @@ fn expr(e: &Expr) -> String {
|
||||
dst_wide,
|
||||
a,
|
||||
} => format!("fcvt.{kind:?} {}->{} {a:?}", fw(*src_wide), fw(*dst_wide)),
|
||||
Expr::Mx { mx, e } => format!("mx({mx:?}) {}", expr(e)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -411,6 +411,7 @@ pub fn expr(e: &Expr) -> String {
|
||||
fw(*dst_wide),
|
||||
v(*a)
|
||||
),
|
||||
Expr::Mx { mx, e } => format!("mx {} {}", v(*mx), expr(e)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -990,6 +991,14 @@ fn parse_expr(k: &mut Toks) -> Result<Expr, String> {
|
||||
a: k.val()?,
|
||||
}
|
||||
}
|
||||
"mx" => {
|
||||
let mx = k.val()?;
|
||||
let e = parse_expr(k)?;
|
||||
if !e.mx_able() {
|
||||
return Err(k.err("mx: not an operation MXCSR rounds".to_string()));
|
||||
}
|
||||
Expr::Mx { mx, e: Box::new(e) }
|
||||
}
|
||||
other => return Err(k.err(format!("unknown expression `{other}`"))),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -134,14 +134,38 @@ impl<'a> Lifter<'a> {
|
||||
})
|
||||
}
|
||||
|
||||
/// A float lane operation; one MXCSR rounds (arithmetic, see
|
||||
/// [`Self::mx`]) under the guest's MXCSR.
|
||||
fn vbin(&mut self, op: VLaneBinOp, lane: VLane, a: Value, b: Value) -> Value {
|
||||
self.b.def(Expr::VLaneBin { op, lane, a, b })
|
||||
self.mx(Expr::VLaneBin { op, lane, a, b })
|
||||
}
|
||||
|
||||
/// A lane operation, under MXCSR when it is a square root. (A round to
|
||||
/// nearest is under MXCSR only when the instruction says so:
|
||||
/// [`Self::vun_mx`].)
|
||||
fn vun(&mut self, op: VLaneUnOp, lane: VLane, a: Value) -> Value {
|
||||
if op == VLaneUnOp::Sqrt {
|
||||
return self.mx(Expr::VLaneUn { op, lane, a });
|
||||
}
|
||||
self.b.def(Expr::VLaneUn { op, lane, a })
|
||||
}
|
||||
|
||||
/// A lane operation under the guest's MXCSR.
|
||||
fn vun_mx(&mut self, op: VLaneUnOp, lane: VLane, a: Value) -> Value {
|
||||
self.mx(Expr::VLaneUn { op, lane, a })
|
||||
}
|
||||
|
||||
/// `e` as SSE computes it: under the guest's MXCSR (its rounding
|
||||
/// control, FTZ and DAZ) when it is an operation MXCSR controls
|
||||
/// ([`Expr::mx_able`]), else as it is.
|
||||
fn mx(&mut self, e: Expr) -> Value {
|
||||
if !e.mx_able() {
|
||||
return self.b.def(e);
|
||||
}
|
||||
let mx = self.b.read(GuestReg::Mxcsr);
|
||||
self.b.def(Expr::Mx { mx, e: Box::new(e) })
|
||||
}
|
||||
|
||||
fn vbit(&mut self, op: VBinOp, a: Value, b: Value) -> Value {
|
||||
self.b.def(Expr::VBin { op, a, b })
|
||||
}
|
||||
@@ -192,7 +216,7 @@ impl<'a> Lifter<'a> {
|
||||
}
|
||||
|
||||
fn fbin(&mut self, op: FpBinOp, double: bool, a: Value, b: Value) -> Value {
|
||||
self.b.def(Expr::FpBin { op, double, a, b })
|
||||
self.mx(Expr::FpBin { op, double, a, b })
|
||||
}
|
||||
|
||||
fn fcmp(&mut self, op: FpCmpOp, double: bool, a: Value, b: Value) -> Value {
|
||||
@@ -337,7 +361,7 @@ impl<'a> Lifter<'a> {
|
||||
let x = if truncate {
|
||||
x
|
||||
} else {
|
||||
self.b.def(Expr::FpUn {
|
||||
self.mx(Expr::FpUn {
|
||||
op: FpUnOp::RoundNearest,
|
||||
double: src_double,
|
||||
a: x,
|
||||
@@ -617,7 +641,7 @@ impl<'a> Lifter<'a> {
|
||||
self.wr_x(i.op0(), r);
|
||||
} else {
|
||||
// cvtpi2ps: the low two floats; the upper two are kept.
|
||||
let c = self.b.def(Expr::VCvt {
|
||||
let c = self.mx(Expr::VCvt {
|
||||
op: VCvtOp::IntToFp { signed: true },
|
||||
from: VLane::I32,
|
||||
a: s,
|
||||
@@ -799,7 +823,7 @@ impl<'a> Lifter<'a> {
|
||||
let dv = self.rd_x(i.op0());
|
||||
let a = self.rd_scalar_src(i.op1(), lane);
|
||||
let r = match i.op {
|
||||
Op::Sqrtp => self.b.def(Expr::FpUn {
|
||||
Op::Sqrtp => self.mx(Expr::FpUn {
|
||||
op: FpUnOp::Sqrt,
|
||||
double,
|
||||
a,
|
||||
@@ -921,15 +945,18 @@ impl<'a> Lifter<'a> {
|
||||
Op::Roundp => {
|
||||
let Operand::Imm(imm) = i.op2() else { return self.illegal() };
|
||||
let (vop, sop) = Self::round_op(imm);
|
||||
// Bit 2: MXCSR's rounding control decides.
|
||||
let by_mx = imm & 4 != 0;
|
||||
if i.scalar {
|
||||
let dv = self.rd_x(i.op0());
|
||||
let a = self.rd_scalar_src(i.op1(), lane);
|
||||
let r = self.b.def(Expr::FpUn { op: sop, double, a });
|
||||
let e = Expr::FpUn { op: sop, double, a };
|
||||
let r = if by_mx { self.mx(e) } else { self.b.def(e) };
|
||||
let r = self.with_scalar(dv, lane, r);
|
||||
self.wr_x(i.op0(), r);
|
||||
} else {
|
||||
let a = self.rd_x(i.op1());
|
||||
let r = self.vun(vop, vl, a);
|
||||
let r = if by_mx { self.vun_mx(vop, vl, a) } else { self.vun(vop, vl, a) };
|
||||
self.wr_x(i.op0(), r);
|
||||
}
|
||||
}
|
||||
@@ -988,7 +1015,7 @@ impl<'a> Lifter<'a> {
|
||||
Op::Cvtsi2s => {
|
||||
let w = i.width;
|
||||
let v = self.rd(i.op1(), w);
|
||||
let f = self.b.def(Expr::FpCvt {
|
||||
let f = self.mx(Expr::FpCvt {
|
||||
kind: FpConvert::IntToFp { signed: true },
|
||||
src_wide: w == Width::Q,
|
||||
dst_wide: double,
|
||||
@@ -1007,7 +1034,7 @@ impl<'a> Lifter<'a> {
|
||||
Op::Cvts2s => {
|
||||
// lane is the SOURCE type.
|
||||
let x = self.rd_scalar_src(i.op1(), lane);
|
||||
let r = self.b.def(Expr::FpCvt {
|
||||
let r = self.mx(Expr::FpCvt {
|
||||
kind: FpConvert::FpToFp,
|
||||
src_wide: double,
|
||||
dst_wide: !double,
|
||||
@@ -1020,7 +1047,7 @@ impl<'a> Lifter<'a> {
|
||||
}
|
||||
Op::Cvtp2p => {
|
||||
let s = self.rd_x_n(i.op1(), if double { 16 } else { 8 });
|
||||
let r = self.b.def(Expr::VCvt {
|
||||
let r = self.mx(Expr::VCvt {
|
||||
op: if double { VCvtOp::DemoteF64 } else { VCvtOp::PromoteF32 },
|
||||
from: vl,
|
||||
a: s,
|
||||
@@ -1042,7 +1069,7 @@ impl<'a> Lifter<'a> {
|
||||
self.wr_x(i.op0(), r);
|
||||
} else {
|
||||
let s = self.rd_x(i.op1());
|
||||
let r = self.b.def(Expr::VCvt {
|
||||
let r = self.mx(Expr::VCvt {
|
||||
op: VCvtOp::IntToFp { signed: true },
|
||||
from: VLane::I32,
|
||||
a: s,
|
||||
@@ -1056,7 +1083,7 @@ impl<'a> Lifter<'a> {
|
||||
let s = if truncate {
|
||||
s
|
||||
} else {
|
||||
self.vun(VLaneUnOp::Nearest, vl, s)
|
||||
self.vun_mx(VLaneUnOp::Nearest, vl, s)
|
||||
};
|
||||
let r = self.b.def(Expr::VCvt {
|
||||
op: VCvtOp::FpToIntSat { signed: true },
|
||||
|
||||
@@ -29,6 +29,7 @@ fn main() {
|
||||
dispatch_miss: 7,
|
||||
safepoint: 8,
|
||||
x87: 9,
|
||||
mx: 10,
|
||||
block_ty: 0,
|
||||
entry_ty: 0,
|
||||
blocks: &blocks,
|
||||
|
||||
@@ -115,6 +115,7 @@ fn main() {
|
||||
dispatch_miss: 7,
|
||||
safepoint: 8,
|
||||
x87: 9,
|
||||
mx: 10,
|
||||
block_ty: 0,
|
||||
entry_ty: 0,
|
||||
blocks: &ids,
|
||||
|
||||
@@ -2267,6 +2267,7 @@ export class Gex {
|
||||
safepoint: single.on ? this.#singleSafepoint(e) : e.gex_safepoint,
|
||||
fma: e.gex_fma,
|
||||
x87: e.gex_x87,
|
||||
mx: e.gex_mx,
|
||||
// A counting unit's table calls (cgen.rs `gex_icall_note`).
|
||||
icall_note: e.gex_icall_note,
|
||||
};
|
||||
@@ -2479,6 +2480,7 @@ export class Gex {
|
||||
safepoint: single.on ? this.#singleSafepoint(e) : e.gex_safepoint,
|
||||
fma: e.gex_fma,
|
||||
x87: e.gex_x87,
|
||||
mx: e.gex_mx,
|
||||
// A counting unit's table calls (cgen.rs `gex_icall_note`).
|
||||
icall_note: e.gex_icall_note,
|
||||
};
|
||||
|
||||
@@ -13956,6 +13956,13 @@ pub extern "C" fn gex_fma(_ctx: i32, a: i64, b: i64, c: i64, double: i32) -> i64
|
||||
}
|
||||
}
|
||||
|
||||
/// An SSE operation under a non-default MXCSR (`gex_abi::imports::MX`):
|
||||
/// exact, pure.
|
||||
#[no_mangle]
|
||||
pub extern "C" fn gex_mx(_ctx: i32, code: i32, a: i64, b: i64, mx: i32) -> i64 {
|
||||
gex_kernel::f80::sse::op(code as u32, a as u64, b as u64, mx as u32) as i64
|
||||
}
|
||||
|
||||
/// An x87 operation the inline fast path left to the host (see
|
||||
/// `gex_abi::imports::X87`): the first result returned, an extended
|
||||
/// result's sign and exponent left at `ctx::X87_TMP + 8` and the
|
||||
|
||||
@@ -4996,6 +4996,11 @@ fn register_imports(linker: &mut Linker<HostState>) -> Result<()> {
|
||||
})()) },
|
||||
)?;
|
||||
|
||||
// An SSE operation under a non-default MXCSR (`gex_abi::imports::MX`).
|
||||
linker.func_wrap(m, imports::MX, |_ctx: i32, code: i32, a: i64, b: i64, mx: i32| -> i64 {
|
||||
gex_kernel::f80::sse::op(code as u32, a as u64, b as u64, mx as u32) as i64
|
||||
})?;
|
||||
|
||||
// The counting flavor's table-call note (`gex_cgen::image::icall_count`):
|
||||
// natively a debugging run only, so nothing is tallied.
|
||||
linker.func_wrap(m, imports::ICALL_NOTE, |_ctx: i32, _ret: i64, _target: i64| {})?;
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
/* The rounding and denormal controls: SSE arithmetic and conversions under
|
||||
* each MXCSR rounding mode and with FTZ and DAZ, and x87 arithmetic and
|
||||
* stores under each rounding and precision control of the control word.
|
||||
* Each line is one operation in one mode, a hash of its results over every
|
||||
* pair of a set of operands (denormals, values a rounding mode moves, ones
|
||||
* that overflow or underflow), compared with the hardware's.
|
||||
*
|
||||
* The modes are set with ldmxcsr / fldcw and left set across the operations,
|
||||
* which run on operands loaded after the mode is set (volatile loads behind
|
||||
* an asm with a memory clobber), so the compiler neither folds nor hoists
|
||||
* them. The SSE part is x86-64's; the x87 part runs on both. */
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#if defined(__x86_64__)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
static uint64_t h;
|
||||
static void mix(uint64_t v) {
|
||||
for (int i = 0; i < 8; i++) h = (h ^ ((v >> (8 * i)) & 0xff)) * 0x100000001b3ull;
|
||||
}
|
||||
static uint64_t dbits(double d) { uint64_t u; memcpy(&u, &d, 8); return u; }
|
||||
static uint32_t fbits(float f) { uint32_t u; memcpy(&u, &f, 4); return u; }
|
||||
|
||||
static volatile double D[] = {
|
||||
1.0, 1.0 / 3.0, -2.0 / 7.0, 0.1, -1e-300, 1e300, -1.7976931348623157e308,
|
||||
2.2250738585072014e-308, 1.5 * 2.2250738585072014e-308, -4.9406564584124654e-324,
|
||||
1.1125369292536007e-308, 3.0e-310, 1e-160, -7.5, 1e16 + 3.0, 0.0, 1e-39, -3.5e-41,
|
||||
/* x * 2^-1022 is tiny for the first (it rounds to the smallest normal
|
||||
* only in the denormals), and the next two's product is not. */
|
||||
0.99999999999999989, 1.0000000000000002, 2.2250738585072009e-308,
|
||||
};
|
||||
static volatile float F[] = {
|
||||
1.0f, 1.0f / 3.0f, -2.0f / 7.0f, 0.1f, -1e-30f, 1e30f, -3.4028235e38f,
|
||||
1.17549435e-38f, 1.5f * 1.17549435e-38f, -1.4e-45f, 5.9e-39f, 3.0e-40f, 1e-20f, -7.5f,
|
||||
16777217.0f, 0.0f, 0.99999994f, 1.00000012f, 1.17549421e-38f,
|
||||
};
|
||||
static volatile int64_t I[] = {
|
||||
1, -3, 16777217, -16777219, 9007199254740993ll, -9007199254740995ll,
|
||||
0x7fffffffffffffffll, 123456789123ll, 2147483647, -2147483647 - 1, 0, 33554435,
|
||||
};
|
||||
#define ND (int)(sizeof D / sizeof D[0])
|
||||
#define NF (int)(sizeof F / sizeof F[0])
|
||||
#define NI (int)(sizeof I / sizeof I[0])
|
||||
|
||||
#define FENCE() __asm__ volatile("" ::: "memory")
|
||||
|
||||
#if defined(__x86_64__)
|
||||
static void setmx(uint32_t v) { __asm__ volatile("ldmxcsr %0" : : "m"(v) : "memory"); }
|
||||
|
||||
#define LINE(name, ...) do { h = 0xcbf29ce484222325ull; __VA_ARGS__; printf("mx %04x %-10s %016llx\n", mode, name, (unsigned long long)h); } while (0)
|
||||
#define PAIRS_D(expr) for (int i = 0; i < ND; i++) for (int j = 0; j < ND; j++) { FENCE(); double a = D[i], b = D[j]; (void)b; __m128d A = _mm_set_sd(a), B = _mm_set_sd(b); (void)A; (void)B; mix(expr); }
|
||||
#define PAIRS_F(expr) for (int i = 0; i < NF; i++) for (int j = 0; j < NF; j++) { FENCE(); float a = F[i], b = F[j]; (void)b; __m128 A = _mm_set_ss(a), B = _mm_set_ss(b); (void)A; (void)B; mix(expr); }
|
||||
#define EACH_D(expr) for (int i = 0; i < ND; i++) { FENCE(); double a = D[i]; __m128d A = _mm_set_sd(a); (void)A; mix(expr); }
|
||||
#define EACH_F(expr) for (int i = 0; i < NF; i++) { FENCE(); float a = F[i]; __m128 A = _mm_set_ss(a); (void)A; mix(expr); }
|
||||
#define EACH_I(expr) for (int i = 0; i < NI; i++) { FENCE(); int64_t n = I[i]; (void)n; mix(expr); }
|
||||
/* Packed: four (two) lanes from consecutive operands. */
|
||||
#define PACK_F(k) _mm_set_ps(F[(k) % NF], F[((k) + 3) % NF], F[((k) + 7) % NF], F[((k) + 11) % NF])
|
||||
#define PACK_D(k) _mm_set_pd(D[(k) % ND], D[((k) + 5) % ND])
|
||||
#define PAIRS_PF(expr) for (int i = 0; i < NF; i++) for (int j = 0; j < NF; j++) { FENCE(); __m128 A = PACK_F(i), B = PACK_F(j); __m128i r = _mm_castps_si128(expr); mix((uint64_t)_mm_cvtsi128_si64(r)); mix((uint64_t)_mm_cvtsi128_si64(_mm_unpackhi_epi64(r, r))); }
|
||||
#define PAIRS_PD(expr) for (int i = 0; i < ND; i++) for (int j = 0; j < ND; j++) { FENCE(); __m128d A = PACK_D(i), B = PACK_D(j); __m128i r = _mm_castpd_si128(expr); mix((uint64_t)_mm_cvtsi128_si64(r)); mix((uint64_t)_mm_cvtsi128_si64(_mm_unpackhi_epi64(r, r))); }
|
||||
#define EACH_PI(expr) for (int i = 0; i < NF; i++) { FENCE(); __m128 A = PACK_F(i); __m128d AD = PACK_D(i); __m128i AI = _mm_set_epi32((int)I[i % NI], (int)I[(i + 2) % NI], (int)I[(i + 3) % NI], (int)I[(i + 5) % NI]); (void)A; (void)AD; (void)AI; __m128i r = (expr); mix((uint64_t)_mm_cvtsi128_si64(r)); mix((uint64_t)_mm_cvtsi128_si64(_mm_unpackhi_epi64(r, r))); }
|
||||
|
||||
/* roundsd/roundps by MXCSR's mode (SSE4.1, which every x86-64 host that
|
||||
* runs this has). */
|
||||
__attribute__((target("sse4.1"))) static void sse41(uint32_t mode) {
|
||||
LINE("roundsd", EACH_D(dbits(_mm_cvtsd_f64(_mm_round_sd(A, A, _MM_FROUND_CUR_DIRECTION)))));
|
||||
LINE("roundps", EACH_PI(_mm_castps_si128(_mm_round_ps(A, _MM_FROUND_CUR_DIRECTION))));
|
||||
}
|
||||
|
||||
static void sse(uint32_t mode) {
|
||||
setmx(0x1f80 | mode);
|
||||
LINE("addsd", PAIRS_D(dbits(_mm_cvtsd_f64(_mm_add_sd(A, B)))));
|
||||
LINE("subsd", PAIRS_D(dbits(_mm_cvtsd_f64(_mm_sub_sd(A, B)))));
|
||||
LINE("mulsd", PAIRS_D(dbits(_mm_cvtsd_f64(_mm_mul_sd(A, B)))));
|
||||
LINE("divsd", PAIRS_D(dbits(_mm_cvtsd_f64(_mm_div_sd(A, B)))));
|
||||
LINE("sqrtsd", EACH_D(dbits(_mm_cvtsd_f64(_mm_sqrt_sd(A, A)))));
|
||||
LINE("addss", PAIRS_F(fbits(_mm_cvtss_f32(_mm_add_ss(A, B)))));
|
||||
LINE("subss", PAIRS_F(fbits(_mm_cvtss_f32(_mm_sub_ss(A, B)))));
|
||||
LINE("mulss", PAIRS_F(fbits(_mm_cvtss_f32(_mm_mul_ss(A, B)))));
|
||||
LINE("divss", PAIRS_F(fbits(_mm_cvtss_f32(_mm_div_ss(A, B)))));
|
||||
LINE("sqrtss", EACH_F(fbits(_mm_cvtss_f32(_mm_sqrt_ss(A)))));
|
||||
LINE("addps", PAIRS_PF(_mm_add_ps(A, B)));
|
||||
LINE("subps", PAIRS_PF(_mm_sub_ps(A, B)));
|
||||
LINE("mulps", PAIRS_PF(_mm_mul_ps(A, B)));
|
||||
LINE("divps", PAIRS_PF(_mm_div_ps(A, B)));
|
||||
LINE("sqrtps", PAIRS_PF(_mm_sqrt_ps(A)));
|
||||
LINE("addpd", PAIRS_PD(_mm_add_pd(A, B)));
|
||||
LINE("mulpd", PAIRS_PD(_mm_mul_pd(A, B)));
|
||||
LINE("divpd", PAIRS_PD(_mm_div_pd(A, B)));
|
||||
LINE("sqrtpd", PAIRS_PD(_mm_sqrt_pd(A)));
|
||||
LINE("cvtsd2ss", EACH_D(fbits(_mm_cvtss_f32(_mm_cvtsd_ss(_mm_setzero_ps(), A)))));
|
||||
LINE("cvtss2sd", EACH_F(dbits(_mm_cvtsd_f64(_mm_cvtss_sd(_mm_setzero_pd(), A)))));
|
||||
LINE("cvtsd2si", EACH_D((uint64_t)_mm_cvtsd_si64(A)));
|
||||
LINE("cvtsd2si32", EACH_D((uint32_t)_mm_cvtsd_si32(A)));
|
||||
LINE("cvtss2si", EACH_F((uint64_t)_mm_cvtss_si64(A)));
|
||||
LINE("cvtsi2sd", EACH_I(dbits(_mm_cvtsd_f64(_mm_cvtsi64_sd(_mm_setzero_pd(), n)))));
|
||||
LINE("cvtsi2ss", EACH_I(fbits(_mm_cvtss_f32(_mm_cvtsi64_ss(_mm_setzero_ps(), n)))));
|
||||
LINE("cvtsi2ss32", EACH_I(fbits(_mm_cvtss_f32(_mm_cvtsi32_ss(_mm_setzero_ps(), (int)n)))));
|
||||
LINE("cvtpd2ps", EACH_PI(_mm_castps_si128(_mm_cvtpd_ps(AD))));
|
||||
LINE("cvtps2pd", EACH_PI(_mm_castpd_si128(_mm_cvtps_pd(A))));
|
||||
LINE("cvtps2dq", EACH_PI(_mm_cvtps_epi32(A)));
|
||||
LINE("cvtpd2dq", EACH_PI(_mm_cvtpd_epi32(AD)));
|
||||
LINE("cvtdq2ps", EACH_PI(_mm_castps_si128(_mm_cvtepi32_ps(AI))));
|
||||
sse41(mode);
|
||||
setmx(0x1f80);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* x87: the control word's rounding (bits 10-11) and precision (8-9). */
|
||||
static void setcw(uint16_t v) { __asm__ volatile("fldcw %0" : : "m"(v) : "memory"); }
|
||||
#define XLINE(name, ...) do { h = 0xcbf29ce484222325ull; __VA_ARGS__; printf("cw %04x %-10s %016llx\n", cw, name, (unsigned long long)h); } while (0)
|
||||
#define XPAIRS(expr) for (int i = 0; i < ND; i++) for (int j = 0; j < ND; j++) { FENCE(); long double a = D[i], b = D[j]; long double r = (expr); FENCE(); volatile long double v = r; uint64_t m; uint16_t e; memcpy(&m, (const void *)&v, 8); memcpy(&e, (const char *)&v + 8, 2); mix(m); mix(e); }
|
||||
|
||||
static void x87(uint16_t cw) {
|
||||
setcw(cw);
|
||||
XLINE("fadd", XPAIRS(a + b));
|
||||
XLINE("fmul", XPAIRS(a * b));
|
||||
XLINE("fdiv", XPAIRS(a / b));
|
||||
XLINE("fadd3", XPAIRS(a / 3 + b));
|
||||
XLINE("fst32", for (int i = 0; i < ND; i++) for (int j = 0; j < ND; j++) { FENCE(); long double a = D[i], b = D[j]; volatile float f = (float)(a * b); mix(fbits(f)); });
|
||||
XLINE("fst64", for (int i = 0; i < ND; i++) for (int j = 0; j < ND; j++) { FENCE(); long double a = D[i], b = D[j]; volatile double d = (double)(a / b); mix(dbits(d)); });
|
||||
XLINE("fistp", for (int i = 0; i < ND; i++) { FENCE(); long double a = D[i] * 3.7L; int64_t n; __asm__ volatile("fistpll %0" : "=m"(n) : "t"(a) : "st"); mix((uint64_t)n); });
|
||||
setcw(0x37f);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
#if defined(__x86_64__)
|
||||
static const uint32_t modes[] = {0x0000, 0x2000, 0x4000, 0x6000, 0x8000, 0x0040, 0x8040, 0xe040};
|
||||
for (unsigned k = 0; k < sizeof modes / sizeof modes[0]; k++) sse(modes[k]);
|
||||
#endif
|
||||
static const uint16_t cws[] = {0x037f, 0x077f, 0x0b7f, 0x0f7f, 0x007f, 0x027f, 0x0e7f, 0x047f};
|
||||
for (unsigned k = 0; k < sizeof cws / sizeof cws[0]; k++) x87(cws[k]);
|
||||
return 0;
|
||||
}
|
||||
@@ -220,7 +220,11 @@ SEED64=web/public/gex/seed.x86.m64.shared.wasm
|
||||
# sig-pause: pause, sigsuspend, and ppoll, poll, pselect and select on no
|
||||
# descriptors block until a second thread's signal ends them with EINTR
|
||||
# (i386's pause is a call of its own).
|
||||
PAGE_PROGRAMS=${PAGE_PROGRAMS:-"fork-dirs dense-lazy dense-holes spawn-threads served-entries page-cross-entry tgkill-futex exec-tail mem64-io mem64-net mem64-console mem64-smc mem64-reserve retire-during-lift close-while-reading thread-signals signal-storm exec-signals clock-monotonic main-thread-exit pi-mutex tcp-loopback pwait-sigmask sa-restart thread-churn kernel-contend exec-futex sleep-threads sendmsg-rights scm-fork ioctl-threads fdtable-threads mmap-threads codemap-threads proc-threads fs-threads sig-state-threads lift-threads fdmisc-threads task-ids sock-threads exec-dethread fork-thread-ctx fork-maps-threads fork-twice-thread quiet-threads fdobj-threads cred-threads kill-threads sigwait-threads poll-timeout tid-threads mount-threads exit-threads many-workers threads-many stale-loop-head file-stale-loop stale-cross-page xattr mem64-wx mmap-coherent mmap-remap mmap-writeback null-access sig-regs-threads sig-pause"}
|
||||
# fp-modes: SSE arithmetic, square roots, roundings and conversions under
|
||||
# each MXCSR rounding mode with FTZ and DAZ (wasm rounds only to nearest:
|
||||
# the other modes take the exact slow path), and x87 arithmetic and stores
|
||||
# under each rounding and precision control; under the function JIT too.
|
||||
PAGE_PROGRAMS=${PAGE_PROGRAMS:-"fork-dirs dense-lazy dense-holes spawn-threads served-entries page-cross-entry tgkill-futex exec-tail mem64-io mem64-net mem64-console mem64-smc mem64-reserve retire-during-lift close-while-reading thread-signals signal-storm exec-signals clock-monotonic main-thread-exit pi-mutex tcp-loopback pwait-sigmask sa-restart thread-churn kernel-contend exec-futex sleep-threads sendmsg-rights scm-fork ioctl-threads fdtable-threads mmap-threads codemap-threads proc-threads fs-threads sig-state-threads lift-threads fdmisc-threads task-ids sock-threads exec-dethread fork-thread-ctx fork-maps-threads fork-twice-thread quiet-threads fdobj-threads cred-threads kill-threads sigwait-threads poll-timeout tid-threads mount-threads exit-threads many-workers threads-many stale-loop-head file-stale-loop stale-cross-page xattr mem64-wx mmap-coherent mmap-remap mmap-writeback null-access sig-regs-threads sig-pause fp-modes"}
|
||||
for src in "$HERE"/*.c; do
|
||||
name=$(basename "$src" .c)
|
||||
# mem64-*: a 64-bit guest memory (docs/mem64.md), x86-64 only.
|
||||
@@ -231,7 +235,7 @@ for src in "$HERE"/*.c; do
|
||||
# *smc*: self-modifying code, under the function JIT as well (which
|
||||
# must leave writable-and-executable code to tier-1's guards).
|
||||
tiers="t1"
|
||||
case $name in *smc* | copy-loop | null-access | sig-regs-threads) tiers="t1 fjit" ;; esac
|
||||
case $name in *smc* | copy-loop | null-access | sig-regs-threads | fp-modes) tiers="t1 fjit" ;; esac
|
||||
case " $PAGE_PROGRAMS " in *" $name "*)
|
||||
if [ -f "$SEED64" ] && command -v node >/dev/null; then tiers="$tiers page"; fi ;;
|
||||
esac
|
||||
|
||||
Reference in new issue
Block a user