Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 21:01:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1571998 - Wasm return calls implementation. r=rhunt

Browse Source 
Changes masm to add return call operations, including platform specific
wasmCollapseFrame operation.

Included basic tests to verify functionality.

Differential Revision: https://phabricator.services.mozilla.com/D166353
This commit is contained in:
Yury Delendik 2023-08-01 16:45:49 +00:00
parent 3be6491b54
commit f4b7a2da28
33 changed files with 1895 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
js/moz.configure
View File

@ -685,6 +685,35 @@ def wasm_function_references(value, no_experimental):
set_config("ENABLE_WASM_FUNCTION_REFERENCES", wasm_function_references)
set_define("ENABLE_WASM_FUNCTION_REFERENCES", wasm_function_references)
# Support for WebAssembly tail-calls.
# ===========================
@depends(milestone.is_nightly)
def default_wasm_tail_calls(is_nightly):
if is_nightly:
return True
option(
"--enable-wasm-tail-calls",
default=default_wasm_tail_calls,
help="{Enable|Disable} WebAssembly tail-calls",
)
@depends("--enable-wasm-tail-calls", target)
def wasm_tail_calls(value, target):
if not value:
return
if target.cpu in ("x86", "x86_64", "arm", "aarch64"):
return True
set_config("ENABLE_WASM_TAIL_CALLS", wasm_tail_calls)
set_define("ENABLE_WASM_TAIL_CALLS", wasm_tail_calls)
# Support for WebAssembly GC.
# ===========================

15
js/public/WasmFeatures.h
View File

@ -85,6 +85,11 @@
#else
# define WASM_MEMORY_CONTROL_ENABLED 0
#endif
#ifdef ENABLE_WASM_TAIL_CALLS
# define WASM_TAIL_CALLS_ENABLED 1
#else
# define WASM_TAIL_CALLS_ENABLED 0
#endif
#ifdef ENABLE_WASM_MOZ_INTGEMM
# define WASM_MOZ_INTGEMM_ENABLED 1
#else
@ -184,6 +189,16 @@ enum class WasmFeatureStage {
/* flag force enable */ false, \
/* shell flag */ "multi-memory", \
/* preference name */ "multi_memory") \
FEATURE( \
/* capitalized name */ TailCalls, \
/* lower case name */ tailCalls, \
/* stage */ WasmFeatureStage::Experimental, \
/* compile predicate */ WASM_TAIL_CALLS_ENABLED, \
/* compiler predicate */ BaselineAvailable(cx), \
/* flag predicate */ !IsFuzzingIon(cx), \
/* flag force enable */ false, \
/* shell flag */ "tail-calls", \
/* preference name */ "tail_calls") \
FEATURE( \
/* capitalized name */ MozIntGemm, \
/* lower case name */ mozIntGemm, \

1
js/src/jit-test/tests/wasm/tail-calls/directives.txt Normal file
View File

@ -0,0 +1 @@
|jit-test| --wasm-tail-calls; test-also=--wasm-compiler=baseline; skip-if: !wasmTailCallsEnabled(); include:wasm.js

54
js/src/jit-test/tests/wasm/tail-calls/exceptions.js Normal file
View File

@ -0,0 +1,54 @@
// |jit-test| --wasm-exceptions; skip-if: !wasmExceptionsEnabled()
// Simple test with return_call.
var ins = wasmEvalText(`(module
(tag $exn)
(func $f0 (result i32)
throw $exn
)
(func $f1 (result i32)
try
return_call $f0
catch $exn
i32.const 3
return
end
i32.const 4
)
(func (export "main") (result i32)
call $f1
)
)`);
assertErrorMessage(() => ins.exports.main(), WebAssembly.Exception, /.*/);
// Test if return stub, created by introducing the slow path,
// not interfering with exception propagation.
var ins0 = wasmEvalText(`(module
(tag $exn)
(func $f0 (result i32)
throw $exn
)
(func (export "f") (result i32)
call $f0
)
(export "t" (tag $exn))
)`);
var ins = wasmEvalText(`(module
(import "" "t" (tag $exn))
(import "" "f" (func $f0 (result i32)))
(func $f1 (result i32)
try
return_call $f0
catch $exn
i32.const 3
return
end
i32.const 4
)
(func (export "main") (result i32)
call $f1
)
)`, {"":ins0.exports,});
assertErrorMessage(() => ins.exports.main(), WebAssembly.Exception, /.*/);

170
js/src/jit-test/tests/wasm/tail-calls/gc.js Normal file
View File

@ -0,0 +1,170 @@
// |jit-test| --wasm-function-references; --wasm-gc; skip-if: !wasmGcEnabled() || getBuildConfiguration().simulator
// Tests GC references passed as arguments during return calls.
// Similar to js/src/jit-test/tests/wasm/gc/trailers-gc-stress.js
let base = wasmEvalText(`(module
;; A simple pseudo-random number generator.
;; Produces numbers in the range 0 .. 2^16-1.
(global $rngState (export "rngState")
(mut i32) (i32.const 1)
)
(func $rand (export "rand") (result i32)
(local $t i32)
;; update $rngState
(local.set $t (global.get $rngState))
(local.set $t (i32.mul (local.get $t) (i32.const 1103515245)))
(local.set $t (i32.add (local.get $t) (i32.const 12345)))
(global.set $rngState (local.get $t))
;; pull 16 random bits out of it
(local.set $t (i32.shr_u (local.get $t) (i32.const 15)))
(local.set $t (i32.and (local.get $t) (i32.const 0xFFFF)))
(local.get $t)
)
;; Array types
(type $tArrayI32 (array (mut i32))) ;; "secondary array" above
(type $tArrayArrayI32 (array (mut (ref null $tArrayI32)))) ;; "primary array"
(func $createSecondaryArrayLoop (export "createSecondaryArrayLoop")
(param $i i32) (param $arr (ref $tArrayI32))
(result (ref $tArrayI32))
(block $cont
(br_if $cont (i32.ge_u (local.get $i) (array.len (local.get $arr))))
(array.set $tArrayI32 (local.get $arr) (local.get $i) (call $rand))
(return_call $createSecondaryArrayLoop
(i32.add (local.get $i) (i32.const 1))
(local.get $arr))
)
(local.get $arr)
)
;; Create an array ("secondary array") containing random numbers, with a
;; size between 1 and 50, also randomly chosen.
(func $createSecondaryArray (export "createSecondaryArray")
(result (ref $tArrayI32))
(return_call $createSecondaryArrayLoop
(i32.const 0)
(array.new $tArrayI32
(i32.const 0)
(i32.add (i32.rem_u (call $rand) (i32.const 50)) (i32.const 1)))
)
)
(func $createPrimaryArrayLoop (export "createPrimaryArrayLoop")
(param $i i32) (param $arrarr (ref $tArrayArrayI32))
(result (ref $tArrayArrayI32))
(block $cont
(br_if $cont (i32.ge_u (local.get $i) (array.len (local.get $arrarr))))
(array.set $tArrayArrayI32 (local.get $arrarr)
(local.get $i) (call $createSecondaryArray))
(return_call $createPrimaryArrayLoop
(i32.add (local.get $i) (i32.const 1))
(local.get $arrarr))
)
(local.get $arrarr)
)
)`);
let t =
`(module
;; Array types (the same as in the base)
(type $tArrayI32 (array (mut i32))) ;; "secondary array" above
(type $tArrayArrayI32 (array (mut (ref null $tArrayI32)))) ;; "primary array"
(import "" "rngState" (global $rngState (mut i32)))
(import "" "rand" (func $rand (result i32)))
(import "" "createSecondaryArrayLoop"
(func $createSecondaryArrayLoop
(param $i i32) (param $arr (ref $tArrayI32))
(result (ref $tArrayI32))))
(import "" "createPrimaryArrayLoop"
(func $createPrimaryArrayLoop
(param $i i32) (param $arrarr (ref $tArrayArrayI32))
(result (ref $tArrayArrayI32))))
;; Create an array ("secondary array") containing random numbers, with a
;; size between 1 and 50, also randomly chosen.
;; (Copy of the base one to create trampoline)
(func $createSecondaryArray (export "createSecondaryArray")
(result (ref $tArrayI32))
(return_call $createSecondaryArrayLoop
(i32.const 0)
(array.new $tArrayI32
(i32.const 0)
(i32.add (i32.rem_u (call $rand) (i32.const 50)) (i32.const 1)))
)
)
;; Create an array (the "primary array") of 1500 elements of
;; type ref-of-tArrayI32.
(func $createPrimaryArray (export "createPrimaryArray")
(result (ref $tArrayArrayI32))
(return_call $createPrimaryArrayLoop
(i32.const 0)
(array.new $tArrayArrayI32 (ref.null $tArrayI32) (i32.const 1500)))
)
;; Use $createPrimaryArray to create an initial array. Then randomly replace
;; elements for a while.
(func $churn (export "churn") (param $thresh i32) (result i32)
(local $i i32)
(local $j i32)
(local $finalSum i32)
(local $arrarr (ref $tArrayArrayI32))
(local $arr (ref null $tArrayI32))
(local $arrLen i32)
(local.set $arrarr (call $createPrimaryArray))
;; This loop iterates 500,000 times. Each iteration, it chooses
;; a randomly element in $arrarr and replaces it with a new
;; random array of 32-bit ints.
(loop $cont
;; make $j be a random number in 0 .. $thresh-1.
;; Then replace that index in $arrarr with a new random arrayI32.
(local.set $j (i32.rem_u (call $rand) (local.get $thresh)))
(array.set $tArrayArrayI32 (local.get $arrarr)
(local.get $j) (call $createSecondaryArray))
(local.set $i (i32.add (local.get $i) (i32.const 1)))
(br_if $cont (i32.lt_u (local.get $i) (i32.const 500000)))
)
;; Finally, compute a checksum by summing all the numbers
;; in all secondary arrays. This simply assumes that all of the refs to
;; secondary arrays are non-null, which isn't per-se guaranteed by the
;; previous loop, but it works in this case because the RNG
;; produces each index value to overwrite at least once.
(local.set $finalSum (i32.const 0))
(local.set $i (i32.const 0)) ;; loop var for the outer loop
(loop $outer
;; body of outer loop
;; $arr = $arrarr[i]
(local.set $arr (array.get $tArrayArrayI32 (local.get $arrarr)
(local.get $i)))
;; iterate over $arr
(local.set $arrLen (array.len (local.get $arr)))
(local.set $j (i32.const 0)) ;; loop var for the inner loop
(loop $inner
;; body of inner loop
(local.set $finalSum
(i32.rotl (local.get $finalSum) (i32.const 1)))
(local.set $finalSum
(i32.xor (local.get $finalSum)
(array.get $tArrayI32 (local.get $arr)
(local.get $j))))
;; loop control for the inner loop
(local.set $j (i32.add (local.get $j) (i32.const 1)))
(br_if $inner (i32.lt_u (local.get $j) (local.get $arrLen)))
)
;; loop control for the outer loop
(local.set $i (i32.add (local.get $i) (i32.const 1)))
(br_if $outer (i32.lt_u (local.get $i) (i32.const 1500)))
)
;; finally, roll in the final value of the RNG state
(i32.xor (local.get $finalSum) (global.get $rngState))
)
)`;
let i = wasmEvalText(t, {"": base.exports,});
let fns = i.exports;
assertEq(fns.churn(800), -575895114);
assertEq(fns.churn(1200), -1164697516);

305
js/src/jit-test/tests/wasm/tail-calls/return_call.js Normal file
View File

@ -0,0 +1,305 @@
var ins = wasmEvalText(`(module
(func $fac-acc (export "fac-acc") (param i64 i64) (result i64)
(if (result i64) (i64.eqz (local.get 0))
(then (local.get 1))
(else
(return_call $fac-acc
(i64.sub (local.get 0) (i64.const 1))
(i64.mul (local.get 0) (local.get 1))
)
)
)
)
(func (export "main") (param i64) (result i64)
(call $fac-acc (local.get 0) (i64.const 1))
)
)`);
// Check return call via wasm function
assertEq(ins.exports.main(5n), 120n);
// Check return call directly via interpreter stub
const fac = ins.exports["fac-acc"];
assertEq(fac(4n, 1n), 24n);
// Check return call directly via jit stub
check_stub1: {
let options = getJitCompilerOptions();
if (!options["baseline.enable"]) break check_stub1;
const check = function() {
fac(4n, 1n);
};
for (let i = options["baseline.warmup.trigger"] + 1; i--;)
check();
}
// Return call of an import
var ins0 = wasmEvalText(`(module
(func $fac-acc (export "fac-acc") (param i64 i64) (result i64)
(if (result i64) (i64.eqz (local.get 0))
(then (local.get 1))
(else
(return_call $fac-acc
(i64.sub (local.get 0) (i64.const 1))
(i64.mul (local.get 0) (local.get 1))
)
)
)
)
)`);
var ins = wasmEvalText(`(module
(import "" "fac-acc" (func $fac-acc (param i64 i64) (result i64)))
(func (export "fac") (param i64) (result i64)
local.get 0
i64.const 1
return_call $fac-acc
)
(func (export "main") (result i64)
i64.const 4
call 1
)
)`, {"": {"fac-acc": ins0.exports["fac-acc"],}});
assertEq(ins.exports.main(), 24n);
assertEq(ins.exports.fac(3n, 1n), 6n);
check_stub2: {
let options = getJitCompilerOptions();
if (!options["baseline.enable"]) break check_stub2;
const check = function() {
ins.exports.fac(3n, 1n)
};
for (let i = options["baseline.warmup.trigger"] + 1; i--;)
check();
}
// Check with parameters area growth
var ins0 = wasmEvalText(`(module
(func $fac-acc (export "fac-acc") (param i64 i64 i64 i64 i64 i64 i64 i64) (result i64)
(if (result i64) (i64.eqz (local.get 0))
(then (local.get 1))
(else
(return_call $fac-acc
(i64.sub (local.get 0) (i64.const 1))
(i64.mul (local.get 0) (local.get 1))
i64.const 1 i64.const 2 i64.const 3 i64.const 4 i64.const 5 i64.const 6
)
)
)
)
)`);
var ins = wasmEvalText(`(module
(import "" "fac-acc" (func $fac-acc (param i64 i64 i64 i64 i64 i64 i64 i64) (result i64)))
(func (export "fac") (param i64) (result i64)
local.get 0
i64.const 1
i64.const 1 i64.const 2 i64.const 3 i64.const 4 i64.const 5 i64.const 6
return_call $fac-acc
)
(func (export "main") (result i64)
i64.const 5
call 1
)
)`, {"": {"fac-acc": ins0.exports["fac-acc"],}});
assertEq(ins.exports.main(), 120n);
assertEq(ins.exports.fac(3n, 1n), 6n);
check_stub3: {
let options = getJitCompilerOptions();
if (!options["baseline.enable"]) break check_stub3;
const check = function() {
ins.exports.fac(4n, 1n)
};
for (let i = options["baseline.warmup.trigger"] + 1; i--;)
check();
}
// Test multi-value returns.
var ins = wasmEvalText(`(module
(memory (export "memory") 1 1)
(func $rec (export "rec") (param i32 i32 i32 i32 i32 i32 i32) (result i32 i32 f32 f32)
(local f32 i32)
(if (result i32 i32 f32 f32) (i32.ge_u (local.get 0) (local.get 1))
(then
(local.get 5)
(local.get 6)
(local.tee 7 (f32.div (f32.convert_i32_u (local.get 3)) (f32.convert_i32_u (local.get 2))))
(f32.sqrt
(f32.sub
(f32.div (f32.convert_i32_u (local.get 4)) (f32.convert_i32_u (local.get 2)))
(f32.mul (local.get 7) (local.get 7))
)
)
)
(else
(return_call $rec
(i32.add (local.get 0) (i32.const 1))
(local.get 1)
(i32.add (local.get 2) (i32.const 1))
(i32.add (local.get 3) (local.tee 8 (i32.load8_u (local.get 0))))
(i32.add (local.get 4) (i32.mul (local.get 8) (local.get 8)))
(if (result i32) (i32.gt_s (local.get 5) (local.get 8))
(then (local.get 8)) (else (local.get 5))
)
(if (result i32) (i32.lt_s (local.get 6) (local.get 8))
(then (local.get 8)) (else (local.get 6))
)
)
)
)
)
(func $main (export "main") (result i32 i32 f32 f32)
(call $rec
(i32.const 0)
(i32.const 6)
(i32.const 0)
(i32.const 0)
(i32.const 0)
(i32.const 1000)
(i32.const -1000)
)
)
(data (i32.const 0) "\\02\\13\\22\\04\\08\\30")
)`);
const main = ins.exports["main"];
assertEq(""+ main(), "2,48,19.16666603088379,16.836633682250977");
assertEq("" + ins.exports.rec(1, 5, 0, 0, 0, 1000, -1000), "4,34,16.25,11.627016067504883");
check_stub3: {
let options = getJitCompilerOptions();
if (!options["baseline.enable"]) break check_stub3;
const check = function() {
ins.exports.rec(1, 5, 0, 0, 0, 1000, -1000);
};
for (let i = options["baseline.warmup.trigger"] + 1; i--;)
check();
}
// Handling trap.
var ins = wasmEvalText(`(module
(func $fac-acc (export "fac") (param i64 i64) (result i64)
(if (result i64) (i64.eqz (local.get 0))
(then (unreachable))
(else
(return_call $fac-acc
(i64.sub (local.get 0) (i64.const 1))
(i64.mul (local.get 0) (local.get 1))
)
)
)
)
(func (export "main") (param i64) (result i64)
(call $fac-acc (local.get 0) (i64.const 1))
)
)`);
assertErrorMessage(() => ins.exports.main(4n), WebAssembly.RuntimeError, /unreachable executed/);
assertErrorMessage(() => ins.exports.fac(3n, 1n), WebAssembly.RuntimeError, /unreachable executed/);
// Performance and stack growth: calculating sum of numbers 1..40000000
var ins = wasmEvalText(`(module
(func $sum (param i32 i64) (result i64)
local.get 0
i32.eqz
if
local.get 1
return
else
local.get 0
i32.const 1
i32.sub
local.get 1
local.get 0
i64.extend_i32_s
i64.add
return_call $sum
end
unreachable
)
(func (export "main") (param i32) (result i64)
local.get 0
i64.const 0
call $sum
)
)`);
if (getBuildConfiguration().simulator) {
assertEq(ins.exports.main(400000), 80000200000n);
} else {
assertEq(ins.exports.main(40000000), 800000020000000n);
}
// GC/externref shall not cling to the trampoline frame.
// The `return_call` caller will create a trampoline because the callee is
// an import. The caller will create a GC object and will hold in its frame
// and a WeakMap.
// Test if the created object is in the WeakMap even after gc().
var wm = new WeakMap();
var ins = wasmEvalText(`(module
(import "" "test" (func $test))
(func $sum (param i32 i64) (result i64)
local.get 0
i32.eqz
if
call $test
local.get 1
return
else
local.get 0
i32.const 1
i32.sub
local.get 1
local.get 0
i64.extend_i32_s
i64.add
return_call $sum
end
unreachable
)
(export "sum" (func $sum))
)`, {"": {
test() {
gc();
assertEq(nondeterministicGetWeakMapKeys(wm).length, 0);
}
}});
var ins2 = wasmEvalText(`(module
(import "" "add_ref" (func $add_ref (result externref)))
(import "" "use_ref" (func $use_ref (param externref)))
(import "" "sum" (func $sum (param i32 i64) (result i64)))
(global $g1 (mut i32) (i32.const 0))
(func (export "main_gc") (param i32) (result i64)
(local $ref externref)
call $add_ref
local.set $ref
local.get $ref
call $use_ref
block
global.get $g1
br_if 0
local.get 0
i64.const 0
return_call $sum
end
local.get $ref
call $use_ref
i64.const -1
)
)`, {"": {
sum: ins.exports.sum,
add_ref() {
const obj = {}; wm.set(obj, 'foo'); return obj;
},
use_ref(obj) {
assertEq(nondeterministicGetWeakMapKeys(wm).length, 1);
},
}});
assertEq(ins2.exports.main_gc(400000), 80000200000n);
assertEq(nondeterministicGetWeakMapKeys(wm).length, 0);

155
js/src/jit-test/tests/wasm/tail-calls/return_call_indirect.js Normal file
View File

@ -0,0 +1,155 @@
var ins0 = wasmEvalText(`(module
(func $fac-acc (export "fac-acc") (param i64 i64) (result i64)
(if (result i64) (i64.eqz (local.get 0))
(then (local.get 1))
(else
(return_call $fac-acc
(i64.sub (local.get 0) (i64.const 1))
(i64.mul (local.get 0) (local.get 1))
)
)
)
)
)`);
var ins = wasmEvalText(`(module
(import "" "fac-acc" (func $fac-acc (param i64 i64) (result i64)))
(type $ty (func (param i64 i64) (result i64)))
(table $t 1 1 funcref)
(func $f (export "fac") (param i64) (result i64)
local.get 0
i64.const 1
i32.const 0
return_call_indirect $t (type $ty)
)
(elem $t (i32.const 0) $fac-acc)
(func (export "main") (result i64)
i64.const 5
call $f
)
)`, {"": {"fac-acc": ins0.exports["fac-acc"]}});
// Check return call via wasm function
assertEq(ins.exports.main(5n), 120n);
// Check return call directly via interpreter stub
const fac = ins.exports["fac"];
assertEq(fac(4n, 1n), 24n);
// Check return call directly via jit stub
check_stub1: {
let options = getJitCompilerOptions();
if (!options["baseline.enable"]) break check_stub1;
const check = function() {
fac(4n, 1n);
};
for (let i = options["baseline.warmup.trigger"] + 1; i--;)
check();
}
// Invalid func type
var ins = wasmEvalText(`(module
(import "" "fac-acc" (func $fac-acc (param i64 i64) (result i64)))
(type $ty (func (param i64 i64) (result i64)))
(table $t 1 1 funcref)
(func $f (export "fac") (param i64) (result i64)
local.get 0
i64.const 1
i32.const 0
return_call_indirect $t
)
(elem $t (i32.const 0) $fac-acc)
(func (export "main") (result i64)
i64.const 5
call $f
)
)`, {"": {"fac-acc": ins0.exports["fac-acc"]}});
assertErrorMessage(() => ins.exports.main(), WebAssembly.RuntimeError, /indirect call signature mismatch/);
assertErrorMessage(() => ins.exports.fac(6n), WebAssembly.RuntimeError, /indirect call signature mismatch/);
// Invalid func type, but testing when entry directly does invalid return_call_indirect.
var wasm = wasmTextToBinary(`(module
(global $g (export "g") (mut i32) (i32.const 0))
(table 1 1 funcref)
(type $ft (func (param f64)))
(func $f (export "f")
unreachable
)
(func $test (export "test")
global.get $g
br_if 0
f64.const 0.0
i32.const 0
return_call_indirect (type $ft)
)
(elem (i32.const 0) $f)
)`);
var ins = new WebAssembly.Instance(new WebAssembly.Module(wasm));
function check_err() { ins.exports.test(); }
assertErrorMessage(check_err, WebAssembly.RuntimeError, /indirect call signature mismatch/);
var ins = new WebAssembly.Instance(new WebAssembly.Module(wasm));
check_stub2: {
let options = getJitCompilerOptions();
if (!options["baseline.enable"]) break check_stub2;
ins.exports.g.value = 1;
for (let i = options["baseline.warmup.trigger"] + 1; i--;)
check_err();
ins.exports.g.value = 0;
assertErrorMessage(check_err, WebAssembly.RuntimeError, /indirect call signature mismatch/);
}
var ins = new WebAssembly.Instance(new WebAssembly.Module(wasm));
check_stub3: {
let options = getJitCompilerOptions();
if (!options["ion.enable"]) break check_stub3;
ins.exports.g.value = 1;
var check_err2 = function() { for (var i = 0; i < 5; i++) ins.exports.test(); };
for (let i = options["ion.warmup.trigger"]+2; i--;)
check_err2();
ins.exports.g.value = 0;
assertErrorMessage(() => check_err2(), WebAssembly.RuntimeError, /indirect call signature mismatch/);
}
// Performance and stack growth: calculating sum of numbers 1..40000000
var ins = wasmEvalText(`(module
(table 1 1 funcref)
(type $rec (func (param i32 i64) (result i64)))
(func $sum (param i32 i64) (result i64)
local.get 0
i32.eqz
if
local.get 1
return
else
local.get 0
i32.const 1
i32.sub
local.get 1
local.get 0
i64.extend_i32_s
i64.add
i32.const 0
return_call_indirect (type $rec)
end
unreachable
)
(elem (i32.const 0) $sum)
(func (export "main") (param i32) (result i64)
local.get 0
i64.const 0
i32.const 0
return_call_indirect (type $rec)
)
)`);
if (getBuildConfiguration().simulator) {
assertEq(ins.exports.main(400000), 80000200000n);
} else {
assertEq(ins.exports.main(40000000), 800000020000000n);
}

4
js/src/jit-test/tests/wasm/widening-i32-after-call.js
View File

@ -35,13 +35,13 @@ var ins = wasmEvalText(`
switch (wasmCompileMode()) {
case "ion":
assertEq(wasmDis(ins.exports.wasm2wasm, {tier:'stable', asString:true}).match(/call.*\n.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.wasm2import, {tier:'stable', asString:true}).match(/call.*\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.wasm2import, {tier:'stable', asString:true}).match(/call.*\n.*or \$0x00, %r14\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.wasmIndirect, {tier:'stable', asString:true}).match(/call.*\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.instanceCall, {tier:'stable', asString:true}).match(/call.*\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
break;
case "baseline":
assertEq(wasmDis(ins.exports.wasm2wasm, {tier:'stable', asString:true}).match(/call.*\n.*lea.*%rsp\n.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.wasm2import, {tier:'stable', asString:true}).match(/call.*\n.*lea.*%rsp\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.wasm2import, {tier:'stable', asString:true}).match(/call.*\n.*or \$0x00, %r14\n.*lea.*%rsp\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.wasmIndirect, {tier:'stable', asString:true}).match(/call.*\n.*lea.*%rsp\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
assertEq(wasmDis(ins.exports.instanceCall, {tier:'stable', asString:true}).match(/call.*\n.*lea.*%rsp\n(?:.*movq.*\n)*.*mov %eax, %eax/).length, 1);
break;

538
js/src/jit/MacroAssembler.cpp
View File

@ -4607,6 +4607,361 @@ std::pair<CodeOffset, uint32_t> MacroAssembler::wasmReserveStackChecked(
return std::pair<CodeOffset, uint32_t>(trapInsnOffset, amount);
}
#ifdef ENABLE_WASM_TAIL_CALLS
static void MoveDataBlock(MacroAssembler& masm, Register base, int32_t from,
int32_t to, uint32_t size) {
MOZ_ASSERT(base != masm.getStackPointer());
if (from == to || size == 0) {
return; // noop
}
# ifdef JS_CODEGEN_ARM64
vixl::UseScratchRegisterScope temps(&masm);
const Register scratch = temps.AcquireX().asUnsized();
# elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_X86)
static constexpr Register scratch = ABINonArgReg0;
masm.push(scratch);
# elif !defined(JS_CODEGEN_NONE)
const Register scratch = ScratchReg;
# else
const Register scratch = InvalidReg;
# endif
if (to < from) {
for (uint32_t i = 0; i < size; i += sizeof(void*)) {
masm.loadPtr(Address(base, from + i), scratch);
masm.storePtr(scratch, Address(base, to + i));
}
} else {
for (uint32_t i = size; i > 0;) {
i -= sizeof(void*);
masm.loadPtr(Address(base, from + i), scratch);
masm.storePtr(scratch, Address(base, to + i));
}
}
# if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_X86)
masm.pop(scratch);
# endif
}
struct ReturnCallTrampolineData {
# ifdef JS_CODEGEN_ARM
uint32_t trampolineOffset;
# else
CodeLabel trampoline;
# endif
};
static ReturnCallTrampolineData MakeReturnCallTrampoline(MacroAssembler& masm) {
uint32_t savedPushed = masm.framePushed();
// Build simple trampoline code: load the instance slot from the frame,
// restore FP, and return to prevous caller.
ReturnCallTrampolineData data;
# ifdef JS_CODEGEN_ARM
data.trampolineOffset = masm.currentOffset();
# else
masm.bind(&data.trampoline);
# endif
masm.setFramePushed(
AlignBytes(wasm::FrameWithInstances::sizeOfInstanceFieldsAndShadowStack(),
WasmStackAlignment));
# ifdef ENABLE_WASM_TAIL_CALLS
masm.wasmMarkSlowCall();
# endif
masm.loadPtr(
Address(masm.getStackPointer(), WasmCallerInstanceOffsetBeforeCall),
InstanceReg);
masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
masm.moveToStackPtr(FramePointer);
# ifdef JS_CODEGEN_ARM64
masm.pop(FramePointer, lr);
masm.Mov(PseudoStackPointer64, vixl::sp);
masm.abiret();
# else
masm.pop(FramePointer);
masm.ret();
# endif
masm.setFramePushed(savedPushed);
return data;
}
// CollapseWasmFrame methods merge frames fields: callee parameters, instance
// slots, and caller RA. See the diagram below. The C0 is the previous caller,
// the C1 is the caller of the return call, and the C2 is the callee.
//
// +-------------------+ +--------------------+
// |C0 instance slots | |C0 instance slots |
// +-------------------+ -+ +--------------------+ -+
// | RA | | | RA | |
// +-------------------+ | C0 +--------------------+ |C0
// | FP | v | FP | v
// +-------------------+ +--------------------+
// |C0 private frame | |C0 private frame |
// +-------------------+ +--------------------+
// |C1 results area | |C1/C2 results area |
// +-------------------+ +--------------------+
// |C1 parameters | |? trampoline frame |
// +-------------------+ +--------------------+
// |C1 instance slots | |C2 parameters |
// +-------------------+ -+ +--------------------+
// |C0 RA | | |C2 instance slots’ |
// +-------------------+ | C1 +--------------------+ -+
// |C0 FP | v |C0 RA’ | |
// +-------------------+ +--------------------+ | C2
// |C1 private frame | |C0 FP’ | v
// +-------------------+ +--------------------+ <= start of C2
// |C2 parameters |
// +-------------------+
// |C2 instance slots |
// +-------------------+ <= call C2
//
// The C2 parameters are moved in place of the C1 parameters, and the
// C1 frame data is removed. The instance slots, return address, and
// frame pointer to the C0 callsite are saved or adjusted.
//
// For cross-instance calls, the trampoline frame will be introduced
// if the C0 callsite has no ability to restore instance registers and realm.
static void CollapseWasmFrameFast(MacroAssembler& masm,
const ReturnCallAdjustmentInfo& retCallInfo) {
uint32_t framePushedAtStart = masm.framePushed();
static_assert(sizeof(wasm::Frame) == 2 * sizeof(void*));
// The instance slots + stack arguments are expected to be padded and
// aligned to the WasmStackAlignment boundary. There is no data expected
// in the padded region, such as results stack area or locals, to avoid
// unwanted stack growth.
uint32_t newSlotsAndStackArgBytes =
AlignBytes(retCallInfo.newSlotsAndStackArgBytes, WasmStackAlignment);
uint32_t oldSlotsAndStackArgBytes =
AlignBytes(retCallInfo.oldSlotsAndStackArgBytes, WasmStackAlignment);
static constexpr Register tempForCaller = ABINonArgReg1;
static constexpr Register tempForFP = ABINonArgReg3;
# ifdef JS_USE_LINK_REGISTER
static constexpr Register tempForRA = lr;
# else
static constexpr Register tempForRA = ABINonArgReg2;
masm.push(tempForRA);
# endif
// Load the FP, RA, and instance slots into registers to preserve them while
// the new frame is collapsed over the current one.
masm.loadPtr(Address(FramePointer, wasm::Frame::callerFPOffset()), tempForFP);
masm.loadPtr(Address(FramePointer, wasm::Frame::returnAddressOffset()),
tempForRA);
bool copyCallerSlot = oldSlotsAndStackArgBytes != newSlotsAndStackArgBytes;
if (copyCallerSlot) {
masm.loadPtr(
Address(FramePointer, wasm::FrameWithInstances::callerInstanceOffset()),
tempForCaller);
}
// Copy parameters data, ignoring shadow data and instance slots.
// Make all offsets relative to the FramePointer.
int32_t newArgSrc = -framePushedAtStart;
int32_t newArgDest =
sizeof(wasm::Frame) + oldSlotsAndStackArgBytes - newSlotsAndStackArgBytes;
const uint32_t SlotsSize =
wasm::FrameWithInstances::sizeOfInstanceFieldsAndShadowStack();
MoveDataBlock(masm, FramePointer, newArgSrc + SlotsSize,
newArgDest + SlotsSize,
retCallInfo.newSlotsAndStackArgBytes - SlotsSize);
// Copy caller instance slots from the current frame.
if (copyCallerSlot) {
masm.storePtr(
tempForCaller,
Address(FramePointer, newArgDest + WasmCallerInstanceOffsetBeforeCall));
}
// Store current instance as the new callee instance slot.
masm.storePtr(
InstanceReg,
Address(FramePointer, newArgDest + WasmCalleeInstanceOffsetBeforeCall));
# ifdef JS_USE_LINK_REGISTER
// RA is already in its place, just move stack.
masm.addToStackPtr(Imm32(framePushedAtStart + newArgDest));
# else
// Push RA to new frame: store RA, restore temp, and move stack.
int32_t newFrameOffset = newArgDest - sizeof(wasm::Frame);
masm.storePtr(tempForRA,
Address(FramePointer,
newFrameOffset + wasm::Frame::returnAddressOffset()));
masm.pop(tempForRA);
masm.addToStackPtr(Imm32(framePushedAtStart + newFrameOffset +
wasm::Frame::returnAddressOffset()));
# endif
masm.movePtr(tempForFP, FramePointer);
}
static void CollapseWasmFrameSlow(MacroAssembler& masm,
const ReturnCallAdjustmentInfo& retCallInfo,
wasm::CallSiteDesc desc,
ReturnCallTrampolineData data) {
uint32_t framePushedAtStart = masm.framePushed();
static constexpr Register tempForCaller = ABINonArgReg1;
static constexpr Register tempForFP = ABINonArgReg3;
static_assert(sizeof(wasm::Frame) == 2 * sizeof(void*));
// The hidden frame will "break" after wasm::Frame data fields.
// Calculate sum of wasm stack alignment before and after the break as
// the size to reserve.
const uint32_t HiddenFrameAfterSize =
AlignBytes(wasm::FrameWithInstances::sizeOfInstanceFieldsAndShadowStack(),
WasmStackAlignment);
const uint32_t HiddenFrameSize =
AlignBytes(sizeof(wasm::Frame), WasmStackAlignment) +
HiddenFrameAfterSize;
// If it is not slow, prepare two frame: one is regular wasm frame, and
// another one is hidden. The hidden frame contains one instance slots
// for unwind and recovering pinned registers.
// The instance slots + stack arguments are expected to be padded and
// aligned to the WasmStackAlignment boundary. There is no data expected
// in the padded region, such as results stack area or locals, to avoid
// unwanted stack growth.
// The Hidden frame will be inserted with this constraint too.
uint32_t newSlotsAndStackArgBytes =
AlignBytes(retCallInfo.newSlotsAndStackArgBytes, WasmStackAlignment);
uint32_t oldSlotsAndStackArgBytes =
AlignBytes(retCallInfo.oldSlotsAndStackArgBytes, WasmStackAlignment);
// Make all offsets relative to the FramePointer.
int32_t newArgSrc = -framePushedAtStart;
int32_t newArgDest = sizeof(wasm::Frame) + oldSlotsAndStackArgBytes -
HiddenFrameSize - newSlotsAndStackArgBytes;
int32_t hiddenFrameArgsDest =
sizeof(wasm::Frame) + oldSlotsAndStackArgBytes - HiddenFrameAfterSize;
// It will be possible to overwrite data (on the top of the stack) due to
// the added hidden frame, reserve needed space.
uint32_t reserved = newArgDest - int32_t(sizeof(void*)) < newArgSrc
? newArgSrc - newArgDest + sizeof(void*)
: 0;
masm.reserveStack(reserved);
# ifdef JS_USE_LINK_REGISTER
static constexpr Register tempForRA = lr;
# else
static constexpr Register tempForRA = ABINonArgReg2;
masm.push(tempForRA);
# endif
// Load FP, RA and instance slots to preserve them from being overwritten.
masm.loadPtr(Address(FramePointer, wasm::Frame::callerFPOffset()), tempForFP);
masm.loadPtr(Address(FramePointer, wasm::Frame::returnAddressOffset()),
tempForRA);
masm.loadPtr(
Address(FramePointer, newArgSrc + WasmCallerInstanceOffsetBeforeCall),
tempForCaller);
// Copy parameters data, ignoring shadow data and instance slots.
const uint32_t SlotsSize =
wasm::FrameWithInstances::sizeOfInstanceFieldsAndShadowStack();
MoveDataBlock(masm, FramePointer, newArgSrc + SlotsSize,
newArgDest + SlotsSize,
retCallInfo.newSlotsAndStackArgBytes - SlotsSize);
// Form hidden frame for trampoline.
int32_t newFPOffset = hiddenFrameArgsDest - sizeof(wasm::Frame);
masm.storePtr(
tempForRA,
Address(FramePointer, newFPOffset + wasm::Frame::returnAddressOffset()));
// Copy original FP.
masm.storePtr(
tempForFP,
Address(FramePointer, newFPOffset + wasm::Frame::callerFPOffset()));
// Set up instance slots.
masm.storePtr(
tempForCaller,
Address(FramePointer,
newFPOffset + wasm::FrameWithInstances::calleeInstanceOffset()));
masm.storePtr(
tempForCaller,
Address(FramePointer, newArgDest + WasmCallerInstanceOffsetBeforeCall));
masm.storePtr(
InstanceReg,
Address(FramePointer, newArgDest + WasmCalleeInstanceOffsetBeforeCall));
# ifdef JS_CODEGEN_ARM
// ARM has no CodeLabel -- calculate PC directly.
masm.mov(pc, tempForRA);
masm.computeEffectiveAddress(
Address(tempForRA,
int32_t(data.trampolineOffset - masm.currentOffset() - 4)),
tempForRA);
masm.append(desc, CodeOffset(data.trampolineOffset));
# else
masm.mov(&data.trampoline, tempForRA);
masm.addCodeLabel(data.trampoline);
// Add slow trampoline callsite description, to be annotated in
// stack/frame iterators.
masm.append(desc, *data.trampoline.target());
# endif
# ifdef JS_USE_LINK_REGISTER
masm.freeStack(reserved);
// RA is already in its place, just move stack.
masm.addToStackPtr(Imm32(framePushedAtStart + newArgDest));
# else
// Push RA to new frame: store RA, restore temp, and move stack.
int32_t newFrameOffset = newArgDest - sizeof(wasm::Frame);
masm.storePtr(tempForRA,
Address(FramePointer,
newFrameOffset + wasm::Frame::returnAddressOffset()));
masm.pop(tempForRA);
masm.freeStack(reserved);
masm.addToStackPtr(Imm32(framePushedAtStart + newFrameOffset +
wasm::Frame::returnAddressOffset()));
# endif
// Point FramePointer to hidden frame.
masm.computeEffectiveAddress(Address(FramePointer, newFPOffset),
FramePointer);
}
void MacroAssembler::wasmCollapseFrameFast(
const ReturnCallAdjustmentInfo& retCallInfo) {
CollapseWasmFrameFast(*this, retCallInfo);
}
void MacroAssembler::wasmCollapseFrameSlow(
const ReturnCallAdjustmentInfo& retCallInfo, wasm::CallSiteDesc desc) {
static constexpr Register temp1 = ABINonArgReg1;
static constexpr Register temp2 = ABINonArgReg3;
// Check if RA has slow marker. If there is no marker, generate a trampoline
// frame to restore register state when this tail call returns.
Label slow, done;
loadPtr(Address(FramePointer, wasm::Frame::returnAddressOffset()), temp1);
wasmCheckSlowCallsite(temp1, &slow, temp1, temp2);
CollapseWasmFrameFast(*this, retCallInfo);
jump(&done);
ReturnCallTrampolineData data = MakeReturnCallTrampoline(*this);
bind(&slow);
CollapseWasmFrameSlow(*this, retCallInfo, desc, data);
bind(&done);
}
#endif // ENABLE_WASM_TAIL_CALLS
CodeOffset MacroAssembler::wasmCallImport(const wasm::CallSiteDesc& desc,
const wasm::CalleeDesc& callee) {
storePtr(InstanceReg,
@ -4644,9 +4999,69 @@ CodeOffset MacroAssembler::wasmCallImport(const wasm::CallSiteDesc& desc,
Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
loadWasmPinnedRegsFromInstance();
return call(desc, ABINonArgReg0);
CodeOffset res = call(desc, ABINonArgReg0);
#ifdef ENABLE_WASM_TAIL_CALLS
wasmMarkSlowCall();
#endif
return res;
}
#ifdef ENABLE_WASM_TAIL_CALLS
CodeOffset MacroAssembler::wasmReturnCallImport(
const wasm::CallSiteDesc& desc, const wasm::CalleeDesc& callee,
const ReturnCallAdjustmentInfo& retCallInfo) {
storePtr(InstanceReg,
Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
// Load the callee, before the caller's registers are clobbered.
uint32_t instanceDataOffset = callee.importInstanceDataOffset();
loadPtr(
Address(InstanceReg, wasm::Instance::offsetInData(
instanceDataOffset +
offsetof(wasm::FuncImportInstanceData, code))),
ABINonArgReg0);
# if !defined(JS_CODEGEN_NONE) && !defined(JS_CODEGEN_WASM32)
static_assert(ABINonArgReg0 != InstanceReg, "by constraint");
# endif
// Switch to the callee's realm.
loadPtr(
Address(InstanceReg, wasm::Instance::offsetInData(
instanceDataOffset +
offsetof(wasm::FuncImportInstanceData, realm))),
ABINonArgReg1);
loadPtr(Address(InstanceReg, wasm::Instance::offsetOfCx()), ABINonArgReg2);
storePtr(ABINonArgReg1, Address(ABINonArgReg2, JSContext::offsetOfRealm()));
// Switch to the callee's instance and pinned registers and make the call.
loadPtr(Address(InstanceReg,
wasm::Instance::offsetInData(
instanceDataOffset +
offsetof(wasm::FuncImportInstanceData, instance))),
InstanceReg);
storePtr(InstanceReg,
Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
loadWasmPinnedRegsFromInstance();
wasm::CallSiteDesc stubDesc(desc.lineOrBytecode(),
wasm::CallSiteDesc::ReturnStub);
wasmCollapseFrameSlow(retCallInfo, stubDesc);
jump(ABINonArgReg0);
return CodeOffset(currentOffset());
}
CodeOffset MacroAssembler::wasmReturnCall(
const wasm::CallSiteDesc& desc, uint32_t funcDefIndex,
const ReturnCallAdjustmentInfo& retCallInfo) {
wasmCollapseFrameFast(retCallInfo);
CodeOffset offset = farJumpWithPatch();
append(desc, offset, funcDefIndex);
return offset;
}
#endif // ENABLE_WASM_TAIL_CALLS
CodeOffset MacroAssembler::wasmCallBuiltinInstanceMethod(
const wasm::CallSiteDesc& desc, const ABIArg& instanceArg,
wasm::SymbolicAddress builtin, wasm::FailureMode failureMode) {
@ -4843,6 +5258,9 @@ void MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc,
calleeScratch);
*slowCallOffset = call(desc, calleeScratch);
#ifdef ENABLE_WASM_TAIL_CALLS
wasmMarkSlowCall();
#endif
// Restore registers and realm and join up with the fast path.
@ -4874,6 +5292,124 @@ void MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc,
bind(&done);
}
#ifdef ENABLE_WASM_TAIL_CALLS
void MacroAssembler::wasmReturnCallIndirect(
const wasm::CallSiteDesc& desc, const wasm::CalleeDesc& callee,
Label* boundsCheckFailedLabel, Label* nullCheckFailedLabel,
mozilla::Maybe<uint32_t> tableSize,
const ReturnCallAdjustmentInfo& retCallInfo) {
CodeOffset t;
CodeOffset* fastCallOffset = &t;
CodeOffset* slowCallOffset = &t;
static_assert(sizeof(wasm::FunctionTableElem) == 2 * sizeof(void*),
"Exactly two pointers or index scaling won't work correctly");
MOZ_ASSERT(callee.which() == wasm::CalleeDesc::WasmTable);
const int shift = sizeof(wasm::FunctionTableElem) == 8 ? 3 : 4;
wasm::BytecodeOffset trapOffset(desc.lineOrBytecode());
const Register calleeScratch = WasmTableCallScratchReg0;
const Register index = WasmTableCallIndexReg;
// Check the table index and throw if out-of-bounds.
//
// Frequently the table size is known, so optimize for that. Otherwise
// compare with a memory operand when that's possible. (There's little sense
// in hoisting the load of the bound into a register at a higher level and
// reusing that register, because a hoisted value would either have to be
// spilled and re-loaded before the next call_indirect, or would be abandoned
// because we could not trust that a hoisted value would not have changed.)
if (boundsCheckFailedLabel) {
if (tableSize.isSome()) {
branch32(Assembler::Condition::AboveOrEqual, index, Imm32(*tableSize),
boundsCheckFailedLabel);
} else {
branch32(
Assembler::Condition::BelowOrEqual,
Address(InstanceReg, wasm::Instance::offsetInData(
callee.tableLengthInstanceDataOffset())),
index, boundsCheckFailedLabel);
}
}
// Write the functype-id into the ABI functype-id register.
const wasm::CallIndirectId callIndirectId = callee.wasmTableSigId();
switch (callIndirectId.kind()) {
case wasm::CallIndirectIdKind::Global:
loadPtr(Address(InstanceReg, wasm::Instance::offsetInData(
callIndirectId.instanceDataOffset())),
WasmTableCallSigReg);
break;
case wasm::CallIndirectIdKind::Immediate:
move32(Imm32(callIndirectId.immediate()), WasmTableCallSigReg);
break;
case wasm::CallIndirectIdKind::AsmJS:
case wasm::CallIndirectIdKind::None:
break;
}
// Load the base pointer of the table and compute the address of the callee in
// the table.
loadPtr(
Address(InstanceReg, wasm::Instance::offsetInData(
callee.tableFunctionBaseInstanceDataOffset())),
calleeScratch);
shiftIndex32AndAdd(index, shift, calleeScratch);
// Load the callee instance and decide whether to take the fast path or the
// slow path.
Label fastCall;
Label done;
const Register newInstanceTemp = WasmTableCallScratchReg1;
loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, instance)),
newInstanceTemp);
branchPtr(Assembler::Equal, InstanceReg, newInstanceTemp, &fastCall);
// Slow path: Save context, check for null, setup new context.
storePtr(InstanceReg,
Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
movePtr(newInstanceTemp, InstanceReg);
# ifdef WASM_HAS_HEAPREG
// Use the null pointer exception resulting from loading HeapReg from a null
// instance to handle a call to a null slot.
MOZ_ASSERT(nullCheckFailedLabel == nullptr);
loadWasmPinnedRegsFromInstance(mozilla::Some(trapOffset));
# else
MOZ_ASSERT(nullCheckFailedLabel != nullptr);
branchTestPtr(Assembler::Zero, InstanceReg, InstanceReg,
nullCheckFailedLabel);
loadWasmPinnedRegsFromInstance();
# endif
switchToWasmInstanceRealm(index, WasmTableCallScratchReg1);
loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, code)),
calleeScratch);
wasm::CallSiteDesc stubDesc(desc.lineOrBytecode(),
wasm::CallSiteDesc::ReturnStub);
wasmCollapseFrameSlow(retCallInfo, stubDesc);
jump(calleeScratch);
*slowCallOffset = CodeOffset(currentOffset());
// Fast path: just load the code pointer and go.
bind(&fastCall);
loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, code)),
calleeScratch);
wasmCollapseFrameFast(retCallInfo);
jump(calleeScratch);
*fastCallOffset = CodeOffset(currentOffset());
}
#endif // ENABLE_WASM_TAIL_CALLS
void MacroAssembler::wasmCallRef(const wasm::CallSiteDesc& desc,
const wasm::CalleeDesc& callee,
CodeOffset* fastCallOffset,

50
js/src/jit/MacroAssembler.h
View File

@ -294,6 +294,21 @@ struct AllocSiteInput
explicit AllocSiteInput(Register reg) : Base(reg) {}
};
#ifdef ENABLE_WASM_TAIL_CALLS
// Instance slots (including ShadowStackArea) and arguments size information
// from two neighboring frames.
// Used in Wasm tail calls to remove frame.
struct ReturnCallAdjustmentInfo {
uint32_t newSlotsAndStackArgBytes;
uint32_t oldSlotsAndStackArgBytes;
ReturnCallAdjustmentInfo(uint32_t newSlotsAndStackArgBytes,
uint32_t oldSlotsAndStackArgBytes)
: newSlotsAndStackArgBytes(newSlotsAndStackArgBytes),
oldSlotsAndStackArgBytes(oldSlotsAndStackArgBytes) {}
};
#endif // ENABLE_WASM_TAIL_CALLS
// [SMDOC] Code generation invariants (incomplete)
//
// ## 64-bit GPRs carrying 32-bit values
@ -3796,6 +3811,26 @@ class MacroAssembler : public MacroAssemblerSpecific {
CodeOffset wasmCallImport(const wasm::CallSiteDesc& desc,
const wasm::CalleeDesc& callee);
#ifdef ENABLE_WASM_TAIL_CALLS
CodeOffset wasmReturnCallImport(const wasm::CallSiteDesc& desc,
const wasm::CalleeDesc& callee,
const ReturnCallAdjustmentInfo& retCallInfo);
CodeOffset wasmReturnCall(const wasm::CallSiteDesc& desc,
uint32_t funcDefIndex,
const ReturnCallAdjustmentInfo& retCallInfo);
void wasmCollapseFrameSlow(const ReturnCallAdjustmentInfo& retCallInfo,
wasm::CallSiteDesc desc);
void wasmCollapseFrameFast(const ReturnCallAdjustmentInfo& retCallInfo);
void wasmCheckSlowCallsite(Register ra, Label* notSlow, Register temp1,
Register temp2) DEFINED_ON(x86, x64, arm, arm64);
void wasmMarkSlowCall() DEFINED_ON(x86, x64, arm, arm64);
#endif
// WasmTableCallIndexReg must contain the index of the indirect call. This is
// for wasm calls only.
//
@ -3815,6 +3850,21 @@ class MacroAssembler : public MacroAssemblerSpecific {
mozilla::Maybe<uint32_t> tableSize,
CodeOffset* fastCallOffset, CodeOffset* slowCallOffset);
#ifdef ENABLE_WASM_TAIL_CALLS
// WasmTableCallIndexReg must contain the index of the indirect call. This is
// for wasm calls only.
//
// `boundsCheckFailedLabel` is non-null iff a bounds check is required.
// `nullCheckFailedLabel` is non-null only on platforms that can't fold the
// null check into the rest of the call instructions.
void wasmReturnCallIndirect(const wasm::CallSiteDesc& desc,
const wasm::CalleeDesc& callee,
Label* boundsCheckFailedLabel,
Label* nullCheckFailedLabel,
mozilla::Maybe<uint32_t> tableSize,
const ReturnCallAdjustmentInfo& retCallInfo);
#endif // ENABLE_WASM_TAIL_CALLS
// This function takes care of loading the callee's instance and address from
// pinned reg.
void wasmCallRef(const wasm::CallSiteDesc& desc,

16
js/src/jit/arm/MacroAssembler-arm.cpp
View File

@ -6019,6 +6019,22 @@ void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
addPtr(indexTemp32, pointer);
}
#ifdef ENABLE_WASM_TAIL_CALLS
void MacroAssembler::wasmMarkSlowCall() { ma_and(lr, lr, lr); }
const int32_t SlowCallMarker = 0xe00ee00e;
void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
Register temp1, Register temp2) {
MOZ_ASSERT(temp1 != temp2);
// Check if RA has slow marker.
load32(Address(ra, 0), temp2);
ma_mov(Imm32(SlowCallMarker), temp1, Always);
ma_cmp(temp2, temp1);
j(Assembler::NotEqual, notSlow);
}
#endif // ENABLE_WASM_TAIL_CALLS
//}}} check_macroassembler_style
void MacroAssemblerARM::wasmTruncateToInt32(FloatRegister input,

14
js/src/jit/arm64/MacroAssembler-arm64.cpp
View File

@ -3419,6 +3419,20 @@ void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
Operand(ARMRegister(indexTemp32, 64), vixl::LSL, shift));
}
#ifdef ENABLE_WASM_TAIL_CALLS
void MacroAssembler::wasmMarkSlowCall() { Mov(x28, x28); }
const int32_t SlowCallMarker = 0xaa1c03fc;
void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
Register temp1, Register temp2) {
MOZ_ASSERT(ra != temp2);
Ldr(W(temp2), MemOperand(X(ra), 0));
Cmp(W(temp2), Operand(SlowCallMarker));
B(Assembler::NotEqual, notSlow);
}
#endif // ENABLE_WASM_TAIL_CALLS
//}}} check_macroassembler_style
} // namespace jit

16
js/src/jit/x64/MacroAssembler-x64.cpp
View File

@ -1598,6 +1598,22 @@ void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
}
}
#ifdef ENABLE_WASM_TAIL_CALLS
void MacroAssembler::wasmMarkSlowCall() {
static_assert(InstanceReg == r14);
orPtr(Imm32(0), r14);
}
const int32_t SlowCallMarker = 0x00ce8349; // OR r14, 0
void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
Register temp1, Register temp2) {
// Check if RA has slow marker.
cmp32(Address(ra, 0), Imm32(SlowCallMarker));
j(Assembler::NotEqual, notSlow);
}
#endif // ENABLE_WASM_TAIL_CALLS
// ========================================================================
// Integer compare-then-conditionally-load/move operations.

16
js/src/jit/x86/MacroAssembler-x86.cpp
View File

@ -1826,4 +1826,20 @@ void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
bind(&notOk);
}
#ifdef ENABLE_WASM_TAIL_CALLS
void MacroAssembler::wasmMarkSlowCall() {
static_assert(esi == InstanceReg);
or32(esi, esi);
}
const int32_t SlowCallMarker = 0xf60b; // OR esi, esi
void MacroAssembler::wasmCheckSlowCallsite(Register ra, Label* notSlow,
Register temp1, Register temp2) {
// Check if RA has slow marker.
cmp16(Address(ra, 0), Imm32(SlowCallMarker));
j(Assembler::NotEqual, notSlow);
}
#endif // ENABLE_WASM_TAIL_CALLS
//}}} check_macroassembler_style

1
js/src/jsapi-tests/moz.build
View File

@ -134,6 +134,7 @@ UNIFIED_SOURCES += [
"testUncaughtSymbol.cpp",
"testUTF8.cpp",
"testWasmLEB128.cpp",
"testWasmReturnCalls.cpp",
"testWeakMap.cpp",
"testWindowNonConfigurable.cpp",
]

91
js/src/jsapi-tests/testWasmReturnCalls.cpp Normal file
View File

@ -0,0 +1,91 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
*/
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "jit/MacroAssembler.h"
#include "jsapi-tests/tests.h"
#include "jsapi-tests/testsJit.h"
#include "jit/MacroAssembler-inl.h"
using namespace js;
using namespace js::jit;
#if defined(ENABLE_WASM_TAIL_CALLS) && !defined(JS_CODEGEN_NONE)
// Check if wasmMarkSlowCall produces the byte sequence that can
// wasmCheckSlowCallsite detect.
BEGIN_TEST(testWasmCheckSlowCallMarkerHit) {
js::LifoAlloc lifo(4096);
TempAllocator alloc(&lifo);
JitContext jc(cx);
StackMacroAssembler masm(cx, alloc);
AutoCreatedBy acb(masm, __func__);
PrepareJit(masm);
Label check, fail, end;
masm.call(&check);
masm.wasmMarkSlowCall();
masm.jump(&end);
masm.bind(&check);
# ifdef JS_USE_LINK_REGISTER
static constexpr Register ra = lr;
# else
static constexpr Register ra = ABINonArgReg2;
masm.loadPtr(Address(StackPointer, 0), ra);
# endif
masm.wasmCheckSlowCallsite(ra, &fail, ABINonArgReg1, ABINonArgReg2);
masm.abiret();
masm.bind(&fail);
masm.printf("Failed\n");
masm.breakpoint();
masm.bind(&end);
return ExecuteJit(cx, masm);
}
END_TEST(testWasmCheckSlowCallMarkerHit)
// Check if wasmCheckSlowCallsite does not detect non-marked slow calls.
BEGIN_TEST(testWasmCheckSlowCallMarkerMiss) {
js::LifoAlloc lifo(4096);
TempAllocator alloc(&lifo);
JitContext jc(cx);
StackMacroAssembler masm(cx, alloc);
AutoCreatedBy acb(masm, __func__);
PrepareJit(masm);
Label check, fast, end;
masm.call(&check);
masm.nop();
masm.jump(&end);
masm.bind(&check);
# ifdef JS_USE_LINK_REGISTER
static constexpr Register ra = lr;
# else
static constexpr Register ra = ABINonArgReg2;
masm.loadPtr(Address(StackPointer, 0), ra);
# endif
masm.wasmCheckSlowCallsite(ra, &fast, ABINonArgReg1, ABINonArgReg2);
masm.printf("Failed\n");
masm.breakpoint();
masm.bind(&fast);
masm.abiret();
masm.bind(&end);
return ExecuteJit(cx, masm);
}
END_TEST(testWasmCheckSlowCallMarkerMiss)
#endif // ENABLE_WASM_TAIL_CALLS

5
js/src/vm/JitActivation.cpp
View File

@ -228,7 +228,10 @@ void js::jit::JitActivation::startWasmTrap(wasm::Trap trap,
bool unwound;
wasm::UnwindState unwindState;
MOZ_RELEASE_ASSERT(wasm::StartUnwinding(state, &unwindState, &unwound));
MOZ_ASSERT(unwound == (trap == wasm::Trap::IndirectCallBadSig));
// With return calls, it is possible to not unwind when there is only an
// entry left on the stack, e.g. the return call trampoline that is created
// to restore realm before returning to the interpreter entry stub.
MOZ_ASSERT_IF(unwound, trap == wasm::Trap::IndirectCallBadSig);
void* pc = unwindState.pc;
const wasm::Frame* fp = wasm::Frame::fromUntaggedWasmExitFP(unwindState.fp);

43
js/src/wasm/WasmBCClass-inl.h
View File

@ -77,6 +77,49 @@ uint32_t BaseCompiler::instanceOffsetOfBoundsCheckLimit(
offsetof(MemoryInstanceData, boundsCheckLimit));
}
// The results parameter for BaseCompiler::emitCallArgs is used for
// regular Wasm calls.
struct NormalCallResults final {
const StackResultsLoc& results;
explicit NormalCallResults(const StackResultsLoc& results)
: results(results) {}
inline uint32_t onStackCount() const { return results.count(); }
inline StackResults stackResults() const { return results.stackResults(); }
inline void getStackResultArea(BaseStackFrame fr, RegPtr dest) const {
fr.computeOutgoingStackResultAreaPtr(results, dest);
}
};
// The results parameter for BaseCompiler::emitCallArgs is used for
// Wasm return/tail calls.
struct TailCallResults final {
bool hasStackResults;
explicit TailCallResults(const FuncType& funcType) {
hasStackResults =
ABIResultIter::HasStackResults(ResultType::Vector(funcType.results()));
}
inline uint32_t onStackCount() const { return 0; }
inline StackResults stackResults() const {
return hasStackResults ? StackResults::HasStackResults
: StackResults::NoStackResults;
}
inline void getStackResultArea(BaseStackFrame fr, RegPtr dest) const {
fr.loadIncomingStackResultAreaPtr(dest);
}
};
// The results parameter for BaseCompiler::emitCallArgs is used when
// no result (area) is expected.
struct NoCallResults final {
inline uint32_t onStackCount() const { return 0; }
inline StackResults stackResults() const {
return StackResults::NoStackResults;
}
inline void getStackResultArea(BaseStackFrame fr, RegPtr dest) const {
MOZ_CRASH();
}
};
} // namespace wasm
} // namespace js

11
js/src/wasm/WasmBCClass.h
View File

@ -955,7 +955,8 @@ struct BaseCompiler final {
bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
const Stk& indexVal, const FunctionCall& call,
CodeOffset* fastCallOffset, CodeOffset* slowCallOffset);
bool tailCall, CodeOffset* fastCallOffset,
CodeOffset* slowCallOffset);
CodeOffset callImport(unsigned instanceDataOffset, const FunctionCall& call);
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
void callRef(const Stk& calleeRef, const FunctionCall& call,
@ -1396,13 +1397,17 @@ struct BaseCompiler final {
False
};
[[nodiscard]] bool emitCallArgs(const ValTypeVector& argTypes,
const StackResultsLoc& results,
// The typename T for emitCallArgs can be one of the following:
// NormalCallResults, TailCallResults, or NoCallResults.
template <typename T>
[[nodiscard]] bool emitCallArgs(const ValTypeVector& argTypes, T results,
FunctionCall* baselineCall,
CalleeOnStack calleeOnStack);
[[nodiscard]] bool emitCall();
[[nodiscard]] bool emitReturnCall();
[[nodiscard]] bool emitCallIndirect();
[[nodiscard]] bool emitReturnCallIndirect();
[[nodiscard]] bool emitUnaryMathBuiltinCall(SymbolicAddress callee,
ValType operandType);
[[nodiscard]] bool emitGetLocal();

19
js/src/wasm/WasmBCFrame.h
View File

@ -725,7 +725,9 @@ class BaseStackFrame final : public BaseStackFrameAllocator {
}
void loadInstancePtr(Register dst) {
masm.loadPtr(Address(sp_, stackOffset(instancePointerOffset_)), dst);
// Sometimes loadInstancePtr is used in context when SP is not sync is FP,
// e.g. just after tail calls returns.
masm.loadPtr(Address(FramePointer, -instancePointerOffset_), dst);
}
void storeInstancePtr(Register instance) {
@ -941,21 +943,24 @@ class BaseStackFrame final : public BaseStackFrameAllocator {
uint32_t bytes, Register temp) {
MOZ_ASSERT(destHeight < srcHeight);
MOZ_ASSERT(bytes % sizeof(uint32_t) == 0);
uint32_t destOffset = stackOffset(destHeight) + bytes;
uint32_t srcOffset = stackOffset(srcHeight) + bytes;
// The shuffleStackResultsTowardFP is used when SP/framePushed is not
// tracked by the compiler, e.g. after possible return call -- use
// FramePointer instead of sp_.
int32_t destOffset = int32_t(-destHeight + bytes);
int32_t srcOffset = int32_t(-srcHeight + bytes);
while (bytes >= sizeof(intptr_t)) {
destOffset -= sizeof(intptr_t);
srcOffset -= sizeof(intptr_t);
bytes -= sizeof(intptr_t);
masm.loadPtr(Address(sp_, srcOffset), temp);
masm.storePtr(temp, Address(sp_, destOffset));
masm.loadPtr(Address(FramePointer, srcOffset), temp);
masm.storePtr(temp, Address(FramePointer, destOffset));
}
if (bytes) {
MOZ_ASSERT(bytes == sizeof(uint32_t));
destOffset -= sizeof(uint32_t);
srcOffset -= sizeof(uint32_t);
masm.load32(Address(sp_, srcOffset), temp);
masm.store32(temp, Address(sp_, destOffset));
masm.load32(Address(FramePointer, srcOffset), temp);
masm.store32(temp, Address(FramePointer, destOffset));
}
}

164
js/src/wasm/WasmBaselineCompile.cpp
View File

@ -1575,9 +1575,18 @@ class OutOfLineAbortingTrap : public OutOfLineCode {
}
};
#ifdef ENABLE_WASM_TAIL_CALLS
static ReturnCallAdjustmentInfo BuildReturnCallAdjustmentInfo(
const FuncType& callerType, const FuncType& calleeType) {
return ReturnCallAdjustmentInfo(
StackArgAreaSizeUnaligned(ArgTypeVector(calleeType)),
StackArgAreaSizeUnaligned(ArgTypeVector(callerType)));
}
#endif
bool BaseCompiler::callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
const Stk& indexVal, const FunctionCall& call,
CodeOffset* fastCallOffset,
bool tailCall, CodeOffset* fastCallOffset,
CodeOffset* slowCallOffset) {
CallIndirectId callIndirectId =
CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex);
@ -1604,8 +1613,19 @@ bool BaseCompiler::callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex,
}
nullCheckFailed = nullref->entry();
#endif
masm.wasmCallIndirect(desc, callee, oob->entry(), nullCheckFailed,
mozilla::Nothing(), fastCallOffset, slowCallOffset);
if (!tailCall) {
masm.wasmCallIndirect(desc, callee, oob->entry(), nullCheckFailed,
mozilla::Nothing(), fastCallOffset, slowCallOffset);
} else {
#ifdef ENABLE_WASM_TAIL_CALLS
ReturnCallAdjustmentInfo retCallInfo = BuildReturnCallAdjustmentInfo(
this->funcType(), (*moduleEnv_.types)[funcTypeIndex].funcType());
masm.wasmReturnCallIndirect(desc, callee, oob->entry(), nullCheckFailed,
mozilla::Nothing(), retCallInfo);
#else
MOZ_CRASH("not available");
#endif
}
return true;
}
@ -4562,8 +4582,8 @@ bool BaseCompiler::emitReturn() {
return true;
}
bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes,
const StackResultsLoc& results,
template <typename T>
bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes, T results,
FunctionCall* baselineCall,
CalleeOnStack calleeOnStack) {
MOZ_ASSERT(!deadCode_);
@ -4574,7 +4594,7 @@ bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes,
startCallArgs(StackArgAreaSizeUnaligned(args), baselineCall);
// Args are deeper on the stack than the stack result area, if any.
size_t argsDepth = results.count();
size_t argsDepth = results.onStackCount();
// They're deeper than the callee too, for callIndirect.
if (calleeOnStack == CalleeOnStack::True) {
argsDepth++;
@ -4590,11 +4610,11 @@ bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes,
ABIArg argLoc = baselineCall->abi.next(MIRType::Pointer);
if (argLoc.kind() == ABIArg::Stack) {
ScratchPtr scratch(*this);
fr.computeOutgoingStackResultAreaPtr(results, scratch);
results.getStackResultArea(fr, scratch);
masm.storePtr(scratch, Address(masm.getStackPointer(),
argLoc.offsetFromArgBase()));
} else {
fr.computeOutgoingStackResultAreaPtr(results, RegPtr(argLoc.gpr()));
results.getStackResultArea(fr, RegPtr(argLoc.gpr()));
}
}
}
@ -4755,7 +4775,7 @@ bool BaseCompiler::emitCall() {
import ? RestoreRegisterStateAndRealm::True
: RestoreRegisterStateAndRealm::False);
if (!emitCallArgs(funcType.args(), results, &baselineCall,
if (!emitCallArgs(funcType.args(), NormalCallResults(results), &baselineCall,
CalleeOnStack::False)) {
return false;
}
@ -4782,6 +4802,59 @@ bool BaseCompiler::emitCall() {
return pushCallResults(baselineCall, resultType, results);
}
#ifdef ENABLE_WASM_TAIL_CALLS
bool BaseCompiler::emitReturnCall() {
uint32_t funcIndex;
BaseNothingVector args_{};
BaseNothingVector unused_values{};
if (!iter_.readReturnCall(&funcIndex, &args_, &unused_values)) {
return false;
}
if (deadCode_) {
return true;
}
sync();
const FuncType& funcType = *moduleEnv_.funcs[funcIndex].type;
bool import = moduleEnv_.funcIsImport(funcIndex);
uint32_t numArgs = funcType.args().length();
FunctionCall baselineCall{};
beginCall(baselineCall, UseABI::Wasm,
import ? RestoreRegisterStateAndRealm::True
: RestoreRegisterStateAndRealm::False);
if (!emitCallArgs(funcType.args(), TailCallResults(funcType), &baselineCall,
CalleeOnStack::False)) {
return false;
}
ReturnCallAdjustmentInfo retCallInfo =
BuildReturnCallAdjustmentInfo(this->funcType(), funcType);
if (import) {
CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Import);
CalleeDesc callee = CalleeDesc::import(
moduleEnv_.offsetOfFuncImportInstanceData(funcIndex));
masm.wasmReturnCallImport(desc, callee, retCallInfo);
} else {
CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::ReturnFunc);
masm.wasmReturnCall(desc, funcIndex, retCallInfo);
}
MOZ_ASSERT(stackMapGenerator_.framePushedExcludingOutboundCallArgs.isSome());
stackMapGenerator_.framePushedExcludingOutboundCallArgs.reset();
popValueStackBy(numArgs);
deadCode_ = true;
return true;
}
#endif
bool BaseCompiler::emitCallIndirect() {
uint32_t funcTypeIndex;
uint32_t tableIndex;
@ -4815,7 +4888,7 @@ bool BaseCompiler::emitCallIndirect() {
// MacroAssembler::wasmCallIndirect).
beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False);
if (!emitCallArgs(funcType.args(), results, &baselineCall,
if (!emitCallArgs(funcType.args(), NormalCallResults(results), &baselineCall,
CalleeOnStack::True)) {
return false;
}
@ -4824,7 +4897,7 @@ bool BaseCompiler::emitCallIndirect() {
CodeOffset fastCallOffset;
CodeOffset slowCallOffset;
if (!callIndirect(funcTypeIndex, tableIndex, callee, baselineCall,
&fastCallOffset, &slowCallOffset)) {
/*tailCall*/ false, &fastCallOffset, &slowCallOffset)) {
return false;
}
if (!createStackMap("emitCallIndirect", fastCallOffset)) {
@ -4844,6 +4917,58 @@ bool BaseCompiler::emitCallIndirect() {
return pushCallResults(baselineCall, resultType, results);
}
#ifdef ENABLE_WASM_TAIL_CALLS
bool BaseCompiler::emitReturnCallIndirect() {
uint32_t funcTypeIndex;
uint32_t tableIndex;
Nothing callee_;
BaseNothingVector args_{};
BaseNothingVector unused_values{};
if (!iter_.readReturnCallIndirect(&funcTypeIndex, &tableIndex, &callee_,
&args_, &unused_values)) {
return false;
}
if (deadCode_) {
return true;
}
sync();
const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType();
// Stack: ... arg1 .. argn callee
uint32_t numArgs = funcType.args().length() + 1;
FunctionCall baselineCall{};
// State and realm are restored as needed by by callIndirect (really by
// MacroAssembler::wasmCallIndirect).
beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False);
if (!emitCallArgs(funcType.args(), TailCallResults(funcType), &baselineCall,
CalleeOnStack::True)) {
return false;
}
const Stk& callee = peek(0);
CodeOffset fastCallOffset;
CodeOffset slowCallOffset;
if (!callIndirect(funcTypeIndex, tableIndex, callee, baselineCall,
/*tailCall*/ true, &fastCallOffset, &slowCallOffset)) {
return false;
}
MOZ_ASSERT(stackMapGenerator_.framePushedExcludingOutboundCallArgs.isSome());
stackMapGenerator_.framePushedExcludingOutboundCallArgs.reset();
popValueStackBy(numArgs);
deadCode_ = true;
return true;
}
#endif
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
bool BaseCompiler::emitCallRef() {
const FuncType* funcType;
@ -4875,7 +5000,7 @@ bool BaseCompiler::emitCallRef() {
// MacroAssembler::wasmCallRef).
beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False);
if (!emitCallArgs(funcType->args(), results, &baselineCall,
if (!emitCallArgs(funcType->args(), NormalCallResults(results), &baselineCall,
CalleeOnStack::True)) {
return false;
}
@ -4940,12 +5065,11 @@ bool BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee,
ValType retType = operandType;
uint32_t numArgs = signature.length();
size_t stackSpace = stackConsumed(numArgs);
StackResultsLoc noStackResults;
FunctionCall baselineCall{};
beginCall(baselineCall, UseABI::Builtin, RestoreRegisterStateAndRealm::False);
if (!emitCallArgs(signature, noStackResults, &baselineCall,
if (!emitCallArgs(signature, NoCallResults(), &baselineCall,
CalleeOnStack::False)) {
return false;
}
@ -9207,6 +9331,18 @@ bool BaseCompiler::emitBody() {
CHECK_NEXT(emitCall());
case uint16_t(Op::CallIndirect):
CHECK_NEXT(emitCallIndirect());
#ifdef ENABLE_WASM_TAIL_CALLS
case uint16_t(Op::ReturnCall):
if (!moduleEnv_.tailCallsEnabled()) {
return iter_.unrecognizedOpcode(&op);
}
CHECK_NEXT(emitReturnCall());
case uint16_t(Op::ReturnCallIndirect):
if (!moduleEnv_.tailCallsEnabled()) {
return iter_.unrecognizedOpcode(&op);
}
CHECK_NEXT(emitReturnCallIndirect());
#endif
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
case uint16_t(Op::CallRef):
if (!moduleEnv_.functionReferencesEnabled()) {

9
js/src/wasm/WasmBuiltins.cpp
View File

@ -628,6 +628,15 @@ bool wasm::HandleThrow(JSContext* cx, WasmFrameIter& iter,
const wasm::TryNote* tryNote = code.lookupTryNote((void*)pc, &tier);
if (tryNote) {
#ifdef ENABLE_WASM_TAIL_CALLS
// Skip tryNote if pc is at return stub generated by
// wasmCollapseFrameSlow.
const CallSite* site = code.lookupCallSite((void*)pc);
if (site && site->kind() == CallSite::ReturnStub) {
continue;
}
#endif
cx->clearPendingException();
RootedAnyRef ref(cx, AnyRef::null());
if (!BoxAnyRef(cx, exn, &ref)) {

6
js/src/wasm/WasmCodegenTypes.h
View File

@ -419,6 +419,8 @@ class CallSiteDesc {
IndirectFast, // dynamically determined to be same-instance
FuncRef, // call using direct function reference
FuncRefFast, // call using direct function reference within same-instance
ReturnFunc, // return call to a specific function
ReturnStub, // return call trampoline
Symbolic, // call to a single symbolic callee
EnterFrame, // call to a enter frame handler
LeaveFrame, // call to a leave frame handler
@ -443,8 +445,10 @@ class CallSiteDesc {
bool isImportCall() const { return kind() == CallSiteDesc::Import; }
bool isIndirectCall() const { return kind() == CallSiteDesc::Indirect; }
bool isFuncRefCall() const { return kind() == CallSiteDesc::FuncRef; }
bool isReturnStub() const { return kind() == CallSiteDesc::ReturnStub; }
bool mightBeCrossInstance() const {
return isImportCall() || isIndirectCall() || isFuncRefCall();
return isImportCall() || isIndirectCall() || isFuncRefCall() ||
isReturnStub();
}
};

2
js/src/wasm/WasmConstants.h
View File

@ -262,6 +262,8 @@ enum class Op {
// Call operators
Call = 0x10,
CallIndirect = 0x11,
ReturnCall = 0x12,
ReturnCallIndirect = 0x13,
CallRef = 0x14,
// Additional exception operators

6
js/src/wasm/WasmFeatures.cpp
View File

@ -197,13 +197,17 @@ bool wasm::IonDisabledByFeatures(JSContext* cx, bool* isDisabled,
JSStringBuilder* reason) {
// Ion has no debugging support.
bool debug = WasmDebuggerActive(cx);
bool tailCalls = WasmTailCallsFlag(cx);
if (reason) {
char sep = 0;
if (debug && !Append(reason, "debug", &sep)) {
return false;
}
if (tailCalls && !Append(reason, "tail-calls", &sep)) {
return false;
}
}
*isDisabled = debug;
*isDisabled = debug || tailCalls;
return true;
}

4
js/src/wasm/WasmFrame.h
View File

@ -370,6 +370,10 @@ class FrameWithInstances : public Frame {
return sizeof(wasm::FrameWithInstances) - sizeof(wasm::Frame);
}
constexpr static uint32_t sizeOfInstanceFieldsAndShadowStack() {
return sizeOfInstanceFields() + js::jit::ShadowStackSpace;
}
constexpr static uint32_t calleeInstanceOffset() {
return offsetof(FrameWithInstances, calleeInstance_) +
js::jit::ShadowStackSpace;

16
js/src/wasm/WasmFrameIter.cpp
View File

@ -1044,7 +1044,7 @@ void ProfilingFrameIterator::initFromExitFP(const Frame* fp) {
MOZ_ASSERT(!done());
}
static bool isSignatureCheckFail(uint32_t offsetInCode,
static bool IsSignatureCheckFail(uint32_t offsetInCode,
const CodeRange* codeRange) {
if (!codeRange->isFunction()) {
return false;
@ -1060,6 +1060,17 @@ static bool isSignatureCheckFail(uint32_t offsetInCode,
(offsetInCode - codeRange->funcCheckedCallEntry()) > SetFP;
}
static bool CanUnwindSignatureCheck(uint8_t* fp) {
const auto* frame = Frame::fromUntaggedWasmExitFP(fp);
uint8_t* const pc = frame->returnAddress();
const CodeRange* codeRange;
const Code* code = LookupCode(pc, &codeRange);
// If a JIT call or JIT/interpreter entry was found,
// unwinding is not possible.
return code && !codeRange->isEntry();
}
const Instance* js::wasm::GetNearestEffectiveInstance(const Frame* fp) {
while (true) {
uint8_t* returnAddress = fp->returnAddress();
@ -1309,7 +1320,8 @@ bool js::wasm::StartUnwinding(const RegisterState& registers,
AssertMatchesCallSite(fixedPC, fixedFP);
#endif
} else {
if (isSignatureCheckFail(offsetInCode, codeRange)) {
if (IsSignatureCheckFail(offsetInCode, codeRange) &&
CanUnwindSignatureCheck(fp)) {
// Frame has been pushed and FP has been set.
const auto* frame = Frame::fromUntaggedWasmExitFP(fp);
fixedFP = frame->rawCaller();

15
js/src/wasm/WasmGenerator.cpp
View File

@ -466,13 +466,24 @@ bool ModuleGenerator::linkCallSites() {
case CallSiteDesc::LeaveFrame:
case CallSiteDesc::FuncRef:
case CallSiteDesc::FuncRefFast:
case CallSiteDesc::ReturnStub:
break;
case CallSiteDesc::ReturnFunc:
case CallSiteDesc::Func: {
auto patch = [this, callSite](uint32_t callerOffset,
uint32_t calleeOffset) {
if (callSite.kind() == CallSiteDesc::ReturnFunc) {
masm_.patchFarJump(CodeOffset(callerOffset), calleeOffset);
} else {
MOZ_ASSERT(callSite.kind() == CallSiteDesc::Func);
masm_.patchCall(callerOffset, calleeOffset);
}
};
if (funcIsCompiled(target.funcIndex())) {
uint32_t calleeOffset =
funcCodeRange(target.funcIndex()).funcUncheckedCallEntry();
if (InRange(callerOffset, calleeOffset)) {
masm_.patchCall(callerOffset, calleeOffset);
patch(callerOffset, calleeOffset);
break;
}
}
@ -499,7 +510,7 @@ bool ModuleGenerator::linkCallSites() {
}
}
masm_.patchCall(callerOffset, p->value());
patch(callerOffset, p->value());
break;
}
}

4
js/src/wasm/WasmOpIter.cpp
View File

@ -250,8 +250,12 @@ OpKind wasm::Classify(OpBytes op) {
return OpKind::TableSet;
case Op::Call:
return OpKind::Call;
case Op::ReturnCall:
return OpKind::ReturnCall;
case Op::CallIndirect:
return OpKind::CallIndirect;
case Op::ReturnCallIndirect:
return OpKind::ReturnCallIndirect;
case Op::CallRef:
WASM_FUNCTION_REFERENCES_OP(OpKind::CallRef);
case Op::Return:

111
js/src/wasm/WasmOpIter.h
View File

@ -161,7 +161,9 @@ enum class OpKind {
SetGlobal,
TeeGlobal,
Call,
ReturnCall,
CallIndirect,
ReturnCallIndirect,
# ifdef ENABLE_WASM_FUNCTION_REFERENCES
CallRef,
# endif
@ -685,6 +687,15 @@ class MOZ_STACK_CLASS OpIter : private Policy {
[[nodiscard]] bool readCallIndirect(uint32_t* funcTypeIndex,
uint32_t* tableIndex, Value* callee,
ValueVector* argValues);
#ifdef ENABLE_WASM_TAIL_CALLS
[[nodiscard]] bool readReturnCall(uint32_t* funcTypeIndex,
ValueVector* argValues,
ValueVector* values);
[[nodiscard]] bool readReturnCallIndirect(uint32_t* funcTypeIndex,
uint32_t* tableIndex, Value* callee,
ValueVector* argValues,
ValueVector* values);
#endif
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
[[nodiscard]] bool readCallRef(const FuncType** funcType, Value* callee,
ValueVector* argValues);
@ -2440,6 +2451,44 @@ inline bool OpIter<Policy>::readCall(uint32_t* funcTypeIndex,
return push(ResultType::Vector(funcType.results()));
}
#ifdef ENABLE_WASM_TAIL_CALLS
template <typename Policy>
inline bool OpIter<Policy>::readReturnCall(uint32_t* funcTypeIndex,
ValueVector* argValues,
ValueVector* values) {
MOZ_ASSERT(Classify(op_) == OpKind::ReturnCall);
if (!readVarU32(funcTypeIndex)) {
return fail("unable to read call function index");
}
if (*funcTypeIndex >= env_.funcs.length()) {
return fail("callee index out of range");
}
const FuncType& funcType = *env_.funcs[*funcTypeIndex].type;
if (!popCallArgs(funcType.args(), argValues)) {
return false;
}
if (!push(ResultType::Vector(funcType.results()))) {
return false;
}
Control& body = controlStack_[0];
MOZ_ASSERT(body.kind() == LabelKind::Body);
// Pop function results as the instruction will cause a return.
if (!popWithType(body.resultType(), values)) {
return false;
}
afterUnconditionalBranch();
return true;
}
#endif
template <typename Policy>
inline bool OpIter<Policy>::readCallIndirect(uint32_t* funcTypeIndex,
uint32_t* tableIndex,
@ -2487,6 +2536,68 @@ inline bool OpIter<Policy>::readCallIndirect(uint32_t* funcTypeIndex,
return push(ResultType::Vector(funcType.results()));
}
#ifdef ENABLE_WASM_TAIL_CALLS
template <typename Policy>
inline bool OpIter<Policy>::readReturnCallIndirect(uint32_t* funcTypeIndex,
uint32_t* tableIndex,
Value* callee,
ValueVector* argValues,
ValueVector* values) {
MOZ_ASSERT(Classify(op_) == OpKind::ReturnCallIndirect);
MOZ_ASSERT(funcTypeIndex != tableIndex);
if (!readVarU32(funcTypeIndex)) {
return fail("unable to read return_call_indirect signature index");
}
if (*funcTypeIndex >= env_.numTypes()) {
return fail("signature index out of range");
}
if (!readVarU32(tableIndex)) {
return fail("unable to read return_call_indirect table index");
}
if (*tableIndex >= env_.tables.length()) {
// Special case this for improved user experience.
if (!env_.tables.length()) {
return fail("can't return_call_indirect without a table");
}
return fail("table index out of range for return_call_indirect");
}
if (!env_.tables[*tableIndex].elemType.isFuncHierarchy()) {
return fail("indirect calls must go through a table of 'funcref'");
}
if (!popWithType(ValType::I32, callee)) {
return false;
}
const TypeDef& typeDef = env_.types->type(*funcTypeIndex);
if (!typeDef.isFuncType()) {
return fail("expected signature type");
}
const FuncType& funcType = typeDef.funcType();
if (!popCallArgs(funcType.args(), argValues)) {
return false;
}
if (!push(ResultType::Vector(funcType.results()))) {
return false;
}
Control& body = controlStack_[0];
MOZ_ASSERT(body.kind() == LabelKind::Body);
// Pop function results as the instruction will cause a return.
if (!popWithType(body.resultType(), values)) {
return false;
}
afterUnconditionalBranch();
return true;
}
#endif
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
template <typename Policy>
inline bool OpIter<Policy>::readCallRef(const FuncType** funcType,

19
js/src/wasm/WasmStubs.cpp
View File

@ -553,8 +553,6 @@ static const LiveRegisterSet NonVolatileRegs =
FloatRegisterSet(FloatRegisters::NonVolatileMask));
#endif
static const unsigned NumExtraPushed = 2; // instance and argv
#ifdef JS_CODEGEN_ARM64
static const unsigned WasmPushSize = 16;
#else
@ -793,9 +791,9 @@ static bool GenerateInterpEntry(MacroAssembler& masm, const FuncExport& fe,
masm.moveStackPtrTo(FramePointer);
masm.setFramePushed(0);
#ifdef JS_CODEGEN_ARM64
const size_t FakeFramePushed = 0;
DebugOnly<size_t> fakeFramePushed = 0;
#else
const size_t FakeFramePushed = sizeof(void*);
DebugOnly<size_t> fakeFramePushed = sizeof(void*);
masm.Push(scratch);
#endif
@ -827,14 +825,19 @@ static bool GenerateInterpEntry(MacroAssembler& masm, const FuncExport& fe,
// Save 'argv' on the stack so that we can recover it after the call.
WasmPush(masm, argv);
MOZ_ASSERT(masm.framePushed() ==
NumExtraPushed * WasmPushSize + FakeFramePushed);
MOZ_ASSERT(masm.framePushed() == 2 * WasmPushSize + fakeFramePushed,
"expected instance, argv, and fake frame");
uint32_t frameSizeBeforeCall = masm.framePushed();
// Align (missing) results area to WasmStackAlignment boudary. Return calls
// expect arguments to not overlap with results or other slots.
unsigned aligned =
AlignBytes(masm.framePushed() + FakeFrameSize, WasmStackAlignment);
masm.reserveStack(aligned - masm.framePushed() + FakeFrameSize);
// Reserve stack space for the wasm call.
unsigned argDecrement = StackDecrementForCall(
WasmStackAlignment, masm.framePushed() + FakeFrameSize,
StackArgBytesForWasmABI(funcType));
WasmStackAlignment, aligned, StackArgBytesForWasmABI(funcType));
masm.reserveStack(argDecrement);
// Copy parameters out of argv and into the wasm ABI registers/stack-slots.

21
js/src/wasm/WasmValidate.cpp
View File

@ -214,6 +214,27 @@ static bool DecodeFunctionBodyExprs(const ModuleEnvironment& env,
CHECK(iter.readCallIndirect(&unusedIndex, &unusedIndex2, &nothing,
&unusedArgs));
}
#ifdef ENABLE_WASM_TAIL_CALLS
case uint16_t(Op::ReturnCall): {
if (!env.tailCallsEnabled()) {
return iter.unrecognizedOpcode(&op);
}
uint32_t unusedIndex;
NothingVector unusedArgs{};
NothingVector unusedValues{};
CHECK(iter.readReturnCall(&unusedIndex, &unusedArgs, &unusedValues));
}
case uint16_t(Op::ReturnCallIndirect): {
if (!env.tailCallsEnabled()) {
return iter.unrecognizedOpcode(&op);
}
uint32_t unusedIndex, unusedIndex2;
NothingVector unusedArgs{};
NothingVector unusedValues{};
CHECK(iter.readReturnCallIndirect(&unusedIndex, &unusedIndex2, &nothing,
&unusedArgs, &unusedValues));
}
#endif
#ifdef ENABLE_WASM_FUNCTION_REFERENCES
case uint16_t(Op::CallRef): {
if (!env.functionReferencesEnabled()) {

7
modules/libpref/init/StaticPrefList.yaml
View File

@ -7651,6 +7651,13 @@
mirror: always
#endif // defined(ENABLE_WASM_MEMORY64)
#if defined(ENABLE_WASM_TAIL_CALLS)
- name: javascript.options.wasm_tail_calls
type: bool
value: @IS_NIGHTLY_BUILD@
mirror: always
#endif // defined(ENABLE_WASM_TAIL_CALLS)
# Support for pretenuring allocations based on their allocation site.
- name: javascript.options.site_based_pretenuring
type: bool