mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-01-27 07:34:20 +00:00
Bug 1627854 - Add documentation for spec-test importing. r=lth
The process of importing spec-tests is confusing and has some tricks that I should write down. The files are also scattered around, so I've tried to move them all into one spot. * Moves wasm/Makefile to jit-test/etc/wasm * Moves jit-test/etc/wasm-config[-lock].toml to jit-test/etc/wasm * Adds jit-test/etc/wasm/README.md Differential Revision: https://phabricator.services.mozilla.com/D69902 --HG-- rename : js/src/wasm/Makefile => js/src/jit-test/etc/wasm/Makefile rename : js/src/jit-test/etc/wasm-config-lock.toml => js/src/jit-test/etc/wasm/config-lock.toml rename : js/src/jit-test/etc/wasm-config.toml => js/src/jit-test/etc/wasm/config.toml extra : moz-landing-system : lando
This commit is contained in:
Ryan Hunt
parent
1a005eb490
commit
f33032205b
@ -57,7 +57,7 @@ _OPT\.OBJ/
|
||||
^js/src/tests/results-.*\.(html|txt)$
|
||||
^js/src/devtools/rootAnalysis/t/out
|
||||
# SpiderMonkey clone of the wasm-generate-testsuite repository
|
||||
^js/src/wasm/wasm-generate-testsuite
|
||||
^js/src/jit-test/etc/wasm/wasm-generate-testsuite/
|
||||
|
||||
# Java HTML5 parser classes
|
||||
^parser/html/java/(html|java)parser/
|
||||
|
||||
26
js/src/jit-test/etc/wasm/Makefile
Normal file
26
js/src/jit-test/etc/wasm/Makefile
Normal file
@ -0,0 +1,26 @@
|
||||
.PHONY: update run expectations
|
||||
|
||||
warning = '\# Wasm Spec Tests\n\nThese tests are autogenerated using a tool, do not edit.\n\nSee `jit-test/etc/wasm/` for more information.'
|
||||
|
||||
update:
|
||||
[ -d ./wasm-generate-testsuite ] || git clone https://github.com/eqrion/wasm-generate-testsuite ./wasm-generate-testsuite
|
||||
cp ./config.toml ./wasm-generate-testsuite/config.toml
|
||||
cp ./config-lock.toml ./wasm-generate-testsuite/config-lock.toml
|
||||
(cd ./wasm-generate-testsuite && RUST_LOG=info cargo run)
|
||||
cp ./wasm-generate-testsuite/config-lock.toml ./config-lock.toml
|
||||
rm -r ../../tests/wasm/spec
|
||||
cp -R wasm-generate-testsuite/tests/js ../../tests/wasm/spec
|
||||
echo $(warning) > ../../tests/wasm/spec/README.md
|
||||
[ ! -d ./spec-tests.patch ] || (cd ../../tests/wasm/spec && patch -u -p7 < ../../../etc/wasm/spec-tests.patch)
|
||||
rm -r ../../../../../testing/web-platform/mozilla/tests/wasm
|
||||
cp -R wasm-generate-testsuite/tests/wpt ../../../../../testing/web-platform/mozilla/tests/wasm
|
||||
echo $(warning) > ../../../../../testing/web-platform/mozilla/tests/wasm/README.md
|
||||
|
||||
run:
|
||||
@[ -z $(MOZCONFIG) ] && echo "You need to define the MOZCONFIG env variable first."
|
||||
@[ -z $(MOZCONFIG) ] || ../../../../../mach wpt /_mozilla/wasm
|
||||
|
||||
expectations:
|
||||
@[ -z $(MOZCONFIG) ] && echo "You need to define the MOZCONFIG env variable first." || true
|
||||
@[ -z $(MOZCONFIG) ] || ../../../../../mach wpt /_mozilla/wasm --log-raw /tmp/expectations.log || true
|
||||
@[ -z $(MOZCONFIG) ] || ../../../../../mach wpt-update /tmp/expectations.log --no-patch
|
||||
113
js/src/jit-test/etc/wasm/README.md
Normal file
113
js/src/jit-test/etc/wasm/README.md
Normal file
@ -0,0 +1,113 @@
|
||||
# Wasm Spec Tests
|
||||
|
||||
This directory contains scripts and configs to manage the in-tree import of the
|
||||
wasm spec testsuite.
|
||||
|
||||
## Format of tests
|
||||
|
||||
Spec tests are given in `test/core` of the `spec` repository as `.wast` files.
|
||||
A `.wast` file is a superset of the `.wat` format with commands for running
|
||||
tests.
|
||||
|
||||
The spec interpreter can natively consume `.wast` files to run the tests, or
|
||||
generate `.js` files which rely on the WebAssembly JS-API plus a harness file
|
||||
to implement unit-test functionality.
|
||||
|
||||
We rely on the spec interpreter to generate `.js` files for us to run.
|
||||
|
||||
## Running tests
|
||||
|
||||
Tests are imported to `jit-test` and `wpt` to be run in both the JS shell or the
|
||||
browser.
|
||||
|
||||
To run under `jit-test`:
|
||||
```bash
|
||||
cd js/src
|
||||
./jit-test.py path_to_build/dist/bin/js wasm/spec/
|
||||
```
|
||||
|
||||
To run under `wpt` (generally not necessary):
|
||||
```bash
|
||||
./mach web-platform-test testing/web-platform/mozilla/tests/wasm/
|
||||
```
|
||||
|
||||
## Test importing
|
||||
|
||||
There are many proposals in addition to the canonical spec. Each proposal is a
|
||||
fork of the canonical spec and may modify any amount of files.
|
||||
|
||||
This causes a challenge for any engine implementing multiple proposals, as any
|
||||
proposal may conflict (and often will) with each other. This makes it
|
||||
infeasible to merge all spec and proposal repos together.
|
||||
|
||||
Testing each proposal separately in full isn't an attractive option either, as
|
||||
most tests are unchanged and that would cause a significant amount of wasted
|
||||
computation time.
|
||||
|
||||
For this reason, we use a tool [1] to generate a set of separate test-suites
|
||||
that are 'pruned' to obtain a minimal set of tests. The tool works by merging
|
||||
each proposal with the proposal it is based off of and removing tests that
|
||||
have not changed.
|
||||
|
||||
[1] https://github.com/eqrion/wasm-generate-testsuite
|
||||
|
||||
### Configuration
|
||||
|
||||
The import tool relies on `config.toml` for the list of proposals to import,
|
||||
and `config-lock.toml` for a list of commits to use for each proposal.
|
||||
|
||||
The lock file makes test importing deterministic and controllable. This is
|
||||
useful as proposals often make inconvenient and breaking changes.
|
||||
|
||||
### Operation
|
||||
|
||||
```bash
|
||||
# Add, remove, or modify proposals
|
||||
vim config.toml
|
||||
# Remove locks for any proposals you wish to pull the latest changes on
|
||||
vim config-lock.toml
|
||||
# Import the tests
|
||||
make update
|
||||
# View the tests that were imported
|
||||
hg stat
|
||||
# Run the imported tests and note failures
|
||||
./jit-test.py dist/bin/js wasm/spec/
|
||||
# Exclude test failures
|
||||
vim config.toml
|
||||
# Re-import the tests to exclude failing tests
|
||||
make update
|
||||
# Commit the changes
|
||||
hg commit
|
||||
```
|
||||
|
||||
### Debugging test failures
|
||||
|
||||
Imported tests use the binary format, which is inconvenient for understanding
|
||||
why a test is failing.
|
||||
|
||||
Luckily, each assertion in an imported test contains the line of the source file
|
||||
that it corresponds with.
|
||||
|
||||
Unfortunately, the '.wast' files are not commited in the tree and so you must
|
||||
use the import tool to get the original source.
|
||||
|
||||
Follow the steps in 'Operation' to get a `wasm-generate-testsuite` repo with
|
||||
the '.wast' files of each proposal. All proposals are stored in a single git
|
||||
repo named `specs/`. Each proposal is a branch, and you can find all tests
|
||||
under `test/core`.
|
||||
|
||||
### Debugging import failures
|
||||
|
||||
Proposals can often have conflicts with their upstream proposals. This is okay,
|
||||
and the test importer will fallback to building tests on an unmerged tree.
|
||||
|
||||
This will likely result in extra tests being imported due to spurious
|
||||
differences between the proposal and upstream, but generally is okay.
|
||||
|
||||
It's also possible that a proposal is 'broken' and fails to generate '.js' test
|
||||
files with the spec interpreter. The best way to debug this is to check out the
|
||||
proposal repo and debug why `./test/build.py` is failing.
|
||||
|
||||
The import tool uses `RUST_LOG` to output debug information. `Makefile`
|
||||
automatically uses `RUST_LOG=info`. Change that to `RUST_LOG=debug` to get
|
||||
verbose output of all the commands run.
|
||||
5
js/src/jit-test/tests/wasm/spec/README.md
Normal file
5
js/src/jit-test/tests/wasm/spec/README.md
Normal file
@ -0,0 +1,5 @@
|
||||
# Wasm Spec Tests
|
||||
|
||||
These tests are autogenerated using a tool, do not edit.
|
||||
|
||||
See `jit-test/etc/wasm/` for more information.
|
||||
@ -1,49 +0,0 @@
|
||||
.PHONY: help update run expectations
|
||||
|
||||
help:
|
||||
@echo "Script to regenerate wasm test cases (JS and WPT) using the wasm-generate-testsuite tool."
|
||||
@echo ""
|
||||
@echo "- 'make update' runs wasm-generate-testsuite and puts the results in"
|
||||
@echo " the right directories, then updates the WPT manifest, if needed."
|
||||
@echo ""
|
||||
@echo " The wasm-generate-testsuite repo is needed under js/src/wasm (not"
|
||||
@echo " checked in). It can be a symbolic link or a real directory; if it's"
|
||||
@echo " not there, the Makefile will clone the repo from the sources."
|
||||
@echo ""
|
||||
@echo " Generation of a testsuite is driven by js/src/jit-test/etc/wasm-config.toml"
|
||||
@echo " If you need to exclude a test, change test directives, or add a"
|
||||
@echo " proposal; that is the file to modify."
|
||||
@echo ""
|
||||
@echo " Generated testsuites are made reproducible using 'wasm-config-lock.toml'"
|
||||
@echo " which contains pinned commits from the last build. New testsuites will"
|
||||
@echo " remain on those commits by default. To update to the latest version for"
|
||||
@echo " a repo, delete it from the lock file and do a build."
|
||||
@echo ""
|
||||
@echo "- 'MOZCONFIG=/path/to/bin/firefox make run' runs the WPT test cases and prints a"
|
||||
@echo " summary of the failures in the console."
|
||||
@echo ""
|
||||
@echo "- 'MOZCONFIG=/path/to/bin/firefox make expectations' runs the WPT test cases and"
|
||||
@echo " updates the expectations (known failures)."
|
||||
@echo ""
|
||||
@echo "Choose a rule: update, run, or expectations."
|
||||
|
||||
update:
|
||||
[ -d ./wasm-generate-testsuite ] || git clone https://github.com/eqrion/wasm-generate-testsuite ./wasm-generate-testsuite
|
||||
cp ../jit-test/etc/wasm-config.toml ./wasm-generate-testsuite/config.toml
|
||||
cp ../jit-test/etc/wasm-config-lock.toml ./wasm-generate-testsuite/config-lock.toml
|
||||
(cd ./wasm-generate-testsuite && cargo run)
|
||||
cp ./wasm-generate-testsuite/config-lock.toml ../jit-test/etc/wasm-config-lock.toml
|
||||
rm -r ../jit-test/tests/wasm/spec
|
||||
cp -R wasm-generate-testsuite/tests/js ../jit-test/tests/wasm/spec
|
||||
[ ! -d ../jit-test/etc/wasm-spec-tests.patch ] || (cd ../jit-test/tests/wasm/spec && patch -u -p7 < ../../../etc/wasm-spec-tests.patch)
|
||||
rm -r ../../../testing/web-platform/mozilla/tests/wasm
|
||||
cp -R wasm-generate-testsuite/tests/wpt ../../../testing/web-platform/mozilla/tests/wasm
|
||||
|
||||
run:
|
||||
@[ -z $(MOZCONFIG) ] && echo "You need to define the MOZCONFIG env variable first."
|
||||
@[ -z $(MOZCONFIG) ] || ../../../mach wpt /_mozilla/wasm
|
||||
|
||||
expectations:
|
||||
@[ -z $(MOZCONFIG) ] && echo "You need to define the MOZCONFIG env variable first." || true
|
||||
@[ -z $(MOZCONFIG) ] || ../../../mach wpt /_mozilla/wasm --log-raw /tmp/expectations.log || true
|
||||
@[ -z $(MOZCONFIG) ] || ../../../mach wpt-update /tmp/expectations.log --no-patch
|
||||
5
testing/web-platform/mozilla/tests/wasm/README.md
Normal file
5
testing/web-platform/mozilla/tests/wasm/README.md
Normal file
@ -0,0 +1,5 @@
|
||||
# Wasm Spec Tests
|
||||
|
||||
These tests are autogenerated using a tool, do not edit.
|
||||
|
||||
See `jit-test/etc/wasm/` for more information.
|
||||
Loading…
x
Reference in New Issue
Block a user