From 1d0db0076b3f7c2afaae9a9fe8716d19f473d14a Mon Sep 17 00:00:00 2001 From: James Graham Date: Thu, 4 Sep 2014 12:52:43 +0100 Subject: [PATCH] Bug 945222 - Initial import of web-platform-tests testsuite 4/4 : Integration with build system, mach and mozharness, r=ahal,gps --HG-- extra : rebase_source : 0d704f5a3edcbcef56d15b76ef26aae1b7232c4d --- .gitignore | 3 + .hgignore | 3 + build/mach_bootstrap.py | 3 + .../mozharness/web_platform_tests_config.py | 11 + testing/profiles/Makefile.in | 15 +- testing/testsuite-targets.mk | 6 + testing/web-platform/Makefile.in | 33 +++ testing/web-platform/README.md | 193 ++++++++++++++++++ testing/web-platform/fetchlogs.py | 95 +++++++++ testing/web-platform/mach_commands.py | 133 ++++++++++++ testing/web-platform/moz.build | 6 + testing/web-platform/runtests.py | 14 ++ testing/web-platform/update.py | 17 ++ testing/web-platform/wptrunner.ini | 11 + toolkit/toolkit.mozbuild | 1 + 15 files changed, 534 insertions(+), 10 deletions(-) create mode 100644 testing/config/mozharness/web_platform_tests_config.py create mode 100644 testing/web-platform/Makefile.in create mode 100644 testing/web-platform/README.md create mode 100644 testing/web-platform/fetchlogs.py create mode 100644 testing/web-platform/mach_commands.py create mode 100644 testing/web-platform/moz.build create mode 100644 testing/web-platform/runtests.py create mode 100644 testing/web-platform/update.py create mode 100644 testing/web-platform/wptrunner.ini diff --git a/.gitignore b/.gitignore index a88d6f2ae5a9..acd30a47ed36 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,6 @@ GTAGS GRTAGS GSYMS GPATH + +# Git clone directory for updating web-platform-tests +testing/web-platform/sync/ diff --git a/.hgignore b/.hgignore index 37cda439bf49..1fed800c1155 100644 --- a/.hgignore +++ b/.hgignore @@ -75,3 +75,6 @@ GPATH # Unit tests for Loop ^browser/components/loop/standalone/content/config\.js$ ^browser/components/loop/standalone/node_modules/ + +# Git clone directory for updating web-platform-tests +^testing/web-platform/sync/ diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py index 9523c134a67c..6cafd195e824 100644 --- a/build/mach_bootstrap.py +++ b/build/mach_bootstrap.py @@ -43,6 +43,8 @@ SEARCH_PATHS = [ 'xpcom/idl-parser', 'testing', 'testing/xpcshell', + 'testing/web-platform', + 'testing/web-platform/harness', 'testing/marionette/client', 'testing/marionette/client/marionette', 'testing/marionette/transport', @@ -80,6 +82,7 @@ MACH_MODULES = [ 'testing/mochitest/mach_commands.py', 'testing/xpcshell/mach_commands.py', 'testing/talos/mach_commands.py', + 'testing/web-platform/mach_commands.py', 'testing/xpcshell/mach_commands.py', 'tools/docs/mach_commands.py', 'tools/mercurial/mach_commands.py', diff --git a/testing/config/mozharness/web_platform_tests_config.py b/testing/config/mozharness/web_platform_tests_config.py new file mode 100644 index 000000000000..eea9a667a3be --- /dev/null +++ b/testing/config/mozharness/web_platform_tests_config.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +config = { + "options": [ + "--prefs-root=%(test_path)s/prefs", + "--processes=1", + "--config=%(test_path)s/wptrunner.ini", + ], +} diff --git a/testing/profiles/Makefile.in b/testing/profiles/Makefile.in index d42d0f9e907e..8d337b0d0a95 100644 --- a/testing/profiles/Makefile.in +++ b/testing/profiles/Makefile.in @@ -1,20 +1,15 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - -include $(topsrcdir)/config/rules.mk - MOCHITEST_PROFILE_FILES = \ prefs_general.js \ prefs_b2g_unittest.js \ webapps_mochitest.json \ $(NULL) +WPT_PROFILE_FILES = $(MOCHITEST_PROFILE_FILES) -_DEST_DIR = $(DEPTH)/_tests/testing/mochitest/profile_data -libs:: $(MOCHITEST_PROFILE_FILES) - $(PYTHON) $(topsrcdir)/config/nsinstall.py $^ $(_DEST_DIR) +MOCHITEST_PROFILE_DEST = $(DEPTH)/_tests/testing/mochitest/profile_data +WPT_PROFILE_DEST = $(DEPTH)/_tests/web-platform/prefs -stage-package: PKG_STAGE = $(DIST)/test-stage -stage-package: - $(NSINSTALL) -D $(PKG_STAGE)/ - @(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(MOCHITEST_PROFILE_FILES)) | (cd $(PKG_STAGE)/mochitest/profile_data && tar -xf -) +INSTALL_TARGETS += MOCHITEST_PROFILE +INSTALL_TARGETS += WPT_PROFILE diff --git a/testing/testsuite-targets.mk b/testing/testsuite-targets.mk index ab7ec4ac69e0..41dc906d6362 100644 --- a/testing/testsuite-targets.mk +++ b/testing/testsuite-targets.mk @@ -403,6 +403,7 @@ package-tests: \ stage-cppunittests \ stage-jittest \ stage-steeplechase \ + stage-web-platform-tests \ $(NULL) else # This staging area has been built for us by universal/flight.mk @@ -547,6 +548,10 @@ stage-marionette: make-stage-dir stage-mozbase: make-stage-dir $(MAKE) -C $(DEPTH)/testing/mozbase stage-package + +stage-web-platform-tests: make-stage-dir + $(MAKE) -C $(DEPTH)/testing/web-platform stage-package + .PHONY: \ mochitest \ mochitest-plain \ @@ -573,5 +578,6 @@ stage-mozbase: make-stage-dir stage-modules \ stage-marionette \ stage-steeplechase \ + stage-web-platform-tests \ $(NULL) diff --git a/testing/web-platform/Makefile.in b/testing/web-platform/Makefile.in new file mode 100644 index 000000000000..30769e7cce30 --- /dev/null +++ b/testing/web-platform/Makefile.in @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +WPT_TESTS = \ + tests \ + $(NULL) + +WPT_METADATA = \ + meta \ + $(NULL) + + +_DEST_DIR = $(DEPTH)/_tests/web-platform +PKG_STAGE = $(DIST)/test-stage + +WEBPLATFORM_FILES = \ + runtests.py \ + wptrunner.ini \ + $(NULL) +WEBPLATFORM_DEST = $(_DEST_DIR) +INSTALL_TARGETS += WEBPLATFORM + +libs:: + $(INSTALL) $(topsrcdir)/testing/web-platform/harness/wptrunner $(_DEST_DIR) +libs:: $(WPT_TESTS) + $(INSTALL) $(foreach f,$^,"$f") $(_DEST_DIR) +libs:: $(WPT_METADATA) + $(INSTALL) $(foreach f,$^,"$f") $(_DEST_DIR) + +stage-package: + $(NSINSTALL) -D $(PKG_STAGE)/web-platform + @cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/web-platform + @(cd $(DEPTH)/_tests/ && tar $(TAR_CREATE_FLAGS) - web-platform) | (cd $(PKG_STAGE) && tar -xf -) diff --git a/testing/web-platform/README.md b/testing/web-platform/README.md new file mode 100644 index 000000000000..d8da42193bd3 --- /dev/null +++ b/testing/web-platform/README.md @@ -0,0 +1,193 @@ +web-platform-tests +================== + +This directory contains the W3C +[web-platform-tests](http://github.com/w3c/web-platform-tests). They +can be run using `mach`: + + mach web-platform-tests + +To limit the testrun to certain directories use the `--include` option; +for example: + + mach web-platform-tests --include=dom + +The testsuite contains a mix of javascript tests and reftests. To +limit the type of tests that get run, use `--test-type=testharness` for +javascript tests or `--test-type=reftest` for reftests. + +FAQ +--- + +* I fixed a bug and some tests have started to pass. How do I fix the + UNEXPECTED-PASS messages when web-platform-tests is run? + + You need to update the expectation data for those tests. See the + section on expectations below. + +* I want to write some new tests for the web-platform-tests + testsuite. How do I do that? + + See the section on tests below. At the moment you will have to + submit the tests directly to the W3C and wait for them to be + imported into the Mozilla tree. + +* A test is unstable; how do I disable it? + + See the section on disabling tests. + +Directories +----------- + +`tests/` contains the tests themselves. This is a copy of a certain +revision of web-platform-tests. The contents of this directory must +not be modified locally as any modifications will be overwritten +whenever a new upstream sync is performed. + +`harness/` contains the [wptrunner](http://github.com/w3c/wptrunner) +test runner. Again the contents of this directory will be overwritten +on update. + +`meta/` contains Gecko-specific expectation data. This is explained in +the following section. + +Expectation Data +---------------- + +With the tests coming from upstream, it is not guaranteed that they +all pass in Gecko-based browsers. For this reason it is necessary to +provide metadata about the expected results of each test. This is +provided in a set of manifest files in the `meta/` subdirectories. + +There is one manifest file per test with "non-default" +expectations. By default tests are expected to PASS, and tests with +subtests are expected to have an overall status of OK. The manifest +file of a test has the same path as the test file but under the `meta` +directory rather than the `tests` directory and has the suffix `.ini`. + +The format of these files is similar to `ini` files, but with a couple +of important differences; sections can be nested using indentation, +and only `:` is permitted as a key-value separator. For example the +expectation file for a test with one failing subtest and one erroring +subtest might look like: + + [filename.html] + type: testharness + + [Subtest name for failing test] + expected: FAIL + + [Subtest name for erroring test] + expected: ERROR + +Expectations can also be made platform-specific using a simple +python-like conditional syntax e.g. for a test that times out on linux +but otherwise fails: + + [filename.html] + type: reftest + expected: + if os == "linux": TIMEOUT + FAIL + +The available variables for the conditions are those provided by +[mozinfo](http://mozbase.readthedocs.org/en/latest/mozinfo.html). + +For more information on manifest files, see the +[wptrunner documentation](http://wptrunner.readthedocs.org/en/latest/expectation.html). + +Autogenerating Expectation Data +------------------------------- + +After changing some code it may be necessary to update the expectation +data for the relevant tests. This can of course be done manually, but +tools are available to automate much of the process. + +First one must run the tests that have changed status, and save the +raw log output to a file: + + mach web-platform-tests --include=url/of/test.html --log-raw=new_results.log + +Then the `web-platform-tests-update` command may be run using this log +data to update the expectation files: + + mach web-platform-tests-update --no-check-clean new_results.log + +By default this only updates the results data for the current +platform. To forcibly overwrite all existing result data, use the +`--ignore-existing` option to the update command. + +Disabling Tests +--------------- + +Tests are disabled using the same manifest files used to set +expectation values. For example, if a test is unstable on Windows, it +can be disabled using an ini file with the contents: + + [filename.html] + type: testharness + disabled: + if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1234567 + +Test Format +----------- + +Javascript tests are written using +[testharness.js](http://github.com/w3c/testharness.js/). Reftests are +similar to standard Gecko reftests without an explicit manifest file, +but with in-test or filename conventions for identifying the +reference. + +New tests must presently be submitted upstream before they can be run +on Mozilla infrastructure. This situation is expected to be temporary. + +Full documentation on test authoring and submission can be found on +[testthewebforward.org](http://testthewebforward.org/docs). + +Running Tests In Other Browsers +------------------------------- + +web-platform-tests is cross browser, and the runner is compatible with +multiple browsers. Therefore it's possible to check the behaviour of +tests in other browsers. This is somewhat more involved than running +them in Firefox since extra dependencies may be required. For example +to test in Chrome: + +1. Download the chromedriver binary and place it somewhere sensible + e.g. `~/bin` + +2. In your gecko source tree activate the virtualenv created by mach, + since this has most dependencies already installed. This is typically + in objdir/_virtualenv and is activated via e.g. + + source objdir/_virtualenv/bin/activate + +3. Install the extra requirements: + + cd testing/web-platform/harness + pip install -r requirements_chrome.txt + +4. Edit the config file `testing/web-platform/wptrunner.ini` so that + Chrome support is enabled by changing the section that reads: + + [products] + firefox = + + to read + + [products] + firefox = + chrome = + + (alternatively create a new config file elsewhere and use the + `--config` option to `runtests.py` to point wptrunner at this config + file). + +5. Run `runtests.py` using the location of chromedriver as + the binary: + + cd testing/web-platform + python runtests.py --product=chrome --binary=~/bin/chromedriver --log-mach=- + +By default this will use the same test checkout and metadata as are in +the Gecko tree, so it's easy to compare behaviour relative to Firefox. diff --git a/testing/web-platform/fetchlogs.py b/testing/web-platform/fetchlogs.py new file mode 100644 index 000000000000..b7cb025b5790 --- /dev/null +++ b/testing/web-platform/fetchlogs.py @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import cStringIO +import gzip +import json +import os +import requests +import urlparse + +treeherder_base = "https://treeherder.mozilla.org/" + +"""Simple script for downloading structured logs from treeherder. + +For the moment this is specialised to work with web-platform-tests +logs; in due course it should move somewhere generic and get hooked +up to mach or similar""" + +# Interpretation of the "job" list from +# https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18 + +def create_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("branch", action="store", + help="Branch on which jobs ran") + parser.add_argument("commit", + action="store", + help="Commit hash for push") + + return parser + +def download(url, prefix, dest, force_suffix=True): + if dest is None: + dest = "." + + if prefix and not force_suffix: + name = os.path.join(dest, prefix + ".log") + else: + name = None + counter = 0 + + while not name or os.path.exists(name): + counter += 1 + sep = "" if not prefix else "-" + name = os.path.join(dest, prefix + sep + str(counter) + ".log") + + with open(name, "wb") as f: + resp = requests.get(url) + f.write(resp.text.encode(resp.encoding)) + +def get_blobber_url(branch, job): + job_id = job[8] + resp = requests.get(urlparse.urljoin(treeherder_base, + "/api/project/%s/artifact/?job_id=%i&name=Job%%20Info" % (branch, + job_id))) + job_data = resp.json() + print job_data + if job_data: + assert len(job_data) == 1 + job_data = job_data[0] + try: + details = job_data["blob"]["job_details"] + for item in details: + if item["value"] == "wpt_structured_full.log": + return item["url"] + except: + return None + + +def get_structured_logs(branch, commit, dest=None): + resp = requests.get(urlparse.urljoin(treeherder_base, "/api/project/%s/resultset/?revision=%s" % (branch, + commit))) + job_data = resp.json() + + for result in job_data["results"]: + for platform in result["platforms"]: + for group in platform["groups"]: + for job in group["jobs"]: + job_type_name = job[13] + if job_type_name.startswith("W3C Web Platform") or job_type_name == "unknown": + url = get_blobber_url(branch, job) + if url: + prefix = job[14] # platform + download(url, prefix, None) + +def main(): + parser = create_parser() + args = parser.parse_args() + + get_structured_logs(args.branch, args.commit) + +if __name__ == "__main__": + main() diff --git a/testing/web-platform/mach_commands.py b/testing/web-platform/mach_commands.py new file mode 100644 index 000000000000..14c39003c4ee --- /dev/null +++ b/testing/web-platform/mach_commands.py @@ -0,0 +1,133 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Integrates the web-platform-tests test runner with mach. + +from __future__ import unicode_literals, print_function + +import os +import sys + +from mozbuild.base import ( + MachCommandBase, + MachCommandConditions as conditions, + MozbuildObject, +) + +from mach.decorators import ( + CommandProvider, + Command, +) + +from wptrunner import wptcommandline + +# This should probably be consolidated with similar classes in other test +# runners. +class InvalidTestPathError(Exception): + """Exception raised when the test path is not valid.""" + +class WebPlatformTestsRunner(MozbuildObject): + """Run web platform tests.""" + + def setup_kwargs(self, kwargs): + build_path = os.path.join(self.topobjdir, 'build') + if build_path not in sys.path: + sys.path.append(build_path) + + if kwargs["config"] is None: + kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini') + + if kwargs["binary"] is None: + kwargs["binary"] = os.path.join(self.get_binary_path('app')) + + if kwargs["prefs_root"] is None: + kwargs["prefs_root"] = os.path.join(self.topobjdir, '_tests', 'web-platform', "prefs") + + kwargs["capture_stdio"] = True + + kwargs = wptcommandline.check_args(kwargs) + + def run_tests(self, **kwargs): + from wptrunner import wptrunner + + self.setup_kwargs(kwargs) + + logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) + result = wptrunner.run_tests(**kwargs) + + return int(not result) + + def list_test_groups(self, **kwargs): + from wptrunner import wptrunner + + self.setup_kwargs(kwargs) + + wptrunner.list_test_groups(**kwargs) + +class WebPlatformTestsUpdater(MozbuildObject): + """Update web platform tests.""" + def run_update(self, **kwargs): + from wptrunner import update + + if kwargs["config"] is None: + kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini') + + wptcommandline.set_from_config(kwargs) + + update.run_update(**kwargs) + +class WebPlatformTestsReduce(WebPlatformTestsRunner): + + def run_reduce(self, **kwargs): + from wptrunner import reduce + + self.setup_kwargs(kwargs) + + kwargs["capture_stdio"] = True + logger = reduce.setup_logging(kwargs, {"mach": sys.stdout}) + tests = reduce.do_reduce(**kwargs) + + if not tests: + logger.warning("Test was not unstable") + + for item in tests: + logger.info(item.id) + +@CommandProvider +class MachCommands(MachCommandBase): + @Command("web-platform-tests", + category="testing", + conditions=[conditions.is_firefox], + parser=wptcommandline.create_parser(["firefox"])) + def run_web_platform_tests(self, **params): + self.setup() + wpt_runner = self._spawn(WebPlatformTestsRunner) + + if params["list_test_groups"]: + return wpt_runner.list_test_groups(**params) + else: + return wpt_runner.run_tests(**params) + + @Command("web-platform-tests-update", + category="testing", + conditions=[conditions.is_firefox], + parser=wptcommandline.create_parser_update()) + def update_web_platform_tests(self, **params): + self.setup() + self.virtualenv_manager.install_pip_package('html5lib==0.99') + wpt_updater = self._spawn(WebPlatformTestsUpdater) + return wpt_updater.run_update(**params) + + def setup(self): + self._activate_virtualenv() + self.virtualenv_manager.install_pip_package('py==1.4.14') + + @Command("web-platform-tests-reduce", + category="testing", + conditions=[conditions.is_firefox], + parser=wptcommandline.create_parser_reduce(["firefox"])) + def unstable_web_platform_tests(self, **params): + self.setup() + wpt_reduce = self._spawn(WebPlatformTestsReduce) + return wpt_reduce.run_reduce(**params) diff --git a/testing/web-platform/moz.build b/testing/web-platform/moz.build new file mode 100644 index 000000000000..895d11993cfb --- /dev/null +++ b/testing/web-platform/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + diff --git a/testing/web-platform/runtests.py b/testing/web-platform/runtests.py new file mode 100644 index 000000000000..ca0d51329386 --- /dev/null +++ b/testing/web-platform/runtests.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys + +from wptrunner import wptrunner + +if __name__ == "__main__": + success = wptrunner.main() + if not success: + sys.exit(1) diff --git a/testing/web-platform/update.py b/testing/web-platform/update.py new file mode 100644 index 000000000000..961f80fa61ea --- /dev/null +++ b/testing/web-platform/update.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import os + +here = os.path.dirname(__file__) +os.path.abspath(sys.path.insert(0, os.path.join(here, "harness"))) + +from wptrunner import update + +if __name__ == "__main__": + success = update.main() + sys.exit(0 if success else 1) diff --git a/testing/web-platform/wptrunner.ini b/testing/web-platform/wptrunner.ini new file mode 100644 index 000000000000..193e050ee2b3 --- /dev/null +++ b/testing/web-platform/wptrunner.ini @@ -0,0 +1,11 @@ +[products] +firefox = + +[web-platform-tests] +remote_url = https://github.com/w3c/web-platform-tests.git +branch = master +sync_path = sync + +[paths] +tests = tests +metadata = meta diff --git a/toolkit/toolkit.mozbuild b/toolkit/toolkit.mozbuild index 560ebafd3fb0..976b1b7c735a 100644 --- a/toolkit/toolkit.mozbuild +++ b/toolkit/toolkit.mozbuild @@ -182,6 +182,7 @@ if CONFIG['ENABLE_TESTS']: 'testing/profiles', 'testing/mozbase', 'testing/modules', + 'testing/web-platform', ]) if CONFIG['MOZ_WEBRTC'] and CONFIG['MOZ_WIDGET_TOOLKIT'] != 'gonk':