Bug 945222 - Initial import of web-platform-tests testsuite 4/4 : Integration with build system, mach and mozharness, r=ahal,gps

--HG--
extra : rebase_source : 0d704f5a3edcbcef56d15b76ef26aae1b7232c4d

.gitignore

@@ -61,3 +61,6 @@ GTAGS
 GRTAGS
 GSYMS
 GPATH
+
+# Git clone directory for updating web-platform-tests
+testing/web-platform/sync/

.hgignore

@@ -75,3 +75,6 @@ GPATH
 # Unit tests for Loop
 ^browser/components/loop/standalone/content/config\.js$
 ^browser/components/loop/standalone/node_modules/
+
+# Git clone directory for updating web-platform-tests
+^testing/web-platform/sync/

build/mach_bootstrap.py

@@ -43,6 +43,8 @@ SEARCH_PATHS = [
     'xpcom/idl-parser',
     'testing',
     'testing/xpcshell',
+    'testing/web-platform',
+    'testing/web-platform/harness',
     'testing/marionette/client',
     'testing/marionette/client/marionette',
     'testing/marionette/transport',
@@ -80,6 +82,7 @@ MACH_MODULES = [
     'testing/mochitest/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'testing/talos/mach_commands.py',
+    'testing/web-platform/mach_commands.py',
     'testing/xpcshell/mach_commands.py',
     'tools/docs/mach_commands.py',
     'tools/mercurial/mach_commands.py',

testing/config/mozharness/web_platform_tests_config.py

@@ -0,0 +1,11 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+config = {
+    "options": [
+        "--prefs-root=%(test_path)s/prefs",
+        "--processes=1",
+        "--config=%(test_path)s/wptrunner.ini",
+    ],
+}

testing/profiles/Makefile.in

@@ -1,20 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-include $(topsrcdir)/config/rules.mk
-
 MOCHITEST_PROFILE_FILES = \
   prefs_general.js \
   prefs_b2g_unittest.js \
   webapps_mochitest.json \
   $(NULL)
+WPT_PROFILE_FILES = $(MOCHITEST_PROFILE_FILES)
 
-_DEST_DIR = $(DEPTH)/_tests/testing/mochitest/profile_data
-libs:: $(MOCHITEST_PROFILE_FILES)
-	$(PYTHON) $(topsrcdir)/config/nsinstall.py $^ $(_DEST_DIR)
+MOCHITEST_PROFILE_DEST = $(DEPTH)/_tests/testing/mochitest/profile_data
+WPT_PROFILE_DEST = $(DEPTH)/_tests/web-platform/prefs
 
-stage-package: PKG_STAGE = $(DIST)/test-stage
-stage-package:
-	$(NSINSTALL) -D $(PKG_STAGE)/
-	@(cd $(srcdir) && tar $(TAR_CREATE_FLAGS) - $(MOCHITEST_PROFILE_FILES)) | (cd $(PKG_STAGE)/mochitest/profile_data && tar -xf -)
+INSTALL_TARGETS += MOCHITEST_PROFILE
+INSTALL_TARGETS += WPT_PROFILE

testing/testsuite-targets.mk

@@ -403,6 +403,7 @@ package-tests: \
   stage-cppunittests \
   stage-jittest \
   stage-steeplechase \
+  stage-web-platform-tests \
   $(NULL)
 else
 # This staging area has been built for us by universal/flight.mk
@@ -547,6 +548,10 @@ stage-marionette: make-stage-dir
 
 stage-mozbase: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mozbase stage-package
+
+stage-web-platform-tests: make-stage-dir
+	$(MAKE) -C $(DEPTH)/testing/web-platform stage-package
+
 .PHONY: \
   mochitest \
   mochitest-plain \
@@ -573,5 +578,6 @@ stage-mozbase: make-stage-dir
   stage-modules \
   stage-marionette \
   stage-steeplechase \
+  stage-web-platform-tests \
   $(NULL)
 

testing/web-platform/Makefile.in

@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+WPT_TESTS = \
+  tests \
+  $(NULL)
+
+WPT_METADATA = \
+  meta \
+  $(NULL)
+
+
+_DEST_DIR = $(DEPTH)/_tests/web-platform
+PKG_STAGE = $(DIST)/test-stage
+
+WEBPLATFORM_FILES = \
+  runtests.py \
+  wptrunner.ini \
+  $(NULL)
+WEBPLATFORM_DEST = $(_DEST_DIR)
+INSTALL_TARGETS += WEBPLATFORM
+
+libs::
+	$(INSTALL) $(topsrcdir)/testing/web-platform/harness/wptrunner $(_DEST_DIR)
+libs:: $(WPT_TESTS)
+	$(INSTALL) $(foreach f,$^,"$f") $(_DEST_DIR)
+libs:: $(WPT_METADATA)
+	$(INSTALL) $(foreach f,$^,"$f") $(_DEST_DIR)
+
+stage-package:
+	$(NSINSTALL) -D $(PKG_STAGE)/web-platform
+	@cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/web-platform
+	@(cd $(DEPTH)/_tests/ && tar $(TAR_CREATE_FLAGS) - web-platform) | (cd $(PKG_STAGE) && tar -xf -)

testing/web-platform/README.md

@@ -0,0 +1,193 @@
+web-platform-tests
+==================
+
+This directory contains the W3C
+[web-platform-tests](http://github.com/w3c/web-platform-tests). They
+can be run using `mach`:
+
+    mach web-platform-tests
+
+To limit the testrun to certain directories use the `--include` option;
+for example:
+
+    mach web-platform-tests --include=dom
+
+The testsuite contains a mix of javascript tests and reftests. To
+limit the type of tests that get run, use `--test-type=testharness` for
+javascript tests or `--test-type=reftest` for reftests.
+
+FAQ
+---
+
+* I fixed a bug and some tests have started to pass. How do I fix the
+  UNEXPECTED-PASS messages when web-platform-tests is run?
+
+  You need to update the expectation data for those tests. See the
+  section on expectations below.
+
+* I want to write some new tests for the web-platform-tests
+  testsuite. How do I do that?
+
+  See the section on tests below. At the moment you will have to
+  submit the tests directly to the W3C and wait for them to be
+  imported into the Mozilla tree.
+
+* A test is unstable; how do I disable it?
+
+  See the section on disabling tests.
+
+Directories
+-----------
+
+`tests/` contains the tests themselves. This is a copy of a certain
+revision of web-platform-tests. The contents of this directory must
+not be modified locally as any modifications will be overwritten
+whenever a new upstream sync is performed.
+
+`harness/` contains the [wptrunner](http://github.com/w3c/wptrunner)
+test runner. Again the contents of this directory will be overwritten
+on update.
+
+`meta/` contains Gecko-specific expectation data. This is explained in
+the following section.
+
+Expectation Data
+----------------
+
+With the tests coming from upstream, it is not guaranteed that they
+all pass in Gecko-based browsers. For this reason it is necessary to
+provide metadata about the expected results of each test. This is
+provided in a set of manifest files in the `meta/` subdirectories.
+
+There is one manifest file per test with "non-default"
+expectations. By default tests are expected to PASS, and tests with
+subtests are expected to have an overall status of OK. The manifest
+file of a test has the same path as the test file but under the `meta`
+directory rather than the `tests` directory and has the suffix `.ini`.
+
+The format of these files is similar to `ini` files, but with a couple
+of important differences; sections can be nested using indentation,
+and only `:` is permitted as a key-value separator. For example the
+expectation file for a test with one failing subtest and one erroring
+subtest might look like:
+
+    [filename.html]
+        type: testharness
+
+        [Subtest name for failing test]
+            expected: FAIL
+
+        [Subtest name for erroring test]
+            expected: ERROR
+
+Expectations can also be made platform-specific using a simple
+python-like conditional syntax e.g. for a test that times out on linux
+but otherwise fails:
+
+    [filename.html]
+        type: reftest
+        expected:
+            if os == "linux": TIMEOUT
+            FAIL
+
+The available variables for the conditions are those provided by
+[mozinfo](http://mozbase.readthedocs.org/en/latest/mozinfo.html).
+
+For more information on manifest files, see the
+[wptrunner documentation](http://wptrunner.readthedocs.org/en/latest/expectation.html).
+
+Autogenerating Expectation Data
+-------------------------------
+
+After changing some code it may be necessary to update the expectation
+data for the relevant tests. This can of course be done manually, but
+tools are available to automate much of the process.
+
+First one must run the tests that have changed status, and save the
+raw log output to a file:
+
+    mach web-platform-tests --include=url/of/test.html --log-raw=new_results.log
+
+Then the `web-platform-tests-update` command may be run using this log
+data to update the expectation files:
+
+    mach web-platform-tests-update --no-check-clean new_results.log
+
+By default this only updates the results data for the current
+platform. To forcibly overwrite all existing result data, use the
+`--ignore-existing` option to the update command.
+
+Disabling Tests
+---------------
+
+Tests are disabled using the same manifest files used to set
+expectation values. For example, if a test is unstable on Windows, it
+can be disabled using an ini file with the contents:
+
+    [filename.html]
+        type: testharness
+        disabled:
+            if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1234567
+
+Test Format
+-----------
+
+Javascript tests are written using
+[testharness.js](http://github.com/w3c/testharness.js/). Reftests are
+similar to standard Gecko reftests without an explicit manifest file,
+but with in-test or filename conventions for identifying the
+reference.
+
+New tests must presently be submitted upstream before they can be run
+on Mozilla infrastructure. This situation is expected to be temporary.
+
+Full documentation on test authoring and submission can be found on
+[testthewebforward.org](http://testthewebforward.org/docs).
+
+Running Tests In Other Browsers
+-------------------------------
+
+web-platform-tests is cross browser, and the runner is compatible with
+multiple browsers. Therefore it's possible to check the behaviour of
+tests in other browsers. This is somewhat more involved than running
+them in Firefox since extra dependencies may be required. For example
+to test in Chrome:
+
+1. Download the chromedriver binary and place it somewhere sensible
+   e.g. `~/bin`
+
+2. In your gecko source tree activate the virtualenv created by mach,
+   since this has most dependencies already installed. This is typically
+   in objdir/_virtualenv and is activated via e.g.
+
+        source objdir/_virtualenv/bin/activate
+
+3. Install the extra requirements:
+
+        cd testing/web-platform/harness
+        pip install -r requirements_chrome.txt
+
+4. Edit the config file `testing/web-platform/wptrunner.ini` so that
+   Chrome support is enabled by changing the section that reads:
+
+        [products]
+        firefox =
+
+   to read
+
+        [products]
+        firefox =
+        chrome =
+
+   (alternatively create a new config file elsewhere and use the
+   `--config` option to `runtests.py` to point wptrunner at this config
+   file).
+
+5. Run `runtests.py` using the location of chromedriver as
+   the binary:
+
+        cd testing/web-platform
+        python runtests.py --product=chrome --binary=~/bin/chromedriver --log-mach=-
+
+By default this will use the same test checkout and metadata as are in
+the Gecko tree, so it's easy to compare behaviour relative to Firefox.

testing/web-platform/fetchlogs.py

@@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import cStringIO
+import gzip
+import json
+import os
+import requests
+import urlparse
+
+treeherder_base = "https://treeherder.mozilla.org/"
+
+"""Simple script for downloading structured logs from treeherder.
+
+For the moment this is specialised to work with web-platform-tests
+logs; in due course it should move somewhere generic and get hooked
+up to mach or similar"""
+
+# Interpretation of the "job" list from
+# https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18
+
+def create_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("branch", action="store",
+                        help="Branch on which jobs ran")
+    parser.add_argument("commit",
+                        action="store",
+                        help="Commit hash for push")
+
+    return parser
+
+def download(url, prefix, dest, force_suffix=True):
+    if dest is None:
+        dest = "."
+
+    if prefix and not force_suffix:
+        name = os.path.join(dest, prefix + ".log")
+    else:
+        name = None
+    counter = 0
+
+    while not name or os.path.exists(name):
+        counter += 1
+        sep = "" if not prefix else "-"
+        name = os.path.join(dest, prefix + sep + str(counter) + ".log")
+
+    with open(name, "wb") as f:
+        resp = requests.get(url)
+        f.write(resp.text.encode(resp.encoding))
+
+def get_blobber_url(branch, job):
+    job_id = job[8]
+    resp = requests.get(urlparse.urljoin(treeherder_base,
+                                         "/api/project/%s/artifact/?job_id=%i&name=Job%%20Info" % (branch,
+                                                                                                   job_id)))
+    job_data = resp.json()
+    print job_data
+    if job_data:
+        assert len(job_data) == 1
+        job_data = job_data[0]
+        try:
+            details = job_data["blob"]["job_details"]
+            for item in details:
+                if item["value"] == "wpt_structured_full.log":
+                    return item["url"]
+        except:
+            return None
+
+
+def get_structured_logs(branch, commit, dest=None):
+    resp = requests.get(urlparse.urljoin(treeherder_base, "/api/project/%s/resultset/?revision=%s" % (branch,
+                                                                                                      commit)))
+    job_data = resp.json()
+
+    for result in job_data["results"]:
+        for platform in result["platforms"]:
+            for group in platform["groups"]:
+                for job in group["jobs"]:
+                    job_type_name = job[13]
+                    if job_type_name.startswith("W3C Web Platform") or job_type_name == "unknown":
+                        url = get_blobber_url(branch, job)
+                        if url:
+                            prefix = job[14] # platform
+                            download(url, prefix, None)
+
+def main():
+    parser = create_parser()
+    args = parser.parse_args()
+
+    get_structured_logs(args.branch, args.commit)
+
+if __name__ == "__main__":
+    main()

testing/web-platform/mach_commands.py

@@ -0,0 +1,133 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Integrates the web-platform-tests test runner with mach.
+
+from __future__ import unicode_literals, print_function
+
+import os
+import sys
+
+from mozbuild.base import (
+    MachCommandBase,
+    MachCommandConditions as conditions,
+    MozbuildObject,
+)
+
+from mach.decorators import (
+    CommandProvider,
+    Command,
+)
+
+from wptrunner import wptcommandline
+
+# This should probably be consolidated with similar classes in other test
+# runners.
+class InvalidTestPathError(Exception):
+    """Exception raised when the test path is not valid."""
+
+class WebPlatformTestsRunner(MozbuildObject):
+    """Run web platform tests."""
+
+    def setup_kwargs(self, kwargs):
+        build_path = os.path.join(self.topobjdir, 'build')
+        if build_path not in sys.path:
+            sys.path.append(build_path)
+
+        if kwargs["config"] is None:
+            kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
+
+        if kwargs["binary"] is None:
+            kwargs["binary"] = os.path.join(self.get_binary_path('app'))
+
+        if kwargs["prefs_root"] is None:
+            kwargs["prefs_root"] = os.path.join(self.topobjdir, '_tests', 'web-platform', "prefs")
+
+        kwargs["capture_stdio"] = True
+
+        kwargs = wptcommandline.check_args(kwargs)
+
+    def run_tests(self, **kwargs):
+        from wptrunner import wptrunner
+
+        self.setup_kwargs(kwargs)
+
+        logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
+        result = wptrunner.run_tests(**kwargs)
+
+        return int(not result)
+
+    def list_test_groups(self, **kwargs):
+        from wptrunner import wptrunner
+
+        self.setup_kwargs(kwargs)
+
+        wptrunner.list_test_groups(**kwargs)
+
+class WebPlatformTestsUpdater(MozbuildObject):
+    """Update web platform tests."""
+    def run_update(self, **kwargs):
+        from wptrunner import update
+
+        if kwargs["config"] is None:
+            kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
+
+        wptcommandline.set_from_config(kwargs)
+
+        update.run_update(**kwargs)
+
+class WebPlatformTestsReduce(WebPlatformTestsRunner):
+
+    def run_reduce(self, **kwargs):
+        from wptrunner import reduce
+
+        self.setup_kwargs(kwargs)
+
+        kwargs["capture_stdio"] = True
+        logger = reduce.setup_logging(kwargs, {"mach": sys.stdout})
+        tests = reduce.do_reduce(**kwargs)
+
+        if not tests:
+            logger.warning("Test was not unstable")
+
+        for item in tests:
+            logger.info(item.id)
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+    @Command("web-platform-tests",
+             category="testing",
+             conditions=[conditions.is_firefox],
+             parser=wptcommandline.create_parser(["firefox"]))
+    def run_web_platform_tests(self, **params):
+        self.setup()
+        wpt_runner = self._spawn(WebPlatformTestsRunner)
+
+        if params["list_test_groups"]:
+            return wpt_runner.list_test_groups(**params)
+        else:
+            return wpt_runner.run_tests(**params)
+
+    @Command("web-platform-tests-update",
+             category="testing",
+             conditions=[conditions.is_firefox],
+             parser=wptcommandline.create_parser_update())
+    def update_web_platform_tests(self, **params):
+        self.setup()
+        self.virtualenv_manager.install_pip_package('html5lib==0.99')
+        wpt_updater = self._spawn(WebPlatformTestsUpdater)
+        return wpt_updater.run_update(**params)
+
+    def setup(self):
+        self._activate_virtualenv()
+        self.virtualenv_manager.install_pip_package('py==1.4.14')
+
+    @Command("web-platform-tests-reduce",
+             category="testing",
+             conditions=[conditions.is_firefox],
+             parser=wptcommandline.create_parser_reduce(["firefox"]))
+    def unstable_web_platform_tests(self, **params):
+        self.setup()
+        wpt_reduce = self._spawn(WebPlatformTestsReduce)
+        return wpt_reduce.run_reduce(**params)

testing/web-platform/moz.build

@@ -0,0 +1,6 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+

testing/web-platform/runtests.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+
+from wptrunner import wptrunner
+
+if __name__ == "__main__":
+    success = wptrunner.main()
+    if not success:
+        sys.exit(1)

testing/web-platform/update.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import os
+
+here = os.path.dirname(__file__)
+os.path.abspath(sys.path.insert(0, os.path.join(here, "harness")))
+
+from wptrunner import update
+
+if __name__ == "__main__":
+    success = update.main()
+    sys.exit(0 if success else 1)

testing/web-platform/wptrunner.ini

@@ -0,0 +1,11 @@
+[products]
+firefox =
+
+[web-platform-tests]
+remote_url = https://github.com/w3c/web-platform-tests.git
+branch = master
+sync_path = sync
+
+[paths]
+tests = tests
+metadata = meta

toolkit/toolkit.mozbuild

@@ -182,6 +182,7 @@ if CONFIG['ENABLE_TESTS']:
         'testing/profiles',
         'testing/mozbase',
         'testing/modules',
+        'testing/web-platform',
     ])
 
     if CONFIG['MOZ_WEBRTC'] and CONFIG['MOZ_WIDGET_TOOLKIT'] != 'gonk':