Bug 1473915 - Set up infra so we can move the wpt-manifest out of tree r=gps

Changes the wpt manifest path to the topobjdir instead so it can be moved out of tree.
Other changes so that the manifest download and update, and |mach wpt| and |mach test <wpt-test>| work with the new path.
The manifest is also downloaded and updated when creating the tests-archive to ensure that it exists when we run tests on TC.

MozReview-Commit-ID: Fp6UsKJjhTU

Differential Revision: https://phabricator.services.mozilla.com/D5312

--HG--
extra : moz-landing-system : lando

python/mozbuild/mozbuild/action/download_wpt_manifest.py

@@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This action is used to generate the wpt manifest
+
+import os
+import sys
+
+import buildconfig
+
+
+def main():
+    print("Downloading wpt manifest")
+    man_path = os.path.join(buildconfig.topobjdir, '_tests', 'web-platform')
+    sys.path.insert(0, buildconfig.topsrcdir)
+    import manifestdownload
+    manifestdownload.run(man_path, buildconfig.topsrcdir, force=True)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

python/mozbuild/mozbuild/frontend/context.py

@@ -44,7 +44,6 @@ from ..testing import (
     all_test_flavors,
     read_manifestparser_manifest,
     read_reftest_manifest,
-    read_wpt_manifest,
 )
 
 import mozpack.path as mozpath
@@ -895,12 +894,8 @@ def TypedListWithAction(typ, action):
             super(_TypedListWithAction, self).__init__(action=_action, *args)
     return _TypedListWithAction
 
-WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest",
-                                          [("manifest_path", unicode),
-                                           ("test_root", unicode)])
 ManifestparserManifestList = OrderedPathListWithAction(read_manifestparser_manifest)
 ReftestManifestList = OrderedPathListWithAction(read_reftest_manifest)
-WptManifestList = TypedListWithAction(WebPlatformTestManifest, read_wpt_manifest)
 
 OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList)
 OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()),
@@ -1882,10 +1877,6 @@ VARIABLES = {
         These are commonly named crashtests.list.
         """),
 
-    'WEB_PLATFORM_TESTS_MANIFESTS': (WptManifestList, list,
-        """List of (manifest_path, test_path) defining web-platform-tests.
-        """),
-
     'WEBRTC_SIGNALLING_TEST_MANIFESTS': (ManifestparserManifestList, list,
         """List of manifest files defining WebRTC signalling tests.
         """),

python/mozbuild/mozbuild/frontend/emitter.py

@@ -83,7 +83,6 @@ from .reader import SandboxValidationError
 from ..testing import (
     TEST_MANIFESTS,
     REFTEST_FLAVORS,
-    WEB_PLATFORM_TESTS_FLAVORS,
     SupportFilesConverter,
 )
 
@@ -1458,11 +1457,6 @@ class TreeMetadataEmitter(LoggingMixin):
                 for obj in self._process_reftest_manifest(context, flavor, path, manifest):
                     yield obj
 
-        for flavor in WEB_PLATFORM_TESTS_FLAVORS:
-            for path, manifest in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []):
-                for obj in self._process_web_platform_tests_manifest(context, path, manifest):
-                    yield obj
-
     def _process_test_manifest(self, context, info, manifest_path, mpmanifest):
         flavor, install_root, install_subdir, package_tests = info
 
@@ -1591,40 +1585,6 @@ class TreeMetadataEmitter(LoggingMixin):
 
         yield obj
 
-    def _process_web_platform_tests_manifest(self, context, paths, manifest):
-        manifest_path, tests_root = paths
-        manifest_full_path = mozpath.normpath(mozpath.join(
-            context.srcdir, manifest_path))
-        manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path,
-            context.config.topsrcdir))
-        tests_root = mozpath.normpath(mozpath.join(context.srcdir, tests_root))
-
-        # Create a equivalent TestManifest object
-        obj = TestManifest(context, manifest_full_path, manifest,
-                           flavor="web-platform-tests",
-                           relpath=mozpath.join(manifest_reldir,
-                                                mozpath.basename(manifest_path)),
-                           install_prefix="web-platform/")
-
-
-        for test_type, path, tests in manifest:
-            path = mozpath.join(tests_root, path)
-            if test_type not in ["testharness", "reftest", "wdspec"]:
-                continue
-
-            for test in tests:
-                obj.tests.append({
-                    'path': path,
-                    'here': mozpath.dirname(path),
-                    'manifest': manifest_path,
-                    'name': test.id,
-                    'head': '',
-                    'support-files': '',
-                    'subsuite': '',
-                })
-
-        yield obj
-
     def _process_jar_manifests(self, context):
         jar_manifests = context.get('JAR_MANIFESTS', [])
         if len(jar_manifests) > 1:

taskcluster/ci/source-test/mozlint.yml

@@ -139,13 +139,11 @@ wptlint-gecko:
     treeherder:
         symbol: W
     run:
-        mach: lint -l wpt -l wpt_manifest -f treeherder
+        mach: lint -l wpt -f treeherder
     when:
         files-changed:
             - 'testing/web-platform/tests/**'
             - 'testing/web-platform/mozilla/tests/**'
-            - 'testing/web-platform/meta/MANIFEST.json'
-            - 'testing/web-platform/mozilla/meta/MANIFEST.json'
 
 yaml:
     description: yamllint run over the gecko codebase

testing/mozbase/moztest/moztest/resolve.py

@@ -282,10 +282,13 @@ class TestMetadata(object):
     configuration.
     """
 
-    def __init__(self, all_tests, test_defaults=None):
+    def __init__(self, all_tests, srcdir, test_defaults=None):
         self._tests_by_path = OrderedDefaultDict(list)
         self._tests_by_flavor = defaultdict(set)
         self._test_dirs = set()
+        self._objdir = os.path.abspath(os.path.join(all_tests, os.pardir))
+        self._wpt_loaded = False
+        self._srcdir = srcdir
 
         with open(all_tests, 'rb') as fh:
             test_data = pickle.load(fh)
@@ -378,6 +381,9 @@ class TestMetadata(object):
 
         candidate_paths = set()
 
+        if any(self.is_wpt_path(path) for path in paths):
+            self.add_wpt_manifest_data()
+
         for path in sorted(paths):
             if path is None:
                 candidate_paths |= set(self._tests_by_path.keys())
@@ -405,6 +411,61 @@ class TestMetadata(object):
             for test in fltr(tests):
                 yield test
 
+    def is_wpt_path(self, path):
+        if path is None:
+            return True
+        if mozpath.match(path, "testing/web-platform/tests/**"):
+            return True
+        if mozpath.match(path, "testing/web-platform/mozilla/tests/**"):
+            return True
+        return False
+
+    def add_wpt_manifest_data(self):
+        if self._wpt_loaded:
+            return
+
+        wpt_path = os.path.join(self._srcdir, "testing", "web-platform")
+        wptrunner_path = os.path.join(wpt_path, "tests", "tools", "wptrunner")
+        manifest_path = os.path.join(self._objdir, "_tests", "web-platform")
+
+        sys.path = [wpt_path, wptrunner_path] + sys.path
+
+        import manifestdownload
+        import wptrunner
+        from wptrunner.wptrunner import testloader
+
+        manifestdownload.run(manifest_path, self._srcdir)
+
+        kwargs = {"config": os.path.join(self._objdir, "_tests", "web-platform",
+                                         "wptrunner.local.ini"),
+                  "tests_root": None,
+                  "metadata_root": None}
+
+        wptrunner.wptcommandline.set_from_config(kwargs)
+        manifests = testloader.ManifestLoader(kwargs["test_paths"]).load()
+
+        for manifest, data in manifests.iteritems():
+            tests_root = data["tests_path"]
+            for test_type, path, tests in manifest:
+                path = os.path.join(tests_root, path)
+                if test_type not in ["testharness", "reftest", "wdspec"]:
+                    continue
+                for test in tests:
+                    self._tests_by_path[path].append({
+                        "path": path,
+                        "flavor": "web-platform-tests",
+                        "here": os.path.dirname(path),
+                        "manifest": data["manifest_path"],
+                        "name": test.id,
+                        "file_relpath": path,
+                        "head": "",
+                        "support-files": "",
+                        "subsuite": test_type,
+                        "dir_relpath": os.path.relpath(os.path.dirname(path), wpt_path)
+                        })
+
+        self._wpt_loaded = True
+
 
 class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
@@ -428,6 +489,7 @@ class TestResolver(MozbuildObject):
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
+                                   self.topsrcdir,
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 

testing/testsuite-targets.mk

@@ -153,8 +153,11 @@ endif
 package-tests-prepare-dest:
 	$(NSINSTALL) -D $(test_archive_dir)
 
+download-wpt-manifest:
+	$(call py_action,download_wpt_manifest)
+
 define package_archive
-package-tests-$(1): stage-all package-tests-prepare-dest
+package-tests-$(1): stage-all package-tests-prepare-dest download-wpt-manifest
 	$$(call py_action,test_archive, \
 		$(1) \
 		'$$(abspath $$(test_archive_dir))/$$(PKG_BASENAME).$(1).tests.$(2)')
@@ -268,6 +271,7 @@ check::
   xpcshell-tests \
   jstestbrowser \
   package-tests \
+  download-wpt-manifest \
   package-tests-prepare-dest \
   package-tests-common \
   make-stage-dir \

testing/web-platform/mach_commands.py

@@ -39,7 +39,7 @@ class WebPlatformTestsRunnerSetup(MozbuildObject):
             sys.path.append(build_path)
 
         if kwargs["config"] is None:
-            kwargs["config"] = os.path.join(here, 'wptrunner.ini')
+            kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
 
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.join(self.topsrcdir, 'testing', 'profiles')
@@ -136,7 +136,7 @@ class WebPlatformTestsUpdater(MozbuildObject):
         from update import updatecommandline
 
         if kwargs["config"] is None:
-            kwargs["config"] = os.path.join(self.topsrcdir, 'testing', 'web-platform', 'wptrunner.ini')
+            kwargs["config"] = os.path.join(self.topobjdir, '_tests', 'web-platform', 'wptrunner.local.ini')
         if kwargs["product"] is None:
             kwargs["product"] = "firefox"
 
@@ -293,21 +293,22 @@ testing/web-platform/tests for tests that may be shared
 
 
 class WPTManifestUpdater(MozbuildObject):
-    def run_update(self, check_clean=False, rebuild=False, **kwargs):
+    def run_update(self, rebuild=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
-        manifestupdate.update(logger, wpt_dir, check_clean, rebuild)
+        config_dir = os.path.abspath(os.path.join(self.topobjdir, '_tests', 'web-platform'))
+        manifestupdate.update(logger, wpt_dir, rebuild, config_dir)
 
 
 class WPTManifestDownloader(MozbuildObject):
-    def run_download(self, path=None, tests_root=None, force=False, **kwargs):
+    def run_download(self, manifest_update=True, force=False, **kwargs):
         import manifestdownload
         from wptrunner import wptlogging
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
-        wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
-        manifestdownload.run(logger, wpt_dir, self.topsrcdir, force)
+        wpt_dir = os.path.abspath(os.path.join(self.topobjdir, '_tests', 'web-platform'))
+        manifestdownload.run(wpt_dir, self.topsrcdir, logger, force, manifest_update)
 
 
 def create_parser_update():
@@ -371,6 +372,8 @@ class MachCommands(MachCommandBase):
                 params["include"].append(item["name"])
             del params["test_objects"]
 
+        self.wpt_manifest_download(**params)
+        params["manifest_update"] = False
         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
         wpt_runner = WebPlatformTestsRunner(wpt_setup)
         return wpt_runner.run(**params)
@@ -437,7 +440,6 @@ class MachCommands(MachCommandBase):
         wpt_manifest_updater = self._spawn(WPTManifestUpdater)
         return wpt_manifest_updater.run_update(**params)
 
-
     @Command("wpt-manifest-download",
              category="testing",
              parser=create_parser_manifest_download)

testing/web-platform/manifestdownload.py

@@ -1,13 +1,16 @@
 from __future__ import absolute_import
 
 import argparse
-import json
 import os
 from datetime import datetime, timedelta
 import tarfile
-import vcs
 import requests
+import vcs
 from cStringIO import StringIO
+import logging
+
+HEADERS = {'User-Agent': "wpt manifest download"}
+
 
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
@@ -24,7 +27,7 @@ def git_commits(repo_root):
     git = vcs.Git.get_func(repo_root)
     for item in git("log", "--format=%H", "-n50", "testing/web-platform/tests",
                     "testing/web-platform/mozilla/tests").splitlines():
-        yield git("cinnabar", "git2hg", item)
+        yield git("cinnabar", "git2hg", item).strip()
 
 
 def get_commits(logger, repo_root):
@@ -38,13 +41,15 @@ def get_commits(logger, repo_root):
     return False
 
 
-def should_download(logger, manifest_path, rebuild_time=timedelta(days=5)):
+def should_download(logger, manifest_paths, rebuild_time=timedelta(days=5)):
     # TODO: Improve logic for when to download. Maybe if x revisions behind?
-    if not os.path.exists(manifest_path):
-        return True
-    mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
-    if mtime < datetime.now() - rebuild_time:
-        return True
+    for manifest_path in manifest_paths:
+        if not os.path.exists(manifest_path):
+            return True
+        mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
+        if mtime < datetime.now() - rebuild_time:
+            return True
+
     logger.info("Skipping manifest download because existing file is recent")
     return False
 
@@ -57,14 +62,33 @@ def taskcluster_url(logger, commits):
               'revision.{changeset}.source.manifest-upload')
 
     for revision in commits:
-        req = requests.get(cset_url.format(changeset=revision),
-                           headers={'Accept': 'application/json'})
-
-        req.raise_for_status()
+        if revision == 40 * "0":
+            continue
+        try:
+            req_headers = HEADERS.copy()
+            req_headers.update({'Accept': 'application/json'})
+            req = requests.get(cset_url.format(changeset=revision),
+                               headers=req_headers)
+            req.raise_for_status()
+        except requests.exceptions.RequestException:
+            if req.status_code == 404:
+                # The API returns a 404 if it can't find a changeset for the revision.
+                continue
+            else:
+                return False
 
         result = req.json()
-        [cset] = result['pushes'].values()[0]['changesets']
-        req = requests.get(tc_url.format(changeset=cset))
+
+        pushes = result['pushes']
+        if not pushes:
+            continue
+        [cset] = pushes.values()[0]['changesets']
+
+        try:
+            req = requests.get(tc_url.format(changeset=cset),
+                               headers=HEADERS)
+        except requests.exceptions.RequestException:
+            return False
 
         if req.status_code == 200:
             return tc_url.format(changeset=cset)
@@ -77,21 +101,25 @@ def taskcluster_url(logger, commits):
 
 
 def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
-    if not force and not should_download(logger, os.path.join(wpt_dir, "meta", "MANIFEST.json")):
-        return False
+    manifest_path = os.path.join(wpt_dir, "meta", "MANIFEST.json")
+    mozilla_manifest_path = os.path.join(wpt_dir, "mozilla", "meta", "MANIFEST.json")
+
+    if not force and not should_download(logger, [manifest_path, mozilla_manifest_path]):
+        return True
 
     commits = commits_func()
     if not commits:
         return False
-    url = url_func(logger, commits) + "/artifacts/public/manifests.tar.gz"
 
+    url = url_func(logger, commits)
     if not url:
         logger.warning("No generated manifest found")
         return False
+    url+= "/artifacts/public/manifests.tar.gz"
 
     logger.info("Downloading manifest from %s" % url)
     try:
-        req = requests.get(url)
+        req = requests.get(url, headers=HEADERS)
     except Exception:
         logger.warning("Downloading pregenerated manifest failed")
         return False
@@ -108,8 +136,8 @@ def download_manifest(logger, wpt_dir, commits_func, url_func, force=False):
         logger.warning("Failed to decompress downloaded file")
         return False
 
-    os.utime(os.path.join(wpt_dir, "meta", "MANIFEST.json"), None)
-    os.utime(os.path.join(wpt_dir, "mozilla", "meta", "MANIFEST.json"), None)
+    os.utime(manifest_path, None)
+    os.utime(mozilla_manifest_path, None)
 
     logger.info("Manifest downloaded")
     return True
@@ -122,6 +150,9 @@ def create_parser():
     parser.add_argument(
         "--force", action="store_true",
         help="Always download, even if the existing manifest is recent")
+    parser.add_argument(
+        "--no-manifest-update", action="store_false", dest="manifest_update",
+        default=True, help="Don't update the downloaded manifest")
     return parser
 
 
@@ -130,6 +161,63 @@ def download_from_taskcluster(logger, wpt_dir, repo_root, force=False):
                              taskcluster_url, force)
 
 
-def run(logger, wpt_dir, repo_root, force=False):
+def generate_config(path):
+    """Generate the local wptrunner.ini file to use locally"""
+    import ConfigParser
+    here = os.path.split(os.path.abspath(__file__))[0]
+    config_path = os.path.join(here, 'wptrunner.ini')
+    path = os.path.join(path, 'wptrunner.local.ini')
+
+    if os.path.exists(path):
+        return True
+
+    parser = ConfigParser.SafeConfigParser()
+    success = parser.read(config_path)
+    assert config_path in success, success
+
+    parser.set('manifest:upstream', 'tests', os.path.join(here, 'tests'))
+    parser.set('manifest:mozilla', 'tests', os.path.join(here, 'mozilla', 'tests'))
+    parser.set('paths', 'prefs', os.path.join(os.getcwd(), 'testing', 'profiles'))
+
+    with open(path, 'wb') as config_file:
+        parser.write(config_file)
+    return True
+
+
+def update_manifest(logger, config_dir, manifest_update=True):
+    if manifest_update:
+        logger.info("Updating manifests")
+        import manifestupdate
+        here = os.path.split(os.path.abspath(__file__))[0]
+        return manifestupdate.update(logger, here, config_dir=config_dir) is 0
+    else:
+        logger.info("Skipping manifest update")
+        return True
+
+def check_dirs(logger, success, wpt_dir):
+    if success:
+        return
+    else:
+        logger.info("Could not download manifests.")
+        logger.info("Generating from scratch instead.")
+        try:
+            os.mkdir(os.path.join(wpt_dir, "meta"))
+        except OSError:
+            pass
+        try:
+            os.makedirs(os.path.join(wpt_dir, "mozilla", "meta"))
+        except OSError:
+            pass
+
+
+def run(wpt_dir, repo_root, logger=None, force=False, manifest_update=True):
+    if not logger:
+        logger = logging.getLogger(__name__)
+        handler = logging.FileHandler(os.devnull)
+        logger.addHandler(handler)
+
     success = download_from_taskcluster(logger, wpt_dir, repo_root, force)
+    check_dirs(logger, success, wpt_dir)
+    generate_config(wpt_dir)
+    success |= update_manifest(logger, wpt_dir, manifest_update)
     return 0 if success else 1

testing/web-platform/manifestupdate.py

@@ -16,10 +16,9 @@ def do_delayed_imports(wpt_dir):
     import manifest
 
 
+
 def create_parser():
     p = argparse.ArgumentParser()
-    p.add_argument("--check-clean", action="store_true",
-                   help="Check that updating the manifest doesn't lead to any changes")
     p.add_argument("--rebuild", action="store_true",
                    help="Rebuild the manifest from scratch")
     commandline.add_logging_group(p)
@@ -27,22 +26,25 @@ def create_parser():
     return p
 
 
-def update(logger, wpt_dir, check_clean=True, rebuild=False):
+def update(logger, wpt_dir, rebuild=False, config_dir=None,):
     localpaths = imp.load_source("localpaths",
                                  os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
-    kwargs = {"config": os.path.join(wpt_dir, "wptrunner.ini"),
+
+    if not config_dir or not os.path.exists(os.path.join(config_dir, 'wptrunner.local.ini')):
+        config_dir = wpt_dir
+        config_name = "wptrunner.ini"
+    else:
+        config_name = "wptrunner.local.ini"
+
+    kwargs = {"config": os.path.join(config_dir, config_name),
               "tests_root": None,
               "metadata_root": None}
 
     set_from_config(kwargs)
     config = kwargs["config"]
     test_paths = get_test_paths(config)
-
     do_delayed_imports(wpt_dir)
 
-    if check_clean:
-        return _check_clean(logger, test_paths)
-
     return _update(logger, test_paths, rebuild)
 
 
@@ -50,7 +52,7 @@ def _update(logger, test_paths, rebuild):
     for url_base, paths in test_paths.iteritems():
         manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
         m = None
-        if not rebuild:
+        if not rebuild and os.path.exists(manifest_path):
             try:
                 m = manifest.manifest.load(paths["tests_path"], manifest_path)
             except manifest.manifest.ManifestVersionMismatch:
@@ -62,90 +64,6 @@ def _update(logger, test_paths, rebuild):
     return 0
 
 
-def _check_clean(logger, test_paths):
-    manifests_by_path = {}
-    rv = 0
-    for url_base, paths in test_paths.iteritems():
-        tests_path = paths["tests_path"]
-        manifest_path = os.path.join(paths["metadata_path"], "MANIFEST.json")
-        old_manifest = manifest.manifest.load(tests_path, manifest_path)
-        new_manifest = manifest.manifest.Manifest.from_json(tests_path,
-                                                            old_manifest.to_json())
-        manifest.update.update(tests_path, new_manifest, working_copy=True)
-        manifests_by_path[manifest_path] = (old_manifest, new_manifest)
-
-    for manifest_path, (old_manifest, new_manifest) in manifests_by_path.iteritems():
-        if not diff_manifests(logger, manifest_path, old_manifest, new_manifest):
-            rv = 1
-    if rv:
-        logger.error("Manifest %s is outdated, use |mach wpt-manifest-update| to fix." % manifest_path)
-
-    return rv
-
-
-def diff_manifests(logger, manifest_path, old_manifest, new_manifest):
-    """Lint the differences between old and new versions of a
-    manifest. Differences are considered significant (and so produce
-    lint errors) if they produce a meaningful difference in the actual
-    tests run.
-
-    :param logger: mozlog logger to use for output
-    :param manifest_path: Path to the manifest being linted
-    :param old_manifest: Manifest object representing the initial manifest
-    :param new_manifest: Manifest object representing the updated manifest
-    """
-    logger.info("Diffing old and new manifests %s" % manifest_path)
-    old_items, new_items = defaultdict(set), defaultdict(set)
-    for manifest, items in [(old_manifest, old_items),
-                            (new_manifest, new_items)]:
-        for test_type, path, tests in manifest:
-            for test in tests:
-                test_id = [test.id]
-                test_id.extend(tuple(item) if isinstance(item, list) else item
-                               for item in test.meta_key())
-                if hasattr(test, "references"):
-                    test_id.extend(tuple(item) for item in test.references)
-                test_id = tuple(test_id)
-                items[path].add((test_type, test_id))
-
-    old_paths = set(old_items.iterkeys())
-    new_paths = set(new_items.iterkeys())
-
-    added_paths = new_paths - old_paths
-    deleted_paths = old_paths - new_paths
-
-    common_paths = new_paths & old_paths
-
-    clean = True
-
-    for path in added_paths:
-        clean = False
-        log_error(logger, manifest_path, "%s in source but not in manifest." % path)
-    for path in deleted_paths:
-        clean = False
-        log_error(logger, manifest_path, "%s in manifest but removed from source." % path)
-
-    for path in common_paths:
-        old_tests = old_items[path]
-        new_tests = new_items[path]
-        added_tests = new_tests - old_tests
-        removed_tests = old_tests - new_tests
-        if added_tests or removed_tests:
-            clean = False
-            log_error(logger, manifest_path, "%s changed test types or metadata" % path)
-
-    if clean:
-        # Manifest currently has some list vs tuple inconsistencies that break
-        # a simple equality comparison.
-        new_paths = {(key, value[0], value[1])
-                     for (key, value) in new_manifest.to_json()["paths"].iteritems()}
-        old_paths = {(key, value[0], value[1])
-                     for (key, value) in old_manifest.to_json()["paths"].iteritems()}
-        if old_paths != new_paths:
-            logger.warning("Manifest %s contains correct tests but file hashes changed; please update" % manifest_path)
-
-    return clean
-
 def log_error(logger, manifest_path, msg):
     logger.lint_error(path=manifest_path,
                       message=msg,

testing/web-platform/moz.build

@@ -4,11 +4,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-WEB_PLATFORM_TESTS_MANIFESTS += [
-    ('meta/MANIFEST.json', 'tests/'),
-    ('mozilla/meta/MANIFEST.json', 'mozilla/tests/')
-]
-
 TEST_HARNESS_FILES['web-platform'] += [
     'mach_commands_base.py',
     'mach_test_package_commands.py',

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -47,7 +47,7 @@ def create_parser(product_choices=None):
 
 TEST is either the full path to a test file to run, or the URL of a test excluding
 scheme host and port.""")
-    parser.add_argument("--manifest-update", action="store_true", default=None,
+    parser.add_argument("--manifest-update", action="store_true", default=True,
                         help="Regenerate the test manifest.")
     parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update",
                         help="Prevent regeneration of the test manifest.")