Bug 1356227 - Add support for excluding paths when doing a wpt sync, r=ato

MozReview-Commit-ID: 6zadFuRDmVD

testing/web-platform/harness/wptrunner/update/sync.py

@@ -2,7 +2,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import fnmatch
 import os
+import re
 import shutil
 import sys
 import uuid
@@ -46,7 +48,7 @@ POSSIBILITY OF SUCH DAMAGE.
 """
 
 
-def copy_wpt_tree(tree, dest):
+def copy_wpt_tree(tree, dest, excludes=None, includes=None):
     """Copy the working copy of a Tree to a destination directory.
 
     :param tree: The Tree to copy.
@@ -55,9 +57,24 @@ def copy_wpt_tree(tree, dest):
         assert os.path.isdir(dest)
 
     shutil.rmtree(dest)
+
     os.mkdir(dest)
 
+    if excludes is None:
+        excludes = []
+
+    excludes = [re.compile(fnmatch.translate(item)) for item in excludes]
+
+    if includes is None:
+        includes = []
+
+    includes = [re.compile(fnmatch.translate(item)) for item in includes]
+
     for tree_path in tree.paths():
+        if (any(item.match(tree_path) for item in excludes) and
+            not any(item.match(tree_path) for item in includes)):
+            continue
+
         source_path = os.path.join(tree.root, tree_path)
         dest_path = os.path.join(dest, tree_path)
 
@@ -83,6 +100,7 @@ def add_license(dest):
     with open(os.path.join(dest, "LICENSE"), "w") as f:
         f.write(bsd_license)
 
+
 class UpdateCheckout(Step):
     """Pull changes from upstream into the local sync tree."""
 
@@ -146,7 +164,9 @@ class CopyWorkTree(Step):
 
     def create(self, state):
         copy_wpt_tree(state.sync_tree,
-                      state.tests_path)
+                      state.tests_path,
+                      excludes=state.path_excludes,
+                      includes=state.path_includes)
 
 
 class CreateSyncPatch(Step):

testing/web-platform/harness/wptrunner/update/tree.py

@@ -316,9 +316,8 @@ class GitTree(object):
 
         for repo_path in repo_paths:
             paths = vcs.git("ls-tree", "-r", "--name-only", "HEAD", repo=repo_path).split("\n")
-            rel_path = os.path.relpath(repo_path, self.root)
-            rv.extend(os.path.join(rel_path, item.strip()) for item in paths if item.strip())
-
+            rv.extend(os.path.relpath(os.path.join(repo_path, item), self.root) for item in paths
+                      if item.strip())
         return rv
 
     def submodules(self):

testing/web-platform/harness/wptrunner/update/update.py

@@ -77,6 +77,8 @@ class SyncFromUpstream(Step):
             state.target_rev = kwargs["rev"]
             state.no_patch = kwargs["no_patch"]
             state.suite_name = kwargs["suite_name"]
+            state.path_excludes = kwargs["exclude"]
+            state.path_includes = kwargs["include"]
             runner = SyncFromUpstreamRunner(self.logger, state)
             runner.run()
 

testing/web-platform/harness/wptrunner/wptcommandline.py

@@ -403,6 +403,10 @@ def create_parser_update(product_choices=None):
     parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
     parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
     parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
+    parser.add_argument("--exclude", action="store", nargs="*",
+                        help="List of glob-style paths to exclude when syncing tests")
+    parser.add_argument("--include", action="store", nargs="*",
+                        help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
     # Should make this required iff run=logfile
     parser.add_argument("run_log", nargs="*", type=abs_path,
                         help="Log file from run of tests")

testing/web-platform/mach_commands.py

@@ -95,6 +95,13 @@ class WebPlatformTestsUpdater(MozbuildObject):
         if kwargs["product"] is None:
             kwargs["product"] = "firefox"
 
+        if kwargs["sync"]:
+            if not kwargs["exclude"]:
+                kwargs["exclude"] = ["css/*"]
+            if not kwargs["include"]:
+                kwargs["include"] = ["css/css-timing-1/*", "css/css-animations-1/*", "css/css-transitions-1/*"]
+
+
         updatecommandline.check_args(kwargs)
         logger = update.setup_logging(kwargs, {"mach": sys.stdout})
 
@@ -244,7 +251,6 @@ class WPTManifestUpdater(MozbuildObject):
     def run_update(self, check_clean=False, **kwargs):
         import manifestupdate
         from wptrunner import wptlogging
-
         logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
         wpt_dir = os.path.abspath(os.path.join(self.topsrcdir, 'testing', 'web-platform'))
         manifestupdate.update(logger, wpt_dir, check_clean)