Bug 1232259 - Update to latest wptrunner, a=testonly

testing/web-platform/harness/docs/expectation.rst

@@ -203,6 +203,10 @@ When used for expectation data, manifests have the following format:
    the (sub)test is disabled and should either not be run (for tests)
    or that its results should be ignored (subtests).
 
+ * A key ``restart-after`` which can be set to any value to indicate that
+   the runner should restart the browser after running this test (e.g. to
+   clear out unwanted state).
+
  * Variables ``debug``, ``os``, ``version``, ``processor`` and
    ``bits`` that describe the configuration of the browser under
    test. ``debug`` is a boolean indicating whether a build is a debug

testing/web-platform/harness/wptrunner/browsers/firefox.py

@@ -28,7 +28,8 @@ __wptrunner__ = {"product": "firefox",
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options",
-                 "run_info_extras": "run_info_extras"}
+                 "run_info_extras": "run_info_extras",
+                 "update_properties": "update_properties"}
 
 
 def check_args(**kwargs):
@@ -71,9 +72,14 @@ def env_options():
             "certificate_domain": "web-platform.test",
             "supports_debugger": True}
 
+
 def run_info_extras(**kwargs):
     return {"e10s": kwargs["gecko_e10s"]}
 
+
+def update_properties():
+    return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"}
+
 class FirefoxBrowser(Browser):
     used_ports = set()
 

testing/web-platform/harness/wptrunner/browsers/servo.py

@@ -17,7 +17,9 @@ __wptrunner__ = {"product": "servo",
                               "reftest": "ServoRefTestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
-                 "env_options": "env_options"}
+                 "env_options": "env_options",
+                 "run_info_extras": "run_info_extras",
+                 "update_properties": "update_properties"}
 
 
 def check_args(**kwargs):
@@ -47,8 +49,16 @@ def env_options():
             "supports_debugger": True}
 
 
+def run_info_extras(**kwargs):
+    return {"backend": kwargs["servo_backend"]}
+
+
+def update_properties():
+    return ["debug", "os", "version", "processor", "bits", "backend"], None
+
+
 def render_arg(render_backend):
-    return {"cpu": "--cpu"}[render_backend]
+    return {"cpu": "--cpu", "webrender": "--webrender"}[render_backend]
 
 
 class ServoBrowser(NullBrowser):

testing/web-platform/harness/wptrunner/browsers/servodriver.py

@@ -23,7 +23,9 @@ __wptrunner__ = {"product": "servodriver",
                               "reftest": "ServoWebDriverRefTestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
-                 "env_options": "env_options"}
+                 "env_options": "env_options",
+                 "run_info_extras": "run_info_extras",
+                 "update_properties": "update_properties"}
 
 hosts_text = """127.0.0.1 web-platform.test
 127.0.0.1 www.web-platform.test
@@ -59,6 +61,14 @@ def env_options():
             "supports_debugger": True}
 
 
+def run_info_extras(**kwargs):
+    return {"backend": kwargs["servo_backend"]}
+
+
+def update_properties():
+    return ["debug", "os", "version", "processor", "bits", "backend"], None
+
+
 def make_hosts_file():
     hosts_fd, hosts_path = tempfile.mkstemp()
     with os.fdopen(hosts_fd, "w") as f:

testing/web-platform/harness/wptrunner/manifestexpected.py

@@ -29,10 +29,10 @@ def data_cls_getter(output_node, visited_node):
     raise ValueError
 
 
-def disabled(node):
-    """Boolean indicating whether the test is disabled"""
+def bool_prop(name, node):
+    """Boolean property"""
     try:
-        return node.get("disabled")
+        return node.get(name)
     except KeyError:
         return None
 
@@ -109,7 +109,11 @@ class ExpectedManifest(ManifestItem):
 
     @property
     def disabled(self):
-        return disabled(self)
+        return bool_prop("disabled", self)
+
+    @property
+    def restart_after(self):
+        return bool_prop("restart-after", self)
 
     @property
     def tags(self):
@@ -123,7 +127,11 @@ class ExpectedManifest(ManifestItem):
 class DirectoryManifest(ManifestItem):
     @property
     def disabled(self):
-        return disabled(self)
+        return bool_prop("disabled", self)
+
+    @property
+    def restart_after(self):
+        return bool_prop("restart-after", self)
 
     @property
     def tags(self):
@@ -164,7 +172,11 @@ class TestNode(ManifestItem):
 
     @property
     def disabled(self):
-        return disabled(self)
+        return bool_prop("disabled", self)
+
+    @property
+    def restart_after(self):
+        return bool_prop("restart-after", self)
 
     @property
     def tags(self):

testing/web-platform/harness/wptrunner/manifestupdate.py

@@ -49,13 +49,18 @@ def data_cls_getter(output_node, visited_node):
 
 
 class ExpectedManifest(ManifestItem):
-    def __init__(self, node, test_path=None, url_base=None):
+    def __init__(self, node, test_path=None, url_base=None, property_order=None,
+                 boolean_properties=None):
         """Object representing all the tests in a particular manifest
 
         :param node: AST Node associated with this object. If this is None,
                      a new AST is created to associate with this manifest.
         :param test_path: Path of the test file associated with this manifest.
-        :param url_base: Base url for serving the tests in this manifest
+        :param url_base: Base url for serving the tests in this manifest.
+        :param property_order: List of properties to use in expectation metadata
+                               from most to least significant.
+        :param boolean_properties: Set of properties in property_order that should
+                                   be treated as boolean.
         """
         if node is None:
             node = DataNode(None)
@@ -65,6 +70,8 @@ class ExpectedManifest(ManifestItem):
         self.url_base = url_base
         assert self.url_base is not None
         self.modified = False
+        self.boolean_properties = boolean_properties
+        self.property_order = property_order
 
     def append(self, child):
         ManifestItem.append(self, child)
@@ -229,7 +236,10 @@ class TestNode(ManifestItem):
                     self.set("expected", status, condition=None)
                     final_conditionals.append(self._data["expected"][-1])
             else:
-                for conditional_node, status in group_conditionals(self.new_expected):
+                for conditional_node, status in group_conditionals(
+                        self.new_expected,
+                        property_order=self.root.property_order,
+                        boolean_properties=self.root.boolean_properties):
                     if status != unconditional_status:
                         self.set("expected", status, condition=conditional_node.children[0])
                         final_conditionals.append(self._data["expected"][-1])
@@ -308,18 +318,30 @@ class SubtestNode(TestNode):
         return True
 
 
-def group_conditionals(values):
+def group_conditionals(values, property_order=None, boolean_properties=None):
     """Given a list of Result objects, return a list of
     (conditional_node, status) pairs representing the conditional
     expressions that are required to match each status
 
-    :param values: List of Results"""
+    :param values: List of Results
+    :param property_order: List of properties to use in expectation metadata
+                           from most to least significant.
+    :param boolean_properties: Set of properties in property_order that should
+                               be treated as boolean."""
 
     by_property = defaultdict(set)
     for run_info, status in values:
         for prop_name, prop_value in run_info.iteritems():
             by_property[(prop_name, prop_value)].add(status)
 
+    if property_order is None:
+        property_order = ["debug", "os", "version", "processor", "bits"]
+
+    if boolean_properties is None:
+        boolean_properties = set(["debug"])
+    else:
+        boolean_properties = set(boolean_properties)
+
     # If we have more than one value, remove any properties that are common
     # for all the values
     if len(values) > 1:
@@ -328,11 +350,9 @@ def group_conditionals(values):
                 del by_property[key]
 
     properties = set(item[0] for item in by_property.iterkeys())
-
-    prop_order = ["debug", "e10s", "os", "version", "processor", "bits"]
     include_props = []
 
-    for prop in prop_order:
+    for prop in property_order:
         if prop in properties:
             include_props.append(prop)
 
@@ -343,28 +363,33 @@ def group_conditionals(values):
         if prop_set in conditions:
             continue
 
-        expr = make_expr(prop_set, status)
+        expr = make_expr(prop_set, status, boolean_properties=boolean_properties)
         conditions[prop_set] = (expr, status)
 
     return conditions.values()
 
 
-def make_expr(prop_set, status):
+def make_expr(prop_set, status, boolean_properties=None):
     """Create an AST that returns the value ``status`` given all the
-    properties in prop_set match."""
+    properties in prop_set match.
+
+    :param prop_set: tuple of (property name, value) pairs for each
+                     property in this expression and the value it must match
+    :param status: Status on RHS when all the given properties match
+    :param boolean_properties: Set of properties in property_order that should
+                               be treated as boolean.
+    """
     root = ConditionalNode()
 
     assert len(prop_set) > 0
 
-    no_value_props = set(["debug", "e10s"])
-
     expressions = []
     for prop, value in prop_set:
         number_types = (int, float, long)
         value_cls = (NumberNode
                      if type(value) in number_types
                      else StringNode)
-        if prop not in no_value_props:
+        if prop not in boolean_properties:
             expressions.append(
                 BinaryExpressionNode(
                     BinaryOperatorNode("=="),
@@ -397,24 +422,32 @@ def make_expr(prop_set, status):
     return root
 
 
-def get_manifest(metadata_root, test_path, url_base):
+def get_manifest(metadata_root, test_path, url_base, property_order=None,
+                 boolean_properties=None):
     """Get the ExpectedManifest for a particular test path, or None if there is no
     metadata stored for that test path.
 
     :param metadata_root: Absolute path to the root of the metadata directory
     :param test_path: Path to the test(s) relative to the test root
     :param url_base: Base url for serving the tests in this manifest
-    """
+    :param property_order: List of properties to use in expectation metadata
+                           from most to least significant.
+    :param boolean_properties: Set of properties in property_order that should
+                               be treated as boolean."""
     manifest_path = expected.expected_path(metadata_root, test_path)
     try:
         with open(manifest_path) as f:
-            return compile(f, test_path, url_base)
+            return compile(f, test_path, url_base, property_order=property_order,
+                           boolean_properties=boolean_properties)
     except IOError:
         return None
 
 
-def compile(manifest_file, test_path, url_base):
+def compile(manifest_file, test_path, url_base, property_order=None,
+            boolean_properties=None):
     return conditional.compile(manifest_file,
                                data_cls_getter=data_cls_getter,
                                test_path=test_path,
-                               url_base=url_base)
+                               url_base=url_base,
+                               property_order=property_order,
+                               boolean_properties=boolean_properties)

testing/web-platform/harness/wptrunner/metadata.py

@@ -32,7 +32,7 @@ def load_test_manifests(serve_root, test_paths):
 
 def update_expected(test_paths, serve_root, log_file_names,
                     rev_old=None, rev_new="HEAD", ignore_existing=False,
-                    sync_root=None):
+                    sync_root=None, property_order=None, boolean_properties=None):
     """Update the metadata files for web-platform-tests based on
     the results obtained in a previous run"""
 
@@ -51,7 +51,9 @@ def update_expected(test_paths, serve_root, log_file_names,
 
     expected_map_by_manifest = update_from_logs(manifests,
                                                 *log_file_names,
-                                                ignore_existing=ignore_existing)
+                                                ignore_existing=ignore_existing,
+                                                property_order=property_order,
+                                                boolean_properties=boolean_properties)
 
     for test_manifest, expected_map in expected_map_by_manifest.iteritems():
         url_base = manifests[test_manifest]["url_base"]
@@ -127,14 +129,19 @@ def unexpected_changes(manifests, change_data, files_changed):
 
 
 def update_from_logs(manifests, *log_filenames, **kwargs):
-    ignore_existing = kwargs.pop("ignore_existing", False)
+    ignore_existing = kwargs.get("ignore_existing", False)
+    property_order = kwargs.get("property_order")
+    boolean_properties = kwargs.get("boolean_properties")
 
     expected_map = {}
     id_test_map = {}
 
     for test_manifest, paths in manifests.iteritems():
-        expected_map_manifest, id_path_map_manifest = create_test_tree(paths["metadata_path"],
-                                                                       test_manifest)
+        expected_map_manifest, id_path_map_manifest = create_test_tree(
+            paths["metadata_path"],
+            test_manifest,
+            property_order=property_order,
+            boolean_properties=boolean_properties)
         expected_map[test_manifest] = expected_map_manifest
         id_test_map.update(id_path_map_manifest)
 
@@ -284,15 +291,22 @@ class ExpectedUpdater(object):
         del self.test_cache[test_id]
 
 
-def create_test_tree(metadata_path, test_manifest):
+def create_test_tree(metadata_path, test_manifest, property_order=None,
+                     boolean_properties=None):
     expected_map = {}
     id_test_map = {}
     exclude_types = frozenset(["stub", "helper", "manual"])
     include_types = set(manifest.item_types) - exclude_types
     for test_path, tests in test_manifest.itertypes(*include_types):
-        expected_data = load_expected(test_manifest, metadata_path, test_path, tests)
+        expected_data = load_expected(test_manifest, metadata_path, test_path, tests,
+                                      property_order=property_order,
+                                      boolean_properties=boolean_properties)
         if expected_data is None:
-            expected_data = create_expected(test_manifest, test_path, tests)
+            expected_data = create_expected(test_manifest,
+                                            test_path,
+                                            tests,
+                                            property_order=property_order,
+                                            boolean_properties=boolean_properties)
 
         for test in tests:
             id_test_map[test.id] = (test_manifest, test)
@@ -301,17 +315,23 @@ def create_test_tree(metadata_path, test_manifest):
     return expected_map, id_test_map
 
 
-def create_expected(test_manifest, test_path, tests):
-    expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base)
+def create_expected(test_manifest, test_path, tests, property_order=None,
+                    boolean_properties=None):
+    expected = manifestupdate.ExpectedManifest(None, test_path, test_manifest.url_base,
+                                               property_order=property_order,
+                                               boolean_properties=boolean_properties)
     for test in tests:
         expected.append(manifestupdate.TestNode.create(test.item_type, test.id))
     return expected
 
 
-def load_expected(test_manifest, metadata_path, test_path, tests):
+def load_expected(test_manifest, metadata_path, test_path, tests, property_order=None,
+                  boolean_properties=None):
     expected_manifest = manifestupdate.get_manifest(metadata_path,
                                                     test_path,
-                                                    test_manifest.url_base)
+                                                    test_manifest.url_base,
+                                                    property_order=property_order,
+                                                    boolean_properties=boolean_properties)
     if expected_manifest is None:
         return
 

testing/web-platform/harness/wptrunner/products.py

@@ -55,3 +55,18 @@ def load_product(config, product):
             browser_cls, browser_kwargs,
             executor_classes, executor_kwargs,
             env_options, run_info_extras)
+
+
+def load_product_update(config, product):
+    """Return tuple of (property_order, boolean_properties) indicating the
+    run_info properties to use when constructing the expectation data for
+    this product. None for either key indicates that the default keys
+    appropriate for distinguishing based on platform will be used."""
+
+    module = product_module(config, product)
+    data = module.__wptrunner__
+
+    update_properties = (getattr(module, data["update_properties"])()
+                         if "update_properties" in data else (None, None))
+
+    return update_properties

testing/web-platform/harness/wptrunner/testrunner.py

@@ -524,7 +524,8 @@ class TestRunnerManager(threading.Thread):
 
         self.test = None
 
-        restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
+        restart_before_next = (test.restart_after or
+                               file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
                                subtest_unexpected or is_unexpected)
 
         if (self.pause_after_test or

testing/web-platform/harness/wptrunner/update/metadata.py

@@ -4,10 +4,21 @@
 
 import os
 
-from .. import metadata
+from .. import metadata, products
 
 from base import Step, StepRunner
 
+class GetUpdatePropertyList(Step):
+    provides = ["property_order", "boolean_properties"]
+
+
+    def create(self, state):
+        property_order, boolean_properties = products.load_product_update(
+            state.config, state.product)
+        state.property_order = property_order
+        state.boolean_properties = boolean_properties
+
+
 class UpdateExpected(Step):
     """Do the metadata update on the local checkout"""
 
@@ -24,7 +35,9 @@ class UpdateExpected(Step):
                                                      state.run_log,
                                                      rev_old=None,
                                                      ignore_existing=state.ignore_existing,
-                                                     sync_root=sync_root)
+                                                     sync_root=sync_root,
+                                                     property_order=state.property_order,
+                                                     boolean_properties=state.boolean_properties)
 
 
 class CreateMetadataPatch(Step):
@@ -57,5 +70,6 @@ class CreateMetadataPatch(Step):
 
 class MetadataUpdateRunner(StepRunner):
     """(Sub)Runner for updating metadata"""
-    steps = [UpdateExpected,
+    steps = [GetUpdatePropertyList,
+             UpdateExpected,
              CreateMetadataPatch]

testing/web-platform/harness/wptrunner/update/update.py

@@ -91,6 +91,8 @@ class UpdateMetadata(Step):
             state.ignore_existing = kwargs["ignore_existing"]
             state.no_patch = kwargs["no_patch"]
             state.suite_name = kwargs["suite_name"]
+            state.product = kwargs["product"]
+            state.config = kwargs["config"]
             runner = MetadataUpdateRunner(self.logger, state)
             runner.run()
 

testing/web-platform/harness/wptrunner/wptcommandline.py

@@ -338,12 +338,25 @@ def check_args(kwargs):
 
     return kwargs
 
+def check_args_update(kwargs):
+    set_from_config(kwargs)
 
-def create_parser_update():
+    if kwargs["product"] is None:
+        kwargs["product"] = "firefox"
+
+def create_parser_update(product_choices=None):
     from mozlog.structured import commandline
 
+    import products
+
+    if product_choices is None:
+        config_data = config.load()
+        product_choices = products.products_enabled(config_data)
+
     parser = argparse.ArgumentParser("web-platform-tests-update",
                                      description="Update script for web-platform-tests tests.")
+    parser.add_argument("--product", action="store", choices=product_choices,
+                        default=None, help="Browser for which metadata is being updated")
     parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
     parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
                         help="Path to the folder containing test metadata"),
@@ -386,7 +399,7 @@ def parse_args():
 def parse_args_update():
     parser = create_parser_update()
     rv = vars(parser.parse_args())
-    set_from_config(rv)
+    check_args_update(rv)
     return rv
 
 

testing/web-platform/harness/wptrunner/wpttest.py

@@ -149,6 +149,14 @@ class Test(object):
                 return disabled
         return None
 
+    @property
+    def restart_after(self):
+        for meta in self.itermeta(None):
+            restart_after = meta.restart_after
+            if restart_after is not None:
+                return True
+        return False
+
     @property
     def tags(self):
         tags = set()