Bug 1567000 - Add --update-intermittent and --remove-intermittent arguments to the command line for the web-platform-tests harness r=jgraham

`--update-intermittent` and `--remove-intermittent` have been added to the command line arguments for the web-platform-tests
`update` option. This adds expected intermittent statuses to the test metadata.

Differential Revision: https://phabricator.services.mozilla.com/D38871

--HG--
extra : moz-landing-system : lando

testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py

@@ -54,11 +54,12 @@ class RunInfo(object):
 
 def update_expected(test_paths, serve_root, log_file_names,
                     update_properties, rev_old=None, rev_new="HEAD",
-                    full_update=False, sync_root=None, disable_intermittent=None):
+                    full_update=False, sync_root=None, disable_intermittent=None,
+                    update_intermittent=False, remove_intermittent=False):
     """Update the metadata files for web-platform-tests based on
     the results obtained in a previous run or runs
 
-    If stability is not None, assume log_file_names refers to logs from repeated
+    If disable_intermittent is not None, assume log_file_names refers to logs from repeated
     test jobs, disable tests that don't behave as expected on all runs"""
     do_delayed_imports(serve_root)
 
@@ -67,6 +68,8 @@ def update_expected(test_paths, serve_root, log_file_names,
     for metadata_path, updated_ini in update_from_logs(id_test_map,
                                                        update_properties,
                                                        disable_intermittent,
+                                                       update_intermittent,
+                                                       remove_intermittent,
                                                        full_update,
                                                        *log_file_names):
 
@@ -138,7 +141,7 @@ def unexpected_changes(manifests, change_data, files_changed):
 #   for each conditional:
 #      If all the new values match (or there aren't any) retain that conditional
 #      If any new values mismatch:
-#           If stability and any repeated values don't match, disable the test
+#           If disable_intermittent and any repeated values don't match, disable the test
 #           else mark the test as needing human attention
 #   Check if all the RHS values are the same; if so collapse the conditionals
 
@@ -217,8 +220,8 @@ def load_test_data(test_paths):
     return id_test_map
 
 
-def update_from_logs(id_test_map, update_properties, disable_intermittent, full_update,
-                     *log_filenames):
+def update_from_logs(id_test_map, update_properties, disable_intermittent, update_intermittent,
+                     remove_intermittent, full_update, *log_filenames):
 
     updater = ExpectedUpdater(id_test_map)
 
@@ -227,7 +230,8 @@ def update_from_logs(id_test_map, update_properties, disable_intermittent, full_
         with open(log_filename) as f:
             updater.update_from_log(f)
 
-    for item in update_results(id_test_map, update_properties, disable_intermittent, full_update):
+    for item in update_results(id_test_map, update_properties, full_update,
+                               disable_intermittent, update_intermittent, remove_intermittent):
         yield item
 
 

testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py

@@ -28,7 +28,9 @@ class UpdateExpected(Step):
                                  rev_old=None,
                                  full_update=state.full_update,
                                  sync_root=sync_root,
-                                 disable_intermittent=state.disable_intermittent)
+                                 disable_intermittent=state.disable_intermittent,
+                                 update_intermittent=state.update_intermittent,
+                                 remove_intermittent=state.remove_intermittent)
 
 
 class CreateMetadataPatch(Step):

testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py

@@ -88,6 +88,8 @@ class UpdateMetadata(Step):
         with state.push(["local_tree", "sync_tree", "paths", "serve_root"]):
             state.run_log = kwargs["run_log"]
             state.disable_intermittent = kwargs["disable_intermittent"]
+            state.update_intermittent = kwargs["update_intermittent"]
+            state.remove_intermittent = kwargs["remove_intermittent"]
             state.patch = kwargs["patch"]
             state.suite_name = kwargs["suite_name"]
             state.product = kwargs["product"]

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -611,9 +611,13 @@ def create_parser_update(product_choices=None):
                         help="Sync the tests with the latest from upstream (implies --patch)")
     parser.add_argument("--full", action="store_true", default=False,
                         help=("For all tests that are updated, remove any existing conditions and missing subtests"))
-    parser.add_argument("--disable_intermittent", nargs="?", action="store", const="unstable", default=None,
+    parser.add_argument("--disable-intermittent", nargs="?", action="store", const="unstable", default=None,
         help=("Reason for disabling tests. When updating test results, disable tests that have "
               "inconsistent results across many runs with the given reason."))
+    parser.add_argument("--update-intermittent", action="store_true", default=False,
+                        help=("Update test metadata with expected intermittent statuses."))
+    parser.add_argument("--remove-intermittent", action="store_true", default=False,
+                        help=("Remove obsolete intermittent statuses from expected statuses."))
     parser.add_argument("--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True,
                         help=("Don't remove metadata files that no longer correspond to a test file"))
     parser.add_argument("--no-store-state", action="store_false", dest="store_state",