From 32e52422c36efbd85a2d5c61cb49eab6eccabba0 Mon Sep 17 00:00:00 2001 From: Nikki Sharpley Date: Wed, 24 Jul 2019 12:30:05 +0000 Subject: [PATCH] Bug 1567000 - Add --update-intermittent and --remove-intermittent arguments to the command line for the web-platform-tests harness r=jgraham `--update-intermittent` and `--remove-intermittent` have been added to the command line arguments for the web-platform-tests `update` option. This adds expected intermittent statuses to the test metadata. Differential Revision: https://phabricator.services.mozilla.com/D38871 --HG-- extra : moz-landing-system : lando --- .../tests/tools/wptrunner/wptrunner/metadata.py | 16 ++++++++++------ .../tools/wptrunner/wptrunner/update/metadata.py | 4 +++- .../tools/wptrunner/wptrunner/update/update.py | 2 ++ .../tools/wptrunner/wptrunner/wptcommandline.py | 6 +++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py index 2dfe78ec26f1..abedb01d6dec 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/metadata.py @@ -54,11 +54,12 @@ class RunInfo(object): def update_expected(test_paths, serve_root, log_file_names, update_properties, rev_old=None, rev_new="HEAD", - full_update=False, sync_root=None, disable_intermittent=None): + full_update=False, sync_root=None, disable_intermittent=None, + update_intermittent=False, remove_intermittent=False): """Update the metadata files for web-platform-tests based on the results obtained in a previous run or runs - If stability is not None, assume log_file_names refers to logs from repeated + If disable_intermittent is not None, assume log_file_names refers to logs from repeated test jobs, disable tests that don't behave as expected on all runs""" do_delayed_imports(serve_root) @@ -67,6 +68,8 @@ def update_expected(test_paths, serve_root, log_file_names, for metadata_path, updated_ini in update_from_logs(id_test_map, update_properties, disable_intermittent, + update_intermittent, + remove_intermittent, full_update, *log_file_names): @@ -138,7 +141,7 @@ def unexpected_changes(manifests, change_data, files_changed): # for each conditional: # If all the new values match (or there aren't any) retain that conditional # If any new values mismatch: -# If stability and any repeated values don't match, disable the test +# If disable_intermittent and any repeated values don't match, disable the test # else mark the test as needing human attention # Check if all the RHS values are the same; if so collapse the conditionals @@ -217,8 +220,8 @@ def load_test_data(test_paths): return id_test_map -def update_from_logs(id_test_map, update_properties, disable_intermittent, full_update, - *log_filenames): +def update_from_logs(id_test_map, update_properties, disable_intermittent, update_intermittent, + remove_intermittent, full_update, *log_filenames): updater = ExpectedUpdater(id_test_map) @@ -227,7 +230,8 @@ def update_from_logs(id_test_map, update_properties, disable_intermittent, full_ with open(log_filename) as f: updater.update_from_log(f) - for item in update_results(id_test_map, update_properties, disable_intermittent, full_update): + for item in update_results(id_test_map, update_properties, full_update, + disable_intermittent, update_intermittent, remove_intermittent): yield item diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py index 63070051dce5..4bc70d311040 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/metadata.py @@ -28,7 +28,9 @@ class UpdateExpected(Step): rev_old=None, full_update=state.full_update, sync_root=sync_root, - disable_intermittent=state.disable_intermittent) + disable_intermittent=state.disable_intermittent, + update_intermittent=state.update_intermittent, + remove_intermittent=state.remove_intermittent) class CreateMetadataPatch(Step): diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py index 9f7c93e2120d..f3a9207ae6a0 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/update/update.py @@ -88,6 +88,8 @@ class UpdateMetadata(Step): with state.push(["local_tree", "sync_tree", "paths", "serve_root"]): state.run_log = kwargs["run_log"] state.disable_intermittent = kwargs["disable_intermittent"] + state.update_intermittent = kwargs["update_intermittent"] + state.remove_intermittent = kwargs["remove_intermittent"] state.patch = kwargs["patch"] state.suite_name = kwargs["suite_name"] state.product = kwargs["product"] diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py index 4f17e2d7d4d6..d060f4bca2ad 100644 --- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py +++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py @@ -611,9 +611,13 @@ def create_parser_update(product_choices=None): help="Sync the tests with the latest from upstream (implies --patch)") parser.add_argument("--full", action="store_true", default=False, help=("For all tests that are updated, remove any existing conditions and missing subtests")) - parser.add_argument("--disable_intermittent", nargs="?", action="store", const="unstable", default=None, + parser.add_argument("--disable-intermittent", nargs="?", action="store", const="unstable", default=None, help=("Reason for disabling tests. When updating test results, disable tests that have " "inconsistent results across many runs with the given reason.")) + parser.add_argument("--update-intermittent", action="store_true", default=False, + help=("Update test metadata with expected intermittent statuses.")) + parser.add_argument("--remove-intermittent", action="store_true", default=False, + help=("Remove obsolete intermittent statuses from expected statuses.")) parser.add_argument("--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True, help=("Don't remove metadata files that no longer correspond to a test file")) parser.add_argument("--no-store-state", action="store_false", dest="store_state",