Bug 1405141 - Add stability checking to wptrunner, r=gbrown

This adds a --verify flag that is compatible with other Mozilla test
harnesses i.e. it runs each test 10 times without restarting and then
runs it 5 times with restarts, and then repeats with chaos mode
enabled.

This uses the code from, and can replace, the |wpt run --stability|
flag from upstream although that has different default behaviour
(running 10 times with restarts). More work is needed to avoid
duplicating all the code, however.

MozReview-Commit-ID: 7oUEwJk7uhZ

testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py

@@ -74,6 +74,9 @@ scheme host and port.""")
     mode_group.add_argument("--list-tests", action="store_true",
                             default=False,
                             help="List all tests that will run")
+    mode_group.add_argument("--verify", action="store_true",
+                            default=False,
+                            help="Run a stability check on the selected tests")
 
     test_selection_group = parser.add_argument_group("Test Selection")
     test_selection_group.add_argument("--test-types", action="store",

testing/web-platform/tests/tools/wptrunner/wptrunner/wptrunner.py

@@ -261,6 +261,12 @@ def run_tests(config, test_paths, product, **kwargs):
                 logger.suite_end()
     return unexpected_total == 0
 
+
+def check_stability(**kwargs):
+    import stability
+    return stability.check_stability(logger, **kwargs)
+
+
 def start(**kwargs):
     if kwargs["list_test_groups"]:
         list_test_groups(**kwargs)
@@ -268,9 +274,12 @@ def start(**kwargs):
         list_disabled(**kwargs)
     elif kwargs["list_tests"]:
         list_tests(**kwargs)
+    elif kwargs["verify"]:
+        check_stability(**kwargs)
     else:
         return not run_tests(**kwargs)
 
+
 def main():
     """Main entry point when calling from the command line"""
     kwargs = wptcommandline.parse_args()