Bug 1416410 - Record wdspec harness outcomes. r=jgraham

This fixes the problem that wdspec tests fail silently when there
is a problem with collecting the file in pytest.  For example,
if the Python test file contains a SyntaxError we currently fail
to record this as ERROR, and instead report it as OK.

MozReview-Commit-ID: 5W46gaLQa0c

--HG--
extra : rebase_source : de7be091949f792be9d72cacac5c630e53cf2c3a

testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py

@@ -357,12 +357,10 @@ class WdspecExecutor(TestExecutor):
         return (test.result_cls(*data), [])
 
     def do_wdspec(self, session_config, path, timeout):
-        harness_result = ("OK", None)
-        subtest_results = pytestrunner.run(path,
-                                           self.server_config,
-                                           session_config,
-                                           timeout=timeout)
-        return (harness_result, subtest_results)
+        return pytestrunner.run(path,
+                                self.server_config,
+                                session_config,
+                                timeout=timeout)
 
     def do_delayed_imports(self):
         global pytestrunner

testing/web-platform/tests/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py

@@ -1,4 +1,5 @@
-"""Provides interface to deal with pytest.
+"""
+Provides interface to deal with pytest.
 
 Usage::
 
@@ -24,7 +25,8 @@ def do_delayed_imports():
 
 
 def run(path, server_config, session_config, timeout=0):
-    """Run Python test at ``path`` in pytest.  The provided ``session``
+    """
+    Run Python test at ``path`` in pytest.  The provided ``session``
     is exposed as a fixture available in the scope of the test functions.
 
     :param path: Path to the test file.
@@ -33,36 +35,51 @@ def run(path, server_config, session_config, timeout=0):
     :param timeout: Duration before interrupting potentially hanging
         tests.  If 0, there is no timeout.
 
-    :returns: List of subtest results, which are tuples of (test id,
-        status, message, stacktrace).
+    :returns: (<harness result>, [<subtest result>, ...]),
+        where <subtest result> is (test id, status, message, stacktrace).
     """
-
     if pytest is None:
         do_delayed_imports()
 
-    recorder = SubtestResultRecorder()
-
     os.environ["WD_HOST"] = session_config["host"]
     os.environ["WD_PORT"] = str(session_config["port"])
     os.environ["WD_CAPABILITIES"] = json.dumps(session_config["capabilities"])
     os.environ["WD_SERVER_CONFIG"] = json.dumps(server_config)
 
-    plugins = [recorder]
-
-    # TODO(ato): Deal with timeouts
+    harness = HarnessResultRecorder()
+    subtests = SubtestResultRecorder()
 
     with TemporaryDirectory() as cache:
-        pytest.main(["--strict",  # turn warnings into errors
-                     "--verbose",  # show each individual subtest
-                     "--capture", "no",  # enable stdout/stderr from tests
-                     "--basetemp", cache,  # temporary directory
-                     "--showlocals",  # display contents of variables in local scope
-                     "-p", "no:mozlog",  # use the WPT result recorder
-                     "-p", "no:cacheprovider",  # disable state preservation across invocations
-                     path],
-                    plugins=plugins)
+        try:
+            pytest.main(["--strict",  # turn warnings into errors
+                         "--verbose",  # show each individual subtest
+                         "--capture", "no",  # enable stdout/stderr from tests
+                         "--basetemp", cache,  # temporary directory
+                         "--showlocals",  # display contents of variables in local scope
+                         "-p", "no:mozlog",  # use the WPT result recorder
+                         "-p", "no:cacheprovider",  # disable state preservation across invocations
+                         path],
+                        plugins=[harness, subtests])
+        except Exception as e:
+            harness.outcome = ("ERROR", str(e))
 
-    return recorder.results
+    return (harness.outcome, subtests.results)
+
+
+class HarnessResultRecorder(object):
+    outcomes = {
+        "failed": "ERROR",
+        "passed": "OK",
+        "skipped": "SKIP",
+    }
+
+    def __init__(self):
+        # we are ok unless told otherwise
+        self.outcome = ("OK", None)
+
+    def pytest_collectreport(self, report):
+        harness_result = self.outcomes[report.outcome]
+        self.outcome = (harness_result, None)
 
 
 class SubtestResultRecorder(object):