diff --git a/testing/web-platform/harness/wptrunner/browsers/b2g_setup/certtest_app.zip b/testing/web-platform/harness/wptrunner/browsers/b2g_setup/certtest_app.zip index ade8879b2128..f9cbd5300ad6 100644 Binary files a/testing/web-platform/harness/wptrunner/browsers/b2g_setup/certtest_app.zip and b/testing/web-platform/harness/wptrunner/browsers/b2g_setup/certtest_app.zip differ diff --git a/testing/web-platform/harness/wptrunner/executors/executorservo.py b/testing/web-platform/harness/wptrunner/executors/executorservo.py index 865abb4f05f9..37ba19f7a540 100644 --- a/testing/web-platform/harness/wptrunner/executors/executorservo.py +++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. +import base64 import hashlib import json import os @@ -21,6 +22,19 @@ from .base import (ExecutorException, reftest_result_converter) from .process import ProcessTestExecutor +hosts_text = """127.0.0.1 web-platform.test +127.0.0.1 www.web-platform.test +127.0.0.1 www1.web-platform.test +127.0.0.1 www2.web-platform.test +127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test +127.0.0.1 xn--lve-6lad.web-platform.test +""" + +def make_hosts_file(): + hosts_fd, hosts_path = tempfile.mkstemp() + with os.fdopen(hosts_fd, "w") as f: + f.write(hosts_text) + return hosts_path class ServoTestharnessExecutor(ProcessTestExecutor): convert_result = testharness_result_converter @@ -34,6 +48,14 @@ class ServoTestharnessExecutor(ProcessTestExecutor): self.result_data = None self.result_flag = None self.protocol = Protocol(self, browser) + self.hosts_path = make_hosts_file() + + def teardown(self): + try: + os.unlink(self.hosts_path) + except OSError: + pass + ProcessTestExecutor.teardown(self) def do_test(self, test): self.result_data = None @@ -47,38 +69,46 @@ class ServoTestharnessExecutor(ProcessTestExecutor): if self.debug_args: self.command = list(self.debug_args) + self.command + env = os.environ.copy() + env["HOST_FILE"] = self.hosts_path self.proc = ProcessHandler(self.command, processOutputLine=[self.on_output], - onFinish=self.on_finish) - self.proc.run() + onFinish=self.on_finish, + env=env) - timeout = test.timeout * self.timeout_multiplier + try: + self.proc.run() - # Now wait to get the output we expect, or until we reach the timeout - if self.debug_args is None and not self.pause_after_test: - wait_timeout = timeout + 5 - else: - wait_timeout = None - self.result_flag.wait(wait_timeout) + timeout = test.timeout * self.timeout_multiplier - proc_is_running = True - if self.result_flag.is_set() and self.result_data is not None: - self.result_data["test"] = test.url - result = self.convert_result(test, self.result_data) - else: - if self.proc.proc.poll() is not None: - result = (test.result_cls("CRASH", None), []) - proc_is_running = False + # Now wait to get the output we expect, or until we reach the timeout + if self.debug_args is None and not self.pause_after_test: + wait_timeout = timeout + 5 else: - result = (test.result_cls("TIMEOUT", None), []) + wait_timeout = None + self.result_flag.wait(wait_timeout) - if proc_is_running: - if self.pause_after_test: - self.logger.info("Pausing until the browser exits") - self.proc.wait() + proc_is_running = True + if self.result_flag.is_set() and self.result_data is not None: + self.result_data["test"] = test.url + result = self.convert_result(test, self.result_data) else: - self.proc.kill() + if self.proc.proc.poll() is not None: + result = (test.result_cls("CRASH", None), []) + proc_is_running = False + else: + result = (test.result_cls("TIMEOUT", None), []) + + if proc_is_running: + if self.pause_after_test: + self.logger.info("Pausing until the browser exits") + self.proc.wait() + else: + self.proc.kill() + except KeyboardInterrupt: + self.proc.kill() + raise return result @@ -121,6 +151,7 @@ class ServoRefTestExecutor(ProcessTestExecutor): def __init__(self, browser, server_config, binary=None, timeout_multiplier=1, screenshot_cache=None, debug_args=None, pause_after_test=False): + ProcessTestExecutor.__init__(self, browser, server_config, @@ -131,8 +162,13 @@ class ServoRefTestExecutor(ProcessTestExecutor): self.screenshot_cache = screenshot_cache self.implementation = RefTestImplementation(self) self.tempdir = tempfile.mkdtemp() + self.hosts_path = make_hosts_file() def teardown(self): + try: + os.unlink(self.hosts_path) + except OSError: + pass os.rmdir(self.tempdir) ProcessTestExecutor.teardown(self) @@ -143,10 +179,20 @@ class ServoRefTestExecutor(ProcessTestExecutor): self.command = [self.binary, "--cpu", "--hard-fail", "--exit", "--output=%s" % output_path, full_url] + env = os.environ.copy() + env["HOST_FILE"] = self.hosts_path + self.proc = ProcessHandler(self.command, - processOutputLine=[self.on_output]) - self.proc.run() - rv = self.proc.wait(timeout=test.timeout) + processOutputLine=[self.on_output], + env=env) + + try: + self.proc.run() + rv = self.proc.wait(timeout=test.timeout) + except KeyboardInterrupt: + self.proc.kill() + raise + if rv is None: self.proc.kill() return False, ("EXTERNAL-TIMEOUT", None) @@ -157,7 +203,7 @@ class ServoRefTestExecutor(ProcessTestExecutor): with open(output_path) as f: # Might need to strip variable headers or something here data = f.read() - return True, data + return True, base64.b64encode(data) def do_test(self, test): result = self.implementation.run_test(test) diff --git a/testing/web-platform/harness/wptrunner/update/metadata.py b/testing/web-platform/harness/wptrunner/update/metadata.py index 34b630df9c16..dbcc9f999511 100644 --- a/testing/web-platform/harness/wptrunner/update/metadata.py +++ b/testing/web-platform/harness/wptrunner/update/metadata.py @@ -39,10 +39,10 @@ class CreateMetadataPatch(Step): if sync_tree is not None: name = "web-platform-tests_update_%s_metadata" % sync_tree.rev - message = "Update web-platform-tests expected data to revision %s" % sync_tree.rev + message = "Update %s expected data to revision %s" % (state.suite_name, sync_tree.rev) else: name = "web-platform-tests_update_metadata" - message = "Update web-platform-tests expected data" + message = "Update %s expected data" % state.suite_name local_tree.create_patch(name, message) diff --git a/testing/web-platform/harness/wptrunner/update/sync.py b/testing/web-platform/harness/wptrunner/update/sync.py index db36f1874aa9..6a58696d5450 100644 --- a/testing/web-platform/harness/wptrunner/update/sync.py +++ b/testing/web-platform/harness/wptrunner/update/sync.py @@ -154,7 +154,7 @@ class CreateSyncPatch(Step): sync_tree = state.sync_tree local_tree.create_patch("web-platform-tests_update_%s" % sync_tree.rev, - "Update web-platform-tests to revision %s" % sync_tree.rev) + "Update %s to revision %s" % (state.suite_name, sync_tree.rev)) local_tree.add_new(os.path.relpath(state.tests_path, local_tree.root)) updated = local_tree.update_patch(include=[state.tests_path, diff --git a/testing/web-platform/harness/wptrunner/update/update.py b/testing/web-platform/harness/wptrunner/update/update.py index ad89ef5f4b3f..3ef4b3f1c07d 100644 --- a/testing/web-platform/harness/wptrunner/update/update.py +++ b/testing/web-platform/harness/wptrunner/update/update.py @@ -13,8 +13,9 @@ from .. import environment as env from base import Step, StepRunner, exit_clean, exit_unclean from state import State -def setup_paths(logger, test_paths): - env.do_delayed_imports(logger, test_paths) +def setup_paths(sync_path): + sys.path.insert(0, sync_path) + from tools import localpaths class LoadConfig(Step): """Step for loading configuration from the ini file and kwargs.""" @@ -68,9 +69,11 @@ class SyncFromUpstream(Step): state.sync_tree = GitTree(root=state.sync["path"]) kwargs = state.kwargs - with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree", "sync_tree"]): + with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree", + "sync_tree"]): state.target_rev = kwargs["rev"] state.no_patch = kwargs["no_patch"] + state.suite_name = kwargs["suite_name"] runner = SyncFromUpstreamRunner(self.logger, state) runner.run() @@ -87,6 +90,7 @@ class UpdateMetadata(Step): state.run_log = kwargs["run_log"] state.ignore_existing = kwargs["ignore_existing"] state.no_patch = kwargs["no_patch"] + state.suite_name = kwargs["suite_name"] runner = MetadataUpdateRunner(self.logger, state) runner.run() @@ -109,8 +113,11 @@ class WPTUpdate(object): """ self.runner_cls = runner_cls self.serve_root = kwargs["test_paths"]["/"]["tests_path"] - #This must be before we try to reload state - setup_paths(logger, kwargs["test_paths"]) + + if not kwargs["sync"]: + setup_paths(self.serve_root) + else: + setup_paths(kwargs["sync_path"]) self.state = State(logger) self.kwargs = kwargs diff --git a/testing/web-platform/harness/wptrunner/wptcommandline.py b/testing/web-platform/harness/wptrunner/wptcommandline.py index 59da40e3a548..1622f87db99e 100644 --- a/testing/web-platform/harness/wptrunner/wptcommandline.py +++ b/testing/web-platform/harness/wptrunner/wptcommandline.py @@ -66,7 +66,7 @@ def create_parser(product_choices=None): type=abs_path, help="Binary to run tests against") parser.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") - parser.add_argument("--processes", action="store", type=int, default=1, + parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, @@ -83,7 +83,7 @@ def create_parser(product_choices=None): help="Don't capture stdio and write to logging") parser.add_argument("--product", action="store", choices=product_choices, - default="firefox", help="Browser against which to run tests") + default=None, help="Browser against which to run tests") parser.add_argument("--list-test-groups", action="store_true", default=False, @@ -170,6 +170,7 @@ def set_from_config(kwargs): config_path = kwargs["config"] kwargs["config_path"] = config_path + kwargs["config"] = config.read(kwargs["config_path"]) keys = {"paths": [("prefs", "prefs_root", True), @@ -204,6 +205,8 @@ def set_from_config(kwargs): kwargs["test_paths"]["/"] = {} kwargs["test_paths"]["/"]["metadata_path"] = kwargs["metadata_root"] + kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests") + def get_test_paths(config): # Set up test_paths @@ -250,6 +253,9 @@ def check_args(kwargs): print "Fatal: %s path %s is not a directory" % (name, path) sys.exit(1) + if kwargs["product"] is None: + kwargs["product"] = "firefox" + if kwargs["test_list"]: if kwargs["include"] is not None: kwargs["include"].extend(kwargs["test_list"]) @@ -268,6 +274,9 @@ def check_args(kwargs): else: kwargs["chunk_type"] = "none" + if kwargs["processes"] is None: + kwargs["processes"] = 1 + if kwargs["debugger"] is not None: debug_args, interactive = debugger_arguments(kwargs["debugger"], kwargs["debugger_args"])