Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 05:41:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1363428 - Support Marionette reftest implementation in wptrunner, r=maja_zf

Browse Source 
Add an InternalReftestImplmentation that runs reftests using the
built-in Marionette reftest primitives rather than screenshots. This
is actiivated using the --internal-reftest switch, although that may
become the default in the future.

MozReview-Commit-ID: 6HxGuBsTITf

--HG--
extra : rebase_source : 7b5416cfc1a26c54432346de3822a2685b41f9c9
This commit is contained in:
James Graham 2017-05-10 11:41:33 +01:00
parent c9e960f859
commit 685665d899
4 changed files with 96 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
View File

@ -65,6 +65,7 @@ def browser_kwargs(test_type, run_info_data, **kwargs):
return {"binary": kwargs["binary"],
"prefs_root": kwargs["prefs_root"],
"extra_prefs": kwargs["extra_prefs"],
"test_type": test_type,
"debug_info": kwargs["debug_info"],
"symbols_path": kwargs["symbols_path"],
"stackwalk_binary": kwargs["stackwalk_binary"],
@ -88,6 +89,9 @@ def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
executor_kwargs["timeout_multiplier"] = get_timeout_multiplier(test_type,
run_info_data,
**kwargs)
if test_type == "reftest":
executor_kwargs["reftest_internal"] = kwargs["reftest_internal"]
executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"]
if test_type == "wdspec":
executor_kwargs["binary"] = kwargs["binary"]
executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
@ -131,13 +135,14 @@ class FirefoxBrowser(Browser):
init_timeout = 60
shutdown_timeout = 60
def __init__(self, logger, binary, prefs_root, extra_prefs=None, debug_info=None,
def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False, stackfix_dir=None,
binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.test_type = test_type
self.extra_prefs = extra_prefs
self.marionette_port = None
self.runner = None
@ -185,11 +190,13 @@ class FirefoxBrowser(Browser):
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False,
"dom.send_after_paint_to_content": True,
"layout.interruptible-reflow.enabled": False})
"dom.send_after_paint_to_content": True})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):

6
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
View File

@ -215,6 +215,12 @@ class RefTestImplementation(object):
self.screenshot_cache = self.executor.screenshot_cache
self.message = None
def setup(self):
pass
def teardown(self):
pass
@property
def logger(self):
return self.executor.logger

86
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
View File

@ -467,8 +467,9 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
class MarionetteRefTestExecutor(RefTestExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
screenshot_cache=None, close_after_done=True,
debug_info=None, **kwargs):
debug_info=None, reftest_internal=False,
reftest_screenshot="unexpected",
**kwargs):
"""Marionette-based executor for reftests"""
RefTestExecutor.__init__(self,
browser,
@ -477,7 +478,12 @@ class MarionetteRefTestExecutor(RefTestExecutor):
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = MarionetteProtocol(self, browser)
self.implementation = RefTestImplementation(self)
self.implementation = (InternalRefTestImplementation
if reftest_internal
else RefTestImplementation)(self)
self.implementation_kwargs = ({"screenshot": reftest_screenshot} if
reftest_internal else {})
self.close_after_done = close_after_done
self.has_window = False
self.original_pref_values = {}
@ -487,6 +493,16 @@ class MarionetteRefTestExecutor(RefTestExecutor):
with open(os.path.join(here, "reftest-wait.js")) as f:
self.wait_script = f.read()
def setup(self, runner):
super(self.__class__, self).setup(runner)
self.implementation.setup(**self.implementation_kwargs)
def teardown(self):
self.implementation.teardown()
handle = self.protocol.marionette.window_handles[0]
self.protocol.marionette.switch_to_window(handle)
super(self.__class__, self).teardown()
def is_alive(self):
return self.protocol.is_alive
@ -494,16 +510,17 @@ class MarionetteRefTestExecutor(RefTestExecutor):
self.protocol.on_environment_change(self.last_environment, new_environment)
def do_test(self, test):
if self.close_after_done and self.has_window:
self.protocol.marionette.close()
self.protocol.marionette.switch_to_window(
self.protocol.marionette.window_handles[-1])
self.has_window = False
if not isinstance(self.implementation, InternalRefTestImplementation):
if self.close_after_done and self.has_window:
self.protocol.marionette.close()
self.protocol.marionette.switch_to_window(
self.protocol.marionette.window_handles[-1])
self.has_window = False
if not self.has_window:
self.protocol.marionette.execute_script(self.script)
self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
self.has_window = True
if not self.has_window:
self.protocol.marionette.execute_script(self.script)
self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
self.has_window = True
result = self.implementation.run_test(test)
return self.convert_result(test, result)
@ -536,6 +553,49 @@ class MarionetteRefTestExecutor(RefTestExecutor):
return screenshot
class InternalRefTestImplementation(object):
def __init__(self, executor):
self.timeout_multiplier = executor.timeout_multiplier
self.executor = executor
@property
def logger(self):
return self.executor.logger
def setup(self, screenshot="unexpected"):
data = {"screenshot": screenshot}
if self.executor.queue_metadata is not None:
data["urlCount"] = {urlparse.urljoin(self.executor.server_url(key[0]), key[1]):value
for key, value in self.executor.queue_metadata.get("url_count", {}).iteritems()
if value > 1}
self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME)
self.executor.protocol.marionette._send_message("reftest:setup", data)
def run_test(self, test):
viewport_size = test.viewport_size
dpi = test.dpi
references = self.get_references(test)
rv = self.executor.protocol.marionette._send_message("reftest:run",
{"test": self.executor.test_url(test),
"references": references,
"expected": test.expected(),
"timeout": test.timeout * 1000})["value"]
return rv
def get_references(self, node):
rv = []
for item, relation in node.references:
rv.append([self.executor.test_url(item), self.get_references(item), relation])
return rv
def teardown(self):
try:
self.executor.protocol.marionette._send_message("reftest:teardown", {})
self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CONTENT)
except socket.error:
pass
class WdspecRun(object):
def __init__(self, func, session, path, timeout):
self.func = func
@ -579,7 +639,7 @@ class WdspecRun(object):
class MarionetteWdspecExecutor(WdspecExecutor):
def __init__(self, browser, server_config, webdriver_binary,
timeout_multiplier=1, close_after_done=True, debug_info=None,
capabilities=None, webdriver_args=None, binary=None):
capabilities=None, webdriver_args=None, binary=None, **kwargs):
self.do_delayed_imports()
WdspecExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,

7
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
View File

@ -184,6 +184,13 @@ scheme host and port.""")
help="Enable leak checking")
gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
help="Number of parallel threads to use for stylo")
gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
default=None, help="Enable reftest runner implemented inside Marionette")
gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
help="Disable reftest runner implemented inside Marionette")
gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
choices=["always", "fail", "unexpected"], default="unexpected",
help="With --reftest-internal, when to take a screenshot")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument("--user-stylesheet",