2012-10-02 08:24:12 +00:00
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
|
2015-06-22 00:39:09 +00:00
|
|
|
from __future__ import absolute_import, unicode_literals
|
2012-10-02 08:24:12 +00:00
|
|
|
|
|
|
|
import os
|
2012-11-27 22:29:03 +00:00
|
|
|
import re
|
2013-08-29 18:23:07 +00:00
|
|
|
import sys
|
2016-09-13 13:26:09 +00:00
|
|
|
from argparse import Namespace
|
2012-10-02 08:24:12 +00:00
|
|
|
|
2012-11-07 00:57:41 +00:00
|
|
|
from mozbuild.base import (
|
|
|
|
MachCommandBase,
|
2013-08-29 18:23:07 +00:00
|
|
|
MachCommandConditions as conditions,
|
2012-11-07 00:57:41 +00:00
|
|
|
MozbuildObject,
|
|
|
|
)
|
2012-10-10 18:08:09 +00:00
|
|
|
|
2012-11-07 00:57:41 +00:00
|
|
|
from mach.decorators import (
|
2012-10-10 18:08:09 +00:00
|
|
|
CommandProvider,
|
|
|
|
Command,
|
|
|
|
)
|
|
|
|
|
2012-12-06 18:03:25 +00:00
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
parser = None
|
|
|
|
|
2016-01-04 21:27:37 +00:00
|
|
|
|
2012-10-10 18:08:09 +00:00
|
|
|
class ReftestRunner(MozbuildObject):
|
2012-10-02 08:24:12 +00:00
|
|
|
"""Easily run reftests.
|
|
|
|
|
|
|
|
This currently contains just the basics for running reftests. We may want
|
|
|
|
to hook up result parsing, etc.
|
|
|
|
"""
|
2013-08-29 18:23:07 +00:00
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
MozbuildObject.__init__(self, *args, **kwargs)
|
|
|
|
|
|
|
|
# TODO Bug 794506 remove once mach integrates with virtualenv.
|
|
|
|
build_path = os.path.join(self.topobjdir, 'build')
|
|
|
|
if build_path not in sys.path:
|
|
|
|
sys.path.append(build_path)
|
|
|
|
|
|
|
|
self.tests_dir = os.path.join(self.topobjdir, '_tests')
|
|
|
|
self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
|
2012-10-02 08:24:12 +00:00
|
|
|
|
2012-11-27 22:29:03 +00:00
|
|
|
def _make_shell_string(self, s):
|
|
|
|
return "'%s'" % re.sub("'", r"'\''", s)
|
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
def _setup_objdir(self, args):
|
2016-01-04 21:27:37 +00:00
|
|
|
# reftest imports will happen from the objdir
|
|
|
|
sys.path.insert(0, self.reftest_dir)
|
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
if not args.tests:
|
2016-01-04 21:27:37 +00:00
|
|
|
test_subdir = {
|
|
|
|
"reftest": os.path.join('layout', 'reftests'),
|
|
|
|
"crashtest": os.path.join('layout', 'crashtest'),
|
2016-09-13 13:26:09 +00:00
|
|
|
}[args.suite]
|
|
|
|
args.tests = [test_subdir]
|
2016-01-04 21:27:37 +00:00
|
|
|
|
|
|
|
tests = os.path.join(self.reftest_dir, 'tests')
|
|
|
|
if not os.path.isdir(tests):
|
|
|
|
os.symlink(self.topsrcdir, tests)
|
|
|
|
|
2015-08-25 13:07:23 +00:00
|
|
|
def run_desktop_test(self, **kwargs):
|
2015-10-14 21:20:20 +00:00
|
|
|
"""Runs a reftest, in desktop Firefox."""
|
2015-08-25 13:07:23 +00:00
|
|
|
import runreftest
|
2012-10-02 08:24:12 +00:00
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
args = Namespace(**kwargs)
|
|
|
|
if args.suite not in ('reftest', 'crashtest', 'jstestbrowser'):
|
2012-10-02 08:24:12 +00:00
|
|
|
raise Exception('None or unrecognized reftest suite type.')
|
|
|
|
|
2015-08-25 13:07:23 +00:00
|
|
|
default_manifest = {
|
|
|
|
"reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
|
|
|
|
"crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
|
|
|
|
"jstestbrowser": (self.topobjdir, "dist", "test-stage", "jsreftest", "tests",
|
|
|
|
"jstests.list")
|
|
|
|
}
|
2014-04-14 20:51:46 +00:00
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
args.extraProfileFiles.append(os.path.join(self.topobjdir, "dist", "plugins"))
|
|
|
|
args.symbolsPath = os.path.join(self.topobjdir, "crashreporter-symbols")
|
2017-09-26 20:36:43 +00:00
|
|
|
args.sandboxReadWhitelist.extend([self.topsrcdir, self.topobjdir])
|
2014-04-14 20:51:46 +00:00
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
if not args.tests:
|
|
|
|
args.tests = [os.path.join(*default_manifest[args.suite])]
|
2014-11-11 17:16:53 +00:00
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
if args.suite == "jstestbrowser":
|
|
|
|
args.extraProfileFiles.append(os.path.join(self.topobjdir, "dist",
|
2017-02-14 19:50:56 +00:00
|
|
|
"test-stage", "jsreftest",
|
|
|
|
"tests", "user.js"))
|
2014-05-23 21:07:01 +00:00
|
|
|
|
2015-08-25 13:07:23 +00:00
|
|
|
self.log_manager.enable_unstructured()
|
2015-10-14 22:31:19 +00:00
|
|
|
try:
|
2016-09-13 13:26:09 +00:00
|
|
|
rv = runreftest.run_test_harness(parser, args)
|
2015-10-14 22:31:19 +00:00
|
|
|
finally:
|
|
|
|
self.log_manager.disable_unstructured()
|
2013-08-29 18:23:07 +00:00
|
|
|
|
2015-08-25 13:07:23 +00:00
|
|
|
return rv
|
2013-04-17 19:06:36 +00:00
|
|
|
|
2015-10-14 21:20:20 +00:00
|
|
|
def run_android_test(self, **kwargs):
|
|
|
|
"""Runs a reftest, in Firefox for Android."""
|
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
args = Namespace(**kwargs)
|
|
|
|
if args.suite not in ('reftest', 'crashtest', 'jstestbrowser'):
|
2015-10-14 21:20:20 +00:00
|
|
|
raise Exception('None or unrecognized reftest suite type.')
|
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
self._setup_objdir(args)
|
2016-09-13 13:26:04 +00:00
|
|
|
import remotereftest
|
|
|
|
|
2015-10-14 21:20:20 +00:00
|
|
|
default_manifest = {
|
|
|
|
"reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
|
|
|
|
"crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
|
|
|
|
"jstestbrowser": ("jsreftest", "tests", "jstests.list")
|
|
|
|
}
|
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
if not args.tests:
|
|
|
|
args.tests = [os.path.join(*default_manifest[args.suite])]
|
2015-10-14 21:20:20 +00:00
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
args.extraProfileFiles.append(
|
2015-11-06 19:01:33 +00:00
|
|
|
os.path.join(self.topsrcdir, "mobile", "android", "fonts"))
|
2015-10-14 21:20:20 +00:00
|
|
|
|
2016-07-04 17:00:15 +00:00
|
|
|
hyphenation_path = os.path.join(self.topsrcdir, "intl", "locales")
|
|
|
|
|
|
|
|
for (dirpath, dirnames, filenames) in os.walk(hyphenation_path):
|
|
|
|
for filename in filenames:
|
|
|
|
if filename.endswith('.dic'):
|
2016-09-13 13:26:09 +00:00
|
|
|
args.extraProfileFiles.append(os.path.join(dirpath, filename))
|
|
|
|
|
|
|
|
if not args.httpdPath:
|
|
|
|
args.httpdPath = os.path.join(self.tests_dir, "modules")
|
|
|
|
if not args.symbolsPath:
|
|
|
|
args.symbolsPath = os.path.join(self.topobjdir, "crashreporter-symbols")
|
|
|
|
if not args.xrePath:
|
|
|
|
args.xrePath = os.environ.get("MOZ_HOST_BIN")
|
|
|
|
if not args.app:
|
|
|
|
args.app = self.substs["ANDROID_PACKAGE_NAME"]
|
|
|
|
if not args.utilityPath:
|
|
|
|
args.utilityPath = args.xrePath
|
|
|
|
args.ignoreWindowSize = True
|
|
|
|
args.printDeviceInfo = False
|
2015-10-14 21:20:20 +00:00
|
|
|
|
2017-10-24 21:17:25 +00:00
|
|
|
from mozrunner.devices.android_device import grant_runtime_permissions, get_adb_path
|
2016-08-05 20:40:08 +00:00
|
|
|
grant_runtime_permissions(self)
|
2016-02-01 22:52:34 +00:00
|
|
|
|
2017-10-24 21:17:25 +00:00
|
|
|
if not args.adb_path:
|
|
|
|
args.adb_path = get_adb_path(self)
|
|
|
|
|
2015-10-14 21:20:20 +00:00
|
|
|
# A symlink and some path manipulations are required so that test
|
|
|
|
# manifests can be found both locally and remotely (via a url)
|
|
|
|
# using the same relative path.
|
2016-09-13 13:26:09 +00:00
|
|
|
if args.suite == "jstestbrowser":
|
2015-10-14 21:20:20 +00:00
|
|
|
staged_js_dir = os.path.join(self.topobjdir, "dist", "test-stage", "jsreftest")
|
|
|
|
tests = os.path.join(self.reftest_dir, 'jsreftest')
|
|
|
|
if not os.path.isdir(tests):
|
|
|
|
os.symlink(staged_js_dir, tests)
|
2016-09-13 13:26:09 +00:00
|
|
|
args.extraProfileFiles.append(os.path.join(staged_js_dir, "tests", "user.js"))
|
2015-10-14 21:20:20 +00:00
|
|
|
else:
|
|
|
|
tests = os.path.join(self.reftest_dir, "tests")
|
|
|
|
if not os.path.isdir(tests):
|
|
|
|
os.symlink(self.topsrcdir, tests)
|
2016-09-13 13:26:09 +00:00
|
|
|
for i, path in enumerate(args.tests):
|
2015-10-14 21:20:20 +00:00
|
|
|
# Non-absolute paths are relative to the packaged directory, which
|
|
|
|
# has an extra tests/ at the start
|
|
|
|
if os.path.exists(os.path.abspath(path)):
|
|
|
|
path = os.path.relpath(path, os.path.join(self.topsrcdir))
|
2016-09-13 13:26:09 +00:00
|
|
|
args.tests[i] = os.path.join('tests', path)
|
2015-10-14 21:20:20 +00:00
|
|
|
|
|
|
|
self.log_manager.enable_unstructured()
|
2015-10-14 22:31:19 +00:00
|
|
|
try:
|
2016-09-13 13:07:05 +00:00
|
|
|
rv = remotereftest.run_test_harness(parser, args)
|
2015-10-14 22:31:19 +00:00
|
|
|
finally:
|
|
|
|
self.log_manager.disable_unstructured()
|
2015-10-14 21:20:20 +00:00
|
|
|
|
|
|
|
return rv
|
2015-09-08 14:30:08 +00:00
|
|
|
|
2016-01-04 21:27:37 +00:00
|
|
|
|
2015-09-08 14:30:08 +00:00
|
|
|
def process_test_objects(kwargs):
|
|
|
|
"""|mach test| works by providing a test_objects argument, from
|
|
|
|
which the test path must be extracted and converted into a normal
|
|
|
|
reftest tests argument."""
|
|
|
|
|
|
|
|
if "test_objects" in kwargs:
|
|
|
|
if kwargs["tests"] is None:
|
|
|
|
kwargs["tests"] = []
|
|
|
|
kwargs["tests"].extend(item["path"] for item in kwargs["test_objects"])
|
|
|
|
del kwargs["test_objects"]
|
|
|
|
|
2016-01-04 21:27:37 +00:00
|
|
|
|
2015-10-14 21:20:20 +00:00
|
|
|
def get_parser():
|
2017-07-19 18:27:10 +00:00
|
|
|
import reftestcommandline
|
|
|
|
|
2016-09-13 13:26:09 +00:00
|
|
|
global parser
|
2015-10-14 22:31:19 +00:00
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
2015-10-14 21:20:20 +00:00
|
|
|
build_obj = MozbuildObject.from_environment(cwd=here)
|
|
|
|
if conditions.is_android(build_obj):
|
2016-09-13 13:26:09 +00:00
|
|
|
parser = reftestcommandline.RemoteArgumentsParser()
|
2015-10-14 21:20:20 +00:00
|
|
|
else:
|
2016-09-13 13:26:09 +00:00
|
|
|
parser = reftestcommandline.DesktopArgumentsParser()
|
|
|
|
return parser
|
2015-09-08 14:30:08 +00:00
|
|
|
|
2016-01-04 21:27:37 +00:00
|
|
|
|
2012-10-10 18:08:09 +00:00
|
|
|
@CommandProvider
|
2012-11-07 00:57:41 +00:00
|
|
|
class MachCommands(MachCommandBase):
|
2015-08-25 13:07:23 +00:00
|
|
|
@Command('reftest',
|
|
|
|
category='testing',
|
|
|
|
description='Run reftests (layout and graphics correctness).',
|
2015-10-14 21:20:20 +00:00
|
|
|
parser=get_parser)
|
2015-08-25 13:07:23 +00:00
|
|
|
def run_reftest(self, **kwargs):
|
|
|
|
kwargs["suite"] = "reftest"
|
|
|
|
return self._run_reftest(**kwargs)
|
|
|
|
|
|
|
|
@Command('jstestbrowser',
|
|
|
|
category='testing',
|
|
|
|
description='Run js/src/tests in the browser.',
|
2015-10-14 21:20:20 +00:00
|
|
|
parser=get_parser)
|
2015-08-25 13:07:23 +00:00
|
|
|
def run_jstestbrowser(self, **kwargs):
|
|
|
|
self._mach_context.commands.dispatch("build",
|
|
|
|
self._mach_context,
|
|
|
|
what=["stage-jstests"])
|
|
|
|
kwargs["suite"] = "jstestbrowser"
|
|
|
|
return self._run_reftest(**kwargs)
|
|
|
|
|
|
|
|
@Command('crashtest',
|
|
|
|
category='testing',
|
|
|
|
description='Run crashtests (Check if crashes on a page).',
|
2015-10-14 21:20:20 +00:00
|
|
|
parser=get_parser)
|
2015-08-25 13:07:23 +00:00
|
|
|
def run_crashtest(self, **kwargs):
|
|
|
|
kwargs["suite"] = "crashtest"
|
|
|
|
return self._run_reftest(**kwargs)
|
|
|
|
|
|
|
|
def _run_reftest(self, **kwargs):
|
2017-04-07 18:53:19 +00:00
|
|
|
kwargs["topsrcdir"] = self.topsrcdir
|
2015-09-08 14:30:08 +00:00
|
|
|
process_test_objects(kwargs)
|
2012-10-10 18:08:09 +00:00
|
|
|
reftest = self._spawn(ReftestRunner)
|
2015-10-14 21:20:20 +00:00
|
|
|
if conditions.is_android(self):
|
|
|
|
from mozrunner.devices.android_device import verify_android_device
|
|
|
|
verify_android_device(self, install=True, xre=True)
|
|
|
|
return reftest.run_android_test(**kwargs)
|
2015-08-25 13:07:23 +00:00
|
|
|
return reftest.run_desktop_test(**kwargs)
|