Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 05:41:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out 7 changesets (bug 1181516, bug 1198257, bug 1193257, bug 1194166, bug 1193223, bug 1193224, bug 1181520) for test bustages on Android on a CLOSED TREE

Browse Source 
Backed out changeset efe681ac063f (bug 1194166)
Backed out changeset bfcbce193ccf (bug 1193257)
Backed out changeset bb035419fd9c (bug 1193224)
Backed out changeset eb2d48b71c37 (bug 1198257)
Backed out changeset 0d6532c1e99e (bug 1193223)
Backed out changeset da3dd2f650d0 (bug 1181516)
Backed out changeset 29c33bbaac01 (bug 1181520)
This commit is contained in:
Carsten "Tomcat" Book 2015-09-11 14:18:09 +02:00
parent 2ed00693e3
commit b3ee47117c
31 changed files with 1935 additions and 1884 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
layout/tools/reftest/Makefile.in
View File

@ -7,7 +7,6 @@ _DEST_DIR = $(DEPTH)/_tests/reftest
_HARNESS_FILES = \
$(srcdir)/runreftest.py \
$(srcdir)/reftestcommandline.py \
$(srcdir)/remotereftest.py \
$(srcdir)/runreftestb2g.py \
$(srcdir)/b2g_desktop.py \

23
layout/tools/reftest/b2g_desktop.py
View File

@ -9,7 +9,7 @@ import sys
here = os.path.abspath(os.path.dirname(__file__))
from runreftest import RefTest
from runreftest import RefTest, ReftestOptions
from marionette_driver import expected
from marionette_driver.by import By
@ -51,10 +51,12 @@ class B2GDesktopReftest(RefTest):
f.close()
self.marionette.execute_script(self.test_script)
def run_tests(self, tests, options):
manifests = self.resolver.resolveManifests(options, tests)
def run_tests(self, test_path, options):
reftestlist = self.getManifestPath(test_path)
if not reftestlist.startswith('file://'):
reftestlist = 'file://%s' % reftestlist
self.profile = self.create_profile(options, manifests,
self.profile = self.create_profile(options, reftestlist,
profile_to_clone=options.profile)
env = self.buildBrowserEnv(options, self.profile.profile)
kp_kwargs = { 'processOutputLine': [self._on_output],
@ -105,8 +107,8 @@ class B2GDesktopReftest(RefTest):
log.info("%s | Running tests: end.", os.path.basename(__file__))
return status
def create_profile(self, options, manifests, profile_to_clone=None):
profile = RefTest.createReftestProfile(self, options, manifests,
def create_profile(self, options, reftestlist, profile_to_clone=None):
profile = RefTest.createReftestProfile(self, options, reftestlist,
profile_to_clone=profile_to_clone)
prefs = {}
@ -134,6 +136,7 @@ class B2GDesktopReftest(RefTest):
prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
prefs["reftest.browser.iframe.enabled"] = False
prefs["reftest.remote"] = False
prefs["reftest.uri"] = "%s" % reftestlist
# Set a future policy version to avoid the telemetry prompt.
prefs["toolkit.telemetry.prompted"] = 999
prefs["toolkit.telemetry.notifiedOptOut"] = 999
@ -189,7 +192,7 @@ class MuletReftest(B2GDesktopReftest):
Wait(self.marionette, timeout).until(expected.element_present(
By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
def run_desktop_reftests(parser, options):
def run_desktop_reftests(parser, options, args):
marionette_args = {}
if options.marionette:
host, port = options.marionette.split(':')
@ -201,7 +204,9 @@ def run_desktop_reftests(parser, options):
else:
reftest = B2GDesktopReftest(marionette_args)
parser.validate(options, reftest)
options = ReftestOptions.verifyCommonOptions(parser, options, reftest)
if options == None:
sys.exit(1)
# add a -bin suffix if b2g-bin exists, but just b2g was specified
if options.app[-4:] != '-bin':
@ -214,4 +219,4 @@ def run_desktop_reftests(parser, options):
if options.desktop and not options.profile:
raise Exception("must specify --profile when specifying --desktop")
sys.exit(reftest.run_tests(options.tests, options))
sys.exit(reftest.run_tests(args[0], options))

478
layout/tools/reftest/mach_commands.py
View File

@ -23,7 +23,8 @@ from mach.decorators import (
Command,
)
import reftestcommandline
DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
ADB_NOT_FOUND = '''
The %s command requires the adb binary to be on your path.
@ -80,10 +81,43 @@ class ReftestRunner(MozbuildObject):
self.tests_dir = os.path.join(self.topobjdir, '_tests')
self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
def _manifest_file(self, suite):
"""Returns the manifest file used for a given test suite."""
files = {
'reftest': 'reftest.list',
'reftest-ipc': 'reftest.list',
'crashtest': 'crashtests.list',
'crashtest-ipc': 'crashtests.list',
'jstestbrowser': 'jstests.list'
}
assert suite in files
return files[suite]
def _find_manifest(self, suite, test_file):
"""Return a tuple of (manifest-path, filter-string) for running test_file.
test_file can be a relative path to a single test file or manifest from
the top source directory, an absolute path to the same, or a directory
containing a manifest.
"""
assert test_file
path_arg = self._wrap_path_argument(test_file)
relpath = path_arg.relpath()
if os.path.isdir(path_arg.srcdir_path()):
return (mozpath.join(relpath, self._manifest_file(suite)), None)
if relpath.endswith('.list'):
return (relpath, None)
return (self._find_manifest(suite, mozpath.dirname(test_file))[0],
mozpath.basename(test_file))
def _make_shell_string(self, s):
return "'%s'" % re.sub("'", r"'\''", s)
def run_b2g_test(self, b2g_home=None, xre_path=None, **kwargs):
def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,
suite=None, filter=None, **kwargs):
"""Runs a b2g reftest.
filter is a regular expression (in JS syntax, as could be passed to the
@ -96,189 +130,324 @@ class ReftestRunner(MozbuildObject):
suite is the type of reftest to run. It can be one of ('reftest',
'crashtest').
"""
if kwargs["suite"] not in ('reftest', 'crashtest'):
if suite not in ('reftest', 'crashtest'):
raise Exception('None or unrecognized reftest suite type.')
sys.path.insert(0, self.reftest_dir)
test_subdir = {"reftest": os.path.join('layout', 'reftests'),
"crashtest": os.path.join('layout', 'crashtest')}[kwargs["suite"]]
# Find the manifest file
if not kwargs["tests"]:
if not os.path.exists(os.path.join(self.topsrcdir, test_subdir)):
test_file = mozpath.relpath(os.path.abspath(test_subdir),
self.topsrcdir)
kwargs["tests"] = [test_subdir]
if not test_file:
if suite == 'reftest':
test_file = mozpath.join('layout', 'reftests')
elif suite == 'crashtest':
test_file = mozpath.join('testing', 'crashtest')
if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
test_file = mozpath.relpath(os.path.abspath(test_file),
self.topsrcdir)
(manifest, single_file_filter) = self._find_manifest(suite, test_file)
if not os.path.exists(mozpath.join(self.topsrcdir, manifest)):
raise Exception('No manifest file was found at %s.' % manifest)
if single_file_filter:
if filter:
raise Exception('Cannot run single files in conjunction with --filter')
filter = single_file_filter
# Need to chdir to reftest_dir otherwise imports fail below.
os.chdir(self.reftest_dir)
# The imp module can spew warnings if the modules below have
# already been imported, ignore them.
with warnings.catch_warnings():
warnings.simplefilter('ignore')
import imp
path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
with open(path, 'r') as fh:
imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
import reftest
# Set up the reftest options.
parser = reftest.B2GOptions()
options, args = parser.parse_args([])
# Tests need to be served from a subdirectory of the server. Symlink
# topsrcdir here to get around this.
tests = os.path.join(self.reftest_dir, 'tests')
if not os.path.isdir(tests):
os.symlink(self.topsrcdir, tests)
args.insert(0, os.path.join('tests', manifest))
for i, path in enumerate(kwargs["tests"]):
# Non-absolute paths are relative to the packaged directory, which
# has an extra tests/ at the start
if os.path.exists(os.path.abspath(path)):
path = os.path.relpath(path, os.path.join(self.topsrcdir))
kwargs["tests"][i] = os.path.join('tests', path)
for k, v in kwargs.iteritems():
setattr(options, k, v)
if conditions.is_b2g_desktop(self):
return self.run_b2g_desktop(**kwargs)
if self.substs.get('ENABLE_MARIONETTE') != '1':
print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
self.mozconfig['path']))
return 1
return self.run_b2g_remote(b2g_home, xre_path, **kwargs)
def run_b2g_desktop(self, **kwargs):
if self.substs.get('ENABLE_MARIONETTE') != '1':
print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
self.mozconfig['path']))
return 1
if not kwargs["profile"]:
gaia_profile = os.environ.get('GAIA_PROFILE')
if not gaia_profile:
options.profile = options.profile or os.environ.get('GAIA_PROFILE')
if not options.profile:
print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')
return 1
kwargs["profile"] = gaia_profile
if os.path.isfile(os.path.join(options.profile, 'extensions', \
'httpd@gaiamobile.org')):
print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
options.profile))
return 1
if os.path.isfile(os.path.join(kwargs["profile"], 'extensions',
'httpd@gaiamobile.org')):
print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
kwargs["profile"]))
return 1
options.desktop = True
options.app = self.get_binary_path()
if options.oop:
options.browser_arg = '-oop'
if not options.app.endswith('-bin'):
options.app = '%s-bin' % options.app
if not os.path.isfile(options.app):
options.app = options.app[:-len('-bin')]
kwargs["desktop"] = True
kwargs["app"] = self.get_binary_path()
if kwargs["oop"]:
options.browser_arg = '-oop'
if not kwargs["app"].endswith('-bin'):
kwargs["app"] = '%s-bin' % options.app
if not os.path.isfile(kwargs["app"]):
options.app = kwargs["app"][:-len('-bin')]
return reftest.run_desktop_reftests(parser, options, args)
return runreftestb2g.run(**kwargs)
def run_b2g_remote(self, b2g_home, xre_path, **kwargs):
import runreftestb2g
try:
which.which('adb')
except which.WhichError:
# TODO Find adb automatically if it isn't on the path
raise Exception(ADB_NOT_FOUND % ('%s-remote' % kwargs["suite"], b2g_home))
raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
kwargs["b2gPath"] = b2g_home
kwargs["logdir"] = self.reftest_dir
kwargs["httpdPath"] = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
kwargs["xrePath"] = xre_path
kwargs["ignoreWindowSize"] = True
options.b2gPath = b2g_home
options.logdir = self.reftest_dir
options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
options.xrePath = xre_path
options.ignoreWindowSize = True
options.filter = filter
# Don't enable oop for crashtest until they run oop in automation
if kwargs["suite"] == 'reftest':
kwargs["oop"] = True
if suite == 'reftest':
options.oop = True
return runreftestb2g.run_remote(**kwargs)
return reftest.run_remote_reftests(parser, options, args)
def run_desktop_test(self, **kwargs):
"""Runs a reftest."""
import runreftest
def run_desktop_test(self, test_file=None, filter=None, suite=None,
debugger=None, debugger_args=None, parallel=False, shuffle=False,
e10s=False, extraPrefs=None, this_chunk=None, total_chunks=None):
"""Runs a reftest.
if kwargs["suite"] not in ('reftest', 'crashtest', 'jstestbrowser'):
test_file is a path to a test file. It can be a relative path from the
top source directory, an absolute filename, or a directory containing
test files.
filter is a regular expression (in JS syntax, as could be passed to the
RegExp constructor) to select which reftests to run from the manifest.
suite is the type of reftest to run. It can be one of ('reftest',
'crashtest', 'jstestbrowser').
debugger is the program name (in $PATH) or the full path of the
debugger to run.
debugger_args are the arguments passed to the debugger.
parallel indicates whether tests should be run in parallel or not.
shuffle indicates whether to run tests in random order.
"""
if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc', 'jstestbrowser'):
raise Exception('None or unrecognized reftest suite type.')
default_manifest = {
"reftest": (self.topsrcdir, "layout", "reftests", "reftest.list"),
"crashtest": (self.topsrcdir, "testing", "crashtest", "crashtests.list"),
"jstestbrowser": (self.topobjdir, "dist", "test-stage", "jsreftest", "tests",
"jstests.list")
}
env = {}
extra_args = []
kwargs["extraProfileFiles"] = [os.path.join(self.topobjdir, "dist", "plugins")]
kwargs["symbolsPath"] = os.path.join(self.topobjdir, "crashreporter-symbols")
if test_file:
(path, single_file_filter) = self._find_manifest(suite, test_file)
if not os.path.exists(mozpath.join(self.topsrcdir, path)):
raise Exception('No manifest file was found at %s.' % path)
if single_file_filter:
if filter:
raise Exception('Cannot run single files in conjunction with --filter')
filter = single_file_filter
env[b'TEST_PATH'] = path
if filter:
extra_args.extend(['--filter', self._make_shell_string(filter)])
if not kwargs["tests"]:
kwargs["tests"] = [os.path.join(*default_manifest[kwargs["suite"]])]
pass_thru = False
if kwargs["suite"] == "jstestbrowser":
kwargs["extraProfileFiles"].append(os.path.join(self.topobjdir, "dist",
"test-stage", "jsreftest",
"tests", "user.js"))
if debugger:
extra_args.append('--debugger=\'%s\'' % debugger)
pass_thru = True
if debugger_args:
# Use _make_shell_string (which quotes) so that we
# handle multiple args being passed to the debugger.
extra_args.extend(['--debugger-args', self._make_shell_string(debugger_args)])
else:
if debugger_args:
print("--debugger-args passed, but no debugger specified.")
return 1
if not kwargs["runTestsInParallel"]:
kwargs["logFile"] = "%s.log" % kwargs["suite"]
if parallel:
extra_args.append('--run-tests-in-parallel')
#Remove the stdout handler from the internal logger and let mach deal with it
runreftest.log.removeHandler(runreftest.log.handlers[0])
self.log_manager.enable_unstructured()
rv = runreftest.run(**kwargs)
self.log_manager.disable_unstructured()
if shuffle:
extra_args.append('--shuffle')
return rv
if e10s:
extra_args.append('--e10s')
if extraPrefs:
for pref in extraPrefs:
extra_args.extend(['--setpref', pref])
if this_chunk:
extra_args.append('--this-chunk=%s' % this_chunk)
if total_chunks:
extra_args.append('--total-chunks=%s' % total_chunks)
if extra_args:
args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]
args.extend(extra_args)
env[b'EXTRA_TEST_ARGS'] = ' '.join(args)
# TODO hook up harness via native Python
return self._run_make(directory='.', target=suite, append_env=env,
pass_thru=pass_thru, ensure_exit_code=False)
def process_test_objects(kwargs):
"""|mach test| works by providing a test_objects argument, from
which the test path must be extracted and converted into a normal
reftest tests argument."""
def ReftestCommand(func):
"""Decorator that adds shared command arguments to reftest commands."""
if "test_objects" in kwargs:
if kwargs["tests"] is None:
kwargs["tests"] = []
kwargs["tests"].extend(item["path"] for item in kwargs["test_objects"])
del kwargs["test_objects"]
debugger = CommandArgument('--debugger', metavar='DEBUGGER',
help=DEBUGGER_HELP)
func = debugger(func)
debugger_args = CommandArgument('--debugger-args', metavar='DEBUGGER_ARGS',
help='Arguments to pass to the debugger.')
func = debugger_args(func)
flter = CommandArgument('--filter', metavar='REGEX',
help='A JS regular expression to match test URLs against, to select '
'a subset of tests to run.')
func = flter(func)
path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
help='Reftest manifest file, or a directory in which to select '
'reftest.list. If omitted, the entire test suite is executed.')
func = path(func)
parallel = CommandArgument('--parallel', action='store_true',
help='Run tests in parallel.')
func = parallel(func)
shuffle = CommandArgument('--shuffle', action='store_true',
help='Run tests in random order.')
func = shuffle(func)
e10s = CommandArgument('--e10s', action='store_true',
help='Use content processes.')
func = e10s(func)
extraPrefs = CommandArgument('--setpref', action='append',
default=[], dest='extraPrefs', metavar='PREF=VALUE',
help='Set prefs in the reftest profile.')
func = extraPrefs(func)
totalChunks = CommandArgument('--total-chunks',
help = 'How many chunks to split the tests up into.')
func = totalChunks(func)
thisChunk = CommandArgument('--this-chunk',
help = 'Which chunk to run between 1 and --total-chunks.')
func = thisChunk(func)
return func
def B2GCommand(func):
"""Decorator that adds shared command arguments to b2g reftest commands."""
busybox = CommandArgument('--busybox', default=None,
help='Path to busybox binary to install on device')
func = busybox(func)
logdir = CommandArgument('--logdir', default=None,
help='directory to store log files')
func = logdir(func)
sdcard = CommandArgument('--sdcard', default="10MB",
help='Define size of sdcard: 1MB, 50MB...etc')
func = sdcard(func)
emulator_res = CommandArgument('--emulator-res', default='800x1000',
help='Emulator resolution of the format \'<width>x<height>\'')
func = emulator_res(func)
marionette = CommandArgument('--marionette', default=None,
help='host:port to use when connecting to Marionette')
func = marionette(func)
totalChunks = CommandArgument('--total-chunks', dest='totalChunks',
type = int,
help = 'How many chunks to split the tests up into.')
func = totalChunks(func)
thisChunk = CommandArgument('--this-chunk', dest='thisChunk',
type = int,
help = 'Which chunk to run between 1 and --total-chunks.')
func = thisChunk(func)
flter = CommandArgument('--filter', metavar='REGEX',
help='A JS regular expression to match test URLs against, to select '
'a subset of tests to run.')
func = flter(func)
oop = CommandArgument('--enable-oop', action='store_true', dest='oop',
help = 'Run tests in out-of-process mode.')
func = oop(func)
path = CommandArgument('test_file', default=None, nargs='?',
metavar='TEST',
help='Test to run. Can be specified as a single file, a ' \
'directory, or omitted. If omitted, the entire test suite is ' \
'executed.')
func = path(func)
return func
@CommandProvider
class MachCommands(MachCommandBase):
@Command('reftest',
category='testing',
description='Run reftests (layout and graphics correctness).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_reftest(self, **kwargs):
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
@Command('reftest', category='testing', description='Run reftests (layout and graphics correctness).')
@ReftestCommand
def run_reftest(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest', **kwargs)
@Command('jstestbrowser',
category='testing',
description='Run js/src/tests in the browser.',
parser=reftestcommandline.DesktopArgumentsParser)
def run_jstestbrowser(self, **kwargs):
self._mach_context.commands.dispatch("build",
self._mach_context,
what=["stage-jstests"])
kwargs["suite"] = "jstestbrowser"
return self._run_reftest(**kwargs)
@Command('jstestbrowser', category='testing',
description='Run js/src/tests in the browser.')
@ReftestCommand
def run_jstestbrowser(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='jstestbrowser', **kwargs)
@Command('reftest-ipc',
category='testing',
description='Run IPC reftests (layout and graphics correctness, separate process).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_ipc(self, **kwargs):
kwargs["ipc"] = True
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
@Command('reftest-ipc', category='testing',
description='Run IPC reftests (layout and graphics correctness, separate process).')
@ReftestCommand
def run_ipc(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)
@Command('crashtest',
category='testing',
description='Run crashtests (Check if crashes on a page).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_crashtest(self, **kwargs):
kwargs["suite"] = "crashtest"
return self._run_reftest(**kwargs)
@Command('crashtest', category='testing',
description='Run crashtests (Check if crashes on a page).')
@ReftestCommand
def run_crashtest(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='crashtest', **kwargs)
@Command('crashtest-ipc',
category='testing',
description='Run IPC crashtests (Check if crashes on a page, separate process).',
parser=reftestcommandline.DesktopArgumentsParser)
def run_crashtest_ipc(self, **kwargs):
kwargs["ipc"] = True
kwargs["suite"] = "crashtest"
return self._run_reftest(**kwargs)
@Command('crashtest-ipc', category='testing',
description='Run IPC crashtests (Check if crashes on a page, separate process).')
@ReftestCommand
def run_crashtest_ipc(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
def _run_reftest(self, **kwargs):
process_test_objects(kwargs)
def _run_reftest(self, test_file=None, suite=None, **kwargs):
reftest = self._spawn(ReftestRunner)
return reftest.run_desktop_test(**kwargs)
return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
# TODO For now b2g commands will only work with the emulator,
@ -297,31 +466,27 @@ class B2GCommands(MachCommandBase):
setattr(self, attr, getattr(context, attr, None))
@Command('reftest-remote', category='testing',
description='Run a remote reftest (b2g layout and graphics correctness, remote device).',
conditions=[conditions.is_b2g, is_emulator],
parser=reftestcommandline.B2GArgumentParser)
def run_reftest_remote(self, **kwargs):
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
description='Run a remote reftest (b2g layout and graphics correctness, remote device).',
conditions=[conditions.is_b2g, is_emulator])
@B2GCommand
def run_reftest_remote(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest', **kwargs)
@Command('reftest-b2g-desktop', category='testing',
description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).',
conditions=[conditions.is_b2g_desktop],
parser=reftestcommandline.B2GArgumentParser)
def run_reftest_b2g_desktop(self, **kwargs):
kwargs["suite"] = "reftest"
return self._run_reftest(**kwargs)
description='Run a b2g desktop reftest (b2g desktop layout and graphics correctness).',
conditions=[conditions.is_b2g_desktop])
@B2GCommand
def run_reftest_b2g_desktop(self, test_file, **kwargs):
return self._run_reftest(test_file, suite='reftest', **kwargs)
@Command('crashtest-remote', category='testing',
description='Run a remote crashtest (Check if b2g crashes on a page, remote device).',
conditions=[conditions.is_b2g, is_emulator],
parser=reftestcommandline.B2GArgumentParser)
description='Run a remote crashtest (Check if b2g crashes on a page, remote device).',
conditions=[conditions.is_b2g, is_emulator])
@B2GCommand
def run_crashtest_remote(self, test_file, **kwargs):
kwargs["suite"] = "crashtest"
return self._run_reftest(**kwargs)
return self._run_reftest(test_file, suite='crashtest', **kwargs)
def _run_reftest(self, **kwargs):
process_test_objects(kwargs)
def _run_reftest(self, test_file=None, suite=None, **kwargs):
if self.device_name:
if self.device_name.startswith('emulator'):
emulator = 'arm'
@ -330,4 +495,5 @@ class B2GCommands(MachCommandBase):
kwargs['emulator'] = emulator
reftest = self._spawn(ReftestRunner)
return reftest.run_b2g_test(self.b2g_home, self.xre_path, **kwargs)
return reftest.run_b2g_test(self.b2g_home, self.xre_path,
test_file, suite=suite, **kwargs)

33
layout/tools/reftest/reftest-cmdline.js
View File

@ -21,6 +21,37 @@ RefTestCmdLineHandler.prototype =
/* nsICommandLineHandler */
handle : function handler_handle(cmdLine) {
var args = { };
args.wrappedJSObject = args;
try {
var uristr = cmdLine.handleFlagWithParam("reftest", false);
if (uristr == null)
return;
try {
args.uri = cmdLine.resolveURI(uristr).spec;
}
catch (e) {
return;
}
}
catch (e) {
cmdLine.handleFlag("reftest", true);
}
try {
var nocache = cmdLine.handleFlag("reftestnocache", false);
args.nocache = nocache;
}
catch (e) {
}
try {
var skipslowtests = cmdLine.handleFlag("reftestskipslowtests", false);
args.skipslowtests = skipslowtests;
}
catch (e) {
}
/* Ignore the platform's online/offline status while running reftests. */
var ios = Components.classes["@mozilla.org/network/io-service;1"]
.getService(Components.interfaces.nsIIOService2);
@ -47,7 +78,7 @@ RefTestCmdLineHandler.prototype =
function loadReftests() {
wwatch.openWindow(null, "chrome://reftest/content/reftest.xul", "_blank",
"chrome,dialog=no,all", {});
"chrome,dialog=no,all", args);
}
var remote = false;

99
layout/tools/reftest/reftest.js
View File

@ -48,7 +48,7 @@ var gShuffle = false;
var gTotalChunks = 0;
var gThisChunk = 0;
var gContainingWindow = null;
var gURLFilterRegex = {};
var gURLFilterRegex = null;
const FOCUS_FILTER_ALL_TESTS = "all";
const FOCUS_FILTER_NEEDS_FOCUS_TESTS = "needs-focus";
const FOCUS_FILTER_NON_NEEDS_FOCUS_TESTS = "non-needs-focus";
@ -69,7 +69,6 @@ var gCanvas1, gCanvas2;
// RecordResult.
var gCurrentCanvas = null;
var gURLs;
var gManifestsLoaded = {};
// Map from URI spec to the number of times it remains to be used
var gURIUseCounts;
// Map from URI spec to the canvas rendered for that URI
@ -390,6 +389,10 @@ function InitAndStartRefTests()
gThisChunk = 0;
}
try {
gURLFilterRegex = new RegExp(prefs.getCharPref("reftest.filter"));
} catch(e) {}
try {
gFocusFilterMode = prefs.getCharPref("reftest.focusFilterMode");
} catch(e) {}
@ -446,7 +449,8 @@ function Shuffle(array)
function StartTests()
{
var manifests;
var uri;
#if BOOTSTRAP
/* These prefs are optional, so we don't need to spit an error to the log */
try {
var prefs = Components.classes["@mozilla.org/preferences-service;1"].
@ -473,32 +477,41 @@ function StartTests()
gRunSlowTests = false;
}
try {
uri = prefs.getCharPref("reftest.uri");
} catch(e) {
uri = "";
}
if (uri == "") {
gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.uri pref. Please ensure your profile is setup properly\n");
DoneTests();
}
#else
try {
// Need to read the manifest once we have gHttpServerPort..
var args = window.arguments[0].wrappedJSObject;
if ("nocache" in args && args["nocache"])
gNoCanvasCache = true;
if ("skipslowtests" in args && args.skipslowtests)
gRunSlowTests = false;
uri = args.uri;
} catch (e) {
++gTestResults.Exception;
gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | EXCEPTION: " + ex + "\n");
DoneTests();
}
#endif
if (gShuffle) {
gNoCanvasCache = true;
}
gURLs = [];
try {
var manifests = JSON.parse(prefs.getCharPref("reftest.manifests"));
gURLFilterRegex = manifests[null];
} catch(e) {
gDumpLog("REFTEST TEST-UNEXPECTED-FAIL | | Unable to find reftest.manifests pref. Please ensure your profile is setup properly\n");
DoneTests();
}
try {
var globalFilter = manifests.hasOwnProperty("") ? new RegExp(manifests[""]) : null;
var manifestURLs = Object.keys(manifests);
// Ensure we read manifests from higher up the directory tree first so that we
// process includes before reading the included manifest again
manifestURLs.sort(function(a,b) {return a.length - b.length})
manifestURLs.forEach(function(manifestURL) {
gDumpLog("Readings manifest" + manifestURL + "\n");
var filter = manifests[manifestURL] ? new RegExp(manifests[manifestURL]) : null;
ReadTopManifest(manifestURL, [globalFilter, filter, false]);
});
ReadTopManifest(uri);
BuildUseCounts();
// Filter tests which will be skipped to get a more even distribution when chunking
@ -751,25 +764,18 @@ function AddPrefSettings(aWhere, aPrefName, aPrefValExpression, aSandbox, aTestP
return true;
}
function ReadTopManifest(aFileURL, aFilter)
function ReadTopManifest(aFileURL)
{
gURLs = new Array();
var url = gIOService.newURI(aFileURL, null, null);
if (!url)
throw "Expected a file or http URL for the manifest.";
ReadManifest(url, EXPECTED_PASS, aFilter);
ReadManifest(url, EXPECTED_PASS);
}
function AddTestItem(aTest, aFilter)
function AddTestItem(aTest)
{
if (!aFilter)
aFilter = [null, [], false];
globalFilter = aFilter[0];
manifestFilter = aFilter[1];
invertManifest = aFilter[2];
if ((globalFilter && !globalFilter.test(aTest.url1.spec)) ||
(manifestFilter &&
!(invertManifest ^ manifestFilter.test(aTest.url1.spec))))
if (gURLFilterRegex && !gURLFilterRegex.test(aTest.url1.spec))
return;
if (gFocusFilterMode == FOCUS_FILTER_NEEDS_FOCUS_TESTS &&
!aTest.needsFocus)
@ -782,19 +788,8 @@ function AddTestItem(aTest, aFilter)
// Note: If you materially change the reftest manifest parsing,
// please keep the parser in print-manifest-dirs.py in sync.
function ReadManifest(aURL, inherited_status, aFilter)
function ReadManifest(aURL, inherited_status)
{
// Ensure each manifest is only read once. This assumes that manifests that are
// included with an unusual inherited_status or filters will be read via their
// include before they are read directly in the case of a duplicate
if (gManifestsLoaded.hasOwnProperty(aURL.spec)) {
if (gManifestsLoaded[aURL.spec] === null)
return;
else
aFilter = [aFilter[0], aFilter[1], true];
}
gManifestsLoaded[aURL.spec] = aFilter[1];
var secMan = CC[NS_SCRIPTSECURITYMANAGER_CONTRACTID]
.getService(CI.nsIScriptSecurityManager);
@ -1010,7 +1005,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
var incURI = gIOService.newURI(items[1], null, listURL);
secMan.checkLoadURIWithPrincipal(principal, incURI,
CI.nsIScriptSecurityManager.DISALLOW_SCRIPT);
ReadManifest(incURI, expected_status, aFilter);
ReadManifest(incURI, expected_status);
} else if (items[0] == TYPE_LOAD) {
if (items.length != 2)
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to load";
@ -1040,7 +1035,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
fuzzyMaxPixels: fuzzy_max_pixels,
url1: testURI,
url2: null,
chaosMode: chaosMode }, aFilter);
chaosMode: chaosMode });
} else if (items[0] == TYPE_SCRIPT) {
if (items.length != 2)
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to script";
@ -1067,7 +1062,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
fuzzyMaxPixels: fuzzy_max_pixels,
url1: testURI,
url2: null,
chaosMode: chaosMode }, aFilter);
chaosMode: chaosMode });
} else if (items[0] == TYPE_REFTEST_EQUAL || items[0] == TYPE_REFTEST_NOTEQUAL) {
if (items.length != 3)
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": incorrect number of arguments to " + items[0];
@ -1097,7 +1092,7 @@ function ReadManifest(aURL, inherited_status, aFilter)
fuzzyMaxPixels: fuzzy_max_pixels,
url1: testURI,
url2: refURI,
chaosMode: chaosMode }, aFilter);
chaosMode: chaosMode });
} else {
throw "Error in manifest file " + aURL.spec + " line " + lineNo + ": unknown test type " + items[0];
}

740
layout/tools/reftest/reftestcommandline.py
View File

@ -1,740 +0,0 @@
import argparse
import os
from collections import OrderedDict
from urlparse import urlparse
here = os.path.abspath(os.path.dirname(__file__))
class ReftestArgumentsParser(argparse.ArgumentParser):
def __init__(self, **kwargs):
super(ReftestArgumentsParser, self).__init__(**kwargs)
# Try to import a MozbuildObject. Success indicates that we are
# running from a source tree. This allows some defaults to be set
# from the source tree.
try:
from mozbuild.base import MozbuildObject
self.build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
self.build_obj = None
self.add_argument("--xre-path",
action="store",
type=str,
dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
self.add_argument("--symbols-path",
action="store",
type=str,
dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
self.add_argument("--debugger",
action="store",
dest="debugger",
help="use the given debugger to launch the application")
self.add_argument("--debugger-args",
action="store",
dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
self.add_argument("--debugger-interactive",
action="store_true",
dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_argument("--appname",
action="store",
type=str,
dest="app",
default=None,
help="absolute path to application, overriding default")
self.add_argument("--extra-profile-file",
action="append",
dest="extraProfileFiles",
default=[],
help="copy specified files/dirs to testing profile")
self.add_argument("--timeout",
action="store",
dest="timeout",
type=int,
default=5 * 60, # 5 minutes per bug 479518
help="reftest will timeout in specified number of seconds. [default %(default)s].")
self.add_argument("--leak-threshold",
action="store",
type=int,
dest="defaultLeakThreshold",
default=0,
help="fail if the number of bytes leaked in default "
"processes through refcounted objects (or bytes "
"in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
"is greater than the given number")
self.add_argument("--utility-path",
action="store",
type=str,
dest="utilityPath",
default="bindir",
help="absolute path to directory containing utility "
"programs (xpcshell, ssltunnel, certutil)")
self.add_argument("--total-chunks",
type=int,
dest="totalChunks",
help="how many chunks to split the tests up into")
self.add_argument("--this-chunk",
type=int,
dest="thisChunk",
help="which chunk to run between 1 and --total-chunks")
self.add_argument("--log-file",
action="store",
type=str,
dest="logFile",
default=None,
help="file to log output to in addition to stdout")
self.add_argument("--skip-slow-tests",
dest="skipSlowTests",
action="store_true",
default=False,
help="skip tests marked as slow when running")
self.add_argument("--ignore-window-size",
dest="ignoreWindowSize",
action="store_true",
default=False,
help="ignore the window size, which may cause spurious failures and passes")
self.add_argument("--install-extension",
action="append",
dest="extensionsToInstall",
default=[],
help="install the specified extension in the testing profile. "
"The extension file's name should be <id>.xpi where <id> is "
"the extension's id as indicated in its install.rdf. "
"An optional path can be specified too.")
self.add_argument("--setenv",
action="append",
type=str,
default=[],
dest="environment",
metavar="NAME=VALUE",
help="sets the given variable in the application's "
"environment")
self.add_argument("--filter",
action="store",
type=str,
dest="filter",
help="specifies a regular expression (as could be passed to the JS "
"RegExp constructor) to test against URLs in the reftest manifest; "
"only test items that have a matching test URL will be run.")
self.add_argument("--shuffle",
action="store_true",
default=False,
dest="shuffle",
help="run reftests in random order")
self.add_argument("--focus-filter-mode",
action="store",
type=str,
dest="focusFilterMode",
default="all",
help="filters tests to run by whether they require focus. "
"Valid values are `all', `needs-focus', or `non-needs-focus'. "
"Defaults to `all'.")
self.add_argument("--e10s",
action="store_true",
default=False,
dest="e10s",
help="enables content processes")
self.add_argument("--setpref",
action="append",
type=str,
default=[],
dest="extraPrefs",
metavar="PREF=VALUE",
help="defines an extra user preference")
self.add_argument("--reftest-extension-path",
action="store",
dest="reftestExtensionPath",
help="Path to the reftest extension")
self.add_argument("--special-powers-extension-path",
action="store",
dest="specialPowersExtensionPath",
help="Path to the special powers extension")
self.add_argument("--suite",
choices=["reftest", "crashtest", "jstestbrowser"],
default=None,
help=argparse.SUPPRESS)
self.add_argument("tests",
metavar="TEST_PATH",
nargs="*",
help="Path to test file, manifest file, or directory containing tests")
def get_ip(self):
import moznetwork
if os.name != "nt":
return moznetwork.get_ip()
else:
self.error(
"ERROR: you must specify a --remote-webserver=<ip address>\n")
def set_default_suite(self, options):
manifests = OrderedDict([("reftest.list", "reftest"),
("crashtests.list", "crashtest"),
("jstests.list", "jstestbrowser")])
for test_path in options.tests:
file_name = os.path.basename(test_path)
if file_name in manifests:
options.suite = manifests[file_name]
return
for test_path in options.tests:
for manifest_file, suite in manifests.iteritems():
if os.path.exists(os.path.join(test_path, manifest_file)):
options.suite = suite
return
self.error("Failed to determine test suite; supply --suite to set this explicitly")
def validate(self, options, reftest):
import sys
if not options.tests:
# Can't just set this in the argument parser because mach will set a default
self.error("Must supply at least one path to a manifest file, test directory, or test file to run.")
if options.suite is None:
self.set_default_suite(options)
if options.totalChunks is not None and options.thisChunk is None:
self.error(
"thisChunk must be specified when totalChunks is specified")
if options.totalChunks:
if not 1 <= options.thisChunk <= options.totalChunks:
self.error("thisChunk must be between 1 and totalChunks")
if options.logFile:
options.logFile = reftest.getFullPath(options.logFile)
if options.xrePath is not None:
if not os.access(options.xrePath, os.F_OK):
self.error("--xre-path '%s' not found" % options.xrePath)
if not os.path.isdir(options.xrePath):
self.error("--xre-path '%s' is not a directory" %
options.xrePath)
options.xrePath = reftest.getFullPath(options.xrePath)
if options.reftestExtensionPath is None:
if self.build_obj is not None:
options.reftestExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
"reftest", "reftest")
else:
options.reftestExtensionPath = os.path.join(here, "reftest")
if (options.specialPowersExtensionPath is None and
options.suite in ["crashtest", "jstestbrowser"]):
if self.build_obj is not None:
options.specialPowersExtensionPath = os.path.join(self.build_obj.topobjdir, "_tests",
"reftest", "specialpowers")
else:
options.specialPowersExtensionPath = os.path.join(
here, "specialpowers")
options.leakThresholds = {
"default": options.defaultLeakThreshold,
"tab": 5000, # See dependencies of bug 1051230.
}
class DesktopArgumentsParser(ReftestArgumentsParser):
def __init__(self, **kwargs):
super(DesktopArgumentsParser, self).__init__(**kwargs)
self.add_argument("--run-tests-in-parallel",
action="store_true",
default=False,
dest="runTestsInParallel",
help="run tests in parallel if possible")
self.add_argument("--ipc",
action="store_true",
default=False,
help="Run in out-of-processes mode")
def _prefs_oop(self):
import mozinfo
prefs = ["layers.async-pan-zoom.enabled=true",
"browser.tabs.remote.autostart=true"]
if mozinfo.os == "win":
prefs.append("layers.acceleration.disabled=true")
return prefs
def _prefs_gpu(self):
if mozinfo.os != "win":
return ["layers.acceleration.force-enabled=true"]
return []
def validate(self, options, reftest):
super(DesktopArgumentsParser, self).validate(options, reftest)
if options.ipc:
for item in self._prefs_oop():
if item not in options.extraPrefs:
options.extraPrefs.append(item)
if options.runTestsInParallel:
if options.logFile is not None:
self.error("cannot specify logfile with parallel tests")
if options.totalChunks is not None or options.thisChunk is not None:
self.error(
"cannot specify thisChunk or totalChunks with parallel tests")
if options.focusFilterMode != "all":
self.error("cannot specify focusFilterMode with parallel tests")
if options.debugger is not None:
self.error("cannot specify a debugger with parallel tests")
if not options.tests:
self.error("No test files specified.")
if options.app is None:
bin_dir = (self.build_obj.get_binary_path() if
self.build_obj and self.build_obj.substs[
'MOZ_BUILD_APP'] != 'mobile/android'
else None)
if bin_dir:
options.app = bin_dir
else:
self.error(
"could not find the application path, --appname must be specified")
options.app = reftest.getFullPath(options.app)
if not os.path.exists(options.app):
self.error("""Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/reftest/runreftest.py?"""
% {"app": options.app})
if options.xrePath is None:
options.xrePath = os.path.dirname(options.app)
if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
options.symbolsPath = reftest.getFullPath(options.symbolsPath)
options.utilityPath = reftest.getFullPath(options.utilityPath)
class B2GArgumentParser(ReftestArgumentsParser):
def __init__(self, **kwargs):
super(B2GArgumentParser, self).__init__(**kwargs)
self.add_argument("--browser-arg",
action="store",
type=str,
dest="browser_arg",
help="Optional command-line arg to pass to the browser")
self.add_argument("--b2gpath",
action="store",
type=str,
dest="b2gPath",
help="path to B2G repo or qemu dir")
self.add_argument("--marionette",
action="store",
type=str,
dest="marionette",
help="host:port to use when connecting to Marionette")
self.add_argument("--emulator",
action="store",
type=str,
dest="emulator",
help="Architecture of emulator to use: x86 or arm")
self.add_argument("--emulator-res",
action="store",
type=str,
dest="emulator_res",
help="Emulator resolution of the format '<width>x<height>'")
self.add_argument("--no-window",
action="store_true",
dest="noWindow",
default=False,
help="Pass --no-window to the emulator")
self.add_argument("--adbpath",
action="store",
type=str,
dest="adb_path",
default="adb",
help="path to adb")
self.add_argument("--deviceIP",
action="store",
type=str,
dest="deviceIP",
help="ip address of remote device to test")
self.add_argument("--devicePort",
action="store",
type=str,
dest="devicePort",
default="20701",
help="port of remote device to test")
self.add_argument("--remote-logfile",
action="store",
type=str,
dest="remoteLogFile",
help="Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.")
self.add_argument("--remote-webserver",
action="store",
type=str,
dest="remoteWebServer",
help="ip address where the remote web server is hosted at")
self.add_argument("--http-port",
action="store",
type=str,
dest="httpPort",
help="ip address where the remote web server is hosted at")
self.add_argument("--ssl-port",
action="store",
type=str,
dest="sslPort",
help="ip address where the remote web server is hosted at")
self.add_argument("--pidfile",
action="store",
type=str,
dest="pidFile",
default="",
help="name of the pidfile to generate")
self.add_argument("--gecko-path",
action="store",
type=str,
dest="geckoPath",
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
self.add_argument("--logdir",
action="store",
type=str,
dest="logdir",
help="directory to store log files")
self.add_argument('--busybox',
action='store',
type=str,
dest='busybox',
help="Path to busybox binary to install on device")
self.add_argument("--httpd-path",
action="store",
type=str,
dest="httpdPath",
help="path to the httpd.js file")
self.add_argument("--profile",
action="store",
type=str,
dest="profile",
help="for desktop testing, the path to the "
"gaia profile to use")
self.add_argument("--desktop",
action="store_true",
dest="desktop",
default=False,
help="Run the tests on a B2G desktop build")
self.add_argument("--mulet",
action="store_true",
dest="mulet",
default=False,
help="Run the tests on a B2G desktop build")
self.add_argument("--enable-oop",
action="store_true",
dest="oop",
default=False,
help="Run the tests out of process")
self.set_defaults(remoteTestRoot=None,
logFile="reftest.log",
autorun=True,
closeWhenDone=True,
testPath="")
def validate_remote(self, options, automation):
if not options.app:
options.app = automation.DEFAULT_APP
if not options.remoteTestRoot:
options.remoteTestRoot = automation._devicemanager.deviceRoot + \
"/reftest"
options.remoteProfile = options.remoteTestRoot + "/profile"
productRoot = options.remoteTestRoot + "/" + automation._product
if options.utilityPath is None:
options.utilityPath = productRoot + "/bin"
if not options.httpPort:
options.httpPort = automation.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = automation.DEFAULT_SSL_PORT
if options.remoteWebServer is None:
options.remoteWebServer = self.get_ip()
options.webServer = options.remoteWebServer
if options.geckoPath and not options.emulator:
self.error(
"You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
self.error("You must specify --emulator if you specify --logdir")
if options.remoteLogFile is None:
options.remoteLogFile = "reftest.log"
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as
# above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
options.xrePath = os.path.abspath(options.xrePath)
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from from the xre bundle, as
# required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.xrePath, "components")
return options
class RemoteArgumentsParser(ReftestArgumentsParser):
def __init__(self, **kwargs):
super(RemoteArgumentsParser, self).__init__()
# app, xrePath and utilityPath variables are set in main function
self.set_defaults(logFile="reftest.log",
app="",
xrePath="",
utilityPath="",
localLogName=None)
self.add_argument("--remote-app-path",
action="store",
type=str,
dest="remoteAppPath",
help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
self.add_argument("--deviceIP",
action="store",
type=str,
dest="deviceIP",
help="ip address of remote device to test")
self.add_argument("--deviceSerial",
action="store",
type=str,
dest="deviceSerial",
help="adb serial number of remote device to test")
self.add_argument("--devicePort",
action="store",
type=str,
default="20701",
dest="devicePort",
help="port of remote device to test")
self.add_argument("--remote-product-name",
action="store",
type=str,
dest="remoteProductName",
default="fennec",
help="Name of product to test - either fennec or firefox, defaults to fennec")
self.add_argument("--remote-webserver",
action="store",
type=str,
dest="remoteWebServer",
help="IP Address of the webserver hosting the reftest content")
self.add_argument("--http-port",
action="store",
type=str,
dest="httpPort",
help="port of the web server for http traffic")
self.add_argument("--ssl-port",
action="store",
type=str,
dest="sslPort",
help="Port for https traffic to the web server")
self.add_argument("--remote-logfile",
action="store",
type=str,
dest="remoteLogFile",
default="reftest.log",
help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
self.add_argument("--pidfile",
action="store",
type=str,
dest="pidFile",
default="",
help="name of the pidfile to generate")
self.add_argument("--bootstrap",
action="store_true",
dest="bootstrap",
default=False,
help="test with a bootstrap addon required for native Fennec")
self.add_argument("--dm_trans",
action="store",
type=str,
dest="dm_trans",
default="sut",
help="the transport to use to communicate with device: [adb|sut]; default=sut")
self.add_argument("--remoteTestRoot",
action="store",
type=str,
dest="remoteTestRoot",
help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
self.add_argument("--httpd-path",
action="store",
type=str,
dest="httpdPath",
help="path to the httpd.js file")
def validate_remote(self, options, automation):
# Ensure our defaults are set properly for everything we can infer
if not options.remoteTestRoot:
options.remoteTestRoot = automation._devicemanager.deviceRoot + \
'/reftest'
options.remoteProfile = options.remoteTestRoot + "/profile"
if options.remoteWebServer is None:
options.remoteWebServer = self.get_ip()
# Verify that our remotewebserver is set properly
if options.remoteWebServer == '127.0.0.1':
self.error("ERROR: Either you specified the loopback for the remote webserver or ",
"your local IP cannot be detected. Please provide the local ip in --remote-webserver")
if not options.httpPort:
options.httpPort = automation.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = automation.DEFAULT_SSL_PORT
# One of remoteAppPath (relative path to application) or the app (executable) must be
# set, but not both. If both are set, we destroy the user's selection for app
# so instead of silently destroying a user specificied setting, we
# error.
if options.remoteAppPath and options.app:
self.error(
"ERROR: You cannot specify both the remoteAppPath and the app")
elif options.remoteAppPath:
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif options.app is None:
# Neither remoteAppPath nor app are set -- error
self.error("ERROR: You must specify either appPath or app")
if options.xrePath is None:
self.error(
"ERROR: You must specify the path to the controller xre directory")
else:
# Ensure xrepath is a full path
options.xrePath = os.path.abspath(options.xrePath)
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as
# above.
if options.logFile:
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
if options.pidFile != "":
with open(options.pidFile, 'w') as f:
f.write(str(os.getpid()))
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from hostutils.zip, as required in
# bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.utilityPath, "components")
if not options.ignoreWindowSize:
parts = automation._devicemanager.getInfo(
'screen')['screen'][0].split()
width = int(parts[0].split(':')[1])
height = int(parts[1].split(':')[1])
if (width < 1050 or height < 1050):
self.error("ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (
width, height))

222
layout/tools/reftest/remotereftest.py
View File

@ -9,9 +9,10 @@ import tempfile
import traceback
# We need to know our current directory so that we can serve our test files from it.
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
from runreftest import RefTest, ReftestResolver
from runreftest import RefTest
from runreftest import ReftestOptions
from automation import Automation
import devicemanager
import droid
@ -19,26 +20,159 @@ import mozinfo
import moznetwork
from remoteautomation import RemoteAutomation, fennecLogcatFilters
import reftestcommandline
class RemoteOptions(ReftestOptions):
def __init__(self, automation):
ReftestOptions.__init__(self)
self.automation = automation
class RemoteReftestResolver(ReftestResolver):
def absManifestPath(self, path):
script_abs_path = os.path.join(SCRIPT_DIRECTORY, path)
if os.path.exists(script_abs_path):
rv = script_abs_path
elif os.path.exists(os.path.abspath(path)):
rv = os.path.abspath(path)
defaults = {}
defaults["logFile"] = "reftest.log"
# app, xrePath and utilityPath variables are set in main function
defaults["app"] = ""
defaults["xrePath"] = ""
defaults["utilityPath"] = ""
defaults["runTestsInParallel"] = False
self.add_option("--remote-app-path", action="store",
type = "string", dest = "remoteAppPath",
help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.")
defaults["remoteAppPath"] = None
self.add_option("--deviceIP", action="store",
type = "string", dest = "deviceIP",
help = "ip address of remote device to test")
defaults["deviceIP"] = None
self.add_option("--deviceSerial", action="store",
type = "string", dest = "deviceSerial",
help = "adb serial number of remote device to test")
defaults["deviceSerial"] = None
self.add_option("--devicePort", action="store",
type = "string", dest = "devicePort",
help = "port of remote device to test")
defaults["devicePort"] = 20701
self.add_option("--remote-product-name", action="store",
type = "string", dest = "remoteProductName",
help = "Name of product to test - either fennec or firefox, defaults to fennec")
defaults["remoteProductName"] = "fennec"
self.add_option("--remote-webserver", action="store",
type = "string", dest = "remoteWebServer",
help = "IP Address of the webserver hosting the reftest content")
defaults["remoteWebServer"] = moznetwork.get_ip()
self.add_option("--http-port", action = "store",
type = "string", dest = "httpPort",
help = "port of the web server for http traffic")
defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
self.add_option("--ssl-port", action = "store",
type = "string", dest = "sslPort",
help = "Port for https traffic to the web server")
defaults["sslPort"] = automation.DEFAULT_SSL_PORT
self.add_option("--remote-logfile", action="store",
type = "string", dest = "remoteLogFile",
help = "Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.")
defaults["remoteLogFile"] = None
self.add_option("--pidfile", action = "store",
type = "string", dest = "pidFile",
help = "name of the pidfile to generate")
defaults["pidFile"] = ""
self.add_option("--bootstrap", action="store_true", dest = "bootstrap",
help = "test with a bootstrap addon required for native Fennec")
defaults["bootstrap"] = False
self.add_option("--dm_trans", action="store",
type = "string", dest = "dm_trans",
help = "the transport to use to communicate with device: [adb|sut]; default=sut")
defaults["dm_trans"] = "sut"
self.add_option("--remoteTestRoot", action = "store",
type = "string", dest = "remoteTestRoot",
help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
defaults["remoteTestRoot"] = None
self.add_option("--httpd-path", action = "store",
type = "string", dest = "httpdPath",
help = "path to the httpd.js file")
defaults["httpdPath"] = None
defaults["localLogName"] = None
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options):
if options.runTestsInParallel:
self.error("Cannot run parallel tests here")
# Ensure our defaults are set properly for everything we can infer
if not options.remoteTestRoot:
options.remoteTestRoot = self.automation._devicemanager.deviceRoot + '/reftest'
options.remoteProfile = options.remoteTestRoot + "/profile"
# Verify that our remotewebserver is set properly
if (options.remoteWebServer == None or
options.remoteWebServer == '127.0.0.1'):
print "ERROR: Either you specified the loopback for the remote webserver or ",
print "your local IP cannot be detected. Please provide the local ip in --remote-webserver"
return None
# One of remoteAppPath (relative path to application) or the app (executable) must be
# set, but not both. If both are set, we destroy the user's selection for app
# so instead of silently destroying a user specificied setting, we error.
if (options.remoteAppPath and options.app):
print "ERROR: You cannot specify both the remoteAppPath and the app"
return None
elif (options.remoteAppPath):
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif (options.app == None):
# Neither remoteAppPath nor app are set -- error
print "ERROR: You must specify either appPath or app"
return None
if (options.xrePath == None):
print "ERROR: You must specify the path to the controller xre directory"
return None
else:
print >> sys.stderr, "Could not find manifest %s" % script_abs_path
sys.exit(1)
return os.path.normpath(rv)
# Ensure xrepath is a full path
options.xrePath = os.path.abspath(options.xrePath)
def manifestURL(self, options, path):
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
# It's possible for this url to have a leading "..", but reftest.js will fix that up
relPath = os.path.relpath(path, SCRIPT_DIRECTORY)
return "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, relPath)
# Default to <deviceroot>/reftest/reftest.log
if (options.remoteLogFile == None):
options.remoteLogFile = 'reftest.log'
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
if (options.pidFile != ""):
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from hostutils.zip, as required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.utilityPath, "components")
# TODO: Copied from main, but I think these are no longer used in a post xulrunner world
#options.xrePath = options.remoteTestRoot + self.automation._product + '/xulrunner'
#options.utilityPath = options.testRoot + self.automation._product + '/bin'
return options
class ReftestServer:
""" Web server used to serve Reftests, for closer fidelity to the real web.
@ -126,7 +260,6 @@ class ReftestServer:
class RemoteReftest(RefTest):
remoteApp = ''
resolver_cls = RemoteReftestResolver
def __init__(self, automation, devicemanager, options, scriptDir):
RefTest.__init__(self)
@ -209,12 +342,8 @@ class RemoteReftest(RefTest):
def stopWebServer(self, options):
self.server.stop()
def createReftestProfile(self, options, manifest):
profile = RefTest.createReftestProfile(self,
options,
manifest,
server=options.remoteWebServer,
port=options.httpPort)
def createReftestProfile(self, options, reftestlist):
profile = RefTest.createReftestProfile(self, options, reftestlist, server=options.remoteWebServer, port=options.httpPort)
profileDir = profile.profile
prefs = {}
@ -226,6 +355,7 @@ class RemoteReftest(RefTest):
# Set a future policy version to avoid the telemetry prompt.
prefs["toolkit.telemetry.prompted"] = 999
prefs["toolkit.telemetry.notifiedOptOut"] = 999
prefs["reftest.uri"] = "%s" % reftestlist
prefs["datareporting.policy.dataSubmissionPolicyBypassAcceptance"] = True
# Point the url-classifier to the local testing server for fast failures
@ -280,6 +410,9 @@ class RemoteReftest(RefTest):
print "Automation Error: Failed to copy extra files to device"
raise
def getManifestPath(self, path):
return path
def printDeviceInfo(self, printLogcat=False):
try:
if printLogcat:
@ -340,10 +473,10 @@ class RemoteReftest(RefTest):
except:
print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
def main():
def main(args):
automation = RemoteAutomation(None)
parser = reftestcommandline.RemoteArgumentsParser()
options = parser.parse_args()
parser = RemoteOptions(automation)
options, args = parser.parse_args()
if (options.dm_trans == 'sut' and options.deviceIP == None):
print "Error: If --dm_trans = sut, you must provide a device IP to connect to via the --deviceIP option"
@ -369,7 +502,18 @@ def main():
automation.setProduct(options.remoteProductName)
# Set up the defaults and ensure options are set
parser.validate_remote(options, automation)
options = parser.verifyRemoteOptions(options)
if (options == None):
print "ERROR: Invalid options specified, use --help for a list of valid options"
return 1
if not options.ignoreWindowSize:
parts = dm.getInfo('screen')['screen'][0].split()
width = int(parts[0].split(':')[1])
height = int(parts[1].split(':')[1])
if (width < 1050 or height < 1050):
print "ERROR: Invalid screen resolution %sx%s, please adjust to 1366x1050 or higher" % (width, height)
return 1
# Check that Firefox is installed
expected = options.app.split('/')[-1]
@ -382,7 +526,7 @@ def main():
automation.setRemoteProfile(options.remoteProfile)
automation.setRemoteLog(options.remoteLogFile)
reftest = RemoteReftest(automation, dm, options, SCRIPT_DIRECTORY)
parser.validate(options, reftest)
options = parser.verifyCommonOptions(options, reftest)
if mozinfo.info['debug']:
print "changing timeout for remote debug reftests from %s to 600 seconds" % options.timeout
@ -391,6 +535,17 @@ def main():
# Hack in a symbolic link for jsreftest
os.system("ln -s ../jsreftest " + str(os.path.join(SCRIPT_DIRECTORY, "jsreftest")))
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
manifest = args[0]
if os.path.exists(os.path.join(SCRIPT_DIRECTORY, args[0])):
manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + args[0]
elif os.path.exists(args[0]):
manifestPath = os.path.abspath(args[0]).split(SCRIPT_DIRECTORY)[1].strip('/')
manifest = "http://" + str(options.remoteWebServer) + ":" + str(options.httpPort) + "/" + manifestPath
else:
print "ERROR: Could not find test manifest '%s'" % manifest
return 1
# Start the webserver
retVal = reftest.startWebServer(options)
if retVal:
@ -406,8 +561,11 @@ def main():
# manifest = "http://" + options.remoteWebServer + "/reftests/layout/reftests/reftest-sanity/reftest.list"
retVal = 0
try:
cmdlineArgs = ["-reftest", manifest]
if options.bootstrap:
cmdlineArgs = []
dm.recordLogcat()
retVal = reftest.runTests(options.tests, options)
retVal = reftest.runTests(manifest, options, cmdlineArgs)
except:
print "Automation Error: Exception caught while running tests"
traceback.print_exc()
@ -420,5 +578,5 @@ def main():
return retVal
if __name__ == "__main__":
sys.exit(main())
sys.exit(main(sys.argv[1:]))

358
layout/tools/reftest/runreftest.py
View File

@ -6,8 +6,9 @@
Runs the reftest test harness.
"""
from optparse import OptionParser
from urlparse import urlparse
import collections
import json
import multiprocessing
import os
import re
@ -18,9 +19,8 @@ import sys
import threading
SCRIPT_DIRECTORY = os.path.abspath(
os.path.realpath(os.path.dirname(__file__)))
if SCRIPT_DIRECTORY not in sys.path:
sys.path.insert(0, SCRIPT_DIRECTORY)
os.path.realpath(os.path.dirname(sys.argv[0])))
sys.path.insert(0, SCRIPT_DIRECTORY)
import mozcrash
import mozdebug
@ -32,7 +32,13 @@ import mozrunner
from mozrunner.utils import get_stack_fixer_function, test_environment
from mozscreenshot import printstatus, dump_screen
import reftestcommandline
here = os.path.abspath(os.path.dirname(__file__))
try:
from mozbuild.base import MozbuildObject
build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
build_obj = None
# set up logging handler a la automation.py.in for compatability
import logging
@ -126,86 +132,14 @@ class ReftestThread(threading.Thread):
if summaryHeadRegex.search(line) is None:
yield line
class ReftestResolver(object):
def defaultManifest(self, suite):
return {"reftest": "reftest.list",
"crashtest": "crashtests.list",
"jstestbrowser": "jstests.list"}[suite]
def directoryManifest(self, suite, path):
return os.path.join(path, self.defaultManifest(suite))
def findManifest(self, suite, test_file, subdirs=True):
"""Return a tuple of (manifest-path, filter-string) for running test_file.
test_file is a path to a test or a manifest file
"""
rv = []
default_manifest = self.defaultManifest(suite)
if not os.path.isabs(test_file):
test_file = self.absManifestPath(test_file)
if os.path.isdir(test_file):
for dirpath, dirnames, filenames in os.walk(test_file):
if default_manifest in filenames:
rv.append((os.path.join(dirpath, default_manifest), None))
# We keep recursing into subdirectories which means that in the case
# of include directives we get the same manifest multiple times.
# However reftest.js will only read each manifest once
elif test_file.endswith('.list'):
if os.path.exists(test_file):
rv = [(test_file, None)]
else:
dirname, pathname = os.path.split(test_file)
found = True
while not os.path.exists(os.path.join(dirname, default_manifest)):
dirname, suffix = os.path.split(dirname)
pathname = os.path.join(suffix, pathname)
if os.path.dirname(dirname) == dirname:
found = False
break
if found:
rv = [(os.path.join(dirname, default_manifest),
r".*(?:/|\\)%s$" % pathname)]
return rv
def absManifestPath(self, path):
return os.path.normpath(os.path.abspath(path))
def manifestURL(self, options, path):
return "file://%s" % path
def resolveManifests(self, options, tests):
suite = options.suite
manifests = {}
for testPath in tests:
for manifest, filter_str in self.findManifest(suite, testPath):
manifest = self.manifestURL(options, manifest)
if manifest not in manifests:
manifests[manifest] = set()
manifests[manifest].add(filter_str)
for key in manifests.iterkeys():
if os.path.split(key)[1] != self.defaultManifest(suite):
print >> sys.stderr, "Invalid manifest for suite %s, %s" %(options.suite, key)
sys.exit(1)
if None in manifests[key]:
manifests[key] = None
else:
manifests[key] = "|".join(list(manifests[key]))
return manifests
class RefTest(object):
oldcwd = os.getcwd()
resolver_cls = ReftestResolver
def __init__(self):
self.update_mozinfo()
self.lastTestSeen = 'reftest'
self.haveDumpedScreen = False
self.resolver = self.resolver_cls()
def update_mozinfo(self):
"""walk up directories to find mozinfo.json update the info"""
@ -224,15 +158,30 @@ class RefTest(object):
"Get an absolute path relative to self.oldcwd."
return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
def createReftestProfile(self, options, manifests, server='localhost', port=0,
profile_to_clone=None):
"""Sets up a profile for reftest.
def getManifestPath(self, path):
"Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest"
path = self.getFullPath(path)
if os.path.isdir(path):
defaultManifestPath = os.path.join(path, 'reftest.list')
if os.path.exists(defaultManifestPath):
path = defaultManifestPath
else:
defaultManifestPath = os.path.join(path, 'crashtests.list')
if os.path.exists(defaultManifestPath):
path = defaultManifestPath
return path
:param options: Object containing command line options
:param manifests: Dictionary of the form {manifest_path: [filters]}
:param server: Server name to use for http tests
:param profile_to_clone: Path to a profile to use as the basis for the
test profile"""
def makeJSString(self, s):
return '"%s"' % re.sub(r'([\\"])', r'\\\1', s)
def createReftestProfile(self, options, manifest, server='localhost', port=0,
special_powers=True, profile_to_clone=None):
"""
Sets up a profile for reftest.
'manifest' is the path to the reftest.list file we want to test with. This is used in
the remote subclass in remotereftest.py so we can write it to a preference for the
bootstrap extension.
"""
locations = mozprofile.permissions.ServerLocations()
locations.add_host(server, scheme='http', port=port)
@ -251,10 +200,11 @@ class RefTest(object):
prefs['reftest.logFile'] = options.logFile
if options.ignoreWindowSize:
prefs['reftest.ignoreWindowSize'] = True
if options.filter:
prefs['reftest.filter'] = options.filter
if options.shuffle:
prefs['reftest.shuffle'] = True
prefs['reftest.focusFilterMode'] = options.focusFilterMode
prefs['reftest.manifests'] = json.dumps(manifests)
# Ensure that telemetry is disabled, so we don't connect to the telemetry
# server in the middle of the tests.
@ -299,10 +249,13 @@ class RefTest(object):
thispref[1].strip())
# install the reftest extension bits into the profile
addons = [options.reftestExtensionPath]
addons = []
addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest"))
if options.specialPowersExtensionPath is not None:
addons.append(options.specialPowersExtensionPath)
# I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with
# release engineering and landing on multiple branches at once.
if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')):
addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers'))
# SpecialPowers requires insecure automation-only features that we
# put behind a pref.
prefs['security.turn_off_all_security_so_that_viruses_can_take_over_this_computer'] = True
@ -383,7 +336,7 @@ class RefTest(object):
if profileDir:
shutil.rmtree(profileDir, True)
def runTests(self, tests, options, cmdlineArgs=None):
def runTests(self, testPath, options, cmdlineArgs=None):
# Despite our efforts to clean up servers started by this script, in practice
# we still see infrequent cases where a process is orphaned and interferes
# with future tests, typically because the old server is keeping the port in use.
@ -392,12 +345,8 @@ class RefTest(object):
self.killNamedOrphans('ssltunnel')
self.killNamedOrphans('xpcshell')
manifests = self.resolver.resolveManifests(options, tests)
if options.filter:
manifests[""] = options.filter
if not hasattr(options, "runTestsInParallel") or not options.runTestsInParallel:
return self.runSerialTests(manifests, options, cmdlineArgs)
if not options.runTestsInParallel:
return self.runSerialTests(testPath, options, cmdlineArgs)
cpuCount = multiprocessing.cpu_count()
@ -428,6 +377,7 @@ class RefTest(object):
jobArgs.remove("--run-tests-in-parallel")
except:
pass
jobArgs.insert(-1, "--no-run-tests-in-parallel")
jobArgs[0:0] = [sys.executable, "-u"]
threads = [ReftestThread(args) for args in perProcessArgs[1:]]
@ -650,7 +600,7 @@ class RefTest(object):
status = 1
return status
def runSerialTests(self, manifests, options, cmdlineArgs=None):
def runSerialTests(self, testPath, options, cmdlineArgs=None):
debuggerInfo = None
if options.debugger:
debuggerInfo = mozdebug.get_debugger_info(options.debugger, options.debuggerArgs,
@ -658,9 +608,10 @@ class RefTest(object):
profileDir = None
try:
reftestlist = self.getManifestPath(testPath)
if cmdlineArgs == None:
cmdlineArgs = []
profile = self.createReftestProfile(options, manifests)
cmdlineArgs = ['-reftest', reftestlist]
profile = self.createReftestProfile(options, reftestlist)
profileDir = profile.profile # name makes more sense
# browser environment
@ -708,26 +659,209 @@ class RefTest(object):
continue
def run(**kwargs):
# Mach gives us kwargs; this is a way to turn them back into an
# options object
parser = reftestcommandline.DesktopArgumentsParser()
reftest = RefTest()
parser.set_defaults(**kwargs)
options = parser.parse_args(kwargs["tests"])
parser.validate(options, reftest)
return reftest.runTests(options.tests, options)
class ReftestOptions(OptionParser):
def __init__(self):
OptionParser.__init__(self)
defaults = {}
self.add_option("--xre-path",
action="store", type="string", dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
self.add_option("--symbols-path",
action="store", type="string", dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
self.add_option("--debugger",
action="store", dest="debugger",
help="use the given debugger to launch the application")
self.add_option("--debugger-args",
action="store", dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
self.add_option("--debugger-interactive",
action="store_true", dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_option("--appname",
action="store", type="string", dest="app",
help="absolute path to application, overriding default")
# Certain paths do not make sense when we're cross compiling Fennec. This
# logic is cribbed from the example in
# python/mozbuild/mozbuild/mach_commands.py.
defaults['app'] = build_obj.get_binary_path() if \
build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None
self.add_option("--extra-profile-file",
action="append", dest="extraProfileFiles",
default=[],
help="copy specified files/dirs to testing profile")
self.add_option("--timeout",
action="store", dest="timeout", type="int",
default=5 * 60, # 5 minutes per bug 479518
help="reftest will timeout in specified number of seconds. [default %default s].")
self.add_option("--leak-threshold",
action="store", type="int", dest="defaultLeakThreshold",
default=0,
help="fail if the number of bytes leaked in default "
"processes through refcounted objects (or bytes "
"in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
"is greater than the given number")
self.add_option("--utility-path",
action="store", type="string", dest="utilityPath",
help="absolute path to directory containing utility "
"programs (xpcshell, ssltunnel, certutil)")
defaults["utilityPath"] = build_obj.bindir if \
build_obj and build_obj.substs['MOZ_BUILD_APP'] != 'mobile/android' else None
self.add_option("--total-chunks",
type="int", dest="totalChunks",
help="how many chunks to split the tests up into")
defaults["totalChunks"] = None
self.add_option("--this-chunk",
type="int", dest="thisChunk",
help="which chunk to run between 1 and --total-chunks")
defaults["thisChunk"] = None
self.add_option("--log-file",
action="store", type="string", dest="logFile",
default=None,
help="file to log output to in addition to stdout")
defaults["logFile"] = None
self.add_option("--skip-slow-tests",
dest="skipSlowTests", action="store_true",
help="skip tests marked as slow when running")
defaults["skipSlowTests"] = False
self.add_option("--ignore-window-size",
dest="ignoreWindowSize", action="store_true",
help="ignore the window size, which may cause spurious failures and passes")
defaults["ignoreWindowSize"] = False
self.add_option("--install-extension",
action="append", dest="extensionsToInstall",
help="install the specified extension in the testing profile. "
"The extension file's name should be <id>.xpi where <id> is "
"the extension's id as indicated in its install.rdf. "
"An optional path can be specified too.")
defaults["extensionsToInstall"] = []
self.add_option("--run-tests-in-parallel",
action="store_true", dest="runTestsInParallel",
help="run tests in parallel if possible")
self.add_option("--no-run-tests-in-parallel",
action="store_false", dest="runTestsInParallel",
help="do not run tests in parallel")
defaults["runTestsInParallel"] = False
self.add_option("--setenv",
action="append", type="string",
dest="environment", metavar="NAME=VALUE",
help="sets the given variable in the application's "
"environment")
defaults["environment"] = []
self.add_option("--filter",
action="store", type="string", dest="filter",
help="specifies a regular expression (as could be passed to the JS "
"RegExp constructor) to test against URLs in the reftest manifest; "
"only test items that have a matching test URL will be run.")
defaults["filter"] = None
self.add_option("--shuffle",
action="store_true", dest="shuffle",
help="run reftests in random order")
defaults["shuffle"] = False
self.add_option("--focus-filter-mode",
action="store", type="string", dest="focusFilterMode",
help="filters tests to run by whether they require focus. "
"Valid values are `all', `needs-focus', or `non-needs-focus'. "
"Defaults to `all'.")
defaults["focusFilterMode"] = "all"
self.add_option("--e10s",
action="store_true",
dest="e10s",
help="enables content processes")
defaults["e10s"] = False
self.add_option("--setpref",
action="append", type="string",
default=[],
dest="extraPrefs", metavar="PREF=VALUE",
help="defines an extra user preference")
self.set_defaults(**defaults)
def verifyCommonOptions(self, options, reftest):
if options.totalChunks is not None and options.thisChunk is None:
self.error("thisChunk must be specified when totalChunks is specified")
if options.totalChunks:
if not 1 <= options.thisChunk <= options.totalChunks:
self.error("thisChunk must be between 1 and totalChunks")
if options.logFile:
options.logFile = reftest.getFullPath(options.logFile)
if options.xrePath is not None:
if not os.access(options.xrePath, os.F_OK):
self.error("--xre-path '%s' not found" % options.xrePath)
if not os.path.isdir(options.xrePath):
self.error("--xre-path '%s' is not a directory" %
options.xrePath)
options.xrePath = reftest.getFullPath(options.xrePath)
if options.runTestsInParallel:
if options.logFile is not None:
self.error("cannot specify logfile with parallel tests")
if options.totalChunks is not None and options.thisChunk is None:
self.error("cannot specify thisChunk or totalChunks with parallel tests")
if options.focusFilterMode != "all":
self.error("cannot specify focusFilterMode with parallel tests")
if options.debugger is not None:
self.error("cannot specify a debugger with parallel tests")
options.leakThresholds = {
"default": options.defaultLeakThreshold,
"tab": 5000, # See dependencies of bug 1051230.
}
return options
def main():
parser = reftestcommandline.DesktopArgumentsParser()
parser = ReftestOptions()
reftest = RefTest()
options = parser.parse_args()
parser.validate(options, reftest)
options, args = parser.parse_args()
if len(args) != 1:
print >>sys.stderr, "No reftest.list specified."
sys.exit(1)
sys.exit(reftest.runTests(options.tests, options))
options = parser.verifyCommonOptions(options, reftest)
if options.app is None:
parser.error("could not find the application path, --appname must be specified")
options.app = reftest.getFullPath(options.app)
if not os.path.exists(options.app):
print """Error: Path %(app)s doesn't exist.
Are you executing $objdir/_tests/reftest/runreftest.py?""" \
% {"app": options.app}
sys.exit(1)
if options.xrePath is None:
options.xrePath = os.path.dirname(options.app)
if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
options.symbolsPath = reftest.getFullPath(options.symbolsPath)
options.utilityPath = reftest.getFullPath(options.utilityPath)
sys.exit(reftest.runTests(args[0], options))
if __name__ == "__main__":
main()

254
layout/tools/reftest/runreftestb2g.py
View File

@ -10,18 +10,207 @@ import traceback
# We need to know our current directory so that we can serve our test files from it.
here = os.path.abspath(os.path.dirname(__file__))
if here not in sys.path:
sys.path.insert(0, here)
from automation import Automation
from b2gautomation import B2GRemoteAutomation
from b2g_desktop import run_desktop_reftests
from remotereftest import RemoteReftestResolver, ReftestServer
from runreftest import RefTest
import reftestcommandline
from runreftest import ReftestOptions
from remotereftest import ReftestServer
from mozdevice import DeviceManagerADB, DMError
from marionette import Marionette
import moznetwork
class B2GOptions(ReftestOptions):
def __init__(self, **kwargs):
defaults = {}
ReftestOptions.__init__(self)
# This is only used for procName in run_remote_reftests.
defaults["app"] = Automation.DEFAULT_APP
self.add_option("--browser-arg", action="store",
type = "string", dest = "browser_arg",
help = "Optional command-line arg to pass to the browser")
defaults["browser_arg"] = None
self.add_option("--b2gpath", action="store",
type = "string", dest = "b2gPath",
help = "path to B2G repo or qemu dir")
defaults["b2gPath"] = None
self.add_option("--marionette", action="store",
type = "string", dest = "marionette",
help = "host:port to use when connecting to Marionette")
defaults["marionette"] = None
self.add_option("--emulator", action="store",
type="string", dest = "emulator",
help = "Architecture of emulator to use: x86 or arm")
defaults["emulator"] = None
self.add_option("--emulator-res", action="store",
type="string", dest = "emulator_res",
help = "Emulator resolution of the format '<width>x<height>'")
defaults["emulator_res"] = None
self.add_option("--no-window", action="store_true",
dest = "noWindow",
help = "Pass --no-window to the emulator")
defaults["noWindow"] = False
self.add_option("--adbpath", action="store",
type = "string", dest = "adb_path",
help = "path to adb")
defaults["adb_path"] = "adb"
self.add_option("--deviceIP", action="store",
type = "string", dest = "deviceIP",
help = "ip address of remote device to test")
defaults["deviceIP"] = None
self.add_option("--devicePort", action="store",
type = "string", dest = "devicePort",
help = "port of remote device to test")
defaults["devicePort"] = 20701
self.add_option("--remote-logfile", action="store",
type = "string", dest = "remoteLogFile",
help = "Name of log file on the device relative to the device root. PLEASE ONLY USE A FILENAME.")
defaults["remoteLogFile"] = None
self.add_option("--remote-webserver", action = "store",
type = "string", dest = "remoteWebServer",
help = "ip address where the remote web server is hosted at")
defaults["remoteWebServer"] = None
self.add_option("--http-port", action = "store",
type = "string", dest = "httpPort",
help = "ip address where the remote web server is hosted at")
defaults["httpPort"] = None
self.add_option("--ssl-port", action = "store",
type = "string", dest = "sslPort",
help = "ip address where the remote web server is hosted at")
defaults["sslPort"] = None
self.add_option("--pidfile", action = "store",
type = "string", dest = "pidFile",
help = "name of the pidfile to generate")
defaults["pidFile"] = ""
self.add_option("--gecko-path", action="store",
type="string", dest="geckoPath",
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
defaults["geckoPath"] = None
self.add_option("--logdir", action="store",
type="string", dest="logdir",
help="directory to store log files")
defaults["logdir"] = None
self.add_option('--busybox', action='store',
type='string', dest='busybox',
help="Path to busybox binary to install on device")
defaults['busybox'] = None
self.add_option("--httpd-path", action = "store",
type = "string", dest = "httpdPath",
help = "path to the httpd.js file")
defaults["httpdPath"] = None
self.add_option("--profile", action="store",
type="string", dest="profile",
help="for desktop testing, the path to the "
"gaia profile to use")
defaults["profile"] = None
self.add_option("--desktop", action="store_true",
dest="desktop",
help="Run the tests on a B2G desktop build")
defaults["desktop"] = False
self.add_option("--mulet", action="store_true",
dest="mulet",
help="Run the tests on a B2G desktop build")
defaults["mulet"] = False
self.add_option("--enable-oop", action="store_true",
dest="oop",
help="Run the tests out of process")
defaults["oop"] = False
defaults["remoteTestRoot"] = None
defaults["logFile"] = "reftest.log"
defaults["autorun"] = True
defaults["closeWhenDone"] = True
defaults["testPath"] = ""
defaults["runTestsInParallel"] = False
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options, auto):
if options.runTestsInParallel:
self.error("Cannot run parallel tests here")
if not options.remoteTestRoot:
options.remoteTestRoot = auto._devicemanager.deviceRoot + "/reftest"
options.remoteProfile = options.remoteTestRoot + "/profile"
productRoot = options.remoteTestRoot + "/" + auto._product
if options.utilityPath is None:
options.utilityPath = productRoot + "/bin"
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
print "ERROR: you must specify a --remote-webserver=<ip address>\n"
return None
options.webServer = options.remoteWebServer
if not options.httpPort:
options.httpPort = auto.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = auto.DEFAULT_SSL_PORT
if options.geckoPath and not options.emulator:
self.error("You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
self.error("You must specify --emulator if you specify --logdir")
#if not options.emulator and not options.deviceIP:
# print "ERROR: you must provide a device IP"
# return None
if options.remoteLogFile == None:
options.remoteLogFile = "reftest.log"
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
# Only reset the xrePath if it wasn't provided
if options.xrePath == None:
options.xrePath = options.utilityPath
options.xrePath = os.path.abspath(options.xrePath)
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from from the xre bundle, as required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.xrePath, "components")
return options
class ProfileConfigParser(ConfigParser.RawConfigParser):
"""Subclass of RawConfigParser that outputs .ini files in the exact
@ -54,7 +243,6 @@ class B2GRemoteReftest(RefTest):
localProfile = None
remoteApp = ''
profile = None
resolver_cls = RemoteReftestResolver
def __init__(self, automation, devicemanager, options, scriptDir):
RefTest.__init__(self)
@ -230,9 +418,10 @@ class B2GRemoteReftest(RefTest):
pass
def createReftestProfile(self, options, manifests):
profile = RefTest.createReftestProfile(self, options, manifests,
server=options.remoteWebServer)
def createReftestProfile(self, options, reftestlist):
profile = RefTest.createReftestProfile(self, options, reftestlist,
server=options.remoteWebServer,
special_powers=False)
profileDir = profile.profile
prefs = {}
@ -248,7 +437,7 @@ class B2GRemoteReftest(RefTest):
prefs["network.dns.localDomains"] = "app://test-container.gaiamobile.org"
prefs["reftest.browser.iframe.enabled"] = False
prefs["reftest.remote"] = True
prefs["reftest.uri"] = "%s" % reftestlist
# Set a future policy version to avoid the telemetry prompt.
prefs["toolkit.telemetry.prompted"] = 999
prefs["toolkit.telemetry.notifiedOptOut"] = 999
@ -306,6 +495,9 @@ class B2GRemoteReftest(RefTest):
print "Automation Error: Failed to copy extra files to device"
raise
def getManifestPath(self, path):
return path
def environment(self, **kwargs):
return self.automation.environment(**kwargs)
@ -324,7 +516,7 @@ class B2GRemoteReftest(RefTest):
return status
def run_remote_reftests(parser, options):
def run_remote_reftests(parser, options, args):
auto = B2GRemoteAutomation(None, "fennec", context_chrome=True)
# create our Marionette instance
@ -367,7 +559,11 @@ def run_remote_reftests(parser, options):
dm = DeviceManagerADB(**kwargs)
auto.setDeviceManager(dm)
parser.validate_remote(options, auto)
options = parser.verifyRemoteOptions(options, auto)
if (options == None):
print "ERROR: Invalid options specified, use --help for a list of valid options"
sys.exit(1)
# TODO fix exception
if not options.ignoreWindowSize:
@ -384,7 +580,7 @@ def run_remote_reftests(parser, options):
auto.logFinish = "REFTEST TEST-START | Shutdown"
reftest = B2GRemoteReftest(auto, dm, options, here)
parser.validate(options, reftest)
options = parser.verifyCommonOptions(options, reftest)
logParent = os.path.dirname(options.remoteLogFile)
dm.mkDir(logParent);
@ -394,6 +590,16 @@ def run_remote_reftests(parser, options):
# Hack in a symbolic link for jsreftest
os.system("ln -s %s %s" % (os.path.join('..', 'jsreftest'), os.path.join(here, 'jsreftest')))
# Dynamically build the reftest URL if possible, beware that args[0] should exist 'inside' the webroot
manifest = args[0]
if os.path.exists(os.path.join(here, args[0])):
manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, args[0])
elif os.path.exists(args[0]):
manifestPath = os.path.abspath(args[0]).split(here)[1].strip('/')
manifest = "http://%s:%s/%s" % (options.remoteWebServer, options.httpPort, manifestPath)
else:
print "ERROR: Could not find test manifest '%s'" % manifest
return 1
# Start the webserver
retVal = 1
@ -405,7 +611,11 @@ def run_remote_reftests(parser, options):
if (dm.processExist(procName)):
dm.killProcess(procName)
retVal = reftest.runTests(options.tests, options)
cmdlineArgs = ["-reftest", manifest]
if getattr(options, 'bootstrap', False):
cmdlineArgs = []
retVal = reftest.runTests(manifest, options, cmdlineArgs)
except:
print "Automation Error: Exception caught while running tests"
traceback.print_exc()
@ -419,21 +629,13 @@ def run_remote_reftests(parser, options):
reftest.stopWebServer(options)
return retVal
def run_remote(**kwargs):
# Tests need to be served from a subdirectory of the server. Symlink
# topsrcdir here to get around this.
parser = reftestcommandline.B2GArgumentParser()
parser.set_defaults(**kwargs)
options = parser.parse_args(kwargs["tests"])
return run_remote_reftests(parser, options)
def main():
parser = reftestcommandline.B2GArgumentParser()
options = parser.parse_args()
def main(args=sys.argv[1:]):
parser = B2GOptions()
options, args = parser.parse_args(args)
if options.desktop or options.mulet:
return run_desktop_reftests(parser, options)
return run_remote_reftests(parser, options)
return run_desktop_reftests(parser, options, args)
return run_remote_reftests(parser, options, args)
if __name__ == "__main__":

7
testing/mach_commands.py
View File

@ -96,7 +96,7 @@ TEST_SUITES = {
'reftest': {
'aliases': ('RR', 'rr', 'Rr'),
'mach_command': 'reftest',
'kwargs': {'tests': None},
'kwargs': {'test_file': None},
},
'reftest-ipc': {
'aliases': ('Ripc',),
@ -139,10 +139,7 @@ TEST_FLAVORS = {
'mach_command': 'mochitest',
'kwargs': {'flavor': 'mochitest', 'test_paths': []},
},
'reftest': {
'mach_command': 'reftest',
'kwargs': {'tests': []}
},
'reftest': { },
'steeplechase': { },
'web-platform-tests': {
'mach_command': 'web-platform-tests',

11
testing/mozharness/configs/android/android_panda_releng.py
View File

@ -97,9 +97,8 @@ config = {
"--http-port=%(http_port)s",
"--ssl-port=%(ssl_port)s",
"--symbols-path=%(symbols_path)s",
"--suite=crashtest",
"reftest/tests/testing/crashtest/crashtests.list"
],
"tests": ["reftest/tests/testing/crashtest/crashtests.list",],
"run_filename": "remotereftest.py",
"testsdir": "reftest"
},
@ -129,12 +128,11 @@ config = {
"--ignore-window-size",
"--bootstrap",
"--extra-profile-file=jsreftest/tests/user.js",
"jsreftest/tests/jstests.list",
"--http-port=%(http_port)s",
"--ssl-port=%(ssl_port)s",
"--symbols-path=%(symbols_path)s",
"--suite=jstestbrowser",
"--symbols-path=%(symbols_path)s"
],
"tests": ["jsreftest/tests/jstests.list",],
"run_filename": "remotereftest.py",
"testsdir": "reftest"
},
@ -168,9 +166,8 @@ config = {
"--http-port=%(http_port)s",
"--ssl-port=%(ssl_port)s",
"--symbols-path=%(symbols_path)s",
"--suite=reftest",
"reftest/tests/layout/reftests/reftest.list"
],
"tests": ["reftest/tests/layout/reftests/reftest.list"],
"run_filename": "remotereftest.py",
"testsdir": "reftest"
},

57
testing/mozharness/configs/android/androidarm.py
View File

@ -140,9 +140,8 @@ config = {
"%(modules_dir)s",
"--symbols-path=%(symbols_path)s",
"--total-chunks=16",
"--suite=reftest",
"tests/layout/reftests/reftest.list",
],
"tests": ["tests/layout/reftests/reftest.list"],
},
"crashtest": {
"run_filename": "remotereftest.py",
@ -162,9 +161,8 @@ config = {
"%(modules_dir)s",
"--symbols-path=%(symbols_path)s",
"--total-chunks=2",
"--suite=crashtest",
"tests/testing/crashtest/crashtests.list",
],
"tests": ["tests/testing/crashtest/crashtests.list"],
},
"jsreftest": {
"run_filename": "remotereftest.py",
@ -183,11 +181,10 @@ config = {
"--httpd-path",
"%(modules_dir)s",
"--symbols-path=%(symbols_path)s",
"../jsreftest/tests/jstests.list",
"--total-chunks=6",
"--extra-profile-file=jsreftest/tests/user.js",
"--suite=jstestbrowser",
],
"tests": ["../jsreftest/tests/jstests.list"],
},
"xpcshell": {
"run_filename": "remotexpcshelltests.py",
@ -318,67 +315,83 @@ config = {
},
"reftest-1": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=1"],
"extra_args": ["--total-chunks=16", "--this-chunk=1",
"tests/layout/reftests/reftest.list"]
},
"reftest-2": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=2"],
"extra_args": ["--total-chunks=16", "--this-chunk=2",
"tests/layout/reftests/reftest.list"]
},
"reftest-3": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=3"],
"extra_args": ["--total-chunks=16", "--this-chunk=3",
"tests/layout/reftests/reftest.list"]
},
"reftest-4": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=4"],
"extra_args": ["--total-chunks=16", "--this-chunk=4",
"tests/layout/reftests/reftest.list"]
},
"reftest-5": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=5"],
"extra_args": ["--total-chunks=16", "--this-chunk=5",
"tests/layout/reftests/reftest.list"]
},
"reftest-6": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=6"],
"extra_args": ["--total-chunks=16", "--this-chunk=6",
"tests/layout/reftests/reftest.list"]
},
"reftest-7": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=7"],
"extra_args": ["--total-chunks=16", "--this-chunk=7",
"tests/layout/reftests/reftest.list"]
},
"reftest-8": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=8"],
"extra_args": ["--total-chunks=16", "--this-chunk=8",
"tests/layout/reftests/reftest.list"]
},
"reftest-9": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=9"],
"extra_args": ["--total-chunks=16", "--this-chunk=9",
"tests/layout/reftests/reftest.list"]
},
"reftest-10": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=10"],
"extra_args": ["--total-chunks=16", "--this-chunk=10",
"tests/layout/reftests/reftest.list"]
},
"reftest-11": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=11"],
"extra_args": ["--total-chunks=16", "--this-chunk=11",
"tests/layout/reftests/reftest.list"]
},
"reftest-12": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=12"],
"extra_args": ["--total-chunks=16", "--this-chunk=12",
"tests/layout/reftests/reftest.list"]
},
"reftest-13": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=13"],
"extra_args": ["--total-chunks=16", "--this-chunk=13",
"tests/layout/reftests/reftest.list"]
},
"reftest-14": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=14"],
"extra_args": ["--total-chunks=16", "--this-chunk=14",
"tests/layout/reftests/reftest.list"]
},
"reftest-15": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=15"],
"extra_args": ["--total-chunks=16", "--this-chunk=15",
"tests/layout/reftests/reftest.list"]
},
"reftest-16": {
"category": "reftest",
"extra_args": ["--total-chunks=16", "--this-chunk=16"],
"extra_args": ["--total-chunks=16", "--this-chunk=16",
"tests/layout/reftests/reftest.list"]
},
"crashtest-1": {
"category": "crashtest",

153
testing/mozharness/configs/android/androidarm_4_3.py
View File

@ -138,9 +138,8 @@ config = {
"--httpd-path", "%(modules_dir)s",
"--symbols-path=%(symbols_path)s",
"--total-chunks=16",
"--suite=reftest",
"tests/layout/reftests/reftest.list",
],
"tests": ["tests/layout/reftests/reftest.list",],
},
"crashtest": {
"run_filename": "remotereftest.py",
@ -159,9 +158,8 @@ config = {
"%(modules_dir)s",
"--symbols-path=%(symbols_path)s",
"--total-chunks=2",
"--suite=crashtest",
"tests/testing/crashtest/crashtests.list",
],
"tests": ["tests/testing/crashtest/crashtests.list",],
},
"jsreftest": {
"run_filename": "remotereftest.py",
@ -175,11 +173,10 @@ config = {
"--utility-path=%(utility_path)s", "--http-port=%(http_port)s",
"--ssl-port=%(ssl_port)s", "--httpd-path", "%(modules_dir)s",
"--symbols-path=%(symbols_path)s",
"../jsreftest/tests/jstests.list",
"--total-chunks=6",
"--extra-profile-file=jsreftest/tests/user.js",
"--suite=jstestbrowser",
],
"tests": ["../jsreftest/tests/jstests.list",],
},
"xpcshell": {
"run_filename": "remotexpcshelltests.py",
@ -378,195 +375,243 @@ config = {
},
"reftest-1": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=1"],
"extra_args": ["--total-chunks=48", "--this-chunk=1",
"tests/layout/reftests/reftest.list"]
},
"reftest-2": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=2"],
"extra_args": ["--total-chunks=48", "--this-chunk=2",
"tests/layout/reftests/reftest.list"]
},
"reftest-3": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=3"],
"extra_args": ["--total-chunks=48", "--this-chunk=3",
"tests/layout/reftests/reftest.list"]
},
"reftest-4": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=4"],
"extra_args": ["--total-chunks=48", "--this-chunk=4",
"tests/layout/reftests/reftest.list"]
},
"reftest-5": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=5"],
"extra_args": ["--total-chunks=48", "--this-chunk=5",
"tests/layout/reftests/reftest.list"]
},
"reftest-6": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=6"],
"extra_args": ["--total-chunks=48", "--this-chunk=6",
"tests/layout/reftests/reftest.list"]
},
"reftest-7": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=7"],
"extra_args": ["--total-chunks=48", "--this-chunk=7",
"tests/layout/reftests/reftest.list"]
},
"reftest-8": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=8"],
"extra_args": ["--total-chunks=48", "--this-chunk=8",
"tests/layout/reftests/reftest.list"]
},
"reftest-9": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=9"],
"extra_args": ["--total-chunks=48", "--this-chunk=9",
"tests/layout/reftests/reftest.list"]
},
"reftest-10": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=10"],
"extra_args": ["--total-chunks=48", "--this-chunk=10",
"tests/layout/reftests/reftest.list"]
},
"reftest-11": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=11"],
"extra_args": ["--total-chunks=48", "--this-chunk=11",
"tests/layout/reftests/reftest.list"]
},
"reftest-12": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=12"],
"extra_args": ["--total-chunks=48", "--this-chunk=12",
"tests/layout/reftests/reftest.list"]
},
"reftest-13": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=13"],
"extra_args": ["--total-chunks=48", "--this-chunk=13",
"tests/layout/reftests/reftest.list"]
},
"reftest-14": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=14"],
"extra_args": ["--total-chunks=48", "--this-chunk=14",
"tests/layout/reftests/reftest.list"]
},
"reftest-15": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=15"],
"extra_args": ["--total-chunks=48", "--this-chunk=15",
"tests/layout/reftests/reftest.list"]
},
"reftest-16": {
"category": "reftest",
"extra_args": ["--total-chunks=48", "--this-chunk=16"],
"extra_args": ["--total-chunks=48", "--this-chunk=16",
"tests/layout/reftests/reftest.list"]
},
"reftest-17": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=17"],
"extra_args": ["--total-chunks=48", "--this-chunk=17",
"tests/layout/reftests/reftest.list"]
},
"reftest-18": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=18"],
"extra_args": ["--total-chunks=48", "--this-chunk=18",
"tests/layout/reftests/reftest.list"]
},
"reftest-19": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=19"],
"extra_args": ["--total-chunks=48", "--this-chunk=19",
"tests/layout/reftests/reftest.list"]
},
"reftest-20": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=20"],
"extra_args": ["--total-chunks=48", "--this-chunk=20",
"tests/layout/reftests/reftest.list"]
},
"reftest-21": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=21"],
"extra_args": ["--total-chunks=48", "--this-chunk=21",
"tests/layout/reftests/reftest.list"]
},
"reftest-22": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=22"],
"extra_args": ["--total-chunks=48", "--this-chunk=22",
"tests/layout/reftests/reftest.list"]
},
"reftest-23": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=23"],
"extra_args": ["--total-chunks=48", "--this-chunk=23",
"tests/layout/reftests/reftest.list"]
},
"reftest-24": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=24"],
"extra_args": ["--total-chunks=48", "--this-chunk=24",
"tests/layout/reftests/reftest.list"]
},
"reftest-25": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=25"],
"extra_args": ["--total-chunks=48", "--this-chunk=25",
"tests/layout/reftests/reftest.list"]
},
"reftest-26": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=26"],
"extra_args": ["--total-chunks=48", "--this-chunk=26",
"tests/layout/reftests/reftest.list"]
},
"reftest-27": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=27"],
"extra_args": ["--total-chunks=48", "--this-chunk=27",
"tests/layout/reftests/reftest.list"]
},
"reftest-28": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=28"],
"extra_args": ["--total-chunks=48", "--this-chunk=28",
"tests/layout/reftests/reftest.list"]
},
"reftest-29": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=29"],
"extra_args": ["--total-chunks=48", "--this-chunk=29",
"tests/layout/reftests/reftest.list"]
},
"reftest-30": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=30"],
"extra_args": ["--total-chunks=48", "--this-chunk=30",
"tests/layout/reftests/reftest.list"]
},
"reftest-31": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=31"],
"extra_args": ["--total-chunks=48", "--this-chunk=31",
"tests/layout/reftests/reftest.list"]
},
"reftest-32": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=32"],
"extra_args": ["--total-chunks=48", "--this-chunk=32",
"tests/layout/reftests/reftest.list"]
},
"reftest-33": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=33"],
"extra_args": ["--total-chunks=48", "--this-chunk=33",
"tests/layout/reftests/reftest.list"]
},
"reftest-34": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=34"],
"extra_args": ["--total-chunks=48", "--this-chunk=34",
"tests/layout/reftests/reftest.list"]
},
"reftest-35": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=35"],
"extra_args": ["--total-chunks=48", "--this-chunk=35",
"tests/layout/reftests/reftest.list"]
},
"reftest-36": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=36"],
"extra_args": ["--total-chunks=48", "--this-chunk=36",
"tests/layout/reftests/reftest.list"]
},
"reftest-37": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=37"],
"extra_args": ["--total-chunks=48", "--this-chunk=37",
"tests/layout/reftests/reftest.list"]
},
"reftest-38": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=38"],
"extra_args": ["--total-chunks=48", "--this-chunk=38",
"tests/layout/reftests/reftest.list"]
},
"reftest-39": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=39"],
"extra_args": ["--total-chunks=48", "--this-chunk=39",
"tests/layout/reftests/reftest.list"]
},
"reftest-40": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=40"],
"extra_args": ["--total-chunks=48", "--this-chunk=40",
"tests/layout/reftests/reftest.list"]
},
"reftest-41": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=41"],
"extra_args": ["--total-chunks=48", "--this-chunk=41",
"tests/layout/reftests/reftest.list"]
},
"reftest-42": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=42"],
"extra_args": ["--total-chunks=48", "--this-chunk=42",
"tests/layout/reftests/reftest.list"]
},
"reftest-43": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=43"],
"extra_args": ["--total-chunks=48", "--this-chunk=43",
"tests/layout/reftests/reftest.list"]
},
"reftest-44": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=44"],
"extra_args": ["--total-chunks=48", "--this-chunk=44",
"tests/layout/reftests/reftest.list"]
},
"reftest-45": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=45"],
"extra_args": ["--total-chunks=48", "--this-chunk=45",
"tests/layout/reftests/reftest.list"]
},
"reftest-46": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=46"],
"extra_args": ["--total-chunks=48", "--this-chunk=46",
"tests/layout/reftests/reftest.list"]
},
"reftest-47": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=47"],
"extra_args": ["--total-chunks=48", "--this-chunk=47",
"tests/layout/reftests/reftest.list"]
},
"reftest-48": {
"category": "reftest",
"extra args": ["--total-chunks=48", "--this-chunk=48"],
"extra_args": ["--total-chunks=48", "--this-chunk=48",
"tests/layout/reftests/reftest.list"]
},
"crashtest-1": {
"category": "crashtest",

36
testing/mozharness/configs/android/androidx86.py
View File

@ -135,11 +135,8 @@ config = {
"test_suite_definitions": {
"jsreftest": {
"category": "reftest",
"tests": ["../jsreftest/tests/jstests.list"],
"extra_args": [
"--suite=jstestbrowser",
"--extra-profile-file=jsreftest/tests/user.js"
]
"extra_args": ["../jsreftest/tests/jstests.list",
"--extra-profile-file=jsreftest/tests/user.js"]
},
"mochitest-1": {
"category": "mochitest",
@ -155,33 +152,22 @@ config = {
},
"reftest-1": {
"category": "reftest",
"extra_args": [
"--suite=reftest",
"--total-chunks=3",
"--this-chunk=1",
],
"tests": ["tests/layout/reftests/reftest.list"],
"extra_args": ["--total-chunks=3", "--this-chunk=1",
"tests/layout/reftests/reftest.list"]
},
"reftest-2": {
"extra_args": [
"--suite=reftest",
"--total-chunks=3",
"--this-chunk=2",
],
"tests": ["tests/layout/reftests/reftest.list"],
"category": "reftest",
"extra_args": ["--total-chunks=3", "--this-chunk=2",
"tests/layout/reftests/reftest.list"]
},
"reftest-3": {
"extra_args": [
"--suite=reftest",
"--total-chunks=3",
"--this-chunk=3",
],
"tests": ["tests/layout/reftests/reftest.list"],
"category": "reftest",
"extra_args": ["--total-chunks=3", "--this-chunk=3",
"tests/layout/reftests/reftest.list"]
},
"crashtest": {
"category": "reftest",
"extra_args": ["--suite=crashtest"],
"tests": ["tests/testing/crashtest/crashtests.list"]
"extra_args": ["tests/testing/crashtest/crashtests.list"]
},
"xpcshell": {
"category": "xpcshell",

7
testing/mozharness/configs/b2g/emulator_automation_config.py
View File

@ -74,9 +74,8 @@ config = {
"--busybox=%(busybox)s",
"--total-chunks=%(total_chunks)s",
"--this-chunk=%(this_chunk)s",
"--suite=crashtest",
"tests/testing/crashtest/crashtests.list"
],
"tests": ["tests/testing/crashtest/crashtests.list",],
"run_filename": "runreftestb2g.py",
"testsdir": "reftest"
},
@ -95,8 +94,8 @@ config = {
"--total-chunks=%(total_chunks)s",
"--this-chunk=%(this_chunk)s",
"--extra-profile-file=jsreftest/tests/user.js",
"jsreftest/tests/jstests.list"
],
"tests": ["jsreftest/tests/jstests.list",],
"run_filename": "remotereftest.py",
"testsdir": "reftest"
},
@ -162,8 +161,8 @@ config = {
"--total-chunks=%(total_chunks)s",
"--this-chunk=%(this_chunk)s",
"--enable-oop",
"tests/layout/reftests/reftest.list"
],
"tests": ["tests/layout/reftests/reftest.list",],
"run_filename": "runreftestsb2g.py",
"testsdir": "reftest"
},

90
testing/mozharness/configs/unittests/linux_unittest.py
View File

@ -2,14 +2,14 @@ import os
import platform
# OS Specifics
ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
ABS_WORK_DIR = os.path.join(os.getcwd(), 'build')
BINARY_PATH = os.path.join(ABS_WORK_DIR, "firefox", "firefox-bin")
INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
XPCSHELL_NAME = "xpcshell"
EXE_SUFFIX = ""
EXE_SUFFIX = ''
DISABLE_SCREEN_SAVER = True
ADJUST_MOUSE_AND_SCREEN = False
if platform.architecture()[0] == "64bit":
if platform.architecture()[0] == '64bit':
TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
else:
@ -20,9 +20,9 @@ else:
config = {
"buildbot_json_path": "buildprops.json",
"exes": {
"python": "/tools/buildbot/bin/python",
"virtualenv": ["/tools/buildbot/bin/python", "/tools/misc-python/virtualenv.py"],
"tooltool.py": "/tools/tooltool.py",
'python': '/tools/buildbot/bin/python',
'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
'tooltool.py': "/tools/tooltool.py",
},
"find_links": [
"http://pypi.pvt.build.mozilla.org/pub",
@ -199,63 +199,35 @@ config = {
},
# local reftest suites
"all_reftest_suites": {
"reftest": {
"options": ["--suite=reftest"],
"tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
},
"crashtest": {
"options": ["--suite=crashtest"],
"tests": ["tests/reftest/tests/testing/crashtest/crashtests.list"]
},
"jsreftest": {
"options":["--extra-profile-file=tests/jsreftest/tests/user.js",
"--suite=jstestbrowser"],
"tests": ["tests/jsreftest/tests/jstests.list"]
},
"reftest-ipc": {
"env": {
"MOZ_OMTC_ENABLED": "1",
"MOZ_DISABLE_CONTEXT_SHARING_GLX": "1"
},
"options": ["--suite=reftest",
"--setpref=browser.tabs.remote=true",
"--setpref=browser.tabs.remote.autostart=true",
"--setpref=layers.offmainthreadcomposition.testing.enabled=true",
"--setpref=layers.async-pan-zoom.enabled=true"],
"tests": ["tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list"]
},
"reftest-no-accel": {
"options": ["--suite=reftest",
"--setpref=layers.acceleration.force-enabled=disabled"],
"tests": ["tests/reftest/tests/layout/reftests/reftest.list"]},
"crashtest-ipc": {
"env": {
"MOZ_OMTC_ENABLED": "1",
"MOZ_DISABLE_CONTEXT_SHARING_GLX": "1"
},
"options": ["--suite=crashtest",
"--setpref=browser.tabs.remote=true",
"--setpref=browser.tabs.remote.autostart=true",
"--setpref=layers.offmainthreadcomposition.testing.enabled=true",
"--setpref=layers.async-pan-zoom.enabled=true"],
"tests": ["tests/reftest/tests/testing/crashtest/crashtests.list"]
},
"reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
"crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
"jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
"reftest-ipc": {'env': {'MOZ_OMTC_ENABLED': '1',
'MOZ_DISABLE_CONTEXT_SHARING_GLX': '1'},
'options': ['--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.offmainthreadcomposition.testing.enabled=true',
'--setpref=layers.async-pan-zoom.enabled=true',
'tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list']},
"reftest-no-accel": ['--setpref=layers.acceleration.force-enabled=disabled',
'tests/reftest/tests/layout/reftests/reftest.list'],
"crashtest-ipc": {'env': {'MOZ_OMTC_ENABLED': '1',
'MOZ_DISABLE_CONTEXT_SHARING_GLX': '1'},
'options': ['--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.offmainthreadcomposition.testing.enabled=true',
'--setpref=layers.async-pan-zoom.enabled=true',
'tests/reftest/tests/testing/crashtest/crashtests.list']},
},
"all_xpcshell_suites": {
"xpcshell": {
"options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--manifest=tests/xpcshell/tests/all-test-dirs.list"],
"tests": []
},
"xpcshell-addons": {
"options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--tag=addons",
"--manifest=tests/xpcshell/tests/all-test-dirs.list"],
"tests": []
},
"xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
"%(abs_app_dir)s/" + XPCSHELL_NAME],
"xpcshell-addons": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
"--tag=addons",
"%(abs_app_dir)s/" + XPCSHELL_NAME]
},
"all_cppunittest_suites": {
"cppunittest": ["tests/cppunittest"]
"cppunittest": ['tests/cppunittest']
},
"all_gtest_suites": {
"gtest": []

51
testing/mozharness/configs/unittests/mac_unittest.py
View File

@ -166,45 +166,24 @@ config = {
},
# local reftest suites
"all_reftest_suites": {
"reftest": {
'options': ["--suite=reftest"],
'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
},
"crashtest": {
'options': ["--suite=crashtest"],
'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
},
"jsreftest": {
'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
'tests': ["tests/jsreftest/tests/jstests.list"]
},
"reftest-ipc": {
'options': ['--suite=reftest',
'--setpref=browser.tabs.remote=true',
"reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
"crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
"jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
"reftest-ipc": ['--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.async-pan-zoom.enabled=true'],
'tests': ['tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list']
},
"crashtest-ipc": {
'options': ['--suite=crashtest',
'--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.async-pan-zoom.enabled=true'],
'tests': ['tests/reftest/tests/testing/crashtest/crashtests.list']
},
'--setpref=layers.async-pan-zoom.enabled=true',
'tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list'],
"crashtest-ipc": ['--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.async-pan-zoom.enabled=true',
'tests/reftest/tests/testing/crashtest/crashtests.list'],
},
"all_xpcshell_suites": {
"xpcshell": {
'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--manifest=tests/xpcshell/tests/all-test-dirs.list"],
'tests': []
},
"xpcshell-addons": {
'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--tag=addons",
"--manifest=tests/xpcshell/tests/all-test-dirs.list"],
'tests': []
},
"xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
"%(abs_app_dir)s/" + XPCSHELL_NAME],
"xpcshell-addons": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
"--tag=addons",
"%(abs_app_dir)s/" + XPCSHELL_NAME]
},
"all_cppunittest_suites": {
"cppunittest": ['tests/cppunittest']

66
testing/mozharness/configs/unittests/win_unittest.py
View File

@ -176,56 +176,28 @@ config = {
},
# local reftest suites
"all_reftest_suites": {
"reftest": {
'options': ["--suite=reftest"],
'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
},
"crashtest": {
'options': ["--suite=crashtest"],
'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
},
"jsreftest": {
'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
'tests': ["tests/jsreftest/tests/jstests.list"]
},
"reftest-ipc": {
'options': ['--suite=reftest',
'--setpref=browser.tabs.remote=true',
"reftest": ["tests/reftest/tests/layout/reftests/reftest.list"],
"crashtest": ["tests/reftest/tests/testing/crashtest/crashtests.list"],
"jsreftest": ["--extra-profile-file=tests/jsreftest/tests/user.js", "tests/jsreftest/tests/jstests.list"],
"reftest-ipc": ['--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.async-pan-zoom.enabled=true'],
'tests': ['tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list']
},
"reftest-no-accel": {
"options": ["--suite=reftest",
"--setpref=gfx.direct2d.disabled=true",
"--setpref=layers.acceleration.disabled=true"],
"tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
},
"reftest-omtc": {
"options": ["--suite=reftest",
"--setpref=layers.offmainthreadcomposition.enabled=true"],
"tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
},
"crashtest-ipc": {
"options": ["--suite=crashtest",
'--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.async-pan-zoom.enabled=true'],
"tests": ['tests/reftest/tests/testing/crashtest/crashtests.list'],
},
'--setpref=layers.async-pan-zoom.enabled=true',
'tests/reftest/tests/layout/reftests/reftest-sanity/reftest.list'],
"reftest-no-accel": ["--setpref=gfx.direct2d.disabled=true", "--setpref=layers.acceleration.disabled=true",
"tests/reftest/tests/layout/reftests/reftest.list"],
"reftest-omtc": ["--setpref=layers.offmainthreadcomposition.enabled=true",
"tests/reftest/tests/layout/reftests/reftest.list"],
"crashtest-ipc": ['--setpref=browser.tabs.remote=true',
'--setpref=browser.tabs.remote.autostart=true',
'--setpref=layers.async-pan-zoom.enabled=true',
'tests/reftest/tests/testing/crashtest/crashtests.list'],
},
"all_xpcshell_suites": {
"xpcshell": {
'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--manifest=tests/xpcshell/tests/all-test-dirs.list"],
'tests': []
},
"xpcshell-addons": {
'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
"--tag=addons",
"--manifest=tests/xpcshell/tests/all-test-dirs.list"],
'tests': []
},
"xpcshell": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
"%(abs_app_dir)s/" + XPCSHELL_NAME],
"xpcshell-addons": ["--manifest=tests/xpcshell/tests/all-test-dirs.list",
"--tag=addons",
"%(abs_app_dir)s/" + XPCSHELL_NAME]
},
"all_cppunittest_suites": {
"cppunittest": ['tests/cppunittest']

9
testing/mozharness/scripts/android_emulator_unittest.py
View File

@ -483,15 +483,6 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
continue
cmd.append(arg)
tests = None
if "tests" in self.test_suite_definitions[self.test_suite]:
tests = self.test_suite_definitions[self.test_suite]["tests"]
elif "tests" in self.config["suite_definitions"][suite_category]:
tests = self.config["suite_definitions"][suite_category]["tests"]
if tests:
cmd.extend(tests)
return cmd
def _tooltool_fetch(self, url):

5
testing/mozharness/scripts/androidx86_emulator_unittest.py
View File

@ -452,11 +452,6 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
continue
cmd.append(arg)
if "tests" in self.suite_definitions[suite_name]:
cmd.extend(self.suite_definitions[suite_name]["tests"])
elif "tests" in self.tree_config["suite_definitions"][suite_category]:
cmd.extend(self.tree_config["suite_definitions"][suite_category]["tests"])
return cmd
def preflight_run_tests(self):

7
testing/mozharness/scripts/b2g_desktop_unittest.py
View File

@ -154,7 +154,7 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
}
if suite not in self.config["suite_definitions"]:
self.fatal("'%s' not defined in the config!" % suite)
self.fatal("'%s' not defined in the config!" % suite),
options = self.config["suite_definitions"][suite]["options"]
if options:
@ -162,11 +162,6 @@ class B2GDesktopTest(BlobUploadMixin, TestingMixin, MercurialScript):
option = option % str_format_values
if not option.endswith('None'):
cmd.append(option)
tests = self.config["suite_definitions"][suite].get("tests")
if tests:
cmd.extend(tests)
return cmd
def download_and_extract(self):

8
testing/mozharness/scripts/b2g_emulator_unittest.py
View File

@ -281,11 +281,6 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
option = option % str_format_values
if not option.endswith('None'):
cmd.append(option)
tests = self.config["suite_definitions"][suite].get("tests", [])
if tests:
cmd.extend(tests)
return cmd
def _query_adb(self):
@ -341,9 +336,8 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
self.fatal("Don't know how to run --test-suite '%s'!" % suite)
cmd = self._query_abs_base_cmd(suite)
cmd = self.append_harness_extra_args(cmd)
cwd = dirs['abs_%s_dir' % suite]
cmd = self.append_harness_extra_args(cmd)
# TODO we probably have to move some of the code in
# scripts/desktop_unittest.py and scripts/marionette.py to

4
testing/mozharness/scripts/desktop_unittest.py
View File

@ -588,8 +588,8 @@ class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMix
options_list = []
env = {}
if isinstance(suites[suite], dict):
options_list = suites[suite]['options'] + suites[suite].get("tests", [])
env = copy.deepcopy(suites[suite].get('env', {}))
options_list = suites[suite]['options']
env = copy.deepcopy(suites[suite]['env'])
else:
options_list = suites[suite]

4
testing/testsuite-targets.mk
View File

@ -174,7 +174,7 @@ REMOTE_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/remotereftest.py \
--dm_trans=$(DM_TRANS) --ignore-window-size \
--app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
--httpd-path=_tests/modules \
$(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
$(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) '$(1)' | tee ./$@.log
RUN_REFTEST_B2G = rm -f ./$@.log && $(PYTHON) _tests/reftest/runreftestb2g.py \
--remote-webserver=10.0.2.2 --b2gpath=${B2G_PATH} --adbpath=${ADB_PATH} \
@ -209,7 +209,7 @@ reftest-remote:
echo 'please prepare your host with the environment variable TEST_DEVICE'; \
else \
ln -s $(abspath $(topsrcdir)) _tests/reftest/tests; \
$(call REMOTE_REFTEST,'tests/$(TEST_PATH)'); \
$(call REMOTE_REFTEST,tests/$(TEST_PATH)); \
$(CHECK_TEST_ERROR); \
fi

1
testing/xpcshell/Makefile.in
View File

@ -10,7 +10,6 @@ TEST_HARNESS_FILES := \
runxpcshelltests.py \
remotexpcshelltests.py \
runtestsb2g.py \
xpcshellcommandline.py \
head.js \
node-spdy \
moz-spdy \

390
testing/xpcshell/mach_commands.py
View File

@ -25,7 +25,8 @@ from mach.decorators import (
Command,
)
from xpcshellcommandline import parser_desktop, parser_remote, parser_b2g
_parser = argparse.ArgumentParser()
structured.commandline.add_logging_group(_parser)
ADB_NOT_FOUND = '''
The %s command requires the adb binary to be on your path.
@ -39,7 +40,6 @@ BUSYBOX_URLS = {
'x86': 'http://www.busybox.net/downloads/binaries/latest/busybox-i686'
}
here = os.path.abspath(os.path.dirname(__file__))
if sys.version_info[0] < 3:
unicode_type = unicode
@ -55,9 +55,20 @@ class InvalidTestPathError(Exception):
class XPCShellRunner(MozbuildObject):
"""Run xpcshell tests."""
def run_suite(self, **kwargs):
return self._run_xpcshell_harness(**kwargs)
from manifestparser import TestManifest
manifest = TestManifest(manifests=[os.path.join(self.topobjdir,
'_tests', 'xpcshell', 'xpcshell.ini')])
def run_test(self, **kwargs):
return self._run_xpcshell_harness(manifest=manifest, **kwargs)
def run_test(self, test_paths, interactive=False,
keep_going=False, sequential=False, shuffle=False,
debugger=None, debuggerArgs=None, debuggerInteractive=None,
jsDebugger=False, jsDebuggerPort=None,
rerun_failures=False, test_objects=None, verbose=False,
log=None, test_tags=None, dump_tests=None,
# ignore parameters from other platforms' options
**kwargs):
"""Runs an individual xpcshell test."""
from mozbuild.testing import TestResolver
from manifestparser import TestManifest
@ -71,15 +82,65 @@ class XPCShellRunner(MozbuildObject):
if os.path.isdir(src_build_path):
sys.path.append(src_build_path)
self.run_suite(**kwargs)
if test_paths == 'all':
self.run_suite(interactive=interactive,
keep_going=keep_going, shuffle=shuffle, sequential=sequential,
debugger=debugger, debuggerArgs=debuggerArgs,
debuggerInteractive=debuggerInteractive,
jsDebugger=jsDebugger, jsDebuggerPort=jsDebuggerPort,
rerun_failures=rerun_failures,
verbose=verbose, log=log, test_tags=test_tags, dump_tests=dump_tests)
return
elif test_paths:
test_paths = [self._wrap_path_argument(p).relpath() for p in test_paths]
if test_objects:
tests = test_objects
else:
resolver = self._spawn(TestResolver)
tests = list(resolver.resolve_tests(paths=test_paths,
flavor='xpcshell'))
if not tests:
raise InvalidTestPathError('We could not find an xpcshell test '
'for the passed test path. Please select a path that is '
'a test file or is a directory containing xpcshell tests.')
# Dynamically write out a manifest holding all the discovered tests.
manifest = TestManifest()
manifest.tests.extend(tests)
args = {
'interactive': interactive,
'keep_going': keep_going,
'shuffle': shuffle,
'sequential': sequential,
'debugger': debugger,
'debuggerArgs': debuggerArgs,
'debuggerInteractive': debuggerInteractive,
'jsDebugger': jsDebugger,
'jsDebuggerPort': jsDebuggerPort,
'rerun_failures': rerun_failures,
'manifest': manifest,
'verbose': verbose,
'log': log,
'test_tags': test_tags,
'dump_tests': dump_tests,
}
return self._run_xpcshell_harness(**args)
def _run_xpcshell_harness(self, manifest,
test_path=None, shuffle=False, interactive=False,
keep_going=False, sequential=False,
debugger=None, debuggerArgs=None, debuggerInteractive=None,
jsDebugger=False, jsDebuggerPort=None,
rerun_failures=False, verbose=False, log=None, test_tags=None,
dump_tests=None):
def _run_xpcshell_harness(self, **kwargs):
# Obtain a reference to the xpcshell test runner.
import runxpcshelltests
log = kwargs.pop("log")
xpcshell = runxpcshelltests.XPCShellTests(log=log)
self.log_manager.enable_unstructured()
@ -87,41 +148,60 @@ class XPCShellRunner(MozbuildObject):
modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
# We want output from the test to be written immediately if we are only
# running a single test.
single_test = (kwargs["testPaths"] is not None or
single_test = (test_path is not None or
(manifest and len(manifest.test_paths())==1))
sequential = kwargs["sequential"] or single_test
sequential = sequential or single_test
if kwargs["xpcshell"] is None:
kwargs["xpcshell"] = self.get_binary_path('xpcshell')
args = {
'manifest': manifest,
'xpcshell': self.get_binary_path('xpcshell'),
'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
'interactive': interactive,
'keepGoing': keep_going,
'logfiles': False,
'sequential': sequential,
'shuffle': shuffle,
'testsRootDir': tests_dir,
'testingModulesDir': modules_dir,
'profileName': 'firefox',
'verbose': verbose or single_test,
'xunitFilename': os.path.join(self.statedir, 'xpchsell.xunit.xml'),
'xunitName': 'xpcshell',
'pluginsPath': os.path.join(self.distdir, 'plugins'),
'debugger': debugger,
'debuggerArgs': debuggerArgs,
'debuggerInteractive': debuggerInteractive,
'jsDebugger': jsDebugger,
'jsDebuggerPort': jsDebuggerPort,
'test_tags': test_tags,
'dump_tests': dump_tests,
'utility_path': self.bindir,
}
if kwargs["mozInfo"] is None:
kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')
if test_path is not None:
args['testPath'] = test_path
if kwargs["symbolsPath"] is None:
kwargs["symbolsPath"] = os.path.join(self.distdir, 'crashreporter-symbols')
if kwargs["logfiles"] is None:
kwargs["logfiles"] = False
if kwargs["profileName"] is None:
kwargs["profileName"] = "firefox"
if kwargs["pluginsPath"] is None:
kwargs['pluginsPath'] = os.path.join(self.distdir, 'plugins')
if kwargs["utility_path"] is None:
kwargs['utility_path'] = self.bindir
if kwargs["manifest"] is None:
kwargs["manifest"] = os.path.join(tests_dir, "xpcshell.ini")
if kwargs["failure_manifest"] is None:
kwargs["failure_manifest"] = os.path.join(self.statedir, 'xpcshell.failures.ini')
# A failure manifest is written by default. If --rerun-failures is
# specified and a prior failure manifest is found, the prior manifest
# will be run. A new failure manifest is always written over any
# prior failure manifest.
failure_manifest_path = os.path.join(self.statedir, 'xpcshell.failures.ini')
rerun_manifest_path = os.path.join(self.statedir, 'xpcshell.rerun.ini')
if os.path.exists(failure_manifest_path) and rerun_failures:
shutil.move(failure_manifest_path, rerun_manifest_path)
args['manifest'] = rerun_manifest_path
elif os.path.exists(failure_manifest_path):
os.remove(failure_manifest_path)
elif rerun_failures:
print("No failures were found to re-run.")
return 0
args['failureManifest'] = failure_manifest_path
# Python through 2.7.2 has issues with unicode in some of the
# arguments. Work around that.
filtered_args = {}
for k, v in kwargs.iteritems():
for k, v in args.items():
if isinstance(v, unicode_type):
v = v.encode('utf-8')
@ -157,7 +237,12 @@ class AndroidXPCShellRunner(MozbuildObject):
return dm
"""Run Android xpcshell tests."""
def run_test(self, **kwargs):
def run_test(self,
test_paths, keep_going,
devicemanager, ip, port, remote_test_root, no_setup, local_apk,
test_objects=None, log=None,
# ignore parameters from other platforms' options
**kwargs):
# TODO Bug 794506 remove once mach integrates with virtualenv.
build_path = os.path.join(self.topobjdir, 'build')
if build_path not in sys.path:
@ -165,53 +250,60 @@ class AndroidXPCShellRunner(MozbuildObject):
import remotexpcshelltests
dm = self.get_devicemanager(kwargs["dm_trans"], kwargs["deviceIP"], kwargs["devicePort"],
kwargs["remoteTestRoot"])
dm = self.get_devicemanager(devicemanager, ip, port, remote_test_root)
log = kwargs.pop("log")
self.log_manager.enable_unstructured()
if kwargs["xpcshell"] is None:
kwargs["xpcshell"] = "xpcshell"
if not kwargs["objdir"]:
kwargs["objdir"] = self.topobjdir
if not kwargs["localLib"]:
kwargs["localLib"] = os.path.join(self.topobjdir, 'dist/fennec')
if not kwargs["localBin"]:
kwargs["localBin"] = os.path.join(self.topobjdir, 'dist/bin')
if not kwargs["testingModulesDir"]:
kwargs["testingModulesDir"] = os.path.join(self.topobjdir, '_tests/modules')
if not kwargs["mozInfo"]:
kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')
if not kwargs["manifest"]:
kwargs["manifest"] = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell.ini')
if not kwargs["symbolsPath"]:
kwargs["symbolsPath"] = os.path.join(self.distdir, 'crashreporter-symbols')
if not kwargs["localAPK"]:
for file_name in os.listdir(os.path.join(kwargs["objdir"], "dist")):
if file_name.endswith(".apk") and file_name.startswith("fennec"):
kwargs["localAPK"] = os.path.join(kwargs["objdir"], "dist", file_name)
print ("using APK: %s" % kwargs["localAPK"])
options = remotexpcshelltests.RemoteXPCShellOptions()
options.shuffle = False
options.sequential = True
options.interactive = False
options.debugger = None
options.debuggerArgs = None
options.setup = not no_setup
options.keepGoing = keep_going
options.objdir = self.topobjdir
options.localLib = os.path.join(self.topobjdir, 'dist/fennec')
options.localBin = os.path.join(self.topobjdir, 'dist/bin')
options.testingModulesDir = os.path.join(self.topobjdir, '_tests/modules')
options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
options.manifest = os.path.join(self.topobjdir, '_tests/xpcshell/xpcshell.ini')
options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
if local_apk:
options.localAPK = local_apk
else:
for file in os.listdir(os.path.join(options.objdir, "dist")):
if file.endswith(".apk") and file.startswith("fennec"):
options.localAPK = os.path.join(options.objdir, "dist")
options.localAPK = os.path.join(options.localAPK, file)
print ("using APK: " + options.localAPK)
break
else:
raise Exception("You must specify an APK")
options = argparse.Namespace(**kwargs)
xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, log)
if test_paths == 'all':
testdirs = []
options.testPath = None
options.verbose = False
elif test_objects:
if len(test_objects) > 1:
print('Warning: only the first test will be used.')
testdirs = test_objects[0]['dir_relpath']
options.testPath = test_objects[0]['path']
options.verbose = True
else:
if len(test_paths) > 1:
print('Warning: only the first test path argument will be used.')
testdirs = test_paths[0]
options.testPath = test_paths[0]
options.verbose = True
result = xpcshell.runTests(testClass=remotexpcshelltests.RemoteXPCShellTestThread,
mobileArgs=xpcshell.mobileArgs,
**vars(options))
xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, testdirs, log)
result = xpcshell.runTests(xpcshell='xpcshell',
testClass=remotexpcshelltests.RemoteXPCShellTestThread,
testdirs=testdirs,
mobileArgs=xpcshell.mobileArgs,
**options.__dict__)
self.log_manager.disable_unstructured()
return int(not result)
@ -258,66 +350,57 @@ class B2GXPCShellRunner(MozbuildObject):
f.write(data.read())
return busybox_path
def run_test(self, **kwargs):
def run_test(self, test_paths, b2g_home=None, busybox=None, device_name=None,
test_objects=None, log=None,
# ignore parameters from other platforms' options
**kwargs):
try:
import which
which.which('adb')
except which.WhichError:
# TODO Find adb automatically if it isn't on the path
print(ADB_NOT_FOUND % ('mochitest-remote', kwargs["b2g_home"]))
print(ADB_NOT_FOUND % ('mochitest-remote', b2g_home))
sys.exit(1)
test_path = None
if test_objects:
if len(test_objects) > 1:
print('Warning: Only the first test will be used.')
test_path = self._wrap_path_argument(test_objects[0]['path'])
elif test_paths:
if len(test_paths) > 1:
print('Warning: Only the first test path will be used.')
test_path = self._wrap_path_argument(test_paths[0]).relpath()
import runtestsb2g
parser = runtestsb2g.B2GOptions()
options, args = parser.parse_args([])
log = kwargs.pop("log")
self.log_manager.enable_unstructured()
options.b2g_path = b2g_home
options.busybox = busybox or os.environ.get('BUSYBOX')
options.localLib = self.bin_dir
options.localBin = self.bin_dir
options.logdir = self.xpcshell_dir
options.manifest = os.path.join(self.xpcshell_dir, 'xpcshell.ini')
options.mozInfo = os.path.join(self.topobjdir, 'mozinfo.json')
options.objdir = self.topobjdir
options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols'),
options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
options.testsRootDir = self.xpcshell_dir
options.testPath = test_path
options.use_device_libs = True
if kwargs["xpcshell"] is None:
kwargs["xpcshell"] = "xpcshell"
if kwargs["b2g_path"] is None:
kwargs["b2g_path"] = kwargs["b2g_home"]
if kwargs["busybox"] is None:
kwargs["busybox"] = os.environ.get('BUSYBOX')
if kwargs["busybox"] is None:
kwargs["busybox"] = self._download_busybox(kwargs["b2g_home"], kwargs["emulator"])
options.emulator = 'arm'
if device_name.startswith('emulator'):
if 'x86' in device_name:
options.emulator = 'x86'
if kwargs["localLib"] is None:
kwargs["localLib"] = self.bin_dir
if kwargs["localBin"] is None:
kwargs["localBin"] = self.bin_dir
if kwargs["logdir"] is None:
kwargs["logdir"] = self.xpcshell_dir
if kwargs["manifest"] is None:
kwargs["manifest"] = os.path.join(self.xpcshell_dir, 'xpcshell.ini')
if kwargs["mozInfo"] is None:
kwargs["mozInfo"] = os.path.join(self.topobjdir, 'mozinfo.json')
if kwargs["objdir"] is None:
kwargs["objdir"] = self.topobjdir
if kwargs["symbolsPath"] is None:
kwargs["symbolsPath"] = os.path.join(self.distdir, 'crashreporter-symbols')
if kwargs["testingModulesDir"] is None:
kwargs["testingModulesDir"] = os.path.join(self.tests_dir, 'modules')
if kwargs["use_device_libs"] is None:
kwargs["use_device_libs"] = True
if not options.busybox:
options.busybox = self._download_busybox(b2g_home, options.emulator)
if kwargs["device_name"].startswith('emulator') and 'x86' in kwargs["device_name"]:
kwargs["emulator"] = 'x86'
parser = parser_b2g()
options = argparse.Namespace(**kwargs)
rv = runtestsb2g.run_remote_xpcshell(parser, options, log)
self.log_manager.disable_unstructured()
return rv
def get_parser():
build_obj = MozbuildObject.from_environment(cwd=here)
if conditions.is_android(build_obj):
return parser_remote()
elif conditions.is_b2g(build_obj):
return parser_b2g()
else:
return parser_desktop()
return runtestsb2g.run_remote_xpcshell(parser, options, args, log)
@CommandProvider
class MachCommands(MachCommandBase):
@ -328,10 +411,61 @@ class MachCommands(MachCommandBase):
setattr(self, attr, getattr(context, attr, None))
@Command('xpcshell-test', category='testing',
description='Run XPCOM Shell tests (API direct unit testing)',
conditions=[lambda *args: True],
parser=get_parser)
description='Run XPCOM Shell tests (API direct unit testing)',
conditions=[lambda *args: True],
parser=_parser)
@CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
help='Test to run. Can be specified as a single JS file, a directory, '
'or omitted. If omitted, the entire test suite is executed.')
@CommandArgument('--verbose', '-v', action='store_true',
help='Provide full output from each test process.')
@CommandArgument("--debugger", default=None, metavar='DEBUGGER',
help = "Run xpcshell under the given debugger.")
@CommandArgument("--debugger-args", default=None, metavar='ARGS', type=str,
dest = "debuggerArgs",
help = "pass the given args to the debugger _before_ "
"the application on the command line")
@CommandArgument("--debugger-interactive", action = "store_true",
dest = "debuggerInteractive",
help = "prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
@CommandArgument("--jsdebugger", dest="jsDebugger", action="store_true",
help="Waits for a devtools JS debugger to connect before "
"starting the test.")
@CommandArgument("--jsdebugger-port", dest="jsDebuggerPort",
type=int, default=6000,
help="The port to listen on for a debugger connection if "
"--jsdebugger is specified (default=6000).")
@CommandArgument('--interactive', '-i', action='store_true',
help='Open an xpcshell prompt before running tests.')
@CommandArgument('--keep-going', '-k', action='store_true',
help='Continue running tests after a SIGINT is received.')
@CommandArgument('--sequential', action='store_true',
help='Run the tests sequentially.')
@CommandArgument('--shuffle', '-s', action='store_true',
help='Randomize the execution order of tests.')
@CommandArgument('--rerun-failures', action='store_true',
help='Reruns failures from last time.')
@CommandArgument('--tag', action='append', dest='test_tags',
help='Filter out tests that don\'t have the given tag. Can be used '
'multiple times in which case the test must contain at least one '
'of the given tags.')
@CommandArgument('--dump-tests', default=None, type=str, dest='dump_tests',
help='Specify path to a filename to dump all the tests that will be run')
@CommandArgument('--devicemanager', default='adb', type=str,
help='(Android) Type of devicemanager to use for communication: adb or sut')
@CommandArgument('--ip', type=str, default=None,
help='(Android) IP address of device')
@CommandArgument('--port', type=int, default=20701,
help='(Android) Port of device')
@CommandArgument('--remote_test_root', type=str, default=None,
help='(Android) Remote test root such as /mnt/sdcard or /data/local')
@CommandArgument('--no-setup', action='store_true',
help='(Android) Do not copy files to device')
@CommandArgument('--local-apk', type=str, default=None,
help='(Android) Use specified Fennec APK')
@CommandArgument('--busybox', type=str, default=None,
help='(B2G) Path to busybox binary (speeds up installation of tests).')
def run_xpcshell_test(self, **params):
from mozbuild.controller.building import BuildDriver

146
testing/xpcshell/remotexpcshelltests.py
View File

@ -16,8 +16,6 @@ import mozdevice
import mozfile
import mozinfo
from xpcshellcommandline import parser_remote
here = os.path.dirname(os.path.abspath(__file__))
def remoteJoin(path1, path2):
@ -107,7 +105,7 @@ class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
return (list(sanitize_list(test['head'], 'head')),
list(sanitize_list(test['tail'], 'tail')))
def buildXpcsCmd(self):
def buildXpcsCmd(self, testdir):
# change base class' paths to remote paths and use base class to build command
self.xpcshell = remoteJoin(self.remoteBinDir, "xpcw")
self.headJSPath = remoteJoin(self.remoteScriptsDir, 'head.js')
@ -115,7 +113,7 @@ class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
self.httpdManifest = remoteJoin(self.remoteComponentsDir, 'httpd.manifest')
self.testingModulesDir = self.remoteModulesDir
self.testharnessdir = self.remoteScriptsDir
xpcshell.XPCShellTestThread.buildXpcsCmd(self)
xpcshell.XPCShellTestThread.buildXpcsCmd(self, testdir)
# remove "-g <dir> -a <dir>" and add "--greomni <apk>"
del(self.xpcsCmd[1:5])
if self.options.localAPK:
@ -218,7 +216,7 @@ class RemoteXPCShellTestThread(xpcshell.XPCShellTestThread):
# via devicemanager.
class XPCShellRemote(xpcshell.XPCShellTests, object):
def __init__(self, devmgr, options, log):
def __init__(self, devmgr, options, args, log):
xpcshell.XPCShellTests.__init__(self, log)
# Add Android version (SDK level) to mozinfo so that manifest entries
@ -519,8 +517,8 @@ class XPCShellRemote(xpcshell.XPCShellTests, object):
self.device.removeDir(self.remoteMinidumpDir)
self.device.mkDir(self.remoteMinidumpDir)
def buildTestList(self, test_tags=None, test_paths=None):
xpcshell.XPCShellTests.buildTestList(self, test_tags=test_tags, test_paths=test_paths)
def buildTestList(self, test_tags=None):
xpcshell.XPCShellTests.buildTestList(self, test_tags=test_tags)
uniqueTestPaths = set([])
for test in self.alltests:
uniqueTestPaths.add(test['here'])
@ -529,37 +527,90 @@ class XPCShellRemote(xpcshell.XPCShellTests, object):
remoteScriptDir = remoteJoin(self.remoteScriptsDir, abbrevTestDir)
self.pathMapping.append(PathMapping(testdir, remoteScriptDir))
def verifyRemoteOptions(parser, options):
if options.localLib is None:
if options.localAPK and options.objdir:
for path in ['dist/fennec', 'fennec/lib']:
options.localLib = os.path.join(options.objdir, path)
if os.path.isdir(options.localLib):
break
else:
parser.error("Couldn't find local library dir, specify --local-lib-dir")
elif options.objdir:
options.localLib = os.path.join(options.objdir, 'dist/bin')
elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
# assume tests are being run from a tests.zip
options.localLib = os.path.abspath(os.path.join(here, '..', 'bin'))
else:
parser.error("Couldn't find local library dir, specify --local-lib-dir")
class RemoteXPCShellOptions(xpcshell.XPCShellOptions):
if options.localBin is None:
if options.objdir:
for path in ['dist/bin', 'bin']:
options.localBin = os.path.join(options.objdir, path)
if os.path.isdir(options.localBin):
break
def __init__(self):
xpcshell.XPCShellOptions.__init__(self)
defaults = {}
self.add_option("--deviceIP", action="store",
type = "string", dest = "deviceIP",
help = "ip address of remote device to test")
defaults["deviceIP"] = None
self.add_option("--devicePort", action="store",
type = "string", dest = "devicePort",
help = "port of remote device to test")
defaults["devicePort"] = 20701
self.add_option("--dm_trans", action="store",
type = "string", dest = "dm_trans",
help = "the transport to use to communicate with device: [adb|sut]; default=sut")
defaults["dm_trans"] = "sut"
self.add_option("--objdir", action="store",
type = "string", dest = "objdir",
help = "local objdir, containing xpcshell binaries")
defaults["objdir"] = None
self.add_option("--apk", action="store",
type = "string", dest = "localAPK",
help = "local path to Fennec APK")
defaults["localAPK"] = None
self.add_option("--noSetup", action="store_false",
dest = "setup",
help = "do not copy any files to device (to be used only if device is already setup)")
defaults["setup"] = True
self.add_option("--local-lib-dir", action="store",
type = "string", dest = "localLib",
help = "local path to library directory")
defaults["localLib"] = None
self.add_option("--local-bin-dir", action="store",
type = "string", dest = "localBin",
help = "local path to bin directory")
defaults["localBin"] = None
self.add_option("--remoteTestRoot", action = "store",
type = "string", dest = "remoteTestRoot",
help = "remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
defaults["remoteTestRoot"] = None
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options):
if options.localLib is None:
if options.localAPK and options.objdir:
for path in ['dist/fennec', 'fennec/lib']:
options.localLib = os.path.join(options.objdir, path)
if os.path.isdir(options.localLib):
break
else:
self.error("Couldn't find local library dir, specify --local-lib-dir")
elif options.objdir:
options.localLib = os.path.join(options.objdir, 'dist/bin')
elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
# assume tests are being run from a tests.zip
options.localLib = os.path.abspath(os.path.join(here, '..', 'bin'))
else:
parser.error("Couldn't find local binary dir, specify --local-bin-dir")
elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
# assume tests are being run from a tests.zip
options.localBin = os.path.abspath(os.path.join(here, '..', 'bin'))
else:
parser.error("Couldn't find local binary dir, specify --local-bin-dir")
return options
self.error("Couldn't find local library dir, specify --local-lib-dir")
if options.localBin is None:
if options.objdir:
for path in ['dist/bin', 'bin']:
options.localBin = os.path.join(options.objdir, path)
if os.path.isdir(options.localBin):
break
else:
self.error("Couldn't find local binary dir, specify --local-bin-dir")
elif os.path.isfile(os.path.join(here, '..', 'bin', 'xpcshell')):
# assume tests are being run from a tests.zip
options.localBin = os.path.abspath(os.path.join(here, '..', 'bin'))
else:
self.error("Couldn't find local binary dir, specify --local-bin-dir")
return options
class PathMapping:
@ -568,13 +619,14 @@ class PathMapping:
self.remote = remoteDir
def main():
if sys.version_info < (2,7):
print >>sys.stderr, "Error: You must use python version 2.7 or newer but less than 3.0"
sys.exit(1)
parser = parser_remote()
parser = RemoteXPCShellOptions()
commandline.add_logging_group(parser)
options = parser.parse_args()
options, args = parser.parse_args()
if not options.localAPK:
for file in os.listdir(os.path.join(options.objdir, "dist")):
if (file.endswith(".apk") and file.startswith("fennec")):
@ -586,11 +638,16 @@ def main():
print >>sys.stderr, "Error: please specify an APK"
sys.exit(1)
options = verifyRemoteOptions(parser, options)
options = parser.verifyRemoteOptions(options)
log = commandline.setup_logging("Remote XPCShell",
options,
{"tbpl": sys.stdout})
if len(args) < 1 and options.manifest is None:
print >>sys.stderr, """Usage: %s <test dirs>
or: %s --manifest=test.manifest """ % (sys.argv[0], sys.argv[0])
sys.exit(1)
if options.dm_trans == "adb":
if options.deviceIP:
dm = mozdevice.DroidADB(options.deviceIP, options.devicePort, packageName=None, deviceRoot=options.remoteTestRoot)
@ -606,17 +663,16 @@ def main():
print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
sys.exit(1)
if options.xpcshell is None:
options.xpcshell = "xpcshell"
xpcsh = XPCShellRemote(dm, options, log)
xpcsh = XPCShellRemote(dm, options, args, log)
# we don't run concurrent tests on mobile
options.sequential = True
if not xpcsh.runTests(testClass=RemoteXPCShellTestThread,
if not xpcsh.runTests(xpcshell='xpcshell',
testClass=RemoteXPCShellTestThread,
testdirs=args[0:],
mobileArgs=xpcsh.mobileArgs,
**vars(options)):
**options.__dict__):
sys.exit(1)

111
testing/xpcshell/runtestsb2g.py
View File

@ -9,15 +9,14 @@ import os
sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
import traceback
import remotexpcshelltests
from remotexpcshelltests import RemoteXPCShellTestThread, XPCShellRemote
from remotexpcshelltests import RemoteXPCShellTestThread, XPCShellRemote, RemoteXPCShellOptions
from mozdevice import devicemanagerADB, DMError
from mozlog import commandline
DEVICE_TEST_ROOT = '/data/local/tests'
from marionette import Marionette
from xpcshellcommandline import parser_b2g
class B2GXPCShellTestThread(RemoteXPCShellTestThread):
# Overridden
@ -76,19 +75,87 @@ class B2GXPCShellRemote(XPCShellRemote):
self.env['LD_LIBRARY_PATH'] = '/system/b2g'
self.options.use_device_libs = True
def verifyRemoteOptions(parser, options):
if options.b2g_path is None:
parser.error("Need to specify a --b2gpath")
class B2GOptions(RemoteXPCShellOptions):
if options.geckoPath and not options.emulator:
parser.error("You must specify --emulator if you specify --gecko-path")
def __init__(self):
RemoteXPCShellOptions.__init__(self)
defaults = {}
if options.logdir and not options.emulator:
parser.error("You must specify --emulator if you specify --logdir")
return remotexpcshelltests.verifyRemoteOptions(parser, options)
self.add_option('--b2gpath', action='store',
type='string', dest='b2g_path',
help="Path to B2G repo or qemu dir")
defaults['b2g_path'] = None
def run_remote_xpcshell(parser, options, log):
options = verifyRemoteOptions(parser, options)
self.add_option('--emupath', action='store',
type='string', dest='emu_path',
help="Path to emulator folder (if different "
"from b2gpath")
self.add_option('--no-clean', action='store_false',
dest='clean',
help="Do not clean TESTROOT. Saves [lots of] time")
defaults['clean'] = True
defaults['emu_path'] = None
self.add_option('--emulator', action='store',
type='string', dest='emulator',
help="Architecture of emulator to use: x86 or arm")
defaults['emulator'] = None
self.add_option('--no-window', action='store_true',
dest='no_window',
help="Pass --no-window to the emulator")
defaults['no_window'] = False
self.add_option('--adbpath', action='store',
type='string', dest='adb_path',
help="Path to adb")
defaults['adb_path'] = 'adb'
self.add_option('--address', action='store',
type='string', dest='address',
help="host:port of running Gecko instance to connect to")
defaults['address'] = None
self.add_option('--use-device-libs', action='store_true',
dest='use_device_libs',
help="Don't push .so's")
defaults['use_device_libs'] = False
self.add_option("--gecko-path", action="store",
type="string", dest="geckoPath",
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
defaults["geckoPath"] = None
self.add_option("--logdir", action="store",
type="string", dest="logdir",
help="directory to store log files")
defaults["logdir"] = None
self.add_option('--busybox', action='store',
type='string', dest='busybox',
help="Path to busybox binary to install on device")
defaults['busybox'] = None
defaults["remoteTestRoot"] = DEVICE_TEST_ROOT
defaults['dm_trans'] = 'adb'
defaults['debugger'] = None
defaults['debuggerArgs'] = None
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options):
if options.b2g_path is None:
self.error("Need to specify a --b2gpath")
if options.geckoPath and not options.emulator:
self.error("You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
self.error("You must specify --emulator if you specify --logdir")
return RemoteXPCShellOptions.verifyRemoteOptions(self, options)
def run_remote_xpcshell(parser, options, args, log):
options = parser.verifyRemoteOptions(options)
# Create the Marionette instance
kwargs = {}
@ -128,18 +195,16 @@ def run_remote_xpcshell(parser, options, log):
if not options.remoteTestRoot:
options.remoteTestRoot = dm.deviceRoot
xpcsh = B2GXPCShellRemote(dm, options, log)
xpcsh = B2GXPCShellRemote(dm, options, args, log)
# we don't run concurrent tests on mobile
options.sequential = True
if options.xpcshell is None:
options.xpcshell = "xpcshell"
try:
if not xpcsh.runTests(testClass=B2GXPCShellTestThread,
mobileArgs=xpcsh.mobileArgs,
**vars(options)):
if not xpcsh.runTests(xpcshell='xpcshell', testdirs=args[0:],
testClass=B2GXPCShellTestThread,
mobileArgs=xpcsh.mobileArgs,
**options.__dict__):
sys.exit(1)
except:
print "Automation Error: Exception caught while running tests"
@ -147,13 +212,13 @@ def run_remote_xpcshell(parser, options, log):
sys.exit(1)
def main():
parser = parser_b2g()
parser = B2GOptions()
commandline.add_logging_group(parser)
options = parser.parse_args()
options, args = parser.parse_args()
log = commandline.setup_logging("Remote XPCShell",
options,
{"tbpl": sys.stdout})
run_remote_xpcshell(parser, options, log)
run_remote_xpcshell(parser, options, args, log)
# You usually run this like :
# python runtestsb2g.py --emulator arm --b2gpath $B2GPATH --manifest $MANIFEST [--xre-path $MOZ_HOST_BIN

245
testing/xpcshell/runxpcshelltests.py
View File

@ -23,7 +23,7 @@ import traceback
from collections import deque, namedtuple
from distutils import dir_util
from multiprocessing import cpu_count
from argparse import ArgumentParser
from optparse import OptionParser
from subprocess import Popen, PIPE, STDOUT
from tempfile import mkdtemp, gettempdir
from threading import (
@ -40,9 +40,6 @@ except Exception:
HAVE_PSUTIL = False
from automation import Automation
from xpcshellcommandline import parser_desktop
SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
HARNESS_TIMEOUT = 5 * 60
@ -65,7 +62,7 @@ if os.path.isdir(mozbase):
sys.path.append(os.path.join(mozbase, package))
from manifestparser import TestManifest
from manifestparser.filters import chunk_by_slice, tags, pathprefix
from manifestparser.filters import chunk_by_slice, tags
from mozlog import commandline
import mozcrash
import mozinfo
@ -103,7 +100,7 @@ def markGotSIGINT(signum, stackFrame):
class XPCShellTestThread(Thread):
def __init__(self, test_object, event, cleanup_dir_list, retry=True,
app_dir_key=None, interactive=False,
tests_root_dir=None, app_dir_key=None, interactive=False,
verbose=False, pStdout=None, pStderr=None, keep_going=False,
log=None, **kwargs):
Thread.__init__(self)
@ -133,6 +130,7 @@ class XPCShellTestThread(Thread):
self.failureManifest = kwargs.get('failureManifest')
self.stack_fixer_function = kwargs.get('stack_fixer_function')
self.tests_root_dir = tests_root_dir
self.app_dir_key = app_dir_key
self.interactive = interactive
self.verbose = verbose
@ -415,7 +413,7 @@ class XPCShellTestThread(Thread):
return (list(sanitize_list(headlist, 'head')),
list(sanitize_list(taillist, 'tail')))
def buildXpcsCmd(self):
def buildXpcsCmd(self, testdir):
"""
Load the root head.js file as the first file in our test path, before other head, test, and tail files.
On a remote system, we overload this to add additional command line arguments, so this gets overloaded.
@ -617,7 +615,7 @@ class XPCShellTestThread(Thread):
self.tempDir = self.setupTempDir()
self.mozInfoJSPath = self.setupMozinfoJS()
self.buildXpcsCmd()
self.buildXpcsCmd(test_dir)
head_files, tail_files = self.getHeadAndTailFiles(self.test_object)
cmdH = self.buildCmdHead(head_files, tail_files, self.xpcsCmd)
@ -770,50 +768,30 @@ class XPCShellTests(object):
self.harness_timeout = HARNESS_TIMEOUT
self.nodeProc = {}
def getTestManifest(self, manifest):
if isinstance(manifest, TestManifest):
return manifest
elif manifest is not None:
manifest = os.path.normpath(os.path.abspath(manifest))
if os.path.isfile(manifest):
return TestManifest([manifest], strict=True)
else:
ini_path = os.path.join(manifest, "xpcshell.ini")
else:
ini_path = os.path.join(SCRIPT_DIR, "tests", "xpcshell.ini")
if os.path.exists(ini_path):
return TestManifest([ini_path], strict=True)
else:
print >> sys.stderr, ("Failed to find manifest at %s; use --manifest "
"to set path explicitly." % (ini_path,))
sys.exit(1)
def buildTestList(self, test_tags=None, test_paths=None):
def buildTestList(self, test_tags=None):
"""
read the xpcshell.ini manifest and set self.alltests to be
an array of test objects.
if we are chunking tests, it will be done here as well
"""
if test_paths is None:
test_paths = []
if len(test_paths) == 1 and test_paths[0].endswith(".js"):
self.singleFile = os.path.basename(test_paths[0])
if isinstance(self.manifest, TestManifest):
mp = self.manifest
else:
self.singleFile = None
mp = TestManifest(strict=True)
if self.manifest is None:
for testdir in self.testdirs:
if testdir:
mp.read(os.path.join(testdir, 'xpcshell.ini'))
else:
mp.read(self.manifest)
mp = self.getTestManifest(self.manifest)
self.buildTestPath()
filters = []
if test_tags:
filters.append(tags(test_tags))
if test_paths:
filters.append(pathprefix(test_paths))
if self.singleFile is None and self.totalChunks > 1:
filters.append(chunk_by_slice(self.thisChunk, self.totalChunks))
try:
@ -940,6 +918,31 @@ class XPCShellTests(object):
pStderr = STDOUT
return pStdout, pStderr
def buildTestPath(self):
"""
If we specifiy a testpath, set the self.testPath variable to be the given directory or file.
|testPath| will be the optional path only, or |None|.
|singleFile| will be the optional test only, or |None|.
"""
self.singleFile = None
if self.testPath is not None:
if self.testPath.endswith('.js'):
# Split into path and file.
if self.testPath.find('/') == -1:
# Test only.
self.singleFile = self.testPath
else:
# Both path and test.
# Reuse |testPath| temporarily.
self.testPath = self.testPath.rsplit('/', 1)
self.singleFile = self.testPath[1]
self.testPath = self.testPath[0]
else:
# Path only.
# Simply remove optional ending separator.
self.testPath = self.testPath.rstrip("/")
def verifyDirPath(self, dirname):
"""
Simple wrapper to get the absolute path for a given directory name.
@ -1041,17 +1044,16 @@ class XPCShellTests(object):
return '%s:%s' % (os.path.basename(test_object['ancestor-manifest']), path)
return path
def runTests(self, xpcshell=None, xrePath=None, appPath=None, symbolsPath=None,
manifest=None, testPaths=None, mobileArgs=None,
def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
manifest=None, testdirs=None, testPath=None, mobileArgs=None,
interactive=False, verbose=False, keepGoing=False, logfiles=True,
thisChunk=1, totalChunks=1, debugger=None,
debuggerArgs=None, debuggerInteractive=False,
profileName=None, mozInfo=None, sequential=False, shuffle=False,
testingModulesDir=None, pluginsPath=None,
testsRootDir=None, testingModulesDir=None, pluginsPath=None,
testClass=XPCShellTestThread, failureManifest=None,
log=None, stream=None, jsDebugger=False, jsDebuggerPort=0,
test_tags=None, dump_tests=None, utility_path=None,
rerun_failures=False, failure_manifest=None, **otherOptions):
test_tags=None, dump_tests=None, utility_path=None, **otherOptions):
"""Run xpcshell tests.
|xpcshell|, is the xpcshell executable to use to run the tests.
@ -1061,8 +1063,9 @@ class XPCShellTests(object):
breakpad symbols for processing crashes in tests.
|manifest|, if provided, is a file containing a list of
test directories to run.
|testPaths|, if provided, is a list of paths to files or directories containing
tests to run.
|testdirs|, if provided, is a list of absolute paths of test directories.
No-manifest only option.
|testPath|, if provided, indicates a single path and/or test to run.
|pluginsPath|, if provided, custom plugins directory to be returned from
the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST.
|interactive|, if set to True, indicates to provide an xpcshell prompt
@ -1080,6 +1083,8 @@ class XPCShellTests(object):
directory if running only a subset of tests.
|mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
|shuffle|, if True, execute tests in random order.
|testsRootDir|, absolute path to root directory of all tests. This is used
by xUnit generation to determine the package name of the tests.
|testingModulesDir|, if provided, specifies where JS modules reside.
xpcshell will register a resource handler mapping this path.
|otherOptions| may be present for the convenience of subclasses
@ -1087,6 +1092,9 @@ class XPCShellTests(object):
global gotSIGINT
if testdirs is None:
testdirs = []
# Try to guess modules directory.
# This somewhat grotesque hack allows the buildbot machines to find the
# modules directory without having to configure the buildbot hosts. This
@ -1099,16 +1107,6 @@ class XPCShellTests(object):
if os.path.isdir(possible):
testingModulesDir = possible
if rerun_failures:
if os.path.exists(failure_manifest):
rerun_manifest = os.path.join(os.path.dirname(failure_manifest), "rerun.ini")
shutil.copyfile(failure_manifest, rerun_manifest)
os.remove(failure_manifest)
manifest = rerun_manifest
else:
print >> sys.stderr, "No failures were found to re-run."
sys.exit(1)
if testingModulesDir:
# The resource loader expects native paths. Depending on how we were
# invoked, a UNIX style path may sneak in on Windows. We try to
@ -1137,6 +1135,8 @@ class XPCShellTests(object):
self.appPath = appPath
self.symbolsPath = symbolsPath
self.manifest = manifest
self.testdirs = testdirs
self.testPath = testPath
self.dump_tests = dump_tests
self.interactive = interactive
self.verbose = verbose
@ -1149,7 +1149,11 @@ class XPCShellTests(object):
self.testingModulesDir = testingModulesDir
self.pluginsPath = pluginsPath
self.sequential = sequential
self.failure_manifest = failure_manifest
if not testdirs and not manifest:
# nothing to test!
self.log.error("Error: No test dirs or test manifest specified!")
return False
self.testCount = 0
self.passCount = 0
@ -1202,7 +1206,7 @@ class XPCShellTests(object):
pStdout, pStderr = self.getPipes()
self.buildTestList(test_tags, testPaths)
self.buildTestList(test_tags)
if self.singleFile:
self.sequential = True
@ -1230,7 +1234,7 @@ class XPCShellTests(object):
'logfiles': self.logfiles,
'xpcshell': self.xpcshell,
'xpcsRunArgs': self.xpcsRunArgs,
'failureManifest': self.failure_manifest,
'failureManifest': failureManifest,
'harness_timeout': self.harness_timeout,
'stack_fixer_function': self.stack_fixer_function,
}
@ -1275,10 +1279,13 @@ class XPCShellTests(object):
if self.singleFile and not path.endswith(self.singleFile):
continue
if self.testPath and path.find(self.testPath) == -1:
continue
self.testCount += 1
test = testClass(test_object, self.event, self.cleanup_dir_list,
app_dir_key=appDirKey,
tests_root_dir=testsRootDir, app_dir_key=appDirKey,
interactive=interactive,
verbose=verbose or test_object.get("verbose") == "true",
pStdout=pStdout, pStderr=pStderr,
@ -1370,7 +1377,7 @@ class XPCShellTests(object):
self.log.info("Retrying tests that failed when run in parallel.")
for test_object in self.try_again_list:
test = testClass(test_object, self.event, self.cleanup_dir_list,
retry=False,
retry=False, tests_root_dir=testsRootDir,
app_dir_key=appDirKey, interactive=interactive,
verbose=verbose, pStdout=pStdout, pStderr=pStderr,
keep_going=keepGoing, log=self.log, mobileArgs=mobileArgs,
@ -1423,16 +1430,122 @@ class XPCShellTests(object):
self.log.suite_end()
return self.failCount == 0
class XPCShellOptions(OptionParser):
def __init__(self):
"""Process command line arguments and call runTests() to do the real work."""
OptionParser.__init__(self)
self.add_option("--app-path",
type="string", dest="appPath", default=None,
help="application directory (as opposed to XRE directory)")
self.add_option("--interactive",
action="store_true", dest="interactive", default=False,
help="don't automatically run tests, drop to an xpcshell prompt")
self.add_option("--verbose",
action="store_true", dest="verbose", default=False,
help="always print stdout and stderr from tests")
self.add_option("--keep-going",
action="store_true", dest="keepGoing", default=False,
help="continue running tests after test killed with control-C (SIGINT)")
self.add_option("--logfiles",
action="store_true", dest="logfiles", default=True,
help="create log files (default, only used to override --no-logfiles)")
self.add_option("--dump-tests",
type="string", dest="dump_tests", default=None,
help="Specify path to a filename to dump all the tests that will be run")
self.add_option("--manifest",
type="string", dest="manifest", default=None,
help="Manifest of test directories to use")
self.add_option("--no-logfiles",
action="store_false", dest="logfiles",
help="don't create log files")
self.add_option("--sequential",
action="store_true", dest="sequential", default=False,
help="Run all tests sequentially")
self.add_option("--test-path",
type="string", dest="testPath", default=None,
help="single path and/or test filename to test")
self.add_option("--tests-root-dir",
type="string", dest="testsRootDir", default=None,
help="absolute path to directory where all tests are located. this is typically $(objdir)/_tests")
self.add_option("--testing-modules-dir",
dest="testingModulesDir", default=None,
help="Directory where testing modules are located.")
self.add_option("--test-plugin-path",
type="string", dest="pluginsPath", default=None,
help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
"By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
"to return for NS_APP_PLUGINS_DIR_LIST when queried.")
self.add_option("--total-chunks",
type = "int", dest = "totalChunks", default=1,
help = "how many chunks to split the tests up into")
self.add_option("--this-chunk",
type = "int", dest = "thisChunk", default=1,
help = "which chunk to run between 1 and --total-chunks")
self.add_option("--profile-name",
type = "string", dest="profileName", default=None,
help="name of application profile being tested")
self.add_option("--build-info-json",
type = "string", dest="mozInfo", default=None,
help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
self.add_option("--shuffle",
action="store_true", dest="shuffle", default=False,
help="Execute tests in random order")
self.add_option("--failure-manifest", dest="failureManifest",
action="store",
help="path to file where failure manifest will be written.")
self.add_option("--xre-path",
action = "store", type = "string", dest = "xrePath",
# individual scripts will set a sane default
default = None,
help = "absolute path to directory containing XRE (probably xulrunner)")
self.add_option("--symbols-path",
action = "store", type = "string", dest = "symbolsPath",
default = None,
help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
self.add_option("--debugger",
action = "store", dest = "debugger",
help = "use the given debugger to launch the application")
self.add_option("--debugger-args",
action = "store", dest = "debuggerArgs",
help = "pass the given args to the debugger _before_ "
"the application on the command line")
self.add_option("--debugger-interactive",
action = "store_true", dest = "debuggerInteractive",
help = "prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_option("--jsdebugger", dest="jsDebugger", action="store_true",
help="Waits for a devtools JS debugger to connect before "
"starting the test.")
self.add_option("--jsdebugger-port", type="int", dest="jsDebuggerPort",
default=6000,
help="The port to listen on for a debugger connection if "
"--jsdebugger is specified.")
self.add_option("--tag",
action="append", dest="test_tags",
default=None,
help="filter out tests that don't have the given tag. Can be "
"used multiple times in which case the test must contain "
"at least one of the given tags.")
self.add_option("--utility-path",
action="store", dest="utility_path",
default=None,
help="Path to a directory containing utility programs, such "
"as stack fixer scripts.")
def main():
parser = parser_desktop()
parser = XPCShellOptions()
commandline.add_logging_group(parser)
options = parser.parse_args()
options, args = parser.parse_args()
log = commandline.setup_logging("XPCShell", options, {"tbpl": sys.stdout})
if options.xpcshell is None:
print >> sys.stderr, """Must provide path to xpcshell using --xpcshell"""
if len(args) < 2 and options.manifest is None or \
(len(args) < 1 and options.manifest is not None):
print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
sys.argv[0])
sys.exit(1)
xpcsh = XPCShellTests(log)
@ -1440,7 +1553,7 @@ def main():
print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
sys.exit(1)
if not xpcsh.runTests(**vars(options)):
if not xpcsh.runTests(args[0], testdirs=args[1:], **options.__dict__):
sys.exit(1)
if __name__ == '__main__':

1
testing/xpcshell/selftest.py
View File

@ -422,6 +422,7 @@ tail =
manifest=self.manifest,
mozInfo=mozinfo.info,
shuffle=shuffle,
testsRootDir=self.tempdir,
verbose=verbose,
sequential=True,
utility_path=self.utility_path),

202
testing/xpcshell/xpcshellcommandline.py
View File

@ -1,202 +0,0 @@
import argparse
def add_common_arguments(parser):
parser.add_argument("--app-path",
type=unicode, dest="appPath", default=None,
help="application directory (as opposed to XRE directory)")
parser.add_argument("--interactive",
action="store_true", dest="interactive", default=False,
help="don't automatically run tests, drop to an xpcshell prompt")
parser.add_argument("--verbose",
action="store_true", dest="verbose", default=False,
help="always print stdout and stderr from tests")
parser.add_argument("--keep-going",
action="store_true", dest="keepGoing", default=False,
help="continue running tests after test killed with control-C (SIGINT)")
parser.add_argument("--logfiles",
action="store_true", dest="logfiles", default=True,
help="create log files (default, only used to override --no-logfiles)")
parser.add_argument("--dump-tests", type=str, dest="dump_tests", default=None,
help="Specify path to a filename to dump all the tests that will be run")
parser.add_argument("--manifest",
type=unicode, dest="manifest", default=None,
help="Manifest of test directories to use")
parser.add_argument("--no-logfiles",
action="store_false", dest="logfiles",
help="don't create log files")
parser.add_argument("--sequential",
action="store_true", dest="sequential", default=False,
help="Run all tests sequentially")
parser.add_argument("--testing-modules-dir",
dest="testingModulesDir", default=None,
help="Directory where testing modules are located.")
parser.add_argument("--test-plugin-path",
type=str, dest="pluginsPath", default=None,
help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
"By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
"to return for NS_APP_PLUGINS_DIR_LIST when queried.")
parser.add_argument("--total-chunks",
type=int, dest="totalChunks", default=1,
help="how many chunks to split the tests up into")
parser.add_argument("--this-chunk",
type=int, dest="thisChunk", default=1,
help="which chunk to run between 1 and --total-chunks")
parser.add_argument("--profile-name",
type=str, dest="profileName", default=None,
help="name of application profile being tested")
parser.add_argument("--build-info-json",
type=str, dest="mozInfo", default=None,
help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
parser.add_argument("--shuffle",
action="store_true", dest="shuffle", default=False,
help="Execute tests in random order")
parser.add_argument("--xre-path",
action="store", type=str, dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
parser.add_argument("--symbols-path",
action="store", type=str, dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
parser.add_argument("--debugger",
action="store", dest="debugger",
help="use the given debugger to launch the application")
parser.add_argument("--debugger-args",
action="store", dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
parser.add_argument("--debugger-interactive",
action="store_true", dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
parser.add_argument("--jsdebugger", dest="jsDebugger", action="store_true",
help="Waits for a devtools JS debugger to connect before "
"starting the test.")
parser.add_argument("--jsdebugger-port", type=int, dest="jsDebuggerPort",
default=6000,
help="The port to listen on for a debugger connection if "
"--jsdebugger is specified.")
parser.add_argument("--tag",
action="append", dest="test_tags",
default=None,
help="filter out tests that don't have the given tag. Can be "
"used multiple times in which case the test must contain "
"at least one of the given tags.")
parser.add_argument("--utility-path",
action="store", dest="utility_path",
default=None,
help="Path to a directory containing utility programs, such "
"as stack fixer scripts.")
parser.add_argument("--xpcshell",
action="store", dest="xpcshell",
default=None,
help="Path to xpcshell binary")
# This argument can be just present, or the path to a manifest file. The
# just-present case is usually used for mach which can provide a default
# path to the failure file from the previous run
parser.add_argument("--rerun-failures",
action="store_true",
help="Rerun failures from the previous run, if any")
parser.add_argument("--failure-manifest",
action="store",
help="Path to a manifest file from which to rerun failures "
"(with --rerun-failure) or in which to record failed tests")
parser.add_argument("testPaths", nargs="*", default=None,
help="Paths of tests to run.")
def add_remote_arguments(parser):
parser.add_argument("--deviceIP", action="store", type=str, dest="deviceIP",
help="ip address of remote device to test")
parser.add_argument("--devicePort", action="store", type=str, dest="devicePort",
default=20701, help="port of remote device to test")
parser.add_argument("--dm_trans", action="store", type=str, dest="dm_trans",
choices=["adb", "sut"], default="sut",
help="the transport to use to communicate with device: [adb|sut]; default=sut")
parser.add_argument("--objdir", action="store", type=str, dest="objdir",
help="local objdir, containing xpcshell binaries")
parser.add_argument("--apk", action="store", type=str, dest="localAPK",
help="local path to Fennec APK")
parser.add_argument("--noSetup", action="store_false", dest="setup", default=True,
help="do not copy any files to device (to be used only if device is already setup)")
parser.add_argument("--local-lib-dir", action="store", type=str, dest="localLib",
help="local path to library directory")
parser.add_argument("--local-bin-dir", action="store", type=str, dest="localBin",
help="local path to bin directory")
parser.add_argument("--remoteTestRoot", action="store", type=str, dest="remoteTestRoot",
help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)")
def add_b2g_arguments(parser):
parser.add_argument('--b2gpath', action='store', type=str, dest='b2g_path',
help="Path to B2G repo or qemu dir")
parser.add_argument('--emupath', action='store', type=str, dest='emu_path',
help="Path to emulator folder (if different "
"from b2gpath")
parser.add_argument('--no-clean', action='store_false', dest='clean', default=True,
help="Do not clean TESTROOT. Saves [lots of] time")
parser.add_argument('--emulator', action='store', type=str, dest='emulator',
default="arm", choices=["x86", "arm"],
help="Architecture of emulator to use: x86 or arm")
parser.add_argument('--no-window', action='store_true', dest='no_window', default=False,
help="Pass --no-window to the emulator")
parser.add_argument('--adbpath', action='store', type=str, dest='adb_path',
default="adb", help="Path to adb")
parser.add_argument('--address', action='store', type=str, dest='address',
help="host:port of running Gecko instance to connect to")
parser.add_argument('--use-device-libs', action='store_true', dest='use_device_libs',
default=None, help="Don't push .so's")
parser.add_argument("--gecko-path", action="store", type=str, dest="geckoPath",
help="the path to a gecko distribution that should "
"be installed on the emulator prior to test")
parser.add_argument("--logdir", action="store", type=str, dest="logdir",
help="directory to store log files")
parser.add_argument('--busybox', action='store', type=str, dest='busybox',
help="Path to busybox binary to install on device")
parser.set_defaults(remoteTestRoot="/data/local/tests",
dm_trans="adb")
def parser_desktop():
parser = argparse.ArgumentParser()
add_common_arguments(parser)
return parser
def parser_remote():
parser = argparse.ArgumentParser()
common = parser.add_argument_group("Common Options")
add_common_arguments(common)
remote = parser.add_argument_group("Remote Options")
add_remote_arguments(remote)
return parser
def parser_b2g():
parser = argparse.ArgumentParser()
common = parser.add_argument_group("Common Options")
add_common_arguments(common)
remote = parser.add_argument_group("Remote Options")
add_remote_arguments(remote)
b2g = parser.add_argument_group("B2G Options")
add_b2g_arguments(b2g)
return parser