mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-12-08 14:03:49 +00:00
Bug 987414 - Pass multiple test arguments to mach testing commands; r=ahal
Previously, mach xpcshell-test and mochitest-* were limited to a single test "path" argument. This patch enables multiple arguments to be passed in. TestResolver in the build system has gained the ability to process multiple paths in a single invocation. The mach commands have been modified to utilize this new feature. Only mach code paths that pass manifestdestiny.TestManifest instances into the test runner can accept multiple arguments. This is because there is no other way to pass a custom set of tests into the test runner. If multiple test arguments are used but not supported, a warning is emitted. --HG-- extra : rebase_source : 1ce1328a969f654e7b43a7a0bdd15ed86f5ceb21
This commit is contained in:
Gregory Szorc
parent
6cccf7a13e
commit
4ff214391e
@ -142,7 +142,7 @@ class TestTestMetadata(Base):
|
||||
|
||||
def test_resolve_by_dir(self):
|
||||
t = self._get_test_metadata()
|
||||
self.assertEqual(len(list(t.resolve_tests('services/common'))), 2)
|
||||
self.assertEqual(len(list(t.resolve_tests(paths=['services/common']))), 2)
|
||||
|
||||
def test_resolve_under_path(self):
|
||||
t = self._get_test_metadata()
|
||||
@ -151,6 +151,11 @@ class TestTestMetadata(Base):
|
||||
self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell',
|
||||
under_path='services'))), 2)
|
||||
|
||||
def test_resolve_multiple_paths(self):
|
||||
t = self._get_test_metadata()
|
||||
result = list(t.resolve_tests(paths=['services', 'toolkit']))
|
||||
self.assertEqual(len(result), 4)
|
||||
|
||||
|
||||
class TestTestResolver(Base):
|
||||
FAKE_TOPSRCDIR = '/Users/gps/src/firefox'
|
||||
@ -183,7 +188,7 @@ class TestTestResolver(Base):
|
||||
|
||||
# Pretend we're under '/services' and ask for 'common'. This should
|
||||
# pick up all tests from '/services/common'
|
||||
tests = list(r.resolve_tests(path='common', cwd=os.path.join(r.topsrcdir,
|
||||
tests = list(r.resolve_tests(paths=['common'], cwd=os.path.join(r.topsrcdir,
|
||||
'services')))
|
||||
|
||||
self.assertEqual(len(tests), 2)
|
||||
@ -198,14 +203,14 @@ class TestTestResolver(Base):
|
||||
|
||||
r = self._get_resolver()
|
||||
|
||||
expected = list(r.resolve_tests(path='services'))
|
||||
actual = list(r.resolve_tests(path='services', cwd='/'))
|
||||
expected = list(r.resolve_tests(paths=['services']))
|
||||
actual = list(r.resolve_tests(paths=['services'], cwd='/'))
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
actual = list(r.resolve_tests(path='services', cwd=r.topsrcdir))
|
||||
actual = list(r.resolve_tests(paths=['services'], cwd=r.topsrcdir))
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
actual = list(r.resolve_tests(path='services', cwd=r.topobjdir))
|
||||
actual = list(r.resolve_tests(paths=['services'], cwd=r.topobjdir))
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
|
||||
@ -69,17 +69,16 @@ class TestMetadata(object):
|
||||
for path in sorted(self._tests_by_flavor.get(flavor, [])):
|
||||
yield self._tests_by_path[path]
|
||||
|
||||
def resolve_tests(self, path=None, flavor=None, under_path=None):
|
||||
def resolve_tests(self, paths=None, flavor=None, under_path=None):
|
||||
"""Resolve tests from an identifier.
|
||||
|
||||
This is a generator of dicts describing each test.
|
||||
|
||||
If ``path`` is a known test file, the tests associated with that file
|
||||
are returned. Files can be specified by full path (relative to main
|
||||
directory), or as a file fragment. The lookup simply tests whether
|
||||
the string is in the path of a test file.
|
||||
|
||||
If ``path`` is a directory, the tests in that directory are returned.
|
||||
``paths`` can be an iterable of values to use to identify tests to run.
|
||||
If an entry is a known test file, tests associated with that file are
|
||||
returned (there may be multiple configurations for a single file). If
|
||||
an entry is a directory, all tests in that directory are returned. If
|
||||
the string appears in a known test file, that test file is considered.
|
||||
|
||||
If ``under_path`` is a string, it will be used to filter out tests that
|
||||
aren't in the specified path prefix relative to topsrcdir or the
|
||||
@ -101,30 +100,33 @@ class TestMetadata(object):
|
||||
# Make a copy so modifications don't change the source.
|
||||
yield dict(test)
|
||||
|
||||
path = mozpath.normpath(path) if path else None
|
||||
paths = paths or []
|
||||
paths = [mozpath.normpath(p) for p in paths]
|
||||
if not paths:
|
||||
paths = [None]
|
||||
|
||||
if path in self._test_dirs:
|
||||
candidates = []
|
||||
for p, tests in sorted(self._tests_by_path.items()):
|
||||
if not p.startswith(path):
|
||||
continue
|
||||
candidate_paths = set()
|
||||
|
||||
candidates.extend(tests)
|
||||
for path in sorted(paths):
|
||||
if path is None:
|
||||
candidate_paths |= set(self._tests_by_path.keys())
|
||||
continue
|
||||
|
||||
for test in fltr(candidates):
|
||||
# If the path is a directory, pull in all tests in that directory.
|
||||
if path in self._test_dirs:
|
||||
candidate_paths |= {p for p in self._tests_by_path
|
||||
if p.startswith(path)}
|
||||
continue
|
||||
|
||||
# If it's a test file, add just that file.
|
||||
candidate_paths |= {p for p in self._tests_by_path if path in p}
|
||||
|
||||
for p in sorted(candidate_paths):
|
||||
tests = self._tests_by_path[p]
|
||||
|
||||
for test in fltr(tests):
|
||||
yield test
|
||||
|
||||
return
|
||||
|
||||
# Do file lookup.
|
||||
candidates = []
|
||||
for p, tests in sorted(self._tests_by_path.items()):
|
||||
if path is None or path in p:
|
||||
candidates.extend(tests)
|
||||
|
||||
for test in fltr(candidates):
|
||||
yield test
|
||||
|
||||
|
||||
class TestResolver(MozbuildObject):
|
||||
"""Helper to resolve tests from the current environment to test files."""
|
||||
|
||||
@ -93,19 +93,21 @@ class MochitestRunner(MozbuildObject):
|
||||
self.mochitest_dir = os.path.join(self.tests_dir, 'testing', 'mochitest')
|
||||
self.bin_dir = os.path.join(self.topobjdir, 'dist', 'bin')
|
||||
|
||||
def run_b2g_test(self, test_file=None, b2g_home=None, xre_path=None,
|
||||
def run_b2g_test(self, test_paths=None, b2g_home=None, xre_path=None,
|
||||
total_chunks=None, this_chunk=None, no_window=None,
|
||||
**kwargs):
|
||||
"""Runs a b2g mochitest.
|
||||
|
||||
test_file is a path to a test file. It can be a relative path from the
|
||||
top source directory, an absolute filename, or a directory containing
|
||||
test files.
|
||||
test_paths is an enumerable of paths to tests. It can be a relative path
|
||||
from the top source directory, an absolute filename, or a directory
|
||||
containing test files.
|
||||
"""
|
||||
# Need to call relpath before os.chdir() below.
|
||||
test_path = ''
|
||||
if test_file:
|
||||
test_path = self._wrap_path_argument(test_file).relpath()
|
||||
if test_paths:
|
||||
if len(test_paths) > 1:
|
||||
print('Warning: Only the first test path will be used.')
|
||||
test_path = self._wrap_path_argument(test_paths[0]).relpath()
|
||||
|
||||
# TODO without os.chdir, chained imports fail below
|
||||
os.chdir(self.mochitest_dir)
|
||||
@ -181,7 +183,7 @@ class MochitestRunner(MozbuildObject):
|
||||
options.xrePath = xre_path
|
||||
return mochitest.run_remote_mochitests(parser, options)
|
||||
|
||||
def run_desktop_test(self, context, suite=None, test_file=None, debugger=None,
|
||||
def run_desktop_test(self, context, suite=None, test_paths=None, debugger=None,
|
||||
debugger_args=None, slowscript=False, shuffle=False, keep_open=False,
|
||||
rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False,
|
||||
slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None,
|
||||
@ -191,7 +193,7 @@ class MochitestRunner(MozbuildObject):
|
||||
install_extension=None, **kwargs):
|
||||
"""Runs a mochitest.
|
||||
|
||||
test_file is a path to a test file. It can be a relative path from the
|
||||
test_paths are path to tests. They can be a relative path from the
|
||||
top source directory, an absolute filename, or a directory containing
|
||||
test files.
|
||||
|
||||
@ -211,14 +213,13 @@ class MochitestRunner(MozbuildObject):
|
||||
keep_open denotes whether to keep the browser open after tests
|
||||
complete.
|
||||
"""
|
||||
if rerun_failures and test_file:
|
||||
if rerun_failures and test_paths:
|
||||
print('Cannot specify both --rerun-failures and a test path.')
|
||||
return 1
|
||||
|
||||
# Need to call relpath before os.chdir() below.
|
||||
test_path = ''
|
||||
if test_file:
|
||||
test_path = self._wrap_path_argument(test_file).relpath()
|
||||
if test_paths:
|
||||
test_paths = [self._wrap_path_argument(p).relpath() for p in test_paths]
|
||||
|
||||
failure_file_path = os.path.join(self.statedir, 'mochitest_failures.json')
|
||||
|
||||
@ -315,10 +316,10 @@ class MochitestRunner(MozbuildObject):
|
||||
for k, v in kwargs.iteritems():
|
||||
setattr(options, k, v)
|
||||
|
||||
if test_path:
|
||||
if test_paths:
|
||||
resolver = self._spawn(TestResolver)
|
||||
|
||||
tests = list(resolver.resolve_tests(path=test_path, flavor=flavor,
|
||||
tests = list(resolver.resolve_tests(paths=test_paths, flavor=flavor,
|
||||
cwd=context.cwd))
|
||||
|
||||
if not tests:
|
||||
@ -488,7 +489,7 @@ def MochitestCommand(func):
|
||||
help='Specifies the directory in which to place dumped memory reports.')
|
||||
func = dumpOutputDirectory(func)
|
||||
|
||||
path = CommandArgument('test_file', default=None, nargs='?',
|
||||
path = CommandArgument('test_paths', default=None, nargs='*',
|
||||
metavar='TEST',
|
||||
help='Test to run. Can be specified as a single file, a ' \
|
||||
'directory, or omitted. If omitted, the entire test suite is ' \
|
||||
@ -551,7 +552,7 @@ def B2GCommand(func):
|
||||
help='If specified, will only log subtest results on failure or timeout.')
|
||||
func = hide_subtests(func)
|
||||
|
||||
path = CommandArgument('test_file', default=None, nargs='?',
|
||||
path = CommandArgument('test_paths', default=None, nargs='*',
|
||||
metavar='TEST',
|
||||
help='Test to run. Can be specified as a single file, a ' \
|
||||
'directory, or omitted. If omitted, the entire test suite is ' \
|
||||
@ -568,52 +569,52 @@ class MachCommands(MachCommandBase):
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run a plain mochitest.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_plain(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'plain', **kwargs)
|
||||
def run_mochitest_plain(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'plain', **kwargs)
|
||||
|
||||
@Command('mochitest-chrome', category='testing',
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run a chrome mochitest.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_chrome(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'chrome', **kwargs)
|
||||
def run_mochitest_chrome(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'chrome', **kwargs)
|
||||
|
||||
@Command('mochitest-browser', category='testing',
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run a mochitest with browser chrome.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_browser(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'browser', **kwargs)
|
||||
def run_mochitest_browser(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'browser', **kwargs)
|
||||
|
||||
@Command('mochitest-metro', category='testing',
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run a mochitest with metro browser chrome.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_metro(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'metro', **kwargs)
|
||||
def run_mochitest_metro(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'metro', **kwargs)
|
||||
|
||||
@Command('mochitest-a11y', category='testing',
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run an a11y mochitest.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_a11y(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'a11y', **kwargs)
|
||||
def run_mochitest_a11y(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'a11y', **kwargs)
|
||||
|
||||
@Command('webapprt-test-chrome', category='testing',
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run a webapprt chrome mochitest.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_webapprt_chrome(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'webapprt-chrome', **kwargs)
|
||||
def run_mochitest_webapprt_chrome(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'webapprt-chrome', **kwargs)
|
||||
|
||||
@Command('webapprt-test-content', category='testing',
|
||||
conditions=[conditions.is_firefox],
|
||||
description='Run a webapprt content mochitest.')
|
||||
@MochitestCommand
|
||||
def run_mochitest_webapprt_content(self, test_file, **kwargs):
|
||||
return self.run_mochitest(test_file, 'webapprt-content', **kwargs)
|
||||
def run_mochitest_webapprt_content(self, test_paths, **kwargs):
|
||||
return self.run_mochitest(test_paths, 'webapprt-content', **kwargs)
|
||||
|
||||
def run_mochitest(self, test_file, flavor, **kwargs):
|
||||
def run_mochitest(self, test_paths, flavor, **kwargs):
|
||||
from mozbuild.controller.building import BuildDriver
|
||||
|
||||
self._ensure_state_subdir_exists('.')
|
||||
@ -624,7 +625,7 @@ class MachCommands(MachCommandBase):
|
||||
mochitest = self._spawn(MochitestRunner)
|
||||
|
||||
return mochitest.run_desktop_test(self._mach_context,
|
||||
test_file=test_file, suite=flavor, **kwargs)
|
||||
test_paths=test_paths, suite=flavor, **kwargs)
|
||||
|
||||
|
||||
# TODO For now b2g commands will only work with the emulator,
|
||||
@ -650,7 +651,7 @@ class B2GCommands(MachCommandBase):
|
||||
description='Run a remote mochitest.',
|
||||
conditions=[conditions.is_b2g, is_emulator])
|
||||
@B2GCommand
|
||||
def run_mochitest_remote(self, test_file, **kwargs):
|
||||
def run_mochitest_remote(self, test_paths, **kwargs):
|
||||
from mozbuild.controller.building import BuildDriver
|
||||
|
||||
self._ensure_state_subdir_exists('.')
|
||||
@ -660,13 +661,13 @@ class B2GCommands(MachCommandBase):
|
||||
|
||||
mochitest = self._spawn(MochitestRunner)
|
||||
return mochitest.run_b2g_test(b2g_home=self.b2g_home,
|
||||
xre_path=self.xre_path, test_file=test_file, **kwargs)
|
||||
xre_path=self.xre_path, test_paths=test_paths, **kwargs)
|
||||
|
||||
@Command('mochitest-b2g-desktop', category='testing',
|
||||
conditions=[conditions.is_b2g_desktop],
|
||||
description='Run a b2g desktop mochitest.')
|
||||
@B2GCommand
|
||||
def run_mochitest_b2g_desktop(self, test_file, **kwargs):
|
||||
def run_mochitest_b2g_desktop(self, test_paths, **kwargs):
|
||||
from mozbuild.controller.building import BuildDriver
|
||||
|
||||
self._ensure_state_subdir_exists('.')
|
||||
@ -675,4 +676,4 @@ class B2GCommands(MachCommandBase):
|
||||
driver.install_tests(remove=False)
|
||||
|
||||
mochitest = self._spawn(MochitestRunner)
|
||||
return mochitest.run_b2g_test(test_file=test_file, **kwargs)
|
||||
return mochitest.run_b2g_test(test_paths=test_paths, **kwargs)
|
||||
|
||||
@ -62,7 +62,7 @@ class XPCShellRunner(MozbuildObject):
|
||||
|
||||
return self._run_xpcshell_harness(manifest=manifest, **kwargs)
|
||||
|
||||
def run_test(self, test_file, interactive=False,
|
||||
def run_test(self, test_paths, interactive=False,
|
||||
keep_going=False, sequential=False, shuffle=False,
|
||||
debugger=None, debuggerArgs=None, debuggerInteractive=None,
|
||||
rerun_failures=False,
|
||||
@ -77,7 +77,7 @@ class XPCShellRunner(MozbuildObject):
|
||||
if build_path not in sys.path:
|
||||
sys.path.append(build_path)
|
||||
|
||||
if test_file == 'all':
|
||||
if test_paths == ['all']:
|
||||
self.run_suite(interactive=interactive,
|
||||
keep_going=keep_going, shuffle=shuffle, sequential=sequential,
|
||||
debugger=debugger, debuggerArgs=debuggerArgs,
|
||||
@ -86,7 +86,7 @@ class XPCShellRunner(MozbuildObject):
|
||||
return
|
||||
|
||||
resolver = self._spawn(TestResolver)
|
||||
tests = list(resolver.resolve_tests(path=test_file, flavor='xpcshell',
|
||||
tests = list(resolver.resolve_tests(paths=test_paths, flavor='xpcshell',
|
||||
cwd=self.cwd))
|
||||
|
||||
if not tests:
|
||||
@ -218,7 +218,7 @@ class AndroidXPCShellRunner(MozbuildObject):
|
||||
|
||||
"""Run Android xpcshell tests."""
|
||||
def run_test(self,
|
||||
test_file, keep_going,
|
||||
test_paths, keep_going,
|
||||
devicemanager, ip, port, remote_test_root, no_setup, local_apk,
|
||||
# ignore parameters from other platforms' options
|
||||
**kwargs):
|
||||
@ -258,13 +258,15 @@ class AndroidXPCShellRunner(MozbuildObject):
|
||||
else:
|
||||
raise Exception("You must specify an APK")
|
||||
|
||||
if test_file == 'all':
|
||||
if test_paths == ['all']:
|
||||
testdirs = []
|
||||
options.testPath = None
|
||||
options.verbose = False
|
||||
else:
|
||||
testdirs = test_file
|
||||
options.testPath = test_file
|
||||
if len(test_paths) > 1:
|
||||
print('Warning: only the first test path argument will be used.')
|
||||
testdirs = test_paths[0]
|
||||
options.testPath = test_paths[0]
|
||||
options.verbose = True
|
||||
dummy_log = StringIO()
|
||||
xpcshell = remotexpcshelltests.XPCShellRemote(dm, options, args=testdirs, log=dummy_log)
|
||||
@ -322,7 +324,7 @@ class B2GXPCShellRunner(MozbuildObject):
|
||||
f.write(data.read())
|
||||
return busybox_path
|
||||
|
||||
def run_test(self, test_file, b2g_home=None, busybox=None,
|
||||
def run_test(self, test_paths, b2g_home=None, busybox=None,
|
||||
# ignore parameters from other platforms' options
|
||||
**kwargs):
|
||||
try:
|
||||
@ -334,8 +336,11 @@ class B2GXPCShellRunner(MozbuildObject):
|
||||
sys.exit(1)
|
||||
|
||||
test_path = None
|
||||
if test_file:
|
||||
test_path = self._wrap_path_argument(test_file).relpath()
|
||||
if test_paths:
|
||||
if len(test_paths) > 1:
|
||||
print('Warning: Only the first test path will be used.')
|
||||
|
||||
test_path = self._wrap_path_argument(test_paths[0]).relpath()
|
||||
|
||||
import runtestsb2g
|
||||
parser = runtestsb2g.B2GOptions()
|
||||
@ -378,7 +383,7 @@ class MachCommands(MachCommandBase):
|
||||
@Command('xpcshell-test', category='testing',
|
||||
conditions=[is_platform_supported],
|
||||
description='Run XPCOM Shell tests.')
|
||||
@CommandArgument('test_file', default='all', nargs='?', metavar='TEST',
|
||||
@CommandArgument('test_paths', default='all', nargs='*', metavar='TEST',
|
||||
help='Test to run. Can be specified as a single JS file, a directory, '
|
||||
'or omitted. If omitted, the entire test suite is executed.')
|
||||
@CommandArgument("--debugger", default=None, metavar='DEBUGGER',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user