diff --git a/python/mozbuild/mozbuild/test/test_testing.py b/python/mozbuild/mozbuild/test/test_testing.py new file mode 100644 index 000000000000..846ae2ec8f54 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_testing.py @@ -0,0 +1,213 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import shutil +import tempfile +import unittest + +import mozpack.path as mozpath + +from mozfile import NamedTemporaryFile +from mozunit import main + +from mozbuild.base import MozbuildObject +from mozbuild.testing import ( + TestMetadata, + TestResolver, +) + + +ALL_TESTS_JSON = b''' +{ + "accessible/tests/mochitest/actions/test_anchors.html": [ + { + "dir_relpath": "accessible/tests/mochitest/actions", + "expected": "pass", + "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html", + "flavor": "a11y", + "here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions", + "manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini", + "name": "test_anchors.html", + "path": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/test_anchors.html", + "relpath": "test_anchors.html" + } + ], + "services/common/tests/unit/test_async_chain.js": [ + { + "dir_relpath": "services/common/tests/unit", + "file_relpath": "services/common/tests/unit/test_async_chain.js", + "firefox-appdir": "browser", + "flavor": "xpcshell", + "head": "head_global.js head_helpers.js head_http.js", + "here": "/Users/gps/src/firefox/services/common/tests/unit", + "manifest": "/Users/gps/src/firefox/services/common/tests/unit/xpcshell.ini", + "name": "test_async_chain.js", + "path": "/Users/gps/src/firefox/services/common/tests/unit/test_async_chain.js", + "relpath": "test_async_chain.js", + "tail": "" + } + ], + "services/common/tests/unit/test_async_querySpinningly.js": [ + { + "dir_relpath": "services/common/tests/unit", + "file_relpath": "services/common/tests/unit/test_async_querySpinningly.js", + "firefox-appdir": "browser", + "flavor": "xpcshell", + "head": "head_global.js head_helpers.js head_http.js", + "here": "/Users/gps/src/firefox/services/common/tests/unit", + "manifest": "/Users/gps/src/firefox/services/common/tests/unit/xpcshell.ini", + "name": "test_async_querySpinningly.js", + "path": "/Users/gps/src/firefox/services/common/tests/unit/test_async_querySpinningly.js", + "relpath": "test_async_querySpinningly.js", + "tail": "" + } + ], + "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js": [ + { + "dir_relpath": "toolkit/mozapps/update/test/unit", + "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "flavor": "xpcshell", + "generated-files": "head_update.js", + "head": "head_update.js", + "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit", + "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini", + "name": "test_0201_app_launch_apply_update.js", + "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "reason": "bug 820380", + "relpath": "test_0201_app_launch_apply_update.js", + "run-sequentially": "Launches application.", + "skip-if": "toolkit == 'gonk' || os == 'android'", + "support-files": "\\ndata/**\\nxpcshell_updater.ini", + "tail": "" + }, + { + "dir_relpath": "toolkit/mozapps/update/test/unit", + "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "flavor": "xpcshell", + "generated-files": "head_update.js", + "head": "head_update.js head2.js", + "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit", + "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini", + "name": "test_0201_app_launch_apply_update.js", + "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "reason": "bug 820380", + "relpath": "test_0201_app_launch_apply_update.js", + "run-sequentially": "Launches application.", + "skip-if": "toolkit == 'gonk' || os == 'android'", + "support-files": "\\ndata/**\\nxpcshell_updater.ini", + "tail": "" + } + ] +}'''.strip() + + +class Base(unittest.TestCase): + def setUp(self): + self._temp_files = [] + + def tearDown(self): + for f in self._temp_files: + del f + + self._temp_files = [] + + def _get_test_metadata(self): + f = NamedTemporaryFile() + f.write(ALL_TESTS_JSON) + f.flush() + self._temp_files.append(f) + + return TestMetadata(filename=f.name) + + +class TestTestMetadata(Base): + def test_load(self): + t = self._get_test_metadata() + self.assertEqual(len(t._tests_by_path), 4) + + self.assertEqual(len(list(t.tests_with_flavor('xpcshell'))), 3) + self.assertEqual(len(list(t.tests_with_flavor('mochitest-plain'))), 0) + + def test_resolve_all(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests())), 5) + + def test_resolve_filter_flavor(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell'))), 4) + + def test_resolve_by_dir(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests('services/common'))), 2) + + def test_resolve_under_path(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests(under_path='services'))), 2) + + self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell', + under_path='services'))), 2) + + +class TestTestResolver(Base): + FAKE_TOPSRCDIR = '/Users/gps/src/firefox' + + def setUp(self): + Base.setUp(self) + + self._temp_dirs = [] + + def tearDown(self): + Base.tearDown(self) + + for d in self._temp_dirs: + shutil.rmtree(d) + + def _get_resolver(self): + topobjdir = tempfile.mkdtemp() + self._temp_dirs.append(topobjdir) + + with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh: + fh.write(ALL_TESTS_JSON) + + o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) + + return o._spawn(TestResolver) + + def test_cwd_children_only(self): + """If cwd is defined, only resolve tests under the specified cwd.""" + r = self._get_resolver() + + # Pretend we're under '/services' and ask for 'common'. This should + # pick up all tests from '/services/common' + tests = list(r.resolve_tests(path='common', cwd=os.path.join(r.topsrcdir, + 'services'))) + + self.assertEqual(len(tests), 2) + + # Tests should be rewritten to objdir. + for t in tests: + self.assertEqual(t['here'], mozpath.join(r.topobjdir, + '_tests/xpcshell/services/common/tests/unit')) + + def test_various_cwd(self): + """Test various cwd conditions are all equal.""" + + r = self._get_resolver() + + expected = list(r.resolve_tests(path='services')) + actual = list(r.resolve_tests(path='services', cwd='/')) + self.assertEqual(actual, expected) + + actual = list(r.resolve_tests(path='services', cwd=r.topsrcdir)) + self.assertEqual(actual, expected) + + actual = list(r.resolve_tests(path='services', cwd=r.topobjdir)) + self.assertEqual(actual, expected) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py new file mode 100644 index 000000000000..a8aa63c3dc43 --- /dev/null +++ b/python/mozbuild/mozbuild/testing.py @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import json +import os + +import mozpack.path as mozpath + +from .base import MozbuildObject +from .util import DefaultOnReadDict + + +def rewrite_test_base(test, new_base): + """Rewrite paths in a test to be under a new base path. + + This is useful for running tests from a separate location from where they + were defined. + """ + test['here'] = mozpath.join(new_base, test['dir_relpath']) + test['path'] = mozpath.join(new_base, test['file_relpath']) + + return test + + +class TestMetadata(object): + """Holds information about tests. + + This class provides an API to query tests active in the build + configuration. + """ + + def __init__(self, filename=None): + self._tests_by_path = DefaultOnReadDict({}, global_default=[]) + self._tests_by_flavor = DefaultOnReadDict({}, global_default=set()) + self._test_dirs = set() + + if filename: + with open(filename, 'rt') as fh: + d = json.load(fh) + + for path, tests in d.items(): + for metadata in tests: + self._tests_by_path[path].append(metadata) + self._test_dirs.add(os.path.dirname(path)) + + flavor = metadata.get('flavor') + self._tests_by_flavor[flavor].add(path) + + def tests_with_flavor(self, flavor): + """Obtain all tests having the specified flavor. + + This is a generator of dicts describing each test. + """ + + for path in sorted(self._tests_by_flavor.get(flavor, [])): + yield self._tests_by_path[path] + + def resolve_tests(self, path=None, flavor=None, under_path=None): + """Resolve tests from an identifier. + + This is a generator of dicts describing each test. + + If ``path`` is a known test file, the tests associated with that file + are returned. Files can be specified by full path (relative to main + directory), or as a file fragment. The lookup simply tests whether + the string is in the path of a test file. + + If ``path`` is a directory, the tests in that directory are returned. + + If ``under_path`` is a string, it will be used to filter out tests that + aren't in the specified path prefix relative to topsrcdir or the + test's installed dir. + + If ``flavor`` is a string, it will be used to filter returned tests + to only be the flavor specified. A flavor is something like + ``xpcshell``. + """ + def fltr(tests): + for test in tests: + if flavor and test.get('flavor') != flavor: + continue + + if under_path \ + and not test['file_relpath'].startswith(under_path): + continue + + # Make a copy so modifications don't change the source. + yield dict(test) + + path = mozpath.normpath(path) if path else None + + if path in self._test_dirs: + candidates = [] + for p, tests in sorted(self._tests_by_path.items()): + if not p.startswith(path): + continue + + candidates.extend(tests) + + for test in fltr(candidates): + yield test + + return + + # Do file lookup. + candidates = [] + for p, tests in sorted(self._tests_by_path.items()): + if path is None or path in p: + candidates.extend(tests) + + for test in fltr(candidates): + yield test + + +class TestResolver(MozbuildObject): + """Helper to resolve tests from the current environment to test files.""" + + def __init__(self, *args, **kwargs): + MozbuildObject.__init__(self, *args, **kwargs) + + self._tests = TestMetadata(filename=os.path.join(self.topobjdir, + 'all-tests.json')) + self._test_rewrites = { + 'a11y': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'a11y'), + 'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'browser'), + 'chrome': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'chrome'), + 'mochitest': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'tests'), + 'xpcshell': os.path.join(self.topobjdir, '_tests', 'xpcshell'), + } + + def resolve_tests(self, cwd=None, **kwargs): + """Resolve tests in the context of the current environment. + + This is a more intelligent version of TestMetadata.resolve_tests(). + + This function provides additional massaging and filtering of low-level + results. + + Paths in returned tests are automatically translated to the paths in + the _tests directory under the object directory. + + If cwd is defined, we will limit our results to tests under the + directory specified. The directory should be defined as an absolute + path under topsrcdir or topobjdir for it to work properly. + """ + rewrite_base = None + + if cwd: + norm_cwd = mozpath.normpath(cwd) + norm_srcdir = mozpath.normpath(self.topsrcdir) + norm_objdir = mozpath.normpath(self.topobjdir) + + reldir = None + + if norm_cwd.startswith(norm_objdir): + reldir = norm_cwd[len(norm_objdir)+1:] + elif norm_cwd.startswith(norm_srcdir): + reldir = norm_cwd[len(norm_srcdir)+1:] + + result = self._tests.resolve_tests(under_path=reldir, + **kwargs) + + else: + result = self._tests.resolve_tests(**kwargs) + + for test in result: + rewrite_base = self._test_rewrites.get(test['flavor'], None) + + if rewrite_base: + yield rewrite_test_base(test, rewrite_base) + else: + yield test diff --git a/testing/xpcshell/mach_commands.py b/testing/xpcshell/mach_commands.py index ec7c226206ef..cbb4b1f90d2b 100644 --- a/testing/xpcshell/mach_commands.py +++ b/testing/xpcshell/mach_commands.py @@ -58,6 +58,9 @@ class XPCShellRunner(MozbuildObject): # ignore parameters from other platforms' options **kwargs): """Runs an individual xpcshell test.""" + from mozbuild.testing import TestResolver + from manifestparser import TestManifest + # TODO Bug 794506 remove once mach integrates with virtualenv. build_path = os.path.join(self.topobjdir, 'build') if build_path not in sys.path: @@ -71,40 +74,31 @@ class XPCShellRunner(MozbuildObject): rerun_failures=rerun_failures) return - path_arg = self._wrap_path_argument(test_file) + resolver = self._spawn(TestResolver) + tests = list(resolver.resolve_tests(path=test_file, flavor='xpcshell', + cwd=self.cwd)) - test_obj_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell', - path_arg.relpath()) - if os.path.isfile(test_obj_dir): - test_obj_dir = mozpack.path.dirname(test_obj_dir) + if not tests: + raise InvalidTestPathError('We could not find an xpcshell test ' + 'for the passed test path. Please select a path that is ' + 'a test file or is a directory containing xpcshell tests.') - xpcshell_dirs = [] - for base, dirs, files in os.walk(test_obj_dir): - if os.path.exists(mozpack.path.join(base, 'xpcshell.ini')): - xpcshell_dirs.append(base) - - if not xpcshell_dirs: - raise InvalidTestPathError('An xpcshell.ini could not be found ' - 'for the passed test path. Please select a path whose ' - 'directory or subdirectories contain an xpcshell.ini file. ' - 'It is possible you received this error because the tree is ' - 'not built or tests are not enabled.') + # Dynamically write out a manifest holding all the discovered tests. + manifest = TestManifest() + manifest.tests.extend(tests) args = { 'interactive': interactive, 'keep_going': keep_going, 'shuffle': shuffle, 'sequential': sequential, - 'test_dirs': xpcshell_dirs, 'debugger': debugger, 'debuggerArgs': debuggerArgs, 'debuggerInteractive': debuggerInteractive, - 'rerun_failures': rerun_failures + 'rerun_failures': rerun_failures, + 'manifest': manifest, } - if os.path.isfile(path_arg.srcdir_path()): - args['test_path'] = mozpack.path.basename(path_arg.srcdir_path()) - return self._run_xpcshell_harness(**args) def _run_xpcshell_harness(self, test_dirs=None, manifest=None, @@ -339,6 +333,7 @@ class MachCommands(MachCommandBase): xpcshell = self._spawn(AndroidXPCShellRunner) else: xpcshell = self._spawn(XPCShellRunner) + xpcshell.cwd = self._mach_context.cwd try: return xpcshell.run_test(**params) diff --git a/testing/xpcshell/runxpcshelltests.py b/testing/xpcshell/runxpcshelltests.py index f171fa3e03a0..fff8069c9fba 100644 --- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -762,13 +762,17 @@ class XPCShellTests(object): if we are chunking tests, it will be done here as well """ - mp = manifestparser.TestManifest(strict=False) - if self.manifest is None: - for testdir in self.testdirs: - if testdir: - mp.read(os.path.join(testdir, 'xpcshell.ini')) + if isinstance(self.manifest, manifestparser.TestManifest): + mp = self.manifest else: - mp.read(self.manifest) + mp = manifestparser.TestManifest(strict=False) + if self.manifest is None: + for testdir in self.testdirs: + if testdir: + mp.read(os.path.join(testdir, 'xpcshell.ini')) + else: + mp.read(self.manifest) + self.buildTestPath() self.alltests = mp.active_tests(**mozinfo.info)