Bug 920849 - Part 2: Discover xpcshell tests through metadata, not filesystem; r=ted

--HG--
extra : rebase_source : bbe52abe704072c11a3313d356ceedb2d0d31b6b

python/mozbuild/mozbuild/test/test_testing.py

@@ -0,0 +1,213 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import mozpack.path as mozpath
+
+from mozfile import NamedTemporaryFile
+from mozunit import main
+
+from mozbuild.base import MozbuildObject
+from mozbuild.testing import (
+    TestMetadata,
+    TestResolver,
+)
+
+
+ALL_TESTS_JSON = b'''
+{
+    "accessible/tests/mochitest/actions/test_anchors.html": [
+        {
+            "dir_relpath": "accessible/tests/mochitest/actions",
+            "expected": "pass",
+            "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html",
+            "flavor": "a11y",
+            "here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions",
+            "manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini",
+            "name": "test_anchors.html",
+            "path": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/test_anchors.html",
+            "relpath": "test_anchors.html"
+        }
+    ],
+    "services/common/tests/unit/test_async_chain.js": [
+        {
+            "dir_relpath": "services/common/tests/unit",
+            "file_relpath": "services/common/tests/unit/test_async_chain.js",
+            "firefox-appdir": "browser",
+            "flavor": "xpcshell",
+            "head": "head_global.js head_helpers.js head_http.js",
+            "here": "/Users/gps/src/firefox/services/common/tests/unit",
+            "manifest": "/Users/gps/src/firefox/services/common/tests/unit/xpcshell.ini",
+            "name": "test_async_chain.js",
+            "path": "/Users/gps/src/firefox/services/common/tests/unit/test_async_chain.js",
+            "relpath": "test_async_chain.js",
+            "tail": ""
+        }
+    ],
+    "services/common/tests/unit/test_async_querySpinningly.js": [
+        {
+            "dir_relpath": "services/common/tests/unit",
+            "file_relpath": "services/common/tests/unit/test_async_querySpinningly.js",
+            "firefox-appdir": "browser",
+            "flavor": "xpcshell",
+            "head": "head_global.js head_helpers.js head_http.js",
+            "here": "/Users/gps/src/firefox/services/common/tests/unit",
+            "manifest": "/Users/gps/src/firefox/services/common/tests/unit/xpcshell.ini",
+            "name": "test_async_querySpinningly.js",
+            "path": "/Users/gps/src/firefox/services/common/tests/unit/test_async_querySpinningly.js",
+            "relpath": "test_async_querySpinningly.js",
+            "tail": ""
+        }
+    ],
+   "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js": [
+        {
+            "dir_relpath": "toolkit/mozapps/update/test/unit",
+            "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+            "flavor": "xpcshell",
+            "generated-files": "head_update.js",
+            "head": "head_update.js",
+            "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit",
+            "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
+            "name": "test_0201_app_launch_apply_update.js",
+            "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+            "reason": "bug 820380",
+            "relpath": "test_0201_app_launch_apply_update.js",
+            "run-sequentially": "Launches application.",
+            "skip-if": "toolkit == 'gonk' || os == 'android'",
+            "support-files": "\\ndata/**\\nxpcshell_updater.ini",
+            "tail": ""
+        },
+        {
+            "dir_relpath": "toolkit/mozapps/update/test/unit",
+            "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+            "flavor": "xpcshell",
+            "generated-files": "head_update.js",
+            "head": "head_update.js head2.js",
+            "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit",
+            "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini",
+            "name": "test_0201_app_launch_apply_update.js",
+            "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js",
+            "reason": "bug 820380",
+            "relpath": "test_0201_app_launch_apply_update.js",
+            "run-sequentially": "Launches application.",
+            "skip-if": "toolkit == 'gonk' || os == 'android'",
+            "support-files": "\\ndata/**\\nxpcshell_updater.ini",
+            "tail": ""
+        }
+    ]
+}'''.strip()
+
+
+class Base(unittest.TestCase):
+    def setUp(self):
+        self._temp_files = []
+
+    def tearDown(self):
+        for f in self._temp_files:
+            del f
+
+        self._temp_files = []
+
+    def _get_test_metadata(self):
+        f = NamedTemporaryFile()
+        f.write(ALL_TESTS_JSON)
+        f.flush()
+        self._temp_files.append(f)
+
+        return TestMetadata(filename=f.name)
+
+
+class TestTestMetadata(Base):
+    def test_load(self):
+        t = self._get_test_metadata()
+        self.assertEqual(len(t._tests_by_path), 4)
+
+        self.assertEqual(len(list(t.tests_with_flavor('xpcshell'))), 3)
+        self.assertEqual(len(list(t.tests_with_flavor('mochitest-plain'))), 0)
+
+    def test_resolve_all(self):
+        t = self._get_test_metadata()
+        self.assertEqual(len(list(t.resolve_tests())), 5)
+
+    def test_resolve_filter_flavor(self):
+        t = self._get_test_metadata()
+        self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell'))), 4)
+
+    def test_resolve_by_dir(self):
+        t = self._get_test_metadata()
+        self.assertEqual(len(list(t.resolve_tests('services/common'))), 2)
+
+    def test_resolve_under_path(self):
+        t = self._get_test_metadata()
+        self.assertEqual(len(list(t.resolve_tests(under_path='services'))), 2)
+
+        self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell',
+            under_path='services'))), 2)
+
+
+class TestTestResolver(Base):
+    FAKE_TOPSRCDIR = '/Users/gps/src/firefox'
+
+    def setUp(self):
+        Base.setUp(self)
+
+        self._temp_dirs = []
+
+    def tearDown(self):
+        Base.tearDown(self)
+
+        for d in self._temp_dirs:
+            shutil.rmtree(d)
+
+    def _get_resolver(self):
+        topobjdir = tempfile.mkdtemp()
+        self._temp_dirs.append(topobjdir)
+
+        with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh:
+            fh.write(ALL_TESTS_JSON)
+
+        o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
+
+        return o._spawn(TestResolver)
+
+    def test_cwd_children_only(self):
+        """If cwd is defined, only resolve tests under the specified cwd."""
+        r = self._get_resolver()
+
+        # Pretend we're under '/services' and ask for 'common'. This should
+        # pick up all tests from '/services/common'
+        tests = list(r.resolve_tests(path='common', cwd=os.path.join(r.topsrcdir,
+            'services')))
+
+        self.assertEqual(len(tests), 2)
+
+        # Tests should be rewritten to objdir.
+        for t in tests:
+            self.assertEqual(t['here'], mozpath.join(r.topobjdir,
+                '_tests/xpcshell/services/common/tests/unit'))
+
+    def test_various_cwd(self):
+        """Test various cwd conditions are all equal."""
+
+        r = self._get_resolver()
+
+        expected = list(r.resolve_tests(path='services'))
+        actual = list(r.resolve_tests(path='services', cwd='/'))
+        self.assertEqual(actual, expected)
+
+        actual = list(r.resolve_tests(path='services', cwd=r.topsrcdir))
+        self.assertEqual(actual, expected)
+
+        actual = list(r.resolve_tests(path='services', cwd=r.topobjdir))
+        self.assertEqual(actual, expected)
+
+
+if __name__ == '__main__':
+    main()

python/mozbuild/mozbuild/testing.py

@@ -0,0 +1,179 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import json
+import os
+
+import mozpack.path as mozpath
+
+from .base import MozbuildObject
+from .util import DefaultOnReadDict
+
+
+def rewrite_test_base(test, new_base):
+    """Rewrite paths in a test to be under a new base path.
+
+    This is useful for running tests from a separate location from where they
+    were defined.
+    """
+    test['here'] = mozpath.join(new_base, test['dir_relpath'])
+    test['path'] = mozpath.join(new_base, test['file_relpath'])
+
+    return test
+
+
+class TestMetadata(object):
+    """Holds information about tests.
+
+    This class provides an API to query tests active in the build
+    configuration.
+    """
+
+    def __init__(self, filename=None):
+        self._tests_by_path = DefaultOnReadDict({}, global_default=[])
+        self._tests_by_flavor = DefaultOnReadDict({}, global_default=set())
+        self._test_dirs = set()
+
+        if filename:
+            with open(filename, 'rt') as fh:
+                d = json.load(fh)
+
+                for path, tests in d.items():
+                    for metadata in tests:
+                        self._tests_by_path[path].append(metadata)
+                        self._test_dirs.add(os.path.dirname(path))
+
+                        flavor = metadata.get('flavor')
+                        self._tests_by_flavor[flavor].add(path)
+
+    def tests_with_flavor(self, flavor):
+        """Obtain all tests having the specified flavor.
+
+        This is a generator of dicts describing each test.
+        """
+
+        for path in sorted(self._tests_by_flavor.get(flavor, [])):
+            yield self._tests_by_path[path]
+
+    def resolve_tests(self, path=None, flavor=None, under_path=None):
+        """Resolve tests from an identifier.
+
+        This is a generator of dicts describing each test.
+
+        If ``path`` is a known test file, the tests associated with that file
+        are returned. Files can be specified by full path (relative to main
+        directory), or as a file fragment. The lookup simply tests whether
+        the string is in the path of a test file.
+
+        If ``path`` is a directory, the tests in that directory are returned.
+
+        If ``under_path`` is a string, it will be used to filter out tests that
+        aren't in the specified path prefix relative to topsrcdir or the
+        test's installed dir.
+
+        If ``flavor`` is a string, it will be used to filter returned tests
+        to only be the flavor specified. A flavor is something like
+        ``xpcshell``.
+        """
+        def fltr(tests):
+            for test in tests:
+                if flavor and test.get('flavor') != flavor:
+                    continue
+
+                if under_path \
+                    and not test['file_relpath'].startswith(under_path):
+                    continue
+
+                # Make a copy so modifications don't change the source.
+                yield dict(test)
+
+        path = mozpath.normpath(path) if path else None
+
+        if path in self._test_dirs:
+            candidates = []
+            for p, tests in sorted(self._tests_by_path.items()):
+                if not p.startswith(path):
+                    continue
+
+                candidates.extend(tests)
+
+            for test in fltr(candidates):
+                yield test
+
+            return
+
+        # Do file lookup.
+        candidates = []
+        for p, tests in sorted(self._tests_by_path.items()):
+            if path is None or path in p:
+                candidates.extend(tests)
+
+        for test in fltr(candidates):
+            yield test
+
+
+class TestResolver(MozbuildObject):
+    """Helper to resolve tests from the current environment to test files."""
+
+    def __init__(self, *args, **kwargs):
+        MozbuildObject.__init__(self, *args, **kwargs)
+
+        self._tests = TestMetadata(filename=os.path.join(self.topobjdir,
+            'all-tests.json'))
+        self._test_rewrites = {
+            'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
+                'mochitest', 'a11y'),
+            'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing',
+                'mochitest', 'browser'),
+            'chrome': os.path.join(self.topobjdir, '_tests', 'testing',
+                'mochitest', 'chrome'),
+            'mochitest': os.path.join(self.topobjdir, '_tests', 'testing',
+                'mochitest', 'tests'),
+            'xpcshell': os.path.join(self.topobjdir, '_tests', 'xpcshell'),
+        }
+
+    def resolve_tests(self, cwd=None, **kwargs):
+        """Resolve tests in the context of the current environment.
+
+        This is a more intelligent version of TestMetadata.resolve_tests().
+
+        This function provides additional massaging and filtering of low-level
+        results.
+
+        Paths in returned tests are automatically translated to the paths in
+        the _tests directory under the object directory.
+
+        If cwd is defined, we will limit our results to tests under the
+        directory specified. The directory should be defined as an absolute
+        path under topsrcdir or topobjdir for it to work properly.
+        """
+        rewrite_base = None
+
+        if cwd:
+            norm_cwd = mozpath.normpath(cwd)
+            norm_srcdir = mozpath.normpath(self.topsrcdir)
+            norm_objdir = mozpath.normpath(self.topobjdir)
+
+            reldir = None
+
+            if norm_cwd.startswith(norm_objdir):
+                reldir = norm_cwd[len(norm_objdir)+1:]
+            elif norm_cwd.startswith(norm_srcdir):
+                reldir = norm_cwd[len(norm_srcdir)+1:]
+
+            result = self._tests.resolve_tests(under_path=reldir,
+                **kwargs)
+
+        else:
+            result = self._tests.resolve_tests(**kwargs)
+
+        for test in result:
+            rewrite_base = self._test_rewrites.get(test['flavor'], None)
+
+            if rewrite_base:
+                yield rewrite_test_base(test, rewrite_base)
+            else:
+                yield test

testing/xpcshell/mach_commands.py

@@ -58,6 +58,9 @@ class XPCShellRunner(MozbuildObject):
                  # ignore parameters from other platforms' options
                  **kwargs):
         """Runs an individual xpcshell test."""
+        from mozbuild.testing import TestResolver
+        from manifestparser import TestManifest
+
         # TODO Bug 794506 remove once mach integrates with virtualenv.
         build_path = os.path.join(self.topobjdir, 'build')
         if build_path not in sys.path:
@@ -71,40 +74,31 @@ class XPCShellRunner(MozbuildObject):
                            rerun_failures=rerun_failures)
             return
 
-        path_arg = self._wrap_path_argument(test_file)
+        resolver = self._spawn(TestResolver)
+        tests = list(resolver.resolve_tests(path=test_file, flavor='xpcshell',
+            cwd=self.cwd))
 
-        test_obj_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell',
-            path_arg.relpath())
-        if os.path.isfile(test_obj_dir):
-            test_obj_dir = mozpack.path.dirname(test_obj_dir)
+        if not tests:
+            raise InvalidTestPathError('We could not find an xpcshell test '
+                'for the passed test path. Please select a path that is '
+                'a test file or is a directory containing xpcshell tests.')
 
-        xpcshell_dirs = []
-        for base, dirs, files in os.walk(test_obj_dir):
-            if os.path.exists(mozpack.path.join(base, 'xpcshell.ini')):
-                xpcshell_dirs.append(base)
-
-        if not xpcshell_dirs:
-            raise InvalidTestPathError('An xpcshell.ini could not be found '
-                'for the passed test path. Please select a path whose '
-                'directory or subdirectories contain an xpcshell.ini file. '
-                'It is possible you received this error because the tree is '
-                'not built or tests are not enabled.')
+        # Dynamically write out a manifest holding all the discovered tests.
+        manifest = TestManifest()
+        manifest.tests.extend(tests)
 
         args = {
             'interactive': interactive,
             'keep_going': keep_going,
             'shuffle': shuffle,
             'sequential': sequential,
-            'test_dirs': xpcshell_dirs,
             'debugger': debugger,
             'debuggerArgs': debuggerArgs,
             'debuggerInteractive': debuggerInteractive,
-            'rerun_failures': rerun_failures
+            'rerun_failures': rerun_failures,
+            'manifest': manifest,
         }
 
-        if os.path.isfile(path_arg.srcdir_path()):
-            args['test_path'] = mozpack.path.basename(path_arg.srcdir_path())
-
         return self._run_xpcshell_harness(**args)
 
     def _run_xpcshell_harness(self, test_dirs=None, manifest=None,
@@ -339,6 +333,7 @@ class MachCommands(MachCommandBase):
             xpcshell = self._spawn(AndroidXPCShellRunner)
         else:
             xpcshell = self._spawn(XPCShellRunner)
+        xpcshell.cwd = self._mach_context.cwd
 
         try:
             return xpcshell.run_test(**params)

testing/xpcshell/runxpcshelltests.py

@@ -762,13 +762,17 @@ class XPCShellTests(object):
 
           if we are chunking tests, it will be done here as well
         """
-        mp = manifestparser.TestManifest(strict=False)
-        if self.manifest is None:
-            for testdir in self.testdirs:
-                if testdir:
-                    mp.read(os.path.join(testdir, 'xpcshell.ini'))
+        if isinstance(self.manifest, manifestparser.TestManifest):
+            mp = self.manifest
         else:
-            mp.read(self.manifest)
+            mp = manifestparser.TestManifest(strict=False)
+            if self.manifest is None:
+                for testdir in self.testdirs:
+                    if testdir:
+                        mp.read(os.path.join(testdir, 'xpcshell.ini'))
+            else:
+                mp.read(self.manifest)
+
         self.buildTestPath()
 
         self.alltests = mp.active_tests(**mozinfo.info)