Bug 1242051 - Install test files to the objdir lazily rather than with each invocation of mach. r=gps

This moves test installation for test files out of the monolithic install
manifest for $objdir/_tests, and determines the test and support files
to install based on the object derived from all-tests.json. Additionally,
the files resulting from TEST_HARNESS_FILES are installed, as some tests
will depend on them.

As a result, the time to install tests when invoking the test runner will
scale with the number of tests requested to run rather than the entire set
of tests in the tree, resulting in significantly less overhead.

MozReview-Commit-ID: LeIrUVh1yD4

Makefile.in

@@ -193,9 +193,18 @@ endif
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
 
-# For compatibility
 .PHONY: install-tests
-install-tests: install-_tests
+install-tests: install-test-files
+
+# We no longer run "make install-tests" directly before running tests, but we still
+# want to depend on things like config.status, hence this target.
+.PHONY: run-tests-deps
+run-tests-deps: $(install_manifest_depends)
+
+# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
+.PHONY: install-test-files
+install-test-files:
+	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 

build/docs/test_manifests.rst

@@ -99,6 +99,18 @@ support-files
    file is selected from the base name (e.g., ``foo`` for ``/path/foo``).
    Files starting with ``/`` cannot be selected using globbing.
 
+   Some support files are used by tests across multiple directories. In
+   this case, a test depending on a support file from another directory
+   must note that dependency with the path to the required support file
+   in its own **support-files** entry. These use a syntax where paths
+   starting with ``!/`` will indicate the beginning of the path to a
+   shared support file starting from the root of the srcdir. For example,
+   if a manifest at ``dom/base/test/mochitest.ini`` has a support file,
+   ``dom/base/test/server-script.sjs``, and a mochitest in
+   ``dom/workers/test`` depends on that support file, the test manifest
+   at ``dom/workers/test/mochitest.ini`` must include
+   ``!/dom/base/test/server-script.sjs`` in its **support-files** entry.
+
 generated-files
    List of files that are generated as part of the build and don't exist in
    the source tree.

python/mozbuild/mozbuild/backend/common.py

@@ -174,16 +174,13 @@ class TestManager(object):
         self.topsrcdir = mozpath.normpath(config.topsrcdir)
 
         self.tests_by_path = defaultdict(list)
+        self.installs_by_path = defaultdict(list)
+        self.deferred_installs = set()
 
-    def add(self, t, flavor=None, topsrcdir=None):
+    def add(self, t, flavor, topsrcdir):
         t = dict(t)
         t['flavor'] = flavor
 
-        if topsrcdir is None:
-            topsrcdir = self.topsrcdir
-        else:
-            topsrcdir = mozpath.normpath(topsrcdir)
-
         path = mozpath.normpath(t['path'])
         assert mozpath.basedir(path, [topsrcdir])
 
@@ -193,6 +190,16 @@ class TestManager(object):
 
         self.tests_by_path[key].append(t)
 
+    def add_installs(self, obj, topsrcdir):
+        for src, (dest, _) in obj.installs.iteritems():
+            key = src[len(topsrcdir)+1:]
+            self.installs_by_path[key].append((src, dest))
+        for src, pat, dest in obj.pattern_installs:
+            key = mozpath.join(src[len(topsrcdir)+1:], pat)
+            self.installs_by_path[key].append((src, pat, dest))
+        for path in obj.deferred_installs:
+            self.deferred_installs.add(path[2:])
+
 
 class BinariesCollection(object):
     """Tracks state of binaries produced by the build."""
@@ -218,8 +225,8 @@ class CommonBackend(BuildBackend):
 
         if isinstance(obj, TestManifest):
             for test in obj.tests:
-                self._test_manager.add(test, flavor=obj.flavor,
-                    topsrcdir=obj.topsrcdir)
+                self._test_manager.add(test, obj.flavor, obj.topsrcdir)
+            self._test_manager.add_installs(obj, obj.topsrcdir)
 
         elif isinstance(obj, XPIDLFile):
             # TODO bug 1240134 tracks not processing XPIDL files during
@@ -359,6 +366,14 @@ class CommonBackend(BuildBackend):
         with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
             json.dump(self._test_manager.tests_by_path, fh)
 
+        path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
+        with self._write_file(path) as fh:
+            json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
+                       if k in self._test_manager.deferred_installs},
+                      fh,
+                      sort_keys=True,
+                      indent=4)
+
         # Write out a machine-readable file describing binaries.
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
             d = {

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -1042,14 +1042,14 @@ class RecursiveMakeBackend(CommonBackend):
         # the manifest is listed as a duplicate.
         for source, (dest, is_test) in obj.installs.items():
             try:
-                self._install_manifests['_tests'].add_symlink(source, dest)
+                self._install_manifests['_test_files'].add_symlink(source, dest)
             except ValueError:
                 if not obj.dupe_manifest and is_test:
                     raise
 
         for base, pattern, dest in obj.pattern_installs:
             try:
-                self._install_manifests['_tests'].add_pattern_symlink(base,
+                self._install_manifests['_test_files'].add_pattern_symlink(base,
                     pattern, dest)
             except ValueError:
                 if not obj.dupe_manifest:
@@ -1057,7 +1057,7 @@ class RecursiveMakeBackend(CommonBackend):
 
         for dest in obj.external_installs:
             try:
-                self._install_manifests['_tests'].add_optional_exists(dest)
+                self._install_manifests['_test_files'].add_optional_exists(dest)
             except ValueError:
                 if not obj.dupe_manifest:
                     raise

python/mozbuild/mozbuild/controller/building.py

@@ -26,8 +26,12 @@ except Exception:
 
 from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 
+import mozpack.path as mozpath
+
 from ..base import MozbuildObject
 
+from ..testing import install_test_files
+
 from ..compilation.warnings import (
     WarningsCollector,
     WarningsDatabase,
@@ -653,17 +657,19 @@ class CCacheStats(object):
 class BuildDriver(MozbuildObject):
     """Provides a high-level API for build actions."""
 
-    def install_tests(self, remove=True):
-        """Install test files (through manifest)."""
+    def install_tests(self, test_objs):
+        """Install test files."""
 
         if self.is_clobber_needed():
             print(INSTALL_TESTS_CLOBBER.format(
                   clobber_file=os.path.join(self.topobjdir, 'CLOBBER')))
             sys.exit(1)
 
-        env = {}
-        if not remove:
-            env[b'NO_REMOVE'] = b'1'
-
-        self._run_make(target='install-tests', append_env=env, pass_thru=True,
-            print_directory=False)
+        if not test_objs:
+            # If we don't actually have a list of tests to install we install
+            # test and support files wholesale.
+            self._run_make(target='install-test-files', pass_thru=True,
+                           print_directory=False)
+        else:
+            install_test_files(mozpath.normpath(self.topsrcdir), self.topobjdir,
+                               '_tests', test_objs)

python/mozbuild/mozbuild/frontend/data.py

@@ -561,6 +561,10 @@ class TestManifest(ContextDerived):
         # Set of files provided by an external mechanism.
         'external_installs',
 
+        # Set of files required by multiple test directories, whose installation
+        # will be resolved when running tests.
+        'deferred_installs',
+
         # The full path of this manifest file.
         'path',
 
@@ -602,6 +606,7 @@ class TestManifest(ContextDerived):
         self.pattern_installs = []
         self.tests = []
         self.external_installs = set()
+        self.deferred_installs = set()
 
 
 class LocalInclude(ContextDerived):

python/mozbuild/mozbuild/frontend/emitter.py

@@ -83,7 +83,7 @@ from ..testing import (
     TEST_MANIFESTS,
     REFTEST_FLAVORS,
     WEB_PLATFORM_TESTS_FLAVORS,
-    convert_support_files,
+    SupportFilesConverter,
 )
 
 from .context import (
@@ -140,6 +140,7 @@ class TreeMetadataEmitter(LoggingMixin):
 
         self._emitter_time = 0.0
         self._object_count = 0
+        self._test_files_converter = SupportFilesConverter()
 
     def summary(self):
         return ExecutionSummary(
@@ -1097,23 +1098,24 @@ class TreeMetadataEmitter(LoggingMixin):
                                                         defaults['install-to-subdir'],
                                                         mozpath.basename(path))
 
-            # Keep a set of already seen support file patterns, because
-            # repeatedly processing the patterns from the default section
-            # for every test is quite costly (see bug 922517).
-            extras = (('head', set()),
-                      ('tail', set()),
-                      ('support-files', set()),
-                      ('generated-files', set()))
             def process_support_files(test):
-                patterns, installs, external = convert_support_files(extras, test,
-                                                                     install_root,
-                                                                     manifest_dir,
-                                                                     out_dir)
+                install_info = self._test_files_converter.convert_support_files(
+                    test, install_root, manifest_dir, out_dir)
 
-                obj.pattern_installs.extend(patterns)
-                for source, dest in installs:
+                obj.pattern_installs.extend(install_info.pattern_installs)
+                for source, dest in install_info.installs:
                     obj.installs[source] = (dest, False)
-                obj.external_installs |= external
+                obj.external_installs |= install_info.external_installs
+                for install_path in install_info.deferred_installs:
+                    if all(['*' not in install_path,
+                            not os.path.isfile(mozpath.join(context.config.topsrcdir,
+                                                            install_path[2:])),
+                            install_path not in install_info.external_installs]):
+                        raise SandboxValidationError('Error processing test '
+                           'manifest %s: entry in support-files not present '
+                           'in the srcdir: %s' % (path, install_path), context)
+
+                obj.deferred_installs |= install_info.deferred_installs
 
             for test in filtered:
                 obj.tests.append(test)

python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+support-files =
+  another-file.sjs
+  data/**
+
+[test_sub.js]

python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini

@@ -0,0 +1,8 @@
+[DEFAULT]
+support-files =
+  support-file.txt
+  !/child/test_sub.js
+  !/child/another-file.sjs
+  !/child/data/**
+
+[test_foo.js]

python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build

@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['child/browser.ini']

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -539,7 +539,7 @@ class TestRecursiveMakeBackend(BackendTester):
         """Pattern matches in test manifests' support-files should be recorded."""
         env = self._consume('test-manifests-written', RecursiveMakeBackend)
         m = InstallManifest(path=mozpath.join(env.topobjdir,
-            '_build_manifests', 'install', '_tests'))
+            '_build_manifests', 'install', '_test_files'))
 
         # This is not the most robust test in the world, but it gets the job
         # done.
@@ -547,6 +547,46 @@ class TestRecursiveMakeBackend(BackendTester):
         self.assertEqual(len(entries), 1)
         self.assertIn('support/**', entries[0])
 
+    def test_test_manifest_deffered_installs_written(self):
+        """Shared support files are written to their own data file by the backend."""
+        env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        self.assertTrue(os.path.exists(all_tests_path))
+        test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json')
+
+        with open(test_installs_path, 'r') as fh:
+            test_installs = json.load(fh)
+
+        self.assertEqual(set(test_installs.keys()),
+                         set(['child/test_sub.js',
+                              'child/data/**',
+                              'child/another-file.sjs']))
+        for key in test_installs.keys():
+            self.assertIn(key, test_installs)
+
+        test_files_manifest = mozpath.join(env.topobjdir,
+                                           '_build_manifests',
+                                           'install',
+                                           '_test_files')
+
+        # First, read the generated for ini manifest contents.
+        m = InstallManifest(path=test_files_manifest)
+
+        # Then, synthesize one from the test-installs.json file. This should
+        # allow us to re-create a subset of the above.
+        synthesized_manifest = InstallManifest()
+        for item, installs in test_installs.items():
+            for install_info in installs:
+                if len(install_info) == 3:
+                    synthesized_manifest.add_pattern_symlink(*install_info)
+                if len(install_info) == 2:
+                    synthesized_manifest.add_symlink(*install_info)
+
+        self.assertEqual(len(synthesized_manifest), 3)
+        for item, info in synthesized_manifest._dests.items():
+            self.assertIn(item, m)
+            self.assertEqual(info, m._dests[item])
+
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
@@ -769,7 +809,7 @@ class TestRecursiveMakeBackend(BackendTester):
         env = self._consume('test-manifests-duplicate-support-files',
             RecursiveMakeBackend)
 
-        p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_tests')
+        p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_test_files')
         m = InstallManifest(p)
         self.assertIn('testing/mochitest/tests/support-file.txt', m)
 
@@ -824,7 +864,7 @@ class TestRecursiveMakeBackend(BackendTester):
         man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install')
         self.assertTrue(os.path.isdir(man_dir))
 
-        full = mozpath.join(man_dir, '_tests')
+        full = mozpath.join(man_dir, '_test_files')
         self.assertTrue(os.path.exists(full))
 
         m = InstallManifest(path=full)

python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+support-files =
+  another-file.sjs
+  data/**
+
+[test_sub.js]

python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini

@@ -0,0 +1,9 @@
+[DEFAULT]
+support-files =
+  support-file.txt
+  !/child/test_sub.js
+  !/child/another-file.sjs
+  !/child/data/**
+  !/does/not/exist.sjs
+
+[test_foo.js]

python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build

@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['child/browser.ini']

python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+support-files =
+  another-file.sjs
+  data/**
+
+[test_sub.js]

python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini

@@ -0,0 +1,8 @@
+[DEFAULT]
+support-files =
+  support-file.txt
+  !/child/test_sub.js
+  !/child/another-file.sjs
+  !/child/data/**
+
+[test_foo.js]

python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build

@@ -0,0 +1,5 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['child/browser.ini']

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -444,6 +444,33 @@ class TestEmitterBasic(unittest.TestCase):
         paths = sorted([v[0] for v in o.installs.values()])
         self.assertEqual(paths, expected)
 
+    def test_test_manifest_shared_support_files(self):
+        """Support files starting with '!' are given separate treatment, so their
+        installation can be resolved when running tests.
+        """
+        reader = self.reader('test-manifest-shared-support')
+        supported, child = self.read_topsrcdir(reader)
+
+        expected_deferred_installs = {
+            '!/child/test_sub.js',
+            '!/child/another-file.sjs',
+            '!/child/data/**',
+        }
+
+        self.assertEqual(len(supported.installs), 3)
+        self.assertEqual(set(supported.deferred_installs),
+                         expected_deferred_installs)
+        self.assertEqual(len(child.installs), 3)
+        self.assertEqual(len(child.pattern_installs), 1)
+
+    def test_test_manifest_deffered_install_missing(self):
+        """A non-existent shared support file reference produces an error."""
+        reader = self.reader('test-manifest-shared-missing')
+
+        with self.assertRaisesRegexp(SandboxValidationError,
+                                     'entry in support-files not present in the srcdir'):
+            self.read_topsrcdir(reader)
+
     def test_test_manifest_install_to_subdir(self):
         """ """
         reader = self.reader('test-manifest-install-subdir')

python/mozbuild/mozbuild/test/test_testing.py

@@ -239,6 +239,10 @@ class TestTestResolver(Base):
 
         o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
 
+        # Monkey patch the test resolver to avoid tests failing to find make
+        # due to our fake topscrdir.
+        TestResolver._run_make = lambda *a, **b: None
+
         return o._spawn(TestResolver)
 
     def test_cwd_children_only(self):

python/mozbuild/mozbuild/testing.py

@@ -10,6 +10,9 @@ import sys
 
 import mozpack.path as mozpath
 
+from mozpack.copier import FileCopier
+from mozpack.manifests import InstallManifest
+
 from .base import MozbuildObject
 from .util import OrderedDefaultDict
 from collections import defaultdict
@@ -167,6 +170,12 @@ class TestResolver(MozbuildObject):
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
+        # If installing tests is going to result in re-generating the build
+        # backend, we need to do this here, so that the updated contents of
+        # all-tests.json make it to the set of tests to run.
+        self._run_make(target='run-tests-deps', pass_thru=True,
+                       print_directory=False)
+
         self._tests = TestMetadata(filename=os.path.join(self.topobjdir,
             'all-tests.json'))
         self._test_rewrites = {
@@ -287,69 +296,184 @@ def all_test_flavors():
             list(WEB_PLATFORM_TESTS_FLAVORS) +
             ['python'])
 
-def convert_support_files(extras, test, install_root, manifest_dir, out_dir):
-    # Processes a "support-files" entry from a test object and returns
-    # the installs to perform for this test object.
-    #
-    # Arguments:
-    #  extras - Tuples used for the basis of memoization (the same support-files
-    #           in the same manifest always have the same effect).
-    #  test - The test object to process.
-    #  install_root - The directory under $objdir/_tests that will contain
-    #                 the tests for this harness (examples are "testing/mochitest",
-    #                 "xpcshell").
-    #  manifest_dir - Absoulute path to the (srcdir) directory containing the
-    #                 manifest that included this test
-    #  out_dir - The path relative to $objdir/_tests used as the destination for the
-    #            test, based on the relative path to the manifest in the srcdir,
-    #            the install_root, and 'install-to-subdir', if present in the manifest.
-    pattern_installs, installs, external = [], [], set()
-    for thing, seen in extras:
-        value = test.get(thing, '')
-        # We need to memoize on the basis of both the path and the output
-        # directory for the benefit of tests specifying 'install-to-subdir'.
-        if (value, out_dir) in seen:
+class TestInstallInfo(object):
+    def __init__(self):
+        self.pattern_installs = []
+        self.installs = []
+        self.external_installs = set()
+        self.deferred_installs = set()
+
+    def __ior__(self, other):
+        self.pattern_installs.extend(other.pattern_installs)
+        self.installs.extend(other.installs)
+        self.external_installs |= other.external_installs
+        self.deferred_installs |= other.deferred_installs
+        return self
+
+class SupportFilesConverter(object):
+    """Processes a "support-files" entry from a test object, either from
+    a parsed object from a test manifests or its representation in
+    moz.build and returns the installs to perform for this test object.
+
+    Processing the same support files multiple times will not have any further
+    effect, and the structure of the parsed objects from manifests will have a
+    lot of repeated entries, so this class takes care of memoizing.
+    """
+    def __init__(self):
+        self._fields = (('head', set()),
+                        ('tail', set()),
+                        ('support-files', set()),
+                        ('generated-files', set()))
+
+    def convert_support_files(self, test, install_root, manifest_dir, out_dir):
+        # Arguments:
+        #  test - The test object to process.
+        #  install_root - The directory under $objdir/_tests that will contain
+        #                 the tests for this harness (examples are "testing/mochitest",
+        #                 "xpcshell").
+        #  manifest_dir - Absoulute path to the (srcdir) directory containing the
+        #                 manifest that included this test
+        #  out_dir - The path relative to $objdir/_tests used as the destination for the
+        #            test, based on the relative path to the manifest in the srcdir,
+        #            the install_root, and 'install-to-subdir', if present in the manifest.
+        info = TestInstallInfo()
+        for thing, seen in self._fields:
+            value = test.get(thing, '')
+            # We need to memoize on the basis of both the path and the output
+            # directory for the benefit of tests specifying 'install-to-subdir'.
+            if (value, out_dir) in seen:
+                continue
+            seen.add((value, out_dir))
+            for pattern in value.split():
+                if thing == 'generated-files':
+                    info.external_installs.add(mozpath.normpath(mozpath.join(out_dir, pattern)))
+                # '!' indicates our syntax for inter-directory support file
+                # dependencies. These receive special handling in the backend.
+                elif pattern[0] == '!':
+                    info.deferred_installs.add(pattern)
+                # We only support globbing on support-files because
+                # the harness doesn't support * for head and tail.
+                elif '*' in pattern and thing == 'support-files':
+                    info.pattern_installs.append((manifest_dir, pattern, out_dir))
+                # "absolute" paths identify files that are to be
+                # placed in the install_root directory (no globs)
+                elif pattern[0] == '/':
+                    full = mozpath.normpath(mozpath.join(manifest_dir,
+                                                         mozpath.basename(pattern)))
+                    info.installs.append((full, mozpath.join(install_root, pattern[1:])))
+                else:
+                    full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
+                    dest_path = mozpath.join(out_dir, pattern)
+
+                    # If the path resolves to a different directory
+                    # tree, we take special behavior depending on the
+                    # entry type.
+                    if not full.startswith(manifest_dir):
+                        # If it's a support file, we install the file
+                        # into the current destination directory.
+                        # This implementation makes installing things
+                        # with custom prefixes impossible. If this is
+                        # needed, we can add support for that via a
+                        # special syntax later.
+                        if thing == 'support-files':
+                            dest_path = mozpath.join(out_dir,
+                                                     os.path.basename(pattern))
+                        # If it's not a support file, we ignore it.
+                        # This preserves old behavior so things like
+                        # head files doesn't get installed multiple
+                        # times.
+                        else:
+                            continue
+                    info.installs.append((full, mozpath.normpath(dest_path)))
+        return info
+
+def _resolve_installs(paths, topobjdir, manifest):
+    """Using the given paths as keys, find any unresolved installs noted
+    by the build backend corresponding to those keys, and add them
+    to the given manifest.
+    """
+    filename = os.path.join(topobjdir, 'test-installs.json')
+    with open(filename, 'r') as fh:
+        resolved_installs = json.load(fh)
+
+    for path in paths:
+        path = path[2:]
+        if path not in resolved_installs:
+            raise Exception('A cross-directory support file path noted in a '
+                'test manifest does not appear in any other manifest.\n "%s" '
+                'must appear in another test manifest to specify an install '
+                'for "!/%s".' % (path, path))
+        installs = resolved_installs[path]
+        for install_info in installs:
+            try:
+                if len(install_info) == 3:
+                    manifest.add_pattern_symlink(*install_info)
+                if len(install_info) == 2:
+                    manifest.add_symlink(*install_info)
+            except ValueError:
+                # A duplicate value here is pretty likely when running
+                # multiple directories at once, and harmless.
+                pass
+
+def install_test_files(topsrcdir, topobjdir, tests_root, test_objs):
+    """Installs the requested test files to the objdir. This is invoked by
+    test runners to avoid installing tens of thousands of test files when
+    only a few tests need to be run.
+    """
+    flavor_info = {flavor: (root, prefix, install)
+                   for (flavor, root, prefix, install) in TEST_MANIFESTS.values()}
+    objdir_dest = mozpath.join(topobjdir, tests_root)
+
+    converter = SupportFilesConverter()
+    install_info = TestInstallInfo()
+    for o in test_objs:
+        flavor = o['flavor']
+        if flavor not in flavor_info:
+            # This is a test flavor that isn't installed by the build system.
+            continue
+        root, prefix, install = flavor_info[flavor]
+        if not install:
+            # This flavor isn't installed to the objdir.
             continue
-        seen.add((value, out_dir))
-        for pattern in value.split():
-            if thing == 'generated-files':
-                external.add(mozpath.join(out_dir, pattern))
-            # We only support globbing on support-files because
-            # the harness doesn't support * for head and tail.
-            elif '*' in pattern and thing == 'support-files':
-                pattern_installs.append((manifest_dir, pattern, out_dir))
-            # "absolute" paths identify files that are to be
-            # placed in the install_root directory (no globs)
-            elif pattern[0] == '/':
-                full = mozpath.normpath(mozpath.join(manifest_dir,
-                                                     mozpath.basename(pattern)))
-                installs.append((full, mozpath.join(install_root, pattern[1:])))
-            else:
-                full = mozpath.normpath(mozpath.join(manifest_dir, pattern))
-                dest_path = mozpath.join(out_dir, pattern)
 
-                # If the path resolves to a different directory
-                # tree, we take special behavior depending on the
-                # entry type.
-                if not full.startswith(manifest_dir):
-                    # If it's a support file, we install the file
-                    # into the current destination directory.
-                    # This implementation makes installing things
-                    # with custom prefixes impossible. If this is
-                    # needed, we can add support for that via a
-                    # special syntax later.
-                    if thing == 'support-files':
-                        dest_path = mozpath.join(out_dir,
-                                                 os.path.basename(pattern))
-                    # If it's not a support file, we ignore it.
-                    # This preserves old behavior so things like
-                    # head files doesn't get installed multiple
-                    # times.
-                    else:
-                        continue
-                installs.append((full, mozpath.normpath(dest_path)))
+        manifest_path = o['manifest']
+        manifest_dir = mozpath.dirname(manifest_path)
+
+        out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:])
+        file_relpath = o['file_relpath']
+        source = mozpath.join(topsrcdir, file_relpath)
+        dest = mozpath.join(root, prefix, file_relpath)
+        if 'install-to-subdir' in o:
+            out_dir = mozpath.join(out_dir, o['install-to-subdir'])
+            manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path))
+            dest = mozpath.join(out_dir, manifest_relpath)
+
+        install_info.installs.append((source, dest))
+        install_info |= converter.convert_support_files(o, root,
+                                                        manifest_dir,
+                                                        out_dir)
+
+    manifest = InstallManifest()
+
+    for source, dest in set(install_info.installs):
+        if dest in install_info.external_installs:
+            continue
+        manifest.add_symlink(source, dest)
+    for base, pattern, dest in install_info.pattern_installs:
+        manifest.add_pattern_symlink(base, pattern, dest)
+
+    _resolve_installs(install_info.deferred_installs, topobjdir, manifest)
+
+    # Harness files are treated as a monolith and installed each time we run tests.
+    # Fortunately there are not very many.
+    manifest |= InstallManifest(mozpath.join(topobjdir,
+                                             '_build_manifests',
+                                             'install', tests_root))
+    copier = FileCopier()
+    manifest.populate_registry(copier)
+    copier.copy(objdir_dest,
+                remove_unaccounted=False)
 
-    return pattern_installs, installs, external
 
 # Convenience methods for test manifest reading.
 def read_manifestparser_manifest(context, manifest_path):

testing/mach_commands.py

@@ -612,7 +612,6 @@ class PushToTry(MachCommandBase):
         """
 
         from mozbuild.testing import TestResolver
-        from mozbuild.controller.building import BuildDriver
         from autotry import AutoTry
 
         print("mach try is under development, please file bugs blocking 1149670.")
@@ -645,9 +644,6 @@ class PushToTry(MachCommandBase):
         builds, platforms, tests, talos, paths, tags, extra = self.validate_args(**kwargs)
 
         if paths or tags:
-            driver = self._spawn(BuildDriver)
-            driver.install_tests(remove=False)
-
             paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
                      for item in paths]
             paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)

testing/mochitest/mach_commands.py

@@ -385,9 +385,6 @@ class MachCommands(MachCommandBase):
         from mozbuild.controller.building import BuildDriver
         self._ensure_state_subdir_exists('.')
 
-        driver = self._spawn(BuildDriver)
-        driver.install_tests(remove=False)
-
         test_paths = kwargs['test_paths']
         kwargs['test_paths'] = []
 
@@ -410,6 +407,9 @@ class MachCommands(MachCommandBase):
         if resolve_tests:
             tests = mochitest.resolve_tests(test_paths, test_objects, cwd=self._mach_context.cwd)
 
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(tests)
+
         subsuite = kwargs.get('subsuite')
         if subsuite == 'default':
             kwargs['subsuite'] = None
@@ -529,9 +529,6 @@ class RobocopCommands(MachCommandBase):
         from mozbuild.controller.building import BuildDriver
         self._ensure_state_subdir_exists('.')
 
-        driver = self._spawn(BuildDriver)
-        driver.install_tests(remove=False)
-
         test_paths = kwargs['test_paths']
         kwargs['test_paths'] = []
 
@@ -539,6 +536,8 @@ class RobocopCommands(MachCommandBase):
         resolver = self._spawn(TestResolver)
         tests = list(resolver.resolve_tests(paths=test_paths, cwd=self._mach_context.cwd,
                                             flavor='instrumentation', subsuite='robocop'))
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(tests)
 
         if len(tests) < 1:
             print(ROBOCOP_TESTS_NOT_FOUND.format('\n'.join(

testing/testsuite-targets.mk

@@ -243,7 +243,9 @@ ifeq ($(MOZ_WIDGET_TOOLKIT),gonk)
 stage-all: stage-b2g
 endif
 
-make-stage-dir:
+# Prepare _tests before any of the other staging/packaging steps.
+# make-stage-dir is a prerequisite to all the stage-* targets in testsuite-targets.mk.
+make-stage-dir: install-test-files
 	rm -rf $(PKG_STAGE)
 	$(NSINSTALL) -D $(PKG_STAGE)
 	$(NSINSTALL) -D $(PKG_STAGE)/bin

testing/xpcshell/mach_commands.py

@@ -347,13 +347,14 @@ class MachCommands(MachCommandBase):
             m.tests.extend(test_objects)
             params['manifest'] = m
 
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(test_objects)
+
         # We should probably have a utility function to ensure the tree is
         # ready to run tests. Until then, we just create the state dir (in
         # case the tree wasn't built with mach).
         self._ensure_state_subdir_exists('.')
 
-        driver = self._spawn(BuildDriver)
-        driver.install_tests(remove=False)
 
         params['log'] = structured.commandline.setup_logging("XPCShellTests",
                                                              params,