Bug 1337391 - Don't skip all directories that aren't traversed with a DIRS in a moz.build file. r=chmanchester

The way directory traversal is computed relies on the
RecursiveMakeTraversal class, which is used to reproduce the old
traversal order from the old entirely-in-make traversal with DIRS,
PARALLEL_DIRS, etc. because of the undeclared intra-directory
dependencies that are looming here and there.

It's fed through DirectoryTraversal objects emitted by the frontend.
Normally, DirectoryTraversal objects are emitted for a directory,
possibly giving the subdirectories defined in DIRS/TEST_DIRS its
moz.build. But in the case of gyp processing, nothing places the gyp
objdirs in some virtual DIRS of some parent moz.build since bug 1308982.

As a consequence, the corresponding entries in the
RecursiveMakeTraversal instance attached to the backend are not attached
to any parent directory. When subsequently traversing the tree from the
root, they are never found, and end up being skipped, irregarding of
their actual _no_skip status.

It would probably be possible to revert the changes from bug 1308992,
but we might as well not rely on remains from the old ways. So instead,
we make the RecursiveMakeTraversal consider directories without a
declared parent attached directly to the root directory. They don't need
to depend on any other directory anyways.

--HG--
extra : rebase_source : 17403922322a71d490fdea8db0ff16b04983ed7a

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -263,6 +263,7 @@ class RecursiveMakeTraversal(object):
 
     def __init__(self):
         self._traversal = {}
+        self._attached = set()
 
     def add(self, dir, dirs=[], tests=[]):
         """
@@ -273,7 +274,10 @@ class RecursiveMakeTraversal(object):
         subdirs = self._traversal.setdefault(dir, self.SubDirectories())
         for key, value in (('dirs', dirs), ('tests', tests)):
             assert(key in self.SubDirectoryCategories)
+            # Callers give us generators
+            value = list(value)
             getattr(subdirs, key).extend(value)
+            self._attached |= set(value)
 
     @staticmethod
     def default_filter(current, subdirs):
@@ -287,8 +291,7 @@ class RecursiveMakeTraversal(object):
         Helper function to call a filter from compute_dependencies and
         traverse.
         """
-        return filter(current, self._traversal.get(current,
-            self.SubDirectories()))
+        return filter(current, self.get_subdirs(current))
 
     def compute_dependencies(self, filter=None):
         """
@@ -357,7 +360,16 @@ class RecursiveMakeTraversal(object):
         """
         Returns all direct subdirectories under the given directory.
         """
-        return self._traversal.get(dir, self.SubDirectories())
+        result = self._traversal.get(dir, self.SubDirectories())
+        if dir == '':
+            unattached = set(self._traversal) - self._attached - set([''])
+            if unattached:
+                new_result = self.SubDirectories()
+                new_result.dirs.extend(result.dirs)
+                new_result.dirs.extend(sorted(unattached))
+                new_result.tests.extend(result.tests)
+                result = new_result
+        return result
 
 
 class RecursiveMakeBackend(CommonBackend):

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -158,6 +158,38 @@ class TestRecursiveMakeTraversal(unittest.TestCase):
             'I': ('H',),
         })
 
+    def test_traversal_parallel(self):
+        traversal = RecursiveMakeTraversal()
+        traversal.add('', dirs=['A', 'B', 'C'])
+        traversal.add('A')
+        traversal.add('B', dirs=['D', 'E', 'F'])
+        traversal.add('C', dirs=['G', 'H', 'I'])
+        traversal.add('D')
+        traversal.add('E')
+        traversal.add('F')
+        traversal.add('G')
+        traversal.add('H')
+        traversal.add('I')
+        traversal.add('J')
+
+        def filter(current, subdirs):
+            return current, subdirs.dirs, []
+
+        start, deps = traversal.compute_dependencies(filter)
+        self.assertEqual(start, ('A', 'D', 'E', 'F', 'G', 'H', 'I', 'J'))
+        self.assertEqual(deps, {
+            'A': ('',),
+            'B': ('',),
+            'C': ('',),
+            'D': ('B',),
+            'E': ('B',),
+            'F': ('B',),
+            'G': ('C',),
+            'H': ('C',),
+            'I': ('C',),
+            'J': ('',),
+        })
+
 class TestRecursiveMakeBackend(BackendTester):
     def test_basic(self):
         """Ensure the RecursiveMakeBackend works without error."""