Bug 874078 - Move RecursiveMakeBackend.built rule to rules.mk; r=glandium

CLOBBER

@@ -17,7 +17,7 @@
 #
 # Modifying this file will now automatically clobber the buildbot machines \o/
 #
-Landing bug 865806
+Refactored build system core dependencies in bug 874078.
 
 Alternative to clobber is to run ./config.status from the objdir and to
 touch the CLOBBER file in the objdir.

Makefile.in

@@ -38,6 +38,8 @@ DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
 ifndef MOZ_PROFILE_USE
+# We need to explicitly put backend.RecursiveMakeBackend.built here
+# otherwise the rule in rules.mk doesn't run early enough.
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
 	$(RM) -r $(DIST)/sdk
 	$(RM) -r $(DIST)/include
@@ -73,14 +75,6 @@ export::
 	$(MAKE) -C config export
 	$(MAKE) tier_nspr
 
-backend.RecursiveMakeBackend.built:
-	@echo "Updating build backend because of moz.build changes."
-	@$(PYTHON) ./config.status
-
-include backend.RecursiveMakeBackend.built.pp
-
-export MOZBUILD_BACKEND_CHECKED=1
-
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk

build/ConfigStatus.py

@@ -82,7 +82,7 @@ def config_status(topobjdir = '.', topsrcdir = '.',
 
     # Without -n, the current directory is meant to be the top object directory
     if not options.not_topobjdir:
-        topobjdir = '.'
+        topobjdir = os.path.abspath('.')
 
     env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
             non_global_defines=non_global_defines, substs=substs)

config/rules.mk

@@ -626,8 +626,14 @@ endif
 # per traversal, hence the ifdef and the export. This rule needs to come before
 # other rules for the default target or else it may not run in time.
 ifndef MOZBUILD_BACKEND_CHECKED
-default::
-	$(MAKE) -C $(DEPTH) backend.RecursiveMakeBackend.built
+
+$(DEPTH)/backend.RecursiveMakeBackend.built:
+	@echo "Build configuration changed. Regenerating backend."
+	@cd $(DEPTH) && $(PYTHON) ./config.status
+
+include $(DEPTH)/backend.RecursiveMakeBackend.built.pp
+
+default:: $(DEPTH)/backend.RecursiveMakeBackend.built
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif

js/src/build/ConfigStatus.py

@@ -82,7 +82,7 @@ def config_status(topobjdir = '.', topsrcdir = '.',
 
     # Without -n, the current directory is meant to be the top object directory
     if not options.not_topobjdir:
-        topobjdir = '.'
+        topobjdir = os.path.abspath('.')
 
     env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
             non_global_defines=non_global_defines, substs=substs)

js/src/config/rules.mk

@@ -626,8 +626,14 @@ endif
 # per traversal, hence the ifdef and the export. This rule needs to come before
 # other rules for the default target or else it may not run in time.
 ifndef MOZBUILD_BACKEND_CHECKED
-default::
-	$(MAKE) -C $(DEPTH) backend.RecursiveMakeBackend.built
+
+$(DEPTH)/backend.RecursiveMakeBackend.built:
+	@echo "Build configuration changed. Regenerating backend."
+	@cd $(DEPTH) && $(PYTHON) ./config.status
+
+include $(DEPTH)/backend.RecursiveMakeBackend.built.pp
+
+default:: $(DEPTH)/backend.RecursiveMakeBackend.built
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif

python/mozbuild/mozbuild/backend/base.py

@@ -108,6 +108,28 @@ class BuildBackend(LoggingMixin):
         # generation.
         self.backend_input_files = set()
 
+        # Pull in Python files for this package as dependencies so backend
+        # regeneration occurs if any of the code affecting it changes.
+        for name, module in sys.modules.items():
+            if not module or not name.startswith('mozbuild'):
+                continue
+
+            p = module.__file__
+
+            # We need to look at the actual source files as opposed to derived
+            # because there may be nothing loading these modules at build time.
+            # Assuming each .pyc comes from a .py file in the same directory is
+            # not a safe assumption. Hence the assert to catch future changes
+            # in behavior. A better solution likely involves loading all
+            # mozbuild modules at the top of the build to force .pyc
+            # generation.
+            if p.endswith('.pyc'):
+                p = p[0:-1]
+
+            assert os.path.exists(p)
+
+            self.backend_input_files.add((os.path.abspath(p)))
+
         self._environments = {}
         self._environments[environment.topobjdir] = environment
 

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -226,7 +226,12 @@ class RecursiveMakeBackend(BuildBackend):
             'backend.%s.built' % self.__class__.__name__).replace(os.sep, '/')
         backend_deps = FileAvoidWrite('%s.pp' % backend_built_path)
         inputs = sorted(p.replace(os.sep, '/') for p in self.backend_input_files)
-        backend_deps.write('%s: %s\n' % (backend_built_path, ' '.join(inputs)))
+
+        # We need to use $(DEPTH) so the target here matches what's in
+        # rules.mk. If they are different, the dependencies don't get pulled in
+        # properly.
+        backend_deps.write('$(DEPTH)/backend.RecursiveMakeBackend.built: %s\n' %
+            ' '.join(inputs))
         for path in inputs:
             backend_deps.write('%s:\n' % path)