Bug 930896 - Keep track of files generated by a build backend. r=gps

Makefile.in

@@ -32,9 +32,9 @@ DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    .mozconfig.mk
 
 ifndef MOZ_PROFILE_USE
-# We need to explicitly put backend.RecursiveMakeBackend.built here
+# We need to explicitly put backend.RecursiveMakeBackend here
 # otherwise the rule in rules.mk doesn't run early enough.
-libs binaries export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
+libs binaries export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend
 ifndef LIBXUL_SDK
 libs binaries export tools:: js-config-status
 endif
@@ -63,7 +63,7 @@ config.status: $(topsrcdir)/configure
 ifndef LIBXUL_SDK
 .PHONY: js-config-status
 js-config-status:
-	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
+	$(call SUBMAKE,backend.RecursiveMakeBackend,js/src,1)
 endif
 
 install_manifests := bin idl include public private sdk
@@ -71,7 +71,7 @@ install_manifest_depends = \
   CLOBBER \
   $(topsrcdir)/configure \
   config.status \
-  backend.RecursiveMakeBackend.built \
+  backend.RecursiveMakeBackend \
   $(NULL)
 
 ifndef LIBXUL_SDK

config/rules.mk

@@ -602,17 +602,17 @@ ifndef MOZBUILD_BACKEND_CHECKED
 # Makefile/backend.mk is updated as a result of backend regeneration, we
 # actually pick up the changes. This should reduce the amount of
 # required clobbers and is thus the lesser evil.
-Makefile: $(DEPTH)/backend.RecursiveMakeBackend.built
+Makefile: $(DEPTH)/backend.RecursiveMakeBackend
 	@$(TOUCH) $@
 
-$(DEPTH)/backend.RecursiveMakeBackend.built:
+$(DEPTH)/backend.RecursiveMakeBackend:
 	@echo "Build configuration changed. Regenerating backend."
 	@cd $(DEPTH) && $(PYTHON) ./config.status
 	@$(TOUCH) $@
 
-include $(DEPTH)/backend.RecursiveMakeBackend.built.pp
+include $(DEPTH)/backend.RecursiveMakeBackend.pp
 
-default:: $(DEPTH)/backend.RecursiveMakeBackend.built
+default:: $(DEPTH)/backend.RecursiveMakeBackend
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif

dom/bindings/Makefile.in

@@ -250,7 +250,6 @@ GARBAGE += \
   parser.out \
   $(wildcard *-example.h) \
   $(wildcard *-example.cpp) \
-  $(unified_binding_cpp_files) \
   .BindingGen \
   .all-webidl-file-list \
   .generated-events-webidl-files \

ipc/ipdl/Makefile.in

@@ -8,8 +8,6 @@ GARBAGE += ipdl_lextab.py ipdl_yacctab.py $(wildcard *.pyc $(srcdir)/ipdl/*.pyc
 # This file is generated by the moz.build backend.
 include ipdlsrcs.mk
 
-GARBAGE += $(CPPSRCS)
-
 LOCAL_INCLUDES += -I$(DEPTH)/ipc/ipdl/_ipdlheaders
 
 

js/src/Makefile.in

@@ -369,7 +369,7 @@ DIST_GARBAGE = config.cache config.log config.status* \
    $(JS_CONFIG_NAME) js-config.h js-confdefs.h \
    backend.mk config/backend.mk devtools/backend.mk editline/backend.mk \
    gdb/backend.mk jsapi-tests/backend.mk shell/backend.mk tests/backend.mk \
-   backend.RecursiveMakeBackend.built backend.RecursiveMakeBackend.built.pp \
+   backend.RecursiveMakeBackend backend.RecursiveMakeBackend.pp \
    devtools/rootAnalysis/Makefile
 
 distclean::

js/src/config/rules.mk

@@ -602,17 +602,17 @@ ifndef MOZBUILD_BACKEND_CHECKED
 # Makefile/backend.mk is updated as a result of backend regeneration, we
 # actually pick up the changes. This should reduce the amount of
 # required clobbers and is thus the lesser evil.
-Makefile: $(DEPTH)/backend.RecursiveMakeBackend.built
+Makefile: $(DEPTH)/backend.RecursiveMakeBackend
 	@$(TOUCH) $@
 
-$(DEPTH)/backend.RecursiveMakeBackend.built:
+$(DEPTH)/backend.RecursiveMakeBackend:
 	@echo "Build configuration changed. Regenerating backend."
 	@cd $(DEPTH) && $(PYTHON) ./config.status
 	@$(TOUCH) $@
 
-include $(DEPTH)/backend.RecursiveMakeBackend.built.pp
+include $(DEPTH)/backend.RecursiveMakeBackend.pp
 
-default:: $(DEPTH)/backend.RecursiveMakeBackend.built
+default:: $(DEPTH)/backend.RecursiveMakeBackend
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif

python/mozbuild/mozbuild/backend/base.py

@@ -48,6 +48,9 @@ class BackendConsumeSummary(object):
         # The number of unchanged backend files.
         self.unchanged_count = 0
 
+        # The number of deleted backend files.
+        self.deleted_count = 0
+
         # The total wall time this backend spent consuming objects. If
         # the iterable passed into consume() is a generator, this includes the
         # time spent to read moz.build files.
@@ -121,6 +124,17 @@ class BuildBackend(LoggingMixin):
         # generation.
         self.backend_input_files = set()
 
+        # Files generated by the backend.
+        self._backend_output_files = set()
+
+        # Previously generated files.
+        self._backend_output_list_file = os.path.join(environment.topobjdir,
+            'backend.%s' % self.__class__.__name__)
+        self._backend_output_list = set()
+        if os.path.exists(self._backend_output_list_file):
+            self._backend_output_list.update(open(self._backend_output_list_file) \
+                                               .read().split('\n'))
+
         # Pull in Python files for this package as dependencies so backend
         # regeneration occurs if any of the code affecting it changes.
         for name, module in sys.modules.items():
@@ -199,18 +213,34 @@ class BuildBackend(LoggingMixin):
                 self.summary.mozbuild_count = obj.total_file_count
                 self.summary.mozbuild_execution_time = obj.total_execution_time
 
-        # Write out a file indicating when this backend was last generated.
-        age_file = os.path.join(self.environment.topobjdir,
-            'backend.%s.built' % self.__class__.__name__)
-        if self.summary.updated_count or self.summary.created_count or \
-                not os.path.exists(age_file):
-            with open(age_file, 'a'):
-                os.utime(age_file, None)
-
         finished_start = time.time()
         self.consume_finished()
         backend_time += time.time() - finished_start
 
+        # Purge backend files created in previous run, but not created anymore
+        delete_files = self._backend_output_list - self._backend_output_files
+        for path in delete_files:
+            try:
+                os.unlink(os.path.join(self.environment.topobjdir, path))
+                self.summary.deleted_count += 1
+            except OSError:
+                pass
+        # Remove now empty directories
+        for dir in set(os.path.dirname(d) for d in delete_files):
+            try:
+                os.removedirs(dir)
+            except OSError:
+                pass
+
+        # Write out the list of backend files generated, if it changed.
+        if self.summary.deleted_count or self.summary.created_count or \
+                not os.path.exists(self._backend_output_list_file):
+            with open(self._backend_output_list_file, 'w') as fh:
+                fh.write('\n'.join(sorted(self._backend_output_files)))
+        elif self.summary.updated_count:
+            with open(self._backend_output_list_file, 'a'):
+                os.utime(self._backend_output_list_file, None)
+
         self.summary.cpu_time = time.clock() - cpu_start
         self.summary.wall_time = time.time() - time_start
         self.summary.backend_execution_time = backend_time
@@ -259,6 +289,7 @@ class BuildBackend(LoggingMixin):
 
         yield fh
 
+        self._backend_output_files.add(os.path.relpath(fh.name, self.environment.topobjdir))
         existed, updated = fh.close()
         if not existed:
             self.summary.created_count += 1

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -268,10 +268,13 @@ class RecursiveMakeBackend(CommonBackend):
         self._generated_webidl_sources = set()
 
         def detailed(summary):
-            return '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format(
+            s = '{:d} total backend files. {:d} created; {:d} updated; {:d} unchanged'.format(
                 summary.created_count + summary.updated_count +
                 summary.unchanged_count, summary.created_count,
                 summary.updated_count, summary.unchanged_count)
+            if summary.deleted_count:
+                s+= '; {:d} deleted'.format(summary.deleted_count)
+            return s
 
         # This is a little kludgy and could be improved with a better API.
         self.summary.backend_detailed_summary = types.MethodType(detailed,
@@ -703,16 +706,14 @@ class RecursiveMakeBackend(CommonBackend):
 
         # Write out a dependency file used to determine whether a config.status
         # re-run is needed.
-        backend_built_path = os.path.join(self.environment.topobjdir,
-            'backend.%s.built' % self.__class__.__name__).replace(os.sep, '/')
         inputs = sorted(p.replace(os.sep, '/') for p in self.backend_input_files)
 
         # We need to use $(DEPTH) so the target here matches what's in
         # rules.mk. If they are different, the dependencies don't get pulled in
         # properly.
-        with self._write_file('%s.pp' % backend_built_path) as backend_deps:
-            backend_deps.write('$(DEPTH)/backend.RecursiveMakeBackend.built: %s\n' %
-                ' '.join(inputs))
+        with self._write_file('%s.pp' % self._backend_output_list_file) as backend_deps:
+            backend_deps.write('$(DEPTH)/backend.%s: %s\n' %
+                (self.__class__.__name__, ' '.join(inputs)))
             for path in inputs:
                 backend_deps.write('%s:\n' % path)
 

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -153,9 +153,9 @@ class TestRecursiveMakeBackend(BackendTester):
         """Ensure the RecursiveMakeBackend works without error."""
         env = self._consume('stub0', RecursiveMakeBackend)
         self.assertTrue(os.path.exists(os.path.join(env.topobjdir,
-            'backend.RecursiveMakeBackend.built')))
+            'backend.RecursiveMakeBackend')))
         self.assertTrue(os.path.exists(os.path.join(env.topobjdir,
-            'backend.RecursiveMakeBackend.built.pp')))
+            'backend.RecursiveMakeBackend.pp')))
 
     def test_output_files(self):
         """Ensure proper files are generated."""