Bug 1253436 - Write out a machine readable file with binaries metadata; r=glandium

This will make it easier for binaries only archive generation to consult
the list of binaries that are relevant to packaging.

This does overlap somewhat with compile databases. Perhaps someday the
logic could converge.

While I was here, I cleaned up writing of the all-tests.json file to
avoid an intermediate string variable.

This patch adds to_dict methods on some frontend data types. There is
room to improve the serialization of these types. For example, we could
use __slots__ to drive the default formatter. We could also leverage a
custom JSONEncoder class that knows how to call to_dict() on instances.
In the spirit of perfect is the enemy of done, I think this should be
deferred to a follow-up bug.

MozReview-Commit-ID: XVnKB1MNqu

--HG--
extra : rebase_source : 376c93467dde3450c7c21cff918cb34913a3c5fe

python/mozbuild/mozbuild/backend/common.py

@@ -20,6 +20,7 @@ from mozbuild.frontend.context import (
     VARIABLES,
 )
 from mozbuild.frontend.data import (
+    BaseProgram,
     ChromeManifestEntry,
     ConfigFileSubstitution,
     ExampleWebIDLInterface,
@@ -30,6 +31,7 @@ from mozbuild.frontend.data import (
     GeneratedWebIDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
+    SharedLibrary,
     TestManifest,
     TestWebIDLFile,
     UnifiedSources,
@@ -192,6 +194,14 @@ class TestManager(object):
         self.tests_by_path[key].append(t)
 
 
+class BinariesCollection(object):
+    """Tracks state of binaries produced by the build."""
+
+    def __init__(self):
+        self.shared_libraries = []
+        self.programs = []
+
+
 class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
 
@@ -199,6 +209,7 @@ class CommonBackend(BuildBackend):
         self._idl_manager = XPIDLManager(self.environment)
         self._test_manager = TestManager(self.environment)
         self._webidls = WebIDLCollection()
+        self._binaries = BinariesCollection()
         self._configs = set()
         self._ipdl_sources = set()
 
@@ -295,6 +306,15 @@ class CommonBackend(BuildBackend):
                 self._write_unified_files(obj.unified_source_mapping, obj.objdir)
             if hasattr(self, '_process_unified_sources'):
                 self._process_unified_sources(obj)
+
+        elif isinstance(obj, BaseProgram):
+            self._binaries.programs.append(obj)
+            return False
+
+        elif isinstance(obj, SharedLibrary):
+            self._binaries.shared_libraries.append(obj)
+            return False
+
         else:
             return False
 
@@ -335,10 +355,17 @@ class CommonBackend(BuildBackend):
             self.backend_input_files.add(config.source)
 
         # Write out a machine-readable file describing every test.
-        path = mozpath.join(self.environment.topobjdir, 'all-tests.json')
-        with self._write_file(path) as fh:
-            s = json.dumps(self._test_manager.tests_by_path)
-            fh.write(s)
+        topobjdir = self.environment.topobjdir
+        with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
+            json.dump(self._test_manager.tests_by_path, fh)
+
+        # Write out a machine-readable file describing binaries.
+        with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
+            d = {
+                'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
+                'programs': [p.to_dict() for p in self._binaries.programs],
+            }
+            json.dump(d, fh, sort_keys=True, indent=4)
 
     def _handle_webidl_collection(self, webidls):
         if not webidls.all_stems():

python/mozbuild/mozbuild/frontend/data.py

@@ -36,6 +36,9 @@ class TreeMetadata(object):
     """Base class for all data being captured."""
     __slots__ = ()
 
+    def to_dict(self):
+        return {k.lower(): getattr(self, k) for k in self.DICT_ATTRS}
+
 
 class ContextDerived(TreeMetadata):
     """Build object derived from a single Context instance.
@@ -357,6 +360,13 @@ class BaseProgram(Linkable):
     """
     __slots__ = ('program')
 
+    DICT_ATTRS = {
+        'install_target',
+        'kind',
+        'program',
+        'relobjdir',
+    }
+
     def __init__(self, context, program, is_unit_test=False):
         Linkable.__init__(self, context)
 
@@ -457,6 +467,15 @@ class SharedLibrary(Library):
         'symbols_file',
     )
 
+    DICT_ATTRS = {
+        'basename',
+        'import_name',
+        'install_target',
+        'lib_name',
+        'relobjdir',
+        'soname',
+    }
+
     FRAMEWORK = 1
     COMPONENT = 2
     MAX_VARIANT = 3

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -871,6 +871,32 @@ class TestRecursiveMakeBackend(BackendTester):
             'DSO_SONAME := bar\n',
         ])
 
+        self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, 'binaries.json')))
+        with open(mozpath.join(env.topobjdir, 'binaries.json'), 'rb') as fh:
+            binaries = json.load(fh)
+
+        self.assertEqual(binaries, {
+            'programs': [],
+            'shared_libraries': [
+                {
+                    'basename': 'foo',
+                    'import_name': 'foo',
+                    'install_target': 'dist/bin',
+                    'lib_name': 'foo',
+                    'relobjdir': 'foo',
+                    'soname': 'foo',
+                },
+                {
+                    'basename': 'bar',
+                    'import_name': 'bar',
+                    'install_target': 'dist/bin',
+                    'lib_name': 'bar',
+                    'relobjdir': 'bar',
+                    'soname': 'bar',
+                }
+            ],
+        })
+
 
 if __name__ == '__main__':
     main()