Bug 1443208 - Pre: Add force flag to GENERATED_FILES. r=ted.mielczarek

The forces on the system are such that I really need to be able to
FORCE a few RecursiveMake targets in order to make Android and Gradle
use GENERATED_FILES and LOCALIZED_GENERATED_FILES.  Over time, I hope
to avoid FORCE, but that time is not now.

MozReview-Commit-ID: 453FpnihSRK

--HG--
rename : python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data => python/mozbuild/mozbuild/test/backend/data/generated-files-force/foo-data
rename : python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py => python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-bar.py
rename : python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py => python/mozbuild/mozbuild/test/backend/data/generated-files-force/generate-foo.py
rename : python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build => python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build
rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/en-US/localized-input => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/en-US/localized-input
rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/foo-data => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/foo-data
rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/generate-foo.py => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/generate-foo.py
rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/moz.build => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build
rename : python/mozbuild/mozbuild/test/backend/data/localized-generated-files/non-localized-input => python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/non-localized-input
rename : python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build => python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build
rename : python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build => python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build
extra : rebase_source : 6820ac4b377ac596cafbdf065112738376cc473e

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -578,8 +578,15 @@ class RecursiveMakeBackend(CommonBackend):
                     backend_file.write('%s: %s ;\n' % (output, first_output))
                 backend_file.write('GARBAGE += %s\n' % output)
             backend_file.write('EXTRA_MDDEPEND_FILES += %s\n' % dep_file)
+
+            force = ''
+            if obj.force:
+                force = ' FORCE'
+            elif obj.localized:
+                force = ' $(if $(IS_LANGUAGE_REPACK),FORCE)'
+
             if obj.script:
-                backend_file.write("""{output}: {script}{inputs}{backend}{repack_force}
+                backend_file.write("""{output}: {script}{inputs}{backend}{force}
 \t$(REPORT_BUILD)
 \t$(call py_action,file_generate,{locale}{script} {method} {output} $(MDDEPDIR)/{dep_file}{inputs}{flags})
 
@@ -592,7 +599,7 @@ class RecursiveMakeBackend(CommonBackend):
            # so standard mtime dependencies won't work properly when the build is re-run
            # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set
            # in this situation, so simply force the generation to run in that case.
-           repack_force=' $(if $(IS_LANGUAGE_REPACK),FORCE)' if obj.localized else '',
+           force=force,
            locale='--locale=$(AB_CD) ' if obj.localized else '',
            script=obj.script,
            method=obj.method))

python/mozbuild/mozbuild/frontend/context.py

@@ -931,6 +931,7 @@ SchedulingComponents = ContextDerivedTypedRecord(
 GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({
     'script': unicode,
     'inputs': list,
+    'force': bool,
     'flags': list, })
 
 
@@ -1276,8 +1277,8 @@ VARIABLES = {
 
         This variable contains a list of files for the build system to
         generate at export time. The generation method may be declared
-        with optional ``script``, ``inputs`` and ``flags`` attributes on
-        individual entries.
+        with optional ``script``, ``inputs``, ``flags``, and ``force``
+        attributes on individual entries.
         If the optional ``script`` attribute is not present on an entry, it
         is assumed that rules for generating the file are present in
         the associated Makefile.in.
@@ -1315,6 +1316,11 @@ VARIABLES = {
 
         When the ``flags`` attribute is present, the given list of flags is
         passed as extra arguments following the inputs.
+
+        When the ``force`` attribute is present, the file is generated every
+        build, regardless of whether it is stale.  This is special to the
+        RecursiveMake backend and intended for special situations only (e.g.,
+        localization).  Please consult a build peer before using ``force``.
         """),
 
     'DEFINES': (InitializedDefines, dict,

python/mozbuild/mozbuild/frontend/data.py

@@ -1122,9 +1122,11 @@ class GeneratedFile(ContextDerived):
         'flags',
         'required_for_compile',
         'localized',
+        'force',
     )
 
-    def __init__(self, context, script, method, outputs, inputs, flags=(), localized=False):
+    def __init__(self, context, script, method, outputs, inputs,
+                 flags=(), localized=False, force=False):
         ContextDerived.__init__(self, context)
         self.script = script
         self.method = method
@@ -1132,6 +1134,7 @@ class GeneratedFile(ContextDerived):
         self.inputs = inputs
         self.flags = flags
         self.localized = localized
+        self.force = force
 
         suffixes = (
             '.c',

python/mozbuild/mozbuild/frontend/emitter.py

@@ -1391,7 +1391,7 @@ class TreeMetadataEmitter(LoggingMixin):
                     inputs.append(p)
 
                 yield GeneratedFile(context, script, method, outputs, inputs,
-                                    flags.flags, localized=localized)
+                                    flags.flags, localized=localized, force=flags.force)
 
     def _process_test_manifests(self, context):
         for prefix, info in TEST_MANIFESTS.items():

python/mozbuild/mozbuild/test/backend/data/generated-files-force/moz.build

@@ -0,0 +1,14 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ]
+
+bar = GENERATED_FILES['bar.c']
+bar.script = 'generate-bar.py:baz'
+bar.force = True
+
+foo = GENERATED_FILES['foo.c']
+foo.script = 'generate-foo.py'
+foo.inputs = ['foo-data']
+foo.force = False

python/mozbuild/mozbuild/test/backend/data/localized-generated-files-force/moz.build

@@ -0,0 +1,22 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+LOCALIZED_GENERATED_FILES += [ 'foo.xyz' ]
+
+foo = LOCALIZED_GENERATED_FILES['foo.xyz']
+foo.script = 'generate-foo.py'
+foo.inputs = [
+    'en-US/localized-input',
+    'non-localized-input',
+]
+
+LOCALIZED_GENERATED_FILES += [ 'abc.xyz' ]
+
+abc = LOCALIZED_GENERATED_FILES['abc.xyz']
+abc.script = 'generate-foo.py'
+abc.inputs = [
+    'en-US/localized-input',
+    'non-localized-input',
+]
+abc.force = True

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -418,6 +418,36 @@ class TestRecursiveMakeBackend(BackendTester):
         self.maxDiff = None
         self.assertEqual(lines, expected)
 
+    def test_generated_files_force(self):
+        """Ensure GENERATED_FILES with .force is handled properly."""
+        env = self._consume('generated-files-force', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        expected = [
+            'export:: bar.c',
+            'GARBAGE += bar.c',
+            'EXTRA_MDDEPEND_FILES += bar.c.pp',
+            'bar.c: %s/generate-bar.py FORCE' % env.topsrcdir,
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp)' % env.topsrcdir,
+            '',
+            'export:: foo.c',
+            'GARBAGE += foo.c',
+            'EXTRA_MDDEPEND_FILES += foo.c.pp',
+            'foo.c: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir),
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(srcdir)/foo-data)' % (env.topsrcdir),
+            '',
+            'export:: quux.c',
+            'GARBAGE += quux.c',
+            'EXTRA_MDDEPEND_FILES += quux.c.pp',
+        ]
+
+        self.maxDiff = None
+        self.assertEqual(lines, expected)
+
     def test_localized_generated_files(self):
         """Ensure LOCALIZED_GENERATED_FILES is handled properly."""
         env = self._consume('localized-generated-files', RecursiveMakeBackend)
@@ -442,6 +472,33 @@ class TestRecursiveMakeBackend(BackendTester):
         self.maxDiff = None
         self.assertEqual(lines, expected)
 
+    def test_localized_generated_files_force(self):
+        """Ensure LOCALIZED_GENERATED_FILES with .force is handled properly."""
+        env = self._consume('localized-generated-files-force', RecursiveMakeBackend)
+
+        backend_path = mozpath.join(env.topobjdir, 'backend.mk')
+        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
+
+        expected = [
+            'libs:: foo.xyz',
+            'GARBAGE += foo.xyz',
+            'EXTRA_MDDEPEND_FILES += foo.xyz.pp',
+            'foo.xyz: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,
+            '',
+            'libs:: abc.xyz',
+            'GARBAGE += abc.xyz',
+            'EXTRA_MDDEPEND_FILES += abc.xyz.pp',
+            'abc.xyz: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir,
+            '$(REPORT_BUILD)',
+            '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,
+            '',
+        ]
+
+        self.maxDiff = None
+        self.assertEqual(lines, expected)
+
     def test_localized_generated_files_AB_CD(self):
         """Ensure LOCALIZED_GENERATED_FILES is handled properly
         when {AB_CD} and {AB_rCD} are used."""

python/mozbuild/mozbuild/test/frontend/data/generated-files-force/moz.build

@@ -0,0 +1,7 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+GENERATED_FILES += [ 'bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ]
+GENERATED_FILES['bar.c'].force = True
+GENERATED_FILES['foo.c'].force = False

python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-force/moz.build

@@ -0,0 +1,6 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+LOCALIZED_GENERATED_FILES += [ 'abc.ini', ('bar', 'baz') ]
+LOCALIZED_GENERATED_FILES['abc.ini'].force = True

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -452,7 +452,6 @@ class TestEmitterBasic(unittest.TestCase):
                           'AS_DASH_C_FLAG': ''})
         self.maxDiff = maxDiff
 
-
     def test_generated_files(self):
         reader = self.reader('generated-files')
         objs = self.read_topsrcdir(reader)
@@ -461,6 +460,7 @@ class TestEmitterBasic(unittest.TestCase):
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)
             self.assertFalse(o.localized)
+            self.assertFalse(o.force)
 
         expected = ['bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ]
         for o, f in zip(objs, expected):
@@ -470,6 +470,15 @@ class TestEmitterBasic(unittest.TestCase):
             self.assertEqual(o.method, None)
             self.assertEqual(o.inputs, [])
 
+    def test_generated_files_force(self):
+        reader = self.reader('generated-files-force')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 3)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+            self.assertEqual(o.force, 'bar.c' in o.outputs)
+
     def test_localized_generated_files(self):
         reader = self.reader('localized-generated-files')
         objs = self.read_topsrcdir(reader)
@@ -487,6 +496,16 @@ class TestEmitterBasic(unittest.TestCase):
             self.assertEqual(o.method, None)
             self.assertEqual(o.inputs, [])
 
+    def test_localized_generated_files_force(self):
+        reader = self.reader('localized-generated-files-force')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 2)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+            self.assertTrue(o.localized)
+            self.assertEqual(o.force, 'abc.ini' in o.outputs)
+
     def test_localized_files_from_generated(self):
         """Test that using LOCALIZED_GENERATED_FILES and then putting the output in
         LOCALIZED_FILES as an objdir path works.