diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index e97ee820bc79..b830766fbcb3 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -24,6 +24,7 @@ from collections import ( ) from mozbuild.util import ( HierarchicalStringList, + ImmutableStrictOrderingOnAppendList, KeyedDefaultDict, List, ListWithAction, @@ -37,6 +38,8 @@ from mozbuild.util import ( TypedNamedTuple, ) +from .. import schedules + from ..testing import ( all_test_flavors, read_manifestparser_manifest, @@ -559,6 +562,54 @@ def ContextDerivedTypedRecord(*fields): return _TypedRecord +class Schedules(object): + """Similar to a ContextDerivedTypedRecord, but with different behavior + for the properties: + + * VAR.inclusive can only be appended to (+=), and can only contain values + from mozbuild.schedules.INCLUSIVE_COMPONENTS + + * VAR.exclusive can only be assigned to (no +=), and can only contain + values from mozbuild.schedules.ALL_COMPONENTS + """ + __slots__ = ('_exclusive', '_inclusive') + + def __init__(self): + self._inclusive = TypedList(Enum(*schedules.INCLUSIVE_COMPONENTS))() + self._exclusive = ImmutableStrictOrderingOnAppendList(schedules.EXCLUSIVE_COMPONENTS) + + # inclusive is mutable cannot be assigned to (+= only) + @property + def inclusive(self): + return self._inclusive + + @inclusive.setter + def inclusive(self, value): + if value is not self._inclusive: + raise AttributeError("Cannot assign to this value - use += instead") + unexpected = [v for v in value if v not in schedules.INCLUSIVE_COMPONENTS] + if unexpected: + raise Exception("unexpected exclusive component(s) " + ', '.join(unexpected)) + + # exclusive is immuntable but can be set (= only) + @property + def exclusive(self): + return self._exclusive + + @exclusive.setter + def exclusive(self, value): + if not isinstance(value, (tuple, list)): + raise Exception("expected a tuple or list") + unexpected = [v for v in value if v not in schedules.ALL_COMPONENTS] + if unexpected: + raise Exception("unexpected exclusive component(s) " + ', '.join(unexpected)) + self._exclusive = ImmutableStrictOrderingOnAppendList(sorted(value)) + + # components provides a synthetic summary of all components + @property + def components(self): + return list(sorted(set(self._inclusive) | set(self._exclusive))) + @memoize def ContextDerivedTypedHierarchicalStringList(type): """Specialized HierarchicalStringList for use with ContextDerivedValue @@ -629,6 +680,9 @@ DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList), ('flavors', OrderedTestFlavorList)) BugzillaComponent = TypedNamedTuple('BugzillaComponent', [('product', unicode), ('component', unicode)]) +SchedulingComponents = ContextDerivedTypedRecord( + ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)), + ('exclusive', TypedList(unicode, StrictOrderingOnAppendList))) class Files(SubContext): @@ -747,6 +801,35 @@ class Files(SubContext): Would suggest that nsGlobalWindow.cpp is potentially relevant to any plain mochitest. """), + 'SCHEDULES': (Schedules, list, + """Maps source files to the CI tasks that should be scheduled when + they change. The tasks are grouped by named components, and those + names appear again in the taskgraph configuration + `($topsrcdir/taskgraph/). + + Some components are "inclusive", meaning that changes to most files + do not schedule them, aside from those described in a Files + subcontext. For example, py-lint tasks need not be scheduled for + most changes, but should be scheduled when any Python file changes. + Such components are named by appending to `SCHEDULES.inclusive`: + + with Files('**.py'): + SCHEDULES.inclusive += ['py-lint'] + + Other components are 'exclusive', meaning that changes to most + files schedule them, but some files affect only one or two + components. For example, most files schedule builds and tests of + Firefox for Android, OS X, Windows, and Linux, but files under + `mobile/android/` affect Android builds and tests exclusively, so + builds for other operating systems are not needed. Test suites + provide another example: most files schedule reftests, but changes + to reftest scripts need only schedule reftests and no other suites. + + Exclusive components are named by setting `SCHEDULES.exclusive`: + + with Files('mobile/android/**'): + SCHEDULES.exclusive = ['android'] + """), } def __init__(self, parent, pattern=None): diff --git a/python/mozbuild/mozbuild/frontend/mach_commands.py b/python/mozbuild/mozbuild/frontend/mach_commands.py index 4936ae322b98..9b3fb3b4342b 100644 --- a/python/mozbuild/mozbuild/frontend/mach_commands.py +++ b/python/mozbuild/mozbuild/frontend/mach_commands.py @@ -17,6 +17,7 @@ from mach.decorators import ( from mozbuild.base import MachCommandBase import mozpack.path as mozpath +TOPSRCDIR = os.path.abspath(os.path.join(__file__, '../../../../../')) class InvalidPathException(Exception): """Represents an error due to an invalid path.""" @@ -221,3 +222,23 @@ class MozbuildFileCommands(MachCommandBase): reader = self._get_reader(finder=reader_finder) return reader.files_info(allpaths) + + + @SubCommand('file-info', 'schedules', + 'Show the combined SCHEDULES for the files listed.') + @CommandArgument('paths', nargs='+', + help='Paths whose data to query') + def file_info_schedules(self, paths): + """Show what is scheduled by the given files. + + Given a requested set of files (which can be specified using + wildcards), print the total set of scheduled components. + """ + from mozbuild.frontend.reader import EmptyConfig, BuildReader + config = EmptyConfig(TOPSRCDIR) + reader = BuildReader(config) + schedules = set() + for p, m in reader.files_info(paths).items(): + schedules |= set(m['SCHEDULES'].components) + + print(", ".join(schedules)) diff --git a/python/mozbuild/mozbuild/schedules.py b/python/mozbuild/mozbuild/schedules.py new file mode 100644 index 000000000000..ca388e1d731a --- /dev/null +++ b/python/mozbuild/mozbuild/schedules.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Constants for SCHEDULES configuration in moz.build files and for +skip-unless-schedules optimizations in task-graph generation. +""" + +from __future__ import absolute_import, unicode_literals, print_function + +# TODO: ideally these lists could be specified in moz.build itself + +INCLUSIVE_COMPONENTS = [ + 'py-lint', + 'js-lint', + 'yaml-lint', +] +EXCLUSIVE_COMPONENTS = [ + # os families + 'android', + 'linux', + 'macosx', + 'windows', +] +ALL_COMPONENTS = INCLUSIVE_COMPONENTS + EXCLUSIVE_COMPONENTS diff --git a/python/mozbuild/mozbuild/test/frontend/data/schedules/moz.build b/python/mozbuild/mozbuild/test/frontend/data/schedules/moz.build new file mode 100644 index 000000000000..d104c5290711 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/schedules/moz.build @@ -0,0 +1,11 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +with Files('*.win'): + SCHEDULES.exclusive = ['windows'] + +with Files('*.osx'): + SCHEDULES.exclusive = ['macosx'] + +with Files('subd/**.py'): + SCHEDULES.inclusive += ['py-lint'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/schedules/subd/moz.build b/python/mozbuild/mozbuild/test/frontend/data/schedules/subd/moz.build new file mode 100644 index 000000000000..d078a8e69db8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/schedules/subd/moz.build @@ -0,0 +1,2 @@ +with Files('yaml.py'): + SCHEDULES.inclusive += ['yaml-lint'] diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py index 7c2aed9dfc17..67cefd7710e0 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_reader.py +++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -480,6 +480,27 @@ class TestBuildReader(unittest.TestCase): with self.assertRaises(BuildReaderError): reader.files_info(['foo.js']) + def test_schedules(self): + reader = self.reader('schedules') + info = reader.files_info(['somefile', 'foo.win', 'foo.osx', 'subd/aa.py', 'subd/yaml.py']) + # default: all exclusive, no inclusive + self.assertEqual(info['somefile']['SCHEDULES'].inclusive, []) + self.assertEqual(info['somefile']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows']) + # windows-only + self.assertEqual(info['foo.win']['SCHEDULES'].inclusive, []) + self.assertEqual(info['foo.win']['SCHEDULES'].exclusive, ['windows']) + # osx-only + self.assertEqual(info['foo.osx']['SCHEDULES'].inclusive, []) + self.assertEqual(info['foo.osx']['SCHEDULES'].exclusive, ['macosx']) + # top-level moz.build specifies subd/**.py with an inclusive option + self.assertEqual(info['subd/aa.py']['SCHEDULES'].inclusive, ['py-lint']) + self.assertEqual(info['subd/aa.py']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows']) + # Files('yaml.py') in subd/moz.build *overrides* Files('subdir/**.py') + self.assertEqual(info['subd/yaml.py']['SCHEDULES'].inclusive, ['yaml-lint']) + self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, ['android', 'linux', 'macosx', 'windows']) + + self.assertEqual(info['subd/yaml.py']['SCHEDULES'].components, + ['android', 'linux', 'macosx', 'windows', 'yaml-lint']) if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index 3f5274d7d2f7..4ea227dc0632 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -485,6 +485,25 @@ class StrictOrderingOnAppendList(ListMixin, StrictOrderingOnAppendListMixin, elements be ordered. This enforces cleaner style in moz.build files. """ +class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList): + """Like StrictOrderingOnAppendList, but not allowing mutations of the value. + """ + def append(self, elt): + raise Exception("cannot use append on this type") + + def extend(self, iterable): + raise Exception("cannot use extend on this type") + + def __setslice__(self, i, j, iterable): + raise Exception("cannot assign to slices on this type") + + def __setitem__(self, i, elt): + raise Exception("cannot assign to indexes on this type") + + def __iadd__(self, other): + raise Exception("cannot use += on this type") + + class ListWithActionMixin(object): """Mixin to create lists with pre-processing. See ListWithAction.""" def __init__(self, iterable=None, action=None):