From 5555792721459389c7b7df53e1de9b7628a0b6c1 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 1 May 2013 15:36:05 -0700 Subject: [PATCH] Bug 677452 - Add smartmake-like functionality to |mach build DIR|. r=gps --- build/dumbmake-dependencies | 52 +++++++++ python/Makefile.in | 1 + python/mozbuild/dumbmake/README.rst | 38 ++++++ python/mozbuild/dumbmake/__init__.py | 0 python/mozbuild/dumbmake/dumbmake.py | 93 +++++++++++++++ python/mozbuild/dumbmake/test/__init__.py | 0 .../mozbuild/dumbmake/test/test_dumbmake.py | 108 ++++++++++++++++++ python/mozbuild/mozbuild/mach_commands.py | 33 +++++- 8 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 build/dumbmake-dependencies create mode 100644 python/mozbuild/dumbmake/README.rst create mode 100644 python/mozbuild/dumbmake/__init__.py create mode 100644 python/mozbuild/dumbmake/dumbmake.py create mode 100644 python/mozbuild/dumbmake/test/__init__.py create mode 100644 python/mozbuild/dumbmake/test/test_dumbmake.py diff --git a/build/dumbmake-dependencies b/build/dumbmake-dependencies new file mode 100644 index 000000000000..5d7926dd3b23 --- /dev/null +++ b/build/dumbmake-dependencies @@ -0,0 +1,52 @@ +toolkit/library + ipc + netwerk/build + netwerk + storage/build + storage + xpcom + chrome + extensions + docshell/build + docshell + uriloader + modules + widget + gfx + toolkit/components/build + toolkit/components + security/build + security/manager + security/dbm + security/nss + accessible/build + accessible + dom + content + layout + editor + parser + js/src + js/xpconnect + js/xpconnect/loader + mfbt + view + caps + xpfe/appshell + xpfe/components + js + toolkit + rdf/build + embedding + hal + image/build + image + intl/build + intl + media + profile + services + startupcache +browser + toolkit/mozapps/extensions + toolkit/content diff --git a/python/Makefile.in b/python/Makefile.in index 29bcb7ad4e17..bce07dc5bed2 100644 --- a/python/Makefile.in +++ b/python/Makefile.in @@ -16,6 +16,7 @@ test_dirs := \ mozbuild/mozbuild/test/compilation \ mozbuild/mozbuild/test/frontend \ mozbuild/mozpack/test \ + mozbuild/dumbmake/test \ $(NULL) PYTHON_UNIT_TESTS := $(foreach dir,$(test_dirs),$(wildcard $(srcdir)/$(dir)/*.py)) diff --git a/python/mozbuild/dumbmake/README.rst b/python/mozbuild/dumbmake/README.rst new file mode 100644 index 000000000000..7840695adddd --- /dev/null +++ b/python/mozbuild/dumbmake/README.rst @@ -0,0 +1,38 @@ +dumbmake +======== + +*dumbmake* is a simple dependency tracker for make. + +It turns lists of make targets into longer lists of make targets that +include dependencies. For example: + + netwerk, package + +might be turned into + + netwerk, netwerk/build, toolkit/library, package + +The dependency list is read from the plain text file +`topsrcdir/build/dumbmake-dependencies`. The format best described by +example: + + build_this + when_this_changes + +Interpret this to mean that `build_this` is a dependency of +`when_this_changes`. More formally, a line (CHILD) indented more than +the preceding line (PARENT) means that CHILD should trigger building +PARENT. That is, building CHILD will trigger building first CHILD and +then PARENT. + +This structure is recursive: + + build_this_when_either_change + build_this_only_when + this_changes + +This means that `build_this_when_either_change` is a dependency of +`build_this_only_when` and `this_changes`, and `build_this_only_when` +is a dependency of `this_changes`. Building `this_changes` will build +first `this_changes`, then `build_this_only_when`, and finally +`build_this_when_either_change`. diff --git a/python/mozbuild/dumbmake/__init__.py b/python/mozbuild/dumbmake/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/dumbmake/dumbmake.py b/python/mozbuild/dumbmake/dumbmake.py new file mode 100644 index 000000000000..f4b2c2e19411 --- /dev/null +++ b/python/mozbuild/dumbmake/dumbmake.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from collections import OrderedDict +from itertools import groupby +from operator import itemgetter + +WHITESPACE_CHARACTERS = ' \t' + +def indentation(line): + """Number of whitespace (tab and space) characters at start of |line|.""" + i = 0 + while i < len(line): + if line[i] not in WHITESPACE_CHARACTERS: + break + i += 1 + return i + +def dependency_map(lines): + """Return a dictionary with keys that are targets and values that + are ordered lists of targets that should also be built. + + This implementation is O(n^2), but lovely and simple! We walk the + targets in the list, and for each target we walk backwards + collecting its dependencies. To make the walking easier, we + reverse the list so that we are always walking forwards. + + """ + pairs = [(indentation(line), line.strip()) for line in lines] + pairs.reverse() + + deps = {} + + for i, (indent, target) in enumerate(pairs): + if not deps.has_key(target): + deps[target] = [] + + for j in range(i+1, len(pairs)): + ind, tar = pairs[j] + if ind < indent: + indent = ind + if tar not in deps[target]: + deps[target].append(tar) + + return deps + +def all_dependencies(*targets, **kwargs): + """Return a list containing |targets| and all the dependencies of + those targets. + + The relative order of targets is maintained if possible. + + """ + dm = kwargs.pop('dependency_map', None) + if dm is None: + dm = dependency_map(targets) + + all_targets = OrderedDict() # Used as an ordered set. + + for target in targets: + all_targets[target] = True + if target in dm: + for dependency in dm[target]: + # Move element back in the ordered set. + if dependency in all_targets: + del all_targets[dependency] + all_targets[dependency] = True + + return all_targets.keys() + +def add_extra_dependencies(target_pairs, dependency_map): + """Take a list [(make_dir, make_target)] and expand (make_dir, None) + entries with extra make dependencies from |dependency_map|. + + Returns an iterator of pairs (make_dir, make_target). + + """ + for make_target, group in groupby(target_pairs, itemgetter(1)): + # Return non-simple directory targets untouched. + if make_target is not None: + for pair in group: + yield pair + continue + + # Add extra dumbmake dependencies to simple directory targets. + make_dirs = [make_dir for make_dir, _ in group] + new_make_dirs = all_dependencies(*make_dirs, dependency_map=dependency_map) + + for make_dir in new_make_dirs: + yield make_dir, None diff --git a/python/mozbuild/dumbmake/test/__init__.py b/python/mozbuild/dumbmake/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozbuild/dumbmake/test/test_dumbmake.py b/python/mozbuild/dumbmake/test/test_dumbmake.py new file mode 100644 index 000000000000..85d4790abe26 --- /dev/null +++ b/python/mozbuild/dumbmake/test/test_dumbmake.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +import unittest + +from mozunit import ( + main, +) + +from dumbmake.dumbmake import ( + add_extra_dependencies, + all_dependencies, + dependency_map, + indentation, +) + +class TestDumbmake(unittest.TestCase): + def test_indentation(self): + self.assertEqual(indentation(""), 0) + self.assertEqual(indentation("x"), 0) + self.assertEqual(indentation(" x"), 1) + self.assertEqual(indentation("\tx"), 1) + self.assertEqual(indentation(" \tx"), 2) + self.assertEqual(indentation("\t x"), 2) + self.assertEqual(indentation(" x "), 1) + self.assertEqual(indentation("\tx\t"), 1) + self.assertEqual(indentation(" x"), 2) + self.assertEqual(indentation(" x"), 4) + + def test_dependency_map(self): + self.assertEqual(dependency_map([]), {}) + self.assertEqual(dependency_map(["a"]), {"a": []}) + self.assertEqual(dependency_map(["a", "b"]), {"a": [], "b": []}) + self.assertEqual(dependency_map(["a", "b", "c"]), {"a": [], "b": [], "c": []}) + # indentation + self.assertEqual(dependency_map(["a", "\tb", "a", "\tc"]), {"a": [], "b": ["a"], "c": ["a"]}) + self.assertEqual(dependency_map(["a", "\tb", "\t\tc"]), {"a": [], "b": ["a"], "c": ["b", "a"]}) + self.assertEqual(dependency_map(["a", "\tb", "\t\tc", "\td", "\te", "f"]), {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []}) + # irregular indentation + self.assertEqual(dependency_map(["\ta", "b"]), {"a": [], "b": []}) + self.assertEqual(dependency_map(["a", "\t\t\tb", "\t\tc"]), {"a": [], "b": ["a"], "c": ["a"]}) + self.assertEqual(dependency_map(["a", "\t\tb", "\t\t\tc", "\t\td", "\te", "f"]), {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []}) + # repetitions + self.assertEqual(dependency_map(["a", "\tb", "a", "\tb"]), {"a": [], "b": ["a"]}) + self.assertEqual(dependency_map(["a", "\tb", "\t\tc", "b", "\td", "\t\te"]), {"a": [], "b": ["a"], "d": ["b"], "e": ["d", "b"], "c": ["b", "a"]}) + # cycles are okay + self.assertEqual(dependency_map(["a", "\tb", "\t\ta"]), {"a": ["b", "a"], "b": ["a"]}) + + def test_all_dependencies(self): + dm = {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []} + self.assertEqual(all_dependencies("a", dependency_map=dm), ["a"]) + self.assertEqual(all_dependencies("b", dependency_map=dm), ["b", "a"]) + self.assertEqual(all_dependencies("c", "a", "b", dependency_map=dm), ["c", "b", "a"]) + self.assertEqual(all_dependencies("d", dependency_map=dm), ["d", "a"]) + self.assertEqual(all_dependencies("d", "f", "c", dependency_map=dm), ["d", "f", "c", "b", "a"]) + self.assertEqual(all_dependencies("a", "b", dependency_map=dm), ["b", "a"]) + self.assertEqual(all_dependencies("b", "b", dependency_map=dm), ["b", "a"]) + + def test_missing_entry(self): + # a depends on b, which is missing + dm = {"a": ["b"]} + self.assertEqual(all_dependencies("a", dependency_map=dm), ["a", "b"]) + self.assertEqual(all_dependencies("a", "b", dependency_map=dm), ["a", "b"]) + self.assertEqual(all_dependencies("b", dependency_map=dm), ["b"]) + + def test_two_dependencies(self): + dm = {"a": ["c"], "b": ["c"], "c": []} + # suppose a and b both depend on c. Then we want to build a and b before c... + self.assertEqual(all_dependencies("a", "b", dependency_map=dm), ["a", "b", "c"]) + # ... but relative order is preserved. + self.assertEqual(all_dependencies("b", "a", dependency_map=dm), ["b", "a", "c"]) + + def test_nested_dependencies(self): + # a depends on b depends on c depends on d + dm = {"a": ["b", "c", "d"], "b": ["c", "d"], "c": ["d"]} + self.assertEqual(all_dependencies("b", "a", dependency_map=dm), ["a", "b", "c", "d"]) + self.assertEqual(all_dependencies("c", "a", dependency_map=dm), ["a", "b", "c", "d"]) + + def test_add_extra_dependencies(self): + # a depends on b depends on c depends on d + dm = {"a": ["b", "c", "d"], "b": ["c", "d"], "c": ["d"]} + # Edge cases. + self.assertEqual(list(add_extra_dependencies([], dependency_map=dm)), + []) + self.assertEqual(list(add_extra_dependencies([(None, "package")], dependency_map=dm)), + [(None, "package")]) + # Easy expansion. + self.assertEqual(list(add_extra_dependencies([("b", None)], dependency_map=dm)), + [("b", None), ("c", None), ("d", None)]) + # Expansion with two groups -- each group is handled independently. + self.assertEqual(list(add_extra_dependencies([("b", None), + (None, "package"), + ("c", None)], dependency_map=dm)), + [("b", None), ("c", None), ("d", None), + (None, "package"), + ("c", None), ("d", None)]) + # Two groups, no duplicate dependencies in each group. + self.assertEqual(list(add_extra_dependencies([("a", None), ("b", None), + (None, "package"), (None, "install"), + ("c", None), ("d", None)], dependency_map=dm)), + [("a", None), ("b", None), ("c", None), ("d", None), + (None, "package"), (None, "install"), + ("c", None), ("d", None)]) + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 21b459f46ca0..a76c94f8cb07 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -21,8 +21,10 @@ from mozbuild.base import MachCommandBase BUILD_WHAT_HELP = ''' What to build. Can be a top-level make target or a relative directory. If -multiple options are provided, they will be built serially. BUILDING ONLY PARTS -OF THE TREE CAN RESULT IN BAD TREE STATE. USE AT YOUR OWN RISK. +multiple options are provided, they will be built serially. Takes dependency +information from `topsrcdir/build/dumbmake-dependencies` to build additional +targets as needed. BUILDING ONLY PARTS OF THE TREE CAN RESULT IN BAD TREE +STATE. USE AT YOUR OWN RISK. '''.strip() FINDER_SLOW_MESSAGE = ''' @@ -46,7 +48,10 @@ class Build(MachCommandBase): @Command('build', help='Build the tree.') @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP) - def build(self, what=None): + @CommandArgument('-X', '--disable-extra-make-dependencies', + default=False, action='store_true', + help='Do not add extra make dependencies.') + def build(self, what=None, disable_extra_make_dependencies=None): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector @@ -87,6 +92,8 @@ class Build(MachCommandBase): '|mach build| with no arguments.') return 1 + # Collect target pairs. + target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) @@ -96,6 +103,26 @@ class Build(MachCommandBase): if make_dir is None and make_target is None: return 1 + target_pairs.append((make_dir, make_target)) + + # Possibly add extra make depencies using dumbmake. + if not disable_extra_make_dependencies: + from dumbmake.dumbmake import (dependency_map, + add_extra_dependencies) + depfile = os.path.join(self.topsrcdir, 'build', + 'dumbmake-dependencies') + with open(depfile) as f: + dm = dependency_map(f.readlines()) + new_pairs = list(add_extra_dependencies(target_pairs, dm)) + self.log(logging.DEBUG, 'dumbmake', + {'target_pairs': target_pairs, + 'new_pairs': new_pairs}, + 'Added extra dependencies: will build {new_pairs} ' + + 'instead of {target_pairs}.') + target_pairs = new_pairs + + # Build target pairs. + for make_dir, make_target in target_pairs: status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False)