Bug 1207882 - Add an initial partial implementation of a new, faster, build backend. r=gps

This commit is contained in:
Mike Hommey 2015-09-24 10:40:54 +09:00
parent 75bdaecdfd
commit 0e156c41d5
5 changed files with 476 additions and 2 deletions

209
config/faster/rules.mk Normal file
View File

@ -0,0 +1,209 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# /!\ Please make sure to update the following comment when you touch this
# file. Thank you /!\
# The traditional Mozilla build system relied on going through the entire
# build tree a number of times with different targets, and many of the
# things happening at each step required other things happening in previous
# steps without any documentation of those dependencies.
#
# This new build system tries to start afresh by establishing what files or
# operations are needed for the build, and applying the necessary rules to
# have those in place, relying on make dependencies to get them going.
#
# As of writing, only building non-compiled parts of Firefox is supported
# here (a few other things are also left out). This is a starting point, with
# the intent to grow this build system to make it more complete.
#
# This file contains rules and dependencies to get things working. The intent
# is for a Makefile to define some dependencies and variables, and include
# this file. What needs to be defined there, and ends up being generated by
# python/mozbuild/mozbuild/backend/fastermake.py is the following:
# - TOPSRCDIR/TOPOBJDIR, respectively the top source directory and the top
# object directory
# - PYTHON, the path to the python executable
# - ACDEFINES, which contains a set of -Dvar=name to be used during
# preprocessing
# - MOZ_CHROME_FILE_FORMAT, which defines whether to use file copies or
# symbolic links
# - JAR_MN_TARGETS, which defines the targets to use for jar manifest
# processing, see further below
# - PP_TARGETS, which defines the file paths of preprocessed files, see
# further below
# - INSTALL_MANIFESTS, which defines the list of base directories handled
# by install manifests, see further below
# - MANIFEST_TARGETS, which defines the file paths of chrome manifests, see
# further below
#
# A convention used between this file and the Makefile including it is that
# global Make variables names are uppercase, while "local" Make variables
# applied to specific targets are lowercase.
# Targets to be triggered for a default build
default: $(addprefix install-,$(INSTALL_MANIFESTS))
default: $(addprefix jar-,$(JAR_MN_TARGETS))
# Explicit files to be built for a default build
default: $(addprefix $(TOPOBJDIR)/,$(PP_TARGETS))
default: $(addprefix $(TOPOBJDIR)/,$(MANIFEST_TARGETS))
default: $(TOPOBJDIR)/dist/bin/greprefs.js
default: $(TOPOBJDIR)/dist/bin/platform.ini
default: $(TOPOBJDIR)/dist/bin/webapprt/webapprt.ini
.PHONY: FORCE
# Extra define to trigger some workarounds. We should strive to limit the
# use of those. As of writing the only one is in
# toolkit/content/buildconfig.html.
ACDEFINES += -DBUILD_FASTER
# Generic rule to fall back to the recursive make backend
$(TOPOBJDIR)/%: FORCE
$(MAKE) -C $(dir $@) $(notdir $@)
# Files under the faster/ sub-directory, however, are not meant to use the
# fallback
$(TOPOBJDIR)/faster/%: ;
# And files under dist/ are meant to be copied from their first dependency
# if there is no other rule.
$(TOPOBJDIR)/dist/%:
rm -f $@
cp $< $@
# Install files using install manifests
#
# The list of base directories is given in INSTALL_MANIFESTS. The
# corresponding install manifests are named correspondingly, with forward
# slashes replaced with underscores, and prefixed with `install_`. That is,
# the install manifest for `dist/bin` would be `install_dist_bin`.
$(addprefix install-,$(INSTALL_MANIFESTS)): install-%:
$(PYTHON) -m mozbuild.action.process_install_manifest \
--no-remove \
--no-remove-empty-directories \
$(TOPOBJDIR)/$* \
install_$(subst /,_,$*)
# Preprocessed files. Ideally they would be using install manifests but
# right now, it's not possible because of things like APP_BUILDID or
# nsURLFormatter.js.
# Things missing:
# - XULPPFLAGS
#
# The list of preprocessed files is defined in PP_TARGETS. The list is
# relative to TOPOBJDIR.
# The source file for each of those preprocessed files is defined as a Make
# dependency for the $(TOPOBJDIR)/path target. For example:
# PP_TARGETS = foo/bar
# $(TOPOBJDIR)/foo/bar: /path/to/source/for/foo/bar.in
# The file name for the source doesn't need to be different.
# Additionally, extra defines can be specified for a given preprocessing
# by setting the `defines` variable specifically for the given target.
# For example:
# $(TOPOBJDIR)/foo/bar: defines = -Dqux=foobar
$(addprefix $(TOPOBJDIR)/,$(PP_TARGETS)): Makefile
$(addprefix $(TOPOBJDIR)/,$(PP_TARGETS)): $(TOPOBJDIR)/%:
$(PYTHON) -m mozbuild.action.preprocessor \
--depend $(TOPOBJDIR)/faster/.deps/$(subst /,_,$*) \
-DAB_CD=en-US \
$(defines) \
$(ACDEFINES) \
$< \
-o $@
# Include the dependency files from the above preprocessed files rule.
$(foreach pp_target,$(PP_TARGETS), \
$(eval -include $(TOPOBJDIR)/faster/.deps/$(subst /,_,$(pp_target))))
# Install files from jar manifests. Ideally, they would be using install
# manifests, but the code to read jar manifests and emit appropriate
# install manifests is not there yet.
# Things missing:
# - XULPPFLAGS
# - DEFINES from config/config.mk
# - L10N
# - -e when USE_EXTENSION_MANIFEST is set in moz.build
#
# The list given in JAR_MN_TARGETS corresponds to the list of `jar-%` targets
# to be processed, with the `jar-` prefix stripped.
# The Makefile is expected to specify the source jar manifest as a dependency
# to each target. There is no expectation that the `jar-%` target name matches
# the source file name in any way. For example:
# JAR_MN_TARGETS = foo
# jar-foo: /path/to/some/jar.mn
# Additionally, extra defines can be specified for the processing of the jar
# manifest by settig the `defines` variable specifically for the given target.
# For example:
# jar-foo: defines = -Dqux=foo
# The default base path where files are going to be installed is `dist/bin`.
# It is possible to use a different path by setting the `install_target`
# variable. For example:
# jar-foo: install_target = dist/bin/foo
# When processing jar manifests, relative paths given inside a jar manifest
# can be resolved from an object directory. The default path for that object
# directory is the translation of the jar manifest directory path from the
# source directory to the object directory. That is, for
# $(TOPSRCDIR)/path/to/jar.mn, the default would be $(TOPOBJDIR)/path/to.
# In case a different path must be used for the object directory, the `objdir`
# variable can be set. For example:
# jar-foo: objdir=/some/other/path
jar-%: objdir ?= $(dir $(patsubst $(TOPSRCDIR)%,$(TOPOBJDIR)%,$<))
jar-%: install_target ?= dist/bin
jar-%:
cd $(objdir) && \
$(PYTHON) -m mozbuild.action.jar_maker \
-j $(TOPOBJDIR)/$(install_target)/chrome \
-t $(TOPSRCDIR) \
-f $(MOZ_CHROME_FILE_FORMAT) \
-c $(dir $<)/en-US \
-DAB_CD=en-US \
$(defines) \
$(ACDEFINES) \
$<
# Create some chrome manifests
# This rule is forced to run every time because it may be updating files that
# already exit.
#
# The list of chrome manifests is given in MANIFEST_TARGETS, relative to the
# top object directory. The content for those manifests is given in the
# `content` variable associated with the target. For example:
# MANIFEST_TARGETS = foo
# $(TOPOBJDIR)/foo: content = "manifest foo.manifest" "manifest bar.manifest"
$(addprefix $(TOPOBJDIR)/,$(MANIFEST_TARGETS)): FORCE
$(PYTHON) -m mozbuild.action.buildlist \
$@ \
$(content)
# ============================================================================
# Below is a set of additional dependencies and variables used to build things
# that are not supported by data in moz.build.
# GENERATED_FILES are not supported yet, and even if they were, the
# dependencies are missing information.
$(foreach p,linux osx windows,jar-browser-themes-$(p)-jar.mn): \
jar-browser-themes-%-jar.mn: \
$(TOPOBJDIR)/browser/themes/%/tab-selected-end.svg \
$(TOPOBJDIR)/browser/themes/%/tab-selected-start.svg
# These files are manually generated from
# toolkit/components/urlformatter/Makefile.in and are force-included so that
# the corresponding defines don't end up in the command lines.
KEYS = mozilla_api_key google_api_key google_oauth_api_key bing_api_key
$(TOPOBJDIR)/dist/bin/components/nsURLFormatter.js: \
$(addprefix $(TOPOBJDIR)/toolkit/components/urlformatter/, $(KEYS))
$(TOPOBJDIR)/dist/bin/components/nsURLFormatter.js: defines += \
$(addprefix -I $(TOPOBJDIR)/toolkit/components/urlformatter/,$(KEYS))
# Extra dependencies and/or definitions for preprocessed files.
$(TOPOBJDIR)/dist/bin/application.ini: $(TOPOBJDIR)/config/buildid
$(TOPOBJDIR)/dist/bin/application.ini: defines += \
-DAPP_BUILDID=$(shell cat $(TOPOBJDIR)/config/buildid)
# Files to build with the recursive backend and simply copy
$(TOPOBJDIR)/dist/bin/greprefs.js: $(TOPOBJDIR)/modules/libpref/greprefs.js
$(TOPOBJDIR)/dist/bin/platform.ini: $(TOPOBJDIR)/toolkit/xre/platform.ini
$(TOPOBJDIR)/dist/bin/webapprt/webapprt.ini: $(TOPOBJDIR)/webapprt/webapprt.ini

View File

@ -0,0 +1,258 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
from mozbuild.backend.common import CommonBackend
from mozbuild.frontend.data import (
ContextDerived,
Defines,
DistFiles,
FinalTargetFiles,
JARManifest,
JavaScriptModules,
JsPreferenceFile,
Resources,
VariablePassthru,
)
from mozbuild.makeutil import Makefile
from mozbuild.util import OrderedDefaultDict
from mozpack.manifests import InstallManifest
import mozpack.path as mozpath
from collections import OrderedDict
from itertools import chain
class FasterMakeBackend(CommonBackend):
def _init(self):
super(FasterMakeBackend, self)._init()
self._seen_directories = set()
self._defines = dict()
self._jar_manifests = OrderedDict()
self._preprocess_files = OrderedDict()
self._manifest_entries = OrderedDefaultDict(list)
self._install_manifests = OrderedDefaultDict(InstallManifest)
def consume_object(self, obj):
# We currently ignore a lot of object types, so just acknowledge
# everything.
obj.ack()
if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
defines = self._defines.get(obj.objdir, [])
if defines:
defines = list(defines.get_defines())
if isinstance(obj, Defines):
self._defines[obj.objdir] = obj
# We're assuming below that Defines come first for a given objdir,
# which is kind of set in stone from the order things are treated
# in emitter.py.
assert obj.objdir not in self._seen_directories
elif isinstance(obj, JARManifest) and \
obj.install_target.startswith('dist/bin'):
self._jar_manifests[obj.path] = (obj.objdir,
obj.install_target,
defines)
elif isinstance(obj, VariablePassthru) and \
obj.install_target.startswith('dist/bin'):
for f in obj.variables.get('EXTRA_COMPONENTS', {}):
path = mozpath.join(obj.install_target, 'components',
mozpath.basename(f))
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join('components', mozpath.basename(f))
)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
for f in obj.variables.get('EXTRA_PP_COMPONENTS', {}):
path = mozpath.join(obj.install_target, 'components',
mozpath.basename(f))
self._preprocess_files[path] = (obj.srcdir, f, defines)
if f.endswith('.manifest'):
manifest = mozpath.join(obj.install_target,
'chrome.manifest')
self._manifest_entries[manifest].append(
'manifest components/%s' % mozpath.basename(f))
elif isinstance(obj, JavaScriptModules) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.modules.walk():
base = mozpath.join(obj.install_target, 'modules', path)
for f in strings:
if obj.flavor == 'extra':
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join('modules', path, mozpath.basename(f))
)
elif obj.flavor == 'extra_pp':
dest = mozpath.join(base, mozpath.basename(f))
self._preprocess_files[dest] = (obj.srcdir, f, defines)
elif isinstance(obj, JsPreferenceFile) and \
obj.install_target.startswith('dist/bin'):
# The condition for the directory value in config/rules.mk is:
# ifneq (,$(DIST_SUBDIR)$(XPI_NAME)$(LIBXUL_SDK))
# - LIBXUL_SDK is not supported (it likely doesn't work in the
# recursive backend anyways
# - when XPI_NAME is set, obj.install_target will start with
# dist/xpi-stage
# - when DIST_SUBDIR is set, obj.install_target will start with
# dist/bin/$(DIST_SUBDIR)
# So an equivalent condition that is not cumbersome for us and that
# is enough at least for now is checking if obj.install_target is
# different from dist/bin.
if obj.install_target == 'dist/bin':
pref_dir = 'defaults/pref'
else:
pref_dir = 'defaults/preferences'
dest = mozpath.join(obj.install_target, pref_dir,
mozpath.basename(obj.path))
# on win32, pref files need CRLF line endings... see bug 206029
if self.environment.substs['OS_ARCH'] == 'WINNT':
defines.append('--line-endings=crlf')
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
defines.append('--silence-missing-directive-warnings')
self._preprocess_files[dest] = (obj.srcdir, obj.path, defines)
elif isinstance(obj, Resources) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.resources.walk():
base = mozpath.join(obj.install_target, 'res', path)
for f in strings:
flags = strings.flags_for(f)
if flags and flags.preprocess:
dest = mozpath.join(base, mozpath.basename(f))
defines = Defines(obj._context, obj.defines)
defines = list(defines.get_defines())
defines.extend(['--marker', '%'])
self._preprocess_files[dest] = (obj.srcdir, f, defines)
else:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join('res', path, mozpath.basename(f))
)
elif isinstance(obj, FinalTargetFiles) and \
obj.install_target.startswith('dist/bin'):
for path, strings in obj.files.walk():
base = mozpath.join(obj.install_target, path)
for f in strings:
self._install_manifests[obj.install_target].add_symlink(
mozpath.join(obj.srcdir, f),
mozpath.join(path, mozpath.basename(f))
)
elif isinstance(obj, DistFiles) and \
obj.install_target.startswith('dist/bin'):
# We preprocess these, but they don't necessarily have preprocessor
# directives, so tell the preprocessor to not complain about that.
defines.append('--silence-missing-directive-warnings')
for f in obj.files:
dest = mozpath.join(obj.install_target, mozpath.basename(f))
self._preprocess_files[dest] = (obj.srcdir, f, defines)
else:
return
self._seen_directories.add(obj.objdir)
def consume_finished(self):
mk = Makefile()
# Add the default rule at the very beginning.
mk.create_rule(['default'])
mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
# Add a few necessary variables inherited from configure
for var in (
'PYTHON',
'ACDEFINES',
'MOZ_CHROME_FILE_FORMAT',
):
mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
# Add all necessary information for jar manifest processing
jar_mn_targets = []
for path, (objdir, install_target, defines) in \
self._jar_manifests.iteritems():
rel_manifest = mozpath.relpath(path, self.environment.topsrcdir)
target = rel_manifest.replace('/', '-')
assert target not in jar_mn_targets
jar_mn_targets.append(target)
target = 'jar-%s' % target
mk.create_rule([target]).add_dependencies([path])
if objdir != mozpath.join(self.environment.topobjdir,
mozpath.dirname(rel_manifest)):
mk.create_rule([target]).add_dependencies(
['objdir = %s' % objdir])
if install_target != 'dist/bin':
mk.create_rule([target]).add_dependencies(
['install_target = %s' % install_target])
if defines:
mk.create_rule([target]).add_dependencies(
['defines = %s' % ' '.join(defines)])
mk.add_statement('JAR_MN_TARGETS = %s' % ' '.join(jar_mn_targets))
# Add information for chrome manifest generation
manifest_targets = []
for target, entries in self._manifest_entries.iteritems():
manifest_targets.append(target)
target = '$(TOPOBJDIR)/%s' % target
mk.create_rule([target]).add_dependencies(
['content = %s' % ' '.join('"%s"' % e for e in entries)])
mk.add_statement('MANIFEST_TARGETS = %s' % ' '.join(manifest_targets))
# Add information for preprocessed files.
preprocess_targets = []
for target, (srcdir, f, defines) in self._preprocess_files.iteritems():
# This matches what PP_TARGETS do in config/rules.
if target.endswith('.in'):
target = target[:-3]
# PP_TARGETS assumes this is true, but doesn't enforce it.
assert target not in self._preprocess_files
preprocess_targets.append(target)
target = '$(TOPOBJDIR)/%s' % target
mk.create_rule([target]).add_dependencies(
[mozpath.join(srcdir, f)])
if defines:
mk.create_rule([target]).add_dependencies(
['defines = %s' % ' '.join(defines)])
mk.add_statement('PP_TARGETS = %s' % ' '.join(preprocess_targets))
# Add information for install manifests.
mk.add_statement('INSTALL_MANIFESTS = %s'
% ' '.join(self._install_manifests.keys()))
mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
for base, install_manifest in self._install_manifests.iteritems():
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'install_%s' % base.replace('/', '_'))) as fh:
install_manifest.write(fileobj=fh)
with self._write_file(
mozpath.join(self.environment.topobjdir, 'faster',
'Makefile')) as fh:
mk.dump(fh, removal_guard=False)

View File

@ -96,7 +96,8 @@ def config_status(topobjdir='.', topsrcdir='.',
parser.add_option('-d', '--diff', action='store_true',
help='print diffs of changed files.')
parser.add_option('-b', '--backend',
choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse', 'VisualStudio'],
choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse',
'VisualStudio', 'FasterMake'],
default='RecursiveMake',
help='what backend to build (default: RecursiveMake).')
options, args = parser.parse_args()
@ -128,6 +129,9 @@ def config_status(topobjdir='.', topsrcdir='.',
elif options.backend == 'VisualStudio':
from mozbuild.backend.visualstudio import VisualStudioBackend
backend_cls = VisualStudioBackend
elif options.backend == 'FasterMake':
from mozbuild.backend.fastermake import FasterMakeBackend
backend_cls = FasterMakeBackend
the_backend = backend_cls(env)

View File

@ -559,7 +559,8 @@ class Build(MachCommandBase):
# It would be nice to filter the choices below based on
# conditions, but that is for another day.
@CommandArgument('-b', '--backend',
choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse', 'VisualStudio'],
choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse',
'VisualStudio', 'FasterMake'],
default='RecursiveMake',
help='Which backend to build (default: RecursiveMake).')
def build_backend(self, backend='RecursiveMake', diff=False):

View File

@ -54,7 +54,9 @@
<tr>
<td>@CXX@</td>
<td>@CXX_VERSION@</td>
#ifndef BUILD_FASTER
<td>@CXXFLAGS@ @CPPFLAGS@</td>
#endif
</tr>
</tbody>
</table>