mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-28 07:13:20 +00:00
Bug 1563403 - Use 3-tier PGO for local MOZ_PGO=1 builds; r=firefox-build-system-reviewers,dmajor,chmanchester
Local PGO builds now use 3-tier machinery under the hood. Instead of a
single object directory that gets cleaned in between the instrumented
and profile-use builds, now the instrumented build uses a separate
'${objdir}/instrumented' directory. This makes it easier to handle
within mach since we can drive the two builds with environment variables
and keep all build artifacts separate, without needing to do manual
cleanup in between.
Differential Revision: https://phabricator.services.mozilla.com/D50098
--HG--
extra : moz-landing-system : lando
This commit is contained in:
Mike Shal
parent
554a8c6462
commit
25308d30a2
49
Makefile.in
49
Makefile.in
@ -115,11 +115,8 @@ install-test-files:
|
||||
|
||||
include $(topsrcdir)/build/moz-automation.mk
|
||||
|
||||
# dist and _tests should be purged during cleaning. However, we don't want them
|
||||
# purged during PGO builds because they contain some auto-generated files.
|
||||
ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
|
||||
# dist and _tests should be purged during cleaning.
|
||||
GARBAGE_DIRS += dist _tests
|
||||
endif
|
||||
|
||||
# Dummy rule for the cases below where we don't depend on dist/include
|
||||
recurse_pre-export::
|
||||
@ -185,37 +182,6 @@ endif
|
||||
default all::
|
||||
$(call BUILDSTATUS,TIERS $(TIERS) $(if $(MOZ_AUTOMATION),$(MOZ_AUTOMATION_TIERS)))
|
||||
|
||||
# PGO build target.
|
||||
profiledbuild::
|
||||
$(call BUILDSTATUS,TIERS pgo_profile_generate pgo_package pgo_profile pgo_clobber pgo_profile_use)
|
||||
$(call BUILDSTATUS,TIER_START pgo_profile_generate)
|
||||
$(MAKE) default MOZ_PROFILE_GENERATE=1 MOZ_LTO=
|
||||
$(call BUILDSTATUS,TIER_FINISH pgo_profile_generate)
|
||||
$(call BUILDSTATUS,TIER_START pgo_package)
|
||||
$(MAKE) package
|
||||
rm -f jarlog/en-US.log
|
||||
$(call BUILDSTATUS,TIER_FINISH pgo_package)
|
||||
$(call BUILDSTATUS,TIER_START pgo_profile)
|
||||
JARLOG_FILE=jarlog/en-US.log $(PYTHON) $(topsrcdir)/build/pgo/profileserver.py
|
||||
$(call BUILDSTATUS,TIER_FINISH pgo_profile)
|
||||
$(call BUILDSTATUS,TIER_START pgo_clobber)
|
||||
$(MAKE) maybe_clobber_profiledbuild
|
||||
$(call BUILDSTATUS,TIER_FINISH pgo_clobber)
|
||||
$(call BUILDSTATUS,TIER_START pgo_profile_use)
|
||||
$(MAKE) default MOZ_PROFILE_USE=1 MOZ_1TIER_PGO=1
|
||||
$(call BUILDSTATUS,TIER_FINISH pgo_profile_use)
|
||||
|
||||
# Change default target to PGO build if PGO is enabled.
|
||||
ifdef MOZ_PGO
|
||||
ifdef COMPILE_ENVIRONMENT
|
||||
# If one of these is already set in addition to PGO we are doing a single phase
|
||||
# of PGO in isolation, so don't override the default target.
|
||||
ifeq (,$(MOZ_PROFILE_GENERATE)$(MOZ_PROFILE_USE))
|
||||
OVERRIDE_DEFAULT_GOAL := profiledbuild
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
include $(topsrcdir)/config/rules.mk
|
||||
|
||||
ifdef SCCACHE_VERBOSE_STATS
|
||||
@ -281,19 +247,6 @@ update-packaging:
|
||||
package-generated-sources:
|
||||
$(call py_action,package_generated_sources,'$(DIST)/$(PKG_PATH)$(GENERATED_SOURCE_FILE_PACKAGE)')
|
||||
|
||||
# PGO support, but we can't do this test in client.mk
|
||||
# No point in clobbering if PGO has been explicitly disabled.
|
||||
ifdef NO_PROFILE_GUIDED_OPTIMIZE
|
||||
maybe_clobber_profiledbuild:
|
||||
else
|
||||
maybe_clobber_profiledbuild: clean
|
||||
ifneq (,$(findstring clang,$(CC_TYPE)))
|
||||
$(LLVM_PROFDATA) merge -o $(DEPTH)/merged.profdata $(DEPTH)/*.profraw
|
||||
endif
|
||||
endif # NO_PROFILE_GUIDED_OPTIMIZE
|
||||
|
||||
.PHONY: maybe_clobber_profiledbuild
|
||||
|
||||
ifdef JS_STANDALONE
|
||||
# Delegate js-specific rules to js
|
||||
check-%:
|
||||
|
||||
@ -23,17 +23,6 @@ Then::
|
||||
|
||||
This is roughly equivalent to::
|
||||
|
||||
#. Perform a build with *MOZ_PROFILE_GENERATE=1*
|
||||
#. Performing a run of the instrumented binaries
|
||||
#. $ make maybe_clobber_profiledbuild
|
||||
#. Perform a build with *MOZ_PROFILE_USE=1*
|
||||
|
||||
Differences between toolchains
|
||||
==============================
|
||||
|
||||
There are some implementation differences depending on the compiler
|
||||
toolchain being used.
|
||||
|
||||
The *maybe_clobber_profiledbuild* step gets its name because of a
|
||||
difference. On Windows, this step merely moves some *.pgc* files around.
|
||||
Using GCC or Clang, it is equivalent to a *make clean*.
|
||||
#. Perform a build with *--enable-profile-generate* in $topobjdir/instrumented
|
||||
#. Perform a run of the instrumented binaries with build/pgo/profileserver.py
|
||||
#. Perform a build with *--enable-profile-use* in $topobjdir
|
||||
|
||||
@ -11,6 +11,7 @@ llvm_profdata = check_prog('LLVM_PROFDATA', ['llvm-profdata'],
|
||||
paths=toolchain_search_path)
|
||||
|
||||
js_option('--enable-profile-generate',
|
||||
env='MOZ_PROFILE_GENERATE',
|
||||
nargs='?',
|
||||
choices=('cross',),
|
||||
help='Build a PGO instrumented binary')
|
||||
@ -25,6 +26,7 @@ set_define('MOZ_PROFILE_GENERATE',
|
||||
depends_if('--enable-profile-generate')(lambda _: True))
|
||||
|
||||
js_option('--enable-profile-use',
|
||||
env='MOZ_PROFILE_USE',
|
||||
nargs='?',
|
||||
choices=('cross',),
|
||||
help='Use a generated profile during the build')
|
||||
@ -51,7 +53,7 @@ def pgo_profile_path(path, pgo_use, profdata, build_env):
|
||||
topobjdir = topobjdir[:-7]
|
||||
|
||||
if not path:
|
||||
return os.path.join(topobjdir, 'merged.profdata')
|
||||
return os.path.join(topobjdir, 'instrumented', 'merged.profdata')
|
||||
if path and not pgo_use:
|
||||
die('Pass --enable-profile-use to use --with-pgo-profile-path.')
|
||||
if path and not profdata:
|
||||
|
||||
@ -80,6 +80,10 @@ if __name__ == '__main__':
|
||||
port=PORT,
|
||||
options='primary,privileged')
|
||||
|
||||
old_profraw_files = glob.glob('*.profraw')
|
||||
for f in old_profraw_files:
|
||||
os.remove(f)
|
||||
|
||||
with TemporaryDirectory() as profilePath:
|
||||
# TODO: refactor this into mozprofile
|
||||
profile_data_dir = os.path.join(build.topsrcdir, 'testing', 'profiles')
|
||||
@ -151,6 +155,8 @@ if __name__ == '__main__':
|
||||
if jarlog:
|
||||
env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
|
||||
print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
|
||||
if os.path.exists(jarlog):
|
||||
os.remove(jarlog)
|
||||
|
||||
if 'UPLOAD_PATH' in env:
|
||||
process_args['logfile'] = os.path.join(env['UPLOAD_PATH'], 'profile-run-2.log')
|
||||
|
||||
@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from mach.decorators import (
|
||||
CommandArgument,
|
||||
@ -14,6 +15,9 @@ from mach.decorators import (
|
||||
)
|
||||
|
||||
from mozbuild.base import MachCommandBase
|
||||
from mozbuild.util import ensure_subprocess_env
|
||||
from mozbuild.mozconfig import MozconfigLoader
|
||||
import mozpack.path as mozpath
|
||||
|
||||
from mozbuild.backend import (
|
||||
backends,
|
||||
@ -73,6 +77,54 @@ class Build(MachCommandBase):
|
||||
|
||||
self.log_manager.enable_all_structured_loggers()
|
||||
|
||||
loader = MozconfigLoader(self.topsrcdir)
|
||||
mozconfig = loader.read_mozconfig(loader.AUTODETECT)
|
||||
doing_pgo = 'MOZ_PGO=1' in mozconfig['configure_args']
|
||||
append_env = None
|
||||
|
||||
if doing_pgo:
|
||||
if what:
|
||||
raise Exception('Cannot specify targets (%s) in MOZ_PGO=1 builds' %
|
||||
what)
|
||||
instr = self._spawn(BuildDriver)
|
||||
orig_topobjdir = instr._topobjdir
|
||||
instr._topobjdir = mozpath.join(instr._topobjdir, 'instrumented')
|
||||
|
||||
append_env = {'MOZ_PROFILE_GENERATE': '1'}
|
||||
status = instr.build(
|
||||
what=what,
|
||||
disable_extra_make_dependencies=disable_extra_make_dependencies,
|
||||
jobs=jobs,
|
||||
directory=directory,
|
||||
verbose=verbose,
|
||||
keep_going=keep_going,
|
||||
mach_context=self._mach_context,
|
||||
append_env=append_env)
|
||||
if status != 0:
|
||||
return status
|
||||
|
||||
# Packaging the instrumented build is required to get the jarlog
|
||||
# data.
|
||||
status = instr._run_make(
|
||||
directory=".", target='package',
|
||||
silent=not verbose, ensure_exit_code=False,
|
||||
append_env=append_env)
|
||||
if status != 0:
|
||||
return status
|
||||
|
||||
pgo_env = os.environ.copy()
|
||||
pgo_env['LLVM_PROFDATA'] = instr.config_environment.substs.get('LLVM_PROFDATA')
|
||||
pgo_env['JARLOG_FILE'] = mozpath.join(orig_topobjdir, 'jarlog/en-US.log')
|
||||
pgo_cmd = [
|
||||
instr.virtualenv_manager.python_path,
|
||||
mozpath.join(self.topsrcdir, 'build/pgo/profileserver.py'),
|
||||
]
|
||||
subprocess.check_call(pgo_cmd, cwd=instr.topobjdir,
|
||||
env=ensure_subprocess_env(pgo_env))
|
||||
|
||||
# Set the default build to MOZ_PROFILE_USE
|
||||
append_env = {'MOZ_PROFILE_USE': '1'}
|
||||
|
||||
driver = self._spawn(BuildDriver)
|
||||
return driver.build(
|
||||
what=what,
|
||||
@ -81,7 +133,8 @@ class Build(MachCommandBase):
|
||||
directory=directory,
|
||||
verbose=verbose,
|
||||
keep_going=keep_going,
|
||||
mach_context=self._mach_context)
|
||||
mach_context=self._mach_context,
|
||||
append_env=append_env)
|
||||
|
||||
@Command('configure', category='build',
|
||||
description='Configure the tree (run configure and config.status).')
|
||||
|
||||
@ -996,7 +996,8 @@ class BuildDriver(MozbuildObject):
|
||||
self.mach_context = None
|
||||
|
||||
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
|
||||
directory=None, verbose=False, keep_going=False, mach_context=None):
|
||||
directory=None, verbose=False, keep_going=False, mach_context=None,
|
||||
append_env=None):
|
||||
"""Invoke the build backend.
|
||||
|
||||
``what`` defines the thing to build. If not defined, the default
|
||||
@ -1065,7 +1066,8 @@ class BuildDriver(MozbuildObject):
|
||||
print(" Config object not found by mach.")
|
||||
|
||||
config_rc = self.configure(buildstatus_messages=True,
|
||||
line_handler=output.on_line)
|
||||
line_handler=output.on_line,
|
||||
append_env=append_env)
|
||||
|
||||
if config_rc != 0:
|
||||
return config_rc
|
||||
@ -1158,12 +1160,13 @@ class BuildDriver(MozbuildObject):
|
||||
# could potentially be fixed if the build monitor were more
|
||||
# intelligent about encountering undefined state.
|
||||
no_build_status = b'1' if make_dir is not None else b''
|
||||
tgt_env = dict(append_env or {})
|
||||
tgt_env['NO_BUILDSTATUS_MESSAGES'] = no_build_status
|
||||
status = self._run_make(
|
||||
directory=make_dir, target=make_target,
|
||||
line_handler=output.on_line, log=False, print_directory=False,
|
||||
ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
|
||||
append_env={
|
||||
b'NO_BUILDSTATUS_MESSAGES': no_build_status},
|
||||
append_env=tgt_env,
|
||||
keep_going=keep_going)
|
||||
|
||||
if status != 0:
|
||||
@ -1175,7 +1178,8 @@ class BuildDriver(MozbuildObject):
|
||||
status = self._run_client_mk(line_handler=output.on_line,
|
||||
jobs=jobs,
|
||||
verbose=verbose,
|
||||
keep_going=keep_going)
|
||||
keep_going=keep_going,
|
||||
append_env=append_env)
|
||||
|
||||
self.log(logging.WARNING, 'warning_summary',
|
||||
{'count': len(monitor.warnings_database)},
|
||||
@ -1338,7 +1342,7 @@ class BuildDriver(MozbuildObject):
|
||||
return status
|
||||
|
||||
def configure(self, options=None, buildstatus_messages=False,
|
||||
line_handler=None):
|
||||
line_handler=None, append_env=None):
|
||||
# Disable indexing in objdir because it is not necessary and can slow
|
||||
# down builds.
|
||||
mkdir(self.topobjdir, not_indexed=True)
|
||||
@ -1350,12 +1354,13 @@ class BuildDriver(MozbuildObject):
|
||||
line_handler = line_handler or on_line
|
||||
|
||||
options = ' '.join(shell_quote(o) for o in options or ())
|
||||
append_env = {b'CONFIGURE_ARGS': options.encode('utf-8')}
|
||||
append_env = dict(append_env or {})
|
||||
append_env['CONFIGURE_ARGS'] = options
|
||||
|
||||
# Only print build status messages when we have an active
|
||||
# monitor.
|
||||
if not buildstatus_messages:
|
||||
append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1'
|
||||
append_env['NO_BUILDSTATUS_MESSAGES'] = b'1'
|
||||
status = self._run_client_mk(target='configure',
|
||||
line_handler=line_handler,
|
||||
append_env=append_env)
|
||||
|
||||
@ -349,6 +349,12 @@ class MozconfigLoader(object):
|
||||
|
||||
if name == 'MOZ_OBJDIR':
|
||||
result['topobjdir'] = value
|
||||
if parsed['env_before'].get('MOZ_PROFILE_GENERATE') == '1':
|
||||
# If MOZ_OBJDIR is specified in the mozconfig, we need to
|
||||
# make sure that the '/instrumented' directory gets appended
|
||||
# for the first build to avoid an objdir mismatch when
|
||||
# running 'mach package' on Windows.
|
||||
result['topobjdir'] = mozpath.join(result['topobjdir'], 'instrumented')
|
||||
continue
|
||||
|
||||
result['make_extra'].append(o)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user