mirror of
https://github.com/mozilla/gecko-dev.git
synced 2025-02-26 20:30:41 +00:00
Bug 1726782 - Create a new task generating a clang profile for PGO. r=firefox-build-system-reviewers,mhentges
The new task builds a stage 2 compiler and uses it to generate the profile as stage 3, and the profile is used to feed the now separate stage 4 clang build. This will allow us to use this same profile to build mac clangs with PGO. Differential Revision: https://phabricator.services.mozilla.com/D138511
This commit is contained in:
Mike Hommey
parent
f0a2c36a16
commit
3789acd7bb
@ -199,7 +199,7 @@ def build_one_stage(
|
||||
libcxx_include_dir,
|
||||
build_wasm,
|
||||
is_final_stage=False,
|
||||
pgo_phase=None,
|
||||
profile=None,
|
||||
):
|
||||
if not os.path.exists(stage_dir):
|
||||
os.mkdir(stage_dir)
|
||||
@ -316,15 +316,15 @@ def build_one_stage(
|
||||
"-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
|
||||
% os.environ["MACOSX_DEPLOYMENT_TARGET"],
|
||||
]
|
||||
if pgo_phase == "gen":
|
||||
if profile == "gen":
|
||||
# Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
|
||||
cmake_args += [
|
||||
"-DLLVM_BUILD_INSTRUMENTED=IR",
|
||||
"-DLLVM_BUILD_RUNTIME=No",
|
||||
]
|
||||
if pgo_phase == "use":
|
||||
elif profile:
|
||||
cmake_args += [
|
||||
"-DLLVM_PROFDATA_FILE=%s/merged.profdata" % stage_dir,
|
||||
"-DLLVM_PROFDATA_FILE=%s" % profile,
|
||||
]
|
||||
return cmake_args
|
||||
|
||||
@ -469,7 +469,7 @@ def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
|
||||
delete(f)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
@ -573,8 +573,6 @@ if __name__ == "__main__":
|
||||
pgo = config["pgo"]
|
||||
if pgo not in (True, False):
|
||||
raise ValueError("Only boolean values are accepted for pgo.")
|
||||
if pgo and stages != 4:
|
||||
raise ValueError("PGO is only supported in 4-stage builds.")
|
||||
build_type = "Release"
|
||||
if "build_type" in config:
|
||||
build_type = config["build_type"]
|
||||
@ -787,7 +785,6 @@ if __name__ == "__main__":
|
||||
stage2_inst_dir = stage2_dir + "/" + package_name
|
||||
final_stage_dir = stage2_dir
|
||||
final_inst_dir = stage2_inst_dir
|
||||
pgo_phase = "gen" if pgo else None
|
||||
if skip_stages < 1:
|
||||
cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
|
||||
cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
|
||||
@ -810,7 +807,7 @@ if __name__ == "__main__":
|
||||
libcxx_include_dir,
|
||||
build_wasm,
|
||||
is_final_stage=(stages == 2),
|
||||
pgo_phase=pgo_phase,
|
||||
profile="gen" if pgo else None,
|
||||
)
|
||||
|
||||
if stages >= 3 and skip_stages < 3:
|
||||
@ -841,23 +838,30 @@ if __name__ == "__main__":
|
||||
build_wasm,
|
||||
(stages == 3),
|
||||
)
|
||||
if pgo:
|
||||
llvm_profdata = stage2_inst_dir + "/bin/llvm-profdata%s" % exe_ext
|
||||
merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
|
||||
profraw_files = glob.glob(
|
||||
os.path.join(stage2_dir, "build", "profiles", "*.profraw")
|
||||
)
|
||||
run_in(stage3_dir, merge_cmd + profraw_files)
|
||||
if stages == 3:
|
||||
mkdir_p(upload_dir)
|
||||
shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir)
|
||||
return
|
||||
|
||||
if stages >= 4 and skip_stages < 4:
|
||||
stage4_dir = build_dir + "/stage4"
|
||||
stage4_inst_dir = stage4_dir + "/" + package_name
|
||||
final_stage_dir = stage4_dir
|
||||
final_inst_dir = stage4_inst_dir
|
||||
pgo_phase = None
|
||||
profile = None
|
||||
if pgo:
|
||||
pgo_phase = "use"
|
||||
llvm_profdata = stage3_inst_dir + "/bin/llvm-profdata%s" % exe_ext
|
||||
merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
|
||||
profraw_files = glob.glob(
|
||||
os.path.join(stage2_dir, "build", "profiles", "*.profraw")
|
||||
)
|
||||
if not os.path.exists(stage4_dir):
|
||||
os.mkdir(stage4_dir)
|
||||
run_in(stage4_dir, merge_cmd + profraw_files)
|
||||
if skip_stages == 3:
|
||||
profile_dir = os.environ.get("MOZ_FETCHES_DIR", "")
|
||||
else:
|
||||
profile_dir = stage3_dir
|
||||
profile = os.path.join(profile_dir, "merged.profdata")
|
||||
if skip_stages < 3:
|
||||
cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
|
||||
cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
|
||||
@ -880,7 +884,7 @@ if __name__ == "__main__":
|
||||
libcxx_include_dir,
|
||||
build_wasm,
|
||||
(stages == 4),
|
||||
pgo_phase=pgo_phase,
|
||||
profile=profile,
|
||||
)
|
||||
|
||||
if build_clang_tidy:
|
||||
@ -905,3 +909,7 @@ if __name__ == "__main__":
|
||||
|
||||
if not args.skip_tar:
|
||||
build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
4
build/build-clang/profile.json
Normal file
4
build/build-clang/profile.json
Normal file
@ -0,0 +1,4 @@
|
||||
{
|
||||
"stages": "3",
|
||||
"pgo": true
|
||||
}
|
||||
6
build/build-clang/skip-3-stages.json
Normal file
6
build/build-clang/skip-3-stages.json
Normal file
@ -0,0 +1,6 @@
|
||||
{
|
||||
"skip_stages": "3",
|
||||
"cc": "{MOZ_FETCHES_DIR}/clang/bin/clang",
|
||||
"cxx": "{MOZ_FETCHES_DIR}/clang/bin/clang++",
|
||||
"as": "{MOZ_FETCHES_DIR}/clang/bin/clang"
|
||||
}
|
||||
@ -124,6 +124,31 @@ linux64-clang-13-stage1:
|
||||
fetch:
|
||||
- clang-13
|
||||
|
||||
linux64-clang-13-profile:
|
||||
description: "Clang 13 toolchain build"
|
||||
treeherder:
|
||||
symbol: TL(clang13-profile)
|
||||
run:
|
||||
using: toolchain-script
|
||||
script: build-clang.sh
|
||||
arguments:
|
||||
- 'build/build-clang/linux64.json'
|
||||
- 'build/build-clang/clang-13.json'
|
||||
- 'build/build-clang/skip-stage-1.json'
|
||||
- 'build/build-clang/profile.json'
|
||||
resources:
|
||||
- 'build/build-clang/linux64.json'
|
||||
- 'build/build-clang/clang-13.json'
|
||||
- 'build/build-clang/skip-stage-1.json'
|
||||
- 'build/build-clang/profile.json'
|
||||
toolchain-artifact: public/build/merged.profdata
|
||||
fetches:
|
||||
fetch:
|
||||
- clang-13
|
||||
toolchain:
|
||||
- linux64-clang-13-stage1
|
||||
- linux64-toolchain-sysroot
|
||||
|
||||
linux64-clang-13:
|
||||
description: "Clang 13 toolchain build"
|
||||
attributes:
|
||||
@ -137,12 +162,12 @@ linux64-clang-13:
|
||||
arguments:
|
||||
- 'build/build-clang/linux64.json'
|
||||
- 'build/build-clang/clang-13.json'
|
||||
- 'build/build-clang/skip-stage-1.json'
|
||||
- 'build/build-clang/skip-3-stages.json'
|
||||
- 'build/build-clang/4stages-pgo.json'
|
||||
resources:
|
||||
- 'build/build-clang/linux64.json'
|
||||
- 'build/build-clang/clang-13.json'
|
||||
- 'build/build-clang/skip-stage-1.json'
|
||||
- 'build/build-clang/skip-3-stages.json'
|
||||
- 'build/build-clang/4stages-pgo.json'
|
||||
toolchain-alias:
|
||||
- linux64-clang
|
||||
@ -152,6 +177,7 @@ linux64-clang-13:
|
||||
- clang-13
|
||||
toolchain:
|
||||
- linux64-clang-13-stage1
|
||||
- linux64-clang-13-profile
|
||||
- linux64-toolchain-sysroot
|
||||
- wasm32-wasi-compiler-rt-13
|
||||
|
||||
|
||||
@ -51,8 +51,10 @@ python3 $GECKO_PATH/build/build-clang/build-clang.py $CONFIGS
|
||||
|
||||
set -x
|
||||
|
||||
# Put a tarball in the artifacts dir
|
||||
mkdir -p $UPLOAD_DIR
|
||||
cp clang*.tar.* $UPLOAD_DIR
|
||||
if [ -f clang.tar.zst ]; then
|
||||
# Put a tarball in the artifacts dir
|
||||
mkdir -p $UPLOAD_DIR
|
||||
cp clang.tar.zst $UPLOAD_DIR
|
||||
fi
|
||||
|
||||
. $GECKO_PATH/taskcluster/scripts/misc/vs-cleanup.sh
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user