Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-12-15 11:13:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge inbound to mozilla-central. a=merge

Browse Source
This commit is contained in:
Noemi Erli 2019-08-07 12:56:42 +03:00
commit 9ae2049722
74 changed files with 742 additions and 659 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
browser/config/tooltool-manifests/win32/build-clang-cl.manifest
View File

@ -1,34 +0,0 @@
[
{
"version": "Visual Studio 2017 15.8.4 / SDK 10.0.17134.0",
"digest": "ecf1e03f6f98f86775059a43f9e7dc7e326f6643d7c08962d9f614e4f5a65b1ca63fa1cfeb0f1a3c2474bf0d4318dda960b378beb2a44ecf8a91111207f4ece5",
"size": 349626009,
"algorithm": "sha512",
"filename": "vs2017_15.8.4.zip",
"unpack": true
},
{
"version": "MinGit-2.13.3-64-bit",
"size": 21482885,
"digest": "929bb3c07be8487ee519422a312bdbfeec8f4db4b62c49d02f9aad9fd2a66c0ee5fad63d2b06c8744c336dc9d50446fa4457897333ad17ffd783ecabd1e2ddbb",
"algorithm": "sha512",
"filename": "git.zip",
"unpack": true
},
{
"version": "CMake 3.6.2 repack",
"size": 19832889,
"digest": "39b0508b60f655969d1b54c76753b14b5b2e92dab58613c835aed798a6aeb9077a7df78aebc011c2c753661fdc15007d3353c0c8773a53148380e2ec02afb629",
"algorithm": "sha512",
"filename": "cmake362.zip",
"unpack": true
},
{
"version": "Ninja 1.7.1",
"size": 184821,
"digest": "e4f9a1ae624a2630e75264ba37d396d9c7407d6e6aea3763056210ba6e1387908bd31cf4037a6a3661a418e86c4d2761e0c333e6a3bd0d66549d2b0d72d3f43b",
"algorithm": "sha512",
"filename": "ninja171.zip",
"unpack": true
}
]

18
browser/config/tooltool-manifests/win32/gn-build.manifest
View File

@ -1,18 +0,0 @@
[
{
"version": "Visual Studio 2017 15.8.4 / SDK 10.0.17134.0",
"digest": "ecf1e03f6f98f86775059a43f9e7dc7e326f6643d7c08962d9f614e4f5a65b1ca63fa1cfeb0f1a3c2474bf0d4318dda960b378beb2a44ecf8a91111207f4ece5",
"size": 349626009,
"algorithm": "sha512",
"filename": "vs2017_15.8.4.zip",
"unpack": true
},
{
"version": "Ninja 1.7.1",
"size": 184821,
"digest": "e4f9a1ae624a2630e75264ba37d396d9c7407d6e6aea3763056210ba6e1387908bd31cf4037a6a3661a418e86c4d2761e0c333e6a3bd0d66549d2b0d72d3f43b",
"algorithm": "sha512",
"filename": "ninja171.zip",
"unpack": true
}
]

0
browser/config/tooltool-manifests/win64/sccache-build.manifest → browser/config/tooltool-manifests/win64/vs2017.manifest
View File

62
build/build-clang/build-clang.py
View File

@ -37,7 +37,7 @@ def symlink(source, link_name):
def check_run(args):
print(' '.join(args), file=sys.stderr)
print(' '.join(args), file=sys.stderr, flush=True)
if args[0] == 'cmake':
# CMake `message(STATUS)` messages, as appearing in failed source code
# compiles, appear on stdout, so we only capture that.
@ -49,8 +49,8 @@ def check_run(args):
sys.stdout.flush()
r = p.wait()
if r != 0:
cmake_output_re = re.compile("See also \"(.*/CMakeOutput.log)\"")
cmake_error_re = re.compile("See also \"(.*/CMakeError.log)\"")
cmake_output_re = re.compile(b"See also \"(.*/CMakeOutput.log)\"")
cmake_error_re = re.compile(b"See also \"(.*/CMakeError.log)\"")
def find_first_match(re):
for l in lines:
@ -214,16 +214,6 @@ def install_asan_symbols(build_dir, clang_dir):
shutil.copy2(src_path[0], dst_path[0])
def git_clone(base_dir, url, directory, revision):
run_in(base_dir, ["git", "clone", "-n", url, directory])
run_in(os.path.join(base_dir, directory), ["git", "checkout", revision])
def git_update(directory, revision):
run_in(directory, ["git", "remote", "update"])
run_in(directory, ["git", "reset", "--hard", revision])
def is_darwin():
return platform.system() == "Darwin"
@ -551,8 +541,6 @@ if __name__ == "__main__":
parser.add_argument('-c', '--config', required=True,
type=argparse.FileType('r'),
help="Clang configuration file")
parser.add_argument('-b', '--base-dir', required=False,
help="Base directory for code and build artifacts")
parser.add_argument('--clean', required=False,
action='store_true',
help="Clean the build directory")
@ -565,35 +553,11 @@ if __name__ == "__main__":
args = parser.parse_args()
# The directories end up in the debug info, so the easy way of getting
# a reproducible build is to run it in a know absolute directory.
# We use a directory that is registered as a volume in the Docker image.
if args.base_dir:
base_dir = args.base_dir
elif os.environ.get('MOZ_AUTOMATION') and not is_windows():
base_dir = "/builds/worker/workspace/moz-toolchain"
else:
# Handles both the Windows automation case and the local build case
# TODO: Because Windows taskcluster builds are run with distinct
# user IDs for each job, we can't store things in some globally
# accessible directory: one job will run, checkout LLVM to that
# directory, and then if another job runs, the new user won't be
# able to access the previously-checked out code--or be able to
# delete it. So on Windows, we build in the task-specific home
# directory; we will eventually add -fdebug-prefix-map options
# to the LLVM build to bring back reproducibility.
base_dir = os.path.join(os.getcwd(), 'build-clang')
source_dir = base_dir + "/src"
build_dir = base_dir + "/build"
if not os.path.exists(base_dir):
os.makedirs(base_dir)
elif os.listdir(base_dir) and not os.path.exists(os.path.join(base_dir, '.build-clang')):
raise ValueError("Base directory %s exists and is not a build-clang directory. "
"Supply a non-existent or empty directory with --base-dir" % base_dir)
open(os.path.join(base_dir, '.build-clang'), 'a').close()
if not os.path.exists('llvm/LLVMBuild.txt'):
raise Exception('The script must be run from the root directory of the '
'llvm-project tree')
source_dir = os.getcwd()
build_dir = source_dir + "/build"
if args.clean:
shutil.rmtree(build_dir)
@ -623,9 +587,6 @@ if __name__ == "__main__":
config_dir = os.path.dirname(args.config.name)
config = json.load(args.config)
llvm_revision = config["llvm_revision"]
if not re.match(r'^[0-9a-fA-F]{40}$', llvm_revision):
raise ValueError("Incorrect format of the git revision")
stages = 3
if "stages" in config:
stages = int(config["stages"])
@ -699,13 +660,6 @@ if __name__ == "__main__":
if not os.path.exists(source_dir):
os.makedirs(source_dir)
if not args.skip_checkout:
if os.path.exists(os.path.join(source_dir, '.git')):
git_update(source_dir, llvm_revision)
else:
delete(source_dir)
git_clone(base_dir, URL_REPO, source_dir, llvm_revision)
for p in config.get("patches", []):
patch(os.path.join(config_dir, p), source_dir)

9
build/build-clang/clang-4.0-linux64.json
View File

@ -1,13 +1,12 @@
{
"llvm_revision": "449c3ef93afc7a668eb35e67a83717453e28b25a",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"patches": []
}

9
build/build-clang/clang-7-linux64.json
View File

@ -1,14 +1,13 @@
{
"llvm_revision": "d0d8eb2e5415b8be29343e3c17a18e49e67b5551",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"patches": [
"static-llvm-symbolizer.patch",
"find_symbolizer_linux.patch",

41
build/build-clang/clang-8-android.json
View File

@ -1,48 +1,47 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "2",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"android_targets": {
"armv7-linux-android": {
"ndk_toolchain": "/builds/worker/workspace/build/src/android-ndk/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/src/android-ndk/platforms/android-16/arch-arm",
"ndk_toolchain": "/builds/worker/workspace/build/android-ndk/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/android-ndk/platforms/android-16/arch-arm",
"ndk_includes": [
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include/arm-linux-androideabi",
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include"
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include/arm-linux-androideabi",
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include"
],
"api_level": 16
},
"i686-linux-android": {
"ndk_toolchain": "/builds/worker/workspace/build/src/android-ndk/toolchains/x86-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/src/android-ndk/platforms/android-16/arch-x86",
"ndk_toolchain": "/builds/worker/workspace/build/android-ndk/toolchains/x86-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/android-ndk/platforms/android-16/arch-x86",
"ndk_includes": [
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include/i686-linux-android",
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include"
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include/i686-linux-android",
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include"
],
"api_level": 16
},
"aarch64-linux-android": {
"ndk_toolchain": "/builds/worker/workspace/build/src/android-ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/src/android-ndk/platforms/android-21/arch-arm64",
"ndk_toolchain": "/builds/worker/workspace/build/android-ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/android-ndk/platforms/android-21/arch-arm64",
"ndk_includes": [
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include/aarch64-linux-android",
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include"
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include/aarch64-linux-android",
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include"
],
"api_level": 21
},
"x86_64-linux-android": {
"ndk_toolchain": "/builds/worker/workspace/build/src/android-ndk/toolchains/x86_64-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/src/android-ndk/platforms/android-21/arch-x86_64",
"ndk_toolchain": "/builds/worker/workspace/build/android-ndk/toolchains/x86_64-4.9/prebuilt/linux-x86_64",
"ndk_sysroot": "/builds/worker/workspace/build/android-ndk/platforms/android-21/arch-x86_64",
"ndk_includes": [
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include/x86_64-linux-android",
"/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include"
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include/x86_64-linux-android",
"/builds/worker/workspace/build/android-ndk/sysroot/usr/include"
],
"api_level": 21
}

9
build/build-clang/clang-8-linux64-aarch64-cross.json
View File

@ -1,14 +1,13 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"extra_targets": [
"aarch64-unknown-linux-gnu"
],

9
build/build-clang/clang-8-linux64.json
View File

@ -1,14 +1,13 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"patches": [
"static-llvm-symbolizer.patch",
"find_symbolizer_linux.patch",

17
build/build-clang/clang-8-macosx64.json
View File

@ -1,19 +1,18 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"osx_cross_compile": true,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/clang/bin/clang",
"cxx": "/builds/worker/workspace/build/src/clang/bin/clang++",
"as": "/builds/worker/workspace/build/src/clang/bin/clang",
"ar": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-ar",
"ranlib": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-ranlib",
"libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-libtool",
"ld": "/builds/worker/workspace/build/src/clang/bin/clang",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/clang/bin/clang",
"cxx": "/builds/worker/workspace/build/clang/bin/clang++",
"as": "/builds/worker/workspace/build/clang/bin/clang",
"ar": "/builds/worker/workspace/build/cctools/bin/x86_64-apple-darwin-ar",
"ranlib": "/builds/worker/workspace/build/cctools/bin/x86_64-apple-darwin-ranlib",
"libtool": "/builds/worker/workspace/build/cctools/bin/x86_64-apple-darwin-libtool",
"ld": "/builds/worker/workspace/build/clang/bin/clang",
"patches": [
"static-llvm-symbolizer.patch",
"rename_gcov_flush.patch",

9
build/build-clang/clang-8-mingw.json
View File

@ -1,14 +1,13 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"patches": [
"mingwclang-llvm-objcopy-COFF-Remove-a-superfluous-namespace-qua.patch",
"mingwclang-llvm-objcopy-COFF-Add-support-for-removing-sections.patch",

9
build/build-clang/clang-tidy-linux64.json
View File

@ -1,15 +1,14 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
"assertions": false,
"build_clang_tidy": true,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
"as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/gcc/bin/gcc",
"cxx": "/builds/worker/workspace/build/gcc/bin/g++",
"as": "/builds/worker/workspace/build/gcc/bin/gcc",
"patches": [
"clang-tidy-8.patch"
]

17
build/build-clang/clang-tidy-macosx64.json
View File

@ -1,5 +1,4 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": true,
"build_type": "Release",
@ -7,14 +6,14 @@
"build_clang_tidy": true,
"osx_cross_compile": true,
"python_path": "/usr/bin/python2.7",
"gcc_dir": "/builds/worker/workspace/build/src/gcc",
"cc": "/builds/worker/workspace/build/src/clang/bin/clang",
"cxx": "/builds/worker/workspace/build/src/clang/bin/clang++",
"as": "/builds/worker/workspace/build/src/clang/bin/clang",
"ar": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-ar",
"ranlib": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-ranlib",
"libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin-libtool",
"ld": "/builds/worker/workspace/build/src/clang/bin/clang",
"gcc_dir": "/builds/worker/workspace/build/gcc",
"cc": "/builds/worker/workspace/build/clang/bin/clang",
"cxx": "/builds/worker/workspace/build/clang/bin/clang++",
"as": "/builds/worker/workspace/build/clang/bin/clang",
"ar": "/builds/worker/workspace/build/cctools/bin/x86_64-apple-darwin-ar",
"ranlib": "/builds/worker/workspace/build/cctools/bin/x86_64-apple-darwin-ranlib",
"libtool": "/builds/worker/workspace/build/cctools/bin/x86_64-apple-darwin-libtool",
"ld": "/builds/worker/workspace/build/clang/bin/clang",
"patches": [
"clang-tidy-8.patch",
"compiler-rt-no-codesign.patch"

1
build/build-clang/clang-tidy-win64.json
View File

@ -1,5 +1,4 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "1",
"build_libcxx": false,
"build_type": "Release",

1
build/build-clang/clang-win64.json
View File

@ -1,5 +1,4 @@
{
"llvm_revision": "d2298e74235598f15594fe2c99bbac870a507c59",
"stages": "3",
"build_libcxx": false,
"build_type": "Release",

9
build/unix/build-binutils/build-binutils.sh
View File

@ -1,13 +1,10 @@
#!/bin/bash
binutils_version=2.31.1
make_flags="-j$(nproc)"
root_dir="$1"
cd $root_dir
cd binutils-$binutils_version
cd $root_dir/binutils-source
patch -p1 <<'EOF'
From 4476cc67e657d6b26cd453c555a611f1ab956660 Mon Sep 17 00:00:00 2001
@ -73,7 +70,7 @@ for target in $TARGETS; do
mkdir binutils-$target
cd binutils-$target
../binutils-$binutils_version/configure --prefix /tools/binutils/ --disable-gold --disable-ld --disable-binutils --disable-gprof --disable-nls --target=$target || exit 1
../binutils-source/configure --prefix /tools/binutils/ --disable-gold --disable-ld --disable-binutils --disable-gprof --disable-nls --target=$target || exit 1
make $make_flags || exit 1
make install $make_flags DESTDIR=$root_dir || exit 1
@ -86,7 +83,7 @@ cd binutils-objdir
# --enable-targets builds extra target support in ld.
# Enabling aarch64 support brings in arm support, so we don't need to specify that too.
../binutils-$binutils_version/configure --prefix /tools/binutils/ --enable-gold --enable-plugins --disable-nls --enable-targets="$TARGETS" || exit 1
../binutils-source/configure --prefix /tools/binutils/ --enable-gold --enable-plugins --disable-nls --enable-targets="$TARGETS" || exit 1
make $make_flags || exit 1
make install $make_flags DESTDIR=$root_dir || exit 1

8
build/unix/build-gcc/build-gcc.sh
View File

@ -24,7 +24,7 @@ apply_patch() {
pushd $root_dir/$1
shift
else
pushd $root_dir/gcc-$gcc_version
pushd $root_dir/gcc-source
fi
patch -p1 < $1
popd
@ -47,7 +47,7 @@ build_binutils() {
mkdir $root_dir/binutils-objdir
pushd $root_dir/binutils-objdir
../binutils-$binutils_version/configure --prefix=${prefix-/tools/gcc}/ $binutils_configure_flags
../binutils-source/configure --prefix=${prefix-/tools/gcc}/ $binutils_configure_flags
make $make_flags
make install $make_flags DESTDIR=$root_dir
export PATH=$root_dir/${prefix-/tools/gcc}/bin:$PATH
@ -61,7 +61,7 @@ build_gcc() {
mkdir $root_dir/gcc-objdir
pushd $root_dir/gcc-objdir
../gcc-$gcc_version/configure --prefix=${prefix-/tools/gcc} --build=x86_64-unknown-linux-gnu --target="${target}" --enable-languages=c,c++ --disable-nls --disable-gnu-unique-object --enable-__cxa_atexit --with-arch-32=pentiumpro --with-sysroot=/
../gcc-source/configure --prefix=${prefix-/tools/gcc} --build=x86_64-unknown-linux-gnu --target="${target}" --enable-languages=c,c++ --disable-nls --disable-gnu-unique-object --enable-__cxa_atexit --with-arch-32=pentiumpro --with-sysroot=/
make $make_flags
make $make_flags install DESTDIR=$root_dir
@ -75,7 +75,7 @@ build_gcc() {
build_gcc_and_mingw() {
mkdir gcc-objdir
pushd gcc-objdir
../gcc-$gcc_version/configure --prefix=$install_dir --target=i686-w64-mingw32 --with-gnu-ld --with-gnu-as --disable-multilib --enable-threads=posix
../gcc-source/configure --prefix=$install_dir --target=i686-w64-mingw32 --with-gnu-ld --with-gnu-as --disable-multilib --enable-threads=posix
make $make_flags all-gcc
make $make_flags install-gcc
popd

4
mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly
View File

@ -11,6 +11,10 @@ NO_NDK=1
. "$topsrcdir/mobile/android/config/mozconfigs/common"
# Until normal mozconfigs also use MOZ_FETCHES_DIR
ac_add_options --with-android-sdk="$MOZ_FETCHES_DIR/android-sdk-linux"
export NODEJS=$MOZ_FETCHES_DIR/node/bin/node
# We want to download Gradle.
ac_add_options --with-gradle
# We want to use (and populate!) the local Nexus repositories.

84
taskcluster/ci/fetch/toolchains.yml
View File

@ -12,6 +12,9 @@ binutils-2.27:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/EAF1C276A747E9ED86210CBAC3126D3B4AE55E93.key
artifact-name: binutils-source.tar.zst
strip-components: 1
add-prefix: binutils-source/
binutils-2.31.1:
description: binutils 2.31.1 source code
@ -23,6 +26,9 @@ binutils-2.31.1:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-binutils/3A24BC1E8FB409FA9F14371813FCEF89DD9E3C4F.key
artifact-name: binutils-source.tar.zst
strip-components: 1
add-prefix: binutils-source/
gcc-6.4.0:
description: GCC 6.4.0 source code
@ -34,6 +40,9 @@ gcc-6.4.0:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/13975A70E63C361C73AE69EF6EEB81F8981C74C7.key
artifact-name: gcc-source.tar.zst
strip-components: 1
add-prefix: gcc-source/
gcc-7.4.0:
description: GCC 7.4.0 source code
@ -45,6 +54,9 @@ gcc-7.4.0:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/13975A70E63C361C73AE69EF6EEB81F8981C74C7.key
artifact-name: gcc-source.tar.zst
strip-components: 1
add-prefix: gcc-source/
gcc-8.3.0:
description: GCC 8.3.0 source code
@ -56,6 +68,9 @@ gcc-8.3.0:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/33C235A34C46AA3FFB293709A328C3A2C3C45C06.key
artifact-name: gcc-source.tar.zst
strip-components: 1
add-prefix: gcc-source/
gcc-9.1.0:
description: GCC 9.1.0 source code
@ -67,6 +82,9 @@ gcc-9.1.0:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/33C235A34C46AA3FFB293709A328C3A2C3C45C06.key
artifact-name: gcc-source.tar.zst
strip-components: 1
add-prefix: gcc-source/
gmp-5.1.3:
description: GMP 5.1.3 source code
@ -78,6 +96,9 @@ gmp-5.1.3:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/343C2FF0FBEE5EC2EDBEF399F3599FF828C67298.key
artifact-name: gmp-source.tar.zst
strip-components: 1
add-prefix: gmp-source/
gmp-6.1.0:
description: GMP 6.1.0 source code
@ -89,6 +110,9 @@ gmp-6.1.0:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/343C2FF0FBEE5EC2EDBEF399F3599FF828C67298.key
artifact-name: gmp-source.tar.zst
strip-components: 1
add-prefix: gmp-source/
isl-0.15:
description: ISL 0.15 source code
@ -97,6 +121,9 @@ isl-0.15:
url: ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.15.tar.bz2
sha256: 8ceebbf4d9a81afa2b4449113cee4b7cb14a687d7a549a963deb5e2a41458b6b
size: 1574964
artifact-name: isl-source.tar.zst
strip-components: 1
add-prefix: isl-source/
isl-0.16.1:
description: ISL 0.16.1 source code
@ -105,6 +132,9 @@ isl-0.16.1:
url: ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.16.1.tar.bz2
sha256: 412538bb65c799ac98e17e8cfcdacbb257a57362acfaaff254b0fcae970126d2
size: 1626446
artifact-name: isl-source.tar.zst
strip-components: 1
add-prefix: isl-source/
mpc-0.8.2:
description: mpc 0.8.2 source code
@ -116,6 +146,9 @@ mpc-0.8.2:
gpg-signature:
sig-url: "{url}.asc"
key-path: build/unix/build-gcc/AD17A21EF8AED8F1CC02DBD9F7D5C9BF765C61E3.key
artifact-name: mpc-source.tar.zst
strip-components: 1
add-prefix: mpc-source/
mpc-1.0.3:
description: mpc 1.0.3 source code
@ -127,6 +160,9 @@ mpc-1.0.3:
gpg-signature:
sig-url: "{url}.sig"
key-path: build/unix/build-gcc/AD17A21EF8AED8F1CC02DBD9F7D5C9BF765C61E3.key
artifact-name: mpc-source.tar.zst
strip-components: 1
add-prefix: mpc-source/
mpfr-3.1.4:
description: mpfr 3.1.4 source code
@ -138,6 +174,9 @@ mpfr-3.1.4:
gpg-signature:
sig-url: "{url}.asc"
key-path: build/unix/build-gcc/07F3DBBECC1A39605078094D980C197698C3739D.key
artifact-name: mpfr-source.tar.zst
strip-components: 1
add-prefix: mpfr-source/
mpfr-3.1.5:
description: mpfr 3.1.5 source code
@ -149,6 +188,9 @@ mpfr-3.1.5:
gpg-signature:
sig-url: "{url}.asc"
key-path: build/unix/build-gcc/07F3DBBECC1A39605078094D980C197698C3739D.key
artifact-name: mpfr-source.tar.zst
strip-components: 1
add-prefix: mpfr-source/
nasm-2.14.02:
description: nasm 2.14.02 source code
@ -336,3 +378,45 @@ android-rs-glue:
type: git
repo: https://github.com/staktrace/android-rs-glue
revision: 486491e81819c3346d364a93fc1f3c0206d3ece0
clang-4.0:
description: clang 4.0 source code
fetch:
type: git
repo: https://github.com/llvm/llvm-project
revision: 449c3ef93afc7a668eb35e67a83717453e28b25a
clang-7:
description: clang 7 source code
fetch:
type: git
repo: https://github.com/llvm/llvm-project
revision: d0d8eb2e5415b8be29343e3c17a18e49e67b5551
clang-8:
description: clang 8 source code
fetch:
type: git
repo: https://github.com/llvm/llvm-project
revision: d2298e74235598f15594fe2c99bbac870a507c59
ninja:
description: ninja 1.9.0
fetch:
type: static-url
url: https://github.com/ninja-build/ninja/releases/download/v1.9.0/ninja-win.zip
sha256: 2d70010633ddaacc3af4ffbd21e22fae90d158674a09e132e06424ba3ab036e9
size: 254497
artifact-name: ninja.tar.zst
add-prefix: ninja/bin/
cmake:
description: cmake 3.15.1
fetch:
type: static-url
url: https://github.com/Kitware/CMake/releases/download/v3.15.1/cmake-3.15.1-win64-x64.zip
sha256: 82a0edfed4cb0b45b25d2f99e621d3ed4014f66191d8f3c7eadf1d9ccf9c461b
size: 32214192
artifact-name: cmake.tar.zst
strip-components: 1
add-prefix: cmake/

10
taskcluster/ci/toolchain/android.yml
View File

@ -51,7 +51,6 @@ linux64-android-gradle-dependencies:
script: android-gradle-dependencies.sh
sparse-profile: null
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
- 'taskcluster/scripts/misc/android-gradle-dependencies/**'
- '*.gradle'
- 'mobile/android/**/*.gradle'
@ -60,7 +59,8 @@ linux64-android-gradle-dependencies:
- 'mobile/android/gradle.configure'
toolchain-artifact: public/build/android-gradle-dependencies.tar.xz
toolchain-alias: android-gradle-dependencies
toolchains:
# Aliases aren't allowed for toolchains depending on toolchains.
- linux64-android-sdk-linux-repack
- linux64-node
fetches:
toolchain:
# Aliases aren't allowed for toolchains depending on toolchains.
- linux64-android-sdk-linux-repack
- linux64-node

29
taskcluster/ci/toolchain/cbindgen.yml
View File

@ -9,8 +9,6 @@ job-defaults:
max-run-time: 3600
run:
script: build-cbindgen.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/cbindgen.tar.xz
fetches:
fetch:
@ -25,8 +23,9 @@ linux64-cbindgen:
max-run-time: 1800
run:
arguments: ['x86_64-unknown-linux-gnu']
toolchains:
- linux64-rust-1.32
fetches:
toolchain:
- linux64-rust-1.32
macosx64-cbindgen:
treeherder:
@ -39,12 +38,15 @@ macosx64-cbindgen:
- try
run:
arguments: ['x86_64-apple-darwin']
resources:
- taskcluster/scripts/misc/tooltool-download.sh
tooltool-downloads: internal
toolchains:
- linux64-cctools-port
- linux64-clang-8
- linux64-llvm-dsymutil
- linux64-rust-macos-1.32
fetches:
toolchain:
- linux64-cctools-port
- linux64-clang-8
- linux64-llvm-dsymutil
- linux64-rust-macos-1.32
win64-cbindgen:
treeherder:
@ -52,9 +54,12 @@ win64-cbindgen:
worker-type: b-win2012
worker:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
arguments: ['x86_64-pc-windows-msvc']
resources:
- taskcluster/scripts/misc/tooltool-download.sh
toolchain-artifact: public/build/cbindgen.tar.bz2
toolchains:
- win64-rust-1.34
fetches:
toolchain:
- win64-rust-1.34

10
taskcluster/ci/toolchain/cctools-port.yml
View File

@ -7,9 +7,6 @@ job-defaults:
worker-type: b-linux
worker:
max-run-time: 1800
run:
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
fetches:
fetch:
- cctools-port
@ -21,6 +18,7 @@ linux64-cctools-port:
run:
script: build-cctools-port.sh
toolchain-artifact: public/build/cctools.tar.xz
toolchains:
- linux64-clang-8
- linux64-binutils
fetches:
toolchain:
- linux64-clang-8
- linux64-binutils

26
taskcluster/ci/toolchain/clang-tidy.yml
View File

@ -22,6 +22,9 @@ job-defaults:
run-on-projects:
- trunk
- try
fetches:
fetch:
- clang-8
linux64-clang-tidy:
index:
@ -34,9 +37,9 @@ linux64-clang-tidy:
- 'build/build-clang/clang-tidy-linux64.json'
resources:
- 'build/build-clang/clang-tidy-linux64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchains:
- linux64-gcc-6
fetches:
toolchain:
- linux64-gcc-6
macosx64-clang-tidy:
index:
@ -56,11 +59,12 @@ macosx64-clang-tidy:
resources:
- 'build/build-clang/clang-tidy-macosx64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchains:
- linux64-cctools-port
- linux64-clang-8
- linux64-gcc-6
- linux64-node
fetches:
toolchain:
- linux64-cctools-port
- linux64-clang-8
- linux64-gcc-6
- linux64-node
win64-clang-tidy:
description: "Clang-tidy toolchain build"
@ -73,7 +77,7 @@ win64-clang-tidy:
worker:
max-run-time: 7200
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
script: build-clang.sh
arguments:
@ -81,3 +85,7 @@ win64-clang-tidy:
resources:
- 'build/build-clang/clang-tidy-win64.json'
toolchain-artifact: public/build/clang-tidy.tar.bz2
fetches:
fetch:
- cmake
- ninja

94
taskcluster/ci/toolchain/clang.yml
View File

@ -21,10 +21,12 @@ linux64-clang-4.0:
- 'build/build-clang/clang-4.0-linux64.json'
resources:
- 'build/build-clang/clang-4.0-linux64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-gcc-6
fetches:
fetch:
- clang-4.0
toolchain:
- linux64-gcc-6
linux64-clang-7:
description: "Clang 7 toolchain build"
@ -37,10 +39,12 @@ linux64-clang-7:
- 'build/build-clang/clang-7-linux64.json'
resources:
- 'build/build-clang/clang-7-linux64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-gcc-6
fetches:
fetch:
- clang-7
toolchain:
- linux64-gcc-6
linux64-clang-8:
description: "Clang 8 toolchain build"
@ -53,11 +57,13 @@ linux64-clang-8:
- 'build/build-clang/clang-8-linux64.json'
resources:
- 'build/build-clang/clang-8-linux64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-alias: linux64-clang
toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-gcc-6
fetches:
fetch:
- clang-8
toolchain:
- linux64-gcc-6
linux64-clang-8-mingw-x86:
description: "MinGW-Clang Trunk x86 toolchain build"
@ -70,19 +76,16 @@ linux64-clang-8-mingw-x86:
- 'build/build-clang/clang-8-mingw.json'
resources:
- 'build/build-clang/clang-8-mingw.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-alias: linux64-clang-mingw-x86
toolchain-artifact: public/build/clangmingw.tar.xz
worker:
env:
MOZ_FETCHES_DIR: "/builds/worker/workspace/moz-toolchain"
toolchains:
- linux64-gcc-6
fetches:
fetch:
- clang-8
- mingw-w64
- libunwind
- llvm-mingw
toolchain:
- linux64-gcc-6
linux64-clang-8-mingw-x64:
description: "MinGW-Clang Trunk x64 toolchain build"
@ -96,19 +99,16 @@ linux64-clang-8-mingw-x64:
- 'build/build-clang/clang-8-mingw.json'
resources:
- 'build/build-clang/clang-8-mingw.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-alias: linux64-clang-mingw-x64
toolchain-artifact: public/build/clangmingw.tar.xz
worker:
env:
MOZ_FETCHES_DIR: "/builds/worker/workspace/moz-toolchain"
toolchains:
- linux64-gcc-6
fetches:
fetch:
- clang-8
- mingw-w64
- libunwind
- llvm-mingw
toolchain:
- linux64-gcc-6
linux64-clang-8-android-cross:
description: "Clang 8 toolchain build"
@ -121,12 +121,14 @@ linux64-clang-8-android-cross:
- 'build/build-clang/clang-8-android.json'
resources:
- 'build/build-clang/clang-8-android.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-alias: linux64-clang-android-cross
toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-gcc-6
- linux64-android-ndk-linux-repack
fetches:
fetch:
- clang-8
toolchain:
- linux64-gcc-6
- linux64-android-ndk-linux-repack
linux64-clang-8-aarch64-cross:
description: "Clang 8 toolchain build with aarch64 runtime"
@ -143,12 +145,14 @@ linux64-clang-8-aarch64-cross:
- 'build/build-clang/clang-8-linux64-aarch64-cross.json'
resources:
- 'build/build-clang/clang-8-linux64-aarch64-cross.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-alias: linux64-aarch64-cross
toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-binutils
- linux64-gcc-6
fetches:
fetch:
- clang-8
toolchain:
- linux64-binutils
- linux64-gcc-6
linux64-clang-8-macosx-cross:
description: "Clang 8 toolchain build with MacOS Compiler RT libs"
@ -169,10 +173,13 @@ linux64-clang-8-macosx-cross:
toolchain-alias: linux64-clang-macosx-cross
toolchain-artifact: public/build/clang.tar.xz
tooltool-downloads: internal
toolchains:
- linux64-cctools-port
- linux64-clang-8
- linux64-gcc-6
fetches:
fetch:
- clang-8
toolchain:
- linux64-cctools-port
- linux64-clang-8
- linux64-gcc-6
macosx64-clang:
description: "Clang toolchain build"
@ -195,11 +202,14 @@ macosx64-clang:
- 'build/build-clang/clang-8-macosx64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/clang.tar.xz
toolchains:
- linux64-cctools-port
- linux64-clang-8
- linux64-gcc-6
- linux64-node
fetches:
fetch:
- clang-8
toolchain:
- linux64-cctools-port
- linux64-clang-8
- linux64-gcc-6
- linux64-node
win64-clang-cl:
description: "Clang-cl toolchain build"
@ -208,11 +218,17 @@ win64-clang-cl:
worker-type: b-win2012
worker:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/build-clang-cl.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
script: build-clang.sh
arguments:
- 'build/build-clang/clang-win64.json'
resources:
- 'build/build-clang/clang-win64.json'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/clang.tar.bz2
fetches:
fetch:
- clang-8
- cmake
- ninja

16
taskcluster/ci/toolchain/dist-toolchains.yml
View File

@ -11,8 +11,6 @@ job-defaults:
run:
using: toolchain-script
script: build-dist-toolchains.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
clang-dist-toolchain:
treeherder:
@ -22,9 +20,10 @@ clang-dist-toolchain:
run:
arguments: ['clang']
toolchain-artifact: public/build/clang-dist-toolchain.tar.xz
toolchains:
- linux64-clang-8
- linux64-sccache
fetches:
toolchain:
- linux64-clang-8
- linux64-sccache
rustc-dist-toolchain:
treeherder:
@ -34,6 +33,7 @@ rustc-dist-toolchain:
run:
arguments: ['rustc']
toolchain-artifact: public/build/rustc-dist-toolchain.tar.xz
toolchains:
- linux64-rust-macos-1.36
- linux64-sccache
fetches:
toolchain:
- linux64-rust-macos-1.36
- linux64-sccache

12
taskcluster/ci/toolchain/gcc.yml
View File

@ -16,7 +16,7 @@ linux64-gcc-6:
treeherder:
symbol: TL(gcc6)
run:
script: build-gcc-6-linux.sh
script: build-gcc-linux.sh
toolchain-alias: linux64-gcc
fetches:
fetch:
@ -32,7 +32,7 @@ linux64-gcc-7:
treeherder:
symbol: TL(gcc7)
run:
script: build-gcc-7-linux.sh
script: build-gcc-linux.sh
fetches:
fetch:
- binutils-2.31.1
@ -47,7 +47,7 @@ linux64-gcc-8:
treeherder:
symbol: TL(gcc8)
run:
script: build-gcc-8-linux.sh
script: build-gcc-linux.sh
fetches:
fetch:
- binutils-2.31.1
@ -62,7 +62,7 @@ linux64-gcc-9:
treeherder:
symbol: TL(gcc9)
run:
script: build-gcc-9-linux.sh
script: build-gcc-linux.sh
fetches:
fetch:
- binutils-2.31.1
@ -81,8 +81,6 @@ linux64-gcc-sixgill:
resources:
- 'taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh'
toolchain-artifact: public/build/sixgill.tar.xz
toolchains:
- linux64-gcc-6
fetches:
fetch:
- binutils-2.31.1
@ -91,3 +89,5 @@ linux64-gcc-sixgill:
- gmp-5.1.3
- mpc-0.8.2
- mpfr-3.1.5
toolchain:
- linux64-gcc-6

24
taskcluster/ci/toolchain/gn.yml
View File

@ -11,9 +11,7 @@ job-defaults:
- trunk
- try
run:
tooltool-downloads: public
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
- 'taskcluster/scripts/misc/build-gn-common.sh'
toolchain-artifact: public/build/gn.tar.xz
fetches:
@ -25,8 +23,9 @@ linux64-gn:
symbol: TL(gn)
run:
script: build-gn-linux.sh
toolchains:
- linux64-gcc-6
fetches:
toolchain:
- linux64-gcc-6
macosx64-gn:
treeherder:
@ -36,11 +35,13 @@ macosx64-gn:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
script: build-gn-macosx.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
tooltool-downloads: internal
toolchains:
- linux64-cctools-port
- linux64-clang-8
- linux64-node
fetches:
toolchain:
- linux64-cctools-port
- linux64-clang-8
win32-gn:
treeherder:
@ -49,7 +50,12 @@ win32-gn:
worker:
max-run-time: 3600
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/gn-build.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
script: build-gn-win32.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/gn.tar.bz2
fetches:
fetch:
- ninja

21
taskcluster/ci/toolchain/grcov.yml
View File

@ -9,8 +9,6 @@ job-defaults:
max-run-time: 1800
run:
script: build-grcov.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/grcov.tar.xz
fetches:
fetch:
@ -19,8 +17,9 @@ job-defaults:
linux64-grcov:
treeherder:
symbol: TL(grcov)
toolchains:
- linux64-rust-1.32
fetches:
toolchain:
- linux64-rust-1.32
macosx64-grcov:
treeherder:
@ -28,8 +27,9 @@ macosx64-grcov:
run-on-projects:
- trunk
- try
toolchains:
- linux64-rust-1.32
fetches:
toolchain:
- linux64-rust-1.32
win64-grcov:
treeherder:
@ -37,8 +37,11 @@ win64-grcov:
worker-type: b-win2012
worker:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
toolchain-artifact: public/build/grcov.tar.bz2
toolchains:
- win64-rust-1.34
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
fetches:
toolchain:
- win64-rust-1.34

1
taskcluster/ci/toolchain/kind.yml
View File

@ -9,7 +9,6 @@ kind-dependencies:
transforms:
- taskgraph.transforms.try_job:transforms
- taskgraph.transforms.use_toolchains:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.cached_tasks:transforms
- taskgraph.transforms.task:transforms

8
taskcluster/ci/toolchain/mingw.yml
View File

@ -53,12 +53,12 @@ linux64-mingw32-nsis:
- 'build/unix/build-gcc/build-gcc.sh'
- 'taskcluster/scripts/misc/build-gcc-mingw32.sh'
toolchain-artifact: public/build/nsis.tar.xz
toolchains:
- linux64-mingw32-gcc
fetches:
fetch:
- nsis-3.01
- zlib-1.2.11
toolchain:
- linux64-mingw32-gcc
linux64-mingw-fxc2-x86:
description: "fxc2.exe x86 build for MinGW Cross Compile"
@ -70,8 +70,8 @@ linux64-mingw-fxc2-x86:
using: toolchain-script
script: build-mingw-fxc2-x86.sh
toolchain-artifact: public/build/fxc2.tar.xz
toolchains:
- linux64-clang-8-mingw-x86
fetches:
fetch:
- fxc2
toolchain:
- linux64-clang-8-mingw-x86

28
taskcluster/ci/toolchain/minidump_stackwalk.yml
View File

@ -15,7 +15,6 @@ job-defaults:
- 'config/external/zlib'
- 'mfbt'
- 'moz.configure'
- 'taskcluster/scripts/misc/tooltool-download.sh'
- 'testing/tools/fileid'
- 'toolkit/crashreporter'
- 'toolkit/crashreporter/google-breakpad/src/common'
@ -30,10 +29,11 @@ job-defaults:
linux64-minidump-stackwalk:
treeherder:
symbol: TL(stackwalk)
toolchains:
- linux64-clang-8
- linux64-binutils
- linux64-rust-1.31
fetches:
toolchain:
- linux64-clang-8
- linux64-binutils
- linux64-rust-1.31
macosx64-minidump-stackwalk:
treeherder:
@ -43,11 +43,14 @@ macosx64-minidump-stackwalk:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
arguments: ['macosx64']
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
tooltool-downloads: internal
toolchains:
- linux64-cctools-port
- linux64-clang-8
- linux64-rust-macos-1.31
fetches:
toolchain:
- linux64-cctools-port
- linux64-clang-8
- linux64-rust-macos-1.31
win32-minidump-stackwalk:
treeherder:
@ -56,6 +59,7 @@ win32-minidump-stackwalk:
docker-image: {in-tree: mingw32-build}
run:
arguments: ['mingw32']
toolchains:
- linux64-clang-8-mingw-x86
- mingw32-rust-1.31
fetches:
toolchain:
- linux64-clang-8-mingw-x86
- mingw32-rust-1.31

35
taskcluster/ci/toolchain/misc.yml
View File

@ -31,11 +31,11 @@ linux64-llvm-dsymutil:
run:
script: build-llvm-dsymutil.sh
toolchain-artifact: public/build/llvm-dsymutil.tar.xz
toolchains:
- linux64-gcc-6
fetches:
fetch:
- llvm-for-dsymutil
toolchain:
- linux64-gcc-6
linux64-binutils:
description: "Binutils toolchain build"
@ -60,13 +60,12 @@ linux64-hfsplus:
script: build-hfsplus-linux.sh
resources:
- 'build/unix/build-hfsplus/build-hfsplus.sh'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/hfsplus-tools.tar.xz
toolchains:
- linux64-clang-8
fetches:
fetch:
- hfsplus-tools
toolchain:
- linux64-clang-8
linux64-libdmg:
description: "libdmg-hfsplus toolchain build"
@ -83,9 +82,10 @@ linux64-mar-tools:
description: "mar-tools toolchain build"
treeherder:
symbol: TL(mar-tools)
toolchains:
- linux64-clang-7
- linux64-binutils
fetches:
toolchain:
- linux64-clang-7
- linux64-binutils
run:
script: build-mar-tools.sh
sparse-profile: null
@ -108,17 +108,15 @@ linux64-tup:
max-run-time: 3600
run:
script: build-tup-linux.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/tup.tar.xz
run-on-projects:
- trunk
- try
toolchains:
- linux64-gcc-6
fetches:
fetch:
- tup
toolchain:
- linux64-gcc-6
linux64-upx:
description: "UPX build"
@ -149,9 +147,6 @@ browsertime:
description: "npm install browsertime node_modules"
treeherder:
symbol: TL(browsertime)
worker:
env:
NODEJS: "/builds/worker/workspace/build/src/node/bin/node"
run:
script: browsertime.sh
sparse-profile: null
@ -159,8 +154,9 @@ browsertime:
- 'tools/browsertime/package.json'
- 'tools/browsertime/mach_commands.py'
toolchain-artifact: public/build/browsertime.zip
toolchains:
- linux64-node
fetches:
toolchain:
- linux64-node
wrench-deps:
description: "Downloads all the crates needed for building wrench"
@ -173,10 +169,9 @@ wrench-deps:
sparse-profile: null # need all of gfx/wr checked out for this script
resources:
- 'gfx/wr/Cargo.lock'
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/wrench-deps.tar.bz2
toolchains:
- linux64-rust-1.36 # whatever m-c is built with
fetches:
fetch:
- android-rs-glue
toolchain:
- linux64-rust-1.36 # whatever m-c is built with

6
taskcluster/ci/toolchain/nasm.yml
View File

@ -7,8 +7,6 @@ job-defaults:
worker:
max-run-time: 1800
run:
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
toolchain-artifact: public/build/nasm.tar.bz2
win64-nasm:
@ -20,11 +18,11 @@ win64-nasm:
run:
script: build-nasm.sh
arguments: ['win64']
toolchains:
- linux64-clang-8-mingw-x64
fetches:
fetch:
- nasm-2.14.02
toolchain:
- linux64-clang-8-mingw-x64
linux64-nasm:
description: "nasm linux64 build"

16
taskcluster/ci/toolchain/rust-size.yml
View File

@ -8,8 +8,6 @@ job-defaults:
max-run-time: 1800
run:
script: build-rust-size.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
fetches:
fetch:
- rust-size
@ -20,8 +18,9 @@ linux64-rust-size:
worker-type: b-linux
run:
toolchain-artifact: public/build/rust-size.tar.xz
toolchains:
- linux64-rust-1.28
fetches:
toolchain:
- linux64-rust-1.28
win64-rust-size:
treeherder:
@ -29,8 +28,11 @@ win64-rust-size:
worker-type: b-win2012
worker:
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
toolchain-artifact: public/build/rust-size.tar.bz2
toolchains:
- win64-rust-1.28
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
fetches:
toolchain:
- win64-rust-1.28

32
taskcluster/ci/toolchain/sccache.yml
View File

@ -9,8 +9,6 @@ job-defaults:
run:
using: toolchain-script
script: build-sccache.sh
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
fetches:
fetch:
- sccache
@ -23,12 +21,12 @@ linux64-sccache:
max-run-time: 1800
run:
toolchain-artifact: public/build/sccache.tar.xz
toolchains:
- linux64-rust-1.34
- linux64-binutils
fetches:
fetch:
- openssl-1.1.0g
toolchain:
- linux64-rust-1.34
- linux64-binutils
macosx64-sccache:
treeherder:
@ -43,14 +41,17 @@ macosx64-sccache:
- try
run:
arguments: ['x86_64-apple-darwin']
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
tooltool-downloads: internal
toolchain-artifact: public/build/sccache.tar.xz
toolchains:
- linux64-rust-macos-1.34
- linux64-clang-8
- linux64-cctools-port
- linux64-llvm-dsymutil
- linux64-binutils
fetches:
toolchain:
- linux64-rust-macos-1.34
- linux64-clang-8
- linux64-cctools-port
- linux64-llvm-dsymutil
- linux64-binutils
win64-sccache:
treeherder:
@ -59,8 +60,11 @@ win64-sccache:
worker:
max-run-time: 3600
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/sccache-build.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
run:
toolchain-artifact: public/build/sccache.tar.bz2
toolchains:
- win64-rust-1.34
resources:
- 'taskcluster/scripts/misc/tooltool-download.sh'
fetches:
toolchain:
- win64-rust-1.34

4
taskcluster/ci/webrender/kind.yml
View File

@ -97,7 +97,7 @@ jobs:
worker:
docker-image: {in-tree: webrender}
env:
TOOLTOOL_MANIFEST: "/builds/worker/checkouts/gecko/browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
chain-of-trust: true
artifacts:
- type: file
@ -160,7 +160,7 @@ jobs:
worker:
docker-image: {in-tree: webrender}
env:
TOOLTOOL_MANIFEST: "/builds/worker/checkouts/gecko/browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
chain-of-trust: true
artifacts:
- type: file

3
taskcluster/scripts/misc/android-gradle-dependencies.sh
View File

@ -8,9 +8,6 @@ set -v
cd $GECKO_PATH
# Download toolchain artifacts.
. taskcluster/scripts/misc/tooltool-download.sh
. taskcluster/scripts/misc/android-gradle-dependencies/before.sh
export MOZCONFIG=mobile/android/config/mozconfigs/android-api-16-gradle-dependencies/nightly

6
taskcluster/scripts/misc/browsertime.sh
View File

@ -8,11 +8,7 @@ set -v
cd $GECKO_PATH
# Download toolchain artifacts.
. taskcluster/scripts/misc/tooltool-download.sh
# We can't set the path to npm directly, but it's sibling to NODEJS.
export PATH=$PATH:`dirname $NODEJS`
export PATH=$PATH:$MOZ_FETCHES_DIR/node/bin
# We don't install ImageMagick, so this will fail. Continue.
./mach browsertime --setup || true

18
taskcluster/scripts/misc/build-cbindgen.sh
View File

@ -19,26 +19,28 @@ esac
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
if [ -n "$TOOLTOOL_MANIFEST" ]; then
. taskcluster/scripts/misc/tooltool-download.sh
fi
# OSX cross builds are a bit harder
if [ "$TARGET" == "x86_64-apple-darwin" ]; then
export PATH="$GECKO_PATH/llvm-dsymutil/bin:$PATH"
export PATH="$GECKO_PATH/cctools/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/llvm-dsymutil/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/cctools/bin:$PATH"
cat >cross-linker <<EOF
exec $GECKO_PATH/clang/bin/clang -v \
-fuse-ld=$GECKO_PATH/cctools/bin/x86_64-apple-darwin-ld \
exec $MOZ_FETCHES_DIR/clang/bin/clang -v \
-fuse-ld=$MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin-ld \
-mmacosx-version-min=10.11 \
-target $TARGET \
-B $GECKO_PATH/cctools/bin \
-isysroot $GECKO_PATH/MacOSX10.11.sdk \
-B $MOZ_FETCHES_DIR/cctools/bin \
-isysroot $MOZ_FETCHES_DIR/MacOSX10.11.sdk \
"\$@"
EOF
chmod +x cross-linker
export RUSTFLAGS="-C linker=$PWD/cross-linker"
fi
export PATH="$(cd $GECKO_PATH && pwd)/rustc/bin:$PATH"
export PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rustc/bin:$PATH"
cd $MOZ_FETCHES_DIR/cbindgen

6
taskcluster/scripts/misc/build-cctools-port.sh
View File

@ -16,16 +16,14 @@ CROSSTOOLS_CCTOOLS_DIR=$CROSSTOOLS_SOURCE_DIR/cctools
CROSSTOOLS_BUILD_DIR=$WORKSPACE/cctools
LIBTAPI_SOURCE_DIR=$MOZ_FETCHES_DIR/apple-libtapi
LIBTAPI_BUILD_DIR=$WORKSPACE/libtapi-build
CLANG_DIR=$GECKO_PATH/clang
CLANG_DIR=$MOZ_FETCHES_DIR/clang
# Create our directories
mkdir -p $CROSSTOOLS_BUILD_DIR $LIBTAPI_BUILD_DIR
# Fetch clang from tooltool
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
export PATH="$GECKO_PATH/binutils/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH"
# Common setup for libtapi and cctools
export CC=$CLANG_DIR/bin/clang

16
taskcluster/scripts/misc/build-clang-8-linux-macosx-cross.sh
View File

@ -3,31 +3,31 @@ set -x -e -v
# This script is for building clang for Mac OS X targets on a Linux host,
# including native Mac OS X Compiler-RT libraries and llvm-symbolizer.
WORKSPACE=$HOME/workspace
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
# these variables are used in build-clang.py
export CROSS_CCTOOLS_PATH=$GECKO_PATH/cctools
export CROSS_SYSROOT=$GECKO_PATH/MacOSX10.11.sdk
export CROSS_CCTOOLS_PATH=$MOZ_FETCHES_DIR/cctools
export CROSS_SYSROOT=$MOZ_FETCHES_DIR/MacOSX10.11.sdk
export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
# gets a bit too verbose here
set +x
python3 build/build-clang/build-clang.py -c $1 --skip-tar
cd $MOZ_FETCHES_DIR/llvm-project
python3 $GECKO_PATH/build/build-clang/build-clang.py -c $GECKO_PATH/$1 --skip-tar
# We now have a native macosx64 toolchain.
# What we want is a native linux64 toolchain which can target macosx64 and use the sanitizer dylibs.
# Overlay the linux64 toolchain that we used for this build (except llvm-symbolizer).
(
cd "$WORKSPACE/moz-toolchain/build/stage1"
cd build/stage1
# Need the macosx64 native llvm-symbolizer since this gets shipped with sanitizer builds
mv clang/bin/llvm-symbolizer $GECKO_PATH/clang/bin/
cp --remove-destination -lr $GECKO_PATH/clang/* clang/
tar -c -J -f $GECKO_PATH/clang.tar.xz clang
mv clang/bin/llvm-symbolizer $MOZ_FETCHES_DIR/clang/bin/
cp --remove-destination -lr $MOZ_FETCHES_DIR/clang/* clang/
tar -c -J -f $MOZ_FETCHES_DIR/llvm-project/clang.tar.xz clang
)
set -x

24
taskcluster/scripts/misc/build-clang-8-mingw.sh
View File

@ -21,12 +21,9 @@ else
exit 1;
fi
WORKSPACE=$HOME/workspace
TOOLCHAIN_DIR=$WORKSPACE/moz-toolchain
TOOLCHAIN_DIR=$MOZ_FETCHES_DIR/llvm-project
INSTALL_DIR=$TOOLCHAIN_DIR/build/stage3/clang
CROSS_PREFIX_DIR=$INSTALL_DIR/$machine-w64-mingw32
SRC_DIR=$TOOLCHAIN_DIR/src
make_flags="-j$(nproc)"
@ -36,13 +33,9 @@ default_win32_winnt=0x601
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
patch_file="$(pwd)/taskcluster/scripts/misc/mingw-winrt.patch"
prepare() {
mkdir -p $TOOLCHAIN_DIR
touch $TOOLCHAIN_DIR/.build-clang
pushd $MOZ_FETCHES_DIR/mingw-w64
patch -p1 <$patch_file
popd
@ -122,7 +115,7 @@ build_compiler_rt() {
-DCMAKE_C_COMPILER_WORKS=1 \
-DCMAKE_C_COMPILER_TARGET=$compiler_rt_machine-windows-gnu \
-DCOMPILER_RT_DEFAULT_TARGET_ONLY=TRUE \
$SRC_DIR/compiler-rt/lib/builtins
$TOOLCHAIN_DIR/compiler-rt/lib/builtins
make $make_flags
mkdir -p $INSTALL_DIR/lib/clang/$CLANG_VERSION/lib/windows
cp lib/windows/libclang_rt.builtins-$compiler_rt_machine.a $INSTALL_DIR/lib/clang/$CLANG_VERSION/lib/windows/
@ -170,7 +163,7 @@ build_libcxx() {
-DLIBUNWIND_ENABLE_THREADS=TRUE \
-DLIBUNWIND_ENABLE_SHARED=FALSE \
-DLIBUNWIND_ENABLE_CROSS_UNWINDING=FALSE \
-DCMAKE_CXX_FLAGS="${DEBUG_FLAGS} -Wno-dll-attribute-on-redeclaration -nostdinc++ -I$SRC_DIR/libcxx/include -DPSAPI_VERSION=2" \
-DCMAKE_CXX_FLAGS="${DEBUG_FLAGS} -Wno-dll-attribute-on-redeclaration -nostdinc++ -I$TOOLCHAIN_DIR/libcxx/include -DPSAPI_VERSION=2" \
-DCMAKE_C_FLAGS="-Wno-dll-attribute-on-redeclaration" \
$MOZ_FETCHES_DIR/libunwind
make $make_flags
@ -197,12 +190,12 @@ build_libcxx() {
-DLIBCXXABI_ENABLE_THREADS=ON \
-DLIBCXXABI_TARGET_TRIPLE=$machine-w64-mingw32 \
-DLIBCXXABI_ENABLE_SHARED=OFF \
-DLIBCXXABI_LIBCXX_INCLUDES=$SRC_DIR/libcxx/include \
-DLIBCXXABI_LIBCXX_INCLUDES=$TOOLCHAIN_DIR/libcxx/include \
-DLLVM_NO_OLD_LIBSTDCXX=TRUE \
-DCXX_SUPPORTS_CXX11=TRUE \
-DCXX_SUPPORTS_CXX_STD=TRUE \
-DCMAKE_CXX_FLAGS="${DEBUG_FLAGS} -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS -D_LIBCPP_HAS_THREAD_API_WIN32" \
$SRC_DIR/libcxxabi
$TOOLCHAIN_DIR/libcxxabi
make $make_flags VERBOSE=1
popd
@ -233,10 +226,10 @@ build_libcxx() {
-DLIBCXX_ENABLE_FILESYSTEM=OFF \
-DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=TRUE \
-DLIBCXX_CXX_ABI=libcxxabi \
-DLIBCXX_CXX_ABI_INCLUDE_PATHS=$SRC_DIR/libcxxabi/include \
-DLIBCXX_CXX_ABI_INCLUDE_PATHS=$TOOLCHAIN_DIR/libcxxabi/include \
-DLIBCXX_CXX_ABI_LIBRARY_PATH=../libcxxabi/lib \
-DCMAKE_CXX_FLAGS="${DEBUG_FLAGS} -D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS" \
$SRC_DIR/libcxx
$TOOLCHAIN_DIR/libcxx
make $make_flags VERBOSE=1
make $make_flags install
@ -265,7 +258,8 @@ prepare
# gets a bit too verbose here
set +x
python3 build/build-clang/build-clang.py -c $2 --skip-tar
cd $TOOLCHAIN_DIR
python3 $GECKO_PATH/build/build-clang/build-clang.py -c $GECKO_PATH/$2 --skip-tar
set -x

21
taskcluster/scripts/misc/build-clang.sh
View File

@ -8,17 +8,19 @@ JSON_CONFIG="$1"
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
if [ -n "$TOOLTOOL_MANIFEST" ]; then
. taskcluster/scripts/misc/tooltool-download.sh
fi
if [ -d "$GECKO_PATH/binutils/bin" ]; then
export PATH="$GECKO_PATH/binutils/bin:$PATH"
if [ -d "$MOZ_FETCHES_DIR/binutils/bin" ]; then
export PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH"
fi
case "$JSON_CONFIG" in
*macosx64*)
# these variables are used in build-clang.py
export CROSS_CCTOOLS_PATH=$GECKO_PATH/cctools
export CROSS_SYSROOT=$GECKO_PATH/MacOSX10.11.sdk
export CROSS_CCTOOLS_PATH=$MOZ_FETCHES_DIR/cctools
export CROSS_SYSROOT=$MOZ_FETCHES_DIR/MacOSX10.11.sdk
export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
;;
*win64*)
@ -30,10 +32,8 @@ case "$JSON_CONFIG" in
# exists.
export VSINSTALLDIR="${VSPATH}/"
# Add git.exe to the path
export PATH="$(pwd)/cmd:${PATH}"
export PATH="$(cd cmake && pwd)/bin:${PATH}"
export PATH="$(cd ninja && pwd)/bin:${PATH}"
export PATH="$(cd $MOZ_FETCHES_DIR/cmake && pwd)/bin:${PATH}"
export PATH="$(cd $MOZ_FETCHES_DIR/ninja && pwd)/bin:${PATH}"
;;
*linux64*|*android*)
;;
@ -46,7 +46,8 @@ esac
# gets a bit too verbose here
set +x
python3 build/build-clang/build-clang.py -c $1
cd $MOZ_FETCHES_DIR/llvm-project
python3 $GECKO_PATH/build/build-clang/build-clang.py -c $GECKO_PATH/$1
set -x

6
taskcluster/scripts/misc/build-dist-toolchains.sh
View File

@ -4,8 +4,6 @@ set -x -e -v
# This script is for packaging toolchains suitable for use by distributed sccache.
TL_NAME="$1"
cd $GECKO_PATH
mkdir -p $HOME/artifacts
. taskcluster/scripts/misc/tooltool-download.sh
sccache/sccache --package-toolchain $GECKO_PATH/$TL_NAME/bin/$TL_NAME $HOME/artifacts/$TL_NAME-dist-toolchain.tar.xz
$MOZ_FETCHES_DIR/sccache/sccache --package-toolchain $MOZ_FETCHES_DIR/$TL_NAME/bin/$TL_NAME $HOME/artifacts/$TL_NAME-dist-toolchain.tar.xz

28
taskcluster/scripts/misc/build-gcc-7-linux.sh
View File

@ -1,28 +0,0 @@
#!/bin/bash
set -e
# This script is for building GCC 7 for Linux.
root_dir=$MOZ_FETCHES_DIR
data_dir=$GECKO_PATH/build/unix/build-gcc
. $data_dir/build-gcc.sh
gcc_version=7.4.0
gcc_ext=xz
binutils_version=2.31.1
binutils_ext=xz
pushd $root_dir/gcc-$gcc_version
ln -sf ../gmp-6.1.0 gmp
ln -sf ../isl-0.16.1 isl
ln -sf ../mpc-1.0.3 mpc
ln -sf ../mpfr-3.1.4 mpfr
popd
build_binutils
build_gcc
# Put a tarball in the artifacts dir
mkdir -p $UPLOAD_DIR
cp $MOZ_FETCHES_DIR/gcc.tar.* $UPLOAD_DIR

28
taskcluster/scripts/misc/build-gcc-8-linux.sh
View File

@ -1,28 +0,0 @@
#!/bin/bash
set -e
# This script is for building GCC 7 for Linux.
root_dir=$MOZ_FETCHES_DIR
data_dir=$GECKO_PATH/build/unix/build-gcc
. $data_dir/build-gcc.sh
gcc_version=8.3.0
gcc_ext=xz
binutils_version=2.31.1
binutils_ext=xz
pushd $root_dir/gcc-$gcc_version
ln -sf ../gmp-6.1.0 gmp
ln -sf ../isl-0.16.1 isl
ln -sf ../mpc-1.0.3 mpc
ln -sf ../mpfr-3.1.4 mpfr
popd
build_binutils
build_gcc
# Put a tarball in the artifacts dir
mkdir -p $UPLOAD_DIR
cp $MOZ_FETCHES_DIR/gcc.tar.* $UPLOAD_DIR

28
taskcluster/scripts/misc/build-gcc-9-linux.sh
View File

@ -1,28 +0,0 @@
#!/bin/bash
set -e
# This script is for building GCC 7 for Linux.
root_dir=$MOZ_FETCHES_DIR
data_dir=$GECKO_PATH/build/unix/build-gcc
. $data_dir/build-gcc.sh
gcc_version=9.1.0
gcc_ext=xz
binutils_version=2.31.1
binutils_ext=xz
pushd $root_dir/gcc-$gcc_version
ln -sf ../gmp-6.1.0 gmp
ln -sf ../isl-0.16.1 isl
ln -sf ../mpc-1.0.3 mpc
ln -sf ../mpfr-3.1.4 mpfr
popd
build_binutils
build_gcc
# Put a tarball in the artifacts dir
mkdir -p $UPLOAD_DIR
cp $MOZ_FETCHES_DIR/gcc.tar.* $UPLOAD_DIR

17
taskcluster/scripts/misc/build-gcc-6-linux.sh → taskcluster/scripts/misc/build-gcc-linux.sh
View File

@ -1,23 +1,18 @@
#!/bin/bash
set -e
# This script is for building GCC 6 for Linux.
# This script is for building GCC for Linux.
root_dir=$MOZ_FETCHES_DIR
data_dir=$GECKO_PATH/build/unix/build-gcc
. $data_dir/build-gcc.sh
gcc_version=6.4.0
gcc_ext=xz
binutils_version=2.31.1
binutils_ext=xz
pushd $root_dir/gcc-$gcc_version
ln -sf ../gmp-5.1.3 gmp
ln -sf ../isl-0.15 isl
ln -sf ../mpc-0.8.2 mpc
ln -sf ../mpfr-3.1.5 mpfr
pushd $root_dir/gcc-source
ln -sf ../gmp-source gmp
ln -sf ../isl-source isl
ln -sf ../mpc-source mpc
ln -sf ../mpfr-source mpfr
popd
build_binutils

14
taskcluster/scripts/misc/build-gcc-mingw32.sh
View File

@ -8,18 +8,14 @@ data_dir=$GECKO_PATH/build/unix/build-gcc
. $data_dir/build-gcc.sh
gcc_version=6.4.0
gcc_ext=xz
binutils_version=2.27
binutils_ext=bz2
binutils_configure_flags="--target=i686-w64-mingw32"
mingw_version=bcf1f29d6dc80b6025b416bef104d2314fa9be57
pushd $root_dir/gcc-$gcc_version
ln -sf ../gmp-5.1.3 gmp
ln -sf ../isl-0.15 isl
ln -sf ../mpc-0.8.2 mpc
ln -sf ../mpfr-3.1.5 mpfr
pushd $root_dir/gcc-source
ln -sf ../gmp-source gmp
ln -sf ../isl-source isl
ln -sf ../mpc-source mpc
ln -sf ../mpfr-source mpfr
popd
prepare_mingw

25
taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh
View File

@ -11,28 +11,21 @@ root_dir=$MOZ_FETCHES_DIR
build_dir=$GECKO_PATH/build
data_dir=$GECKO_PATH/build/unix/build-gcc
# Download and unpack upstream toolchain artifacts (ie, the gcc binary).
. $(dirname $0)/tooltool-download.sh
gcc_version=6.4.0
gcc_ext=xz
binutils_version=2.28.1
binutils_ext=xz
sixgill_rev=bc0ef9258470
sixgill_repo=https://hg.mozilla.org/users/sfink_mozilla.com/sixgill
. $data_dir/build-gcc.sh
pushd $root_dir/gcc-$gcc_version
ln -sf ../binutils-2.31.1 binutils
ln -sf ../gmp-5.1.3 gmp
ln -sf ../isl-0.15 isl
ln -sf ../mpc-0.8.2 mpc
ln -sf ../mpfr-3.1.5 mpfr
pushd $root_dir/gcc-source
ln -sf ../binutils-source binutils
ln -sf ../gmp-source gmp
ln -sf ../isl-source isl
ln -sf ../mpc-source mpc
ln -sf ../mpfr-source mpfr
popd
export TMPDIR=${TMPDIR:-/tmp/}
export gcc_bindir=$root_dir/src/gcc/bin
export gcc_bindir=$MOZ_FETCHES_DIR/gcc/bin
export gmp_prefix=/tools/gmp
export gmp_dir=$root_dir$gmp_prefix
@ -51,12 +44,12 @@ build_gmp() {
# cc1/cc1plus. So link the plugin statically to libgmp. Except that the
# default static build does not have -fPIC, and will result in a relocation
# error, so build our own. This requires the gcc and related source to be
# in $root_dir/gcc-$gcc_version.
# in $root_dir/gcc-source.
mkdir $root_dir/gmp-objdir || true
(
cd $root_dir/gmp-objdir
$root_dir/gcc-$gcc_version/gmp/configure --disable-shared --with-pic --prefix=$gmp_prefix
$root_dir/gcc-source/gmp/configure --disable-shared --with-pic --prefix=$gmp_prefix
make -j8
make install DESTDIR=$root_dir
)

5
taskcluster/scripts/misc/build-gn-linux.sh
View File

@ -5,8 +5,8 @@ set -e -v
WORKSPACE=$HOME/workspace
COMPRESS_EXT=xz
export CC=$GECKO_PATH/gcc/bin/gcc
export CXX=$GECKO_PATH/gcc/bin/g++
export CC=$MOZ_FETCHES_DIR/gcc/bin/gcc
export CXX=$MOZ_FETCHES_DIR/gcc/bin/g++
export LDFLAGS=-lrt
# Gn build scripts use #!/usr/bin/env python, which will be python 2.6 on
@ -18,5 +18,4 @@ export PATH=$WORKSPACE/python_bin:$PATH
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
. taskcluster/scripts/misc/build-gn-common.sh

10
taskcluster/scripts/misc/build-gn-macosx.sh
View File

@ -6,12 +6,12 @@ set -e -v
WORKSPACE=$HOME/workspace
COMPRESS_EXT=xz
CROSS_CCTOOLS_PATH=$GECKO_PATH/cctools
CROSS_SYSROOT=$GECKO_PATH/MacOSX10.11.sdk
CROSS_CCTOOLS_PATH=$MOZ_FETCHES_DIR/cctools
CROSS_SYSROOT=$MOZ_FETCHES_DIR/MacOSX10.11.sdk
export CC=$GECKO_PATH/clang/bin/clang
export CXX=$GECKO_PATH/clang/bin/clang++
export AR=$GECKO_PATH/clang/bin/llvm-ar
export CC=$MOZ_FETCHES_DIR/clang/bin/clang
export CXX=$MOZ_FETCHES_DIR/clang/bin/clang++
export AR=$MOZ_FETCHES_DIR/clang/bin/llvm-ar
export CFLAGS="-target x86_64-apple-darwin -mlinker-version=137 -B ${CROSS_CCTOOLS_PATH}/bin -isysroot ${CROSS_SYSROOT} -I${CROSS_SYSROOT}/usr/include -iframework ${CROSS_SYSROOT}/System/Library/Frameworks"
export CXXFLAGS="-stdlib=libc++ ${CFLAGS}"
export LDFLAGS="${CXXFLAGS} -Wl,-syslibroot,${CROSS_SYSROOT} -Wl,-dead_strip"

4
taskcluster/scripts/misc/build-gn-win32.sh
View File

@ -8,8 +8,8 @@ COMPRESS_EXT=bz2
cd $GECKO_PATH
export PATH="$GECKO_PATH/ninja/bin:$PATH"
export PATH="$GECKO_PATH/mingw64/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/ninja/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/mingw64/bin:$PATH"
. taskcluster/scripts/misc/vs-setup.sh
. taskcluster/scripts/misc/tooltool-download.sh

6
taskcluster/scripts/misc/build-grcov.sh
View File

@ -20,14 +20,16 @@ esac
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
if [ -n "$TOOLTOOL_MANIFEST" ]; then
. taskcluster/scripts/misc/tooltool-download.sh
fi
# cargo gets mad if the parent directory has a Cargo.toml file in it
if [ -e Cargo.toml ]; then
mv Cargo.toml Cargo.toml.back
fi
PATH="$(cd $GECKO_PATH && pwd)/rustc/bin:$PATH"
PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rustc/bin:$PATH"
pushd $MOZ_FETCHES_DIR/$PROJECT

4
taskcluster/scripts/misc/build-hfsplus-linux.sh
View File

@ -5,9 +5,7 @@ set -x -e -v
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
export PATH=$PATH:$GECKO_PATH/clang/bin
export PATH=$PATH:$MOZ_FETCHES_DIR/clang/bin
build/unix/build-hfsplus/build-hfsplus.sh $MOZ_FETCHES_DIR

6
taskcluster/scripts/misc/build-llvm-dsymutil.sh
View File

@ -5,8 +5,6 @@ set -x -e -v
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
cd $MOZ_FETCHES_DIR/llvm-project/llvm
mkdir build
@ -16,10 +14,10 @@ cmake \
-GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_TARGETS_TO_BUILD=X86 \
-DCMAKE_C_COMPILER=$GECKO_PATH/gcc/bin/gcc \
-DCMAKE_C_COMPILER=$MOZ_FETCHES_DIR/gcc/bin/gcc \
..
export LD_LIBRARY_PATH=$GECKO_PATH/gcc/lib64
export LD_LIBRARY_PATH=$MOZ_FETCHES_DIR/gcc/lib64
ninja dsymutil llvm-symbolizer

4
taskcluster/scripts/misc/build-mar-tools.sh
View File

@ -7,8 +7,6 @@ COMPRESS_EXT=xz
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
export MOZ_OBJDIR=obj-mar
echo ac_add_options --enable-project=tools/update-packaging > .mozconfig
@ -16,7 +14,7 @@ echo ac_add_options --enable-project=tools/update-packaging > .mozconfig
TOOLCHAINS="binutils clang"
for t in $TOOLCHAINS; do
PATH="$GECKO_PATH/$t/bin:$PATH"
PATH="$MOZ_FETCHES_DIR/$t/bin:$PATH"
done
./mach build -v

4
taskcluster/scripts/misc/build-mingw-fxc2-x86.sh
View File

@ -6,9 +6,7 @@ INSTALL_DIR=$WORKSPACE/fxc2
mkdir -p $INSTALL_DIR/bin
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
export PATH="$GECKO_PATH/clang/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/clang/bin:$PATH"
# --------------

5
taskcluster/scripts/misc/build-mingw32-nsis.sh
View File

@ -15,16 +15,13 @@ INSTALL_DIR=$GECKO_PATH/mingw32
mkdir -p $INSTALL_DIR
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
# After tooltool runs, we move the stuff we just downloaded.
# As explained above, we have to build nsis to the directory it
# will eventually be run from, which is the same place we just
# installed our compiler. But at the end of the script we want
# to package up what we just built. If we don't move the compiler,
# we will package up the compiler we downloaded along with the
# stuff we just built.
mv mingw32 mingw32-gcc
mv $MOZ_FETCHES_DIR/mingw32 $GECKO_PATH/mingw32-gcc
export PATH="$GECKO_PATH/mingw32-gcc/bin:$PATH"
cd $MOZ_FETCHES_DIR

8
taskcluster/scripts/misc/build-minidump-stackwalk.sh
View File

@ -7,7 +7,9 @@ COMPRESS_EXT=xz
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
if [ -n "$TOOLTOOL_MANIFEST" ]; then
. taskcluster/scripts/misc/tooltool-download.sh
fi
export MOZ_OBJDIR=obj-minidump
@ -19,7 +21,7 @@ case "$1" in
macosx64)
TOOLCHAINS="cctools rustc clang"
echo ac_add_options --target=x86_64-apple-darwin >> .mozconfig
echo ac_add_options --with-macos-sdk=$GECKO_PATH/MacOSX10.11.sdk >> .mozconfig
echo ac_add_options --with-macos-sdk=$MOZ_FETCHES_DIR/MacOSX10.11.sdk >> .mozconfig
;;
mingw32)
TOOLCHAINS="binutils rustc clang"
@ -34,7 +36,7 @@ mingw32)
esac
for t in $TOOLCHAINS; do
PATH="$GECKO_PATH/$t/bin:$PATH"
PATH="$MOZ_FETCHES_DIR/$t/bin:$PATH"
done
./mach build -v

6
taskcluster/scripts/misc/build-nasm.sh
View File

@ -3,14 +3,10 @@ set -x -e -v
COMPRESS_EXT=bz2
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
cd $MOZ_FETCHES_DIR/nasm-*
case "$1" in
win64)
export PATH="$GECKO_PATH/clang/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/clang/bin:$PATH"
./configure CC=x86_64-w64-mingw32-clang AR=llvm-ar RANLIB=llvm-ranlib --host=x86_64-w64-mingw32
EXE=.exe
;;

6
taskcluster/scripts/misc/build-rust-size.sh
View File

@ -18,14 +18,16 @@ esac
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
if [ -n "$TOOLTOOL_MANIFEST" ]; then
. taskcluster/scripts/misc/tooltool-download.sh
fi
# cargo gets mad if the parent directory has a Cargo.toml file in it
if [ -e Cargo.toml ]; then
mv Cargo.toml Cargo.toml.back
fi
PATH="$(cd $GECKO_PATH && pwd)/rustc/bin:$PATH"
PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rustc/bin:$PATH"
cd $MOZ_FETCHES_DIR/$PROJECT

28
taskcluster/scripts/misc/build-sccache.sh
View File

@ -8,7 +8,7 @@ TARGET="$1"
case "$(uname -s)" in
Linux)
COMPRESS_EXT=xz
PATH="$GECKO_PATH/binutils/bin:$PATH"
PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH"
;;
MINGW*)
UPLOAD_DIR=$PWD/public/build
@ -20,29 +20,31 @@ esac
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
if [ -n "$TOOLTOOL_MANIFEST" ]; then
. taskcluster/scripts/misc/tooltool-download.sh
fi
PATH="$(cd $GECKO_PATH && pwd)/rustc/bin:$PATH"
PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rustc/bin:$PATH"
cd $MOZ_FETCHES_DIR/sccache
case "$(uname -s)" in
Linux)
if [ "$TARGET" == "x86_64-apple-darwin" ]; then
export PATH="$GECKO_PATH/llvm-dsymutil/bin:$PATH"
export PATH="$GECKO_PATH/cctools/bin:$PATH"
cat >$GECKO_PATH/cross-linker <<EOF
exec $GECKO_PATH/clang/bin/clang -v \
-fuse-ld=$GECKO_PATH/cctools/bin/x86_64-apple-darwin-ld \
export PATH="$MOZ_FETCHES_DIR/llvm-dsymutil/bin:$PATH"
export PATH="$MOZ_FETCHES_DIR/cctools/bin:$PATH"
cat >cross-linker <<EOF
exec $MOZ_FETCHES_DIR/clang/bin/clang -v \
-fuse-ld=$MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin-ld \
-mmacosx-version-min=10.11 \
-target $TARGET \
-B $GECKO_PATH/cctools/bin \
-isysroot $GECKO_PATH/MacOSX10.11.sdk \
-B $MOZ_FETCHES_DIR/cctools/bin \
-isysroot $MOZ_FETCHES_DIR/MacOSX10.11.sdk \
"\$@"
EOF
chmod +x $GECKO_PATH/cross-linker
export RUSTFLAGS="-C linker=$GECKO_PATH/cross-linker"
export CC="$GECKO_PATH/clang/bin/clang"
chmod +x cross-linker
export RUSTFLAGS="-C linker=$PWD/cross-linker"
export CC="$MOZ_FETCHES_DIR/clang/bin/clang"
cargo build --features "all" --verbose --release --target $TARGET
else
# We can't use the system openssl; see the sad story in

6
taskcluster/scripts/misc/build-tup-linux.sh
View File

@ -4,11 +4,7 @@ set -e -v
# This script is for building tup on Linux.
COMPRESS_EXT=xz
export PATH=$GECKO_PATH/gcc/bin:$PATH
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
export PATH=$MOZ_FETCHES_DIR/gcc/bin:$PATH
cd $MOZ_FETCHES_DIR/tup

254
taskcluster/scripts/misc/fetch-content
View File

@ -6,6 +6,8 @@
import argparse
import bz2
import concurrent.futures
import contextlib
import datetime
import gzip
import hashlib
import json
@ -14,11 +16,15 @@ import multiprocessing
import os
import pathlib
import random
import stat
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.parse
import urllib.request
import zipfile
try:
import zstandard
@ -38,6 +44,40 @@ class IntegrityError(Exception):
"""Represents an integrity error when downloading a URL."""
def ZstdCompressor(*args, **kwargs):
if not zstandard:
raise ValueError('zstandard Python package not available')
return zstandard.ZstdCompressor(*args, **kwargs)
def ZstdDecompressor(*args, **kwargs):
if not zstandard:
raise ValueError('zstandard Python package not available')
return zstandard.ZstdDecompressor(*args, **kwargs)
@contextlib.contextmanager
def rename_after_close(fname, *args, **kwargs):
"""
Context manager that opens a temporary file to use as a writer,
and closes the file on context exit, renaming it to the expected
file name in case of success, or removing it in case of failure.
Takes the same options as open(), but must be used as a context
manager.
"""
path = pathlib.Path(fname)
tmp = path.with_name('%s.tmp' % path.name)
try:
with tmp.open(*args, **kwargs) as fh:
yield fh
except Exception:
tmp.unlink()
raise
else:
tmp.rename(fname)
# The following is copied from
# https://github.com/mozilla-releng/redo/blob/6d07678a014e0c525e54a860381a165d34db10ff/redo/__init__.py#L15-L85
def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1):
@ -180,20 +220,12 @@ def download_to_path(url, path, sha256=None, size=None):
for _ in retrier(attempts=5, sleeptime=60):
try:
tmp = path.with_name('%s.tmp' % path.name)
log('Downloading %s to %s' % (url, path))
log('Downloading %s to %s' % (url, tmp))
with rename_after_close(path, 'wb') as fh:
for chunk in stream_download(url, sha256=sha256, size=size):
fh.write(chunk)
try:
with tmp.open('wb') as fh:
for chunk in stream_download(url, sha256=sha256, size=size):
fh.write(chunk)
except IntegrityError:
tmp.unlink()
raise
log('Renaming to %s' % path)
tmp.rename(path)
return
except IntegrityError:
raise
@ -240,6 +272,23 @@ def gpg_verify_path(path: pathlib.Path, public_key_data: bytes,
subprocess.run(['gpgconf', '--kill', 'gpg-agent'], env=env)
def open_tar_stream(path: pathlib.Path):
""""""
if path.suffix == '.bz2':
return bz2.open(str(path), 'rb')
elif path.suffix == '.gz':
return gzip.open(str(path), 'rb')
elif path.suffix == '.xz':
return lzma.open(str(path), 'rb')
elif path.suffix == '.zst':
dctx = ZstdDecompressor()
return dctx.stream_reader(path.open('rb'))
elif path.suffix == '.tar':
return path.open('rb')
else:
raise ValueError('unknown archive format for tar file: %s' % path)
def archive_type(path: pathlib.Path):
"""Attempt to identify a path as an extractable archive."""
if path.suffixes[-2:-1] == ['.tar']:
@ -257,27 +306,26 @@ def extract_archive(path, dest_dir, typ):
path = path.resolve()
dest_dir = dest_dir.resolve()
log('Extracting %s to %s' % (path, dest_dir))
t0 = time.time()
# We pipe input to the decompressor program so that we can apply
# custom decompressors that the program may not know about.
if typ == 'tar':
if path.suffix == '.bz2':
ifh = bz2.open(str(path), 'rb')
elif path.suffix == '.gz':
ifh = gzip.open(str(path), 'rb')
elif path.suffix == '.xz':
ifh = lzma.open(str(path), 'rb')
elif path.suffix == '.zst':
if not zstandard:
raise ValueError('zstandard Python package not available')
dctx = zstandard.ZstdDecompressor()
ifh = dctx.stream_reader(path.open('rb'))
elif path.suffix == '.tar':
ifh = path.open('rb')
ifh = open_tar_stream(path)
# On Windows, the tar program doesn't support things like symbolic
# links, while Windows actually support them. The tarfile module in
# python does. So use that. But since it's significantly slower than
# the tar program on Linux, only use tarfile on Windows (tarfile is
# also not much slower on Windows, presumably because of the
# notoriously bad I/O).
if sys.platform == 'win32':
tar = tarfile.open(fileobj=ifh, mode='r|')
tar.extractall(str(dest_dir))
args = []
else:
raise ValueError('unknown archive format for tar file: %s' % path)
args = ['tar', 'xf', '-']
pipe_stdin = True
args = ['tar', 'xf', '-']
pipe_stdin = True
elif typ == 'zip':
# unzip from stdin has wonky behavior. We don't use a pipe for it.
ifh = open(os.devnull, 'rb')
@ -286,27 +334,114 @@ def extract_archive(path, dest_dir, typ):
else:
raise ValueError('unknown archive format: %s' % path)
log('Extracting %s to %s using %r' % (path, dest_dir, args))
t0 = time.time()
if args:
with ifh, subprocess.Popen(args, cwd=str(dest_dir), bufsize=0,
stdin=subprocess.PIPE) as p:
while True:
if not pipe_stdin:
break
with ifh, subprocess.Popen(args, cwd=str(dest_dir), bufsize=0,
stdin=subprocess.PIPE) as p:
while True:
if not pipe_stdin:
break
chunk = ifh.read(131072)
if not chunk:
break
chunk = ifh.read(131072)
if not chunk:
break
p.stdin.write(chunk)
p.stdin.write(chunk)
if p.returncode:
raise Exception('%r exited %d' % (args, p.returncode))
if p.returncode:
raise Exception('%r exited %d' % (args, p.returncode))
log('%s extracted in %.3fs' % (path, time.time() - t0))
def repack_archive(orig: pathlib.Path, dest: pathlib.Path,
strip_components=0, prefix=''):
assert orig != dest
log('Repacking as %s' % dest)
orig_typ = archive_type(orig)
typ = archive_type(dest)
if not orig_typ:
raise Exception('Archive type not supported for %s' % orig.name)
if not typ:
raise Exception('Archive type not supported for %s' % dest.name)
if dest.suffixes[-2:] != ['.tar', '.zst']:
raise Exception('Only producing .tar.zst archives is supported.')
if strip_components or prefix:
def filter(name):
if strip_components:
stripped = '/'.join(name.split('/')[strip_components:])
if not stripped:
raise Exception(
'Stripping %d components would remove files'
% strip_components)
name = stripped
return prefix + name
else:
filter = None
with rename_after_close(dest, 'wb') as fh:
ctx = ZstdCompressor()
if orig_typ == 'zip':
assert typ == 'tar'
zip = zipfile.ZipFile(orig)
# Convert the zip stream to a tar on the fly.
with ctx.stream_writer(fh) as compressor, \
tarfile.open(fileobj=compressor, mode='w:') as tar:
for zipinfo in zip.infolist():
if zipinfo.is_dir():
continue
tarinfo = tarfile.TarInfo()
filename = zipinfo.filename
tarinfo.name = filter(filename) if filter else filename
tarinfo.size = zipinfo.file_size
# Zip files don't have any knowledge of the timezone
# they were created in. Which is not really convenient to
# reliably convert to a timestamp. But we don't really
# care about accuracy, but rather about reproducibility,
# so we pick UTC.
time = datetime.datetime(
*zipinfo.date_time, tzinfo=datetime.timezone.utc)
tarinfo.mtime = time.timestamp()
# 0 is MS-DOS, 3 is UNIX. Only in the latter case do we
# get anything useful for the tar file mode.
if zipinfo.create_system == 3:
mode = zipinfo.external_attr >> 16
else:
mode = 0o0644
tarinfo.mode = stat.S_IMODE(mode)
if stat.S_ISLNK(mode):
tarinfo.type = tarfile.SYMTYPE
tarinfo.linkname = zip.read(filename).decode()
tar.addfile(tarinfo, zip.open(filename))
elif stat.S_ISREG(mode) or stat.S_IFMT(mode) == 0:
tar.addfile(tarinfo, zip.open(filename))
else:
raise Exception('Unsupported file mode %o'
% stat.S_IFMT(mode))
elif orig_typ == 'tar':
if typ == 'zip':
raise Exception('Repacking a tar to zip is not supported')
assert typ == 'tar'
ifh = open_tar_stream(orig)
if filter:
# To apply the filter, we need to open the tar stream and
# tweak it.
origtar = tarfile.open(fileobj=ifh, mode='r|')
with ctx.stream_writer(fh) as compressor, \
tarfile.open(fileobj=compressor, mode='w:') as tar:
for tarinfo in origtar:
if tarinfo.isdir():
continue
tarinfo.name = filter(tarinfo.name)
tar.addfile(tarinfo, origtar.extractfile(tarinfo))
else:
# We only change compression here. The tar stream is unchanged.
ctx.copy_stream(ifh, fh)
def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
"""Fetch a URL and extract it to a destination path.
@ -315,7 +450,7 @@ def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None):
the destination directory.
"""
basename = url.split('/')[-1]
basename = urllib.parse.urlparse(url).path.split('/')[-1]
dest_path = dest_dir / basename
download_to_path(url, dest_path, sha256=sha256, size=size)
@ -350,9 +485,6 @@ def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
if dest_path.suffixes[-2:] != ['.tar', '.zst']:
raise Exception('Only producing .tar.zst archives is supported.')
if not zstandard:
raise ValueError('zstandard Python package not available')
with tempfile.TemporaryDirectory() as td:
temp_dir = pathlib.Path(td)
@ -373,8 +505,8 @@ def git_checkout_archive(dest_path: pathlib.Path, repo: str, commit: str,
'--format=tar', '--prefix=%s/' % prefix, commit
], stdout=subprocess.PIPE)
with open(dest_path, 'wb') as out:
ctx = zstandard.ZstdCompressor()
with rename_after_close(dest_path, 'wb') as out:
ctx = ZstdCompressor()
ctx.copy_stream(proc.stdout, out)
proc.wait()
@ -410,20 +542,32 @@ def command_static_url(args):
dest = pathlib.Path(args.dest)
dest.parent.mkdir(parents=True, exist_ok=True)
basename = urllib.parse.urlparse(args.url).path.split('/')[-1]
if basename.endswith(''.join(dest.suffixes)):
dl_dest = dest
else:
dl_dest = dest.parent / basename
try:
download_to_path(args.url, dest, sha256=args.sha256, size=args.size)
download_to_path(args.url, dl_dest, sha256=args.sha256, size=args.size)
if gpg_sig_url:
gpg_verify_path(dest, gpg_key, gpg_signature)
gpg_verify_path(dl_dest, gpg_key, gpg_signature)
if dl_dest != dest or args.strip_components or args.add_prefix:
repack_archive(dl_dest, dest, args.strip_components, args.add_prefix)
except Exception:
try:
dest.unlink()
dl_dest.unlink()
except FileNotFoundError:
pass
raise
if dl_dest != dest:
log('Removing %s' % dl_dest)
dl_dest.unlink()
def api(root_url, service, version, path):
# taskcluster-lib-urls is not available when this script runs, so
@ -486,6 +630,12 @@ def main():
'URL to fetch')
url.add_argument('--gpg-key-env',
help='Environment variable containing GPG key to validate')
url.add_argument('--strip-components', type=int, default=0,
help='Number of leading components to strip from file '
'names in the downloaded archive')
url.add_argument('--add-prefix', default='',
help='Prefix to add to file names in the downloaded '
'archive')
url.add_argument('url', help='URL to fetch')
url.add_argument('dest', help='Destination path')

15
taskcluster/scripts/misc/tooltool-download.sh
View File

@ -1,6 +1,6 @@
# Fetch a tooltool manifest.
cd $GECKO_PATH
cd $MOZ_FETCHES_DIR
case "`uname -s`" in
Linux)
@ -30,6 +30,17 @@ fi
: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache}
export TOOLTOOL_CACHE
./mach artifact toolchain -v${TOOLTOOL_DL_FLAGS}${TOOLTOOL_MANIFEST:+ --tooltool-manifest "${TOOLTOOL_MANIFEST}"}${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}} --retry 5${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}}
if [ -n "$MOZ_TOOLCHAINS" ]; then
echo This script should not be used for toolchain downloads anymore
echo Use fetches
exit 1
fi
if [ -z "$TOOLTOOL_MANIFEST" ]; then
echo This script should not be used when there is no tooltool manifest set
exit 1
fi
${GECKO_PATH}/mach artifact toolchain -v${TOOLTOOL_DL_FLAGS} --tooltool-manifest "${GECKO_PATH}/${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}} --retry 5
cd $OLDPWD

4
taskcluster/scripts/misc/vs-setup.sh
View File

@ -1,6 +1,6 @@
VSDIR=vs2017_15.8.4
VSPATH="${GECKO_PATH}/${VSDIR}"
UNIX_VSPATH="$(cd ${GECKO_PATH} && pwd)/${VSDIR}"
VSPATH="${MOZ_FETCHES_DIR}/${VSDIR}"
UNIX_VSPATH="$(cd ${MOZ_FETCHES_DIR} && pwd)/${VSDIR}"
SDK_VERSION=10.0.17134.0
export INCLUDE="${VSPATH}/VC/include;${VSPATH}/VC/atlmfc/include;${VSPATH}/SDK/Include/${SDK_VERSION}/ucrt;${VSPATH}/SDK/Include/${SDK_VERSION}/shared;${VSPATH}/SDK/Include/${SDK_VERSION}/um;${VSPATH}/SDK/Include/${SDK_VERSION}/winrt;${VSPATH}/DIA SDK/include"

2
taskcluster/scripts/misc/wr-macos-cross-build-setup.sh
View File

@ -5,7 +5,7 @@ export TARGET_TRIPLE="x86_64-apple-darwin"
source "${GECKO_PATH}/taskcluster/scripts/misc/tooltool-download.sh"
MACOS_SYSROOT="${GECKO_PATH}/MacOSX10.11.sdk"
MACOS_SYSROOT="${MOZ_FETCHES_DIR}/MacOSX10.11.sdk"
CLANGDIR="${MOZ_FETCHES_DIR}/clang"
# Deploy the wrench dependencies

3
taskcluster/scripts/misc/wrench-deps-vendoring.sh
View File

@ -9,8 +9,7 @@ set -x -e -v
UPLOAD_DIR=$HOME/artifacts
cd $GECKO_PATH
. taskcluster/scripts/misc/tooltool-download.sh
export PATH=$PATH:$GECKO_PATH/rustc/bin
export PATH=$PATH:$MOZ_FETCHES_DIR/rustc/bin
cargo install --version 0.1.23 cargo-vendor
cd gfx/wr/
mkdir .cargo

5
taskcluster/scripts/misc/wrench-windows-tests.sh
View File

@ -12,7 +12,7 @@ cd $GECKO_PATH
# This will download the rustc, cmake, ninja, MSVC, and wrench-deps artifacts.
. taskcluster/scripts/misc/tooltool-download.sh
export PATH=$PATH:$MOZ_FETCHES_DIR/rustc/bin:$GECKO_PATH/cmake/bin:$GECKO_PATH/ninja/bin
export PATH=$PATH:$MOZ_FETCHES_DIR/rustc/bin:$MOZ_FETCHES_DIR/cmake/bin:$MOZ_FETCHES_DIR/ninja/bin
. taskcluster/scripts/misc/vs-setup.sh
@ -27,3 +27,6 @@ powershell.exe 'iex (Get-Content -Raw ci-scripts\set-screenresolution.ps1); Set-
export CARGOFLAGS='--verbose --frozen'
export FREETYPE_CMAKE_GENERATOR=Ninja
cmd.exe /c 'ci-scripts\windows-tests.cmd'
# Diagnostic for bug 1571986.
tasklist -M

36
taskcluster/taskgraph/transforms/fetch.py
View File

@ -71,9 +71,21 @@ FETCH_SCHEMA = Schema({
Required('key-path'): basestring,
},
# The name to give to the generated artifact.
# The name to give to the generated artifact. Defaults to the file
# portion of the URL. Using a different extension converts the
# archive to the given type. Only conversion to .tar.zst is
# supported.
Optional('artifact-name'): basestring,
# Strip the given number of path components at the beginning of
# each file entry in the archive.
# Requires an artifact-name ending with .tar.zst.
Optional('strip-components'): int,
# Add the given prefix to each file entry in the archive.
# Requires an artifact-name ending with .tar.zst.
Optional('add-prefix'): basestring,
# IMPORTANT: when adding anything that changes the behavior of the task,
# it is important to update the digest data used to compute cache hits.
},
@ -168,12 +180,24 @@ def create_fetch_url_task(config, job):
if not artifact_name:
artifact_name = fetch['url'].split('/')[-1]
args = [
command = [
'/builds/worker/bin/fetch-content', 'static-url',
]
# Arguments that matter to the cache digest
args = [
'--sha256', fetch['sha256'],
'--size', '%d' % fetch['size'],
]
if fetch.get('strip-components'):
args.extend(['--strip-components', '%d' % fetch['strip-components']])
if fetch.get('add-prefix'):
args.extend(['--add-prefix', fetch['add-prefix']])
command.extend(args)
env = {}
if 'gpg-signature' in fetch:
@ -185,16 +209,16 @@ def create_fetch_url_task(config, job):
gpg_key = fh.read()
env['FETCH_GPG_KEY'] = gpg_key
args.extend([
command.extend([
'--gpg-sig-url', sig_url,
'--gpg-key-env', 'FETCH_GPG_KEY',
])
args.extend([
command.extend([
fetch['url'], '/builds/worker/artifacts/%s' % artifact_name,
])
task = make_base_task(config, name, job['description'], args)
task = make_base_task(config, name, job['description'], command)
task['treeherder']['symbol'] = join_symbol('Fetch', name)
task['worker']['artifacts'] = [{
'type': 'directory',
@ -216,7 +240,7 @@ def create_fetch_url_task(config, job):
# We don't include the GPG signature in the digest because it isn't
# materially important for caching: GPG signatures are supplemental
# trust checking beyond what the shasum already provides.
digest_data=[fetch['sha256'], '%d' % fetch['size'], artifact_name],
digest_data=args + [artifact_name],
)
return task

11
taskcluster/taskgraph/transforms/job/__init__.py
View File

@ -161,6 +161,17 @@ def use_fetches(config, jobs):
artifact_names = {}
aliases = {}
if config.kind == 'toolchain':
jobs = list(jobs)
for job in jobs:
run = job.get('run', {})
label = 'toolchain-{}'.format(job['name'])
get_attribute(
artifact_names, label, run, 'toolchain-artifact')
value = run.get('toolchain-alias')
if value:
aliases['toolchain-{}'.format(value)] = label
for task in config.kind_dependencies_tasks:
if task.kind in ('fetch', 'toolchain'):
get_attribute(

2
taskcluster/taskgraph/transforms/job/common.py
View File

@ -85,7 +85,7 @@ def add_artifacts(config, job, taskdesc, path):
def docker_worker_add_artifacts(config, job, taskdesc):
""" Adds an artifact directory to the task """
path = '{workdir}/artifacts/'.format(**job['run'])
taskdesc['worker']['env']['UPLOAD_DIR'] = path
taskdesc['worker'].setdefault('env', {})['UPLOAD_DIR'] = path
add_artifacts(config, job, taskdesc, path)

5
taskcluster/taskgraph/transforms/job/toolchain.py
View File

@ -132,12 +132,9 @@ def docker_worker_toolchain(config, job, taskdesc):
env.update({
'MOZ_BUILD_DATE': config.params['moz_build_date'],
'MOZ_SCM_LEVEL': config.params['level'],
'MOZ_FETCHES_DIR': workspace,
'GECKO_PATH': gecko_path,
})
# If the task definition hasn't overridden the default value, set to
# the workspace.
if env.get('MOZ_FETCHES_DIR') == 'fetches':
env['MOZ_FETCHES_DIR'] = workspace
attributes = taskdesc.setdefault('attributes', {})
attributes['toolchain-artifact'] = run.pop('toolchain-artifact')