Bug 1914947 - Add stage1 and profile tasks for macosx64 and macosx64-aarch64. r=firefox-build-system-reviewers,sergesanspaille

This splits the clang build in the necessary steps to get native
profiles for Intel and Arm64 mac, similarly to Linux.

Stage 1s are cross-compiled, and the profile step runs natively.

Differential Revision: https://phabricator.services.mozilla.com/D220530

build/build-clang/README

@@ -50,4 +50,4 @@ Environment Variables
 
 The following environment variables are used for cross-compile builds targeting OS X on Linux.
 
-* CROSS_SYSROOT: Path to the OS X SDK directory for cross compile builds.
+* OSX_SYSROOT: Path to the OS X SDK directory for cross compile builds.

build/build-clang/build-clang.py

@@ -189,6 +189,7 @@ def build_one_stage(
     asm,
     ar,
     ranlib,
+    libtool,
     ldflags,
     src_dir,
     stage_dir,
@@ -210,8 +211,13 @@ def build_one_stage(
     def slashify_path(path):
         return path.replace("\\", "/")
 
-    def cmake_base_args(cc, cxx, asm, ar, ranlib, ldflags, inst_dir):
-        machine_targets = targets if is_final_stage and targets else "X86"
+    def cmake_base_args(cc, cxx, asm, ar, ranlib, libtool, ldflags, inst_dir):
+        if is_final_stage and targets:
+            machine_targets = targets
+        elif target.startswith("aarch64-"):
+            machine_targets = "AArch64"
+        else:
+            machine_targets = "X86"
 
         cmake_args = [
             "-GNinja",
@@ -287,20 +293,25 @@ def build_one_stage(
             cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
         if ranlib is not None:
             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
-        if is_darwin(target) and is_cross_compile(target):
+        if libtool is not None:
+            cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
+        if is_darwin(target):
             arch = "arm64" if target.startswith("aarch64") else "x86_64"
+            if is_cross_compile(target):
+                cmake_args += [
+                    "-DCMAKE_SYSTEM_NAME=Darwin",
+                ]
             cmake_args += [
-                "-DCMAKE_SYSTEM_NAME=Darwin",
                 "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"],
-                "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
-                "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
+                "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("OSX_SYSROOT")),
+                "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("OSX_SYSROOT")),
                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
                 "-DCMAKE_MACOSX_RPATH=ON",
                 "-DCMAKE_OSX_ARCHITECTURES=%s" % arch,
                 "-DDARWIN_osx_ARCHS=%s" % arch,
-                "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
+                "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("OSX_SYSROOT")),
                 "-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % target,
                 "-DCMAKE_C_COMPILER_TARGET=%s" % target,
                 "-DCMAKE_CXX_COMPILER_TARGET=%s" % target,
@@ -336,7 +347,7 @@ def build_one_stage(
         return cmake_args
 
     cmake_args = []
-    cmake_args += cmake_base_args(cc, cxx, asm, ar, ranlib, ldflags, inst_dir)
+    cmake_args += cmake_base_args(cc, cxx, asm, ar, ranlib, libtool, ldflags, inst_dir)
     cmake_args += [src_dir]
     build_package(build_dir, cmake_args)
 
@@ -640,6 +651,7 @@ def main():
     # knows how to find it when they are installed alongside each others.
     ar = get_tool(config, "lib" if is_windows(target) else "ar")
     ranlib = None if is_windows(target) else get_tool(config, "ranlib")
+    libtool = get_tool(config, "libtool") if is_darwin(target) else None
 
     if not os.path.exists(source_dir):
         os.makedirs(source_dir)
@@ -733,6 +745,7 @@ def main():
             [asm] + extra_asmflags,
             ar,
             ranlib,
+            libtool,
             extra_ldflags,
             llvm_source_dir,
             stage1_dir,
@@ -758,6 +771,7 @@ def main():
             [asm] + extra_asmflags,
             ar,
             ranlib,
+            libtool,
             extra_ldflags,
             llvm_source_dir,
             stage2_dir,
@@ -784,6 +798,7 @@ def main():
             [asm] + extra_asmflags,
             ar,
             ranlib,
+            libtool,
             extra_ldflags,
             llvm_source_dir,
             stage3_dir,
@@ -826,6 +841,7 @@ def main():
             [asm] + extra_asmflags,
             ar,
             ranlib,
+            libtool,
             extra_ldflags,
             llvm_source_dir,
             stage4_dir,

build/build-clang/macosx64.json

@@ -5,5 +5,6 @@
     "as": "{MOZ_FETCHES_DIR}/clang/bin/clang",
     "ar": "{MOZ_FETCHES_DIR}/clang/bin/llvm-ar",
     "ranlib": "{MOZ_FETCHES_DIR}/clang/bin/llvm-ranlib",
+    "libtool": "{MOZ_FETCHES_DIR}/clang/bin/llvm-libtool-darwin",
     "ld": "{MOZ_FETCHES_DIR}/clang/bin/clang"
 }

taskcluster/kinds/toolchain/clang.yml

@@ -277,6 +277,60 @@ linux64-clang-18:
             - win64-compiler-rt-18
             - win64-aarch64-compiler-rt-18
 
+macosx64-clang-18-stage1:
+    description: "Clang 18 toolchain build"
+    treeherder:
+        symbol: TM(clang-18-stage1)
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/1stage.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/1stage.json'
+        toolchain-artifact: public/build/clang.tar.zst
+    fetches:
+        fetch:
+            - clang-18
+        toolchain:
+            - linux64-clang-18-stage1
+            - macosx64-sdk-toolchain
+
+macosx64-clang-18-profile:
+    description: "Clang 18 toolchain build"
+    treeherder:
+        symbol: TM(clang-18-profile)
+    worker-type: b-osx-1015
+    worker:
+        max-run-time: 14400
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        toolchain-artifact: public/build/merged.profdata
+    fetches:
+        fetch:
+            - clang-18
+            - macosx64-cmake
+            - macosx64-ninja
+        toolchain:
+            - macosx64-clang-18-stage1
+            - macosx64-x64-compiler-rt-18
+            - macosx64-sdk-toolchain
+
 macosx64-clang-18-raw:
     description: "Clang 18 toolchain build"
     treeherder:
@@ -301,7 +355,7 @@ macosx64-clang-18-raw:
             - clang-18
         toolchain:
             - linux64-clang-18-stage1
-            - linux64-clang-18-profile
+            - macosx64-clang-18-profile
             - macosx64-sdk-toolchain
             - macosx64-x64-compiler-rt-18
 
@@ -345,6 +399,64 @@ macosx64-clang-18:
             - win64-compiler-rt-18
             - win64-aarch64-compiler-rt-18
 
+macosx64-aarch64-clang-18-stage1:
+    description: "Clang 18 toolchain build"
+    treeherder:
+        symbol: TM(clang-18-aarch64-stage1)
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/1stage.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/1stage.json'
+        toolchain-artifact: public/build/clang.tar.zst
+    fetches:
+        fetch:
+            - clang-18
+        toolchain:
+            - linux64-clang-18-stage1
+            - macosx64-sdk-toolchain
+
+macosx64-aarch64-clang-18-profile:
+    description: "Clang 18 toolchain build"
+    treeherder:
+        symbol: TM(clang-18-aarch64-profile)
+    worker-type: b-osx-arm64
+    worker:
+        max-run-time: 7200
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-18.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        toolchain-artifact: public/build/merged.profdata
+    fetches:
+        fetch:
+            - clang-18
+            - macosx64-cmake
+            - macosx64-ninja
+        toolchain:
+            - macosx64-clang-18-stage1
+            - macosx64-aarch64-compiler-rt-18
+            - macosx64-sdk-toolchain
+
 macosx64-aarch64-clang-18-raw:
     description: "Clang 18 toolchain build"
     treeherder:
@@ -371,9 +483,9 @@ macosx64-aarch64-clang-18-raw:
             - clang-18
         toolchain:
             - linux64-clang-18-stage1
-            - linux64-clang-18-profile
-            - macosx64-sdk-toolchain
+            - macosx64-aarch64-clang-18-profile
             - macosx64-aarch64-compiler-rt-18
+            - macosx64-sdk-toolchain
 
 macosx64-aarch64-clang-18:
     description: "Clang 18 toolchain repack with MacOS Compiler RT libs"
@@ -756,6 +868,64 @@ linux64-clang-trunk:
             - win64-compiler-rt-trunk
             - win64-aarch64-compiler-rt-trunk
 
+macosx64-clang-trunk-stage1:
+    description: "Clang trunk toolchain build"
+    attributes:
+        cached_task: false
+    treeherder:
+        symbol: TM(clang-trunk-stage1)
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/1stage.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/1stage.json'
+        toolchain-artifact: public/build/clang.tar.zst
+    fetches:
+        fetch:
+            - clang-trunk
+        toolchain:
+            - linux64-clang-trunk-stage1
+            - macosx64-sdk-toolchain
+
+macosx64-clang-trunk-profile:
+    description: "Clang trunk toolchain build"
+    attributes:
+        cached_task: false
+    treeherder:
+        symbol: TM(clang-trunk-profile)
+    worker-type: b-osx-1015
+    worker:
+        max-run-time: 14400
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        toolchain-artifact: public/build/merged.profdata
+    fetches:
+        fetch:
+            - clang-trunk
+            - macosx64-cmake
+            - macosx64-ninja
+        toolchain:
+            - macosx64-clang-trunk-stage1
+            - macosx64-x64-compiler-rt-trunk
+            - macosx64-sdk-toolchain
+
 macosx64-clang-trunk-raw:
     description: "Clang trunk toolchain build"
     attributes:
@@ -782,7 +952,7 @@ macosx64-clang-trunk-raw:
             - clang-trunk
         toolchain:
             - linux64-clang-trunk-stage1
-            - linux64-clang-trunk-profile
+            - macosx64-clang-trunk-profile
             - macosx64-sdk-toolchain
             - macosx64-x64-compiler-rt-trunk
 
@@ -826,6 +996,68 @@ macosx64-clang-trunk:
             - win64-compiler-rt-trunk
             - win64-aarch64-compiler-rt-trunk
 
+macosx64-aarch64-clang-trunk-stage1:
+    description: "Clang trunk toolchain build"
+    attributes:
+        cached_task: false
+    treeherder:
+        symbol: TM(clang-trunk-aarch64-stg1)
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/1stage.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/1stage.json'
+        toolchain-artifact: public/build/clang.tar.zst
+    fetches:
+        fetch:
+            - clang-trunk
+        toolchain:
+            - linux64-clang-trunk-stage1
+            - macosx64-sdk-toolchain
+
+macosx64-aarch64-clang-trunk-profile:
+    description: "Clang trunk toolchain build"
+    attributes:
+        cached_task: false
+    treeherder:
+        symbol: TM(clang-trunk-aarch64-prof)
+    worker-type: b-osx-arm64
+    worker:
+        max-run-time: 7200
+    run:
+        using: toolchain-script
+        script: build-clang.sh
+        arguments:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        resources:
+            - 'build/build-clang/macosx64.json'
+            - 'build/build-clang/macosx64-aarch64.json'
+            - 'build/build-clang/clang-trunk.json'
+            - 'build/build-clang/skip-stage-1.json'
+            - 'build/build-clang/profile.json'
+        toolchain-artifact: public/build/merged.profdata
+    fetches:
+        fetch:
+            - clang-trunk
+            - macosx64-cmake
+            - macosx64-ninja
+        toolchain:
+            - macosx64-clang-trunk-stage1
+            - macosx64-aarch64-compiler-rt-trunk
+            - macosx64-sdk-toolchain
+
 macosx64-aarch64-clang-trunk-raw:
     description: "Clang trunk toolchain build"
     attributes:
@@ -854,9 +1086,9 @@ macosx64-aarch64-clang-trunk-raw:
             - clang-trunk
         toolchain:
             - linux64-clang-trunk-stage1
-            - linux64-clang-trunk-profile
-            - macosx64-sdk-toolchain
+            - macosx64-aarch64-clang-trunk-profile
             - macosx64-aarch64-compiler-rt-trunk
+            - macosx64-sdk-toolchain
 
 macosx64-aarch64-clang-trunk:
     description: "Clang trunk toolchain repack with MacOS Compiler RT libs"

taskcluster/scripts/misc/build-clang.sh

@@ -25,7 +25,7 @@ case "$CONFIGS" in
   echo echo 10.12 >> $ORIGPWD/bin/sw_vers
   chmod +x $ORIGPWD/bin/sw_vers
   # these variables are used in build-clang.py
-  export CROSS_SYSROOT=$(ls -d $MOZ_FETCHES_DIR/MacOSX1*.sdk)
+  export OSX_SYSROOT=$(ls -d $MOZ_FETCHES_DIR/MacOSX1*.sdk)
   export PATH=$PATH:$ORIGPWD/bin
   ;;
 *win64*)
@@ -38,9 +38,6 @@ case "$CONFIGS" in
     # LLVM_ENABLE_DIA_SDK is set if the directory "$ENV{VSINSTALLDIR}DIA SDK"
     # exists.
     export VSINSTALLDIR="${VSPATH}/"
-
-    export PATH="$(cd $MOZ_FETCHES_DIR/cmake && pwd)/bin:${PATH}"
-    export PATH="$(cd $MOZ_FETCHES_DIR/ninja && pwd)/bin:${PATH}"
     ;;
   *)
     export VSINSTALLDIR="$MOZ_FETCHES_DIR/vs"
@@ -55,6 +52,13 @@ case "$CONFIGS" in
   ;;
 esac
 
+if test -d "$MOZ_FETCHES_DIR/cmake"; then
+    export PATH="$(cd $MOZ_FETCHES_DIR/cmake && pwd)/bin:${PATH}"
+fi
+if test -d "$MOZ_FETCHES_DIR/ninja"; then
+    export PATH="$(cd $MOZ_FETCHES_DIR/ninja && pwd)/bin:${PATH}"
+fi
+
 # gets a bit too verbose here
 set +x