Bug 1810014 - Move macOS SDK toolchain task to linux workers. r=firefox-build-system-reviewers,andi

Differential Revision: https://phabricator.services.mozilla.com/D166724

python/mozbuild/mozpack/macpkg.py

@@ -2,6 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# TODO: Eventually consolidate with mozpack.pkg module. This is kept separate
+# for now because of the vast difference in API, and to avoid churn for the
+# users of this module (docker images, macos SDK artifacts) when changes are
+# necessary in mozpack.pkg
 import bz2
 import io
 import lzma

taskcluster/ci/toolchain/macos-sdk.yml

@@ -37,6 +37,7 @@ macosx64-sdk-11.3:
     description: "MacOSX11.3 SDK"
     treeherder:
         symbol: TM(sdk11.3)
+    worker-type: b-linux-gcp
     run:
         script: pack-macos-sdk.sh
         arguments:
@@ -45,3 +46,6 @@ macosx64-sdk-11.3:
             - Library/Developer/CommandLineTools/SDKs/MacOSX11.3.sdk
         toolchain-artifact: project/gecko/mac-sdk/MacOSX11.3.sdk.tar.zst
         toolchain-alias: macosx64-sdk
+        resources:
+            - python/mozbuild/mozpack/macpkg.py
+            - taskcluster/scripts/misc/unpack-sdk.py

taskcluster/docker/system-symbols-mac/Dockerfile

@@ -27,6 +27,8 @@ ADD https://github.com/marco-c/breakpad-mac-update-symbols/raw/21221733edfbcac49
 RUN chmod +x /usr/local/bin/lipo
 
 COPY topsrcdir/tools/crashreporter/system-symbols/mac /builds/worker/mac-update-symbols
+# %include python/mozbuild/mozpack/macpkg.py
+COPY topsrcdir/python/mozbuild/mozpack/macpkg.py /builds/worker/mac-update-symbols/mozpack/
 WORKDIR /builds/worker/mac-update-symbols
 
 RUN chown -R worker:worker /builds/worker

taskcluster/scripts/misc/pack-macos-sdk.sh

@@ -12,7 +12,7 @@ curl -OL $PKG
 shasum -a 256 -c <<EOF
 $SHA256  $(basename $PKG)
 EOF
-pkgutil --expand-full $(basename $PKG) pkg/
-mv pkg/Payload/$SDK_DIR $(basename $SDK_DIR)
+$GECKO_PATH/mach python $(dirname $0)/unpack-sdk.py $(basename $PKG) pkg/
+mv pkg/$SDK_DIR $(basename $SDK_DIR)
 
 $(dirname $0)/pack.sh $(basename $SDK_DIR)

taskcluster/scripts/misc/unpack-sdk.py

@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import stat
+import sys
+
+from mozpack.macpkg import Pbzx, uncpio, unxar
+
+
+def unpkg(pkg_path, out_dir="."):
+    with open(pkg_path, "rb") as pkg:
+        for name, content in unxar(pkg):
+            if name == "Payload":
+                extract_payload(content, out_dir)
+
+
+def extract_payload(fileobj, out_dir="."):
+    for path, mode, content in uncpio(Pbzx(fileobj)):
+        if not path:
+            continue
+        path = os.path.join(out_dir, path.decode())
+        if stat.S_ISDIR(mode):
+            os.makedirs(path, exist_ok=True)
+        else:
+            parent = os.path.dirname(path)
+            if parent:
+                os.makedirs(parent, exist_ok=True)
+
+            if stat.S_ISLNK(mode):
+                os.symlink(content.read(), path)
+            elif stat.S_ISREG(mode):
+                with open(path, "wb") as out:
+                    shutil.copyfileobj(content, out)
+            else:
+                raise Exception(f"File mode {mode:o} is not supported")
+
+
+if __name__ == "__main__":
+    unpkg(*sys.argv[1:])

tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py

@@ -49,7 +49,7 @@ import subprocess
 import tempfile
 import traceback
 
-from macpkg import Pbzx, uncpio, unxar
+from mozpack.macpkg import Pbzx, uncpio, unxar
 from scrapesymbols.gathersymbols import process_paths