Bug 1834414 - Consistent lzma support for taskcluster Python r=glandium

On OSX, depending on the node used to build python, lzma support was
available or not (depending on local setup). Then it could fail at
runtime if another node was used.

Build xz from source and use it as a build dependency to always provide
lzma on OSX, as we already do on Linux and Windows.

As a side effect, provide more verbose output during the post link
process.

Differential Revision: https://phabricator.services.mozilla.com/D191044

taskcluster/ci/fetch/toolchains.yml

@@ -622,6 +622,17 @@ symbol-scrapers:
         repo: http://github.com/mozilla/symbol-scrapers
         revision: 2b0923643e9c859e00d388b363d2e4be1036dfab
 
+xz-5.4.4:
+    description: xz 5.4.4 source code
+    fetch:
+        type: static-url
+        url: https://tukaani.org/xz/xz-5.4.4.tar.gz
+        sha256: aae39544e254cfd27e942d35a048d592959bd7a79f9a624afb0498bb5613bdf8
+        size: 2874706
+        artifact-name: xz-source.tar.zst
+        strip-components: 1
+        add-prefix: xz-source/
+
 cpython-3.8.10:
     description: cpython 3.8.10 source code
     fetch:

taskcluster/ci/toolchain/misc.yml

@@ -181,6 +181,23 @@ linux64-custom-v8:
         ]
         toolchain-artifact: public/build/d8.tar.zst
 
+macosx64-xz:
+    description: "xz development library"
+    worker-type: b-osx-1015
+    treeherder:
+        symbol: TM(xz)
+    worker:
+        max-run-time: 30000
+    run:
+        script: build-xz.sh
+        toolchain-artifact: public/build/xz.tar.zst
+    fetches:
+        fetch:
+            - xz-5.4.4
+        toolchain:
+            - macosx64-clang-toolchain
+            - macosx64-sdk-toolchain
+
 macosx64-custom-car:
     description: "Custom chromium-as-release build for mac"
     worker-type: b-osx-1015

taskcluster/ci/toolchain/python.yml

@@ -62,3 +62,4 @@ macosx64-python-3.8:
         toolchain:
             - macosx64-clang-toolchain
             - macosx64-sdk-toolchain
+            - macosx64-xz

taskcluster/scripts/misc/build-cpython.sh

@@ -13,6 +13,7 @@ set -x
 clang_bindir=${MOZ_FETCHES_DIR}/clang/bin
 clang_libdir=${MOZ_FETCHES_DIR}/clang/lib
 python_src=${MOZ_FETCHES_DIR}/cpython-source
+xz_prefix=${MOZ_FETCHES_DIR}/xz
 
 # Make the compiler-rt available to clang.
 env UPLOAD_DIR= $GECKO_PATH/taskcluster/scripts/misc/repack-clang.sh
@@ -38,9 +39,9 @@ case `uname -s` in
         # NOTE: both CFLAGS and CPPFLAGS need to be set here, otherwise
         # configure step fails.
         sysroot_flags="-isysroot ${MOZ_FETCHES_DIR}/MacOSX${macosx_sdk}.sdk -mmacosx-version-min=${macosx_version_min}"
-        export CPPFLAGS=${sysroot_flags}
+        export CPPFLAGS="${sysroot_flags} -I${xz_prefix}/include"
         export CFLAGS=${sysroot_flags}
-        export LDFLAGS="${LDFLAGS} ${sysroot_flags}"
+        export LDFLAGS="${LDFLAGS} ${sysroot_flags} -L${xz_prefix}/lib"
         configure_flags_extra=--with-openssl=/usr/local/opt/openssl
 
         # see https://bugs.python.org/issue44065
@@ -76,15 +77,24 @@ case `uname -s` in
     Darwin)
         cp /usr/local/opt/openssl/lib/libssl*.dylib ${work_dir}/python/lib/
         cp /usr/local/opt/openssl/lib/libcrypto*.dylib ${work_dir}/python/lib/
+        cp ${xz_prefix}/lib/liblzma.dylib ${work_dir}/python/lib/
 
         # Instruct the loader to search for the lib in rpath instead of the one used during linking
         install_name_tool -change /usr/local/opt/openssl@1.1/lib/libssl.1.1.dylib @rpath/libssl.1.1.dylib ${work_dir}/python/lib/python3.8/lib-dynload/_ssl.cpython-38-darwin.so
         install_name_tool -change /usr/local/opt/openssl@1.1/lib/libcrypto.1.1.dylib @rpath/libcrypto.1.1.dylib ${work_dir}/python/lib/python3.8/lib-dynload/_ssl.cpython-38-darwin.so
+        otool -L ${work_dir}/python/lib/python3.8/lib-dynload/_ssl.cpython-38-darwin.so | grep @rpath/libssl.1.1.dylib
+
+
+        install_name_tool -change /xz/lib/liblzma.5.dylib @rpath/liblzma.5.dylib ${work_dir}/python/lib/python3.8/lib-dynload/_lzma.cpython-38-darwin.so
+        otool -L ${work_dir}/python/lib/python3.8/lib-dynload/_lzma.cpython-38-darwin.so | grep @rpath/liblzma.5.dylib
+
         # Also modify the shipped libssl to use the shipped libcrypto
         install_name_tool -change /usr/local/Cellar/openssl@1.1/1.1.1h/lib/libcrypto.1.1.dylib @rpath/libcrypto.1.1.dylib ${work_dir}/python/lib/libssl.1.1.dylib
+        otool -L ${work_dir}/python/lib/libssl.1.1.dylib | grep @rpath/libcrypto.1.1.dylib
 
         # sanity check
         ${work_dir}/python/bin/python3 -c "import ssl"
+        ${work_dir}/python/bin/python3 -c "import lzma"
         ;;
     Linux)
         cp /usr/lib/x86_64-linux-gnu/libffi.so.* ${work_dir}/python/lib/

taskcluster/scripts/misc/build-xz.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+set -e
+set -x
+
+# Required fetch artifact
+xz_src=${MOZ_FETCHES_DIR}/xz-source
+
+# Actual build
+work_dir=`pwd`
+dest_dir=${work_dir}/tmp-install
+tardir=xz
+
+cd `mktemp -d`
+${xz_src}/configure --prefix=/${tardir} CFLAGS=-O2 ${configure_flags_extra} || { exit_status=$? && cat config.log && exit $exit_status ; }
+export MAKEFLAGS=-j`nproc`
+make
+make DESTDIR=${dest_dir} install
+cd ${dest_dir}
+
+$(dirname $0)/pack.sh ${tardir}