From 9f3f69216da9459e33e33f0ae94cec2e66e38e78 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 9 Sep 2026 21:11:12 +0200 Subject: [PATCH] binaries: strip debug symbols on Linux, bundle only needed botocore models, one OpenSSL The Linux binaries were about 70 MiB as a single file and unpacked to 152 MiB at every start, see #10345. Three changes: Strip the debug symbols on Linux: pythons built with the default CFLAGS carry -g, and the spec had strip=False everywhere, so about half of the Linux binaries was debug info. Enable PyInstaller's strip for the collected shared libraries on Linux (only there: on macOS it would interfere with code signing, and it is not recommended on Windows). Python tracebacks do not need the symbols, only gdb would. Measured on the glibc239 x86_64 bundle: 152 -> 78 MiB unpacked, 69.6 -> 43.4 MiB compressed. Bundle only the botocore service models borg needs: botocore (boto3, s3 extra) ships the JSON service models of all ~430 AWS services, 23 MiB, about 14 MiB in the compressed binary. borg only ever creates an s3 client - also for S3-compatible services like MinIO or Backblaze B2, which just get a different endpoint URL - so the spec keeps only the s3 model, the shared top-level files and the models the credential providers may need (sts, sso, sso-oidc). In the oldglibc_binary job, build cryptography (a paramiko dependency from the sftp extra) from source against the OpenSSL built there, instead of using the manylinux wheel that statically links its own copy of OpenSSL - the binary contained two OpenSSLs. The runner image has the Rust toolchain this needs. The job now checks that the bundled OpenSSL and Python libraries are the ones built there by their GNU build IDs (the files are stripped now), that cryptography links and loads that OpenSSL, that no bundled shared library still carries debug info, that the other botocore models are gone, prints the bundle sizes, and smoke-tests the frozen binary against a MinIO server - the test suite's S3 test runs borg from the venv, not the binary. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 84 ++++++++++++++++++++++++++++++++++++++-- docs/changes.rst | 4 ++ scripts/borg.exe.spec | 35 +++++++++++++++-- 3 files changed, 116 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d79aaac1b..2b56b7959 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -596,6 +596,29 @@ jobs: pip install -r requirements.d/development.lock.txt pip install -r requirements.d/pyinstaller.txt + - name: Build cryptography against the OpenSSL we built + # cryptography (a paramiko dependency, sftp extra) comes as a manylinux + # wheel that statically links its own copy of OpenSSL, which would be a + # second OpenSSL in the binary, see #10345. Build it from source against + # the OpenSSL built above instead - the runner image has the Rust + # toolchain this needs. + env: + # where openssl-sys (the Rust OpenSSL bindings) finds the OpenSSL + # headers and libraries; the link is dynamic unless OPENSSL_STATIC is set. + OPENSSL_DIR: ${{ env.DEPS_PREFIX }} + # make the extension load that OpenSSL at runtime, see the python build above + RUSTFLAGS: -C link-arg=-Wl,-rpath,${{ env.DEPS_PREFIX }}/lib + run: | + set -euxo pipefail + rustc --version + pip install --no-binary cryptography cryptography + # check that the extension is linked against (and loads) that OpenSSL + rust_ext=$(python -c 'import cryptography.hazmat.bindings._rust as m; print(m.__file__)') + ldd "$rust_ext" + ldd "$rust_ext" | grep "libcrypto.so.3 => $DEPS_PREFIX/lib/" + python -c 'from cryptography.hazmat.backends.openssl.backend import backend; print(backend.openssl_version_text())' \ + | grep "^OpenSSL $OPENSSL_VERSION " + - name: Install borgbackup env: # Link borg's crypto extension against the OpenSSL built above - with @@ -637,11 +660,19 @@ jobs: # newer glibc than its name promises. run: | set -euxo pipefail - # borg-dir has everything as plain files, and PyInstaller bundles the - # shared libraries as they are (no strip, no upx there - see the spec): - # the bundled ones must be the files from the prefix. + # borg-dir has everything as plain files: the bundled libraries must be + # the ones from the prefix. PyInstaller strips them (see the spec), so + # compare the GNU build IDs, which identify the link output and survive + # stripping, instead of the files. + build_id() { readelf -n "$1" | grep -o 'Build ID: [0-9a-f]*' || true; } for lib in libcrypto.so.3 libssl.so.3 "libpython${PYTHON_VERSION%.*}.so.1.0"; do - cmp "$DEPS_PREFIX/lib/$lib" "$(find dist/binary/borg-dir -name "$lib")" + ours=$(build_id "$DEPS_PREFIX/lib/$lib") + bundled=$(build_id "$(find dist/binary/borg-dir -name "$lib")") + echo "$lib: prefix [$ours] bundled [$bundled]" + if [ -z "$ours" ] || [ "$ours" != "$bundled" ]; then + echo "::error::the bundled $lib is not the one built in this job" + exit 1 + fi done # the bundled python is the one built above dist/binary/borg-dir/borg.exe debug info @@ -649,6 +680,51 @@ jobs: # no bundled ELF file needs a newer glibc than the one in the binary's name find dist/binary/borg-dir -type f \( -name '*.so' -o -name '*.so.*' -o -name 'borg.exe' \) -print0 \ | xargs -0 python scripts/glibc_check.py "$GLIBC_VERSION" + # cryptography uses the bundled OpenSSL, it does not bring its own (see + # the step that builds it) + readelf -d "$(find dist/binary/borg-dir -name '_rust.abi3.so')" | grep 'NEEDED.*libcrypto.so.3' + # the shared libraries are stripped (see strip_binaries in the spec): + # none of them may still carry debug info + if find dist/binary/borg-dir -type f \( -name '*.so' -o -name '*.so.*' \) -exec file {} + | grep 'with debug_info'; then + echo "::error::bundled shared libraries still carry debug info" + exit 1 + fi + # only the botocore models borg needs are bundled (see the spec) + ls dist/binary/borg-dir/_internal/botocore/data + test -d dist/binary/borg-dir/_internal/botocore/data/s3 + test ! -e dist/binary/borg-dir/_internal/botocore/data/ec2 + du -sh dist/binary/borg-dir + ls -l dist/binary/borg.exe dist/binary/borg.tgz + + - name: Smoke-test the binary against an S3 server (${{ matrix.binary }}) + # The spec bundles only the botocore models borg needs: exercise the s3 + # model and the endpoint files of the frozen binary against a real S3 + # server (MinIO, like the test suite's S3 test - which runs borg from the + # venv, not the binary). + run: | + set -euxo pipefail + case "$(uname -m)" in + x86_64|amd64) srv_url=https://dl.min.io/server/minio/release/linux-amd64/minio ;; + aarch64|arm64) srv_url=https://dl.min.io/server/minio/release/linux-arm64/minio ;; + *) echo "Unsupported arch: $(uname -m)"; exit 1 ;; + esac + curl -fsSL -o "$RUNNER_TEMP/minio" "$srv_url" + chmod +x "$RUNNER_TEMP/minio" + mkdir -p "$RUNNER_TEMP/minio-data" + # default credentials: minioadmin / minioadmin + nohup "$RUNNER_TEMP/minio" server "$RUNNER_TEMP/minio-data" --address 127.0.0.1:9000 > "$RUNNER_TEMP/minio.log" 2>&1 & + for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done + # the bucket has to exist - the venv has boto3 (s3 extra) + python -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", aws_access_key_id="minioadmin", aws_secret_access_key="minioadmin").create_bucket(Bucket="borg")' + export BORG_REPO="s3:minioadmin:minioadmin@http://127.0.0.1:9000/borg/binary-repo" + export BORG_PASSPHRASE=test + borg=dist/binary/borg-dir/borg.exe + $borg repo-create -e aes256-ocb + $borg create --stats archive1 src/borg/testsuite + $borg list archive1 | wc -l + $borg check + $borg extract --dry-run archive1 + $borg repo-delete --force - name: Run tests # No SFTP, S3 or rest-over-ssh test servers here (see native_tests): those diff --git a/docs/changes.rst b/docs/changes.rst index a02b41f25..e4b279f5f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -182,6 +182,10 @@ Other changes: - CI: also build Linux binaries on Ubuntu 24.04 (glibc 2.39) for systems with an older glibc or a CPU below x86-64-v3, with OpenSSL 3.5 and Python 3.14 built from source, #10342. +- binaries: strip the debug symbols from the Linux binaries, bundle only the + botocore (S3) service models borg needs, and build cryptography against the + bundled OpenSSL in the glibc239 binaries instead of bundling a second OpenSSL + with it, #10345. Version 2.0.0b24 (2026-09-02) ----------------------------- diff --git a/scripts/borg.exe.spec b/scripts/borg.exe.spec index e1155cad6..44f581b57 100644 --- a/scripts/borg.exe.spec +++ b/scripts/borg.exe.spec @@ -4,6 +4,15 @@ import os, sys is_win32 = sys.platform.startswith('win32') +is_linux = sys.platform.startswith('linux') + +# Strip the symbol tables and debug info from the collected shared libraries +# (libpython, the extension modules, libcrypto, ...) on Linux: pythons built +# with the default CFLAGS carry -g debug info, which makes up about half of +# the size of the Linux binaries, see #10345. Only on Linux: on macOS the +# strip would interfere with code signing, and it is not recommended on +# Windows. Python tracebacks do not need any of this, only gdb would. +strip_binaries = is_linux # Note: SPEC contains the spec file argument given to pyinstaller here = os.path.dirname(os.path.abspath(SPEC)) @@ -50,6 +59,26 @@ if sys.platform == 'darwin': # mismatch to the installed kernel driver of osxfuse. a.binaries = [b for b in a.binaries if 'libosxfuse' not in b[0]] +# botocore (boto3, s3 extra) ships the JSON service models of all ~430 AWS +# services (23 MiB, ~14 MiB in the compressed binary, see #10345), but borg +# only ever creates an s3 client - also for S3-compatible services like MinIO +# or Backblaze B2, which just get a different endpoint URL. Keep the s3 model, +# the shared top-level files (endpoints.json, partitions.json, ...) and the +# models the credential providers may need: sts (assume-role and web-identity +# profiles), sso and sso-oidc (SSO profiles). Everything else is dead weight. +botocore_needed_services = ('s3', 'sts', 'sso', 'sso-oidc') + + +def is_unneeded_botocore_model(dest_name): + parts = os.path.normpath(dest_name).split(os.sep) + if parts[:2] != ['botocore', 'data'] or len(parts) < 4: + # not a service model - e.g. a shared top-level file like botocore/data/endpoints.json + return False + return parts[2] not in botocore_needed_services + + +a.datas = [d for d in a.datas if not is_unneeded_botocore_model(d[0])] + pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, @@ -59,7 +88,7 @@ exe = EXE(pyz, a.datas, name='borg.exe', debug=False, - strip=False, + strip=strip_binaries, upx=True, console=True, icon='NONE') @@ -74,7 +103,7 @@ slim_exe = EXE(pyz, exclude_binaries=True, name='borg.exe', debug=False, - strip=False, + strip=strip_binaries, upx=False, console=True) @@ -82,6 +111,6 @@ coll = COLLECT(slim_exe, a.binaries, a.zipfiles, a.datas, - strip=False, + strip=strip_binaries, upx=False, name='borg-dir')