mirror of
https://github.com/borgbackup/borg.git
synced 2026-10-12 02:11:36 +00:00
CI: run the S3 tests against moto instead of MinIO, fixes #10361
MinIO withdrew its community binaries, so CI was pinned to its last GitHub releases that still carry assets (#10360) - a frozen snapshot that gets no fixes and could vanish as well. Use moto in server mode (moto_server) instead: it comes from PyPI, so there is no server binary to download, and the pins in the new requirements.d/s3-test-server.txt are kept up to date by dependabot. moto's "server" extra pulls in the dependencies of all AWS services it can mock, for S3 the server only needs flask and flask-cors on top of the "s3" extra. The mc client is not needed either, boto3 creates the bucket. This replaces the S3 server of the native_tests job and the one the oldglibc_binary job smoke-tests the frozen binary against. Also add test_s3_repo_compact: compact rewrites a mixed pack via store.defrag, which for the S3 backend is ranged GETs plus one PUT. A server that ignores the requested range goes unnoticed by test_s3_repo_basics, but makes compact fail. Co-Authored-By: Claude Fable 5.1 <[email protected]>
This commit is contained in:
Thomas Waldmann
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
e7b2de8a20
commit
7a16a03930
+19
-53
@@ -34,17 +34,6 @@ env:
|
||||
# to pytest (pass_env = ["*"]).
|
||||
PY_COLORS: "1" # pytest
|
||||
TOX_COLORED: "yes" # tox's own output
|
||||
# The S3 tests run against a MinIO server. MinIO retired its dl.min.io downloads (they
|
||||
# answer HTTP 410 now) and its releases from RELEASE.2025-10-15T17-29-55Z on carry no
|
||||
# binaries at all, so fetch the last releases that still publish them on GitHub and
|
||||
# check what we downloaded. Bumping a release means updating its sha256 here, too:
|
||||
# the sums are the ones published next to the binaries (<asset>.sha256sum).
|
||||
MINIO_RELEASE: "RELEASE.2025-09-07T16-13-09Z"
|
||||
MINIO_SHA256_LINUX_AMD64: "7c5bd8512c6e966455b1d198209358b2d191c77a83ab377c4073281065fb855f"
|
||||
MINIO_SHA256_LINUX_ARM64: "5c83cd2cf151717ba0243f73e1c7802ff36e272b67144bdd7f1f7d684fd6f03d"
|
||||
MC_RELEASE: "RELEASE.2025-08-13T08-35-41Z"
|
||||
MC_SHA256_LINUX_AMD64: "01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891"
|
||||
MC_SHA256_LINUX_ARM64: "14c8c9616cfce4636add161304353244e8de383b2e2752c0e9dad01d4c27c12c"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
@@ -295,36 +284,21 @@ jobs:
|
||||
echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV
|
||||
echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV
|
||||
|
||||
- name: Install and configure MinIO S3 server (test only)
|
||||
- name: Install and start moto S3 server (test only)
|
||||
# The S3 tests run against moto in server mode: it comes from PyPI (pinned in
|
||||
# requirements.d/s3-test-server.txt), so there is no server binary to download.
|
||||
# It keeps the objects in memory and accepts any credentials.
|
||||
if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') }}
|
||||
run: |
|
||||
set -e
|
||||
arch=$(uname -m)
|
||||
case "$arch" in
|
||||
x86_64|amd64) plat=linux-amd64; srv_sha=$MINIO_SHA256_LINUX_AMD64; cli_sha=$MC_SHA256_LINUX_AMD64 ;;
|
||||
aarch64|arm64) plat=linux-arm64; srv_sha=$MINIO_SHA256_LINUX_ARM64; cli_sha=$MC_SHA256_LINUX_ARM64 ;;
|
||||
*) echo "Unsupported arch: $arch"; exit 1 ;;
|
||||
esac
|
||||
srv_url=https://github.com/minio/minio/releases/download/$MINIO_RELEASE/minio.$plat.$MINIO_RELEASE
|
||||
cli_url=https://github.com/minio/mc/releases/download/$MC_RELEASE/mc.$plat.$MC_RELEASE
|
||||
curl -fsSL -o /usr/local/bin/minio "$srv_url"
|
||||
curl -fsSL -o /usr/local/bin/mc "$cli_url"
|
||||
echo "$srv_sha /usr/local/bin/minio" | sha256sum -c -
|
||||
echo "$cli_sha /usr/local/bin/mc" | sha256sum -c -
|
||||
sudo chmod +x /usr/local/bin/minio /usr/local/bin/mc
|
||||
export PATH=/usr/local/bin:$PATH
|
||||
# Start MinIO on :9000 with default credentials (minioadmin/minioadmin)
|
||||
MINIO_DIR="$GITHUB_WORKSPACE/.minio-data"
|
||||
MINIO_LOG="$GITHUB_WORKSPACE/.minio.log"
|
||||
mkdir -p "$MINIO_DIR"
|
||||
nohup minio server "$MINIO_DIR" --address ":9000" >"$MINIO_LOG" 2>&1 &
|
||||
# Wait for MinIO port to be ready
|
||||
pip install -r requirements.d/s3-test-server.txt
|
||||
nohup moto_server -H 127.0.0.1 -p 9000 > "$GITHUB_WORKSPACE/.moto.log" 2>&1 &
|
||||
# Wait for the moto port to be ready
|
||||
for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done
|
||||
# Configure client and create bucket
|
||||
mc alias set local http://127.0.0.1:9000 minioadmin minioadmin
|
||||
mc mb --ignore-existing local/borg
|
||||
# Create the bucket (boto3 is a moto dependency)
|
||||
python -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", aws_access_key_id="test", aws_secret_access_key="test").create_bucket(Bucket="borg")'
|
||||
# Export S3 test URL for tox via GITHUB_ENV
|
||||
echo "BORG_TEST_S3_REPO=s3:minioadmin:minioadmin@http://127.0.0.1:9000/borg/s3-repo" >> $GITHUB_ENV
|
||||
echo "BORG_TEST_S3_REPO=s3:test:test@http://127.0.0.1:9000/borg/s3-repo" >> $GITHUB_ENV
|
||||
|
||||
- name: Enable store cache (test only)
|
||||
if: ${{ matrix.store_cache == '1' }}
|
||||
@@ -744,27 +718,19 @@ jobs:
|
||||
|
||||
- name: Smoke-test the binary against an S3 server (${{ matrix.binary }})
|
||||
# The spec bundles only the botocore models borg needs: exercise the s3
|
||||
# model and the endpoint files of the frozen binary against a real S3
|
||||
# server (MinIO, like the test suite's S3 test - which runs borg from the
|
||||
# venv, not the binary).
|
||||
# model and the endpoint files of the frozen binary against an S3 server
|
||||
# (moto in server mode, like the test suite's S3 tests - which run borg
|
||||
# from the venv, not the binary). moto gets installed into the venv only
|
||||
# now, after the binary was built from it.
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64) plat=linux-amd64; srv_sha=$MINIO_SHA256_LINUX_AMD64 ;;
|
||||
aarch64|arm64) plat=linux-arm64; srv_sha=$MINIO_SHA256_LINUX_ARM64 ;;
|
||||
*) echo "Unsupported arch: $(uname -m)"; exit 1 ;;
|
||||
esac
|
||||
srv_url=https://github.com/minio/minio/releases/download/$MINIO_RELEASE/minio.$plat.$MINIO_RELEASE
|
||||
curl -fsSL -o "$RUNNER_TEMP/minio" "$srv_url"
|
||||
echo "$srv_sha $RUNNER_TEMP/minio" | sha256sum -c -
|
||||
chmod +x "$RUNNER_TEMP/minio"
|
||||
mkdir -p "$RUNNER_TEMP/minio-data"
|
||||
# default credentials: minioadmin / minioadmin
|
||||
nohup "$RUNNER_TEMP/minio" server "$RUNNER_TEMP/minio-data" --address 127.0.0.1:9000 > "$RUNNER_TEMP/minio.log" 2>&1 &
|
||||
pip install -r requirements.d/s3-test-server.txt
|
||||
# moto accepts any credentials
|
||||
nohup moto_server -H 127.0.0.1 -p 9000 > "$RUNNER_TEMP/moto.log" 2>&1 &
|
||||
for i in $(seq 1 60); do (echo > /dev/tcp/127.0.0.1/9000) >/dev/null 2>&1 && break; sleep 1; done
|
||||
# the bucket has to exist - the venv has boto3 (s3 extra)
|
||||
python -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", aws_access_key_id="minioadmin", aws_secret_access_key="minioadmin").create_bucket(Bucket="borg")'
|
||||
export BORG_REPO="s3:minioadmin:minioadmin@http://127.0.0.1:9000/borg/binary-repo"
|
||||
python -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", aws_access_key_id="test", aws_secret_access_key="test").create_bucket(Bucket="borg")'
|
||||
export BORG_REPO="s3:test:test@http://127.0.0.1:9000/borg/binary-repo"
|
||||
export BORG_PASSPHRASE=test
|
||||
borg=dist/binary/borg-dir/borg.exe
|
||||
$borg repo-create -e aes256-ocb
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# The S3 server the CI tests run against: moto in server mode ("moto_server").
|
||||
# moto's own "server" extra pulls in the dependencies of all the AWS services it can
|
||||
# mock, for S3 the server only needs flask and flask-cors on top of the "s3" extra.
|
||||
moto[s3]==5.2.3
|
||||
flask==3.1.3
|
||||
flask-cors==6.0.5
|
||||
@@ -136,3 +136,35 @@ def test_s3_repo_basics(archiver):
|
||||
list_output = cmd(archiver, "repo-list")
|
||||
assert archive_name not in list_output
|
||||
cmd(archiver, "repo-delete")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not S3_URL, reason="BORG_TEST_S3_REPO not set.")
|
||||
def test_s3_repo_compact(archiver, monkeypatch):
|
||||
# compact rewrites a mixed pack via store.defrag, which for the S3 backend means ranged GETs of
|
||||
# the still used objects plus one PUT of the new pack. A server that ignores the requested range
|
||||
# goes unnoticed by test_s3_repo_basics, but makes compact fail here.
|
||||
# One big pack per create run: all of archive1's chunks land in a single pack, so that pack is
|
||||
# mixed (used + unused objects) once archive1 is deleted below.
|
||||
monkeypatch.setenv("BORG_PACK_MAX_COUNT", "1000")
|
||||
contents_kept = os.urandom(100 * 1024)
|
||||
create_regular_file(archiver.input_path, "file_dropped1", contents=os.urandom(10 * 1024))
|
||||
create_regular_file(archiver.input_path, "file_kept", contents=contents_kept)
|
||||
create_regular_file(archiver.input_path, "file_dropped2", contents=os.urandom(10 * 1024))
|
||||
# an own repository, test_s3_repo_basics might run at the same time (pytest-xdist)
|
||||
archiver.repository_location = S3_URL + "-compact"
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
cmd(archiver, "create", "archive1", "input")
|
||||
# archive2 references only file_kept's chunks, which deduplicate against archive1's pack
|
||||
os.remove(os.path.join(archiver.input_path, "file_dropped1"))
|
||||
os.remove(os.path.join(archiver.input_path, "file_dropped2"))
|
||||
cmd(archiver, "create", "archive2", "input")
|
||||
cmd(archiver, "delete", "-a", "archive1")
|
||||
# threshold 0: every pack with any unused bytes is rewritten
|
||||
output = cmd(archiver, "compact", "-v", "--threshold", "0")
|
||||
assert "Finished compaction" in output
|
||||
cmd(archiver, "check")
|
||||
with changedir("output"):
|
||||
cmd(archiver, "extract", "archive2")
|
||||
with open(os.path.join(archiver.output_path, "input", "file_kept"), "rb") as fd:
|
||||
assert fd.read() == contents_kept
|
||||
cmd(archiver, "repo-delete")
|
||||
|
||||
Reference in New Issue
Block a user