From 2ae65f578be9f0a12bcd035c9b8c897bd72b0e3e Mon Sep 17 00:00:00 2001 From: Benjamin Tissoires Date: Thu, 13 Oct 2022 21:25:38 +0200 Subject: [PATCH] CI: convert to use the new S3 server instead of the legacy minio We don't need to login anymore, but we can't use plain minio commands now. `ci-fairy` got a helper as `s3cp` to keep an almost identical API. Solved Conflicts: .gitlab-ci/common/init-stage2.sh .gitlab-ci/container/lava_build.sh .gitlab-ci/prepare-artifacts.sh src/amd/ci/traces-amd.yml src/freedreno/ci/traces-freedreno.yml src/gallium/frontends/lavapipe/ci/traces-lavapipe.yml Signed-off-by: Benjamin Tissoires (cherry picked from commit 67cee534a88c95a8eb6839f7bcf28a5e6dac8fbf) Part-of: --- .gitlab-ci.yml | 6 ++---- .gitlab-ci/common/init-stage2.sh | 3 +-- .gitlab-ci/container/lava_build.sh | 7 +++---- .gitlab-ci/lava/lava-submit.sh | 3 +-- .gitlab-ci/piglit/piglit-traces.sh | 8 +++----- .gitlab-ci/prepare-artifacts.sh | 3 +-- .gitlab-ci/test/gitlab-ci.yml | 2 +- docs/ci/bare-metal.rst | 6 +++--- src/broadcom/ci/traces-broadcom.yml | 2 +- src/freedreno/ci/gitlab-ci.yml | 2 +- src/freedreno/ci/restricted-traces-freedreno.yml | 2 +- src/gallium/drivers/crocus/ci/traces-crocus.yml | 2 +- src/gallium/drivers/i915/ci/traces-i915.yml | 2 +- src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml | 2 +- src/gallium/drivers/virgl/ci/traces-virgl-iris.yml | 2 +- src/gallium/drivers/virgl/ci/traces-virgl.yml | 2 +- src/intel/ci/traces-iris.yml | 2 +- src/panfrost/ci/traces-panfrost.yml | 2 +- 18 files changed, 25 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38455f1c860..b922852eac7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: rm download-git-cache.sh set +o xtrace CI_JOB_JWT_FILE: /minio_jwt - MINIO_HOST: minio-packet.freedesktop.org + MINIO_HOST: s3.freedesktop.org # per-pipeline artifact storage on MinIO PIPELINE_ARTIFACTS_BASE: ${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID} # per-job artifact storage on MinIO @@ -262,9 +262,7 @@ make git archive: # compress the current folder - tar -cvzf ../$CI_PROJECT_NAME.tar.gz . - # login with the JWT token file - - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" - - ci-fairy minio cp ../$CI_PROJECT_NAME.tar.gz minio://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz + - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$MINIO_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz # Sanity checks of MR settings and commit logs diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh index ff73ec8a06f..beaee1de727 100755 --- a/.gitlab-ci/common/init-stage2.sh +++ b/.gitlab-ci/common/init-stage2.sh @@ -150,8 +150,7 @@ cleanup # upload artifacts if [ -n "$MINIO_RESULTS_UPLOAD" ]; then tar -czf results.tar.gz results/; - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"; - ci-fairy minio cp results.tar.gz minio://"$MINIO_RESULTS_UPLOAD"/results.tar.gz; + ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" results.tar.gz https://"$MINIO_RESULTS_UPLOAD"/results.tar.gz; fi # We still need to echo the hwci: mesa message, as some scripts rely on it, such diff --git a/.gitlab-ci/container/lava_build.sh b/.gitlab-ci/container/lava_build.sh index 776a055a8e9..f9e9e2ac6d6 100755 --- a/.gitlab-ci/container/lava_build.sh +++ b/.gitlab-ci/container/lava_build.sh @@ -244,7 +244,6 @@ popd . .gitlab-ci/container/container_post_build.sh ############### Upload the files! -ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" FILES_TO_UPLOAD="lava-rootfs.tgz \ $KERNEL_IMAGE_NAME" @@ -253,9 +252,9 @@ if [[ -n $DEVICE_TREES ]]; then fi for f in $FILES_TO_UPLOAD; do - ci-fairy minio cp /lava-files/$f \ - minio://${MINIO_PATH}/$f + ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \ + https://${MINIO_PATH}/$f done touch /lava-files/done -ci-fairy minio cp /lava-files/done minio://${MINIO_PATH}/done +ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/done https://${MINIO_PATH}/done diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index a3affe1be66..9b420a1de74 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -23,8 +23,7 @@ KERNEL_IMAGE_BASE_URL="https://${BASE_SYSTEM_HOST_PATH}" \ artifacts/ci-common/generate-env.sh > results/job-rootfs-overlay/set-job-env-vars.sh tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ . -ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" -ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}" +ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}" touch results/lava.log tail -f results/lava.log & diff --git a/.gitlab-ci/piglit/piglit-traces.sh b/.gitlab-ci/piglit/piglit-traces.sh index 0b1438498d9..61658d6c38c 100755 --- a/.gitlab-ci/piglit/piglit-traces.sh +++ b/.gitlab-ci/piglit/piglit-traces.sh @@ -3,7 +3,7 @@ set -ex INSTALL=$(realpath -s "$PWD"/install) -MINIO_ARGS="--credentials=/tmp/.minio_credentials" +MINIO_ARGS="--token-file ${CI_JOB_JWT_FILE}" RESULTS=$(realpath -s "$PWD"/results) mkdir -p "$RESULTS" @@ -103,8 +103,8 @@ replay_minio_upload_images() { __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}" fi - ci-fairy minio cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \ - "minio://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" + ci-fairy s3cp $MINIO_ARGS "$RESULTS/$__PREFIX/$line" \ + "https://${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" done } @@ -139,8 +139,6 @@ if [ "$RUN_CMD_WRAPPER" ]; then RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e" fi -ci-fairy minio login $MINIO_ARGS --token-file "${CI_JOB_JWT_FILE}" - # The replayer doesn't do any size or checksum verification for the traces in # the replayer db, so if we had to restart the system due to intermittent device # errors (or tried to cache replayer-db between runs, which would be nice to diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index 46d4ab0e3a6..25deae6b6cd 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -54,6 +54,5 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then # Pass needed files to the test stage MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz" gzip -c artifacts/install.tar > ${MINIO_ARTIFACT_NAME} - ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}" - ci-fairy minio cp ${MINIO_ARTIFACT_NAME} minio://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME} + ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME} fi diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml index c05c0b49d1b..eba62eb79a6 100644 --- a/.gitlab-ci/test/gitlab-ci.yml +++ b/.gitlab-ci/test/gitlab-ci.yml @@ -79,7 +79,7 @@ - results/summary/ - results/*.txt variables: - PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-public --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE} script: - install/piglit/piglit-traces.sh diff --git a/docs/ci/bare-metal.rst b/docs/ci/bare-metal.rst index 0c8ecb4f3eb..aff81e7a881 100644 --- a/docs/ci/bare-metal.rst +++ b/docs/ci/bare-metal.rst @@ -219,12 +219,12 @@ Enable the site and restart nginx: sudo service nginx restart # First download will hit the internet - wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace + wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace # Second download should be cached. - wget http://localhost/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace + wget http://localhost/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/itoral-gl-terrain-demo/demo.trace Now, set ``download-url`` in your ``traces-*.yml`` entry to something like -``http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public`` +``http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public`` and you should have cached downloads for traces. Add it to ``FDO_HTTP_CACHE_URI=`` in your ``config.toml`` runner environment lines and you can use it for cached artifact downloads instead of going all the way to diff --git a/src/broadcom/ci/traces-broadcom.yml b/src/broadcom/ci/traces-broadcom.yml index 43229c7703c..ea92a16018e 100644 --- a/src/broadcom/ci/traces-broadcom.yml +++ b/src/broadcom/ci/traces-broadcom.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "http://192.168.40.131:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://192.168.40.131:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: - path: 0ad/0ad.trace diff --git a/src/freedreno/ci/gitlab-ci.yml b/src/freedreno/ci/gitlab-ci.yml index 757a46e28ca..ef4dc1d0987 100644 --- a/src/freedreno/ci/gitlab-ci.yml +++ b/src/freedreno/ci/gitlab-ci.yml @@ -271,7 +271,7 @@ a630-traces-restricted: - .freedreno-rules-restricted variables: PIGLIT_REPLAY_DESCRIPTION_FILE: "/install/restricted-traces-freedreno.yml" - PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_host=minio-packet.freedesktop.org --minio_bucket=mesa-tracie-private --role-session-name=${CI_PROJECT_PATH}:${CI_JOB_ID} --jwt-file=${CI_JOB_JWT_FILE} + PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-private --jwt-file=${CI_JOB_JWT_FILE} allow_failure: true a630-traces-performance: diff --git a/src/freedreno/ci/restricted-traces-freedreno.yml b/src/freedreno/ci/restricted-traces-freedreno.yml index 36e5ba336b9..6b7e175d2d9 100644 --- a/src/freedreno/ci/restricted-traces-freedreno.yml +++ b/src/freedreno/ci/restricted-traces-freedreno.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "http://10.42.0.1:8888/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-private/" + download-url: "http://10.42.0.1:8888/cache/?uri=https://s3.freedesktop.org/mesa-tracie-private/" traces: # takes approx. 70 seconds diff --git a/src/gallium/drivers/crocus/ci/traces-crocus.yml b/src/gallium/drivers/crocus/ci/traces-crocus.yml index f6ff283f9fe..d5c6a4a5044 100644 --- a/src/gallium/drivers/crocus/ci/traces-crocus.yml +++ b/src/gallium/drivers/crocus/ci/traces-crocus.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: - path: 0ad/0ad-v2.trace diff --git a/src/gallium/drivers/i915/ci/traces-i915.yml b/src/gallium/drivers/i915/ci/traces-i915.yml index ca6c3afa33d..401b964e6ab 100644 --- a/src/gallium/drivers/i915/ci/traces-i915.yml +++ b/src/gallium/drivers/i915/ci/traces-i915.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" # glmark2 looks like maybe it fails due to wanting an xserver, but we don't have # Xorg in the x86 docker images we're using. diff --git a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml index 52760a4ec41..5218334e5d6 100644 --- a/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml +++ b/src/gallium/drivers/llvmpipe/ci/traces-llvmpipe.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: - path: 0ad/0ad-v2.trace diff --git a/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml b/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml index 27ee05e573c..afe9915ae2f 100644 --- a/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml +++ b/src/gallium/drivers/virgl/ci/traces-virgl-iris.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: # Crash in crosvm: ../src/dispatch_common.c:858: epoxy_get_proc_address: Assertion `0 && "Couldn't find current GLX or EGL context.\n"' failed. diff --git a/src/gallium/drivers/virgl/ci/traces-virgl.yml b/src/gallium/drivers/virgl/ci/traces-virgl.yml index 82396976550..17bf1e3f2cf 100644 --- a/src/gallium/drivers/virgl/ci/traces-virgl.yml +++ b/src/gallium/drivers/virgl/ci/traces-virgl.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "https://s3.freedesktop.org/mesa-tracie-public/" traces: - path: glmark2/desktop:windows=4:effect=blur:blur-radius=5:passes=1:separable=true-v2.trace diff --git a/src/intel/ci/traces-iris.yml b/src/intel/ci/traces-iris.yml index 65896b398e4..d4b948a4d99 100644 --- a/src/intel/ci/traces-iris.yml +++ b/src/intel/ci/traces-iris.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: # - path: 0ad/0ad-v2.trace diff --git a/src/panfrost/ci/traces-panfrost.yml b/src/panfrost/ci/traces-panfrost.yml index 8a19dacba99..4bf851189b0 100644 --- a/src/panfrost/ci/traces-panfrost.yml +++ b/src/panfrost/ci/traces-panfrost.yml @@ -1,5 +1,5 @@ traces-db: - download-url: "http://caching-proxy/cache/?uri=https://minio-packet.freedesktop.org/mesa-tracie-public/" + download-url: "http://caching-proxy/cache/?uri=https://s3.freedesktop.org/mesa-tracie-public/" traces: - path: behdad-glyphy/glyphy-v2.trace