ci: Uprev deqp-runner and use "suite" support to merge softpipe runs.

The deqp-runner "suite" support lets us specify all the deqp variants at once and shard them across gitlab jobs, so we reduce docker overhead and get fewer jobs to review the output of. Acked-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com> Reviewed-by: Tomeu Vizoso <tomeu.vizoso@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12110>
2024-11-23 23:41:13 +00:00 · 2021-07-21 13:12:29 -07:00 · 2021-07-21 13:12:29 -07:00 · 4658a9a479
commit 4658a9a479
parent 1bdab45594
6 changed files with 121 additions and 82 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -407,7 +407,7 @@ debian/android_build:
 debian/x86_test-base:
  extends: debian/x86_build-base
  variables:
-    MESA_IMAGE_TAG: &debian-x86_test-base "2021-07-02-bump-libdrm"
+    MESA_IMAGE_TAG: &debian-x86_test-base "2021-07-30-deqp-runner"

 .use-debian/x86_test-base:
  extends:
@ -480,7 +480,7 @@ fedora/x86_build:
  variables:
    GIT_STRATEGY: fetch
    KERNEL_URL: *kernel-rootfs-url
-    MESA_ROOTFS_TAG: &kernel-rootfs "2021-08-02-rust-bump"
+    MESA_ROOTFS_TAG: &kernel-rootfs "2021-08-03-deqp-runner"
    DISTRIBUTION_TAG: &distribution-tag-arm "${MESA_ROOTFS_TAG}--${MESA_ARTIFACTS_TAG}--${MESA_TEMPLATES_COMMIT}"
  script:
    - .gitlab-ci/container/lava_build.sh
--- a/.gitlab-ci/container/build-deqp-runner.sh
+++ b/.gitlab-ci/container/build-deqp-runner.sh
@ -4,6 +4,6 @@ set -ex

 cargo install --locked deqp-runner \
  -j ${FDO_CI_CONCURRENT:-4} \
-  --version 0.7.2 \
+  --version 0.8.2 \
  --root /usr/local \
  $EXTRA_CARGO_ARGS
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@ -47,47 +47,51 @@ HANG_DETECTION_CMD=""

 # Generate test case list file.
 if [ "$DEQP_VER" = "vk" ]; then
-   cp /deqp/mustpass/vk-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/vk-$DEQP_VARIANT.txt
   DEQP=/deqp/external/vulkancts/modules/vulkan/deqp-vk
   HANG_DETECTION_CMD="/parallel-deqp-runner/build/bin/hang-detection"
 elif [ "$DEQP_VER" = "gles2" -o "$DEQP_VER" = "gles3" -o "$DEQP_VER" = "gles31" -o "$DEQP_VER" = "egl" ]; then
-   cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
   DEQP=/deqp/modules/$DEQP_VER/deqp-$DEQP_VER
   SUITE=dEQP
 elif [ "$DEQP_VER" = "gles2-khr" -o "$DEQP_VER" = "gles3-khr" -o "$DEQP_VER" = "gles31-khr" -o "$DEQP_VER" = "gles32-khr" ]; then
-   cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
   DEQP=/deqp/external/openglcts/modules/glcts
   SUITE=dEQP
 else
-   cp /deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt /tmp/case-list.txt
+   MUSTPASS=/deqp/mustpass/$DEQP_VER-$DEQP_VARIANT.txt
   DEQP=/deqp/external/openglcts/modules/glcts
   SUITE=KHR
 fi

-# If the caselist is too long to run in a reasonable amount of time, let the job
-# specify what fraction (1/n) of the caselist we should run.  Note: N~M is a gnu
-# sed extension to match every nth line (first line is #1).
-if [ -n "$DEQP_FRACTION" ]; then
-   sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt
-fi
+if [ -z "$DEQP_SUITE" ]; then
+    cp $MUSTPASS /tmp/case-list.txt

-# If the job is parallel at the gitab job level, take the corresponding fraction
-# of the caselist.
-if [ -n "$CI_NODE_INDEX" ]; then
-   sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
-fi
+    # If the caselist is too long to run in a reasonable amount of time, let the job
+    # specify what fraction (1/n) of the caselist we should run.  Note: N~M is a gnu
+    # sed extension to match every nth line (first line is #1).
+    if [ -n "$DEQP_FRACTION" ]; then
+       sed -ni 1~$DEQP_FRACTION"p" /tmp/case-list.txt
+    fi

-if [ -n "$DEQP_CASELIST_FILTER" ]; then
-    sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt
-fi
+    # If the job is parallel at the gitab job level, take the corresponding fraction
+    # of the caselist.
+    if [ -n "$CI_NODE_INDEX" ]; then
+       sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
+    fi

-if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then
-    sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt
-fi
+    if [ -n "$DEQP_CASELIST_FILTER" ]; then
+        sed -ni "/$DEQP_CASELIST_FILTER/p" /tmp/case-list.txt
+    fi

-if [ ! -s /tmp/case-list.txt ]; then
-    echo "Caselist generation failed"
-    exit 1
+    if [ -n "$DEQP_CASELIST_INV_FILTER" ]; then
+        sed -ni "/$DEQP_CASELIST_INV_FILTER/!p" /tmp/case-list.txt
+    fi
+
+    if [ ! -s /tmp/case-list.txt ]; then
+        echo "Caselist generation failed"
+        exit 1
+    fi
 fi

 if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then
@ -116,25 +120,6 @@ if [ -n "$DEQP_NO_SAVE_RESULTS" ]; then
   SUMMARY_LIMIT="--summary-limit 0"
 fi

-run_cts() {
-    deqp=$1
-    caselist=$2
-    output=$3
-    deqp-runner \
-        run \
-        --deqp $deqp \
-        --output $RESULTS \
-        --caselist $caselist \
-        --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \
-        --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
-        --testlog-to-xml  /deqp/executor/testlog-to-xml \
-        $JOB \
-        $SUMMARY_LIMIT \
-	$DEQP_RUNNER_OPTIONS \
-        -- \
-        $DEQP_OPTIONS
-}
-
 parse_renderer() {
    RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
    VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
@ -214,7 +199,35 @@ FAILURES_CSV=$RESULTS/failures.csv

 export LD_PRELOAD=$TEST_LD_PRELOAD

-run_cts $DEQP /tmp/case-list.txt $RESULTS_CSV
+if [ -z "$DEQP_SUITE" ]; then
+    deqp-runner \
+        run \
+        --deqp $DEQP \
+        --output $RESULTS \
+        --caselist /tmp/case-list.txt \
+        --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \
+        --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
+        --testlog-to-xml /deqp/executor/testlog-to-xml \
+        $JOB \
+        $SUMMARY_LIMIT \
+	$DEQP_RUNNER_OPTIONS \
+        -- \
+        $DEQP_OPTIONS
+else
+    deqp-runner \
+        suite \
+        --suite $INSTALL/deqp-$DEQP_SUITE.toml \
+        --output $RESULTS \
+        --skips $INSTALL/deqp-all-skips.txt $DEQP_SKIPS \
+        --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
+        --testlog-to-xml /deqp/executor/testlog-to-xml \
+        --fraction-start $CI_NODE_INDEX \
+        --fraction $CI_NODE_TOTAL \
+        $JOB \
+        $SUMMARY_LIMIT \
+	$DEQP_RUNNER_OPTIONS
+fi
+
 DEQP_EXITCODE=$?

 export LD_PRELOAD=
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@ -38,6 +38,7 @@ cp -Rp .gitlab-ci/deqp-*.txt install/
 cp -Rp .gitlab-ci/report-flakes.py install/
 cp -Rp .gitlab-ci/vkd3d-proton install/
 find . -path \*/ci/\*.txt \
+    -o -path \*/ci/\*.toml \
    -o -path \*/ci/\*traces\*.yml \
    | xargs -I '{}' cp -p '{}' install/

--- a/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml
+++ b/src/gallium/drivers/softpipe/ci/deqp-softpipe.toml
@ -0,0 +1,51 @@
+[[deqp]]
+deqp = "/deqp/modules/gles2/deqp-gles2"
+caselists = ["/deqp/mustpass/gles2-master.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+[[deqp]]
+deqp = "/deqp/modules/gles3/deqp-gles3"
+caselists = ["/deqp/mustpass/gles3-master.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+[[deqp]]
+deqp = "/deqp/modules/gles31/deqp-gles31"
+caselists = ["/deqp/mustpass/gles31-master.txt"]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
+
+# Note that KHR-GL3* test sets include all tests from the previous
+# version, so we only need to run one test list (unlike dEQP-GLES,
+# where the test sets are separate).
+[[deqp]]
+deqp = "/deqp/external/openglcts/modules/glcts"
+caselists = [
+    "/deqp/mustpass/gles2-khr-master.txt",
+    "/deqp/mustpass/gles3-khr-master.txt",
+    "/deqp/mustpass/gles31-khr-master.txt",
+    "/deqp/mustpass/gl33-master.txt",
+]
+deqp_args = [
+    "--deqp-surface-width=256",
+    "--deqp-surface-height=256",
+    "--deqp-surface-type=pbuffer",
+    "--deqp-gl-config-name=rgba8888d24s8ms0",
+    "--deqp-visibility=hidden"
+]
--- a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
@ -1,38 +1,21 @@
-softpipe-gles2:
+.softpipe-deqp-test:
  extends:
-    - llvmpipe-gles2
    - .softpipe-rules
+    - .test-gl
+    - .deqp-test
  variables:
    GPU_VERSION: softpipe
+    LIBGL_ALWAYS_SOFTWARE: "true"
    GALLIUM_DRIVER: "softpipe"
    DEQP_EXPECTED_RENDERER: softpipe
    FLAKES_CHANNEL: "#mesa-swrast-ci"

-softpipe-gles3:
+softpipe-deqp:
  variables:
-    DEQP_VER: gles3
-  extends: softpipe-gles2
-
-softpipe-gles31:
+    DEQP_VER: gles2 # for the renderer check
+    DEQP_SUITE: softpipe
  parallel: 2
-  variables:
-    DEQP_VER: gles31
-  extends: softpipe-gles2
-
-softpipe-khr-gles2:
-  variables:
-    DEQP_VER: gles2-khr
-  extends: softpipe-gles2
-
-softpipe-khr-gles3:
-  variables:
-    DEQP_VER: gles3-khr
-  extends: softpipe-gles2
-
-softpipe-khr-gles31:
-  variables:
-    DEQP_VER: gles31-khr
-  extends: softpipe-gles2
+  extends: .softpipe-deqp-test

 softpipe-asan-gles31:
  variables:
@ -40,7 +23,7 @@ softpipe-asan-gles31:
    DEQP_FRACTION: 10
    DEQP_VER: gles31
    TEST_LD_PRELOAD: libasan.so.6
-  extends: softpipe-gles2
+  extends: .softpipe-deqp-test
  needs:
    - debian/x86_test-gl
    - debian-testing-asan
@ -86,12 +69,3 @@ softpipe-piglit-quick:
      -x glsl-uniform-interstage-limits
    PIGLIT_PROFILES: quick_gl quick_shader
    PIGLIT_RESULTS: softpipe-quick
-
-# Note that KHR-GL3* test sets include all tests from the previous
-# version, so we only need to run one test list (unlike dEQP-GLES,
-# where the test sets are separate).
-softpipe-gl:
-  variables:
-    DEQP_VER: gl33
-  extends:
-    - softpipe-gles2