qwen-code/.github/workflows/e2e.yml

name: 'E2E Tests'

permissions:
  contents: 'read'

on:
  # E2E is slow and currently flaky, so it is NOT in the merge queue (gating the
  # serial queue on it would stall every merge). It runs post-merge on `main`,
  # plus a nightly full regression and on-demand. Promote it to `merge_group` +
  # required once a stable subset is carved out.
  push:
    branches:
      - 'main'
      - 'feat/e2e/**'
  schedule:
    - cron: '0 4 * * *' # nightly full regression (~04:00 UTC), guaranteed signal
  workflow_dispatch:

concurrency:
  # Scope the group by event so pushes to `main` coalesce with each other without
  # ever touching the nightly schedule or a manual dispatch — they share the ref
  # but not the event. feat/e2e/** pushes coalesce within their own branch.
  group: |-
    ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }}
  # Do NOT cancel in-progress runs on `main`: a full run takes ~40min while
  # merges land every ~18min (median), so cancelling on every merge starved the
  # suite — over 100 push runs, 67 were cancelled and only 25 ever reported.
  # With cancellation off, GitHub keeps at most ONE pending run per group and
  # cancels the previously pending one, so the queue collapses to the newest
  # tree on its own: the in-flight run always finishes, and each result covers
  # the batch of commits merged since the last one (bisect that range when it
  # goes red). Dev branches keep cancelling superseded runs — nobody bisects
  # those, and the latest push is the only tree of interest.
  cancel-in-progress: |-
    ${{ github.event_name == 'push' && github.ref_name != 'main' }}

jobs:
  # One build feeds every leg. Each of the eleven legs used to build and
  # bundle on its own runner: 4–8 minutes on a hosted VM, 10–17 on a busy
  # pool host, so the six Linux shards spent more of the shared pool
  # rebuilding one commit than testing it. The outputs are platform-
  # independent JavaScript (native modules come from each leg's own
  # `npm ci`), so a hosted VM builds once, off the pool, and every leg
  # downloads the result. See .github/scripts/e2e-build-pack.sh.
  build:
    name: 'Build for E2E'
    runs-on: 'ubuntu-latest'
    timeout-minutes: 30
    # Same fork gate as the legs, so a skipped build skips them all for the
    # same reason rather than failing them on a missing artifact.
    if: |-
      ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          package-manager-cache: false
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Cache pnpm store (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: './.github/actions/pnpm-store-cache'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # The install must not build; the two steps below do it explicitly.
          QWEN_SKIP_PREPARE: '1'
        run: |-
          corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only

      - name: 'Build project'
        run: |-
          npm run build

      - name: 'Bundle CLI for E2E tests'
        run: |-
          npm run bundle

      - name: 'Pack build outputs'
        run: 'bash .github/scripts/e2e-build-pack.sh "${RUNNER_TEMP}/e2e-build.tar.gz"'

      # One bounded retry, same ungated contract as the sandbox-image
      # build retry (run-e2e-tests.sh) and the image build retry
      # (build-and-publish-image.yml): upload-artifact dying with
      # "Upload progress stalled." is a transient runner-to-blobstore
      # network class — run 34208365262 stalled ten minutes on the ~100MB
      # archive and reded the whole run before any leg started. A
      # deterministic failure (missing archive, via if-no-files-found:
      # error) fails both attempts and keeps the job red. The first
      # attempt's continue-on-error keeps that failure from failing the
      # job before the retry runs; the retry's overwrite covers the
      # finalize-then-fail window: v4+ 409s a same-name upload only
      # against an artifact already finalized in this run attempt, and
      # a stall aborts before finalize and reserves nothing — but an
      # attempt that finalized e2e-build and only then reported failure
      # would 409 the retry without it.
      - name: 'Upload build artifact'
        id: 'upload-build'
        uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1
        continue-on-error: true
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build.tar.gz'
          if-no-files-found: 'error'
          # Already gzip-compressed.
          compression-level: 0
          retention-days: 1

      - name: 'Upload build artifact (retry)'
        if: "${{ steps.upload-build.outcome == 'failure' }}"
        uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build.tar.gz'
          if-no-files-found: 'error'
          # Already gzip-compressed.
          compression-level: 0
          retention-days: 1
          overwrite: true

      # An absorbed stall must stay countable: a recovered upload leaves
      # this job's conclusion green, so the main-CI failure tracker (which
      # counts conclusion == "failure" jobs) never records it. The warning
      # annotation keeps it on the check-run record without reddening the
      # job; placed after the retry, the implicit success() gate scopes it
      # to the absorbed case — a double failure reddens the job directly.
      - name: 'Announce absorbed upload failure'
        if: "${{ steps.upload-build.outcome == 'failure' }}"
        run: |-
          echo "::warning::Upload build artifact failed its first attempt and the retry absorbed it — count this toward the #11364 upload-stall recurrence threshold."

  e2e-test-linux:
    name: 'E2E Test (Linux) - ${{ matrix.sandbox }} - shard ${{ matrix.shard }}'
    # Persistent ECS pool with hosted fallback: every trigger here is a
    # trusted context (push to in-repo branches, the nightly schedule, and
    # manual dispatch — no pull_request trigger), and the six-shard matrix
    # at ~16 minutes each was one of the largest remaining hosted consumers.
    # The pool ships docker (measured by qwen-autofix's sandboxed jobs; see
    # the runtime preflight below, per #9556).
    # Kill-switch: MAINTAINER_ECS_RUNNER_DISABLED.
    runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}'
    # A wedged shard must free the pool runner well before GitHub's
    # 360-minute default; 60 matches ci.yml's pool jobs.
    timeout-minutes: 60
    needs:
      - 'build'
    # Skip on fork PRs: forks have no access to repository secrets
    # (OPENAI_*, DOCKERHUB_*), so the matrix would fail unconditionally
    # and show misleading red status. Same-repo PRs run normally.
    if: |-
      ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    env:
      E2E_CONTAINER_OWNER: '${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.shard }}'
    strategy:
      # One failing shard must not cancel the others: a partial matrix hides the
      # rest of the suite and reports an incomplete failure set.
      fail-fast: false
      matrix:
        sandbox:
          - 'sandbox:none'
          - 'sandbox:docker'
        # Keep three-way test concurrency inside one runner so each sandbox
        # shares setup work instead of occupying three pool runners.
        shard:
          - '1/1'
        node-version:
          - '22.x'
    steps:
      # The sandbox:none retry gate below budgets against the whole 60-minute
      # job, not just the test step — setup spent ~29 of those minutes on two
      # of the runners that motivated the retry. Recording the start here is
      # what lets the gate see that spend.
      - name: 'Record job start epoch'
        run: 'echo "E2E_JOB_START_EPOCH=$(date +%s)" >> "${GITHUB_ENV}"'

      # Fail-fast capability preflight (#9556): the docker leg needs a live
      # daemon, and without this check a missing one surfaces only after
      # npm ci + build. Runs on hosted and pool runners alike — both are
      # expected to answer.
      - name: 'Check container runtime'
        if: |-
          ${{ matrix.sandbox == 'sandbox:docker' }}
        run: |-
          set -uo pipefail
          if ! docker info > /dev/null 2>&1; then
            echo "::error::docker daemon is not reachable on ${RUNNER_NAME:-this runner}; the sandbox:docker leg cannot run here."
            exit 1
          fi

      # Pre-checkout heal, mirroring ci.yml: the docker leg's containers can
      # leave root-owned files in the reused pool workspace, and the next
      # checkout's clean would die on EACCES. Hosted runners are ephemeral
      # and never take this branch.
      - name: 'Restore workspace ownership'
        if: "${{ runner.environment == 'self-hosted' }}"
        run: |-
          set -uo pipefail
          RUNNER_UID="$(id -u)"
          RUNNER_GID="$(id -g)"
          if [ "$RUNNER_UID" != "0" ]; then
            chown -R "$RUNNER_UID:$RUNNER_GID" "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chown -R "$RUNNER_UID:$RUNNER_GID" "$GITHUB_WORKSPACE" || echo "::warning::could not restore workspace ownership; checkout may fail on leftover root-owned files"
            # The docker leg's host-side locks live in the runner's home: a
            # root-owned leftover under ~/.cache/qwen-code-ci fails every
            # `exec 9>` against them with EACCES before a test runs (#11990).
            if [ -d "${HOME}/.cache/qwen-code-ci" ]; then
              chown -R "$RUNNER_UID:$RUNNER_GID" "${HOME}/.cache/qwen-code-ci" 2>/dev/null || sudo -n chown -R "$RUNNER_UID:$RUNNER_GID" "${HOME}/.cache/qwen-code-ci" || echo "::warning::could not restore CI cache ownership; docker sandbox locks may stay unreadable"
            fi
          fi
          chmod -R u+rwX "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chmod -R u+rwX "$GITHUB_WORKSPACE" || echo "::warning::could not restore workspace write permissions; checkout may fail on leftover read-only files"

      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      # Hosted-only, mirroring ci.yml: on the pool, nodejs.org may be
      # unreachable through the egress proxy, and the action's post step
      # uploads the npm cache to GitHub — a save that ran 14+ minutes over
      # the pool's slow egress on security-checks' first pool run
      # (2026-08-26). The pool reuses the machine's Node and its persistent
      # npm cache instead.
      - name: 'Set up Node.js ${{ matrix.node-version }} (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version: '${{ matrix.node-version }}'
          package-manager-cache: false
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Cache pnpm store (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: './.github/actions/pnpm-store-cache'

      - name: 'Use pre-installed Node.js (self-hosted)'
        if: "${{ runner.environment == 'self-hosted' }}"
        uses: './.github/actions/self-hosted-node'

      - name: 'Configure persistent npm cache (self-hosted)'
        if: "${{ runner.environment == 'self-hosted' }}"
        run: |-
          cache_dir="${HOME}/.cache/qwen-code/npm"
          mkdir -p "${cache_dir}"
          echo "NPM_CONFIG_CACHE=${cache_dir}" >> "${GITHUB_ENV}"
          echo "Using persistent npm cache at ${cache_dir}"
          du -sh "${cache_dir}" 2>/dev/null || true

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # `npm ci` runs the `prepare` script, which builds and bundles the
          # whole workspace; this leg unpacks the `build` job's outputs
          # instead. (The web-shell job below has no build step of its own,
          # so it keeps building during install.)
          QWEN_SKIP_PREPARE: '1'
        run: |-
          corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only

      # The bundle and workspace dist/ trees come from the `build` job.
      # One bounded retry, same transient runner-to-blobstore contract as
      # the build job's upload retry (#11364): run 35250857883's macOS
      # shard 2/2 died in this step with the build job green behind it
      # (#12125). A deterministic failure — the artifact missing — fails
      # both attempts and keeps the leg red.
      # The first attempt is time-boxed: an absorbed stall otherwise burns
      # the whole job budget before the retry ever runs.
      - name: 'Download build artifact'
        id: 'download-build'
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        continue-on-error: true
        timeout-minutes: 10
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      - name: 'Download build artifact (retry)'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      # An absorbed download stall leaves the leg green, so the main-CI
      # failure tracker never records it — the warning annotation keeps the
      # recurrence countable, mirroring the build job's upload announce.
      - name: 'Announce absorbed download failure'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        run: |-
          echo "::warning::Download build artifact failed its first attempt and the retry absorbed it — count this toward the #12125 download-stall recurrence watch."

      - name: 'Unpack build artifact'
        run: 'bash .github/scripts/e2e-build-unpack.sh "${RUNNER_TEMP}/e2e-build/e2e-build.tar.gz"'

      - name: 'Set up Docker'
        if: |-
          ${{ matrix.sandbox == 'sandbox:docker' && runner.environment == 'github-hosted' }}
        uses: 'docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5' # ratchet:docker/setup-buildx-action@v4

      - name: 'Set up Podman'
        if: |-
          ${{ matrix.sandbox == 'sandbox:podman' }}
        uses: 'redhat-actions/podman-login@4934294ad0449894bcd1e9f191899d7292469603' # ratchet:redhat-actions/podman-login@v1
        with:
          registry: 'docker.io'
          username: '${{ secrets.DOCKERHUB_USERNAME }}'
          password: '${{ secrets.DOCKERHUB_TOKEN }}'

      - name: 'Run E2E tests'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
          KEEP_OUTPUT: 'true'
          VERBOSE: 'true'
          # Mapped for integration-tests/vitest.config.ts, which exempts
          # pressure-flake unhandled errors on shared-pool runners. The test
          # commands below override its one-fork default for this job.
          RUNNER_ENVIRONMENT: '${{ runner.environment }}'
          QWEN_SANDBOX: "${{ matrix.sandbox == 'sandbox:docker' && 'docker' || 'false' }}"
          BUILD_SANDBOX_FLAGS: '--label org.qwen-code.ci.sandbox=true'
          SANDBOX_FLAGS: '--label org.qwen-code.ci.owner=${E2E_CONTAINER_OWNER}'
        run: |-
          exec bash .github/scripts/run-e2e-tests.sh '${{ matrix.sandbox }}' '${{ matrix.shard }}'

      - name: 'Remove job-owned E2E containers'
        if: |-
          ${{ always() && matrix.sandbox == 'sandbox:docker' && runner.environment == 'self-hosted' }}
        run: |-
          set -euo pipefail
          container_ids="$(timeout 30 docker ps -aq --filter "label=org.qwen-code.ci.owner=${E2E_CONTAINER_OWNER}")"
          if [ -n "$container_ids" ]; then
            printf '%s\n' "$container_ids" | xargs -r timeout 60 docker rm -f > /dev/null || true
          fi
          remaining="$(timeout 30 docker ps -aq --filter "label=org.qwen-code.ci.owner=${E2E_CONTAINER_OWNER}")"
          if [ -n "$remaining" ]; then
            echo "::error::E2E containers remain for ${E2E_CONTAINER_OWNER}: ${remaining//$'\n'/,}"
            exit 1
          fi

      # Commit-qualified CI images are pruned under the exclusive daemon lock
      # once they are older than a day. The cleanup is non-blocking so a shard
      # finishing early cannot queue a writer ahead of another shard's reader.
      - name: 'Prune dangling docker images'
        if: |-
          ${{ always() && matrix.sandbox == 'sandbox:docker' && runner.environment == 'self-hosted' }}
        run: |-
          mkdir -p "${HOME}/.cache/qwen-code-ci"
          exec 9>"${HOME}/.cache/qwen-code-ci/docker-sandbox-daemon.lock"
          if flock --nonblock 9; then
            docker image prune --all --force --filter 'label=org.qwen-code.ci.sandbox=true' --filter 'until=24h' 9>&- || echo "::warning::old CI sandbox image cleanup failed on ${RUNNER_NAME:-this runner}"
            docker image prune --force --filter 'until=24h' 9>&- || echo "::warning::docker image prune failed on ${RUNNER_NAME:-this runner}; dangling images may accumulate"
          else
            echo "Docker cleanup skipped because the shared daemon is active"
          fi

  e2e-test-macos:
    name: 'E2E Test - macOS - shard ${{ matrix.shard }}'
    runs-on: 'macos-latest'
    needs:
      - 'build'
    # Skip on fork PRs (no secrets) — see e2e-test-linux above.
    if: |-
      ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    strategy:
      fail-fast: false
      matrix:
        # Two shards, not three: the macOS runner is the slowest to install and
        # build, so a third shard would add more fixed cost than it removes.
        shard:
          - '1/2'
          - '2/2'
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          package-manager-cache: false
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Cache pnpm store (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: './.github/actions/pnpm-store-cache'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # The install must not build; this leg unpacks the `build` job's
          # outputs instead.
          QWEN_SKIP_PREPARE: '1'
        run: |-
          corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only

      # The bundle and workspace dist/ trees come from the `build` job.
      # One bounded retry, same transient runner-to-blobstore contract as
      # the build job's upload retry (#11364): run 35250857883's macOS
      # shard 2/2 died in this step with the build job green behind it
      # (#12125). A deterministic failure — the artifact missing — fails
      # both attempts and keeps the leg red.
      # The first attempt is time-boxed: an absorbed stall otherwise burns
      # the whole job budget before the retry ever runs.
      - name: 'Download build artifact'
        id: 'download-build'
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        continue-on-error: true
        timeout-minutes: 10
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      - name: 'Download build artifact (retry)'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      # An absorbed download stall leaves the leg green, so the main-CI
      # failure tracker never records it — the warning annotation keeps the
      # recurrence countable, mirroring the build job's upload announce.
      - name: 'Announce absorbed download failure'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        run: |-
          echo "::warning::Download build artifact failed its first attempt and the retry absorbed it — count this toward the #12125 download-stall recurrence watch."

      - name: 'Unpack build artifact'
        run: 'bash .github/scripts/e2e-build-unpack.sh "${RUNNER_TEMP}/e2e-build/e2e-build.tar.gz"'

      - name: 'Run E2E tests'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
        run: 'npx cross-env QWEN_E2E_RENDERER=ink VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --exclude "**/chat-transcript-document.test.ts" --shard="${{ matrix.shard }}"'

  e2e-interactive-opentui:
    name: 'E2E Interactive - OpenTUI renderer (bun)'
    runs-on: 'ubuntu-latest'
    # Same rationale as e2e-test-linux: a wedged run must free the runner.
    timeout-minutes: 60
    # Skip on fork PRs (no secrets) — see e2e-test-linux above.
    needs:
      - 'build'
    if: |-
      ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          package-manager-cache: false
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Cache pnpm store (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: './.github/actions/pnpm-store-cache'

      # The opentui renderer leg runs the CLI bundle under bun: the locked
      # @opentui/core loads its native renderer through bun:ffi, and the CI
      # node build cannot load node:ffi. The renderer matrix also pins
      # QWEN_TUI_RENDERER=opentui (plus QWEN_TUI_RENDERER_STRICT) for every
      # spawned CLI, so a boot that silently fell back to ink fails the leg
      # instead of passing as a false green. The version matches
      # DEFAULT_BUN_VERSION in scripts/build-standalone-release.js (the
      # --runtime=bun archive flavor) and is pinned, not 'latest', so an
      # upstream Bun release cannot flip this gating leg red on its own.
      - name: 'Set up Bun'
        uses: 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' # v2.2.0
        with:
          bun-version: '1.3.14'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # The install must not build; this leg unpacks the `build` job's
          # outputs instead.
          QWEN_SKIP_PREPARE: '1'
        run: |-
          corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only

      # The bundle and workspace dist/ trees come from the `build` job.
      # One bounded retry, same transient runner-to-blobstore contract as
      # the build job's upload retry (#11364): run 35250857883's macOS
      # shard 2/2 died in this step with the build job green behind it
      # (#12125). A deterministic failure — the artifact missing — fails
      # both attempts and keeps the leg red.
      # The first attempt is time-boxed: an absorbed stall otherwise burns
      # the whole job budget before the retry ever runs.
      - name: 'Download build artifact'
        id: 'download-build'
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        continue-on-error: true
        timeout-minutes: 10
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      - name: 'Download build artifact (retry)'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      # An absorbed download stall leaves the leg green, so the main-CI
      # failure tracker never records it — the warning annotation keeps the
      # recurrence countable, mirroring the build job's upload announce.
      - name: 'Announce absorbed download failure'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        run: |-
          echo "::warning::Download build artifact failed its first attempt and the retry absorbed it — count this toward the #12125 download-stall recurrence watch."

      - name: 'Unpack build artifact'
        run: 'bash .github/scripts/e2e-build-unpack.sh "${RUNNER_TEMP}/e2e-build/e2e-build.tar.gz"'

      - name: 'Run interactive E2E tests (OpenTUI)'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
          KEEP_OUTPUT: 'true'
          VERBOSE: 'true'
        run: |-
          npx cross-env QWEN_E2E_RENDERER=opentui QWEN_SANDBOX=false vitest run --root ./integration-tests interactive --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts'

  isolated-nightly:
    name: '${{ matrix.label }} (nightly)'
    runs-on: 'ubuntu-latest'
    needs:
      - 'build'
    if: |-
      ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        include:
          - label: 'cron-interactive E2E'
            test_file: 'interactive/cron-interactive.test.ts'
          - label: 'channel-plugin E2E'
            test_file: 'channel-plugin.test.ts'
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          package-manager-cache: false
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Cache pnpm store (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: './.github/actions/pnpm-store-cache'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # The install must not build; this leg unpacks the `build` job's
          # outputs instead.
          QWEN_SKIP_PREPARE: '1'
        run: |-
          corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only

      # The bundle and workspace dist/ trees come from the `build` job.
      # One bounded retry, same transient runner-to-blobstore contract as
      # the build job's upload retry (#11364): run 35250857883's macOS
      # shard 2/2 died in this step with the build job green behind it
      # (#12125). A deterministic failure — the artifact missing — fails
      # both attempts; this leg's deliberate job-level continue-on-error
      # keeps its conclusion green regardless, so the retry exists to run
      # the tests, not to keep the leg red.
      # The first attempt is time-boxed: an absorbed stall otherwise burns
      # the whole job budget before the retry ever runs.
      - name: 'Download build artifact'
        id: 'download-build'
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        continue-on-error: true
        timeout-minutes: 10
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      - name: 'Download build artifact (retry)'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1
        with:
          name: 'e2e-build'
          path: '${{ runner.temp }}/e2e-build'

      # An absorbed download stall leaves the leg green, so the main-CI
      # failure tracker never records it — the warning annotation keeps the
      # recurrence countable, mirroring the build job's upload announce.
      - name: 'Announce absorbed download failure'
        if: "${{ steps.download-build.outcome == 'failure' }}"
        run: |-
          echo "::warning::Download build artifact failed its first attempt and the retry absorbed it — count this toward the #12125 download-stall recurrence watch."

      - name: 'Unpack build artifact'
        run: 'bash .github/scripts/e2e-build-unpack.sh "${RUNNER_TEMP}/e2e-build/e2e-build.tar.gz"'

      - name: 'Run ${{ matrix.label }} tests'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
          KEEP_OUTPUT: 'true'
          VERBOSE: 'true'
        run: 'npx cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests "${{ matrix.test_file }}"'

  web-shell-browser-regression:
    name: 'web-shell Browser Regression'
    runs-on: 'ubuntu-latest'
    if: |-
      ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          package-manager-cache: false
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Cache pnpm store (hosted)'
        if: "${{ runner.environment == 'github-hosted' }}"
        uses: './.github/actions/pnpm-store-cache'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        run: |-
          corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only

      - name: 'Install Playwright Chromium'
        run: |-
          node node_modules/playwright/cli.js install --with-deps chromium
          nested_cli='node_modules/@playwright/test/node_modules/playwright/cli.js'
          if [ -f "${nested_cli}" ]; then
            node "${nested_cli}" install chromium
          fi

      - name: 'Run transcript document browser gate'
        run: 'npx vitest run --root ./integration-tests ./chat-transcript-document.test.ts --retry=0'

      - name: 'Run web-shell browser regression'
        run: 'npm run test:e2e --workspace=packages/web-shell'

      - name: 'Upload web-shell Playwright artifacts'
        if: '${{ always() }}'
        uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1
        with:
          name: 'web-shell-browser-regression'
          path: |-
            packages/web-shell/client/e2e/test-results
            packages/web-shell/client/e2e/playwright-report
          if-no-files-found: 'ignore'