From b383efb8543965707de8a44a5057254cca623ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Wed, 4 Nov 2026 17:41:55 +0800 Subject: [PATCH] fix(ci): widen the built-tree download window for eight shards (#16285) * fix(ci): widen the built-tree download window for eight shards Eight shards now pull the same unit-test-tree artifact at once, so the download cap has to clear the blob's tail latency rather than the median. On main the slow host class took 4m52s, and two consecutive runs expired twice at the five-minute cap before the unpack ran, which failed the required Linux check with no test involved. Raise both attempts to fifteen minutes and pin the pair in the build-to-shard handoff contract. Co-authored-by: Qwen-Coder * fix(ci): size the shard download cap from the measured tail Review flagged that the outgoing comment claimed a pool host had stalled "for over a quarter of an hour", which the new fifteen-minute cap then contradicts. That claim came in with the sharding commit and was never tied to a run; neither incident can prove how far past five minutes the tail went, so the comment now states only the measured distribution. Also correct the number: the slowest completed shard took just under five minutes, and it was a different shard that expired, on two consecutive runs. Add the missing shard-side ceiling contract, mirroring the build job's: every blocking step cap has to fire before the job cap, because a cancelled job skips the failure-gated disk-pressure collector. Co-authored-by: Qwen-Coder --------- Co-authored-by: Qwen-Coder --- .github/workflows/ci.yml | 14 ++++++++--- scripts/tests/ci-platform-lanes.test.js | 33 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53408e48b1a..0684fd82744 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1093,12 +1093,18 @@ jobs: trap 'pkill -TERM -P "$SAMPLER_PID" 2>/dev/null || true; kill "$SAMPLER_PID" 2>/dev/null || true' EXIT corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only - # A pool host's download usually takes under a minute but has stalled - # for over a quarter of an hour; bound it and try once more. + # Eight shards pull this same artifact at once, so the cap has to clear + # the slow tail rather than the median: on main four shards finished + # inside a minute, three needed between 2m40s and just under five + # minutes, and one exceeded the old five-minute cap on two consecutive + # runs, which skipped the unpack and failed the required check. Fifteen + # minutes leaves that measured tail about three times the headroom; if + # main still expires, the fix is fewer concurrent fetches, not a bigger + # cap. - name: 'Download the built tree' id: 'download_tree' if: "${{ needs.test_build.result == 'success' && needs.classify_pr.outputs.skip_ci != 'true' && steps.ci_profile.outputs.ci_profile == 'full' }}" - timeout-minutes: 5 + timeout-minutes: 15 continue-on-error: true uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1 with: @@ -1107,7 +1113,7 @@ jobs: - name: 'Retry downloading the built tree' if: "${{ steps.download_tree.outcome == 'failure' }}" - timeout-minutes: 5 + timeout-minutes: 15 uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1 with: name: 'unit-test-tree' diff --git a/scripts/tests/ci-platform-lanes.test.js b/scripts/tests/ci-platform-lanes.test.js index 695f94bca37..c55741e3616 100644 --- a/scripts/tests/ci-platform-lanes.test.js +++ b/scripts/tests/ci-platform-lanes.test.js @@ -353,6 +353,29 @@ it('keeps every blocking test_build step under the job ceiling', () => { } }); +it('keeps every shard step cap under the job ceiling', () => { + // Same reason as the build job: the job cap cancels the job, and a cancelled + // job skips the failure-gated disk-pressure collector at the end. The caps + // are pinned one by one rather than summed — they are alternatives in + // practice, since a step only runs when the previous one left work to do, + // and install plus the test step already add up past the ceiling by design. + // The job cap therefore stays the outer backstop, not the sum of the parts. + const jobCap = Number(ci.jobs.test['timeout-minutes']); + const steps = new Map( + (ci.jobs.test.steps ?? []).map((step) => [step.name, step]), + ); + for (const name of [ + 'Install dependencies', + 'Download the built tree', + 'Retry downloading the built tree', + 'Run tests and generate reports', + ]) { + const minutes = steps.get(name)?.['timeout-minutes']; + expect(typeof minutes, `${name} has no step timeout`).toBe('number'); + expect(minutes, `${name} cannot outlast the job`).toBeLessThan(jobCap); + } +}); + it('reports the required Linux check only when the build and every shard pass', () => { const gate = ci.jobs.test_ubuntu; expect(gate.name).toBe('Test (ubuntu-latest, Node 22.x)'); @@ -394,6 +417,16 @@ it('pins the build-to-shard artifact handoff on both ends', () => { expect(name).toBeTruthy(); expect(shard.get('Download the built tree').with.name).toBe(name); expect(shard.get('Retry downloading the built tree').with.name).toBe(name); + // Both attempts have to outlast the slow tail, not the median: eight shards + // fetch this same artifact at once, the slowest completed one on main took + // just under five minutes, and one exceeded the old five-minute cap on two + // runs in a row, skipping the unpack entirely. + for (const step of [ + 'Download the built tree', + 'Retry downloading the built tree', + ]) { + expect(shard.get(step)?.['timeout-minutes'], step).toBe(15); + } expect(shard.get('Unpack the built tree').run).toContain( `\${RUNNER_TEMP}/${name}/${name}.tgz`, );