diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53408e48b1a..0684fd82744 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1093,12 +1093,18 @@ jobs: trap 'pkill -TERM -P "$SAMPLER_PID" 2>/dev/null || true; kill "$SAMPLER_PID" 2>/dev/null || true' EXIT corepack pnpm install --frozen-lockfile --prefer-offline --reporter=append-only - # A pool host's download usually takes under a minute but has stalled - # for over a quarter of an hour; bound it and try once more. + # Eight shards pull this same artifact at once, so the cap has to clear + # the slow tail rather than the median: on main four shards finished + # inside a minute, three needed between 2m40s and just under five + # minutes, and one exceeded the old five-minute cap on two consecutive + # runs, which skipped the unpack and failed the required check. Fifteen + # minutes leaves that measured tail about three times the headroom; if + # main still expires, the fix is fewer concurrent fetches, not a bigger + # cap. - name: 'Download the built tree' id: 'download_tree' if: "${{ needs.test_build.result == 'success' && needs.classify_pr.outputs.skip_ci != 'true' && steps.ci_profile.outputs.ci_profile == 'full' }}" - timeout-minutes: 5 + timeout-minutes: 15 continue-on-error: true uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1 with: @@ -1107,7 +1113,7 @@ jobs: - name: 'Retry downloading the built tree' if: "${{ steps.download_tree.outcome == 'failure' }}" - timeout-minutes: 5 + timeout-minutes: 15 uses: 'actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' # v8.0.1 with: name: 'unit-test-tree' diff --git a/scripts/tests/ci-platform-lanes.test.js b/scripts/tests/ci-platform-lanes.test.js index 695f94bca37..c55741e3616 100644 --- a/scripts/tests/ci-platform-lanes.test.js +++ b/scripts/tests/ci-platform-lanes.test.js @@ -353,6 +353,29 @@ it('keeps every blocking test_build step under the job ceiling', () => { } }); +it('keeps every shard step cap under the job ceiling', () => { + // Same reason as the build job: the job cap cancels the job, and a cancelled + // job skips the failure-gated disk-pressure collector at the end. The caps + // are pinned one by one rather than summed — they are alternatives in + // practice, since a step only runs when the previous one left work to do, + // and install plus the test step already add up past the ceiling by design. + // The job cap therefore stays the outer backstop, not the sum of the parts. + const jobCap = Number(ci.jobs.test['timeout-minutes']); + const steps = new Map( + (ci.jobs.test.steps ?? []).map((step) => [step.name, step]), + ); + for (const name of [ + 'Install dependencies', + 'Download the built tree', + 'Retry downloading the built tree', + 'Run tests and generate reports', + ]) { + const minutes = steps.get(name)?.['timeout-minutes']; + expect(typeof minutes, `${name} has no step timeout`).toBe('number'); + expect(minutes, `${name} cannot outlast the job`).toBeLessThan(jobCap); + } +}); + it('reports the required Linux check only when the build and every shard pass', () => { const gate = ci.jobs.test_ubuntu; expect(gate.name).toBe('Test (ubuntu-latest, Node 22.x)'); @@ -394,6 +417,16 @@ it('pins the build-to-shard artifact handoff on both ends', () => { expect(name).toBeTruthy(); expect(shard.get('Download the built tree').with.name).toBe(name); expect(shard.get('Retry downloading the built tree').with.name).toBe(name); + // Both attempts have to outlast the slow tail, not the median: eight shards + // fetch this same artifact at once, the slowest completed one on main took + // just under five minutes, and one exceeded the old five-minute cap on two + // runs in a row, skipping the unpack entirely. + for (const step of [ + 'Download the built tree', + 'Retry downloading the built tree', + ]) { + expect(shard.get(step)?.['timeout-minutes'], step).toBe(15); + } expect(shard.get('Unpack the built tree').run).toContain( `\${RUNNER_TEMP}/${name}/${name}.tgz`, );