From 54fd8e0372900f6dd34fe447ff2d1f372f11c02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Fri, 30 Oct 2026 18:24:15 +0800 Subject: [PATCH] fix(ci): route the E2E build job through the pool-first runner expression (#15885) * fix(ci): route the E2E build job through the pool-first runner expression * fix(ci): carry the full self-hosted contract on the E2E build job * test(ci): pin the pool contract's step order and sync the build-routing design note * test(ci): pin the npm-cache position and heal body on every pool job The pool-contract loop only asserted that `Configure persistent npm cache (self-hosted)` exists, so moving it below `Install dependencies` or dropping its `if:` stayed green while the pool install silently fell back to the default npm cache and re-downloaded through the slow egress the step exists to avoid. It also left the build job's ownership-heal body unpinned: the full body was only asserted on the Linux leg, so dropping the `sudo -n` fallbacks from the build copy stayed green while the next checkout on a pool host still holding root-owned leftovers dies with EACCES. Pin the cache step's position and `if:`, and compare every pool job's heal body against the leg copy the docker-lock ownership suite already pins in full. Three mutants (cache step moved below install, cache `if:` stripped, build heal `sudo -n` dropped) now each fail one test; unmutated 130/130 pass. Co-authored-by: Qwen-Coder Patrol-Run: qwen-pr-closeout/jmvu6810z1y * docs(design): pair the E2E build-once note with its zh-CN version docs/design/README.md asks that updating an existing single-language design adds the missing version and aligns the pair, and this PR rewrites the section that records why the build left the hosted runner. Add e2e-build-once.zh-CN.md with the same section order and the same decisions, and link both directions immediately under each title. Also drop the "only hosted-pinned structural job" claim from the English note: `e2e-interactive-opentui` is hosted-pinned too, which the same note already records two paragraphs down. The remaining sentence keeps the actual cause - every leg waits on `build`, and `build` sat on hosted ubuntu. Co-authored-by: Qwen-Coder Patrol-Run: qwen-pr-closeout/jmvu6810z1y --------- Co-authored-by: Qwen-Coder --- .github/workflows/e2e.yml | 54 ++++++++++++++++-- docs/design/e2e-build-once.md | 56 +++++++++++++------ docs/design/e2e-build-once.zh-CN.md | 76 +++++++++++++++++++++++++ scripts/tests/e2e-workflow.test.js | 86 ++++++++++++++++++++++++++--- 4 files changed, 242 insertions(+), 30 deletions(-) create mode 100644 docs/design/e2e-build-once.zh-CN.md diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 794d21717b0..ddbd69dd105 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -32,31 +32,73 @@ jobs: # pool host, so the six Linux shards spent more of the shared pool # rebuilding one commit than testing it. The outputs are platform- # independent JavaScript (native modules come from each leg's own - # `npm ci`), so a hosted VM builds once, off the pool, and every leg - # downloads the result. See .github/scripts/e2e-build-pack.sh. + # `npm ci`), so one job builds once and every leg downloads the result. + # The build shares the legs' pool-first routing: hosted ubuntu jobs sat + # queued for hours while the pool idled (2026-10-28), and one build per + # push adds no measurable pool load. The OpenTUI interactive leg is still + # hosted-pinned, so a run can still wait on hosted capacity for that leg — + # pool routing takes the wait off the critical first step, it does not + # remove every hosted dependency. See .github/scripts/e2e-build-pack.sh. build: name: 'Build for E2E' - runs-on: 'ubuntu-latest' + runs-on: '${{ (github.repository == ''QwenLM/qwen-code'' && vars.MAINTAINER_ECS_RUNNER_DISABLED != ''true'') && fromJSON(''["self-hosted", "linux", "x64", "ecs-qwen"]'') || fromJSON(''["ubuntu-latest"]'') }}' timeout-minutes: 30 # Same fork gate as the legs, so a skipped build skips them all for the # same reason rather than failing them on a missing artifact. if: |- ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} steps: + - name: 'Restore workspace ownership' + if: "${{ runner.environment == 'self-hosted' }}" + run: |- + set -uo pipefail + RUNNER_UID="$(id -u)" + RUNNER_GID="$(id -g)" + if [ "$RUNNER_UID" != "0" ]; then + chown -R "$RUNNER_UID:$RUNNER_GID" "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chown -R "$RUNNER_UID:$RUNNER_GID" "$GITHUB_WORKSPACE" || echo "::warning::could not restore workspace ownership; checkout may fail on leftover root-owned files" + # The docker leg's host-side locks live in the runner's home: a + # root-owned leftover under ~/.cache/qwen-code-ci fails every + # `exec 9>` against them with EACCES before a test runs (#11990). + if [ -d "${HOME}/.cache/qwen-code-ci" ]; then + chown -R "$RUNNER_UID:$RUNNER_GID" "${HOME}/.cache/qwen-code-ci" 2>/dev/null || sudo -n chown -R "$RUNNER_UID:$RUNNER_GID" "${HOME}/.cache/qwen-code-ci" || echo "::warning::could not restore CI cache ownership; docker sandbox locks may stay unreadable" + fi + fi + chmod -R u+rwX "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chmod -R u+rwX "$GITHUB_WORKSPACE" || echo "::warning::could not restore workspace write permissions; checkout may fail on leftover read-only files" + - name: 'Checkout' uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 - - name: 'Set up Node.js' + # Hosted-only, mirroring the legs: on the pool, nodejs.org may be + # unreachable through the egress proxy, and the action's post step + # uploads the npm cache to GitHub — a save that ran 14+ minutes over + # the pool's slow egress on security-checks' first pool run + # (2026-08-26). The pool reuses the machine's Node and its persistent + # npm cache instead. + - name: 'Set up Node.js (hosted)' + if: "${{ runner.environment == 'github-hosted' }}" uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0 with: node-version-file: '.nvmrc' package-manager-cache: false registry-url: 'https://registry.npmjs.org/' + - name: 'Use pre-installed Node.js (self-hosted)' + if: "${{ runner.environment == 'self-hosted' }}" + uses: './.github/actions/self-hosted-node' + - name: 'Cache pnpm store (hosted)' if: "${{ runner.environment == 'github-hosted' }}" uses: './.github/actions/pnpm-store-cache' + - name: 'Configure persistent npm cache (self-hosted)' + if: "${{ runner.environment == 'self-hosted' }}" + run: |- + cache_dir="${HOME}/.cache/qwen-code/npm" + mkdir -p "${cache_dir}" + echo "NPM_CONFIG_CACHE=${cache_dir}" >> "${GITHUB_ENV}" + echo "Using persistent npm cache at ${cache_dir}" + du -sh "${cache_dir}" 2>/dev/null || true + - name: 'Configure npm for rate limiting' run: |- npm config set fetch-retry-mintimeout 20000 @@ -64,6 +106,10 @@ jobs: npm config set fetch-retries 5 npm config set fetch-timeout 300000 + - name: 'Disk floor gate (self-hosted)' + if: "${{ runner.environment == 'self-hosted' }}" + run: 'bash .github/scripts/check-disk-floor.sh "${GITHUB_WORKSPACE}" "${RUNNER_TEMP:-/tmp}"' + # A bare `corepack pnpm install` has no retry: run 34700339334's # build leg died at this step before any test ran, on a tree whose # install reproduces clean, and every leg behind `needs: [build]` diff --git a/docs/design/e2e-build-once.md b/docs/design/e2e-build-once.md index fe7f7cc3a54..2d10f4087b1 100644 --- a/docs/design/e2e-build-once.md +++ b/docs/design/e2e-build-once.md @@ -1,7 +1,9 @@ # E2E: build once, unpack on every leg -Design note for the `build` job in `.github/workflows/e2e.yml`: one hosted -runner builds and bundles the commit, and the eleven test legs download the +[English](e2e-build-once.md) | [简体中文](e2e-build-once.zh-CN.md) + +Design note for the `build` job in `.github/workflows/e2e.yml`: one job +builds and bundles the commit, and the eleven test legs download the result instead of each rebuilding it. ## Context @@ -18,16 +20,16 @@ run, so this repeats many times a day. ## Decision -- A `build` job on `ubuntu-latest` installs (with `QWEN_SKIP_PREPARE=1`), - builds, bundles, and packs the outputs with - `.github/scripts/e2e-build-pack.sh`: the bundle (`dist/`), every workspace - `dist/` under the roots that root `package.json`'s `workspaces` declares - (node_modules pruned, a matched `dist/` not descended again), and an - `e2e-build.sha` stamp of the commit. The script fails closed — with an - `::error::` line — when the bundle is missing or when the scan finds no - workspace `dist/` at all, the same under-pack contract as release.yml's - Pack Build Outputs. The archive is one gzip tarball uploaded as the - `e2e-build` artifact with one day of retention. +- A `build` job installs (with `QWEN_SKIP_PREPARE=1`), builds, bundles, and + packs the outputs with `.github/scripts/e2e-build-pack.sh`: the bundle + (`dist/`), every workspace `dist/` under the roots that root + `package.json`'s `workspaces` declares (node_modules pruned, a matched + `dist/` not descended again), and an `e2e-build.sha` stamp of the commit. + The script fails closed — with an `::error::` line — when the bundle is + missing or when the scan finds no workspace `dist/` at all, the same + under-pack contract as release.yml's Pack Build Outputs. The archive is one + gzip tarball uploaded as the `e2e-build` artifact with one day of + retention. - Every leg gains `needs: [build]`, keeps its own `npm ci` (node_modules, native modules, and the `generate` step that `prepare` still runs), and replaces its build and bundle steps with a download plus @@ -40,13 +42,31 @@ run, so this repeats many times a day. skips them for the same reason instead of failing them on a missing artifact. -## Why a hosted builder +## Why the build left the hosted runner -The build is platform-independent JavaScript; the platform-specific parts are -native modules that each leg's own `npm ci` installs. A hosted VM builds it in -about four minutes with no pool contention, and the ECS legs then pay a -download rather than a build. Building on the pool would put the one job the -whole run waits on behind the same queue that made the legs slow. +The build was hosted-pinned because it is platform-independent JavaScript — +the platform-specific parts are native modules that each leg's own `npm ci` +installs. A hosted VM built it in about four minutes with no pool contention, +and the ECS legs then paid a download rather than a build. Building on the +pool was expected to put the one job the whole run waits on behind the same +queue that made the legs slow. + +That premise inverted on 2026-10-28. Hosted ubuntu allocations for this +repository degraded across one day from a 47-minute wait (run created 02:19 +UTC, build started 03:06) to 2 hours 16 minutes (build started 04:56) to 2 +hours 52 minutes (run created 05:26 UTC, build started 08:18), while the +self-hosted pool — several hundred runners, mostly idle — went unused. Every +leg waits on `build`, and `build` sat on hosted ubuntu, so each push to `main` +accumulated a queued run and the backlog could not drain. The build now shares +the legs' pool-first routing (the ecs-qwen pool, falling back to hosted ubuntu +only when the pool is disabled) and carries the pool's self-hosted contract: +the workspace-ownership heal before checkout, the pre-installed Node preflight, +the persistent npm cache, and the disk-floor gate. On a probe run of the change +(40935790797), the build scheduled onto the pool 4 seconds after creation and +finished in 6m02s. + +The OpenTUI interactive leg remains hosted-pinned, so a run can still end on +a hosted wait for that leg; rerouting it is a separate decision. ## Measured locally diff --git a/docs/design/e2e-build-once.zh-CN.md b/docs/design/e2e-build-once.zh-CN.md new file mode 100644 index 00000000000..20e00a2435e --- /dev/null +++ b/docs/design/e2e-build-once.zh-CN.md @@ -0,0 +1,76 @@ +# E2E：一次构建，每条 leg 各自解包 + +[English](e2e-build-once.md) | [简体中文](e2e-build-once.zh-CN.md) + +`.github/workflows/e2e.yml` 里 `build` job 的设计说明：由一个 job 构建并打包 +本次提交，十一条测试 leg 下载它的产物，而不是各自重新构建一次。 + +## 背景 + +E2E workflow 的每一条 leg —— 共享 ECS 池上的六个 Linux 分片、两个 macOS 分片、 +OpenTUI 交互式 leg 以及两条 nightly 隔离 leg —— 在跑到第一个测试之前，都要在自己的 +runner 上执行 `npm ci`、`npm run build` 和 `npm run bundle`。托管 VM 上这是 4–8 分钟； +在繁忙的池机器上，光是 build 这一步就要 10–17 分钟（run 33718808438：先构建 10–17 +分钟，再跑 12–39 分钟测试）。于是六个池分片花在「重复构建同一个提交」上的共享机器时间 +比花在测试上还多，并且在一次 21 小时的采样里，E2E 占掉了池子 slot-minutes 的 11%。 +每次推送到 `main` 都会启动一次新的 run，所以这件事每天要重复很多次。 + +## 决策 + +- 一个 `build` job 负责安装（带 `QWEN_SKIP_PREPARE=1`）、构建、打包，并用 + `.github/scripts/e2e-build-pack.sh` 归档产物：主产物（`dist/`）、根 `package.json` + 的 `workspaces` 所声明的每个根目录下的 workspace `dist/`（剔除 node_modules，已匹配的 + `dist/` 不再向下递归），以及一个记录提交号的 `e2e-build.sha` 戳。当主产物缺失，或扫描 + 下来一个 workspace `dist/` 都没有时，脚本会以 `::error::` 行 fail closed —— 与 + release.yml 的 Pack Build Outputs 用的是同一套「打包不足即失败」契约。归档是一个 gzip + tarball，以 `e2e-build` artifact 上传，保留一天。 +- 每条 leg 都加上 `needs: [build]`，保留自己的 `npm ci`（node_modules、原生模块，以及 + `prepare` 仍会执行的 `generate` 步骤），并把自己的 build 与 bundle 步骤换成一次下载加 + `.github/scripts/e2e-build-unpack.sh`。该脚本在解包之前就会拒绝戳了不同提交号的归档， + 或其中没有 `dist/cli.js` 的归档；解包之后还会复查主产物是否就位。 +- docker sandbox 镜像不变：它在 Docker 内部、按每机锁从 checkout 构建，因此不属于归档内容。 +- build job 带着与 leg 相同的 fork 门禁，所以 build 被跳过时，leg 也因同一个原因被跳过， + 而不是因为缺 artifact 而失败。 + +## 为什么 build 离开了托管 runner + +build 原先被钉在托管 runner 上，因为它是平台无关的 JavaScript —— 平台相关的部分是原生 +模块，由各条 leg 自己的 `npm ci` 安装。托管 VM 构建它约四分钟，且不与池子争抢资源，ECS +leg 随后只需付出一次下载的成本。当时的预期是：把构建放到池上，会让整个 run 都要等的那个 +job 排到同一条让 leg 变慢的队列里。 + +这个前提在 2026-10-28 反转了。本仓库的托管 ubuntu 分配在一天之内持续恶化：从等待 47 分钟 +（run 创建于 02:19 UTC，build 03:06 启动），到 2 小时 16 分钟（build 04:56 启动），再到 +2 小时 52 分钟（run 创建于 05:26 UTC，build 08:18 启动）；与此同时，自建池 —— 几百台 +runner，大多空闲 —— 一直没被用上。每条 leg 都要等 `build`，而 `build` 待在托管 ubuntu 上， +于是每次推送 `main` 都会积压一个排队中的 run，堆积无法消化。现在 build 与各条 leg 共用 +pool-first 路由（优先 ecs-qwen 池，仅在池被禁用时回落到托管 ubuntu），并且带上池的 +self-hosted 契约：checkout 之前的 workspace 属主修复、预装 Node 的 preflight、常驻 npm +缓存，以及磁盘水位门禁。在该改动的一次探测 run（40935790797）里，build 在创建后 4 秒就 +被调度到池上，6 分 02 秒完成。 + +OpenTUI 交互式 leg 仍然钉在托管 runner 上，所以一次 run 仍可能因为等它而结束在托管排队上； +重新路由它是另一个独立决策。 + +## 本机实测 + +- 构建 `a8248eaed3` 得到的归档 gzip 压缩后 85 MB、14,408 个文件；打包耗时 16 秒。 +- 同一提交上的一个干净 worktree，用 `QWEN_SKIP_PREPARE=1` 安装（各处都没有 `dist/`）， + 可以解开该归档并从还原出来的产物跑通 no-AK 集成门禁。戳与 `GITHUB_SHA` 不一致的归档， + 解包步骤会直接拒绝，且不解出任何内容。 +- `.github/scripts/e2e-build.test.mjs` 覆盖打包与解包契约（workspace dist/ 选取、 + node_modules 剔除、权限位保留、戳校验），并在 CI 的 helper-test 步骤中运行。 + +## 已考虑的替代方案 + +- **直接用 `upload-artifact` 上传目录。** artifact 存储不保留文件权限位，而且每次 run 要 + 上传 1.4 万个对象；tarball 是一个对象，并且保留权限位。 +- **在池上做同一主机共享的构建缓存。** 早前已被否决（#10129），理由是它会在同一台机器上的并发 + job 之间引入共享可变状态。 +- **只接六个 Linux 分片。** macOS、OpenTUI 和 nightly leg 构建的是同一份 JavaScript，因此 + 它们消费同一个归档；把它们排除在外只会让每次 run 多跑三次构建，信号上没有任何收益。 + +## 后续工作 + +- 如果池上的下载耗时确实成为问题，把归档裁剪到测试真正会解析的那些 workspace `dist/` 目录树。 +- 等队列能力开启后，取消 `main` 上被取代的 run，让一批集中合并只构建并测试最新的那个提交。 diff --git a/scripts/tests/e2e-workflow.test.js b/scripts/tests/e2e-workflow.test.js index 9c62fa2643e..b5743d2609a 100644 --- a/scripts/tests/e2e-workflow.test.js +++ b/scripts/tests/e2e-workflow.test.js @@ -1260,18 +1260,84 @@ describe('e2e workflow', () => { expect(gateIndex).toBeLessThan(names.indexOf('Install dependencies')); }); - it('keeps the gate on every pool-routed job', () => { + it('keeps the self-hosted contract on every pool-routed job', () => { // A new pool-routed leg without the gate reddens here instead of on a - // saturated host; the hosted legs stay ungated on fresh VMs. + // saturated host; the hosted legs stay ungated on fresh VMs. The same + // holds for the Node preflight (nodejs.org may be unreachable through + // the pool's egress proxy) and the ownership heal (docker shards leave + // root-owned files on the runners). const poolJobs = Object.entries(yml.jobs).filter(([, job]) => String(job['runs-on'] ?? '').includes('ecs-qwen'), ); - expect(poolJobs.map(([id]) => id)).toEqual(['e2e-test-linux']); + expect(poolJobs.map(([id]) => id)).toEqual(['build', 'e2e-test-linux']); + // Keep the build aligned with the leg, and pin the workspace fallback + // chains below so editing both copies cannot silently remove them. + const pinnedHeal = yml.jobs['e2e-test-linux'].steps.find( + (step) => step.name === 'Restore workspace ownership', + ); for (const [id, job] of poolJobs) { - const gate = (job.steps ?? []).find( + const steps = job.steps ?? []; + const names = steps.map((step) => step.name); + const checkoutIndex = steps.findIndex((step) => + String(step.uses ?? '').includes('actions/checkout'), + ); + const installIndex = names.indexOf('Install dependencies'); + // Presence alone is not the contract: the heal must precede + // checkout, the gate must sit between checkout and install, and the + // preflight must be the pool's own action — a reordered or + // re-pointed step fails here instead of on a runner. + const heal = steps.find( + (step) => step.name === 'Restore workspace ownership', + ); + expect(heal?.if, id).toBe("${{ runner.environment == 'self-hosted' }}"); + expect(names.indexOf('Restore workspace ownership'), id).toBeLessThan( + checkoutIndex, + ); + const gate = steps.find( (step) => step.name === 'Disk floor gate (self-hosted)', ); expect(gate, id).toBeDefined(); + const gateIndex = names.indexOf('Disk floor gate (self-hosted)'); + expect(gateIndex, id).toBeGreaterThan(checkoutIndex); + expect(gateIndex, id).toBeLessThan(installIndex); + const poolNode = steps.find( + (step) => step.name === 'Use pre-installed Node.js (self-hosted)', + ); + expect(poolNode?.uses, id).toBe('./.github/actions/self-hosted-node'); + const hostedNode = steps.find( + (step) => + typeof step.name === 'string' && + step.name.startsWith('Set up Node.js'), + ); + expect(hostedNode?.if, id).toBe( + "${{ runner.environment == 'github-hosted' }}", + ); + const npmCacheIndex = names.indexOf( + 'Configure persistent npm cache (self-hosted)', + ); + // Presence alone is not the contract here either: below the install it + // exports NPM_CONFIG_CACHE after npm has already picked a cache, so + // the pool re-downloads through the slow egress this step avoids, and + // without the `if:` the hosted lane points npm at a directory no cache + // step ever restores. + expect(npmCacheIndex, id).toBeGreaterThanOrEqual(0); + expect(npmCacheIndex, id).toBeLessThan(installIndex); + expect(steps[npmCacheIndex]?.if, id).toBe( + "${{ runner.environment == 'self-hosted' }}", + ); + expect(steps[npmCacheIndex]?.run, id).toContain( + 'cache_dir="${HOME}/.cache/qwen-code/npm"', + ); + expect(steps[npmCacheIndex]?.run, id).toContain( + 'echo "NPM_CONFIG_CACHE=${cache_dir}" >> "${GITHUB_ENV}"', + ); + expect(heal?.run, id).toBe(pinnedHeal?.run); + expect(heal?.run, id).toContain( + 'chown -R "$RUNNER_UID:$RUNNER_GID" "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chown -R "$RUNNER_UID:$RUNNER_GID" "$GITHUB_WORKSPACE"', + ); + expect(heal?.run, id).toContain( + 'chmod -R u+rwX "$GITHUB_WORKSPACE" 2>/dev/null || sudo -n chmod -R u+rwX "$GITHUB_WORKSPACE"', + ); } }); }); @@ -1279,8 +1345,8 @@ describe('e2e workflow', () => { describe('one build for every leg', () => { // Each leg used to build and bundle on its own runner — 4–8 minutes on a // hosted VM, 10–17 on a busy pool host, eleven times per run. The `build` - // job does it once on a hosted VM and the legs unpack its archive; these - // pins keep a leg from quietly growing its own build back. + // job does it once and the legs unpack its archive; these pins keep a leg + // from quietly growing its own build back. const build = yml.jobs.build; const legs = [ 'e2e-test-linux', @@ -1289,9 +1355,13 @@ describe('e2e workflow', () => { 'isolated-nightly', ]; - it('builds once, on a hosted runner, off the shared pool', () => { + it("builds once, sharing the legs' pool-first runner routing", () => { expect(build.needs).toBeUndefined(); - expect(build['runs-on']).toBe('ubuntu-latest'); + // Hosted ubuntu jobs queued for hours while the pool idled + // (2026-10-28), so the build rides the exact routing expression every + // other pool job uses instead of pinning a hosted-only label. + expect(build['runs-on']).toBe(yml.jobs['e2e-test-linux']['runs-on']); + expect(String(build['runs-on'])).toContain('ecs-qwen'); const names = build.steps.map((step) => step.name); expect(names).toContain('Build project'); expect(names).toContain('Bundle CLI for E2E tests');