From 708ba347d355f199e9f5a6f4987755a3218bb5fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=93=E8=89=AF?= <1204183885@qq.com> Date: Sat, 7 Nov 2026 22:24:33 +0800 Subject: [PATCH 1/2] feat(ci): add disposable external PR runners (#16513) * feat(ci): add disposable external PR runners * test(ci): align platform lanes with disposable runner routing --------- Co-authored-by: yiliang114 <11473889+yiliang114@users.noreply.github.com> --- .github/scripts/ci-runner-routing.test.mjs | 121 ++- .github/workflows/.size-baseline | 1 + .github/workflows/ci.yml | 53 +- .github/workflows/ephemeral-runner-smoke.yml | 23 + .../2026-11-07-ephemeral-container-runners.md | 163 ++++ ...11-07-ephemeral-container-runners.zh-CN.md | 133 +++ ...2026-11-07-ephemeral-runner-model-proxy.md | 165 ++++ ...1-07-ephemeral-runner-model-proxy.zh-CN.md | 116 +++ scripts/ci/ephemeral-runner/Dockerfile | 39 + scripts/ci/ephemeral-runner/README.md | 140 +++ .../ci/ephemeral-runner/config.example.json | 16 + scripts/ci/ephemeral-runner/entrypoint.sh | 8 + scripts/ci/ephemeral-runner/manager.mjs | 620 +++++++++++++ scripts/ci/ephemeral-runner/manager.test.mjs | 822 ++++++++++++++++++ .../qwen-ephemeral-runner.service | 18 + scripts/tests/ci-platform-lanes.test.js | 22 +- .../review-worktree-cleanup-workflow.test.js | 26 +- 17 files changed, 2427 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/ephemeral-runner-smoke.yml create mode 100644 docs/design/2026-11-07-ephemeral-container-runners.md create mode 100644 docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md create mode 100644 docs/design/2026-11-07-ephemeral-runner-model-proxy.md create mode 100644 docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md create mode 100644 scripts/ci/ephemeral-runner/Dockerfile create mode 100644 scripts/ci/ephemeral-runner/README.md create mode 100644 scripts/ci/ephemeral-runner/config.example.json create mode 100644 scripts/ci/ephemeral-runner/entrypoint.sh create mode 100644 scripts/ci/ephemeral-runner/manager.mjs create mode 100644 scripts/ci/ephemeral-runner/manager.test.mjs create mode 100644 scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service diff --git a/.github/scripts/ci-runner-routing.test.mjs b/.github/scripts/ci-runner-routing.test.mjs index 2553a5177ac..14474e24ed3 100644 --- a/.github/scripts/ci-runner-routing.test.mjs +++ b/.github/scripts/ci-runner-routing.test.mjs @@ -8,7 +8,7 @@ // scarce pool (2026-10-31 decision, same accepted risk class as lint). // The association routing for DOWNSTREAM lanes lives in the `pick_runner` // shell step, which publishes `ubuntu_runner` (association-gated) and -// `lint_runner` (every pull request reaches the fleet). The matrix below +// `lint_runner` (untrusted forks default to hosted). The matrix below // pins the gate's unconditional routing AND that ubuntu_runner keeps the // association gate for the test/integration lanes. // @@ -123,6 +123,8 @@ function runPickRunnerOutputs({ eventName, dispatch, fleet, + ephemeralEnabled, + ephemeralArch, }) { const tmp = mkdtempSync(join(tmpdir(), 'pick-runner-')); const outputFile = join(tmp, 'github_output'); @@ -133,6 +135,8 @@ function runPickRunnerOutputs({ ECS_DISABLED: ecsDisabled ? 'true' : '', EVENT_NAME: eventName, LINUX_RUNNER_FLEET: fleet ?? '', + EPHEMERAL_ENABLED: ephemeralEnabled ?? '', + EPHEMERAL_ARCH: ephemeralArch ?? '', DISPATCH_LINUX_RUNNER: dispatch ?? '', GITHUB_OUTPUT: outputFile, }, @@ -161,7 +165,15 @@ function runPickRunnerOutputs({ outputs.split('\n').includes(`lint_runner=${lintRunner}`), `lint_runner not published: ${outputs}`, ); - return { ubuntuRunner, lintRunner }; + const prTest = outputs + .split('\n') + .find((line) => line.startsWith('pr_test_runner=')); + assert.ok(prTest, 'pr_test_runner not published'); + return { + ubuntuRunner, + lintRunner, + prTestRunner: prTest.slice('pr_test_runner='.length), + }; } function runPickRunner(inputs) { @@ -302,11 +314,7 @@ describe('ci.yml lint_runner routing', () => { const ARM = '["self-hosted", "linux", "arm64"]'; const lintRunner = (inputs) => runPickRunnerOutputs(inputs).lintRunner; - it('routes every pull request to the fleet, untrusted forks included', () => { - // The maintainer decision this pins: Lint & Static executes PR code on - // the shared pool for ALL authors. If this matrix ever flips back to - // hosted for the negative associations without a deliberate decision, - // the lane silently regrows its hosted quota spend. + it('keeps untrusted fork lint hosted when the disposable switch is unset', () => { for (const sameRepo of [true, false]) { for (const assoc of ASSOCIATIONS) { assert.equal( @@ -316,8 +324,8 @@ describe('ci.yml lint_runner routing', () => { assoc, eventName: 'pull_request', }), - ECS, - `lint sameRepo=${sameRepo} assoc='${assoc}' must route to ECS`, + sameRepo || TRUSTED.includes(assoc) ? ECS : HOSTED, + `lint sameRepo=${sameRepo} assoc='${assoc}'`, ); } } @@ -354,7 +362,7 @@ describe('ci.yml lint_runner routing', () => { lintRunner({ ecsDisabled: false, sameRepo: false, - assoc: 'NONE', + assoc: 'MEMBER', eventName: 'pull_request', fleet: 'arm64', }), @@ -442,6 +450,99 @@ describe('ci.yml lint_runner routing', () => { }); }); +describe('ci.yml disposable runner routing', () => { + it('keeps all selected fork jobs hosted unless the switch is exactly true', () => { + for (const ephemeralEnabled of [undefined, '', 'false', 'invalid']) { + const out = runPickRunnerOutputs({ + eventName: 'pull_request', + sameRepo: false, + assoc: 'NONE', + ephemeralEnabled, + }); + assert.equal(out.prTestRunner, HOSTED); + assert.equal(out.lintRunner, HOSTED); + } + }); + it('selects ARM64 only when explicitly configured', () => { + const inputs = { + eventName: 'pull_request', + sameRepo: false, + assoc: 'NONE', + ephemeralEnabled: 'true', + }; + const arm = '["self-hosted", "linux", "arm64", "ecs-qwen-ephemeral"]'; + assert.equal( + runPickRunnerOutputs({ ...inputs, ephemeralArch: 'arm64' }).prTestRunner, + arm, + ); + assert.equal( + runPickRunnerOutputs({ ...inputs, ephemeralArch: 'arm64' }).lintRunner, + arm, + ); + assert.equal( + runPickRunnerOutputs({ ...inputs, ephemeralArch: 'invalid' }) + .prTestRunner, + '["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]', + ); + }); + it('routes only untrusted fork jobs to the disposable pool when enabled', () => { + const ephemeral = '["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]'; + for (const eventName of [ + 'pull_request', + 'push', + 'merge_group', + 'workflow_dispatch', + ]) { + for (const sameRepo of [true, false]) { + for (const assoc of ASSOCIATIONS) { + for (const ecsDisabled of [true, false]) { + for (const fleet of ['', 'hosted', 'arm64']) { + const inputs = { eventName, sameRepo, assoc, ecsDisabled, fleet }; + const baseline = runPickRunnerOutputs(inputs); + const enabled = runPickRunnerOutputs({ + ...inputs, + ephemeralEnabled: 'true', + }); + const selected = + eventName === 'pull_request' && + !sameRepo && + !TRUSTED.includes(assoc) && + !ecsDisabled && + fleet !== 'hosted'; + assert.equal( + enabled.prTestRunner, + selected ? ephemeral : baseline.ubuntuRunner, + JSON.stringify(inputs), + ); + assert.equal( + enabled.lintRunner, + selected ? ephemeral : baseline.lintRunner, + JSON.stringify(inputs), + ); + assert.equal(enabled.ubuntuRunner, baseline.ubuntuRunner); + } + } + } + } + } + }); + + it('routes only the intended jobs through the new output', () => { + assert.equal( + ciDoc.jobs.classify_pr.outputs.pr_test_runner, + '${{ steps.pick_runner.outputs.pr_test_runner }}', + ); + for (const name of ['test_build', 'test']) + assert.match(ciDoc.jobs[name]['runs-on'], /pr_test_runner/); + for (const name of ['lint_and_static', 'integration_no_ak']) + assert.match(ciDoc.jobs[name]['runs-on'], /lint_runner/); + for (const [name, job] of Object.entries(ciDoc.jobs)) { + if (!['classify_pr', 'test_build', 'test'].includes(name)) + assert.doesNotMatch(String(job['runs-on']), /pr_test_runner/, name); + } + }); +}); + describe('ci.yml lint HOME isolation', () => { // This guard lives in the routing battery (HELPER_TESTS) because it gates // every full-profile PR; the scripts/tests vitest suites only run after the diff --git a/.github/workflows/.size-baseline b/.github/workflows/.size-baseline index 3af4140f298..55d506c2371 100644 --- a/.github/workflows/.size-baseline +++ b/.github/workflows/.size-baseline @@ -27,6 +27,7 @@ 2038 docs-page-action.yml 10005 dsw-swe-verified-release.yml 40188 e2e.yml +651 ephemeral-runner-smoke.yml 13135 finalize-release.yml 16647 live-host-release.yml 5950 live-host.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 840ecbed56c..9fdf7df6012 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,8 +101,8 @@ env: # re-runs every suite after `npm ci`, and a push to main always classifies # `full` (classify_pr computes a profile only for pull_request), so what # the fast lane skips is still re-checked post-merge. - HELPER_TESTS_DEP_FREE: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs' - HELPER_TESTS: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/classify-release-notes.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/qwen-triage-workflow.test.mjs .github/scripts/assign-issue-owner.test.mjs .github/scripts/auto-minimize-spam.test.mjs .github/scripts/ci-runner-routing.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/assign-pr-owner.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/ci-disk-pressure.test.mjs .github/scripts/e2e-build.test.mjs integration-tests/concurrent-runner/export-html-from-chatrecord-jsonl.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/ecs-runner/review-scratch-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs' + HELPER_TESTS_DEP_FREE: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs scripts/ci/ephemeral-runner/manager.test.mjs' + HELPER_TESTS: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/classify-release-notes.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/qwen-triage-workflow.test.mjs .github/scripts/assign-issue-owner.test.mjs .github/scripts/auto-minimize-spam.test.mjs .github/scripts/ci-runner-routing.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/assign-pr-owner.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/ci-disk-pressure.test.mjs .github/scripts/e2e-build.test.mjs integration-tests/concurrent-runner/export-html-from-chatrecord-jsonl.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/ecs-runner/review-scratch-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs scripts/ci/ephemeral-runner/manager.test.mjs' # The growth ratchet and its vitest mirror compare each workflow against # the PR's base commit to tell "this PR grew the file" apart from "the # baseline went stale on main" (#9904). Wired once here so every lane @@ -143,11 +143,8 @@ jobs: outputs: skip_ci: '${{ steps.release_sync.outputs.skip_ci }}' ubuntu_runner: '${{ steps.pick_runner.outputs.ubuntu_runner }}' - # Lint & Static's own routing: every pull request reaches the fleet, - # including untrusted forks (maintainer decision 2026-10-31 — the lane - # executes PR code on the shared pool and that risk is accepted). The - # association gate in ubuntu_runner still governs every other Linux - # lane; do not "simplify" the two outputs into one. + pr_test_runner: '${{ steps.pick_runner.outputs.pr_test_runner }}' + # Untrusted forks use hosted runners unless the disposable pool is enabled. lint_runner: '${{ steps.pick_runner.outputs.lint_runner }}' ci_profile: '${{ steps.ci_profile.outputs.ci_profile }}' steps: @@ -196,17 +193,16 @@ jobs: # Keep push routing aligned with the classifier above so a hosted queue # does not delay the downstream ECS jobs. # - # lint_runner is the one deliberate divergence: Lint & Static routes - # EVERY pull request to the fleet, untrusted forks included (maintainer - # decision 2026-10-31, risk accepted — see the classify_pr outputs - # comment). The kill-switch and LINUX_RUNNER_FLEET=hosted still force - # hosted, and non-PR events inherit ubuntu_runner unchanged. + # Untrusted forks opt into the disposable pool; otherwise their build, + # test, lint and no-AK integration jobs stay hosted. - name: 'Select Linux runner' id: 'pick_runner' env: SAME_REPO: '${{ github.event.pull_request.head.repo.full_name == github.repository }}' AUTHOR_ASSOCIATION: '${{ github.event.pull_request.author_association }}' ECS_DISABLED: '${{ vars.MAINTAINER_ECS_RUNNER_DISABLED }}' + EPHEMERAL_ENABLED: '${{ vars.EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED }}' + EPHEMERAL_ARCH: '${{ vars.EXTERNAL_PR_EPHEMERAL_ARCH }}' # Optional fleet selector: 'hosted' forces github.com-hosted runners, # 'arm64' routes to the experimental ARM64 ECS fleet, anything else # keeps the x64 ecs-qwen fleet. Empty/unset keeps current behavior. @@ -242,12 +238,16 @@ jobs: fi echo "ubuntu_runner=${ubuntu_runner}" >> "${GITHUB_OUTPUT}" echo "Selected Linux runner: ${ubuntu_runner}" - # Lint & Static routes every pull request to the fleet (maintainer - # decision 2026-10-31); non-PR events inherit ubuntu_runner unchanged. + pr_test_runner="${ubuntu_runner}" lint_runner="${ubuntu_runner}" - if [[ "${EVENT_NAME}" == "pull_request" && "${ECS_DISABLED}" != "true" && "${LINUX_RUNNER_FLEET}" != "hosted" ]]; then - lint_runner="${fleet_runner}" + if [[ "${EPHEMERAL_ENABLED}" == "true" && "${ECS_DISABLED}" != "true" && "${LINUX_RUNNER_FLEET}" != "hosted" && "${EVENT_NAME}" == "pull_request" && "${SAME_REPO}" == "false" && "${trusted_author}" == "false" ]]; then + pr_test_runner='["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]' + if [[ "${EPHEMERAL_ARCH}" == "arm64" ]]; then + pr_test_runner='["self-hosted", "linux", "arm64", "ecs-qwen-ephemeral"]' + fi + lint_runner="${pr_test_runner}" fi + echo "pr_test_runner=${pr_test_runner}" >> "${GITHUB_OUTPUT}" echo "lint_runner=${lint_runner}" >> "${GITHUB_OUTPUT}" echo "Selected lint runner: ${lint_runner}" @@ -423,7 +423,7 @@ jobs: name: 'Build unit-test tree (ubuntu-latest, Node 22.x)' needs: 'classify_pr' if: "${{ !cancelled() && github.event_name != 'schedule' }}" - runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}' + runs-on: '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || ''["ubuntu-latest"]'') }}' timeout-minutes: 45 permissions: contents: 'read' @@ -890,7 +890,7 @@ jobs: # have. So a push run is lint, static analysis and unit tests — # lint/static in `lint_and_static`, unit tests here. if: "${{ !cancelled() && github.event_name != 'schedule' }}" - runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}' + runs-on: '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || ''["ubuntu-latest"]'') }}' # Every Vitest run takes one eighth of its suite's files (`--shard`): the # unsharded lane ran 45-59 minutes, almost all of it the CLI and core # suites one after the other. A shard keeps the old ceiling's headroom @@ -1559,12 +1559,7 @@ jobs: # same release-sync convention: stay running on skip_ci PRs so a required # context still reports, with the per-step guards making it a no-op pass. if: "${{ !cancelled() && github.event_name != 'schedule' }}" - # Reads lint_runner, NOT ubuntu_runner: this lane routes every pull - # request to the fleet, untrusted forks included (maintainer decision - # 2026-10-31 — the lane executes PR code and the shared-pool risk is - # accepted). test and the integration lanes keep the association-gated - # ubuntu_runner. The kill-switch and hosted-fleet overrides still apply, - # inside pick_runner. + # Shares the external-PR hosted/disposable switch with test and integration_no_ak. runs-on: '${{ fromJSON(needs.classify_pr.outputs.lint_runner || ''["ubuntu-latest"]'') }}' # Pool-scoped like test's, not a bare constant: this lane was beheaded twice # running honestly on cold-cache hk3-pool runners — at 30 minutes flat @@ -2982,14 +2977,8 @@ jobs: # A draft pull request skips it (and the browser smoke): it cannot merge, # and `ready_for_review` starts the full run that gates the merge. if: "${{ !cancelled() && needs.classify_pr.outputs.skip_ci != 'true' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') && !github.event.pull_request.draft }}" - # Reads lint_runner, NOT ubuntu_runner: like lint_and_static, this lane - # executes PR code on the pool for every pull request — accepted under the - # same 2026-10-31 decision, and it carries NO secrets by construction - # (every model key is emptied at the integration step; the job's only - # token is the read-only checkout grant). The lane publishes a run-unique - # HOME from its first step (lint's shape) and empties every model key at - # the gate, so the pool exposure is the class already accepted for lint. - # test and integration_cli keep the association-gated ubuntu_runner. + # Shares lint's hosted/disposable routing for untrusted forks. Model keys + # are empty; the job receives only the read-only checkout token. runs-on: '${{ fromJSON(needs.classify_pr.outputs.lint_runner || ''["ubuntu-latest"]'') }}' strategy: fail-fast: false diff --git a/.github/workflows/ephemeral-runner-smoke.yml b/.github/workflows/ephemeral-runner-smoke.yml new file mode 100644 index 00000000000..19896976fa8 --- /dev/null +++ b/.github/workflows/ephemeral-runner-smoke.yml @@ -0,0 +1,23 @@ +name: 'Ephemeral runner smoke' + +on: + workflow_dispatch: + +permissions: + contents: 'read' + +jobs: + smoke: + runs-on: ['ecs-qwen-ephemeral-smoke'] + timeout-minutes: 10 + steps: + - uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + - name: 'Check tools and workspace' + run: | + node --version + git --version + printf 'Runner: %s\n' "$RUNNER_NAME" > runner-smoke.txt + - uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1 + with: + name: 'ephemeral-runner-smoke' + path: 'runner-smoke.txt' diff --git a/docs/design/2026-11-07-ephemeral-container-runners.md b/docs/design/2026-11-07-ephemeral-container-runners.md new file mode 100644 index 00000000000..1c3fbca65e5 --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-container-runners.md @@ -0,0 +1,163 @@ +# Ephemeral container runners for external PRs + +[English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md) + +Date: 2026-11-07. Status: implemented; a five-slot systemd trial is deployed on +hz4. Production external-PR routing remains disabled. + +Ten slots registering and running ten keyless test jobs of a private repository +at the same time on the Hangzhou shared test machine; containers, networks and +registrations were all reclaimed. A subsequent one-slot trial uses a real PAT +and two sequential GitHub Actions jobs to verify registration, container +replacement, workspace separation and absence of the PAT in jobs. A forced +manager exit verified systemd restart and cleanup before replacement. These +trials found and fixed compatibility with gh 2.46 pagination and Docker's quiet +mode overriding formatted slot output. Full PR CI and production isolation +remain unverified, so the shared host is still only a controlled trial host. + +Follow-up model access is described in +[the runner and model-proxy combination](2026-11-07-ephemeral-runner-model-proxy.md); +that proposal is neither implemented nor enabled. + +## Goals and boundaries + +Put the whole GitHub Actions runner into a disposable Docker container. The +container runs one job and is then deleted; the next job gets a new container, +a new runner identity and a new working directory from the same clean image. +Users do not want to maintain a per-command permission allowlist for CI. + +The disposable unit is a job, not a workflow, a step or a PR. Multiple jobs +exchange files through GitHub artifacts and never share a writable working +directory. + +This design addresses run-environment residue and cross-task file and process +contamination. Plain Docker shares the host kernel and gives no virtual-machine +isolation from malicious code; any secret passed into a job can still be read +by that job. The first phase accepts only Linux CI work without long-lived +secrets: no model gateway is built here, and release, autofix, real-model E2E +and tasks that need a Docker daemon are not migrated. + +## Architecture + +```text +External-CI dedicated ECS +├── systemd-managed manager (long-running, trusted) +│ ├── requests JIT runner configuration with the host-only PAT +│ ├── starts, monitors and deletes containers and their isolated networks +│ └── keeps management state, exports logs, recovers interrupted tasks +├── slot 1: container A → one job → delete → container C +└── slot 2: container B → one job → delete → container D + +Each container +├── one JIT runner that connects to GitHub on its own +├── Node.js 22, Git and the CI dependencies +└── its own HOME, working directory, temporary directory and npm cache +``` + +The manager keeps a configured number of slots, at most one container per slot. +The Hangzhou trial ran ten slots at once; the hz4 deployment now maintains five slots. It takes no webhook, schedules no PR +itself, and introduces neither Kubernetes nor a database. GitHub assigns work by +runner label. Each ECS runs its own manager against its local Docker socket, +with a unique `poolId` and a per-host `slots` count. Hosts share the scheduling +labels; no central manager is needed. New names use `ecs-qwen-` for compatibility +with CI worker caps and timeouts; recovery still accepts old `qwen-` identities. + +systemd starts the manager at boot when enabled and restarts unexpected exits +after ten seconds (`Restart=always`). An explicit service stop remains stopped. +This supervises process exits; it does not detect a live but stuck manager. + +Images are produced by a trusted build and pinned by digest. The manager and the +image may only be deployed from versions maintained under maintainer control, +never updated from a job checkout or from PR artifacts. + +Authentication uses a dedicated PAT in the root-only systemd EnvironmentFile, +`/etc/qwen-ephemeral-runner/credentials.env`, as `GH_TOKEN`. A fine-grained token +should select only the target repository and Administration: write, the GitHub +API's required permission. It is not restricted to runner operations alone. +Fine-grained access can be unavailable for outside collaborators or under +organization policy; a broader classic token requires an explicit decision. +The PAT stays on the host, never in the config JSON or a job container. Host +root and Docker administrators remain trusted. Rotate the credential file and +restart the service before expiry; no GitHub App or organization runner +permission is needed for repository-scoped runners. + +## Ownership and lifecycle + +| Object | Owner | Lifetime | +| ------------------------------------- | ----------------------- | ---------------------------------------------------- | +| ECS, Docker daemon, manager | Ops / maintainers | Long term | +| Host-only runner-management PAT | Manager | Never enters a job container | +| Clean image | Trusted image build | Versioned, not writable from a job | +| JIT configuration, runner identity | One container instance | Used for a single job | +| Container writable layer and job data | One container instance | Deleted on destruction | +| Docker bridge network | One container instance | Deleted on destruction | +| Diagnostic logs | Exported by the manager | Size and retention bounded, unreadable by later jobs | + +Normal path: request JIT → create the isolated network and container → runner +comes online → run one job → runner exits → export logs → delete the container +and network → clean up the leftover registration → create the next one. + +JIT provides single-job registration semantics under the normal protocol; the +host manager is responsible for actually destroying the container. The design +must not rely on a malicious job exiting on its own, finishing a final cleanup +step, or keeping the runner intact. A watchdog outside the container enforces a +maximum lifetime, and a manager restart reconciles old resources first. + +If deleting a container fails, that slot stops; if GitHub registration cleanup +fails, the failure is recorded and retried instead of endlessly requesting new +identities. Host failures cannot clean up immediately, so recovery must reclaim +before accepting new work. + +## Minimal but necessary deployment boundary + +- Use an ECS dedicated to external CI; do not share a host with existing runners + that hold long-lived secrets. +- Mount no host home, workspace, management state, credential directory or + Docker socket; use neither privileged mode, host PID nor host networking. No + per-command allowlist is needed inside the container. +- Images declare no persistent VOLUME and the first phase creates no job volume; + all job writes go to the container writable layer. +- Network policy outside the host and unmodifiable from the container blocks the + management interface, sensitive intranets, instance credentials and metadata; + configure only the GitHub, npm and image-service egress actually required. +- Access granted to the old trusted runners must be verified on the GitHub side. + Neither labels nor PR-modifiable YAML are an authorization boundary; an + external PR must not route back into a sensitive pool by editing runs-on. +- Do not open fork secrets because a self-hosted runner is enabled, and do not + introduce a privileged pull_request_target flow that executes PR code. + +## Initial routing decision + +Keep the existing `ubuntu_runner` output and all of its former semantics, and add +a `pr_test_runner` output used by `test_build` and `test`. For an untrusted fork +PR, `EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` selects the disposable pool; +unset, `false`, or any other value selects `ubuntu-latest`. `lint_runner` follows +the same switch for `lint_and_static` and `integration_no_ak`, replacing the old +shared-ECS fallback for these fork jobs. Trusted and same-repository PR routing +stays unchanged. The global kill-switch and hosted setting take precedence. + +The switch selects a destination when a job is evaluated, without querying +runner availability. It cannot move an already queued job; cancel queued work +and start a new workflow run to apply a rollback. Verify unset/false/invalid, +true for both architectures, and the global overrides against the actual shell +selector before rollout. + +`classify_pr`, `web_shell_e2e_smoke`, `integration_cli` and the other workflows +keep their current routing. This avoids migrating browser system dependencies +that the new image has not verified, and leaves the routing image relationship +of the existing classifier job and the other workflows unchanged. + +The new pool label is `ecs-qwen-ephemeral`. Default routing uses x64; setting +`EXTERNAL_PR_EPHEMERAL_ARCH=arm64` selects ARM64 instead. The toolchain of full +PR CI on the new image still has to be verified. + +## References + +- [GitHub ephemeral runner semantics and cleanup responsibility](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#ephemeral-runners-for-autoscaling) +- [GitHub repository-level JIT API](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository) +- [Docker daemon and kernel security boundary](https://docs.docker.com/engine/security/) + +The facts this design relies on were checked against the official documentation +during this discussion; the actual runner version, GitHub account capabilities, +API permissions and ECS network configuration must be confirmed again at +deployment time. diff --git a/docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md b/docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md new file mode 100644 index 00000000000..5378d5bdcdd --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md @@ -0,0 +1,133 @@ +# 外部 PR 的一次性容器 runner + +[English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md) + +日期：2026-11-07。状态：已实现并在 hz4 部署五槽位 systemd 试运行服务； +外部 PR 的生产路由仍未启用。 + +杭州共享测试机上已验证 10 个槽位同时注册，并执行同一个私有测试仓库的 10 个无密钥 +测试 job，容器、网络和注册均已回收。后续单槽位试运行使用真实 PAT， +通过两个串行 GitHub Actions job 验证注册、容器替换、工作区隔离及 job 内 +没有 PAT；强制结束 manager 后也验证了 systemd 重启及补位前清理。 +试运行发现并修复了 gh 2.46 分页兼容性及 Docker quiet 模式覆盖格式化 +槽位输出的问题。完整 PR CI 和生产隔离仍未验证，共享主机只用于受控试验。 + +后续模型访问见 [runner 与模型代理结合方案](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md)； +该方案尚未实现或启用。 + +## 目标和边界 + +将整个 GitHub Actions runner 放进一次性 Docker 容器。容器执行一个 job， +随后删除；下一次从同一个干净镜像创建新容器、新 runner 身份和新工作目录。 +用户不希望为 CI 中的每条命令维护权限白名单。 + +一次性单位是 job，不是 workflow、step 或 PR。多个 job 通过 GitHub artifacts +交换文件，不共享可写工作目录。 + +本设计解决运行环境残留和跨任务文件、进程污染。普通 Docker 共享宿主机内核， +不提供虚拟机级恶意代码隔离；传入 job 的任何密钥仍可能被该 job 读取。 +首期只接无长期密钥的 Linux CI，不建设模型网关，不迁移发布、自动修复、 +真实模型 E2E 或需要 Docker daemon 的任务。 + +## 架构 + +```text +外部 CI 专用 ECS +├── systemd 托管的管理程序（长期运行，可信） +│ ├── 使用宿主机 PAT 申请 JIT runner 配置 +│ ├── 启动、监控、删除容器及其独立网络 +│ └── 保存管理状态、导出日志、恢复异常任务 +├── 槽位 1：容器 A → 一个 job → 删除 → 容器 C +└── 槽位 2：容器 B → 一个 job → 删除 → 容器 D + +每个容器 +├── 一个 JIT runner，主动连接 GitHub +├── Node.js 22、Git 和 CI 依赖 +└── 独立的 HOME、工作目录、临时目录、npm 缓存 +``` + +管理程序按配置维护固定数量的槽位，每个槽位最多一个容器；杭州十槽位试验同时使用 +十个槽位，hz4 部署目前维持五个槽位。不接 webhook、不自行调度 PR、不引入 Kubernetes 或数据库。 +GitHub 根据 runner 标签分配任务。每台 ECS 各运行一个管理程序，只连接本机 +Docker socket，使用独立的 `poolId` 和按主机配置的 `slots`。各主机共用调度 +标签，无需中央管理器。新名称使用 `ecs-qwen-` 前缀以兼容 CI 并发上限和超时； +恢复时仍接受旧的 `qwen-` 身份。 + +启用 systemd 服务后开机启动，意外退出时十秒后重启（`Restart=always`）； +显式停止服务后保持停止。它监督进程退出，不检测进程存活但卡住的情况。 + +镜像由可信构建生成并固定 digest。管理程序和镜像只能从经过维护者控制的版本 +部署，不从任务 checkout 或 PR artifacts 更新。 + +认证使用专用 PAT，以 `GH_TOKEN` 写入仅 root 可读的 systemd EnvironmentFile： +`/etc/qwen-ephemeral-runner/credentials.env`。fine-grained token 应只选择目标 +仓库及 GitHub API 要求的 Administration: write；它不只限于 runner 操作。 +外部协作者身份或组织策略可能使 fine-grained token 不可用，改用权限更宽的 +classic token 必须明确决定。PAT 留在宿主机，不进入配置 JSON 或任务容器。 +宿主机 root 和 Docker 管理员仍属于可信边界。到期前替换凭证文件并重启服务； +仓库级 runner 不需要 GitHub App 或组织级 runner 管理权限。 + +## 所有权和生命周期 + +| 对象 | 所有者 | 生命周期 | +| ---------------------------- | ------------ | ----------------------------------- | +| ECS、Docker daemon、管理程序 | 运维/维护者 | 长期 | +| 宿主机 runner 管理 PAT | 管理程序 | 不进入任务容器 | +| 干净镜像 | 可信镜像构建 | 版本化、不可由 job 回写 | +| JIT 配置、runner 身份 | 单个容器实例 | 只用于一个 job | +| 容器可写层和任务数据 | 单个容器实例 | 销毁时删除 | +| Docker bridge 网络 | 单个容器实例 | 销毁时删除 | +| 诊断日志 | 管理程序导出 | 有大小和保留期限，不被后续 job 读取 | + +正常路径：申请 JIT → 创建独立网络和容器 → runner 上线 → 执行一个 job → +runner 退出 → 导出日志 → 删除容器与网络 → 清理残留注册 → 创建下一份。 + +JIT 提供正常协议下的单任务注册语义；宿主机管理程序负责实际销毁。 +不能依赖恶意任务自行退出、完成最后一个 cleanup step 或保持 runner 完整。 +容器外 watchdog 强制限制最长存活时间，管理程序重启时先对账旧资源。 + +容器删除失败时停止对应槽位；GitHub 注册清理失败时记录并重试，不继续无限 +申请新身份。宿主机故障期间无法即时清理，恢复后必须先回收再接新任务。 + +## 最少但必要的部署边界 + +- 使用外部 CI 专用 ECS；不与持有长期密钥的现有 runner 共用主机。 +- 不挂载宿主机 home、工作区、管理状态、凭证目录、Docker socket；不使用 + privileged、host PID 或 host network。容器内无需逐命令白名单。 +- 镜像不声明持久 VOLUME，首期不创建任务卷；全部任务写入容器可写层。 +- 宿主机外/容器无法修改的网络策略阻止访问管理接口、敏感内网、实例凭证和 + 元数据；配置实际需要的 GitHub、npm、镜像服务出口。 +- 旧可信 runner 的访问权必须在 GitHub 侧检查。标签和 PR 可修改的 YAML 都 + 不是授权边界；外部 PR 不得通过修改 runs-on 绕回敏感池。 +- 不因启用自托管 runner 而开放 fork secrets，也不引入执行 PR 代码的 + pull_request_target 特权流程。 + +## 首期路由决策 + +保留现有 `ubuntu_runner` 输出及其所有旧语义，新增 `pr_test_runner` 输出， +用于 `test_build` 和 `test`。对于非可信作者的外部 fork PR，变量 +`EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` 选择一次性池；未设置、`false` +或其他值选择 `ubuntu-latest`。`lint_runner` 在 `lint_and_static` 和 +`integration_no_ak` 中遵循同一个开关，替代这些 fork 任务原有的共享 ECS +回退行为。可信作者和同仓库 PR 路由不变。全局 kill-switch 和 hosted 设置优先。 + +开关在 job 求值时选择目标，不查询 runner 是否空闲，也不能迁移已排队任务； +回滚时取消已排队任务并启动新的 workflow run。上线前用实际 shell 选择器 +验证未设置/false/非法值、开启时的两种架构，以及全局覆盖开关。 + +`classify_pr`、`web_shell_e2e_smoke`、`integration_cli`、其他 workflow +保持当前路由。这样不把新镜像尚未验证的浏览器系统依赖一起迁入，也不改变 +现有分类 job 与其他 workflow 的路由镜像关系。 + +新池标签为 `ecs-qwen-ephemeral`。默认路由使用 x64，设置变量 +`EXTERNAL_PR_EPHEMERAL_ARCH=arm64` 可改用 ARM64；完整 PR CI 在新镜像上的 +工具链仍须验证。 + +## 依据 + +- [GitHub ephemeral runner 语义与清理责任](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#ephemeral-runners-for-autoscaling) +- [GitHub 仓库级 JIT API](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository) +- [Docker 的 daemon 与内核安全边界](https://docs.docker.com/engine/security/) + +本设计采用的事实已在本次讨论中核对官方文档；部署时再次确认实际 runner +版本、GitHub 账户能力、API 权限和 ECS 网络配置。 diff --git a/docs/design/2026-11-07-ephemeral-runner-model-proxy.md b/docs/design/2026-11-07-ephemeral-runner-model-proxy.md new file mode 100644 index 00000000000..6a24ff14cae --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-runner-model-proxy.md @@ -0,0 +1,165 @@ +# Combining ephemeral runners with a model proxy + +[English](2026-11-07-ephemeral-runner-model-proxy.md) | [简体中文](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md) + +Date: 2026-11-07. Status: proposal and partial verification complete; the +combined service is neither implemented, deployed nor enabled. + +## Goals and scope + +When CI for an external PR needs a real model, every ephemeral runner uses its +own short-lived, restricted proxy credential. The real model AK goes only to the +trusted model proxy and never into a runner container. After the runner finishes +one job, the management service reclaims the container and revokes the +credential. Existing keyless jobs need no change. + +The existing triage/verify execution model is unchanged, no external PR gains +unrestricted model access by default, and Docker is not claimed to be +virtual-machine isolation from malicious code. This proposal provides neither +release capability, model-output trustworthiness, nor prompt-injection +protection. + +## Current code and verification + +- `scripts/ci/ephemeral-runner/manager.mjs` creates a single-job JIT runner for + each fixed slot, gives every container its own bridge, and replenishes after + reclamation. Model-proxy integration does not exist yet. +- `triage`, `tmux-testing` and `verify` in `.github/workflows/qwen-triage.yml` each embed + a model proxy: it listens on host loopback, validates requests with a random + bearer, substitutes the real AK, forwards only `POST .../chat/completions`, + and supports streaming responses and an idle timeout. These are per-workflow + temporary processes, not standalone gateways. +- Using the proxy extracted from the existing `verify` workflow plus a fake + upstream, we verified that a wrong token returns 401, other paths return 403, + a correct token is forwarded with the upstream receiving only the fake real + AK, and streamed data is returned. Two correct requests both succeed: the + current implementation has no per-request revocation. +- Five selected existing triage tests pass, covering authentication and the + streaming idle timeout for the `tmux-testing` and `verify` lanes; the + `triage` lane embeds the same proxy pattern and is in the extraction scope + below. +- On Hangzhou hz4, with a temporary Docker bridge and a single restricted test + container, we verified that `127.0.0.1` inside the container cannot reach a + loopback service on the host. Test resources were cleaned up. This does **not + prove** that a container cannot reach the host bridge address or other host + services. +- No real model AK was used; an actual proxy deployment, credential issuance + and revocation, quotas, model jobs for + external PRs, and production network isolation were not verified. + +## Trust boundary + +```text +GitHub job queue + ↓ dispatched to a labelled ephemeral runner +External-CI dedicated ECS Trusted proxy service environment + runner management service ──protected issue/revoke──→ model proxy admin API + │ │ + └─ runner container ──short-lived token──────→ model endpoint ──real AK──→ upstream +``` + +The model proxy is preferably deployed on a separate trusted host or service; +the real AK does not live on the runner ECS. The host management service holds +the runner-management PAT and the admin credential for the proxy admin API; +neither is mounted, injected into, or exposed to runner containers. The admin +API and the model endpoint are authorized separately and restricted by TLS and +network policy; containers cannot reach the issue/revoke endpoints, the +management service, the Docker socket, sensitive intranets, or ECS metadata. +Runners are allowed to reach only the model endpoint and the CI egress they +need. + +Containers have sudo, so isolating the real AK from the container's root/node +users cannot be relied on. Plain Docker shares the host kernel; untrusted code +still needs a dedicated host, with virtual-machine isolation considered +according to risk. Network policy is enforced by the host or external network +devices that the container cannot modify, and is verified in practice. + +## Credential and slot lifecycle + +The first version issues credentials per **runner instance**, rather than +pretending to issue per PR or per job: the management service does not yet know +which job GitHub will assign when it creates a runner. Model capability needs +its own dedicated runner label and runner group, used only by explicitly +permitted model-test workflows; model tokens must not be injected into the +default keyless runner pool. Runner group permissions on the GitHub side must be +verified: labels or workflow YAML in a PR are not an authorization boundary. + +State machine for each instance: + +1. Generate a non-reusable instance ID. Request a random token through the + protected admin API, setting model, request size, concurrency, call rate, + total volume and an absolute expiry. Store on the proxy only what validation + needs; persistent state and logs never record the plaintext token. +2. Deliver the proxy address and token to the corresponding new container; the + management service itself must not hand management credentials or the + upstream AK to a container. Supply them as `OPENAI_BASE_URL` and + `OPENAI_API_KEY` for compatible tests. The token is considered public to that + job; log redaction is not treated as a security boundary. +3. An idle runner still consumes its credential's lifetime. In the first + version the credential lifetime covers the instance's maximum lifetime, and + an instance left idle too long is reclaimed together with its runner; no + fixed remaining time at job start can be promised. +4. Normal exit, failure, cancellation, timeout and restart recovery all revoke + the token, delete the container and network, and clean up the GitHub + registration. A new instance gets a new token; old tokens are rejected. +5. When issuance is uncertain, do not create a container that can reach the + model; when revocation fails, stop replenishing that slot and retry, with the + absolute expiry as the final backstop. After a management-service or proxy + restart, reconcile first: the restart must not lose the revocation state or + reset budgets for already-issued tokens. + +Any script in the container can use or leak the token and may incur cost before +revocation; environment variables, container metadata, logs and artifacts are +all untrusted for that job. Credential capability must therefore be limited, +and the whole model pool or proxy endpoint must be switchable off immediately. + +## Proxy features and budget + +Extract a testable shared implementation from the existing forwarding logic and +preserve the behaviour of the three triage lanes; the new standalone service adds +credential issuance, revocation, persistent state, statistics and limits +separately, without changing the permissions or triggers of the existing +workflows. Allow only configured upstream targets, models and exact API paths; +requests must not control the upstream URL or pass an upstream authorization +header. Bound request body size, timeout and concurrency, keep streamed +disconnect handling, and prevent a client from triggering arbitrary address +access or unbounded resource use. + +Set per-instance request counts, rates, concurrency and input/output limits, plus +a shared budget across all instances and a hard quota or alert on the upstream +account. Concurrent debits must be atomic and over-limit requests rejected. +Request-count and output-parameter caps are not an exact cost cap; a streamed +response may still incur cost after disconnect. An exact cost cap depends on the +model provider's metering and quota capability and must be verified once the +upstream is chosen. Do not log plaintext credentials, complete prompts or +responses unless a concrete later requirement and data-handling policy exist. + +## Implementation and acceptance + +1. Determine which model tests actually need external-PR execution, and which + upstream model and budget they use; by default open exactly one controlled + test workflow and do not change the existing `integration_no_ak` or other CI + routing. +2. Extract and test the HTTP/streaming core of the existing proxy, preserving + both triage lanes. Implement the standalone proxy's admin API, separate + credentials, permissions, persistence, expiry, revocation and budget. +3. Add optional model-pool lifecycle to the management service; fail closed when + issuance or cleanup is uncertain. Cover token injection, log redaction, + abnormal recovery and old-token invalidation in tests. +4. In an isolated test environment with a fake upstream, one slot and a + controlled private repository, verify two consecutive jobs: the two tokens + differ; normal and streamed calls succeed; wrong, old, revoked and expired + tokens are rejected; over-limit requests, concurrency races, upstream + failures, and management-service/proxy restarts do not bypass the limits. +5. From inside a job, verify that the real AK, the runner-management PAT and the proxy + admin credential are unreadable, and that metadata, the Docker socket, other + containers and host management services are unreachable; verify that the + proxy can reach only the permitted upstream. Both container-cleanup and + credential-revocation failures must stop slot replenishment. +6. Prepare isolated dedicated ECS capacity and proxy services, and measure the + network and GitHub authorization, then run a one-slot trial with a low-budget + real model credential. Only after it passes does a maintainer decide whether + to enable the corresponding model-test routing. + +The existing runner design is described in +[the ephemeral runner design](2026-11-07-ephemeral-container-runners.md). diff --git a/docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md b/docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md new file mode 100644 index 00000000000..feea6dd7684 --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md @@ -0,0 +1,116 @@ +# 一次性 runner 与模型代理的结合方案 + +[English](2026-11-07-ephemeral-runner-model-proxy.md) | [简体中文](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md) + +日期：2026-11-07。状态：方案与局部验证完成；尚未实现组合服务、部署或启用。 + +## 目标与范围 + +外部 PR 的 CI 需要调用真实模型时，每个一次性 runner 使用独立、短期、受限的 +代理凭证。真实模型 AK 只交给可信模型代理，不进入 runner 容器。runner 执行 +一个 job 后，管理服务回收容器并撤销该凭证。原有无 AK 的 job 不需要改变。 + +不改变现有 triage/verify 的运行方式，不让所有外部 PR 默认获得无限制模型 +访问，不承诺 Docker 是对恶意代码的虚拟机级隔离。本方案不提供发布、模型输出 +可信性或 prompt 注入防护。 + +## 当前代码与验证 + +- `scripts/ci/ephemeral-runner/manager.mjs` 为固定槽位创建单 job JIT runner， + 每个容器使用单独的 bridge，回收完毕后再补充。当前没有模型代理集成。 +- `.github/workflows/qwen-triage.yml` 的 `triage`、`tmux-testing` 与 `verify` 各有一份 + 内嵌的模型代理：监听本机 loopback，以随机 bearer 校验请求，替换为真实 AK， + 只转发 `POST .../chat/completions`，支持流式响应与空闲超时。它们是各自 + workflow 的临时进程，不是独立网关。 +- 使用从现有 `verify` workflow 提取的代理和假上游，验证了错误 token 返回 + 401、其他路径返回 403、正确 token 转发且上游只收到假真实 AK，以及流式 + 数据返回。两个正确请求都能成功：现有实现没有按请求数撤销 token 的行为。 +- 现有 triage 测试中选中的 5 项通过，包含认证，以及 `tmux-testing` 与 `verify` + 两条 lane 的流式空闲超时；`triage` lane 使用同一套代理形态，也纳入下述提取范围。 +- 在杭州 hz4 通过一个临时 Docker bridge 和单个受限测试容器验证：容器内 + `127.0.0.1` 不能访问宿主机的 loopback 服务。测试资源已清理。这**不证明** + 容器不能访问宿主机 bridge 地址或其他宿主机服务。 +- 未使用真实模型 AK；未验证实际代理部署、凭证签发与 + 撤销、配额、外部 PR 的模型 job、或生产网络隔离。 + +## 信任边界 + +```text +GitHub job 队列 + ↓ 分派到带标签的一次性 runner +外部 CI 专用 ECS 可信代理服务环境 + runner 管理服务 ──受保护的签发/撤销──→ 模型代理管理接口 + │ │ + └─ runner 容器 ──短期 token────→ 模型调用接口 ──真实 AK──→ 上游 +``` + +模型代理优先部署在独立可信主机或服务，真实 AK 不放在 runner ECS。宿主机 +管理服务保存 runner 管理 PAT 和调用代理管理接口的管理凭证；这些都不挂载、 +注入或暴露给 runner 容器。管理接口与模型调用接口分开授权，并通过 TLS 和 +网络策略限制来源；容器不能访问签发、撤销接口、管理服务、Docker socket、 +敏感内网和 ECS 元数据。runner 仅获准访问模型调用接口及必要 CI 出口。 + +容器有 sudo，不能依赖容器内 root/node 用户隔离真实 AK。普通 Docker 共享 +宿主机内核；涉及不可信代码时仍需专用主机，按风险考虑虚拟机级隔离。网络 +策略由容器不能修改的宿主机或外部网络设备执行并实际验证。 + +## 凭证和槽位生命周期 + +第一版按 **runner 实例**签发，而不是假装按 PR 或 job 签发：管理服务创建 +runner 时还不知道 GitHub 将分配哪个 job。模型能力需使用独立的专用 runner +标签与 runner group，只供经明确允许的模型测试 workflow 使用；不要把模型 +token 注入默认无 AK 的 runner 池。GitHub 侧 runner group 权限必须经过核验， +标签或 PR 中的 workflow YAML 不是授权边界。 + +每个实例的状态机： + +1. 生成不可复用的实例 ID。通过受保护的管理接口申请随机 token，设置模型、 + 请求大小、并发、调用速率、总量及绝对过期时间。只在代理端保存验证所需 + 信息；持久状态和日志不记录明文 token。 +2. 将 proxy 地址和 token 交付给对应新容器；管理服务自身不能将管理凭证或 + 上游 AK 交给容器。以 `OPENAI_BASE_URL`、`OPENAI_API_KEY` 供兼容的测试 + 使用，token 对这个 job 视为已公开，不依赖日志脱敏作为安全边界。 +3. runner 空闲时也占用凭证有效期。第一版凭证有效期覆盖实例最长生命周期， + 空闲太久后连 runner 一起回收；不能承诺任务开始时仍有固定剩余时间。 +4. 正常退出、失败、取消、超时或重启恢复均撤销 token、删除容器和网络、 + 清理 GitHub 注册。新实例获得新 token；旧 token 被拒绝。 +5. 签发结果不确定时，不创建可访问模型的容器；撤销失败时停止对应槽位补充 + 并重试，凭证绝对过期作为最终兜底。管理服务/代理重启后应先对账，不能 + 因重启丢失已签发 token 的撤销状态或重置预算。 + +容器中任何脚本都能使用或泄露 token，且撤销前可能产生费用；环境变量、 +容器元数据、日志和 artifact 都不能被视作对该 job 保密。必须限制凭证能力， +并允许立即整体关闭模型池或代理调用接口。 + +## 代理功能与预算 + +从现有转发逻辑提取可测试的共享实现，保持已有三条 triage lane 行为； +新的独立服务另加凭证签发、吊销、持久状态、统计和限额，不直接修改既有 +workflow 的权限或触发方式。只允许配置好的上游目标、模型和确切 API 路径， +不得让请求控制上游 URL 或传递上游授权头；限制请求体大小、超时和并发， +保留流式断连处理，避免客户端触发任意地址访问或无限资源消耗。 + +按实例设置请求数、速率、并发和输入/输出上限，并设置所有实例共享的总预算 +及上游账户侧的硬额度或告警。并发扣额需要原子处理，拒绝超额请求。请求数 +和输出参数上限并不等于精确费用上限；流式响应可能在断连后仍产生费用。 +精确成本上限依赖模型供应方的计量/限额能力，应在选定上游后验证。不要记录 +明文凭证、完整 prompt 或回复，除非后续有明确需求和数据处理策略。 + +## 实施和验收 + +1. 确定哪些模型测试确实需要外部 PR 执行、上游模型及预算；默认只开放一条 + 受控测试 workflow，先不要改变现有 `integration_no_ak` 或其他 CI 路由。 +2. 提取并测试现有 proxy 的 HTTP/流式核心，保留三个 triage lane 行为。实现 + 独立代理的管理接口、独立凭证、权限、持久化、过期、撤销和预算。 +3. 为管理服务增加可选的模型池生命周期；签发或清理不确定时 fail closed。 + 明确 token 注入、日志脱敏、异常恢复及旧 token 失效的测试。 +4. 在独立测试环境使用假上游、1 个槽位及受控私有仓库，验证两次连续 job： + 两个 token 不同；正常和流式调用通过；错误、旧、撤销和过期 token 被拒； + 超额、并发竞争、上游故障、管理服务/代理重启不绕过限制。 +5. 从 job 内验证不能读到真实 AK、runner 管理 PAT、代理管理凭证，也不能访问元数据、 + Docker socket、其他容器和宿主机管理服务；验证代理只能访问允许的上游。 + 容器清理和凭证撤销失败均阻止槽位补充。 +6. 准备隔离的专用 ECS 和代理服务、实测网络与 GitHub 授权，再用低预算真实 + 模型凭证进行一槽位试运行。通过后由维护者决定是否启用相应模型测试路由。 + +现有 runner 方案见 [一次性 runner 设计](2026-11-07-ephemeral-container-runners.zh-CN.md)。 diff --git a/scripts/ci/ephemeral-runner/Dockerfile b/scripts/ci/ephemeral-runner/Dockerfile new file mode 100644 index 00000000000..9d530fe7929 --- /dev/null +++ b/scripts/ci/ephemeral-runner/Dockerfile @@ -0,0 +1,39 @@ +ARG BASE_IMAGE=node:22-bookworm-slim@sha256:5ec23b7c622b00547753bc6ffd78085dd868a68aeb41c8e0a1418f033e13d96d +FROM ${BASE_IMAGE} + +ARG TARGETARCH +ARG RUNNER_VERSION=2.338.2 +ARG RUNNER_RELEASE_URL=https://github.com/actions/runner/releases/download +ARG DEBIAN_MIRROR=deb.debian.org +ARG PIP_INDEX_URL=https://pypi.org/simple + +RUN sed -i "s|deb.debian.org|${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources \ + && apt-get update -o Acquire::http::Timeout=30 \ + && apt-get install -y --no-install-recommends \ + build-essential ca-certificates curl file gh git git-lfs jq libicu72 \ + libkrb5-3 libssl3 libunwind8 procps python3 python3-pip python3-venv \ + sudo tmux unzip zip zlib1g \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m venv /opt/ci-tools \ + && /opt/ci-tools/bin/pip install --no-cache-dir 'yamllint==1.35.1' \ + && echo 'node ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/runner + +ENV PATH="/opt/ci-tools/bin:${PATH}" + +WORKDIR /home/node/actions-runner +RUN case "${TARGETARCH}" in \ + amd64) runner_arch=x64; runner_sha=baebf204307bac6193480852732f0aba2ae97b6c5597c244f4d5558390147c40 ;; \ + arm64) runner_arch=arm64; runner_sha=670adb2f268fe21dcbb58b3db0e0aa9d5b3bddfe2c651481336e561ff86de945 ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl -fsSL --retry 3 --connect-timeout 15 --max-time 180 -o /tmp/runner.tar.gz \ + "${RUNNER_RELEASE_URL}/v${RUNNER_VERSION}/actions-runner-linux-${runner_arch}-${RUNNER_VERSION}.tar.gz" \ + && echo "${runner_sha} /tmp/runner.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/runner.tar.gz \ + && rm /tmp/runner.tar.gz \ + && chown -R node:node /home/node/actions-runner + +COPY entrypoint.sh /usr/local/bin/ephemeral-runner +RUN chmod 755 /usr/local/bin/ephemeral-runner +USER node +ENTRYPOINT ["/usr/local/bin/ephemeral-runner"] diff --git a/scripts/ci/ephemeral-runner/README.md b/scripts/ci/ephemeral-runner/README.md new file mode 100644 index 00000000000..d1e40510b5a --- /dev/null +++ b/scripts/ci/ephemeral-runner/README.md @@ -0,0 +1,140 @@ +# Ephemeral GitHub Actions runners + +Each slot registers one just-in-time runner and runs it in a new container. +After at most one job, the manager removes the container and runner registration +before creating the next one. While the manager waits on a container it also +enforces a per-slot deadline and removes a container whose runner does not exit; +a manager crash is covered by reconciling leftover resources on the next start. + +Each job has a separate Docker bridge with inter-container communication +disabled. Deleting the network is part of cleanup and a failure blocks slot +replacement. This does not itself prevent access to the host or private +networks: configure and verify the host's external egress controls separately. + +Use a dedicated host for untrusted pull requests. Docker shares the host kernel; +jobs must not receive the host Docker socket, host directories, cloud instance +credentials, internal network access, or long-lived secrets. A runner label is a +scheduling selector, not an access-control boundary. Before enabling external +pull requests, verify they cannot select other sensitive runner pools. + +## Build the image + +The image supports Linux x64 and ARM64. Its Node base is pinned to an OCI index +digest and the runner downloads are checked against pinned SHA-256 digests. +Use BuildKit to select both the base image and runner architecture: + +```sh +docker buildx build --platform linux/arm64 --load -t qwen-ephemeral-runner:trial scripts/ci/ephemeral-runner +``` + +BuildKit supplies `TARGETARCH`. With Docker's legacy builder, build on the target +architecture and pass its matching `TARGETARCH` explicitly (`amd64` or `arm64`). +That argument selects only the runner archive; it does not change the base image +architecture. + +The runner image supports ARM64, but full PR CI on this image has not yet been +verified. The repository also selects ARM64 linters; verify the complete CI +toolchain before routing production work to either architecture. + +Hosts with restricted upstream connectivity can set `DEBIAN_MIRROR`, +`PIP_INDEX_URL` or `RUNNER_RELEASE_URL` at build time. The runner archive is +still verified against its pinned digest when served from a mirror. + +For a real deployment, publish the image to a registry and configure its +immutable `repository@sha256:...` digest. A local `sha256:...` image ID may be +used for a trial. Runner and base versions need routine security updates. + +## Configure the manager + +Run one manager on each host, managing only that host's Docker socket. Give each +host a unique `poolId`; matching runner labels let GitHub share work across the +hosts without a central scheduler. `slots` is a per-host limit, not a fleet total. +New runner names start with `ecs-qwen-` so the existing CI worker caps and test +timeouts apply. Restart recovery also accepts the earlier `qwen-` identities. + +Install Docker, Node 22, `gh`, and `flock` on the host. The manager uses a +dedicated PAT supplied as `GH_TOKEN`. The account must administer the target +repository. Prefer a fine-grained PAT restricted to that repository with +Administration: write and a short expiry. GitHub currently requires this broader +permission for runner registration; it is not a runner-only credential. Some +organization or outside-collaborator policies prevent fine-grained access. Do +not silently replace it with a classic `repo` token, which can cover other +repositories the account can access. + +Store `GH_TOKEN=` in `/etc/qwen-ephemeral-runner/credentials.env`, owned by +root with mode `600`, inside the root-only configuration directory. The supplied +systemd unit loads this file; do not put it in shell history, config JSON, job +environments or the repository. Replace it and restart the service to rotate or +renew the token. The service fails startup if `GH_TOKEN` is missing. Docker child +environments are allowlisted, and only the one-time JIT configuration enters a +job container. Host root and Docker administrators are trusted and can access +host credentials; ordinary container isolation does not change that boundary. + +Copy `config.example.json` to `/etc/qwen-ephemeral-runner/config.json` and +replace the placeholders: repository, actual runner group ID, unique host/pool +ID, immutable image reference, and matching architecture and labels. Use a +unique test label and a controlled test repository for a trial. The supplied +manual smoke workflow selects only `ecs-qwen-ephemeral-smoke`; a trial runner +can register with just that label to avoid matching generic Linux jobs. The example runs +one slot; increase only after measuring the host. Restrict configuration and +credential directories to the manager account. + +The `ttlSeconds` limit starts when the container starts, including idle time. +A job assigned near the deadline can be interrupted. The manager removes and +replaces the container; GitHub remains authoritative for job status. The +manager logs cleanup but does not distinguish expiry from normal completion. +A nonzero or unknown container exit status delays replenishment by 60 seconds; +a successful short job is replenished immediately. GitHub remains authoritative +for job success. Diagnostic export is best effort: stderr is not archived, and +output exceeding the capture limits is discarded. +`minDiskBytes` applies to the Docker data filesystem. Configure and verify +disk quotas or a dedicated data volume as appropriate for the host. + +## Install and operate + +Install `manager.mjs` in `/opt/qwen-ephemeral-runner/`, install the supplied +unit in `/etc/systemd/system/`, and create the configured state/log directories +with permissions restricted to the manager account. From the trusted checkout: + +```sh +sudo install -d /opt/qwen-ephemeral-runner +sudo install -d -m 700 /etc/qwen-ephemeral-runner /var/lib/qwen-ephemeral-runner /var/log/qwen-ephemeral-runner +sudo install -m 644 scripts/ci/ephemeral-runner/manager.mjs /opt/qwen-ephemeral-runner/manager.mjs +sudo install -m 644 scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service /etc/systemd/system/qwen-ephemeral-runner.service +``` + +Finish the configuration and PAT setup above before starting. A Docker-authorized user +has host-level power; the unit is intended for a trusted dedicated host. + +```sh +sudo systemctl daemon-reload +sudo systemctl enable --now qwen-ephemeral-runner.service +sudo systemctl status qwen-ephemeral-runner.service +sudo journalctl -u qwen-ephemeral-runner.service -f +``` + +systemd restarts an unexpectedly exited manager after ten seconds, including a +clean exit. An explicit `systemctl stop` leaves it stopped. This is process +supervision, not hang detection; inspect the journal and runner availability if +slots are not replenishing. + +Stopping the service stops replenishment, waits up to the configured grace +period, and cleans up its containers. The next start reconciles abandoned +resources first. A cleanup or ambiguous registration error blocks that slot; +inspect its state and GitHub runner list before manual reconciliation. Never +remove unrelated runners or run a global Docker prune. + +```sh +sudo systemctl stop qwen-ephemeral-runner.service +``` + +After validating a single runner and its recovery paths, set the repository +variable `EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` to route untrusted fork PR +build, test, lint and no-AK integration jobs to this pool. Unset, `false`, or any +other value keeps those jobs on `ubuntu-latest`. Trusted and same-repository PRs +retain their existing routing. The global kill-switch and hosted-fleet override +still win. Leave the switch off until the host and GitHub access policies have +been verified. Changing it affects newly evaluated jobs; cancel queued work and +start a new workflow run when rolling back. It does not detect idle runners or +migrate a queued job automatically. +Set `EXTERNAL_PR_EPHEMERAL_ARCH=arm64` for an ARM64 pool; the default is x64. diff --git a/scripts/ci/ephemeral-runner/config.example.json b/scripts/ci/ephemeral-runner/config.example.json new file mode 100644 index 00000000000..f00ef88f539 --- /dev/null +++ b/scripts/ci/ephemeral-runner/config.example.json @@ -0,0 +1,16 @@ +{ + "repo": "QwenLM/qwen-code", + "runnerGroupId": 1, + "poolId": "external-ci-host-1", + "image": "registry.example.invalid/qwen-runner@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "labels": ["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"], + "slots": 1, + "cpus": 4, + "memory": "8g", + "pidsLimit": 1024, + "ttlSeconds": 10800, + "shutdownGraceSeconds": 60, + "minDiskBytes": 10737418240, + "stateDir": "/var/lib/qwen-ephemeral-runner", + "logDir": "/var/log/qwen-ephemeral-runner" +} diff --git a/scripts/ci/ephemeral-runner/entrypoint.sh b/scripts/ci/ephemeral-runner/entrypoint.sh new file mode 100644 index 00000000000..34c03f4347b --- /dev/null +++ b/scripts/ci/ephemeral-runner/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${RUNNER_JIT_CONFIG:?RUNNER_JIT_CONFIG is required}" +jit_config="$RUNNER_JIT_CONFIG" +unset RUNNER_JIT_CONFIG +cd /home/node/actions-runner +exec ./run.sh --jitconfig "$jit_config" diff --git a/scripts/ci/ephemeral-runner/manager.mjs b/scripts/ci/ephemeral-runner/manager.mjs new file mode 100644 index 00000000000..4b1d75c9b24 --- /dev/null +++ b/scripts/ci/ephemeral-runner/manager.mjs @@ -0,0 +1,620 @@ +import { randomUUID } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { realpathSync } from 'node:fs'; +import { + lstat, + mkdir, + readFile, + readdir, + rename, + rm, + statfs, + writeFile, +} from 'node:fs/promises'; +import { isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const sleep = (ms) => + new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); + +export function command( + program, + args, + { env = process.env, input, timeout = 30000, binary = false } = {}, +) { + return new Promise((resolveCommand, reject) => { + const child = spawn(program, args, { + env, + stdio: ['pipe', 'pipe', 'pipe'], + }); + const output = []; + let outputSize = 0; + let stderr = ''; + const timer = setTimeout(() => child.kill('SIGKILL'), timeout); + child.stdout.on('data', (chunk) => { + outputSize += chunk.length; + if (outputSize > 4 * 1024 * 1024) child.kill('SIGKILL'); + else output.push(chunk); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + if (stderr.length > 1024 * 1024) child.kill('SIGKILL'); + }); + child.on('error', (error) => { + clearTimeout(timer); + reject(error); + }); + child.on('close', (code) => { + clearTimeout(timer); + if (code === 0 && outputSize <= 4 * 1024 * 1024) + resolveCommand( + binary ? Buffer.concat(output) : Buffer.concat(output).toString(), + ); + else { + const status = Number(stderr.match(/HTTP (\d{3})/)?.[1] || 0); + const error = new Error( + `${program} failed (exit ${code}${status ? `, HTTP ${status}` : ''})`, + ); + error.status = status; + error.exitCode = code; + reject(error); + } + }); + child.stdin.on('error', () => {}); + child.stdin.end(input); + }); +} + +function validate(c) { + if ( + !/^[\w.-]+\/[\w.-]+$/.test(c.repo || '') || + !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,39}$/.test(c.poolId || '') || + !/^(?:[a-zA-Z0-9][a-zA-Z0-9._:/-]*@)?sha256:[a-f0-9]{64}$/.test( + c.image || '', + ) || + !Number.isInteger(c.slots) || + c.slots < 1 || + c.slots > 10 || + !Number.isInteger(c.runnerGroupId) || + c.runnerGroupId < 1 || + !Array.isArray(c.labels) || + !c.labels.length || + !c.labels.every( + (label) => + typeof label === 'string' && /^[a-zA-Z0-9_-]{1,63}$/.test(label), + ) || + typeof c.stateDir !== 'string' || + !isAbsolute(c.stateDir) || + typeof c.logDir !== 'string' || + !isAbsolute(c.logDir) || + (c.dockerSocket !== undefined && + (typeof c.dockerSocket !== 'string' || + !isAbsolute(c.dockerSocket) || + c.dockerSocket.includes('\n'))) || + !Number.isFinite(c.cpus) || + !(c.cpus > 0) || + !/^[1-9]\d*[mMgG]$/.test(c.memory || '') || + !Number.isInteger(c.pidsLimit) || + c.pidsLimit < 1 || + !Number.isFinite(c.ttlSeconds) || + !(c.ttlSeconds > 0) || + !Number.isFinite(c.shutdownGraceSeconds) || + !(c.shutdownGraceSeconds >= 0) || + !Number.isSafeInteger(c.minDiskBytes) || + c.minDiskBytes < 0 + ) + throw new Error('Invalid manager configuration'); +} + +export function createManager(config, dependencies = {}) { + validate(config); + const execute = dependencies.command || command; + const run = (program, args, options = {}) => { + if (program !== 'docker') return execute(program, args, options); + const source = options.env || process.env; + const env = {}; + for (const key of ['PATH', 'HOME', 'DOCKER_CONFIG', 'XDG_RUNTIME_DIR']) { + if (source[key] !== undefined) env[key] = source[key]; + } + env.DOCKER_HOST = `unix://${config.dockerSocket || '/var/run/docker.sock'}`; + if (args[0] === 'create' && options.env?.RUNNER_JIT_CONFIG) + env.RUNNER_JIT_CONFIG = options.env.RUNNER_JIT_CONFIG; + return execute(program, args, { ...options, env }); + }; + const wait = dependencies.sleep || sleep; + const diskStats = dependencies.statfs || statfs; + let stopping = false; + let stopAt = Infinity; + const stateDir = resolve(config.stateDir); + const logDir = resolve(config.logDir); + const lockPath = join(stateDir, 'manager.lock'); + const slotPath = (slot) => join(stateDir, `slot-${slot}.json`); + const log = (message) => console.log(`[${config.poolId}] ${message}`); + function parseJson(text) { + try { + return JSON.parse(text); + } catch { + throw new Error('Invalid JSON response from GitHub'); + } + } + + async function gh(args, input) { + return run('gh', ['api', ...args, ...(input ? ['--input', '-'] : [])], { + env: process.env, + input: input && JSON.stringify(input), + }); + } + + async function save(slot, state) { + const path = slotPath(slot); + const temp = `${path}.${randomUUID()}.tmp`; + await writeFile(temp, JSON.stringify(state), { mode: 0o600 }); + await rename(temp, path); + } + + async function load(slot) { + try { + const raw = await readFile(slotPath(slot), 'utf8'); + let state; + try { + state = JSON.parse(raw); + } catch { + throw new Error('Invalid slot state; manual reconciliation required'); + } + // Reconcile identities created before the ECS-compatible name prefix. + const name = + typeof state?.name === 'string' ? state.name.replace(/^ecs-/, '') : ''; + const prefix = `qwen-${config.poolId}-${slot}-`; + if ( + !name.startsWith(prefix) || + !/^[0-9a-f-]{36}$/.test(name.slice(prefix.length)) || + typeof state.uncertain !== 'boolean' || + (!state.uncertain && + (!Number.isSafeInteger(state.runnerId) || state.runnerId < 1)) || + (state.runnerId !== undefined && + (!Number.isSafeInteger(state.runnerId) || state.runnerId < 1)) + ) + throw new Error('Invalid slot state; manual reconciliation required'); + return state; + } catch (error) { + if (error.code === 'ENOENT') return null; + throw error; + } + } + + async function listContainers(slot) { + const output = await run('docker', [ + 'ps', + '-aq', + '--no-trunc', + '--filter', + `label=qwen.pool=${config.poolId}`, + '--filter', + `label=qwen.slot=${slot}`, + ]); + return output.trim().split('\n').filter(Boolean); + } + + async function listNetworks(slot) { + const args = [ + 'network', + 'ls', + '-q', + '--no-trunc', + '--filter', + `label=qwen.pool=${config.poolId}`, + ]; + if (slot !== undefined) args.push('--filter', `label=qwen.slot=${slot}`); + return (await run('docker', args)).trim().split('\n').filter(Boolean); + } + + async function listRunners() { + const output = await gh([ + '--paginate', + '--jq', + '.runners | @json', + `/repos/${config.repo}/actions/runners?per_page=100`, + ]); + const pages = output.trim().split('\n').map(parseJson); + if ( + !pages.every( + (runners) => + Array.isArray(runners) && + runners.every( + (runner) => + typeof runner?.name === 'string' && + Number.isSafeInteger(runner.id) && + runner.id > 0, + ), + ) + ) + throw new Error('Invalid GitHub runner list'); + return pages.flat(); + } + + async function pruneLogs() { + const files = []; + for (const entry of await readdir(logDir, { withFileTypes: true })) { + if ( + !entry.isFile() || + !entry.name.replace(/^ecs-/, '').startsWith(`qwen-${config.poolId}-`) || + !/\.(log|tar)$/.test(entry.name) + ) + continue; + const path = join(logDir, entry.name); + const info = await lstat(path); + files.push({ path, size: info.size, time: info.mtimeMs }); + } + files.sort((a, b) => b.time - a.time); + let size = 0; + for (const file of files) { + size += file.size; + if (file.time < Date.now() - 7 * 86400000 || size > 100 * 1024 * 1024) + await rm(file.path, { force: true }); + } + } + + async function cleanup(slot, state) { + const containers = await listContainers(slot); + const networks = await listNetworks(slot); + if ((containers.length || networks.length) && !state?.name) + throw new Error( + 'Container has no recorded runner identity; manual reconciliation required', + ); + for (const id of containers) { + try { + const logs = await run('docker', ['logs', '--tail', '10000', id]); + await writeFile( + join( + logDir, + `${state?.name || `slot-${slot}`}-${id.slice(0, 12)}.log`, + ), + logs.slice(-1024 * 1024), + { mode: 0o600 }, + ); + await pruneLogs(); + } catch { + log(`slot ${slot}: container logs unavailable`); + } + try { + const archive = await run( + 'docker', + ['cp', `${id}:/home/node/actions-runner/_diag`, '-'], + { binary: true }, + ); + await writeFile( + join(logDir, `${state.name}-${id.slice(0, 12)}.tar`), + archive, + { mode: 0o600 }, + ); + await pruneLogs(); + } catch { + log( + `slot ${slot}: runner diagnostics unavailable or larger than 4 MiB`, + ); + } + await run('docker', ['rm', '-f', '-v', id]); + } + if ((await listContainers(slot)).length) + throw new Error('Container cleanup failed'); + for (const id of networks) await run('docker', ['network', 'rm', id]); + if ((await listNetworks(slot)).length) + throw new Error('Network cleanup failed'); + if (state?.name) { + const all = await listRunners(); + const runners = all.filter((runner) => runner.name === state.name); + if ( + state.runnerId && + (runners.some((runner) => runner.id !== state.runnerId) || + all.some( + (runner) => + runner.id === state.runnerId && runner.name !== state.name, + )) + ) + throw new Error( + 'Runner identity mismatch; manual reconciliation required', + ); + if ( + runners.length > 1 || + runners.some( + (runner) => !Number.isSafeInteger(runner.id) || runner.id < 1, + ) + ) + throw new Error( + 'Ambiguous runner identity; manual reconciliation required', + ); + if (state.uncertain && !runners.length) + throw new Error( + 'JIT response was lost; runner not found, manual reconciliation required', + ); + for (const runner of runners) { + try { + await gh([ + '-X', + 'DELETE', + `/repos/${config.repo}/actions/runners/${runner.id}`, + ]); + } catch (error) { + if (error.status !== 404) throw error; + } + } + } + await rm(slotPath(slot), { force: true }); + log(`slot ${slot}: cleaned`); + } + + async function cycle(slot) { + let earlyExit = false; + let state = await load(slot); + if ( + state || + (await listContainers(slot)).length || + (await listNetworks(slot)).length + ) + await cleanup(slot, state); + if (stopping) return; + const dockerRoot = ( + await run('docker', ['info', '--format', '{{.DockerRootDir}}']) + ).trim(); + if (!isAbsolute(dockerRoot) || dockerRoot.includes('\n')) + throw new Error('Invalid Docker Root Dir'); + const disk = await diskStats(dockerRoot); + if (disk.bavail * disk.bsize < config.minDiskBytes) + throw new Error( + `Insufficient disk capacity on Docker Root Dir ${dockerRoot}`, + ); + state = { + name: `ecs-qwen-${config.poolId}-${slot}-${randomUUID()}`, + createdAt: new Date().toISOString(), + uncertain: true, + }; + await save(slot, state); + let result; + try { + result = parseJson( + await gh( + [ + '-X', + 'POST', + `/repos/${config.repo}/actions/runners/generate-jitconfig`, + ], + { + name: state.name, + runner_group_id: config.runnerGroupId, + labels: config.labels, + work_folder: '_work', + }, + ), + ); + } catch (error) { + if (error.status >= 400 && error.status < 500 && error.status !== 408) { + await rm(slotPath(slot), { force: true }); + } else { + await cleanup(slot, state); + } + throw error; + } + try { + if ( + result?.runner?.name !== state.name || + !Number.isSafeInteger(result.runner.id) || + result.runner.id < 1 + ) + throw new Error('Invalid JIT runner identity'); + state.runnerId = result.runner.id; + state.uncertain = false; + await save(slot, state); + if (!result.encoded_jit_config) + throw new Error('GitHub returned no JIT configuration'); + const networkId = ( + await run('docker', [ + 'network', + 'create', + '--driver', + 'bridge', + '--opt', + 'com.docker.network.bridge.enable_icc=false', + '--label', + `qwen.pool=${config.poolId}`, + '--label', + `qwen.slot=${slot}`, + '--label', + `qwen.instance=${state.name}`, + state.name, + ]) + ).trim(); + if (!networkId) throw new Error('Docker returned no network ID'); + state.networkId = networkId; + await save(slot, state); + const env = { + ...process.env, + RUNNER_JIT_CONFIG: result.encoded_jit_config, + }; + delete env.GH_TOKEN; + delete env.GITHUB_TOKEN; + const id = ( + await run( + 'docker', + [ + 'create', + '--init', + '--network', + networkId, + '--label', + `qwen.pool=${config.poolId}`, + '--label', + `qwen.slot=${slot}`, + '--label', + `qwen.instance=${state.name}`, + '--cpus', + String(config.cpus), + '--memory', + config.memory, + '--pids-limit', + String(config.pidsLimit), + '--log-driver', + 'json-file', + '--log-opt', + 'max-size=10m', + '--log-opt', + 'max-file=3', + '--env', + 'RUNNER_JIT_CONFIG', + config.image, + ], + { env }, + ) + ).trim(); + state.containerId = id; + await save(slot, state); + if (!id) throw new Error('Docker returned no container ID'); + if (!stopping) { + await run('docker', ['start', id]); + log( + `slot ${slot}: started runner ${state.name}, id ${state.runnerId}, container ${id}`, + ); + const deadline = Date.now() + config.ttlSeconds * 1000; + let exitCode; + const watching = run('docker', ['wait', id], { + timeout: (config.ttlSeconds + 10) * 1000, + }).then( + (output) => { + const status = output.trim(); + exitCode = + /^\d+$/.test(status) && Number(status) <= 255 + ? Number(status) + : null; + return true; + }, + () => { + exitCode = null; + return true; + }, + ); + while (Date.now() < Math.min(deadline, stopAt)) { + if (await Promise.race([watching, wait(1000).then(() => false)])) + break; + } + if (exitCode !== undefined && exitCode !== 0) { + earlyExit = true; + log( + `slot ${slot}: runner exit status ${exitCode ?? 'unknown'}; next JIT request is delayed`, + ); + } + } + } finally { + await cleanup(slot, state); + } + return { earlyExit }; + } + + async function acquireLock() { + await mkdir(stateDir, { recursive: true, mode: 0o700 }); + await mkdir(logDir, { recursive: true, mode: 0o700 }); + const child = spawn('flock', ['-n', lockPath, 'cat'], { + stdio: ['pipe', 'pipe', 'ignore'], + }); + await new Promise((resolveLock, reject) => { + child.once('error', reject); + child.once('exit', () => + reject(new Error('Could not acquire manager lock')), + ); + child.stdout.once('data', resolveLock); + child.stdin.on('error', () => {}); + child.stdin.write('locked\n'); + }); + return child; + } + + async function start() { + const lock = await acquireLock(); + try { + const slots = ( + await run('docker', [ + 'ps', + '-a', + '--filter', + `label=qwen.pool=${config.poolId}`, + '--format', + '{{.Label "qwen.slot"}}', + ]) + ) + .trim() + .split('\n') + .filter(Boolean); + const states = (await readdir(stateDir)) + .map((name) => name.match(/^slot-(\d+)\.json$/)?.[1]) + .filter(Boolean); + const networkSlots = await Promise.all( + (await listNetworks()).map(async (id) => + ( + await run('docker', [ + 'network', + 'inspect', + '--format', + '{{index .Labels "qwen.slot"}}', + id, + ]) + ).trim(), + ), + ); + if ( + [...slots, ...states, ...networkSlots].some( + (slot) => + !/^\d+$/.test(slot) || + Number(slot) < 1 || + Number(slot) > config.slots, + ) + ) { + throw new Error( + 'Pool contains resources outside configured slots; reconcile before starting', + ); + } + await Promise.all( + Array.from({ length: config.slots }, async (_, i) => { + const slot = i + 1; + while (!stopping) { + try { + const result = await cycle(slot); + if (result?.earlyExit && !stopping) { + log(`slot ${slot}: backing off 60 s before replenishing`); + await wait(60000); + } + } catch (error) { + log( + `slot ${slot}: ${error.message}; retrying without replenishing`, + ); + await wait(10000); + } + } + }), + ); + } finally { + lock.stdin.end(); + } + } + function stop() { + stopping = true; + stopAt = Math.min(stopAt, Date.now() + config.shutdownGraceSeconds * 1000); + } + return { start, stop, cycle, cleanup }; +} + +if ( + process.argv[1] && + realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + if (!process.env.GH_TOKEN?.trim()) throw new Error('GH_TOKEN is required'); + const config = JSON.parse( + await readFile(resolve(process.argv[2] || ''), 'utf8'), + ); + const manager = createManager(config); + process.on('SIGTERM', manager.stop); + process.on('SIGINT', manager.stop); + await manager.start(); + } catch (error) { + console.error(`manager: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/scripts/ci/ephemeral-runner/manager.test.mjs b/scripts/ci/ephemeral-runner/manager.test.mjs new file mode 100644 index 00000000000..62c82c5a8d6 --- /dev/null +++ b/scripts/ci/ephemeral-runner/manager.test.mjs @@ -0,0 +1,822 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { command as runCommand, createManager } from './manager.mjs'; + +async function fixture(t) { + const dir = await mkdtemp(join(tmpdir(), 'runner-manager-')); + t.after(() => rm(dir, { recursive: true, force: true })); + await mkdir(join(dir, 'state')); + await mkdir(join(dir, 'logs')); + const config = { + repo: 'owner/repo', + poolId: 'trial', + image: `image@sha256:${'a'.repeat(64)}`, + slots: 1, + runnerGroupId: 1, + labels: ['self-hosted', 'linux', 'ARM64', 'trial-only'], + cpus: 1, + memory: '1g', + pidsLimit: 100, + ttlSeconds: 2, + shutdownGraceSeconds: 0, + minDiskBytes: 0, + stateDir: join(dir, 'state'), + logDir: join(dir, 'logs'), + }; + const calls = []; + const containers = new Map(); + const networks = new Map(); + let runner; + let counter = 0; + let foreignCounter = 0; + let failRemove = false; + let failNetworkRemove = false; + let keepContainerAfterRemove = false; + let loseJit = false; + const labelsFromArgs = (args) => { + const labels = {}; + for (let i = 0; i < args.length; i += 1) { + if (args[i] !== '--label' || typeof args[i + 1] !== 'string') continue; + const at = args[i + 1].indexOf('='); + if (at > 0) labels[args[i + 1].slice(0, at)] = args[i + 1].slice(at + 1); + } + return labels; + }; + const filtersFromArgs = (args) => { + const filters = []; + for (let i = 0; i < args.length; i += 1) { + if (args[i] === '--filter' && typeof args[i + 1] === 'string') + filters.push(args[i + 1].replace(/^label=/, '')); + } + return filters; + }; + const matchesFilters = (entry, args) => + filtersFromArgs(args).every((filter) => { + const at = filter.indexOf('='); + if (at < 0) return true; + return entry.labels[filter.slice(0, at)] === filter.slice(at + 1); + }); + async function command(program, args, options = {}) { + calls.push({ program, args, options }); + if (program === 'gh') { + const path = args.find((v) => v.startsWith('/')); + if (path.includes('generate-jitconfig')) { + const body = JSON.parse(options.input); + runner = { id: ++counter, name: body.name }; + if (loseJit) throw new Error('network failure'); + return JSON.stringify({ runner, encoded_jit_config: 'jit-secret' }); + } + if (args.includes('DELETE')) { + runner = undefined; + return ''; + } + return JSON.stringify(runner ? [runner] : []); + } + if (args[0] === 'ps') { + const matched = [...containers.entries()].filter(([, entry]) => + matchesFilters(entry, args), + ); + if (!matched.length) return ''; + if (args.includes('--format')) + return `${matched.map(([, entry]) => entry.labels['qwen.slot'] ?? '').join('\n')}\n`; + return `${matched.map(([id]) => id).join('\n')}\n`; + } + if (args[0] === 'network' && args[1] === 'ls') { + const matched = [...networks.entries()].filter(([, entry]) => + matchesFilters(entry, args), + ); + return matched.length ? `${matched.map(([id]) => id).join('\n')}\n` : ''; + } + if (args[0] === 'network' && args[1] === 'create') { + const id = `network-${counter}`; + networks.set(id, { labels: labelsFromArgs(args) }); + return id; + } + if (args[0] === 'network' && args[1] === 'rm') { + if (failNetworkRemove) throw new Error('network removal failed'); + networks.delete(args[args.length - 1]); + return ''; + } + if (args[0] === 'network' && args[1] === 'inspect') { + const entry = networks.get(args[args.length - 1]); + return `${entry?.labels['qwen.slot'] ?? ''}\n`; + } + if (args[0] === 'info') return `${dir}\n`; + if (args[0] === 'create') { + const id = `container-${counter}`; + containers.set(id, { labels: labelsFromArgs(args) }); + return id; + } + if (args[0] === 'logs') return 'runner log'; + if (args[0] === 'rm') { + if (failRemove) throw new Error('docker unavailable'); + if (!keepContainerAfterRemove) containers.delete(args[args.length - 1]); + return ''; + } + if (args[0] === 'wait') return '0\n'; + if (args[0] === 'start') return ''; + return ''; + } + return { + config, + calls, + command, + sleep: async () => {}, + addForeignNetwork(labels) { + const id = `network-foreign-${++foreignCounter}`; + networks.set(id, { labels }); + return id; + }, + networkIds: () => [...networks.keys()], + setFailRemove: (v) => { + failRemove = v; + }, + setLoseJit: (v) => { + loseJit = v; + }, + setFailNetworkRemove: (v) => { + failNetworkRemove = v; + }, + setKeepContainerAfterRemove: (v) => { + keepContainerAfterRemove = v; + }, + }; +} + +test('creates a new runner and container per job and keeps GitHub token out of Docker', async (t) => { + const previousToken = process.env.GH_TOKEN; + process.env.GH_TOKEN = 'host-only-pat'; + t.after(() => { + if (previousToken === undefined) delete process.env.GH_TOKEN; + else process.env.GH_TOKEN = previousToken; + }); + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + await manager.cycle(1); + await manager.cycle(1); + const creates = f.calls.filter( + (c) => c.program === 'docker' && c.args[0] === 'create', + ); + const requests = f.calls.filter( + (c) => + c.program === 'gh' && + c.args.includes('/repos/owner/repo/actions/runners/generate-jitconfig'), + ); + assert.ok( + requests.every((c) => + JSON.parse(c.options.input).name.startsWith('ecs-qwen-'), + ), + ); + assert.ok(requests.every((c) => c.options.env.GH_TOKEN === 'host-only-pat')); + assert.equal(creates.length, 2); + assert.notEqual( + JSON.parse(requests[0].options.input).name, + JSON.parse(requests[1].options.input).name, + ); + assert.deepEqual( + JSON.parse(requests[0].options.input).labels, + f.config.labels, + ); + assert.ok( + creates.every( + (c) => + !c.options.env.GH_TOKEN && + c.options.env.RUNNER_JIT_CONFIG === 'jit-secret', + ), + ); + assert.deepEqual( + creates.map((c) => c.args[c.args.indexOf('--network') + 1]), + ['network-1', 'network-2'], + ); + const networks = f.calls.filter( + (c) => + c.program === 'docker' && + c.args[0] === 'network' && + c.args[1] === 'create', + ); + assert.equal(networks.length, 2); + assert.ok( + networks.every((c) => + c.args.includes('com.docker.network.bridge.enable_icc=false'), + ), + ); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length, + 2, + ); +}); + +test('failed cleanup blocks replacement until recovery cleans the old container', async (t) => { + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + f.setFailRemove(true); + await assert.rejects(manager.cycle(1), /docker unavailable/); + await assert.rejects(manager.cycle(1), /docker unavailable/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); + f.setFailRemove(false); + await manager.cycle(1); + const ops = f.calls + .filter((c) => c.program === 'docker') + .map((c) => c.args[0]); + assert.ok(ops.lastIndexOf('rm') > ops.lastIndexOf('create')); + assert.equal(ops.filter((op) => op === 'create').length, 2); +}); + +test('lost JIT response reconciles the recorded name before requesting another', async (t) => { + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + f.setLoseJit(true); + await assert.rejects(manager.cycle(1), /network failure/); + await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), { + code: 'ENOENT', + }); + assert.ok( + f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')), + ); + f.setLoseJit(false); + await manager.cycle(1); +}); + +test('an unconfirmed JIT request blocks replacement when no runner can be found', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + return Promise.reject(new Error('connection lost')); + } + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), /manual reconciliation/); + await assert.rejects(manager.cycle(1), /manual reconciliation/); + const state = JSON.parse( + await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'), + ); + assert.equal(state.uncertain, true); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 0, + ); +}); + +test('a failed container start is cleaned up before another cycle', async (t) => { + const f = await fixture(t); + let fail = true; + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'start' && fail) { + fail = false; + return Promise.reject(new Error('start failed')); + } + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), /start failed/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length, + 1, + ); + await manager.cycle(1); +}); + +test('watchdog removes a container that does not exit', async (t) => { + const f = await fixture(t); + f.config.ttlSeconds = 0.01; + const command = (program, args, options) => + program === 'docker' && args[0] === 'wait' + ? new Promise(() => {}) + : f.command(program, args, options); + await createManager(f.config, { command }).cycle(1); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length, + 1, + ); +}); + +test('checks Docker root disk rather than the state directory', async (t) => { + const f = await fixture(t); + let checked; + const manager = createManager(f.config, { + command: f.command, + statfs: async (path) => { + checked = path; + return { bavail: 0, bsize: 4096 }; + }, + }); + f.config.minDiskBytes = 1; + await assert.rejects(manager.cycle(1), /Insufficient disk capacity/); + assert.equal(checked, join(f.config.stateDir, '..')); + assert.equal(f.calls.filter((c) => c.program === 'gh').length, 0); + const info = f.calls.find( + (c) => c.program === 'docker' && c.args[0] === 'info', + ); + assert.equal(info.options.env.DOCKER_HOST, 'unix:///var/run/docker.sock'); +}); + +test('rejects same-name runner with a different ID without deleting it', async (t) => { + const f = await fixture(t); + const state = { + name: 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + runnerId: 42, + uncertain: false, + }; + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify(state), + ); + const command = (program, args, options) => { + if (program === 'gh' && args.includes('--paginate')) + return Promise.resolve(JSON.stringify([{ name: state.name, id: 43 }])); + return f.command(program, args, options); + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + /identity mismatch/, + ); + assert.ok( + !f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')), + ); + assert.equal( + JSON.parse(await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8')) + .runnerId, + 42, + ); +}); + +test('reconciles a legacy identity on a later page before registering its replacement', async (t) => { + const f = await fixture(t); + const name = 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa'; + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify({ + name, + runnerId: 42, + uncertain: false, + }), + ); + let deleted = false; + const command = (program, args, options) => { + if (program === 'gh' && args.includes('--paginate') && !deleted) { + assert.ok(!args.includes('--slurp')); + assert.equal(args[args.indexOf('--jq') + 1], '.runners | @json'); + return Promise.resolve('[]\n' + JSON.stringify([{ name, id: 42 }])); + } + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/42') + ) { + deleted = true; + } + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + assert.ok( + deleted, + 'legacy registration must be removed before replacement', + ); + } + return f.command(program, args, options); + }; + await createManager(f.config, { command }).cycle(1); + assert.ok(deleted); +}); + +test('startup reclaims a live container before replenishing after a crash', async (t) => { + const f = await fixture(t); + const previousPath = process.env.PATH; + await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', { + mode: 0o755, + }); + process.env.PATH = `${f.config.stateDir}:${previousPath}`; + t.after(() => { + process.env.PATH = previousPath; + }); + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify({ + name: 'ecs-qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + runnerId: 42, + uncertain: false, + }), + ); + let oldContainer = true; + let manager; + const command = async (program, args, options) => { + if (program === 'docker' && args[0] === 'ps') { + if (args.includes('--format')) { + // Docker quiet mode overrides --format, as on the trial host. + return oldContainer + ? args.includes('-aq') + ? 'old-container' + : '1' + : ''; + } + if (oldContainer) return 'old-container'; + } + if ( + program === 'docker' && + args[0] === 'rm' && + args.includes('old-container') + ) { + oldContainer = false; + return ''; + } + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + assert.equal( + oldContainer, + false, + 'remove old container before registration', + ); + } + if (program === 'docker' && args[0] === 'start') manager.stop(); + return f.command(program, args, options); + }; + manager = createManager(f.config, { command }); + await manager.start(); + assert.equal(oldContainer, false); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); +}); + +test('accepts immutable local image IDs and rejects mutable tags and invalid configuration', async (t) => { + const f = await fixture(t); + assert.doesNotThrow(() => + createManager({ ...f.config, image: `sha256:${'b'.repeat(64)}` }), + ); + assert.throws( + () => createManager({ ...f.config, image: 'image:latest' }), + /Invalid manager configuration/, + ); + assert.throws( + () => + createManager({ ...f.config, dockerSocket: 'tcp://example.com:2375' }), + /Invalid manager configuration/, + ); + assert.throws( + () => createManager({ ...f.config, cpus: Infinity }), + /Invalid manager configuration/, + ); +}); + +test('does not expose malformed GitHub responses in errors', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if ( + program === 'gh' && + args.some((arg) => arg.includes('generate-jitconfig')) + ) { + return f + .command(program, args, options) + .then(() => 'malformed-secret-token'); + } + return f.command(program, args, options); + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + (error) => { + assert.match(error.message, /Invalid JSON response/); + assert.doesNotMatch(error.message, /malformed-secret-token/); + return true; + }, + ); +}); + +test('HTTP 429 rejection does not leave uncertain state and the next cycle can retry', async (t) => { + const f = await fixture(t); + const fakeGh = join(f.config.stateDir, 'gh'); + await writeFile( + fakeGh, + '#!/bin/sh\necho "gh: API rate limit exceeded (HTTP 429)" >&2\nexit 1\n', + { mode: 0o755 }, + ); + let rejected = false; + const command = (program, args, options) => { + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') && + !rejected + ) { + rejected = true; + return runCommand(fakeGh, [], { env: {} }); + } + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), (error) => error.status === 429); + await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), { + code: 'ENOENT', + }); + await manager.cycle(1); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); +}); + +test('network cleanup failure prevents new runner and is reconciled on the next cycle', async (t) => { + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + f.setFailNetworkRemove(true); + await assert.rejects(manager.cycle(1), /network removal failed/); + await assert.rejects(manager.cycle(1), /network removal failed/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); + f.setFailNetworkRemove(false); + await manager.cycle(1); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 2, + ); +}); + +test('a lost network create response is recovered using labels', async (t) => { + const f = await fixture(t); + const command = async (program, args, options) => { + const result = await f.command(program, args, options); + if (program === 'docker' && args[0] === 'network' && args[1] === 'create') + throw new Error('network create response lost'); + return result; + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + /network create response lost/, + ); + assert.equal( + f.calls.filter( + (c) => + c.program === 'docker' && c.args[0] === 'network' && c.args[1] === 'rm', + ).length, + 1, + ); + await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), { + code: 'ENOENT', + }); +}); + +test('network deletion is verified even when docker reports success', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'network' && args[1] === 'rm') + return Promise.resolve(''); + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), /Network cleanup failed/); + await assert.rejects(manager.cycle(1), /Network cleanup failed/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); +}); + +test('passes only the docker environment allowlist and the JIT config by name', async (t) => { + const sentinels = { + AWS_SECRET_ACCESS_KEY: 'sentinel-secret', + SOME_DEPLOY_TOKEN: 'sentinel-token', + }; + const previous = {}; + for (const key of Object.keys(sentinels)) { + previous[key] = process.env[key]; + process.env[key] = sentinels[key]; + } + t.after(() => { + for (const key of Object.keys(sentinels)) { + if (previous[key] === undefined) delete process.env[key]; + else process.env[key] = previous[key]; + } + }); + const f = await fixture(t); + await createManager(f.config, { command: f.command }).cycle(1); + const create = f.calls.find( + (c) => c.program === 'docker' && c.args[0] === 'create', + ); + const allowed = [ + 'DOCKER_CONFIG', + 'DOCKER_HOST', + 'HOME', + 'PATH', + 'RUNNER_JIT_CONFIG', + 'XDG_RUNTIME_DIR', + ]; + assert.deepEqual( + Object.keys(create.options.env).filter((key) => !allowed.includes(key)), + [], + ); + assert.ok(!JSON.stringify(create.options.env).includes('sentinel')); + assert.ok(!JSON.stringify(create.args).includes('sentinel')); + assert.equal( + create.args[create.args.indexOf('--log-driver') + 1], + 'json-file', + ); + const at = create.args.indexOf('--env'); + assert.equal(create.args[at + 1], 'RUNNER_JIT_CONFIG'); +}); + +test('labels pool and slot on create and honours recovery filters', async (t) => { + const f = await fixture(t); + const foreign = f.addForeignNetwork({ + 'qwen.pool': 'other', + 'qwen.slot': '9', + }); + const command = async (program, args, options) => { + const result = await f.command(program, args, options); + if (program === 'docker' && args[0] === 'network' && args[1] === 'create') + throw new Error('network create response lost'); + return result; + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + /network create response lost/, + ); + const created = f.calls.find( + (c) => + c.program === 'docker' && + c.args[0] === 'network' && + c.args[1] === 'create', + ); + assert.ok(created.args.includes('qwen.pool=trial')); + assert.ok(created.args.includes('qwen.slot=1')); + const removed = f.calls + .filter( + (c) => + c.program === 'docker' && c.args[0] === 'network' && c.args[1] === 'rm', + ) + .map((c) => c.args[c.args.length - 1]); + assert.deepEqual(removed, ['network-1']); + assert.ok(f.networkIds().includes(foreign)); +}); + +test('container cleanup failure blocks replenishment even when docker reports success', async (t) => { + const f = await fixture(t); + f.setKeepContainerAfterRemove(true); + const manager = createManager(f.config, { command: f.command }); + await assert.rejects(manager.cycle(1), /Container cleanup failed/); + const state = JSON.parse( + await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'), + ); + assert.ok(state.name); + assert.equal(state.runnerId, 1); +}); + +test('rejects corrupted slot state without deleting a runner', async (t) => { + const validName = 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa'; + const cases = [ + { + label: 'foreign prefix', + state: { + name: 'qwen-other-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + runnerId: 42, + uncertain: false, + }, + }, + { + label: 'non-UUID suffix', + state: { + name: 'qwen-trial-1-not-a-uuid', + runnerId: 42, + uncertain: false, + }, + }, + { + label: 'non-boolean uncertain', + state: { name: validName, runnerId: 42, uncertain: 'no' }, + }, + { + label: 'zero runner id', + state: { name: validName, runnerId: 0, uncertain: false }, + }, + ]; + for (const item of cases) { + const f = await fixture(t); + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify(item.state), + ); + await assert.rejects( + createManager(f.config, { command: f.command }).cycle(1), + /Invalid slot state/, + item.label, + ); + assert.ok( + !f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')), + item.label, + ); + } +}); + +test('backs off on a failed container even after the startup window', async (t) => { + const f = await fixture(t); + f.config.ttlSeconds = 120; + let now = Date.now(); + t.mock.method(Date, 'now', () => now); + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'wait') { + now += 31000; + return Promise.resolve('126\n'); + } + return f.command(program, args, options); + }; + const result = await createManager(f.config, { command }).cycle(1); + assert.equal(result.earlyExit, true); +}); + +test('backs off before the next JIT request after a failed container', async (t) => { + const f = await fixture(t); + const previousPath = process.env.PATH; + await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', { + mode: 0o755, + }); + process.env.PATH = `${f.config.stateDir}:${previousPath}`; + t.after(() => { + process.env.PATH = previousPath; + }); + const sleeps = []; + let manager; + let jitRequests = 0; + const command = async (program, args, options) => { + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + jitRequests += 1; + if (jitRequests > 1) manager.stop(); + } + if (program === 'docker' && args[0] === 'wait') return '127\n'; + return f.command(program, args, options); + }; + manager = createManager(f.config, { + command, + sleep: async (ms) => { + sleeps.push(ms); + }, + }); + await manager.start(); + // The 1 s poll that races `docker wait` also goes through the injected sleep; + // the replenishment backoff is the 60 s entry. + assert.equal(sleeps.filter((ms) => ms === 60000).length, 1); + assert.equal(sleeps.at(-1), 60000); + assert.equal(jitRequests, 2); +}); + +test('does not penalize a fast successful container', async (t) => { + const f = await fixture(t); + const result = await createManager(f.config, { command: f.command }).cycle(1); + assert.equal(result.earlyExit, false); +}); + +test('backs off when docker wait cannot determine container status', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'wait') + return Promise.reject(new Error('Docker disconnected')); + return f.command(program, args, options); + }; + const result = await createManager(f.config, { command }).cycle(1); + assert.equal(result.earlyExit, true); +}); + +test('starts through a symlink and reports invalid configuration', async (t) => { + const f = await fixture(t); + const link = join(f.config.stateDir, 'manager.mjs'); + await symlink(new URL('./manager.mjs', import.meta.url), link); + const result = spawnSync( + process.execPath, + [link, join(f.config.stateDir, 'missing.json')], + { + env: { ...process.env, GH_TOKEN: 'test-only' }, + encoding: 'utf8', + }, + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /^manager: /); +}); diff --git a/scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service b/scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service new file mode 100644 index 00000000000..a989541fc84 --- /dev/null +++ b/scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service @@ -0,0 +1,18 @@ +[Unit] +Description=Qwen Code ephemeral GitHub Actions runners +After=docker.service network-online.target +Requires=docker.service +Wants=network-online.target + +[Service] +Type=simple +EnvironmentFile=/etc/qwen-ephemeral-runner/credentials.env +WorkingDirectory=/opt/qwen-ephemeral-runner +ExecStart=/usr/bin/env node /opt/qwen-ephemeral-runner/manager.mjs /etc/qwen-ephemeral-runner/config.json +Restart=always +RestartSec=10 +KillMode=mixed +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/tests/ci-platform-lanes.test.js b/scripts/tests/ci-platform-lanes.test.js index a41581934bc..d941df09207 100644 --- a/scripts/tests/ci-platform-lanes.test.js +++ b/scripts/tests/ci-platform-lanes.test.js @@ -116,8 +116,7 @@ describe('complete macOS test groups', () => { // The extended ceilings below, except the test lane's, are scoped to the // self-hosted pool: `runs-on` falls back to ubuntu-latest when // MAINTAINER_ECS_RUNNER_DISABLED=true (and, for the association-gated lanes, -// for fork PRs from untrusted authors — lint_and_static reads lint_runner, -// which routes every PR to the pool), so those hosted paths keep the +// for untrusted fork PRs with the disposable pool disabled), so hosted paths keep the // pre-contention ceiling. The ARM64 fleet renders // `["self-hosted", "linux", "arm64"]` — no `ecs-qwen` label — so the scope // cannot key on the x64 fleet's name. Evaluate the real timeout expression for @@ -391,8 +390,11 @@ it('reports the required Linux check only when the build and every shard pass', const gate = ci.jobs.test_ubuntu; expect(gate.name).toBe('Test (ubuntu-latest, Node 22.x)'); expect(gate.needs).toEqual(['classify_pr', 'test_build', 'test']); - // On the shards' runners, not a hosted one that can queue it for minutes. - expect(gate['runs-on']).toBe(ci.jobs.test['runs-on']); + // The gate remains on the original association-gated runner, while test + // shards can opt into the disposable pool for untrusted forks. + expect(gate['runs-on']).toBe( + '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || \'["ubuntu-latest"]\') }}', + ); expect(condOf('test_ubuntu')).toBe(condOf('test')); expect(gate.steps[0].run).toBe( 'test "$BUILD_RESULT" = success && test "$SHARDS_RESULT" = success', @@ -862,15 +864,11 @@ describe('post-merge push lane', () => { ).toContain('classify_pr'); }); - it('keeps test on the association-gated output', () => { - // Nothing else reads test's runs-on: switching it to lint_runner would - // hand every untrusted fork the pool while the timeout pins above stayed - // green — timeoutMinutesOn derives each lane's output from its own - // runs-on, so it cannot catch a lane pointed at the wrong output. Pin the - // expression, not an equality with lint_and_static — the two are meant - // to differ. + it('keeps test on the opt-in disposable output', () => { + // The classifier defaults this output to ubuntu_runner, but selects the + // disposable pool for untrusted forks only when explicitly enabled. expect(String(ci.jobs.test['runs-on'])).toBe( - '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || \'["ubuntu-latest"]\') }}', + '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || \'["ubuntu-latest"]\') }}', ); }); diff --git a/scripts/tests/review-worktree-cleanup-workflow.test.js b/scripts/tests/review-worktree-cleanup-workflow.test.js index 280d30999dc..98e75fdbab2 100644 --- a/scripts/tests/review-worktree-cleanup-workflow.test.js +++ b/scripts/tests/review-worktree-cleanup-workflow.test.js @@ -55,16 +55,18 @@ const ciYaml = parse(readFileSync('.github/workflows/ci.yml', 'utf8')); // possibly dirty workspace. Match the pool itself, not just the output // references that usually name it: jobs can also hard-code the shared label // array, and a checkout on either form inherits the same leftovers. -// lint_and_static reads lint_runner (every PR reaches the pool), the other -// lanes ubuntu_runner — both name the same shared workspace. +// lint_and_static reads lint_runner (every PR reaches the pool), the test +// lanes read pr_test_runner, the other lanes ubuntu_runner — all name the same +// shared workspace. pr_test_runner is listed explicitly because jobs that read +// it are also kept in scope by their own sweep step, which would otherwise hide +// a future job that adds the pool and forgets the sweep. // Include retained recovery steps on hosted jobs too, so rerouting cannot // silently restore an untested sweep. Select by properties, not job name. +const sharedPoolSelector = /ubuntu_runner|lint_runner|pr_test_runner|ecs-qwen/; const ciCleanSteps = Object.entries(ciYaml.jobs) .filter( ([, job]) => - (/ubuntu_runner|lint_runner|ecs-qwen/.test( - JSON.stringify(job['runs-on'] ?? ''), - ) || + (sharedPoolSelector.test(JSON.stringify(job['runs-on'] ?? '')) || (job.steps ?? []).some( (s) => s.name === 'Clean stale .qwen before checkout', )) && @@ -381,10 +383,24 @@ describe('review worktree cleanup steps', () => { } }); + it('matches every pr_test_runner job with the shared-pool selector itself', () => { + const poolJobs = Object.entries(ciYaml.jobs).filter(([, job]) => + JSON.stringify(job['runs-on'] ?? '').includes('pr_test_runner'), + ); + expect(poolJobs.length).toBeGreaterThan(0); + for (const [id, job] of poolJobs) { + expect( + sharedPoolSelector.test(JSON.stringify(job['runs-on'] ?? '')), + `job "${id}" runs on pr_test_runner and must match the pool selector`, + ).toBe(true); + } + }); + it('keeps every shared-pool ci.yml checkout sweep pinned to paths.ts', () => { expect(ciCleanSteps.map(({ id }) => id)).toEqual( expect.arrayContaining([ 'test', + 'test_build', 'web_shell_e2e_smoke', 'integration_cli', ]), From 0b54b9be999b68d9c2d322a771ea9c171c733511 Mon Sep 17 00:00:00 2001 From: Shaojin Wen Date: Sat, 7 Nov 2026 22:24:43 +0800 Subject: [PATCH 2/2] fix(managed-agent): restore two runtime-broker classes to their size ratchet (#16542) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ClassSizeRatchetTest fails on main itself: RuntimeBrokerService is 2947 lines against a 2943 ceiling and RuntimeBindingProvisioning is 1327 against 1326. Both overruns come from #16535 replacing inline fully qualified names with imports, which adds import lines while the usage lines keep their count — 4 imports in RuntimeBrokerService and 1 in RuntimeBindingProvisioning — without updating the ceilings that #16501 had just recorded after the extraction. The SDK Java lane runs the merge ref, so every PR that merges main inherits the failure. The ratchet only lets ceilings go down, so reclaim the lines instead of raising the pins: join expressions that were already split across two lines. RuntimeBindingProvisioning's single Objects.equals operand, and the four BrokerValues.requireId assignments in release() and releaseMount(). Every joined line is at most 88 characters; the diff is formatting only and keeps #16535's import style. ClassSizeRatchetTest red before / green after; RuntimeBrokerServiceTest 199 green; runtime-broker checkstyle and spotbugs clean. No pin was edited. --- .../service/RuntimeBindingProvisioning.java | 3 +-- .../runtimebroker/service/RuntimeBrokerService.java | 12 ++++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBindingProvisioning.java b/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBindingProvisioning.java index 9e6cc3e8c3a..005ac11566f 100644 --- a/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBindingProvisioning.java +++ b/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBindingProvisioning.java @@ -1251,8 +1251,7 @@ final class RuntimeBindingProvisioning { && lease.getLeaseId().equals(attestation.getLeaseId()) && lease.getEpoch() == attestation.getEpoch() && request.getScope().equals(attestation.getScope()) - && Objects.equals(request.getStorageId(), - attestation.getStorageId()) + && Objects.equals(request.getStorageId(), attestation.getStorageId()) && seed.getProvisionRequestId().equals( attestation.getProvisionRequestId()); } diff --git a/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBrokerService.java b/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBrokerService.java index 02b6a24897b..f63b43172b0 100644 --- a/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBrokerService.java +++ b/packages/managed-agent-server/runtime-broker/src/main/java/com/alibaba/qwen/code/runtimebroker/service/RuntimeBrokerService.java @@ -2007,10 +2007,8 @@ public final class RuntimeBrokerService implements AutoCloseable, RuntimeBrokerO public CompletionStage release(String harnessSessionId, String runtimeSessionId, boolean drain, RuntimeLifecycleAuthority authority) { requireOpen(closed); - String harnessId = BrokerValues.requireId(harnessSessionId, - "harnessSessionId"); - String runtimeId = BrokerValues.requireId(runtimeSessionId, - "runtimeSessionId"); + String harnessId = BrokerValues.requireId(harnessSessionId, "harnessSessionId"); + String runtimeId = BrokerValues.requireId(runtimeSessionId, "runtimeSessionId"); CompletableFuture local = bindingRegistry.session(runtimeId); if (local == null) { return releasedSession(harnessId, runtimeId, drain, authority); @@ -2057,10 +2055,8 @@ public final class RuntimeBrokerService implements AutoCloseable, RuntimeBrokerO String runtimeSessionId, String mountClaimId, RuntimeLifecycleAuthority authority) { requireOpen(closed); - String harnessId = BrokerValues.requireId(harnessSessionId, - "harnessSessionId"); - String runtimeId = BrokerValues.requireId(runtimeSessionId, - "runtimeSessionId"); + String harnessId = BrokerValues.requireId(harnessSessionId, "harnessSessionId"); + String runtimeId = BrokerValues.requireId(runtimeSessionId, "runtimeSessionId"); String claimId = BrokerValues.requireId(mountClaimId, "mountClaimId"); CompletableFuture local = bindingRegistry.session(runtimeId); if (local == null) {