diff --git a/.github/scripts/ci-runner-routing.test.mjs b/.github/scripts/ci-runner-routing.test.mjs index 2553a5177ac..14474e24ed3 100644 --- a/.github/scripts/ci-runner-routing.test.mjs +++ b/.github/scripts/ci-runner-routing.test.mjs @@ -8,7 +8,7 @@ // scarce pool (2026-10-31 decision, same accepted risk class as lint). // The association routing for DOWNSTREAM lanes lives in the `pick_runner` // shell step, which publishes `ubuntu_runner` (association-gated) and -// `lint_runner` (every pull request reaches the fleet). The matrix below +// `lint_runner` (untrusted forks default to hosted). The matrix below // pins the gate's unconditional routing AND that ubuntu_runner keeps the // association gate for the test/integration lanes. // @@ -123,6 +123,8 @@ function runPickRunnerOutputs({ eventName, dispatch, fleet, + ephemeralEnabled, + ephemeralArch, }) { const tmp = mkdtempSync(join(tmpdir(), 'pick-runner-')); const outputFile = join(tmp, 'github_output'); @@ -133,6 +135,8 @@ function runPickRunnerOutputs({ ECS_DISABLED: ecsDisabled ? 'true' : '', EVENT_NAME: eventName, LINUX_RUNNER_FLEET: fleet ?? '', + EPHEMERAL_ENABLED: ephemeralEnabled ?? '', + EPHEMERAL_ARCH: ephemeralArch ?? '', DISPATCH_LINUX_RUNNER: dispatch ?? '', GITHUB_OUTPUT: outputFile, }, @@ -161,7 +165,15 @@ function runPickRunnerOutputs({ outputs.split('\n').includes(`lint_runner=${lintRunner}`), `lint_runner not published: ${outputs}`, ); - return { ubuntuRunner, lintRunner }; + const prTest = outputs + .split('\n') + .find((line) => line.startsWith('pr_test_runner=')); + assert.ok(prTest, 'pr_test_runner not published'); + return { + ubuntuRunner, + lintRunner, + prTestRunner: prTest.slice('pr_test_runner='.length), + }; } function runPickRunner(inputs) { @@ -302,11 +314,7 @@ describe('ci.yml lint_runner routing', () => { const ARM = '["self-hosted", "linux", "arm64"]'; const lintRunner = (inputs) => runPickRunnerOutputs(inputs).lintRunner; - it('routes every pull request to the fleet, untrusted forks included', () => { - // The maintainer decision this pins: Lint & Static executes PR code on - // the shared pool for ALL authors. If this matrix ever flips back to - // hosted for the negative associations without a deliberate decision, - // the lane silently regrows its hosted quota spend. + it('keeps untrusted fork lint hosted when the disposable switch is unset', () => { for (const sameRepo of [true, false]) { for (const assoc of ASSOCIATIONS) { assert.equal( @@ -316,8 +324,8 @@ describe('ci.yml lint_runner routing', () => { assoc, eventName: 'pull_request', }), - ECS, - `lint sameRepo=${sameRepo} assoc='${assoc}' must route to ECS`, + sameRepo || TRUSTED.includes(assoc) ? ECS : HOSTED, + `lint sameRepo=${sameRepo} assoc='${assoc}'`, ); } } @@ -354,7 +362,7 @@ describe('ci.yml lint_runner routing', () => { lintRunner({ ecsDisabled: false, sameRepo: false, - assoc: 'NONE', + assoc: 'MEMBER', eventName: 'pull_request', fleet: 'arm64', }), @@ -442,6 +450,99 @@ describe('ci.yml lint_runner routing', () => { }); }); +describe('ci.yml disposable runner routing', () => { + it('keeps all selected fork jobs hosted unless the switch is exactly true', () => { + for (const ephemeralEnabled of [undefined, '', 'false', 'invalid']) { + const out = runPickRunnerOutputs({ + eventName: 'pull_request', + sameRepo: false, + assoc: 'NONE', + ephemeralEnabled, + }); + assert.equal(out.prTestRunner, HOSTED); + assert.equal(out.lintRunner, HOSTED); + } + }); + it('selects ARM64 only when explicitly configured', () => { + const inputs = { + eventName: 'pull_request', + sameRepo: false, + assoc: 'NONE', + ephemeralEnabled: 'true', + }; + const arm = '["self-hosted", "linux", "arm64", "ecs-qwen-ephemeral"]'; + assert.equal( + runPickRunnerOutputs({ ...inputs, ephemeralArch: 'arm64' }).prTestRunner, + arm, + ); + assert.equal( + runPickRunnerOutputs({ ...inputs, ephemeralArch: 'arm64' }).lintRunner, + arm, + ); + assert.equal( + runPickRunnerOutputs({ ...inputs, ephemeralArch: 'invalid' }) + .prTestRunner, + '["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]', + ); + }); + it('routes only untrusted fork jobs to the disposable pool when enabled', () => { + const ephemeral = '["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]'; + for (const eventName of [ + 'pull_request', + 'push', + 'merge_group', + 'workflow_dispatch', + ]) { + for (const sameRepo of [true, false]) { + for (const assoc of ASSOCIATIONS) { + for (const ecsDisabled of [true, false]) { + for (const fleet of ['', 'hosted', 'arm64']) { + const inputs = { eventName, sameRepo, assoc, ecsDisabled, fleet }; + const baseline = runPickRunnerOutputs(inputs); + const enabled = runPickRunnerOutputs({ + ...inputs, + ephemeralEnabled: 'true', + }); + const selected = + eventName === 'pull_request' && + !sameRepo && + !TRUSTED.includes(assoc) && + !ecsDisabled && + fleet !== 'hosted'; + assert.equal( + enabled.prTestRunner, + selected ? ephemeral : baseline.ubuntuRunner, + JSON.stringify(inputs), + ); + assert.equal( + enabled.lintRunner, + selected ? ephemeral : baseline.lintRunner, + JSON.stringify(inputs), + ); + assert.equal(enabled.ubuntuRunner, baseline.ubuntuRunner); + } + } + } + } + } + }); + + it('routes only the intended jobs through the new output', () => { + assert.equal( + ciDoc.jobs.classify_pr.outputs.pr_test_runner, + '${{ steps.pick_runner.outputs.pr_test_runner }}', + ); + for (const name of ['test_build', 'test']) + assert.match(ciDoc.jobs[name]['runs-on'], /pr_test_runner/); + for (const name of ['lint_and_static', 'integration_no_ak']) + assert.match(ciDoc.jobs[name]['runs-on'], /lint_runner/); + for (const [name, job] of Object.entries(ciDoc.jobs)) { + if (!['classify_pr', 'test_build', 'test'].includes(name)) + assert.doesNotMatch(String(job['runs-on']), /pr_test_runner/, name); + } + }); +}); + describe('ci.yml lint HOME isolation', () => { // This guard lives in the routing battery (HELPER_TESTS) because it gates // every full-profile PR; the scripts/tests vitest suites only run after the diff --git a/.github/workflows/.size-baseline b/.github/workflows/.size-baseline index 3af4140f298..55d506c2371 100644 --- a/.github/workflows/.size-baseline +++ b/.github/workflows/.size-baseline @@ -27,6 +27,7 @@ 2038 docs-page-action.yml 10005 dsw-swe-verified-release.yml 40188 e2e.yml +651 ephemeral-runner-smoke.yml 13135 finalize-release.yml 16647 live-host-release.yml 5950 live-host.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 840ecbed56c..9fdf7df6012 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,8 +101,8 @@ env: # re-runs every suite after `npm ci`, and a push to main always classifies # `full` (classify_pr computes a profile only for pull_request), so what # the fast lane skips is still re-checked post-merge. - HELPER_TESTS_DEP_FREE: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs' - HELPER_TESTS: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/classify-release-notes.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/qwen-triage-workflow.test.mjs .github/scripts/assign-issue-owner.test.mjs .github/scripts/auto-minimize-spam.test.mjs .github/scripts/ci-runner-routing.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/assign-pr-owner.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/ci-disk-pressure.test.mjs .github/scripts/e2e-build.test.mjs integration-tests/concurrent-runner/export-html-from-chatrecord-jsonl.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/ecs-runner/review-scratch-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs' + HELPER_TESTS_DEP_FREE: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs scripts/ci/ephemeral-runner/manager.test.mjs' + HELPER_TESTS: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/classify-release-notes.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/qwen-triage-workflow.test.mjs .github/scripts/assign-issue-owner.test.mjs .github/scripts/auto-minimize-spam.test.mjs .github/scripts/ci-runner-routing.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/assign-pr-owner.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/ci-disk-pressure.test.mjs .github/scripts/e2e-build.test.mjs integration-tests/concurrent-runner/export-html-from-chatrecord-jsonl.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/ecs-runner/review-scratch-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs scripts/ci/ephemeral-runner/manager.test.mjs' # The growth ratchet and its vitest mirror compare each workflow against # the PR's base commit to tell "this PR grew the file" apart from "the # baseline went stale on main" (#9904). Wired once here so every lane @@ -143,11 +143,8 @@ jobs: outputs: skip_ci: '${{ steps.release_sync.outputs.skip_ci }}' ubuntu_runner: '${{ steps.pick_runner.outputs.ubuntu_runner }}' - # Lint & Static's own routing: every pull request reaches the fleet, - # including untrusted forks (maintainer decision 2026-10-31 — the lane - # executes PR code on the shared pool and that risk is accepted). The - # association gate in ubuntu_runner still governs every other Linux - # lane; do not "simplify" the two outputs into one. + pr_test_runner: '${{ steps.pick_runner.outputs.pr_test_runner }}' + # Untrusted forks use hosted runners unless the disposable pool is enabled. lint_runner: '${{ steps.pick_runner.outputs.lint_runner }}' ci_profile: '${{ steps.ci_profile.outputs.ci_profile }}' steps: @@ -196,17 +193,16 @@ jobs: # Keep push routing aligned with the classifier above so a hosted queue # does not delay the downstream ECS jobs. # - # lint_runner is the one deliberate divergence: Lint & Static routes - # EVERY pull request to the fleet, untrusted forks included (maintainer - # decision 2026-10-31, risk accepted — see the classify_pr outputs - # comment). The kill-switch and LINUX_RUNNER_FLEET=hosted still force - # hosted, and non-PR events inherit ubuntu_runner unchanged. + # Untrusted forks opt into the disposable pool; otherwise their build, + # test, lint and no-AK integration jobs stay hosted. - name: 'Select Linux runner' id: 'pick_runner' env: SAME_REPO: '${{ github.event.pull_request.head.repo.full_name == github.repository }}' AUTHOR_ASSOCIATION: '${{ github.event.pull_request.author_association }}' ECS_DISABLED: '${{ vars.MAINTAINER_ECS_RUNNER_DISABLED }}' + EPHEMERAL_ENABLED: '${{ vars.EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED }}' + EPHEMERAL_ARCH: '${{ vars.EXTERNAL_PR_EPHEMERAL_ARCH }}' # Optional fleet selector: 'hosted' forces github.com-hosted runners, # 'arm64' routes to the experimental ARM64 ECS fleet, anything else # keeps the x64 ecs-qwen fleet. Empty/unset keeps current behavior. @@ -242,12 +238,16 @@ jobs: fi echo "ubuntu_runner=${ubuntu_runner}" >> "${GITHUB_OUTPUT}" echo "Selected Linux runner: ${ubuntu_runner}" - # Lint & Static routes every pull request to the fleet (maintainer - # decision 2026-10-31); non-PR events inherit ubuntu_runner unchanged. + pr_test_runner="${ubuntu_runner}" lint_runner="${ubuntu_runner}" - if [[ "${EVENT_NAME}" == "pull_request" && "${ECS_DISABLED}" != "true" && "${LINUX_RUNNER_FLEET}" != "hosted" ]]; then - lint_runner="${fleet_runner}" + if [[ "${EPHEMERAL_ENABLED}" == "true" && "${ECS_DISABLED}" != "true" && "${LINUX_RUNNER_FLEET}" != "hosted" && "${EVENT_NAME}" == "pull_request" && "${SAME_REPO}" == "false" && "${trusted_author}" == "false" ]]; then + pr_test_runner='["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]' + if [[ "${EPHEMERAL_ARCH}" == "arm64" ]]; then + pr_test_runner='["self-hosted", "linux", "arm64", "ecs-qwen-ephemeral"]' + fi + lint_runner="${pr_test_runner}" fi + echo "pr_test_runner=${pr_test_runner}" >> "${GITHUB_OUTPUT}" echo "lint_runner=${lint_runner}" >> "${GITHUB_OUTPUT}" echo "Selected lint runner: ${lint_runner}" @@ -423,7 +423,7 @@ jobs: name: 'Build unit-test tree (ubuntu-latest, Node 22.x)' needs: 'classify_pr' if: "${{ !cancelled() && github.event_name != 'schedule' }}" - runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}' + runs-on: '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || ''["ubuntu-latest"]'') }}' timeout-minutes: 45 permissions: contents: 'read' @@ -890,7 +890,7 @@ jobs: # have. So a push run is lint, static analysis and unit tests — # lint/static in `lint_and_static`, unit tests here. if: "${{ !cancelled() && github.event_name != 'schedule' }}" - runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}' + runs-on: '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || ''["ubuntu-latest"]'') }}' # Every Vitest run takes one eighth of its suite's files (`--shard`): the # unsharded lane ran 45-59 minutes, almost all of it the CLI and core # suites one after the other. A shard keeps the old ceiling's headroom @@ -1559,12 +1559,7 @@ jobs: # same release-sync convention: stay running on skip_ci PRs so a required # context still reports, with the per-step guards making it a no-op pass. if: "${{ !cancelled() && github.event_name != 'schedule' }}" - # Reads lint_runner, NOT ubuntu_runner: this lane routes every pull - # request to the fleet, untrusted forks included (maintainer decision - # 2026-10-31 — the lane executes PR code and the shared-pool risk is - # accepted). test and the integration lanes keep the association-gated - # ubuntu_runner. The kill-switch and hosted-fleet overrides still apply, - # inside pick_runner. + # Shares the external-PR hosted/disposable switch with test and integration_no_ak. runs-on: '${{ fromJSON(needs.classify_pr.outputs.lint_runner || ''["ubuntu-latest"]'') }}' # Pool-scoped like test's, not a bare constant: this lane was beheaded twice # running honestly on cold-cache hk3-pool runners — at 30 minutes flat @@ -2982,14 +2977,8 @@ jobs: # A draft pull request skips it (and the browser smoke): it cannot merge, # and `ready_for_review` starts the full run that gates the merge. if: "${{ !cancelled() && needs.classify_pr.outputs.skip_ci != 'true' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') && !github.event.pull_request.draft }}" - # Reads lint_runner, NOT ubuntu_runner: like lint_and_static, this lane - # executes PR code on the pool for every pull request — accepted under the - # same 2026-10-31 decision, and it carries NO secrets by construction - # (every model key is emptied at the integration step; the job's only - # token is the read-only checkout grant). The lane publishes a run-unique - # HOME from its first step (lint's shape) and empties every model key at - # the gate, so the pool exposure is the class already accepted for lint. - # test and integration_cli keep the association-gated ubuntu_runner. + # Shares lint's hosted/disposable routing for untrusted forks. Model keys + # are empty; the job receives only the read-only checkout token. runs-on: '${{ fromJSON(needs.classify_pr.outputs.lint_runner || ''["ubuntu-latest"]'') }}' strategy: fail-fast: false diff --git a/.github/workflows/ephemeral-runner-smoke.yml b/.github/workflows/ephemeral-runner-smoke.yml new file mode 100644 index 00000000000..19896976fa8 --- /dev/null +++ b/.github/workflows/ephemeral-runner-smoke.yml @@ -0,0 +1,23 @@ +name: 'Ephemeral runner smoke' + +on: + workflow_dispatch: + +permissions: + contents: 'read' + +jobs: + smoke: + runs-on: ['ecs-qwen-ephemeral-smoke'] + timeout-minutes: 10 + steps: + - uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 + - name: 'Check tools and workspace' + run: | + node --version + git --version + printf 'Runner: %s\n' "$RUNNER_NAME" > runner-smoke.txt + - uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1 + with: + name: 'ephemeral-runner-smoke' + path: 'runner-smoke.txt' diff --git a/docs/design/2026-11-07-ephemeral-container-runners.md b/docs/design/2026-11-07-ephemeral-container-runners.md new file mode 100644 index 00000000000..1c3fbca65e5 --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-container-runners.md @@ -0,0 +1,163 @@ +# Ephemeral container runners for external PRs + +[English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md) + +Date: 2026-11-07. Status: implemented; a five-slot systemd trial is deployed on +hz4. Production external-PR routing remains disabled. + +Ten slots registering and running ten keyless test jobs of a private repository +at the same time on the Hangzhou shared test machine; containers, networks and +registrations were all reclaimed. A subsequent one-slot trial uses a real PAT +and two sequential GitHub Actions jobs to verify registration, container +replacement, workspace separation and absence of the PAT in jobs. A forced +manager exit verified systemd restart and cleanup before replacement. These +trials found and fixed compatibility with gh 2.46 pagination and Docker's quiet +mode overriding formatted slot output. Full PR CI and production isolation +remain unverified, so the shared host is still only a controlled trial host. + +Follow-up model access is described in +[the runner and model-proxy combination](2026-11-07-ephemeral-runner-model-proxy.md); +that proposal is neither implemented nor enabled. + +## Goals and boundaries + +Put the whole GitHub Actions runner into a disposable Docker container. The +container runs one job and is then deleted; the next job gets a new container, +a new runner identity and a new working directory from the same clean image. +Users do not want to maintain a per-command permission allowlist for CI. + +The disposable unit is a job, not a workflow, a step or a PR. Multiple jobs +exchange files through GitHub artifacts and never share a writable working +directory. + +This design addresses run-environment residue and cross-task file and process +contamination. Plain Docker shares the host kernel and gives no virtual-machine +isolation from malicious code; any secret passed into a job can still be read +by that job. The first phase accepts only Linux CI work without long-lived +secrets: no model gateway is built here, and release, autofix, real-model E2E +and tasks that need a Docker daemon are not migrated. + +## Architecture + +```text +External-CI dedicated ECS +├── systemd-managed manager (long-running, trusted) +│ ├── requests JIT runner configuration with the host-only PAT +│ ├── starts, monitors and deletes containers and their isolated networks +│ └── keeps management state, exports logs, recovers interrupted tasks +├── slot 1: container A → one job → delete → container C +└── slot 2: container B → one job → delete → container D + +Each container +├── one JIT runner that connects to GitHub on its own +├── Node.js 22, Git and the CI dependencies +└── its own HOME, working directory, temporary directory and npm cache +``` + +The manager keeps a configured number of slots, at most one container per slot. +The Hangzhou trial ran ten slots at once; the hz4 deployment now maintains five slots. It takes no webhook, schedules no PR +itself, and introduces neither Kubernetes nor a database. GitHub assigns work by +runner label. Each ECS runs its own manager against its local Docker socket, +with a unique `poolId` and a per-host `slots` count. Hosts share the scheduling +labels; no central manager is needed. New names use `ecs-qwen-` for compatibility +with CI worker caps and timeouts; recovery still accepts old `qwen-` identities. + +systemd starts the manager at boot when enabled and restarts unexpected exits +after ten seconds (`Restart=always`). An explicit service stop remains stopped. +This supervises process exits; it does not detect a live but stuck manager. + +Images are produced by a trusted build and pinned by digest. The manager and the +image may only be deployed from versions maintained under maintainer control, +never updated from a job checkout or from PR artifacts. + +Authentication uses a dedicated PAT in the root-only systemd EnvironmentFile, +`/etc/qwen-ephemeral-runner/credentials.env`, as `GH_TOKEN`. A fine-grained token +should select only the target repository and Administration: write, the GitHub +API's required permission. It is not restricted to runner operations alone. +Fine-grained access can be unavailable for outside collaborators or under +organization policy; a broader classic token requires an explicit decision. +The PAT stays on the host, never in the config JSON or a job container. Host +root and Docker administrators remain trusted. Rotate the credential file and +restart the service before expiry; no GitHub App or organization runner +permission is needed for repository-scoped runners. + +## Ownership and lifecycle + +| Object | Owner | Lifetime | +| ------------------------------------- | ----------------------- | ---------------------------------------------------- | +| ECS, Docker daemon, manager | Ops / maintainers | Long term | +| Host-only runner-management PAT | Manager | Never enters a job container | +| Clean image | Trusted image build | Versioned, not writable from a job | +| JIT configuration, runner identity | One container instance | Used for a single job | +| Container writable layer and job data | One container instance | Deleted on destruction | +| Docker bridge network | One container instance | Deleted on destruction | +| Diagnostic logs | Exported by the manager | Size and retention bounded, unreadable by later jobs | + +Normal path: request JIT → create the isolated network and container → runner +comes online → run one job → runner exits → export logs → delete the container +and network → clean up the leftover registration → create the next one. + +JIT provides single-job registration semantics under the normal protocol; the +host manager is responsible for actually destroying the container. The design +must not rely on a malicious job exiting on its own, finishing a final cleanup +step, or keeping the runner intact. A watchdog outside the container enforces a +maximum lifetime, and a manager restart reconciles old resources first. + +If deleting a container fails, that slot stops; if GitHub registration cleanup +fails, the failure is recorded and retried instead of endlessly requesting new +identities. Host failures cannot clean up immediately, so recovery must reclaim +before accepting new work. + +## Minimal but necessary deployment boundary + +- Use an ECS dedicated to external CI; do not share a host with existing runners + that hold long-lived secrets. +- Mount no host home, workspace, management state, credential directory or + Docker socket; use neither privileged mode, host PID nor host networking. No + per-command allowlist is needed inside the container. +- Images declare no persistent VOLUME and the first phase creates no job volume; + all job writes go to the container writable layer. +- Network policy outside the host and unmodifiable from the container blocks the + management interface, sensitive intranets, instance credentials and metadata; + configure only the GitHub, npm and image-service egress actually required. +- Access granted to the old trusted runners must be verified on the GitHub side. + Neither labels nor PR-modifiable YAML are an authorization boundary; an + external PR must not route back into a sensitive pool by editing runs-on. +- Do not open fork secrets because a self-hosted runner is enabled, and do not + introduce a privileged pull_request_target flow that executes PR code. + +## Initial routing decision + +Keep the existing `ubuntu_runner` output and all of its former semantics, and add +a `pr_test_runner` output used by `test_build` and `test`. For an untrusted fork +PR, `EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` selects the disposable pool; +unset, `false`, or any other value selects `ubuntu-latest`. `lint_runner` follows +the same switch for `lint_and_static` and `integration_no_ak`, replacing the old +shared-ECS fallback for these fork jobs. Trusted and same-repository PR routing +stays unchanged. The global kill-switch and hosted setting take precedence. + +The switch selects a destination when a job is evaluated, without querying +runner availability. It cannot move an already queued job; cancel queued work +and start a new workflow run to apply a rollback. Verify unset/false/invalid, +true for both architectures, and the global overrides against the actual shell +selector before rollout. + +`classify_pr`, `web_shell_e2e_smoke`, `integration_cli` and the other workflows +keep their current routing. This avoids migrating browser system dependencies +that the new image has not verified, and leaves the routing image relationship +of the existing classifier job and the other workflows unchanged. + +The new pool label is `ecs-qwen-ephemeral`. Default routing uses x64; setting +`EXTERNAL_PR_EPHEMERAL_ARCH=arm64` selects ARM64 instead. The toolchain of full +PR CI on the new image still has to be verified. + +## References + +- [GitHub ephemeral runner semantics and cleanup responsibility](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#ephemeral-runners-for-autoscaling) +- [GitHub repository-level JIT API](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository) +- [Docker daemon and kernel security boundary](https://docs.docker.com/engine/security/) + +The facts this design relies on were checked against the official documentation +during this discussion; the actual runner version, GitHub account capabilities, +API permissions and ECS network configuration must be confirmed again at +deployment time. diff --git a/docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md b/docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md new file mode 100644 index 00000000000..5378d5bdcdd --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md @@ -0,0 +1,133 @@ +# 外部 PR 的一次性容器 runner + +[English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md) + +日期:2026-11-07。状态:已实现并在 hz4 部署五槽位 systemd 试运行服务; +外部 PR 的生产路由仍未启用。 + +杭州共享测试机上已验证 10 个槽位同时注册,并执行同一个私有测试仓库的 10 个无密钥 +测试 job,容器、网络和注册均已回收。后续单槽位试运行使用真实 PAT, +通过两个串行 GitHub Actions job 验证注册、容器替换、工作区隔离及 job 内 +没有 PAT;强制结束 manager 后也验证了 systemd 重启及补位前清理。 +试运行发现并修复了 gh 2.46 分页兼容性及 Docker quiet 模式覆盖格式化 +槽位输出的问题。完整 PR CI 和生产隔离仍未验证,共享主机只用于受控试验。 + +后续模型访问见 [runner 与模型代理结合方案](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md); +该方案尚未实现或启用。 + +## 目标和边界 + +将整个 GitHub Actions runner 放进一次性 Docker 容器。容器执行一个 job, +随后删除;下一次从同一个干净镜像创建新容器、新 runner 身份和新工作目录。 +用户不希望为 CI 中的每条命令维护权限白名单。 + +一次性单位是 job,不是 workflow、step 或 PR。多个 job 通过 GitHub artifacts +交换文件,不共享可写工作目录。 + +本设计解决运行环境残留和跨任务文件、进程污染。普通 Docker 共享宿主机内核, +不提供虚拟机级恶意代码隔离;传入 job 的任何密钥仍可能被该 job 读取。 +首期只接无长期密钥的 Linux CI,不建设模型网关,不迁移发布、自动修复、 +真实模型 E2E 或需要 Docker daemon 的任务。 + +## 架构 + +```text +外部 CI 专用 ECS +├── systemd 托管的管理程序(长期运行,可信) +│ ├── 使用宿主机 PAT 申请 JIT runner 配置 +│ ├── 启动、监控、删除容器及其独立网络 +│ └── 保存管理状态、导出日志、恢复异常任务 +├── 槽位 1:容器 A → 一个 job → 删除 → 容器 C +└── 槽位 2:容器 B → 一个 job → 删除 → 容器 D + +每个容器 +├── 一个 JIT runner,主动连接 GitHub +├── Node.js 22、Git 和 CI 依赖 +└── 独立的 HOME、工作目录、临时目录、npm 缓存 +``` + +管理程序按配置维护固定数量的槽位,每个槽位最多一个容器;杭州十槽位试验同时使用 +十个槽位,hz4 部署目前维持五个槽位。不接 webhook、不自行调度 PR、不引入 Kubernetes 或数据库。 +GitHub 根据 runner 标签分配任务。每台 ECS 各运行一个管理程序,只连接本机 +Docker socket,使用独立的 `poolId` 和按主机配置的 `slots`。各主机共用调度 +标签,无需中央管理器。新名称使用 `ecs-qwen-` 前缀以兼容 CI 并发上限和超时; +恢复时仍接受旧的 `qwen-` 身份。 + +启用 systemd 服务后开机启动,意外退出时十秒后重启(`Restart=always`); +显式停止服务后保持停止。它监督进程退出,不检测进程存活但卡住的情况。 + +镜像由可信构建生成并固定 digest。管理程序和镜像只能从经过维护者控制的版本 +部署,不从任务 checkout 或 PR artifacts 更新。 + +认证使用专用 PAT,以 `GH_TOKEN` 写入仅 root 可读的 systemd EnvironmentFile: +`/etc/qwen-ephemeral-runner/credentials.env`。fine-grained token 应只选择目标 +仓库及 GitHub API 要求的 Administration: write;它不只限于 runner 操作。 +外部协作者身份或组织策略可能使 fine-grained token 不可用,改用权限更宽的 +classic token 必须明确决定。PAT 留在宿主机,不进入配置 JSON 或任务容器。 +宿主机 root 和 Docker 管理员仍属于可信边界。到期前替换凭证文件并重启服务; +仓库级 runner 不需要 GitHub App 或组织级 runner 管理权限。 + +## 所有权和生命周期 + +| 对象 | 所有者 | 生命周期 | +| ---------------------------- | ------------ | ----------------------------------- | +| ECS、Docker daemon、管理程序 | 运维/维护者 | 长期 | +| 宿主机 runner 管理 PAT | 管理程序 | 不进入任务容器 | +| 干净镜像 | 可信镜像构建 | 版本化、不可由 job 回写 | +| JIT 配置、runner 身份 | 单个容器实例 | 只用于一个 job | +| 容器可写层和任务数据 | 单个容器实例 | 销毁时删除 | +| Docker bridge 网络 | 单个容器实例 | 销毁时删除 | +| 诊断日志 | 管理程序导出 | 有大小和保留期限,不被后续 job 读取 | + +正常路径:申请 JIT → 创建独立网络和容器 → runner 上线 → 执行一个 job → +runner 退出 → 导出日志 → 删除容器与网络 → 清理残留注册 → 创建下一份。 + +JIT 提供正常协议下的单任务注册语义;宿主机管理程序负责实际销毁。 +不能依赖恶意任务自行退出、完成最后一个 cleanup step 或保持 runner 完整。 +容器外 watchdog 强制限制最长存活时间,管理程序重启时先对账旧资源。 + +容器删除失败时停止对应槽位;GitHub 注册清理失败时记录并重试,不继续无限 +申请新身份。宿主机故障期间无法即时清理,恢复后必须先回收再接新任务。 + +## 最少但必要的部署边界 + +- 使用外部 CI 专用 ECS;不与持有长期密钥的现有 runner 共用主机。 +- 不挂载宿主机 home、工作区、管理状态、凭证目录、Docker socket;不使用 + privileged、host PID 或 host network。容器内无需逐命令白名单。 +- 镜像不声明持久 VOLUME,首期不创建任务卷;全部任务写入容器可写层。 +- 宿主机外/容器无法修改的网络策略阻止访问管理接口、敏感内网、实例凭证和 + 元数据;配置实际需要的 GitHub、npm、镜像服务出口。 +- 旧可信 runner 的访问权必须在 GitHub 侧检查。标签和 PR 可修改的 YAML 都 + 不是授权边界;外部 PR 不得通过修改 runs-on 绕回敏感池。 +- 不因启用自托管 runner 而开放 fork secrets,也不引入执行 PR 代码的 + pull_request_target 特权流程。 + +## 首期路由决策 + +保留现有 `ubuntu_runner` 输出及其所有旧语义,新增 `pr_test_runner` 输出, +用于 `test_build` 和 `test`。对于非可信作者的外部 fork PR,变量 +`EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` 选择一次性池;未设置、`false` +或其他值选择 `ubuntu-latest`。`lint_runner` 在 `lint_and_static` 和 +`integration_no_ak` 中遵循同一个开关,替代这些 fork 任务原有的共享 ECS +回退行为。可信作者和同仓库 PR 路由不变。全局 kill-switch 和 hosted 设置优先。 + +开关在 job 求值时选择目标,不查询 runner 是否空闲,也不能迁移已排队任务; +回滚时取消已排队任务并启动新的 workflow run。上线前用实际 shell 选择器 +验证未设置/false/非法值、开启时的两种架构,以及全局覆盖开关。 + +`classify_pr`、`web_shell_e2e_smoke`、`integration_cli`、其他 workflow +保持当前路由。这样不把新镜像尚未验证的浏览器系统依赖一起迁入,也不改变 +现有分类 job 与其他 workflow 的路由镜像关系。 + +新池标签为 `ecs-qwen-ephemeral`。默认路由使用 x64,设置变量 +`EXTERNAL_PR_EPHEMERAL_ARCH=arm64` 可改用 ARM64;完整 PR CI 在新镜像上的 +工具链仍须验证。 + +## 依据 + +- [GitHub ephemeral runner 语义与清理责任](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#ephemeral-runners-for-autoscaling) +- [GitHub 仓库级 JIT API](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository) +- [Docker 的 daemon 与内核安全边界](https://docs.docker.com/engine/security/) + +本设计采用的事实已在本次讨论中核对官方文档;部署时再次确认实际 runner +版本、GitHub 账户能力、API 权限和 ECS 网络配置。 diff --git a/docs/design/2026-11-07-ephemeral-runner-model-proxy.md b/docs/design/2026-11-07-ephemeral-runner-model-proxy.md new file mode 100644 index 00000000000..6a24ff14cae --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-runner-model-proxy.md @@ -0,0 +1,165 @@ +# Combining ephemeral runners with a model proxy + +[English](2026-11-07-ephemeral-runner-model-proxy.md) | [简体中文](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md) + +Date: 2026-11-07. Status: proposal and partial verification complete; the +combined service is neither implemented, deployed nor enabled. + +## Goals and scope + +When CI for an external PR needs a real model, every ephemeral runner uses its +own short-lived, restricted proxy credential. The real model AK goes only to the +trusted model proxy and never into a runner container. After the runner finishes +one job, the management service reclaims the container and revokes the +credential. Existing keyless jobs need no change. + +The existing triage/verify execution model is unchanged, no external PR gains +unrestricted model access by default, and Docker is not claimed to be +virtual-machine isolation from malicious code. This proposal provides neither +release capability, model-output trustworthiness, nor prompt-injection +protection. + +## Current code and verification + +- `scripts/ci/ephemeral-runner/manager.mjs` creates a single-job JIT runner for + each fixed slot, gives every container its own bridge, and replenishes after + reclamation. Model-proxy integration does not exist yet. +- `triage`, `tmux-testing` and `verify` in `.github/workflows/qwen-triage.yml` each embed + a model proxy: it listens on host loopback, validates requests with a random + bearer, substitutes the real AK, forwards only `POST .../chat/completions`, + and supports streaming responses and an idle timeout. These are per-workflow + temporary processes, not standalone gateways. +- Using the proxy extracted from the existing `verify` workflow plus a fake + upstream, we verified that a wrong token returns 401, other paths return 403, + a correct token is forwarded with the upstream receiving only the fake real + AK, and streamed data is returned. Two correct requests both succeed: the + current implementation has no per-request revocation. +- Five selected existing triage tests pass, covering authentication and the + streaming idle timeout for the `tmux-testing` and `verify` lanes; the + `triage` lane embeds the same proxy pattern and is in the extraction scope + below. +- On Hangzhou hz4, with a temporary Docker bridge and a single restricted test + container, we verified that `127.0.0.1` inside the container cannot reach a + loopback service on the host. Test resources were cleaned up. This does **not + prove** that a container cannot reach the host bridge address or other host + services. +- No real model AK was used; an actual proxy deployment, credential issuance + and revocation, quotas, model jobs for + external PRs, and production network isolation were not verified. + +## Trust boundary + +```text +GitHub job queue + ↓ dispatched to a labelled ephemeral runner +External-CI dedicated ECS Trusted proxy service environment + runner management service ──protected issue/revoke──→ model proxy admin API + │ │ + └─ runner container ──short-lived token──────→ model endpoint ──real AK──→ upstream +``` + +The model proxy is preferably deployed on a separate trusted host or service; +the real AK does not live on the runner ECS. The host management service holds +the runner-management PAT and the admin credential for the proxy admin API; +neither is mounted, injected into, or exposed to runner containers. The admin +API and the model endpoint are authorized separately and restricted by TLS and +network policy; containers cannot reach the issue/revoke endpoints, the +management service, the Docker socket, sensitive intranets, or ECS metadata. +Runners are allowed to reach only the model endpoint and the CI egress they +need. + +Containers have sudo, so isolating the real AK from the container's root/node +users cannot be relied on. Plain Docker shares the host kernel; untrusted code +still needs a dedicated host, with virtual-machine isolation considered +according to risk. Network policy is enforced by the host or external network +devices that the container cannot modify, and is verified in practice. + +## Credential and slot lifecycle + +The first version issues credentials per **runner instance**, rather than +pretending to issue per PR or per job: the management service does not yet know +which job GitHub will assign when it creates a runner. Model capability needs +its own dedicated runner label and runner group, used only by explicitly +permitted model-test workflows; model tokens must not be injected into the +default keyless runner pool. Runner group permissions on the GitHub side must be +verified: labels or workflow YAML in a PR are not an authorization boundary. + +State machine for each instance: + +1. Generate a non-reusable instance ID. Request a random token through the + protected admin API, setting model, request size, concurrency, call rate, + total volume and an absolute expiry. Store on the proxy only what validation + needs; persistent state and logs never record the plaintext token. +2. Deliver the proxy address and token to the corresponding new container; the + management service itself must not hand management credentials or the + upstream AK to a container. Supply them as `OPENAI_BASE_URL` and + `OPENAI_API_KEY` for compatible tests. The token is considered public to that + job; log redaction is not treated as a security boundary. +3. An idle runner still consumes its credential's lifetime. In the first + version the credential lifetime covers the instance's maximum lifetime, and + an instance left idle too long is reclaimed together with its runner; no + fixed remaining time at job start can be promised. +4. Normal exit, failure, cancellation, timeout and restart recovery all revoke + the token, delete the container and network, and clean up the GitHub + registration. A new instance gets a new token; old tokens are rejected. +5. When issuance is uncertain, do not create a container that can reach the + model; when revocation fails, stop replenishing that slot and retry, with the + absolute expiry as the final backstop. After a management-service or proxy + restart, reconcile first: the restart must not lose the revocation state or + reset budgets for already-issued tokens. + +Any script in the container can use or leak the token and may incur cost before +revocation; environment variables, container metadata, logs and artifacts are +all untrusted for that job. Credential capability must therefore be limited, +and the whole model pool or proxy endpoint must be switchable off immediately. + +## Proxy features and budget + +Extract a testable shared implementation from the existing forwarding logic and +preserve the behaviour of the three triage lanes; the new standalone service adds +credential issuance, revocation, persistent state, statistics and limits +separately, without changing the permissions or triggers of the existing +workflows. Allow only configured upstream targets, models and exact API paths; +requests must not control the upstream URL or pass an upstream authorization +header. Bound request body size, timeout and concurrency, keep streamed +disconnect handling, and prevent a client from triggering arbitrary address +access or unbounded resource use. + +Set per-instance request counts, rates, concurrency and input/output limits, plus +a shared budget across all instances and a hard quota or alert on the upstream +account. Concurrent debits must be atomic and over-limit requests rejected. +Request-count and output-parameter caps are not an exact cost cap; a streamed +response may still incur cost after disconnect. An exact cost cap depends on the +model provider's metering and quota capability and must be verified once the +upstream is chosen. Do not log plaintext credentials, complete prompts or +responses unless a concrete later requirement and data-handling policy exist. + +## Implementation and acceptance + +1. Determine which model tests actually need external-PR execution, and which + upstream model and budget they use; by default open exactly one controlled + test workflow and do not change the existing `integration_no_ak` or other CI + routing. +2. Extract and test the HTTP/streaming core of the existing proxy, preserving + both triage lanes. Implement the standalone proxy's admin API, separate + credentials, permissions, persistence, expiry, revocation and budget. +3. Add optional model-pool lifecycle to the management service; fail closed when + issuance or cleanup is uncertain. Cover token injection, log redaction, + abnormal recovery and old-token invalidation in tests. +4. In an isolated test environment with a fake upstream, one slot and a + controlled private repository, verify two consecutive jobs: the two tokens + differ; normal and streamed calls succeed; wrong, old, revoked and expired + tokens are rejected; over-limit requests, concurrency races, upstream + failures, and management-service/proxy restarts do not bypass the limits. +5. From inside a job, verify that the real AK, the runner-management PAT and the proxy + admin credential are unreadable, and that metadata, the Docker socket, other + containers and host management services are unreachable; verify that the + proxy can reach only the permitted upstream. Both container-cleanup and + credential-revocation failures must stop slot replenishment. +6. Prepare isolated dedicated ECS capacity and proxy services, and measure the + network and GitHub authorization, then run a one-slot trial with a low-budget + real model credential. Only after it passes does a maintainer decide whether + to enable the corresponding model-test routing. + +The existing runner design is described in +[the ephemeral runner design](2026-11-07-ephemeral-container-runners.md). diff --git a/docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md b/docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md new file mode 100644 index 00000000000..feea6dd7684 --- /dev/null +++ b/docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md @@ -0,0 +1,116 @@ +# 一次性 runner 与模型代理的结合方案 + +[English](2026-11-07-ephemeral-runner-model-proxy.md) | [简体中文](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md) + +日期:2026-11-07。状态:方案与局部验证完成;尚未实现组合服务、部署或启用。 + +## 目标与范围 + +外部 PR 的 CI 需要调用真实模型时,每个一次性 runner 使用独立、短期、受限的 +代理凭证。真实模型 AK 只交给可信模型代理,不进入 runner 容器。runner 执行 +一个 job 后,管理服务回收容器并撤销该凭证。原有无 AK 的 job 不需要改变。 + +不改变现有 triage/verify 的运行方式,不让所有外部 PR 默认获得无限制模型 +访问,不承诺 Docker 是对恶意代码的虚拟机级隔离。本方案不提供发布、模型输出 +可信性或 prompt 注入防护。 + +## 当前代码与验证 + +- `scripts/ci/ephemeral-runner/manager.mjs` 为固定槽位创建单 job JIT runner, + 每个容器使用单独的 bridge,回收完毕后再补充。当前没有模型代理集成。 +- `.github/workflows/qwen-triage.yml` 的 `triage`、`tmux-testing` 与 `verify` 各有一份 + 内嵌的模型代理:监听本机 loopback,以随机 bearer 校验请求,替换为真实 AK, + 只转发 `POST .../chat/completions`,支持流式响应与空闲超时。它们是各自 + workflow 的临时进程,不是独立网关。 +- 使用从现有 `verify` workflow 提取的代理和假上游,验证了错误 token 返回 + 401、其他路径返回 403、正确 token 转发且上游只收到假真实 AK,以及流式 + 数据返回。两个正确请求都能成功:现有实现没有按请求数撤销 token 的行为。 +- 现有 triage 测试中选中的 5 项通过,包含认证,以及 `tmux-testing` 与 `verify` + 两条 lane 的流式空闲超时;`triage` lane 使用同一套代理形态,也纳入下述提取范围。 +- 在杭州 hz4 通过一个临时 Docker bridge 和单个受限测试容器验证:容器内 + `127.0.0.1` 不能访问宿主机的 loopback 服务。测试资源已清理。这**不证明** + 容器不能访问宿主机 bridge 地址或其他宿主机服务。 +- 未使用真实模型 AK;未验证实际代理部署、凭证签发与 + 撤销、配额、外部 PR 的模型 job、或生产网络隔离。 + +## 信任边界 + +```text +GitHub job 队列 + ↓ 分派到带标签的一次性 runner +外部 CI 专用 ECS 可信代理服务环境 + runner 管理服务 ──受保护的签发/撤销──→ 模型代理管理接口 + │ │ + └─ runner 容器 ──短期 token────→ 模型调用接口 ──真实 AK──→ 上游 +``` + +模型代理优先部署在独立可信主机或服务,真实 AK 不放在 runner ECS。宿主机 +管理服务保存 runner 管理 PAT 和调用代理管理接口的管理凭证;这些都不挂载、 +注入或暴露给 runner 容器。管理接口与模型调用接口分开授权,并通过 TLS 和 +网络策略限制来源;容器不能访问签发、撤销接口、管理服务、Docker socket、 +敏感内网和 ECS 元数据。runner 仅获准访问模型调用接口及必要 CI 出口。 + +容器有 sudo,不能依赖容器内 root/node 用户隔离真实 AK。普通 Docker 共享 +宿主机内核;涉及不可信代码时仍需专用主机,按风险考虑虚拟机级隔离。网络 +策略由容器不能修改的宿主机或外部网络设备执行并实际验证。 + +## 凭证和槽位生命周期 + +第一版按 **runner 实例**签发,而不是假装按 PR 或 job 签发:管理服务创建 +runner 时还不知道 GitHub 将分配哪个 job。模型能力需使用独立的专用 runner +标签与 runner group,只供经明确允许的模型测试 workflow 使用;不要把模型 +token 注入默认无 AK 的 runner 池。GitHub 侧 runner group 权限必须经过核验, +标签或 PR 中的 workflow YAML 不是授权边界。 + +每个实例的状态机: + +1. 生成不可复用的实例 ID。通过受保护的管理接口申请随机 token,设置模型、 + 请求大小、并发、调用速率、总量及绝对过期时间。只在代理端保存验证所需 + 信息;持久状态和日志不记录明文 token。 +2. 将 proxy 地址和 token 交付给对应新容器;管理服务自身不能将管理凭证或 + 上游 AK 交给容器。以 `OPENAI_BASE_URL`、`OPENAI_API_KEY` 供兼容的测试 + 使用,token 对这个 job 视为已公开,不依赖日志脱敏作为安全边界。 +3. runner 空闲时也占用凭证有效期。第一版凭证有效期覆盖实例最长生命周期, + 空闲太久后连 runner 一起回收;不能承诺任务开始时仍有固定剩余时间。 +4. 正常退出、失败、取消、超时或重启恢复均撤销 token、删除容器和网络、 + 清理 GitHub 注册。新实例获得新 token;旧 token 被拒绝。 +5. 签发结果不确定时,不创建可访问模型的容器;撤销失败时停止对应槽位补充 + 并重试,凭证绝对过期作为最终兜底。管理服务/代理重启后应先对账,不能 + 因重启丢失已签发 token 的撤销状态或重置预算。 + +容器中任何脚本都能使用或泄露 token,且撤销前可能产生费用;环境变量、 +容器元数据、日志和 artifact 都不能被视作对该 job 保密。必须限制凭证能力, +并允许立即整体关闭模型池或代理调用接口。 + +## 代理功能与预算 + +从现有转发逻辑提取可测试的共享实现,保持已有三条 triage lane 行为; +新的独立服务另加凭证签发、吊销、持久状态、统计和限额,不直接修改既有 +workflow 的权限或触发方式。只允许配置好的上游目标、模型和确切 API 路径, +不得让请求控制上游 URL 或传递上游授权头;限制请求体大小、超时和并发, +保留流式断连处理,避免客户端触发任意地址访问或无限资源消耗。 + +按实例设置请求数、速率、并发和输入/输出上限,并设置所有实例共享的总预算 +及上游账户侧的硬额度或告警。并发扣额需要原子处理,拒绝超额请求。请求数 +和输出参数上限并不等于精确费用上限;流式响应可能在断连后仍产生费用。 +精确成本上限依赖模型供应方的计量/限额能力,应在选定上游后验证。不要记录 +明文凭证、完整 prompt 或回复,除非后续有明确需求和数据处理策略。 + +## 实施和验收 + +1. 确定哪些模型测试确实需要外部 PR 执行、上游模型及预算;默认只开放一条 + 受控测试 workflow,先不要改变现有 `integration_no_ak` 或其他 CI 路由。 +2. 提取并测试现有 proxy 的 HTTP/流式核心,保留三个 triage lane 行为。实现 + 独立代理的管理接口、独立凭证、权限、持久化、过期、撤销和预算。 +3. 为管理服务增加可选的模型池生命周期;签发或清理不确定时 fail closed。 + 明确 token 注入、日志脱敏、异常恢复及旧 token 失效的测试。 +4. 在独立测试环境使用假上游、1 个槽位及受控私有仓库,验证两次连续 job: + 两个 token 不同;正常和流式调用通过;错误、旧、撤销和过期 token 被拒; + 超额、并发竞争、上游故障、管理服务/代理重启不绕过限制。 +5. 从 job 内验证不能读到真实 AK、runner 管理 PAT、代理管理凭证,也不能访问元数据、 + Docker socket、其他容器和宿主机管理服务;验证代理只能访问允许的上游。 + 容器清理和凭证撤销失败均阻止槽位补充。 +6. 准备隔离的专用 ECS 和代理服务、实测网络与 GitHub 授权,再用低预算真实 + 模型凭证进行一槽位试运行。通过后由维护者决定是否启用相应模型测试路由。 + +现有 runner 方案见 [一次性 runner 设计](2026-11-07-ephemeral-container-runners.zh-CN.md)。 diff --git a/scripts/ci/ephemeral-runner/Dockerfile b/scripts/ci/ephemeral-runner/Dockerfile new file mode 100644 index 00000000000..9d530fe7929 --- /dev/null +++ b/scripts/ci/ephemeral-runner/Dockerfile @@ -0,0 +1,39 @@ +ARG BASE_IMAGE=node:22-bookworm-slim@sha256:5ec23b7c622b00547753bc6ffd78085dd868a68aeb41c8e0a1418f033e13d96d +FROM ${BASE_IMAGE} + +ARG TARGETARCH +ARG RUNNER_VERSION=2.338.2 +ARG RUNNER_RELEASE_URL=https://github.com/actions/runner/releases/download +ARG DEBIAN_MIRROR=deb.debian.org +ARG PIP_INDEX_URL=https://pypi.org/simple + +RUN sed -i "s|deb.debian.org|${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources \ + && apt-get update -o Acquire::http::Timeout=30 \ + && apt-get install -y --no-install-recommends \ + build-essential ca-certificates curl file gh git git-lfs jq libicu72 \ + libkrb5-3 libssl3 libunwind8 procps python3 python3-pip python3-venv \ + sudo tmux unzip zip zlib1g \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m venv /opt/ci-tools \ + && /opt/ci-tools/bin/pip install --no-cache-dir 'yamllint==1.35.1' \ + && echo 'node ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/runner + +ENV PATH="/opt/ci-tools/bin:${PATH}" + +WORKDIR /home/node/actions-runner +RUN case "${TARGETARCH}" in \ + amd64) runner_arch=x64; runner_sha=baebf204307bac6193480852732f0aba2ae97b6c5597c244f4d5558390147c40 ;; \ + arm64) runner_arch=arm64; runner_sha=670adb2f268fe21dcbb58b3db0e0aa9d5b3bddfe2c651481336e561ff86de945 ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl -fsSL --retry 3 --connect-timeout 15 --max-time 180 -o /tmp/runner.tar.gz \ + "${RUNNER_RELEASE_URL}/v${RUNNER_VERSION}/actions-runner-linux-${runner_arch}-${RUNNER_VERSION}.tar.gz" \ + && echo "${runner_sha} /tmp/runner.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/runner.tar.gz \ + && rm /tmp/runner.tar.gz \ + && chown -R node:node /home/node/actions-runner + +COPY entrypoint.sh /usr/local/bin/ephemeral-runner +RUN chmod 755 /usr/local/bin/ephemeral-runner +USER node +ENTRYPOINT ["/usr/local/bin/ephemeral-runner"] diff --git a/scripts/ci/ephemeral-runner/README.md b/scripts/ci/ephemeral-runner/README.md new file mode 100644 index 00000000000..d1e40510b5a --- /dev/null +++ b/scripts/ci/ephemeral-runner/README.md @@ -0,0 +1,140 @@ +# Ephemeral GitHub Actions runners + +Each slot registers one just-in-time runner and runs it in a new container. +After at most one job, the manager removes the container and runner registration +before creating the next one. While the manager waits on a container it also +enforces a per-slot deadline and removes a container whose runner does not exit; +a manager crash is covered by reconciling leftover resources on the next start. + +Each job has a separate Docker bridge with inter-container communication +disabled. Deleting the network is part of cleanup and a failure blocks slot +replacement. This does not itself prevent access to the host or private +networks: configure and verify the host's external egress controls separately. + +Use a dedicated host for untrusted pull requests. Docker shares the host kernel; +jobs must not receive the host Docker socket, host directories, cloud instance +credentials, internal network access, or long-lived secrets. A runner label is a +scheduling selector, not an access-control boundary. Before enabling external +pull requests, verify they cannot select other sensitive runner pools. + +## Build the image + +The image supports Linux x64 and ARM64. Its Node base is pinned to an OCI index +digest and the runner downloads are checked against pinned SHA-256 digests. +Use BuildKit to select both the base image and runner architecture: + +```sh +docker buildx build --platform linux/arm64 --load -t qwen-ephemeral-runner:trial scripts/ci/ephemeral-runner +``` + +BuildKit supplies `TARGETARCH`. With Docker's legacy builder, build on the target +architecture and pass its matching `TARGETARCH` explicitly (`amd64` or `arm64`). +That argument selects only the runner archive; it does not change the base image +architecture. + +The runner image supports ARM64, but full PR CI on this image has not yet been +verified. The repository also selects ARM64 linters; verify the complete CI +toolchain before routing production work to either architecture. + +Hosts with restricted upstream connectivity can set `DEBIAN_MIRROR`, +`PIP_INDEX_URL` or `RUNNER_RELEASE_URL` at build time. The runner archive is +still verified against its pinned digest when served from a mirror. + +For a real deployment, publish the image to a registry and configure its +immutable `repository@sha256:...` digest. A local `sha256:...` image ID may be +used for a trial. Runner and base versions need routine security updates. + +## Configure the manager + +Run one manager on each host, managing only that host's Docker socket. Give each +host a unique `poolId`; matching runner labels let GitHub share work across the +hosts without a central scheduler. `slots` is a per-host limit, not a fleet total. +New runner names start with `ecs-qwen-` so the existing CI worker caps and test +timeouts apply. Restart recovery also accepts the earlier `qwen-` identities. + +Install Docker, Node 22, `gh`, and `flock` on the host. The manager uses a +dedicated PAT supplied as `GH_TOKEN`. The account must administer the target +repository. Prefer a fine-grained PAT restricted to that repository with +Administration: write and a short expiry. GitHub currently requires this broader +permission for runner registration; it is not a runner-only credential. Some +organization or outside-collaborator policies prevent fine-grained access. Do +not silently replace it with a classic `repo` token, which can cover other +repositories the account can access. + +Store `GH_TOKEN=` in `/etc/qwen-ephemeral-runner/credentials.env`, owned by +root with mode `600`, inside the root-only configuration directory. The supplied +systemd unit loads this file; do not put it in shell history, config JSON, job +environments or the repository. Replace it and restart the service to rotate or +renew the token. The service fails startup if `GH_TOKEN` is missing. Docker child +environments are allowlisted, and only the one-time JIT configuration enters a +job container. Host root and Docker administrators are trusted and can access +host credentials; ordinary container isolation does not change that boundary. + +Copy `config.example.json` to `/etc/qwen-ephemeral-runner/config.json` and +replace the placeholders: repository, actual runner group ID, unique host/pool +ID, immutable image reference, and matching architecture and labels. Use a +unique test label and a controlled test repository for a trial. The supplied +manual smoke workflow selects only `ecs-qwen-ephemeral-smoke`; a trial runner +can register with just that label to avoid matching generic Linux jobs. The example runs +one slot; increase only after measuring the host. Restrict configuration and +credential directories to the manager account. + +The `ttlSeconds` limit starts when the container starts, including idle time. +A job assigned near the deadline can be interrupted. The manager removes and +replaces the container; GitHub remains authoritative for job status. The +manager logs cleanup but does not distinguish expiry from normal completion. +A nonzero or unknown container exit status delays replenishment by 60 seconds; +a successful short job is replenished immediately. GitHub remains authoritative +for job success. Diagnostic export is best effort: stderr is not archived, and +output exceeding the capture limits is discarded. +`minDiskBytes` applies to the Docker data filesystem. Configure and verify +disk quotas or a dedicated data volume as appropriate for the host. + +## Install and operate + +Install `manager.mjs` in `/opt/qwen-ephemeral-runner/`, install the supplied +unit in `/etc/systemd/system/`, and create the configured state/log directories +with permissions restricted to the manager account. From the trusted checkout: + +```sh +sudo install -d /opt/qwen-ephemeral-runner +sudo install -d -m 700 /etc/qwen-ephemeral-runner /var/lib/qwen-ephemeral-runner /var/log/qwen-ephemeral-runner +sudo install -m 644 scripts/ci/ephemeral-runner/manager.mjs /opt/qwen-ephemeral-runner/manager.mjs +sudo install -m 644 scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service /etc/systemd/system/qwen-ephemeral-runner.service +``` + +Finish the configuration and PAT setup above before starting. A Docker-authorized user +has host-level power; the unit is intended for a trusted dedicated host. + +```sh +sudo systemctl daemon-reload +sudo systemctl enable --now qwen-ephemeral-runner.service +sudo systemctl status qwen-ephemeral-runner.service +sudo journalctl -u qwen-ephemeral-runner.service -f +``` + +systemd restarts an unexpectedly exited manager after ten seconds, including a +clean exit. An explicit `systemctl stop` leaves it stopped. This is process +supervision, not hang detection; inspect the journal and runner availability if +slots are not replenishing. + +Stopping the service stops replenishment, waits up to the configured grace +period, and cleans up its containers. The next start reconciles abandoned +resources first. A cleanup or ambiguous registration error blocks that slot; +inspect its state and GitHub runner list before manual reconciliation. Never +remove unrelated runners or run a global Docker prune. + +```sh +sudo systemctl stop qwen-ephemeral-runner.service +``` + +After validating a single runner and its recovery paths, set the repository +variable `EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` to route untrusted fork PR +build, test, lint and no-AK integration jobs to this pool. Unset, `false`, or any +other value keeps those jobs on `ubuntu-latest`. Trusted and same-repository PRs +retain their existing routing. The global kill-switch and hosted-fleet override +still win. Leave the switch off until the host and GitHub access policies have +been verified. Changing it affects newly evaluated jobs; cancel queued work and +start a new workflow run when rolling back. It does not detect idle runners or +migrate a queued job automatically. +Set `EXTERNAL_PR_EPHEMERAL_ARCH=arm64` for an ARM64 pool; the default is x64. diff --git a/scripts/ci/ephemeral-runner/config.example.json b/scripts/ci/ephemeral-runner/config.example.json new file mode 100644 index 00000000000..f00ef88f539 --- /dev/null +++ b/scripts/ci/ephemeral-runner/config.example.json @@ -0,0 +1,16 @@ +{ + "repo": "QwenLM/qwen-code", + "runnerGroupId": 1, + "poolId": "external-ci-host-1", + "image": "registry.example.invalid/qwen-runner@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "labels": ["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"], + "slots": 1, + "cpus": 4, + "memory": "8g", + "pidsLimit": 1024, + "ttlSeconds": 10800, + "shutdownGraceSeconds": 60, + "minDiskBytes": 10737418240, + "stateDir": "/var/lib/qwen-ephemeral-runner", + "logDir": "/var/log/qwen-ephemeral-runner" +} diff --git a/scripts/ci/ephemeral-runner/entrypoint.sh b/scripts/ci/ephemeral-runner/entrypoint.sh new file mode 100644 index 00000000000..34c03f4347b --- /dev/null +++ b/scripts/ci/ephemeral-runner/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${RUNNER_JIT_CONFIG:?RUNNER_JIT_CONFIG is required}" +jit_config="$RUNNER_JIT_CONFIG" +unset RUNNER_JIT_CONFIG +cd /home/node/actions-runner +exec ./run.sh --jitconfig "$jit_config" diff --git a/scripts/ci/ephemeral-runner/manager.mjs b/scripts/ci/ephemeral-runner/manager.mjs new file mode 100644 index 00000000000..4b1d75c9b24 --- /dev/null +++ b/scripts/ci/ephemeral-runner/manager.mjs @@ -0,0 +1,620 @@ +import { randomUUID } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { realpathSync } from 'node:fs'; +import { + lstat, + mkdir, + readFile, + readdir, + rename, + rm, + statfs, + writeFile, +} from 'node:fs/promises'; +import { isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const sleep = (ms) => + new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); + +export function command( + program, + args, + { env = process.env, input, timeout = 30000, binary = false } = {}, +) { + return new Promise((resolveCommand, reject) => { + const child = spawn(program, args, { + env, + stdio: ['pipe', 'pipe', 'pipe'], + }); + const output = []; + let outputSize = 0; + let stderr = ''; + const timer = setTimeout(() => child.kill('SIGKILL'), timeout); + child.stdout.on('data', (chunk) => { + outputSize += chunk.length; + if (outputSize > 4 * 1024 * 1024) child.kill('SIGKILL'); + else output.push(chunk); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + if (stderr.length > 1024 * 1024) child.kill('SIGKILL'); + }); + child.on('error', (error) => { + clearTimeout(timer); + reject(error); + }); + child.on('close', (code) => { + clearTimeout(timer); + if (code === 0 && outputSize <= 4 * 1024 * 1024) + resolveCommand( + binary ? Buffer.concat(output) : Buffer.concat(output).toString(), + ); + else { + const status = Number(stderr.match(/HTTP (\d{3})/)?.[1] || 0); + const error = new Error( + `${program} failed (exit ${code}${status ? `, HTTP ${status}` : ''})`, + ); + error.status = status; + error.exitCode = code; + reject(error); + } + }); + child.stdin.on('error', () => {}); + child.stdin.end(input); + }); +} + +function validate(c) { + if ( + !/^[\w.-]+\/[\w.-]+$/.test(c.repo || '') || + !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,39}$/.test(c.poolId || '') || + !/^(?:[a-zA-Z0-9][a-zA-Z0-9._:/-]*@)?sha256:[a-f0-9]{64}$/.test( + c.image || '', + ) || + !Number.isInteger(c.slots) || + c.slots < 1 || + c.slots > 10 || + !Number.isInteger(c.runnerGroupId) || + c.runnerGroupId < 1 || + !Array.isArray(c.labels) || + !c.labels.length || + !c.labels.every( + (label) => + typeof label === 'string' && /^[a-zA-Z0-9_-]{1,63}$/.test(label), + ) || + typeof c.stateDir !== 'string' || + !isAbsolute(c.stateDir) || + typeof c.logDir !== 'string' || + !isAbsolute(c.logDir) || + (c.dockerSocket !== undefined && + (typeof c.dockerSocket !== 'string' || + !isAbsolute(c.dockerSocket) || + c.dockerSocket.includes('\n'))) || + !Number.isFinite(c.cpus) || + !(c.cpus > 0) || + !/^[1-9]\d*[mMgG]$/.test(c.memory || '') || + !Number.isInteger(c.pidsLimit) || + c.pidsLimit < 1 || + !Number.isFinite(c.ttlSeconds) || + !(c.ttlSeconds > 0) || + !Number.isFinite(c.shutdownGraceSeconds) || + !(c.shutdownGraceSeconds >= 0) || + !Number.isSafeInteger(c.minDiskBytes) || + c.minDiskBytes < 0 + ) + throw new Error('Invalid manager configuration'); +} + +export function createManager(config, dependencies = {}) { + validate(config); + const execute = dependencies.command || command; + const run = (program, args, options = {}) => { + if (program !== 'docker') return execute(program, args, options); + const source = options.env || process.env; + const env = {}; + for (const key of ['PATH', 'HOME', 'DOCKER_CONFIG', 'XDG_RUNTIME_DIR']) { + if (source[key] !== undefined) env[key] = source[key]; + } + env.DOCKER_HOST = `unix://${config.dockerSocket || '/var/run/docker.sock'}`; + if (args[0] === 'create' && options.env?.RUNNER_JIT_CONFIG) + env.RUNNER_JIT_CONFIG = options.env.RUNNER_JIT_CONFIG; + return execute(program, args, { ...options, env }); + }; + const wait = dependencies.sleep || sleep; + const diskStats = dependencies.statfs || statfs; + let stopping = false; + let stopAt = Infinity; + const stateDir = resolve(config.stateDir); + const logDir = resolve(config.logDir); + const lockPath = join(stateDir, 'manager.lock'); + const slotPath = (slot) => join(stateDir, `slot-${slot}.json`); + const log = (message) => console.log(`[${config.poolId}] ${message}`); + function parseJson(text) { + try { + return JSON.parse(text); + } catch { + throw new Error('Invalid JSON response from GitHub'); + } + } + + async function gh(args, input) { + return run('gh', ['api', ...args, ...(input ? ['--input', '-'] : [])], { + env: process.env, + input: input && JSON.stringify(input), + }); + } + + async function save(slot, state) { + const path = slotPath(slot); + const temp = `${path}.${randomUUID()}.tmp`; + await writeFile(temp, JSON.stringify(state), { mode: 0o600 }); + await rename(temp, path); + } + + async function load(slot) { + try { + const raw = await readFile(slotPath(slot), 'utf8'); + let state; + try { + state = JSON.parse(raw); + } catch { + throw new Error('Invalid slot state; manual reconciliation required'); + } + // Reconcile identities created before the ECS-compatible name prefix. + const name = + typeof state?.name === 'string' ? state.name.replace(/^ecs-/, '') : ''; + const prefix = `qwen-${config.poolId}-${slot}-`; + if ( + !name.startsWith(prefix) || + !/^[0-9a-f-]{36}$/.test(name.slice(prefix.length)) || + typeof state.uncertain !== 'boolean' || + (!state.uncertain && + (!Number.isSafeInteger(state.runnerId) || state.runnerId < 1)) || + (state.runnerId !== undefined && + (!Number.isSafeInteger(state.runnerId) || state.runnerId < 1)) + ) + throw new Error('Invalid slot state; manual reconciliation required'); + return state; + } catch (error) { + if (error.code === 'ENOENT') return null; + throw error; + } + } + + async function listContainers(slot) { + const output = await run('docker', [ + 'ps', + '-aq', + '--no-trunc', + '--filter', + `label=qwen.pool=${config.poolId}`, + '--filter', + `label=qwen.slot=${slot}`, + ]); + return output.trim().split('\n').filter(Boolean); + } + + async function listNetworks(slot) { + const args = [ + 'network', + 'ls', + '-q', + '--no-trunc', + '--filter', + `label=qwen.pool=${config.poolId}`, + ]; + if (slot !== undefined) args.push('--filter', `label=qwen.slot=${slot}`); + return (await run('docker', args)).trim().split('\n').filter(Boolean); + } + + async function listRunners() { + const output = await gh([ + '--paginate', + '--jq', + '.runners | @json', + `/repos/${config.repo}/actions/runners?per_page=100`, + ]); + const pages = output.trim().split('\n').map(parseJson); + if ( + !pages.every( + (runners) => + Array.isArray(runners) && + runners.every( + (runner) => + typeof runner?.name === 'string' && + Number.isSafeInteger(runner.id) && + runner.id > 0, + ), + ) + ) + throw new Error('Invalid GitHub runner list'); + return pages.flat(); + } + + async function pruneLogs() { + const files = []; + for (const entry of await readdir(logDir, { withFileTypes: true })) { + if ( + !entry.isFile() || + !entry.name.replace(/^ecs-/, '').startsWith(`qwen-${config.poolId}-`) || + !/\.(log|tar)$/.test(entry.name) + ) + continue; + const path = join(logDir, entry.name); + const info = await lstat(path); + files.push({ path, size: info.size, time: info.mtimeMs }); + } + files.sort((a, b) => b.time - a.time); + let size = 0; + for (const file of files) { + size += file.size; + if (file.time < Date.now() - 7 * 86400000 || size > 100 * 1024 * 1024) + await rm(file.path, { force: true }); + } + } + + async function cleanup(slot, state) { + const containers = await listContainers(slot); + const networks = await listNetworks(slot); + if ((containers.length || networks.length) && !state?.name) + throw new Error( + 'Container has no recorded runner identity; manual reconciliation required', + ); + for (const id of containers) { + try { + const logs = await run('docker', ['logs', '--tail', '10000', id]); + await writeFile( + join( + logDir, + `${state?.name || `slot-${slot}`}-${id.slice(0, 12)}.log`, + ), + logs.slice(-1024 * 1024), + { mode: 0o600 }, + ); + await pruneLogs(); + } catch { + log(`slot ${slot}: container logs unavailable`); + } + try { + const archive = await run( + 'docker', + ['cp', `${id}:/home/node/actions-runner/_diag`, '-'], + { binary: true }, + ); + await writeFile( + join(logDir, `${state.name}-${id.slice(0, 12)}.tar`), + archive, + { mode: 0o600 }, + ); + await pruneLogs(); + } catch { + log( + `slot ${slot}: runner diagnostics unavailable or larger than 4 MiB`, + ); + } + await run('docker', ['rm', '-f', '-v', id]); + } + if ((await listContainers(slot)).length) + throw new Error('Container cleanup failed'); + for (const id of networks) await run('docker', ['network', 'rm', id]); + if ((await listNetworks(slot)).length) + throw new Error('Network cleanup failed'); + if (state?.name) { + const all = await listRunners(); + const runners = all.filter((runner) => runner.name === state.name); + if ( + state.runnerId && + (runners.some((runner) => runner.id !== state.runnerId) || + all.some( + (runner) => + runner.id === state.runnerId && runner.name !== state.name, + )) + ) + throw new Error( + 'Runner identity mismatch; manual reconciliation required', + ); + if ( + runners.length > 1 || + runners.some( + (runner) => !Number.isSafeInteger(runner.id) || runner.id < 1, + ) + ) + throw new Error( + 'Ambiguous runner identity; manual reconciliation required', + ); + if (state.uncertain && !runners.length) + throw new Error( + 'JIT response was lost; runner not found, manual reconciliation required', + ); + for (const runner of runners) { + try { + await gh([ + '-X', + 'DELETE', + `/repos/${config.repo}/actions/runners/${runner.id}`, + ]); + } catch (error) { + if (error.status !== 404) throw error; + } + } + } + await rm(slotPath(slot), { force: true }); + log(`slot ${slot}: cleaned`); + } + + async function cycle(slot) { + let earlyExit = false; + let state = await load(slot); + if ( + state || + (await listContainers(slot)).length || + (await listNetworks(slot)).length + ) + await cleanup(slot, state); + if (stopping) return; + const dockerRoot = ( + await run('docker', ['info', '--format', '{{.DockerRootDir}}']) + ).trim(); + if (!isAbsolute(dockerRoot) || dockerRoot.includes('\n')) + throw new Error('Invalid Docker Root Dir'); + const disk = await diskStats(dockerRoot); + if (disk.bavail * disk.bsize < config.minDiskBytes) + throw new Error( + `Insufficient disk capacity on Docker Root Dir ${dockerRoot}`, + ); + state = { + name: `ecs-qwen-${config.poolId}-${slot}-${randomUUID()}`, + createdAt: new Date().toISOString(), + uncertain: true, + }; + await save(slot, state); + let result; + try { + result = parseJson( + await gh( + [ + '-X', + 'POST', + `/repos/${config.repo}/actions/runners/generate-jitconfig`, + ], + { + name: state.name, + runner_group_id: config.runnerGroupId, + labels: config.labels, + work_folder: '_work', + }, + ), + ); + } catch (error) { + if (error.status >= 400 && error.status < 500 && error.status !== 408) { + await rm(slotPath(slot), { force: true }); + } else { + await cleanup(slot, state); + } + throw error; + } + try { + if ( + result?.runner?.name !== state.name || + !Number.isSafeInteger(result.runner.id) || + result.runner.id < 1 + ) + throw new Error('Invalid JIT runner identity'); + state.runnerId = result.runner.id; + state.uncertain = false; + await save(slot, state); + if (!result.encoded_jit_config) + throw new Error('GitHub returned no JIT configuration'); + const networkId = ( + await run('docker', [ + 'network', + 'create', + '--driver', + 'bridge', + '--opt', + 'com.docker.network.bridge.enable_icc=false', + '--label', + `qwen.pool=${config.poolId}`, + '--label', + `qwen.slot=${slot}`, + '--label', + `qwen.instance=${state.name}`, + state.name, + ]) + ).trim(); + if (!networkId) throw new Error('Docker returned no network ID'); + state.networkId = networkId; + await save(slot, state); + const env = { + ...process.env, + RUNNER_JIT_CONFIG: result.encoded_jit_config, + }; + delete env.GH_TOKEN; + delete env.GITHUB_TOKEN; + const id = ( + await run( + 'docker', + [ + 'create', + '--init', + '--network', + networkId, + '--label', + `qwen.pool=${config.poolId}`, + '--label', + `qwen.slot=${slot}`, + '--label', + `qwen.instance=${state.name}`, + '--cpus', + String(config.cpus), + '--memory', + config.memory, + '--pids-limit', + String(config.pidsLimit), + '--log-driver', + 'json-file', + '--log-opt', + 'max-size=10m', + '--log-opt', + 'max-file=3', + '--env', + 'RUNNER_JIT_CONFIG', + config.image, + ], + { env }, + ) + ).trim(); + state.containerId = id; + await save(slot, state); + if (!id) throw new Error('Docker returned no container ID'); + if (!stopping) { + await run('docker', ['start', id]); + log( + `slot ${slot}: started runner ${state.name}, id ${state.runnerId}, container ${id}`, + ); + const deadline = Date.now() + config.ttlSeconds * 1000; + let exitCode; + const watching = run('docker', ['wait', id], { + timeout: (config.ttlSeconds + 10) * 1000, + }).then( + (output) => { + const status = output.trim(); + exitCode = + /^\d+$/.test(status) && Number(status) <= 255 + ? Number(status) + : null; + return true; + }, + () => { + exitCode = null; + return true; + }, + ); + while (Date.now() < Math.min(deadline, stopAt)) { + if (await Promise.race([watching, wait(1000).then(() => false)])) + break; + } + if (exitCode !== undefined && exitCode !== 0) { + earlyExit = true; + log( + `slot ${slot}: runner exit status ${exitCode ?? 'unknown'}; next JIT request is delayed`, + ); + } + } + } finally { + await cleanup(slot, state); + } + return { earlyExit }; + } + + async function acquireLock() { + await mkdir(stateDir, { recursive: true, mode: 0o700 }); + await mkdir(logDir, { recursive: true, mode: 0o700 }); + const child = spawn('flock', ['-n', lockPath, 'cat'], { + stdio: ['pipe', 'pipe', 'ignore'], + }); + await new Promise((resolveLock, reject) => { + child.once('error', reject); + child.once('exit', () => + reject(new Error('Could not acquire manager lock')), + ); + child.stdout.once('data', resolveLock); + child.stdin.on('error', () => {}); + child.stdin.write('locked\n'); + }); + return child; + } + + async function start() { + const lock = await acquireLock(); + try { + const slots = ( + await run('docker', [ + 'ps', + '-a', + '--filter', + `label=qwen.pool=${config.poolId}`, + '--format', + '{{.Label "qwen.slot"}}', + ]) + ) + .trim() + .split('\n') + .filter(Boolean); + const states = (await readdir(stateDir)) + .map((name) => name.match(/^slot-(\d+)\.json$/)?.[1]) + .filter(Boolean); + const networkSlots = await Promise.all( + (await listNetworks()).map(async (id) => + ( + await run('docker', [ + 'network', + 'inspect', + '--format', + '{{index .Labels "qwen.slot"}}', + id, + ]) + ).trim(), + ), + ); + if ( + [...slots, ...states, ...networkSlots].some( + (slot) => + !/^\d+$/.test(slot) || + Number(slot) < 1 || + Number(slot) > config.slots, + ) + ) { + throw new Error( + 'Pool contains resources outside configured slots; reconcile before starting', + ); + } + await Promise.all( + Array.from({ length: config.slots }, async (_, i) => { + const slot = i + 1; + while (!stopping) { + try { + const result = await cycle(slot); + if (result?.earlyExit && !stopping) { + log(`slot ${slot}: backing off 60 s before replenishing`); + await wait(60000); + } + } catch (error) { + log( + `slot ${slot}: ${error.message}; retrying without replenishing`, + ); + await wait(10000); + } + } + }), + ); + } finally { + lock.stdin.end(); + } + } + function stop() { + stopping = true; + stopAt = Math.min(stopAt, Date.now() + config.shutdownGraceSeconds * 1000); + } + return { start, stop, cycle, cleanup }; +} + +if ( + process.argv[1] && + realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + if (!process.env.GH_TOKEN?.trim()) throw new Error('GH_TOKEN is required'); + const config = JSON.parse( + await readFile(resolve(process.argv[2] || ''), 'utf8'), + ); + const manager = createManager(config); + process.on('SIGTERM', manager.stop); + process.on('SIGINT', manager.stop); + await manager.start(); + } catch (error) { + console.error(`manager: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/scripts/ci/ephemeral-runner/manager.test.mjs b/scripts/ci/ephemeral-runner/manager.test.mjs new file mode 100644 index 00000000000..62c82c5a8d6 --- /dev/null +++ b/scripts/ci/ephemeral-runner/manager.test.mjs @@ -0,0 +1,822 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { command as runCommand, createManager } from './manager.mjs'; + +async function fixture(t) { + const dir = await mkdtemp(join(tmpdir(), 'runner-manager-')); + t.after(() => rm(dir, { recursive: true, force: true })); + await mkdir(join(dir, 'state')); + await mkdir(join(dir, 'logs')); + const config = { + repo: 'owner/repo', + poolId: 'trial', + image: `image@sha256:${'a'.repeat(64)}`, + slots: 1, + runnerGroupId: 1, + labels: ['self-hosted', 'linux', 'ARM64', 'trial-only'], + cpus: 1, + memory: '1g', + pidsLimit: 100, + ttlSeconds: 2, + shutdownGraceSeconds: 0, + minDiskBytes: 0, + stateDir: join(dir, 'state'), + logDir: join(dir, 'logs'), + }; + const calls = []; + const containers = new Map(); + const networks = new Map(); + let runner; + let counter = 0; + let foreignCounter = 0; + let failRemove = false; + let failNetworkRemove = false; + let keepContainerAfterRemove = false; + let loseJit = false; + const labelsFromArgs = (args) => { + const labels = {}; + for (let i = 0; i < args.length; i += 1) { + if (args[i] !== '--label' || typeof args[i + 1] !== 'string') continue; + const at = args[i + 1].indexOf('='); + if (at > 0) labels[args[i + 1].slice(0, at)] = args[i + 1].slice(at + 1); + } + return labels; + }; + const filtersFromArgs = (args) => { + const filters = []; + for (let i = 0; i < args.length; i += 1) { + if (args[i] === '--filter' && typeof args[i + 1] === 'string') + filters.push(args[i + 1].replace(/^label=/, '')); + } + return filters; + }; + const matchesFilters = (entry, args) => + filtersFromArgs(args).every((filter) => { + const at = filter.indexOf('='); + if (at < 0) return true; + return entry.labels[filter.slice(0, at)] === filter.slice(at + 1); + }); + async function command(program, args, options = {}) { + calls.push({ program, args, options }); + if (program === 'gh') { + const path = args.find((v) => v.startsWith('/')); + if (path.includes('generate-jitconfig')) { + const body = JSON.parse(options.input); + runner = { id: ++counter, name: body.name }; + if (loseJit) throw new Error('network failure'); + return JSON.stringify({ runner, encoded_jit_config: 'jit-secret' }); + } + if (args.includes('DELETE')) { + runner = undefined; + return ''; + } + return JSON.stringify(runner ? [runner] : []); + } + if (args[0] === 'ps') { + const matched = [...containers.entries()].filter(([, entry]) => + matchesFilters(entry, args), + ); + if (!matched.length) return ''; + if (args.includes('--format')) + return `${matched.map(([, entry]) => entry.labels['qwen.slot'] ?? '').join('\n')}\n`; + return `${matched.map(([id]) => id).join('\n')}\n`; + } + if (args[0] === 'network' && args[1] === 'ls') { + const matched = [...networks.entries()].filter(([, entry]) => + matchesFilters(entry, args), + ); + return matched.length ? `${matched.map(([id]) => id).join('\n')}\n` : ''; + } + if (args[0] === 'network' && args[1] === 'create') { + const id = `network-${counter}`; + networks.set(id, { labels: labelsFromArgs(args) }); + return id; + } + if (args[0] === 'network' && args[1] === 'rm') { + if (failNetworkRemove) throw new Error('network removal failed'); + networks.delete(args[args.length - 1]); + return ''; + } + if (args[0] === 'network' && args[1] === 'inspect') { + const entry = networks.get(args[args.length - 1]); + return `${entry?.labels['qwen.slot'] ?? ''}\n`; + } + if (args[0] === 'info') return `${dir}\n`; + if (args[0] === 'create') { + const id = `container-${counter}`; + containers.set(id, { labels: labelsFromArgs(args) }); + return id; + } + if (args[0] === 'logs') return 'runner log'; + if (args[0] === 'rm') { + if (failRemove) throw new Error('docker unavailable'); + if (!keepContainerAfterRemove) containers.delete(args[args.length - 1]); + return ''; + } + if (args[0] === 'wait') return '0\n'; + if (args[0] === 'start') return ''; + return ''; + } + return { + config, + calls, + command, + sleep: async () => {}, + addForeignNetwork(labels) { + const id = `network-foreign-${++foreignCounter}`; + networks.set(id, { labels }); + return id; + }, + networkIds: () => [...networks.keys()], + setFailRemove: (v) => { + failRemove = v; + }, + setLoseJit: (v) => { + loseJit = v; + }, + setFailNetworkRemove: (v) => { + failNetworkRemove = v; + }, + setKeepContainerAfterRemove: (v) => { + keepContainerAfterRemove = v; + }, + }; +} + +test('creates a new runner and container per job and keeps GitHub token out of Docker', async (t) => { + const previousToken = process.env.GH_TOKEN; + process.env.GH_TOKEN = 'host-only-pat'; + t.after(() => { + if (previousToken === undefined) delete process.env.GH_TOKEN; + else process.env.GH_TOKEN = previousToken; + }); + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + await manager.cycle(1); + await manager.cycle(1); + const creates = f.calls.filter( + (c) => c.program === 'docker' && c.args[0] === 'create', + ); + const requests = f.calls.filter( + (c) => + c.program === 'gh' && + c.args.includes('/repos/owner/repo/actions/runners/generate-jitconfig'), + ); + assert.ok( + requests.every((c) => + JSON.parse(c.options.input).name.startsWith('ecs-qwen-'), + ), + ); + assert.ok(requests.every((c) => c.options.env.GH_TOKEN === 'host-only-pat')); + assert.equal(creates.length, 2); + assert.notEqual( + JSON.parse(requests[0].options.input).name, + JSON.parse(requests[1].options.input).name, + ); + assert.deepEqual( + JSON.parse(requests[0].options.input).labels, + f.config.labels, + ); + assert.ok( + creates.every( + (c) => + !c.options.env.GH_TOKEN && + c.options.env.RUNNER_JIT_CONFIG === 'jit-secret', + ), + ); + assert.deepEqual( + creates.map((c) => c.args[c.args.indexOf('--network') + 1]), + ['network-1', 'network-2'], + ); + const networks = f.calls.filter( + (c) => + c.program === 'docker' && + c.args[0] === 'network' && + c.args[1] === 'create', + ); + assert.equal(networks.length, 2); + assert.ok( + networks.every((c) => + c.args.includes('com.docker.network.bridge.enable_icc=false'), + ), + ); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length, + 2, + ); +}); + +test('failed cleanup blocks replacement until recovery cleans the old container', async (t) => { + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + f.setFailRemove(true); + await assert.rejects(manager.cycle(1), /docker unavailable/); + await assert.rejects(manager.cycle(1), /docker unavailable/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); + f.setFailRemove(false); + await manager.cycle(1); + const ops = f.calls + .filter((c) => c.program === 'docker') + .map((c) => c.args[0]); + assert.ok(ops.lastIndexOf('rm') > ops.lastIndexOf('create')); + assert.equal(ops.filter((op) => op === 'create').length, 2); +}); + +test('lost JIT response reconciles the recorded name before requesting another', async (t) => { + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + f.setLoseJit(true); + await assert.rejects(manager.cycle(1), /network failure/); + await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), { + code: 'ENOENT', + }); + assert.ok( + f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')), + ); + f.setLoseJit(false); + await manager.cycle(1); +}); + +test('an unconfirmed JIT request blocks replacement when no runner can be found', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + return Promise.reject(new Error('connection lost')); + } + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), /manual reconciliation/); + await assert.rejects(manager.cycle(1), /manual reconciliation/); + const state = JSON.parse( + await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'), + ); + assert.equal(state.uncertain, true); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 0, + ); +}); + +test('a failed container start is cleaned up before another cycle', async (t) => { + const f = await fixture(t); + let fail = true; + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'start' && fail) { + fail = false; + return Promise.reject(new Error('start failed')); + } + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), /start failed/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length, + 1, + ); + await manager.cycle(1); +}); + +test('watchdog removes a container that does not exit', async (t) => { + const f = await fixture(t); + f.config.ttlSeconds = 0.01; + const command = (program, args, options) => + program === 'docker' && args[0] === 'wait' + ? new Promise(() => {}) + : f.command(program, args, options); + await createManager(f.config, { command }).cycle(1); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length, + 1, + ); +}); + +test('checks Docker root disk rather than the state directory', async (t) => { + const f = await fixture(t); + let checked; + const manager = createManager(f.config, { + command: f.command, + statfs: async (path) => { + checked = path; + return { bavail: 0, bsize: 4096 }; + }, + }); + f.config.minDiskBytes = 1; + await assert.rejects(manager.cycle(1), /Insufficient disk capacity/); + assert.equal(checked, join(f.config.stateDir, '..')); + assert.equal(f.calls.filter((c) => c.program === 'gh').length, 0); + const info = f.calls.find( + (c) => c.program === 'docker' && c.args[0] === 'info', + ); + assert.equal(info.options.env.DOCKER_HOST, 'unix:///var/run/docker.sock'); +}); + +test('rejects same-name runner with a different ID without deleting it', async (t) => { + const f = await fixture(t); + const state = { + name: 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + runnerId: 42, + uncertain: false, + }; + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify(state), + ); + const command = (program, args, options) => { + if (program === 'gh' && args.includes('--paginate')) + return Promise.resolve(JSON.stringify([{ name: state.name, id: 43 }])); + return f.command(program, args, options); + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + /identity mismatch/, + ); + assert.ok( + !f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')), + ); + assert.equal( + JSON.parse(await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8')) + .runnerId, + 42, + ); +}); + +test('reconciles a legacy identity on a later page before registering its replacement', async (t) => { + const f = await fixture(t); + const name = 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa'; + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify({ + name, + runnerId: 42, + uncertain: false, + }), + ); + let deleted = false; + const command = (program, args, options) => { + if (program === 'gh' && args.includes('--paginate') && !deleted) { + assert.ok(!args.includes('--slurp')); + assert.equal(args[args.indexOf('--jq') + 1], '.runners | @json'); + return Promise.resolve('[]\n' + JSON.stringify([{ name, id: 42 }])); + } + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/42') + ) { + deleted = true; + } + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + assert.ok( + deleted, + 'legacy registration must be removed before replacement', + ); + } + return f.command(program, args, options); + }; + await createManager(f.config, { command }).cycle(1); + assert.ok(deleted); +}); + +test('startup reclaims a live container before replenishing after a crash', async (t) => { + const f = await fixture(t); + const previousPath = process.env.PATH; + await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', { + mode: 0o755, + }); + process.env.PATH = `${f.config.stateDir}:${previousPath}`; + t.after(() => { + process.env.PATH = previousPath; + }); + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify({ + name: 'ecs-qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + runnerId: 42, + uncertain: false, + }), + ); + let oldContainer = true; + let manager; + const command = async (program, args, options) => { + if (program === 'docker' && args[0] === 'ps') { + if (args.includes('--format')) { + // Docker quiet mode overrides --format, as on the trial host. + return oldContainer + ? args.includes('-aq') + ? 'old-container' + : '1' + : ''; + } + if (oldContainer) return 'old-container'; + } + if ( + program === 'docker' && + args[0] === 'rm' && + args.includes('old-container') + ) { + oldContainer = false; + return ''; + } + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + assert.equal( + oldContainer, + false, + 'remove old container before registration', + ); + } + if (program === 'docker' && args[0] === 'start') manager.stop(); + return f.command(program, args, options); + }; + manager = createManager(f.config, { command }); + await manager.start(); + assert.equal(oldContainer, false); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); +}); + +test('accepts immutable local image IDs and rejects mutable tags and invalid configuration', async (t) => { + const f = await fixture(t); + assert.doesNotThrow(() => + createManager({ ...f.config, image: `sha256:${'b'.repeat(64)}` }), + ); + assert.throws( + () => createManager({ ...f.config, image: 'image:latest' }), + /Invalid manager configuration/, + ); + assert.throws( + () => + createManager({ ...f.config, dockerSocket: 'tcp://example.com:2375' }), + /Invalid manager configuration/, + ); + assert.throws( + () => createManager({ ...f.config, cpus: Infinity }), + /Invalid manager configuration/, + ); +}); + +test('does not expose malformed GitHub responses in errors', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if ( + program === 'gh' && + args.some((arg) => arg.includes('generate-jitconfig')) + ) { + return f + .command(program, args, options) + .then(() => 'malformed-secret-token'); + } + return f.command(program, args, options); + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + (error) => { + assert.match(error.message, /Invalid JSON response/); + assert.doesNotMatch(error.message, /malformed-secret-token/); + return true; + }, + ); +}); + +test('HTTP 429 rejection does not leave uncertain state and the next cycle can retry', async (t) => { + const f = await fixture(t); + const fakeGh = join(f.config.stateDir, 'gh'); + await writeFile( + fakeGh, + '#!/bin/sh\necho "gh: API rate limit exceeded (HTTP 429)" >&2\nexit 1\n', + { mode: 0o755 }, + ); + let rejected = false; + const command = (program, args, options) => { + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') && + !rejected + ) { + rejected = true; + return runCommand(fakeGh, [], { env: {} }); + } + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), (error) => error.status === 429); + await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), { + code: 'ENOENT', + }); + await manager.cycle(1); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); +}); + +test('network cleanup failure prevents new runner and is reconciled on the next cycle', async (t) => { + const f = await fixture(t); + const manager = createManager(f.config, { command: f.command }); + f.setFailNetworkRemove(true); + await assert.rejects(manager.cycle(1), /network removal failed/); + await assert.rejects(manager.cycle(1), /network removal failed/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); + f.setFailNetworkRemove(false); + await manager.cycle(1); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 2, + ); +}); + +test('a lost network create response is recovered using labels', async (t) => { + const f = await fixture(t); + const command = async (program, args, options) => { + const result = await f.command(program, args, options); + if (program === 'docker' && args[0] === 'network' && args[1] === 'create') + throw new Error('network create response lost'); + return result; + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + /network create response lost/, + ); + assert.equal( + f.calls.filter( + (c) => + c.program === 'docker' && c.args[0] === 'network' && c.args[1] === 'rm', + ).length, + 1, + ); + await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), { + code: 'ENOENT', + }); +}); + +test('network deletion is verified even when docker reports success', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'network' && args[1] === 'rm') + return Promise.resolve(''); + return f.command(program, args, options); + }; + const manager = createManager(f.config, { command }); + await assert.rejects(manager.cycle(1), /Network cleanup failed/); + await assert.rejects(manager.cycle(1), /Network cleanup failed/); + assert.equal( + f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create') + .length, + 1, + ); +}); + +test('passes only the docker environment allowlist and the JIT config by name', async (t) => { + const sentinels = { + AWS_SECRET_ACCESS_KEY: 'sentinel-secret', + SOME_DEPLOY_TOKEN: 'sentinel-token', + }; + const previous = {}; + for (const key of Object.keys(sentinels)) { + previous[key] = process.env[key]; + process.env[key] = sentinels[key]; + } + t.after(() => { + for (const key of Object.keys(sentinels)) { + if (previous[key] === undefined) delete process.env[key]; + else process.env[key] = previous[key]; + } + }); + const f = await fixture(t); + await createManager(f.config, { command: f.command }).cycle(1); + const create = f.calls.find( + (c) => c.program === 'docker' && c.args[0] === 'create', + ); + const allowed = [ + 'DOCKER_CONFIG', + 'DOCKER_HOST', + 'HOME', + 'PATH', + 'RUNNER_JIT_CONFIG', + 'XDG_RUNTIME_DIR', + ]; + assert.deepEqual( + Object.keys(create.options.env).filter((key) => !allowed.includes(key)), + [], + ); + assert.ok(!JSON.stringify(create.options.env).includes('sentinel')); + assert.ok(!JSON.stringify(create.args).includes('sentinel')); + assert.equal( + create.args[create.args.indexOf('--log-driver') + 1], + 'json-file', + ); + const at = create.args.indexOf('--env'); + assert.equal(create.args[at + 1], 'RUNNER_JIT_CONFIG'); +}); + +test('labels pool and slot on create and honours recovery filters', async (t) => { + const f = await fixture(t); + const foreign = f.addForeignNetwork({ + 'qwen.pool': 'other', + 'qwen.slot': '9', + }); + const command = async (program, args, options) => { + const result = await f.command(program, args, options); + if (program === 'docker' && args[0] === 'network' && args[1] === 'create') + throw new Error('network create response lost'); + return result; + }; + await assert.rejects( + createManager(f.config, { command }).cycle(1), + /network create response lost/, + ); + const created = f.calls.find( + (c) => + c.program === 'docker' && + c.args[0] === 'network' && + c.args[1] === 'create', + ); + assert.ok(created.args.includes('qwen.pool=trial')); + assert.ok(created.args.includes('qwen.slot=1')); + const removed = f.calls + .filter( + (c) => + c.program === 'docker' && c.args[0] === 'network' && c.args[1] === 'rm', + ) + .map((c) => c.args[c.args.length - 1]); + assert.deepEqual(removed, ['network-1']); + assert.ok(f.networkIds().includes(foreign)); +}); + +test('container cleanup failure blocks replenishment even when docker reports success', async (t) => { + const f = await fixture(t); + f.setKeepContainerAfterRemove(true); + const manager = createManager(f.config, { command: f.command }); + await assert.rejects(manager.cycle(1), /Container cleanup failed/); + const state = JSON.parse( + await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'), + ); + assert.ok(state.name); + assert.equal(state.runnerId, 1); +}); + +test('rejects corrupted slot state without deleting a runner', async (t) => { + const validName = 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa'; + const cases = [ + { + label: 'foreign prefix', + state: { + name: 'qwen-other-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa', + runnerId: 42, + uncertain: false, + }, + }, + { + label: 'non-UUID suffix', + state: { + name: 'qwen-trial-1-not-a-uuid', + runnerId: 42, + uncertain: false, + }, + }, + { + label: 'non-boolean uncertain', + state: { name: validName, runnerId: 42, uncertain: 'no' }, + }, + { + label: 'zero runner id', + state: { name: validName, runnerId: 0, uncertain: false }, + }, + ]; + for (const item of cases) { + const f = await fixture(t); + await writeFile( + join(f.config.stateDir, 'slot-1.json'), + JSON.stringify(item.state), + ); + await assert.rejects( + createManager(f.config, { command: f.command }).cycle(1), + /Invalid slot state/, + item.label, + ); + assert.ok( + !f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')), + item.label, + ); + } +}); + +test('backs off on a failed container even after the startup window', async (t) => { + const f = await fixture(t); + f.config.ttlSeconds = 120; + let now = Date.now(); + t.mock.method(Date, 'now', () => now); + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'wait') { + now += 31000; + return Promise.resolve('126\n'); + } + return f.command(program, args, options); + }; + const result = await createManager(f.config, { command }).cycle(1); + assert.equal(result.earlyExit, true); +}); + +test('backs off before the next JIT request after a failed container', async (t) => { + const f = await fixture(t); + const previousPath = process.env.PATH; + await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', { + mode: 0o755, + }); + process.env.PATH = `${f.config.stateDir}:${previousPath}`; + t.after(() => { + process.env.PATH = previousPath; + }); + const sleeps = []; + let manager; + let jitRequests = 0; + const command = async (program, args, options) => { + if ( + program === 'gh' && + args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') + ) { + jitRequests += 1; + if (jitRequests > 1) manager.stop(); + } + if (program === 'docker' && args[0] === 'wait') return '127\n'; + return f.command(program, args, options); + }; + manager = createManager(f.config, { + command, + sleep: async (ms) => { + sleeps.push(ms); + }, + }); + await manager.start(); + // The 1 s poll that races `docker wait` also goes through the injected sleep; + // the replenishment backoff is the 60 s entry. + assert.equal(sleeps.filter((ms) => ms === 60000).length, 1); + assert.equal(sleeps.at(-1), 60000); + assert.equal(jitRequests, 2); +}); + +test('does not penalize a fast successful container', async (t) => { + const f = await fixture(t); + const result = await createManager(f.config, { command: f.command }).cycle(1); + assert.equal(result.earlyExit, false); +}); + +test('backs off when docker wait cannot determine container status', async (t) => { + const f = await fixture(t); + const command = (program, args, options) => { + if (program === 'docker' && args[0] === 'wait') + return Promise.reject(new Error('Docker disconnected')); + return f.command(program, args, options); + }; + const result = await createManager(f.config, { command }).cycle(1); + assert.equal(result.earlyExit, true); +}); + +test('starts through a symlink and reports invalid configuration', async (t) => { + const f = await fixture(t); + const link = join(f.config.stateDir, 'manager.mjs'); + await symlink(new URL('./manager.mjs', import.meta.url), link); + const result = spawnSync( + process.execPath, + [link, join(f.config.stateDir, 'missing.json')], + { + env: { ...process.env, GH_TOKEN: 'test-only' }, + encoding: 'utf8', + }, + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /^manager: /); +}); diff --git a/scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service b/scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service new file mode 100644 index 00000000000..a989541fc84 --- /dev/null +++ b/scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service @@ -0,0 +1,18 @@ +[Unit] +Description=Qwen Code ephemeral GitHub Actions runners +After=docker.service network-online.target +Requires=docker.service +Wants=network-online.target + +[Service] +Type=simple +EnvironmentFile=/etc/qwen-ephemeral-runner/credentials.env +WorkingDirectory=/opt/qwen-ephemeral-runner +ExecStart=/usr/bin/env node /opt/qwen-ephemeral-runner/manager.mjs /etc/qwen-ephemeral-runner/config.json +Restart=always +RestartSec=10 +KillMode=mixed +TimeoutStopSec=300 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/tests/ci-platform-lanes.test.js b/scripts/tests/ci-platform-lanes.test.js index a41581934bc..d941df09207 100644 --- a/scripts/tests/ci-platform-lanes.test.js +++ b/scripts/tests/ci-platform-lanes.test.js @@ -116,8 +116,7 @@ describe('complete macOS test groups', () => { // The extended ceilings below, except the test lane's, are scoped to the // self-hosted pool: `runs-on` falls back to ubuntu-latest when // MAINTAINER_ECS_RUNNER_DISABLED=true (and, for the association-gated lanes, -// for fork PRs from untrusted authors — lint_and_static reads lint_runner, -// which routes every PR to the pool), so those hosted paths keep the +// for untrusted fork PRs with the disposable pool disabled), so hosted paths keep the // pre-contention ceiling. The ARM64 fleet renders // `["self-hosted", "linux", "arm64"]` — no `ecs-qwen` label — so the scope // cannot key on the x64 fleet's name. Evaluate the real timeout expression for @@ -391,8 +390,11 @@ it('reports the required Linux check only when the build and every shard pass', const gate = ci.jobs.test_ubuntu; expect(gate.name).toBe('Test (ubuntu-latest, Node 22.x)'); expect(gate.needs).toEqual(['classify_pr', 'test_build', 'test']); - // On the shards' runners, not a hosted one that can queue it for minutes. - expect(gate['runs-on']).toBe(ci.jobs.test['runs-on']); + // The gate remains on the original association-gated runner, while test + // shards can opt into the disposable pool for untrusted forks. + expect(gate['runs-on']).toBe( + '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || \'["ubuntu-latest"]\') }}', + ); expect(condOf('test_ubuntu')).toBe(condOf('test')); expect(gate.steps[0].run).toBe( 'test "$BUILD_RESULT" = success && test "$SHARDS_RESULT" = success', @@ -862,15 +864,11 @@ describe('post-merge push lane', () => { ).toContain('classify_pr'); }); - it('keeps test on the association-gated output', () => { - // Nothing else reads test's runs-on: switching it to lint_runner would - // hand every untrusted fork the pool while the timeout pins above stayed - // green — timeoutMinutesOn derives each lane's output from its own - // runs-on, so it cannot catch a lane pointed at the wrong output. Pin the - // expression, not an equality with lint_and_static — the two are meant - // to differ. + it('keeps test on the opt-in disposable output', () => { + // The classifier defaults this output to ubuntu_runner, but selects the + // disposable pool for untrusted forks only when explicitly enabled. expect(String(ci.jobs.test['runs-on'])).toBe( - '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || \'["ubuntu-latest"]\') }}', + '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || \'["ubuntu-latest"]\') }}', ); }); diff --git a/scripts/tests/review-worktree-cleanup-workflow.test.js b/scripts/tests/review-worktree-cleanup-workflow.test.js index 280d30999dc..98e75fdbab2 100644 --- a/scripts/tests/review-worktree-cleanup-workflow.test.js +++ b/scripts/tests/review-worktree-cleanup-workflow.test.js @@ -55,16 +55,18 @@ const ciYaml = parse(readFileSync('.github/workflows/ci.yml', 'utf8')); // possibly dirty workspace. Match the pool itself, not just the output // references that usually name it: jobs can also hard-code the shared label // array, and a checkout on either form inherits the same leftovers. -// lint_and_static reads lint_runner (every PR reaches the pool), the other -// lanes ubuntu_runner — both name the same shared workspace. +// lint_and_static reads lint_runner (every PR reaches the pool), the test +// lanes read pr_test_runner, the other lanes ubuntu_runner — all name the same +// shared workspace. pr_test_runner is listed explicitly because jobs that read +// it are also kept in scope by their own sweep step, which would otherwise hide +// a future job that adds the pool and forgets the sweep. // Include retained recovery steps on hosted jobs too, so rerouting cannot // silently restore an untested sweep. Select by properties, not job name. +const sharedPoolSelector = /ubuntu_runner|lint_runner|pr_test_runner|ecs-qwen/; const ciCleanSteps = Object.entries(ciYaml.jobs) .filter( ([, job]) => - (/ubuntu_runner|lint_runner|ecs-qwen/.test( - JSON.stringify(job['runs-on'] ?? ''), - ) || + (sharedPoolSelector.test(JSON.stringify(job['runs-on'] ?? '')) || (job.steps ?? []).some( (s) => s.name === 'Clean stale .qwen before checkout', )) && @@ -381,10 +383,24 @@ describe('review worktree cleanup steps', () => { } }); + it('matches every pr_test_runner job with the shared-pool selector itself', () => { + const poolJobs = Object.entries(ciYaml.jobs).filter(([, job]) => + JSON.stringify(job['runs-on'] ?? '').includes('pr_test_runner'), + ); + expect(poolJobs.length).toBeGreaterThan(0); + for (const [id, job] of poolJobs) { + expect( + sharedPoolSelector.test(JSON.stringify(job['runs-on'] ?? '')), + `job "${id}" runs on pr_test_runner and must match the pool selector`, + ).toBe(true); + } + }); + it('keeps every shared-pool ci.yml checkout sweep pinned to paths.ts', () => { expect(ciCleanSteps.map(({ id }) => id)).toEqual( expect.arrayContaining([ 'test', + 'test_build', 'web_shell_e2e_smoke', 'integration_cli', ]),