feat(ci): add disposable external PR runners (#16513)

* feat(ci): add disposable external PR runners

* test(ci): align platform lanes with disposable runner routing

---------

Co-authored-by: yiliang114 <[email protected]>

.github/scripts/ci-runner-routing.test.mjs

@@ -8,7 +8,7 @@
 // scarce pool (2026-10-31 decision, same accepted risk class as lint).
 // The association routing for DOWNSTREAM lanes lives in the `pick_runner`
 // shell step, which publishes `ubuntu_runner` (association-gated) and
-// `lint_runner` (every pull request reaches the fleet). The matrix below
+// `lint_runner` (untrusted forks default to hosted). The matrix below
 // pins the gate's unconditional routing AND that ubuntu_runner keeps the
 // association gate for the test/integration lanes.
 //
@@ -123,6 +123,8 @@ function runPickRunnerOutputs({
   eventName,
   dispatch,
   fleet,
+  ephemeralEnabled,
+  ephemeralArch,
 }) {
   const tmp = mkdtempSync(join(tmpdir(), 'pick-runner-'));
   const outputFile = join(tmp, 'github_output');
@@ -133,6 +135,8 @@ function runPickRunnerOutputs({
       ECS_DISABLED: ecsDisabled ? 'true' : '',
       EVENT_NAME: eventName,
       LINUX_RUNNER_FLEET: fleet ?? '',
+      EPHEMERAL_ENABLED: ephemeralEnabled ?? '',
+      EPHEMERAL_ARCH: ephemeralArch ?? '',
       DISPATCH_LINUX_RUNNER: dispatch ?? '',
       GITHUB_OUTPUT: outputFile,
     },
@@ -161,7 +165,15 @@ function runPickRunnerOutputs({
     outputs.split('\n').includes(`lint_runner=${lintRunner}`),
     `lint_runner not published: ${outputs}`,
   );
-  return { ubuntuRunner, lintRunner };
+  const prTest = outputs
+    .split('\n')
+    .find((line) => line.startsWith('pr_test_runner='));
+  assert.ok(prTest, 'pr_test_runner not published');
+  return {
+    ubuntuRunner,
+    lintRunner,
+    prTestRunner: prTest.slice('pr_test_runner='.length),
+  };
 }
 
 function runPickRunner(inputs) {
@@ -302,11 +314,7 @@ describe('ci.yml lint_runner routing', () => {
   const ARM = '["self-hosted", "linux", "arm64"]';
   const lintRunner = (inputs) => runPickRunnerOutputs(inputs).lintRunner;
 
-  it('routes every pull request to the fleet, untrusted forks included', () => {
-    // The maintainer decision this pins: Lint & Static executes PR code on
-    // the shared pool for ALL authors. If this matrix ever flips back to
-    // hosted for the negative associations without a deliberate decision,
-    // the lane silently regrows its hosted quota spend.
+  it('keeps untrusted fork lint hosted when the disposable switch is unset', () => {
     for (const sameRepo of [true, false]) {
       for (const assoc of ASSOCIATIONS) {
         assert.equal(
@@ -316,8 +324,8 @@ describe('ci.yml lint_runner routing', () => {
             assoc,
             eventName: 'pull_request',
           }),
-          ECS,
-          `lint sameRepo=${sameRepo} assoc='${assoc}' must route to ECS`,
+          sameRepo || TRUSTED.includes(assoc) ? ECS : HOSTED,
+          `lint sameRepo=${sameRepo} assoc='${assoc}'`,
         );
       }
     }
@@ -354,7 +362,7 @@ describe('ci.yml lint_runner routing', () => {
       lintRunner({
         ecsDisabled: false,
         sameRepo: false,
-        assoc: 'NONE',
+        assoc: 'MEMBER',
         eventName: 'pull_request',
         fleet: 'arm64',
       }),
@@ -442,6 +450,99 @@ describe('ci.yml lint_runner routing', () => {
   });
 });
 
+describe('ci.yml disposable runner routing', () => {
+  it('keeps all selected fork jobs hosted unless the switch is exactly true', () => {
+    for (const ephemeralEnabled of [undefined, '', 'false', 'invalid']) {
+      const out = runPickRunnerOutputs({
+        eventName: 'pull_request',
+        sameRepo: false,
+        assoc: 'NONE',
+        ephemeralEnabled,
+      });
+      assert.equal(out.prTestRunner, HOSTED);
+      assert.equal(out.lintRunner, HOSTED);
+    }
+  });
+  it('selects ARM64 only when explicitly configured', () => {
+    const inputs = {
+      eventName: 'pull_request',
+      sameRepo: false,
+      assoc: 'NONE',
+      ephemeralEnabled: 'true',
+    };
+    const arm = '["self-hosted", "linux", "arm64", "ecs-qwen-ephemeral"]';
+    assert.equal(
+      runPickRunnerOutputs({ ...inputs, ephemeralArch: 'arm64' }).prTestRunner,
+      arm,
+    );
+    assert.equal(
+      runPickRunnerOutputs({ ...inputs, ephemeralArch: 'arm64' }).lintRunner,
+      arm,
+    );
+    assert.equal(
+      runPickRunnerOutputs({ ...inputs, ephemeralArch: 'invalid' })
+        .prTestRunner,
+      '["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]',
+    );
+  });
+  it('routes only untrusted fork jobs to the disposable pool when enabled', () => {
+    const ephemeral = '["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]';
+    for (const eventName of [
+      'pull_request',
+      'push',
+      'merge_group',
+      'workflow_dispatch',
+    ]) {
+      for (const sameRepo of [true, false]) {
+        for (const assoc of ASSOCIATIONS) {
+          for (const ecsDisabled of [true, false]) {
+            for (const fleet of ['', 'hosted', 'arm64']) {
+              const inputs = { eventName, sameRepo, assoc, ecsDisabled, fleet };
+              const baseline = runPickRunnerOutputs(inputs);
+              const enabled = runPickRunnerOutputs({
+                ...inputs,
+                ephemeralEnabled: 'true',
+              });
+              const selected =
+                eventName === 'pull_request' &&
+                !sameRepo &&
+                !TRUSTED.includes(assoc) &&
+                !ecsDisabled &&
+                fleet !== 'hosted';
+              assert.equal(
+                enabled.prTestRunner,
+                selected ? ephemeral : baseline.ubuntuRunner,
+                JSON.stringify(inputs),
+              );
+              assert.equal(
+                enabled.lintRunner,
+                selected ? ephemeral : baseline.lintRunner,
+                JSON.stringify(inputs),
+              );
+              assert.equal(enabled.ubuntuRunner, baseline.ubuntuRunner);
+            }
+          }
+        }
+      }
+    }
+  });
+
+  it('routes only the intended jobs through the new output', () => {
+    assert.equal(
+      ciDoc.jobs.classify_pr.outputs.pr_test_runner,
+      '${{ steps.pick_runner.outputs.pr_test_runner }}',
+    );
+    for (const name of ['test_build', 'test'])
+      assert.match(ciDoc.jobs[name]['runs-on'], /pr_test_runner/);
+    for (const name of ['lint_and_static', 'integration_no_ak'])
+      assert.match(ciDoc.jobs[name]['runs-on'], /lint_runner/);
+    for (const [name, job] of Object.entries(ciDoc.jobs)) {
+      if (!['classify_pr', 'test_build', 'test'].includes(name))
+        assert.doesNotMatch(String(job['runs-on']), /pr_test_runner/, name);
+    }
+  });
+});
+
 describe('ci.yml lint HOME isolation', () => {
   // This guard lives in the routing battery (HELPER_TESTS) because it gates
   // every full-profile PR; the scripts/tests vitest suites only run after the

.github/workflows/.size-baseline

@@ -27,6 +27,7 @@
 2038 docs-page-action.yml
 10005 dsw-swe-verified-release.yml
 40188 e2e.yml
+651 ephemeral-runner-smoke.yml
 13135 finalize-release.yml
 16647 live-host-release.yml
 5950 live-host.yml

.github/workflows/ci.yml

@@ -101,8 +101,8 @@ env:
   # re-runs every suite after `npm ci`, and a push to main always classifies
   # `full` (classify_pr computes a profile only for pull_request), so what
   # the fast lane skips is still re-checked post-merge.
-  HELPER_TESTS_DEP_FREE: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs'
-  HELPER_TESTS: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/classify-release-notes.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/qwen-triage-workflow.test.mjs .github/scripts/assign-issue-owner.test.mjs .github/scripts/auto-minimize-spam.test.mjs .github/scripts/ci-runner-routing.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/assign-pr-owner.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/ci-disk-pressure.test.mjs .github/scripts/e2e-build.test.mjs integration-tests/concurrent-runner/export-html-from-chatrecord-jsonl.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/ecs-runner/review-scratch-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs'
+  HELPER_TESTS_DEP_FREE: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs scripts/ci/ephemeral-runner/manager.test.mjs'
+  HELPER_TESTS: '.github/scripts/pr-safety-precheck.test.mjs .github/scripts/cap-release-notes.test.mjs .github/scripts/ci/classify-profile.test.mjs .github/scripts/ci/classify-platform-sensitivity.test.mjs .github/scripts/ci/classify-pr-profile.test.mjs .github/scripts/upsert-bot-comment.test.mjs .github/scripts/ci/main-failure-signature.test.mjs .github/scripts/classify-release-notes.test.mjs .github/scripts/dsw-swe-verified/make-manifest.test.mjs .github/scripts/dsw-swe-verified/make-terminal-bench-manifest.test.mjs .github/scripts/resolve-sandbox-image.test.mjs .github/scripts/update-ecs-runner-qwen-workflow.test.mjs .github/scripts/web-shell-visuals-publish.test.mjs .github/scripts/web-shell-visuals-compose.test.mjs .github/scripts/serve-ab-diff.test.mjs .github/scripts/serve-ab-drive.test.mjs .github/scripts/qwen-triage-workflow.test.mjs .github/scripts/assign-issue-owner.test.mjs .github/scripts/auto-minimize-spam.test.mjs .github/scripts/ci-runner-routing.test.mjs .github/scripts/autofix-status-heartbeat.test.mjs .github/scripts/assign-pr-owner.test.mjs .github/scripts/check-disk-floor.test.mjs .github/scripts/check-lint-gate-freshness.test.mjs .github/scripts/ci-disk-pressure.test.mjs .github/scripts/e2e-build.test.mjs integration-tests/concurrent-runner/export-html-from-chatrecord-jsonl.test.mjs .github/scripts/review-runner-schedule.test.mjs .github/scripts/ecs-runner/qwen-docker-cleanup.test.mjs .github/scripts/ecs-runner/review-scratch-cleanup.test.mjs .github/scripts/check-test-growth.test.mjs .github/scripts/check-ci-budget.test.mjs .github/scripts/ecs-runner/check-font-baseline.test.mjs scripts/ci/ephemeral-runner/manager.test.mjs'
   # The growth ratchet and its vitest mirror compare each workflow against
   # the PR's base commit to tell "this PR grew the file" apart from "the
   # baseline went stale on main" (#9904). Wired once here so every lane
@@ -143,11 +143,8 @@ jobs:
     outputs:
       skip_ci: '${{ steps.release_sync.outputs.skip_ci }}'
       ubuntu_runner: '${{ steps.pick_runner.outputs.ubuntu_runner }}'
-      # Lint & Static's own routing: every pull request reaches the fleet,
-      # including untrusted forks (maintainer decision 2026-10-31 — the lane
-      # executes PR code on the shared pool and that risk is accepted). The
-      # association gate in ubuntu_runner still governs every other Linux
-      # lane; do not "simplify" the two outputs into one.
+      pr_test_runner: '${{ steps.pick_runner.outputs.pr_test_runner }}'
+      # Untrusted forks use hosted runners unless the disposable pool is enabled.
       lint_runner: '${{ steps.pick_runner.outputs.lint_runner }}'
       ci_profile: '${{ steps.ci_profile.outputs.ci_profile }}'
     steps:
@@ -196,17 +193,16 @@ jobs:
       # Keep push routing aligned with the classifier above so a hosted queue
       # does not delay the downstream ECS jobs.
       #
-      # lint_runner is the one deliberate divergence: Lint & Static routes
-      # EVERY pull request to the fleet, untrusted forks included (maintainer
-      # decision 2026-10-31, risk accepted — see the classify_pr outputs
-      # comment). The kill-switch and LINUX_RUNNER_FLEET=hosted still force
-      # hosted, and non-PR events inherit ubuntu_runner unchanged.
+      # Untrusted forks opt into the disposable pool; otherwise their build,
+      # test, lint and no-AK integration jobs stay hosted.
       - name: 'Select Linux runner'
         id: 'pick_runner'
         env:
           SAME_REPO: '${{ github.event.pull_request.head.repo.full_name == github.repository }}'
           AUTHOR_ASSOCIATION: '${{ github.event.pull_request.author_association }}'
           ECS_DISABLED: '${{ vars.MAINTAINER_ECS_RUNNER_DISABLED }}'
+          EPHEMERAL_ENABLED: '${{ vars.EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED }}'
+          EPHEMERAL_ARCH: '${{ vars.EXTERNAL_PR_EPHEMERAL_ARCH }}'
           # Optional fleet selector: 'hosted' forces github.com-hosted runners,
           # 'arm64' routes to the experimental ARM64 ECS fleet, anything else
           # keeps the x64 ecs-qwen fleet. Empty/unset keeps current behavior.
@@ -242,12 +238,16 @@ jobs:
           fi
           echo "ubuntu_runner=${ubuntu_runner}" >> "${GITHUB_OUTPUT}"
           echo "Selected Linux runner: ${ubuntu_runner}"
-          # Lint & Static routes every pull request to the fleet (maintainer
-          # decision 2026-10-31); non-PR events inherit ubuntu_runner unchanged.
+          pr_test_runner="${ubuntu_runner}"
           lint_runner="${ubuntu_runner}"
-          if [[ "${EVENT_NAME}" == "pull_request" && "${ECS_DISABLED}" != "true" && "${LINUX_RUNNER_FLEET}" != "hosted" ]]; then
-            lint_runner="${fleet_runner}"
+          if [[ "${EPHEMERAL_ENABLED}" == "true" && "${ECS_DISABLED}" != "true" && "${LINUX_RUNNER_FLEET}" != "hosted" && "${EVENT_NAME}" == "pull_request" && "${SAME_REPO}" == "false" && "${trusted_author}" == "false" ]]; then
+            pr_test_runner='["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"]'
+            if [[ "${EPHEMERAL_ARCH}" == "arm64" ]]; then
+              pr_test_runner='["self-hosted", "linux", "arm64", "ecs-qwen-ephemeral"]'
+            fi
+            lint_runner="${pr_test_runner}"
           fi
+          echo "pr_test_runner=${pr_test_runner}" >> "${GITHUB_OUTPUT}"
           echo "lint_runner=${lint_runner}" >> "${GITHUB_OUTPUT}"
           echo "Selected lint runner: ${lint_runner}"
 
@@ -423,7 +423,7 @@ jobs:
     name: 'Build unit-test tree (ubuntu-latest, Node 22.x)'
     needs: 'classify_pr'
     if: "${{ !cancelled() && github.event_name != 'schedule' }}"
-    runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}'
+    runs-on: '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || ''["ubuntu-latest"]'') }}'
     timeout-minutes: 45
     permissions:
       contents: 'read'
@@ -890,7 +890,7 @@ jobs:
     # have. So a push run is lint, static analysis and unit tests —
     # lint/static in `lint_and_static`, unit tests here.
     if: "${{ !cancelled() && github.event_name != 'schedule' }}"
-    runs-on: '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || ''["ubuntu-latest"]'') }}'
+    runs-on: '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || ''["ubuntu-latest"]'') }}'
     # Every Vitest run takes one eighth of its suite's files (`--shard`): the
     # unsharded lane ran 45-59 minutes, almost all of it the CLI and core
     # suites one after the other. A shard keeps the old ceiling's headroom
@@ -1559,12 +1559,7 @@ jobs:
     # same release-sync convention: stay running on skip_ci PRs so a required
     # context still reports, with the per-step guards making it a no-op pass.
     if: "${{ !cancelled() && github.event_name != 'schedule' }}"
-    # Reads lint_runner, NOT ubuntu_runner: this lane routes every pull
-    # request to the fleet, untrusted forks included (maintainer decision
-    # 2026-10-31 — the lane executes PR code and the shared-pool risk is
-    # accepted). test and the integration lanes keep the association-gated
-    # ubuntu_runner. The kill-switch and hosted-fleet overrides still apply,
-    # inside pick_runner.
+    # Shares the external-PR hosted/disposable switch with test and integration_no_ak.
     runs-on: '${{ fromJSON(needs.classify_pr.outputs.lint_runner || ''["ubuntu-latest"]'') }}'
     # Pool-scoped like test's, not a bare constant: this lane was beheaded twice
     # running honestly on cold-cache hk3-pool runners — at 30 minutes flat
@@ -2982,14 +2977,8 @@ jobs:
     # A draft pull request skips it (and the browser smoke): it cannot merge,
     # and `ready_for_review` starts the full run that gates the merge.
     if: "${{ !cancelled() && needs.classify_pr.outputs.skip_ci != 'true' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') && !github.event.pull_request.draft }}"
-    # Reads lint_runner, NOT ubuntu_runner: like lint_and_static, this lane
-    # executes PR code on the pool for every pull request — accepted under the
-    # same 2026-10-31 decision, and it carries NO secrets by construction
-    # (every model key is emptied at the integration step; the job's only
-    # token is the read-only checkout grant). The lane publishes a run-unique
-    # HOME from its first step (lint's shape) and empties every model key at
-    # the gate, so the pool exposure is the class already accepted for lint.
-    # test and integration_cli keep the association-gated ubuntu_runner.
+    # Shares lint's hosted/disposable routing for untrusted forks. Model keys
+    # are empty; the job receives only the read-only checkout token.
     runs-on: '${{ fromJSON(needs.classify_pr.outputs.lint_runner || ''["ubuntu-latest"]'') }}'
     strategy:
       fail-fast: false

.github/workflows/ephemeral-runner-smoke.yml

@@ -0,0 +1,23 @@
+name: 'Ephemeral runner smoke'
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: 'read'
+
+jobs:
+  smoke:
+    runs-on: ['ecs-qwen-ephemeral-smoke']
+    timeout-minutes: 10
+    steps:
+      - uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3
+      - name: 'Check tools and workspace'
+        run: |
+          node --version
+          git --version
+          printf 'Runner: %s\n' "$RUNNER_NAME" > runner-smoke.txt
+      - uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1
+        with:
+          name: 'ephemeral-runner-smoke'
+          path: 'runner-smoke.txt'

docs/design/2026-11-07-ephemeral-container-runners.md

@@ -0,0 +1,163 @@
+# Ephemeral container runners for external PRs
+
+[English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md)
+
+Date: 2026-11-07. Status: implemented; a five-slot systemd trial is deployed on
+hz4. Production external-PR routing remains disabled.
+
+Ten slots registering and running ten keyless test jobs of a private repository
+at the same time on the Hangzhou shared test machine; containers, networks and
+registrations were all reclaimed. A subsequent one-slot trial uses a real PAT
+and two sequential GitHub Actions jobs to verify registration, container
+replacement, workspace separation and absence of the PAT in jobs. A forced
+manager exit verified systemd restart and cleanup before replacement. These
+trials found and fixed compatibility with gh 2.46 pagination and Docker's quiet
+mode overriding formatted slot output. Full PR CI and production isolation
+remain unverified, so the shared host is still only a controlled trial host.
+
+Follow-up model access is described in
+[the runner and model-proxy combination](2026-11-07-ephemeral-runner-model-proxy.md);
+that proposal is neither implemented nor enabled.
+
+## Goals and boundaries
+
+Put the whole GitHub Actions runner into a disposable Docker container. The
+container runs one job and is then deleted; the next job gets a new container,
+a new runner identity and a new working directory from the same clean image.
+Users do not want to maintain a per-command permission allowlist for CI.
+
+The disposable unit is a job, not a workflow, a step or a PR. Multiple jobs
+exchange files through GitHub artifacts and never share a writable working
+directory.
+
+This design addresses run-environment residue and cross-task file and process
+contamination. Plain Docker shares the host kernel and gives no virtual-machine
+isolation from malicious code; any secret passed into a job can still be read
+by that job. The first phase accepts only Linux CI work without long-lived
+secrets: no model gateway is built here, and release, autofix, real-model E2E
+and tasks that need a Docker daemon are not migrated.
+
+## Architecture
+
+```text
+External-CI dedicated ECS
+├── systemd-managed manager (long-running, trusted)
+│   ├── requests JIT runner configuration with the host-only PAT
+│   ├── starts, monitors and deletes containers and their isolated networks
+│   └── keeps management state, exports logs, recovers interrupted tasks
+├── slot 1: container A → one job → delete → container C
+└── slot 2: container B → one job → delete → container D
+
+Each container
+├── one JIT runner that connects to GitHub on its own
+├── Node.js 22, Git and the CI dependencies
+└── its own HOME, working directory, temporary directory and npm cache
+```
+
+The manager keeps a configured number of slots, at most one container per slot.
+The Hangzhou trial ran ten slots at once; the hz4 deployment now maintains five slots. It takes no webhook, schedules no PR
+itself, and introduces neither Kubernetes nor a database. GitHub assigns work by
+runner label. Each ECS runs its own manager against its local Docker socket,
+with a unique `poolId` and a per-host `slots` count. Hosts share the scheduling
+labels; no central manager is needed. New names use `ecs-qwen-` for compatibility
+with CI worker caps and timeouts; recovery still accepts old `qwen-` identities.
+
+systemd starts the manager at boot when enabled and restarts unexpected exits
+after ten seconds (`Restart=always`). An explicit service stop remains stopped.
+This supervises process exits; it does not detect a live but stuck manager.
+
+Images are produced by a trusted build and pinned by digest. The manager and the
+image may only be deployed from versions maintained under maintainer control,
+never updated from a job checkout or from PR artifacts.
+
+Authentication uses a dedicated PAT in the root-only systemd EnvironmentFile,
+`/etc/qwen-ephemeral-runner/credentials.env`, as `GH_TOKEN`. A fine-grained token
+should select only the target repository and Administration: write, the GitHub
+API's required permission. It is not restricted to runner operations alone.
+Fine-grained access can be unavailable for outside collaborators or under
+organization policy; a broader classic token requires an explicit decision.
+The PAT stays on the host, never in the config JSON or a job container. Host
+root and Docker administrators remain trusted. Rotate the credential file and
+restart the service before expiry; no GitHub App or organization runner
+permission is needed for repository-scoped runners.
+
+## Ownership and lifecycle
+
+| Object                                | Owner                   | Lifetime                                             |
+| ------------------------------------- | ----------------------- | ---------------------------------------------------- |
+| ECS, Docker daemon, manager           | Ops / maintainers       | Long term                                            |
+| Host-only runner-management PAT       | Manager                 | Never enters a job container                         |
+| Clean image                           | Trusted image build     | Versioned, not writable from a job                   |
+| JIT configuration, runner identity    | One container instance  | Used for a single job                                |
+| Container writable layer and job data | One container instance  | Deleted on destruction                               |
+| Docker bridge network                 | One container instance  | Deleted on destruction                               |
+| Diagnostic logs                       | Exported by the manager | Size and retention bounded, unreadable by later jobs |
+
+Normal path: request JIT → create the isolated network and container → runner
+comes online → run one job → runner exits → export logs → delete the container
+and network → clean up the leftover registration → create the next one.
+
+JIT provides single-job registration semantics under the normal protocol; the
+host manager is responsible for actually destroying the container. The design
+must not rely on a malicious job exiting on its own, finishing a final cleanup
+step, or keeping the runner intact. A watchdog outside the container enforces a
+maximum lifetime, and a manager restart reconciles old resources first.
+
+If deleting a container fails, that slot stops; if GitHub registration cleanup
+fails, the failure is recorded and retried instead of endlessly requesting new
+identities. Host failures cannot clean up immediately, so recovery must reclaim
+before accepting new work.
+
+## Minimal but necessary deployment boundary
+
+- Use an ECS dedicated to external CI; do not share a host with existing runners
+  that hold long-lived secrets.
+- Mount no host home, workspace, management state, credential directory or
+  Docker socket; use neither privileged mode, host PID nor host networking. No
+  per-command allowlist is needed inside the container.
+- Images declare no persistent VOLUME and the first phase creates no job volume;
+  all job writes go to the container writable layer.
+- Network policy outside the host and unmodifiable from the container blocks the
+  management interface, sensitive intranets, instance credentials and metadata;
+  configure only the GitHub, npm and image-service egress actually required.
+- Access granted to the old trusted runners must be verified on the GitHub side.
+  Neither labels nor PR-modifiable YAML are an authorization boundary; an
+  external PR must not route back into a sensitive pool by editing runs-on.
+- Do not open fork secrets because a self-hosted runner is enabled, and do not
+  introduce a privileged pull_request_target flow that executes PR code.
+
+## Initial routing decision
+
+Keep the existing `ubuntu_runner` output and all of its former semantics, and add
+a `pr_test_runner` output used by `test_build` and `test`. For an untrusted fork
+PR, `EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` selects the disposable pool;
+unset, `false`, or any other value selects `ubuntu-latest`. `lint_runner` follows
+the same switch for `lint_and_static` and `integration_no_ak`, replacing the old
+shared-ECS fallback for these fork jobs. Trusted and same-repository PR routing
+stays unchanged. The global kill-switch and hosted setting take precedence.
+
+The switch selects a destination when a job is evaluated, without querying
+runner availability. It cannot move an already queued job; cancel queued work
+and start a new workflow run to apply a rollback. Verify unset/false/invalid,
+true for both architectures, and the global overrides against the actual shell
+selector before rollout.
+
+`classify_pr`, `web_shell_e2e_smoke`, `integration_cli` and the other workflows
+keep their current routing. This avoids migrating browser system dependencies
+that the new image has not verified, and leaves the routing image relationship
+of the existing classifier job and the other workflows unchanged.
+
+The new pool label is `ecs-qwen-ephemeral`. Default routing uses x64; setting
+`EXTERNAL_PR_EPHEMERAL_ARCH=arm64` selects ARM64 instead. The toolchain of full
+PR CI on the new image still has to be verified.
+
+## References
+
+- [GitHub ephemeral runner semantics and cleanup responsibility](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#ephemeral-runners-for-autoscaling)
+- [GitHub repository-level JIT API](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository)
+- [Docker daemon and kernel security boundary](https://docs.docker.com/engine/security/)
+
+The facts this design relies on were checked against the official documentation
+during this discussion; the actual runner version, GitHub account capabilities,
+API permissions and ECS network configuration must be confirmed again at
+deployment time.

docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md

@@ -0,0 +1,133 @@
+# 外部 PR 的一次性容器 runner
+
+[English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md)
+
+日期：2026-11-07。状态：已实现并在 hz4 部署五槽位 systemd 试运行服务；
+外部 PR 的生产路由仍未启用。
+
+杭州共享测试机上已验证 10 个槽位同时注册，并执行同一个私有测试仓库的 10 个无密钥
+测试 job，容器、网络和注册均已回收。后续单槽位试运行使用真实 PAT，
+通过两个串行 GitHub Actions job 验证注册、容器替换、工作区隔离及 job 内
+没有 PAT；强制结束 manager 后也验证了 systemd 重启及补位前清理。
+试运行发现并修复了 gh 2.46 分页兼容性及 Docker quiet 模式覆盖格式化
+槽位输出的问题。完整 PR CI 和生产隔离仍未验证，共享主机只用于受控试验。
+
+后续模型访问见 [runner 与模型代理结合方案](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md)；
+该方案尚未实现或启用。
+
+## 目标和边界
+
+将整个 GitHub Actions runner 放进一次性 Docker 容器。容器执行一个 job，
+随后删除；下一次从同一个干净镜像创建新容器、新 runner 身份和新工作目录。
+用户不希望为 CI 中的每条命令维护权限白名单。
+
+一次性单位是 job，不是 workflow、step 或 PR。多个 job 通过 GitHub artifacts
+交换文件，不共享可写工作目录。
+
+本设计解决运行环境残留和跨任务文件、进程污染。普通 Docker 共享宿主机内核，
+不提供虚拟机级恶意代码隔离；传入 job 的任何密钥仍可能被该 job 读取。
+首期只接无长期密钥的 Linux CI，不建设模型网关，不迁移发布、自动修复、
+真实模型 E2E 或需要 Docker daemon 的任务。
+
+## 架构
+
+```text
+外部 CI 专用 ECS
+├── systemd 托管的管理程序（长期运行，可信）
+│   ├── 使用宿主机 PAT 申请 JIT runner 配置
+│   ├── 启动、监控、删除容器及其独立网络
+│   └── 保存管理状态、导出日志、恢复异常任务
+├── 槽位 1：容器 A → 一个 job → 删除 → 容器 C
+└── 槽位 2：容器 B → 一个 job → 删除 → 容器 D
+
+每个容器
+├── 一个 JIT runner，主动连接 GitHub
+├── Node.js 22、Git 和 CI 依赖
+└── 独立的 HOME、工作目录、临时目录、npm 缓存
+```
+
+管理程序按配置维护固定数量的槽位，每个槽位最多一个容器；杭州十槽位试验同时使用
+十个槽位，hz4 部署目前维持五个槽位。不接 webhook、不自行调度 PR、不引入 Kubernetes 或数据库。
+GitHub 根据 runner 标签分配任务。每台 ECS 各运行一个管理程序，只连接本机
+Docker socket，使用独立的 `poolId` 和按主机配置的 `slots`。各主机共用调度
+标签，无需中央管理器。新名称使用 `ecs-qwen-` 前缀以兼容 CI 并发上限和超时；
+恢复时仍接受旧的 `qwen-` 身份。
+
+启用 systemd 服务后开机启动，意外退出时十秒后重启（`Restart=always`）；
+显式停止服务后保持停止。它监督进程退出，不检测进程存活但卡住的情况。
+
+镜像由可信构建生成并固定 digest。管理程序和镜像只能从经过维护者控制的版本
+部署，不从任务 checkout 或 PR artifacts 更新。
+
+认证使用专用 PAT，以 `GH_TOKEN` 写入仅 root 可读的 systemd EnvironmentFile：
+`/etc/qwen-ephemeral-runner/credentials.env`。fine-grained token 应只选择目标
+仓库及 GitHub API 要求的 Administration: write；它不只限于 runner 操作。
+外部协作者身份或组织策略可能使 fine-grained token 不可用，改用权限更宽的
+classic token 必须明确决定。PAT 留在宿主机，不进入配置 JSON 或任务容器。
+宿主机 root 和 Docker 管理员仍属于可信边界。到期前替换凭证文件并重启服务；
+仓库级 runner 不需要 GitHub App 或组织级 runner 管理权限。
+
+## 所有权和生命周期
+
+| 对象                         | 所有者       | 生命周期                            |
+| ---------------------------- | ------------ | ----------------------------------- |
+| ECS、Docker daemon、管理程序 | 运维/维护者  | 长期                                |
+| 宿主机 runner 管理 PAT       | 管理程序     | 不进入任务容器                      |
+| 干净镜像                     | 可信镜像构建 | 版本化、不可由 job 回写             |
+| JIT 配置、runner 身份        | 单个容器实例 | 只用于一个 job                      |
+| 容器可写层和任务数据         | 单个容器实例 | 销毁时删除                          |
+| Docker bridge 网络           | 单个容器实例 | 销毁时删除                          |
+| 诊断日志                     | 管理程序导出 | 有大小和保留期限，不被后续 job 读取 |
+
+正常路径：申请 JIT → 创建独立网络和容器 → runner 上线 → 执行一个 job →
+runner 退出 → 导出日志 → 删除容器与网络 → 清理残留注册 → 创建下一份。
+
+JIT 提供正常协议下的单任务注册语义；宿主机管理程序负责实际销毁。
+不能依赖恶意任务自行退出、完成最后一个 cleanup step 或保持 runner 完整。
+容器外 watchdog 强制限制最长存活时间，管理程序重启时先对账旧资源。
+
+容器删除失败时停止对应槽位；GitHub 注册清理失败时记录并重试，不继续无限
+申请新身份。宿主机故障期间无法即时清理，恢复后必须先回收再接新任务。
+
+## 最少但必要的部署边界
+
+- 使用外部 CI 专用 ECS；不与持有长期密钥的现有 runner 共用主机。
+- 不挂载宿主机 home、工作区、管理状态、凭证目录、Docker socket；不使用
+  privileged、host PID 或 host network。容器内无需逐命令白名单。
+- 镜像不声明持久 VOLUME，首期不创建任务卷；全部任务写入容器可写层。
+- 宿主机外/容器无法修改的网络策略阻止访问管理接口、敏感内网、实例凭证和
+  元数据；配置实际需要的 GitHub、npm、镜像服务出口。
+- 旧可信 runner 的访问权必须在 GitHub 侧检查。标签和 PR 可修改的 YAML 都
+  不是授权边界；外部 PR 不得通过修改 runs-on 绕回敏感池。
+- 不因启用自托管 runner 而开放 fork secrets，也不引入执行 PR 代码的
+  pull_request_target 特权流程。
+
+## 首期路由决策
+
+保留现有 `ubuntu_runner` 输出及其所有旧语义，新增 `pr_test_runner` 输出，
+用于 `test_build` 和 `test`。对于非可信作者的外部 fork PR，变量
+`EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` 选择一次性池；未设置、`false`
+或其他值选择 `ubuntu-latest`。`lint_runner` 在 `lint_and_static` 和
+`integration_no_ak` 中遵循同一个开关，替代这些 fork 任务原有的共享 ECS
+回退行为。可信作者和同仓库 PR 路由不变。全局 kill-switch 和 hosted 设置优先。
+
+开关在 job 求值时选择目标，不查询 runner 是否空闲，也不能迁移已排队任务；
+回滚时取消已排队任务并启动新的 workflow run。上线前用实际 shell 选择器
+验证未设置/false/非法值、开启时的两种架构，以及全局覆盖开关。
+
+`classify_pr`、`web_shell_e2e_smoke`、`integration_cli`、其他 workflow
+保持当前路由。这样不把新镜像尚未验证的浏览器系统依赖一起迁入，也不改变
+现有分类 job 与其他 workflow 的路由镜像关系。
+
+新池标签为 `ecs-qwen-ephemeral`。默认路由使用 x64，设置变量
+`EXTERNAL_PR_EPHEMERAL_ARCH=arm64` 可改用 ARM64；完整 PR CI 在新镜像上的
+工具链仍须验证。
+
+## 依据
+
+- [GitHub ephemeral runner 语义与清理责任](https://docs.github.com/en/actions/reference/runners/self-hosted-runners#ephemeral-runners-for-autoscaling)
+- [GitHub 仓库级 JIT API](https://docs.github.com/en/rest/actions/self-hosted-runners#create-configuration-for-a-just-in-time-runner-for-a-repository)
+- [Docker 的 daemon 与内核安全边界](https://docs.docker.com/engine/security/)
+
+本设计采用的事实已在本次讨论中核对官方文档；部署时再次确认实际 runner
+版本、GitHub 账户能力、API 权限和 ECS 网络配置。

docs/design/2026-11-07-ephemeral-runner-model-proxy.md

@@ -0,0 +1,165 @@
+# Combining ephemeral runners with a model proxy
+
+[English](2026-11-07-ephemeral-runner-model-proxy.md) | [简体中文](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md)
+
+Date: 2026-11-07. Status: proposal and partial verification complete; the
+combined service is neither implemented, deployed nor enabled.
+
+## Goals and scope
+
+When CI for an external PR needs a real model, every ephemeral runner uses its
+own short-lived, restricted proxy credential. The real model AK goes only to the
+trusted model proxy and never into a runner container. After the runner finishes
+one job, the management service reclaims the container and revokes the
+credential. Existing keyless jobs need no change.
+
+The existing triage/verify execution model is unchanged, no external PR gains
+unrestricted model access by default, and Docker is not claimed to be
+virtual-machine isolation from malicious code. This proposal provides neither
+release capability, model-output trustworthiness, nor prompt-injection
+protection.
+
+## Current code and verification
+
+- `scripts/ci/ephemeral-runner/manager.mjs` creates a single-job JIT runner for
+  each fixed slot, gives every container its own bridge, and replenishes after
+  reclamation. Model-proxy integration does not exist yet.
+- `triage`, `tmux-testing` and `verify` in `.github/workflows/qwen-triage.yml` each embed
+  a model proxy: it listens on host loopback, validates requests with a random
+  bearer, substitutes the real AK, forwards only `POST .../chat/completions`,
+  and supports streaming responses and an idle timeout. These are per-workflow
+  temporary processes, not standalone gateways.
+- Using the proxy extracted from the existing `verify` workflow plus a fake
+  upstream, we verified that a wrong token returns 401, other paths return 403,
+  a correct token is forwarded with the upstream receiving only the fake real
+  AK, and streamed data is returned. Two correct requests both succeed: the
+  current implementation has no per-request revocation.
+- Five selected existing triage tests pass, covering authentication and the
+  streaming idle timeout for the `tmux-testing` and `verify` lanes; the
+  `triage` lane embeds the same proxy pattern and is in the extraction scope
+  below.
+- On Hangzhou hz4, with a temporary Docker bridge and a single restricted test
+  container, we verified that `127.0.0.1` inside the container cannot reach a
+  loopback service on the host. Test resources were cleaned up. This does **not
+  prove** that a container cannot reach the host bridge address or other host
+  services.
+- No real model AK was used; an actual proxy deployment, credential issuance
+  and revocation, quotas, model jobs for
+  external PRs, and production network isolation were not verified.
+
+## Trust boundary
+
+```text
+GitHub job queue
+    ↓ dispatched to a labelled ephemeral runner
+External-CI dedicated ECS                    Trusted proxy service environment
+  runner management service ──protected issue/revoke──→ model proxy admin API
+       │                                              │
+       └─ runner container ──short-lived token──────→ model endpoint ──real AK──→ upstream
+```
+
+The model proxy is preferably deployed on a separate trusted host or service;
+the real AK does not live on the runner ECS. The host management service holds
+the runner-management PAT and the admin credential for the proxy admin API;
+neither is mounted, injected into, or exposed to runner containers. The admin
+API and the model endpoint are authorized separately and restricted by TLS and
+network policy; containers cannot reach the issue/revoke endpoints, the
+management service, the Docker socket, sensitive intranets, or ECS metadata.
+Runners are allowed to reach only the model endpoint and the CI egress they
+need.
+
+Containers have sudo, so isolating the real AK from the container's root/node
+users cannot be relied on. Plain Docker shares the host kernel; untrusted code
+still needs a dedicated host, with virtual-machine isolation considered
+according to risk. Network policy is enforced by the host or external network
+devices that the container cannot modify, and is verified in practice.
+
+## Credential and slot lifecycle
+
+The first version issues credentials per **runner instance**, rather than
+pretending to issue per PR or per job: the management service does not yet know
+which job GitHub will assign when it creates a runner. Model capability needs
+its own dedicated runner label and runner group, used only by explicitly
+permitted model-test workflows; model tokens must not be injected into the
+default keyless runner pool. Runner group permissions on the GitHub side must be
+verified: labels or workflow YAML in a PR are not an authorization boundary.
+
+State machine for each instance:
+
+1. Generate a non-reusable instance ID. Request a random token through the
+   protected admin API, setting model, request size, concurrency, call rate,
+   total volume and an absolute expiry. Store on the proxy only what validation
+   needs; persistent state and logs never record the plaintext token.
+2. Deliver the proxy address and token to the corresponding new container; the
+   management service itself must not hand management credentials or the
+   upstream AK to a container. Supply them as `OPENAI_BASE_URL` and
+   `OPENAI_API_KEY` for compatible tests. The token is considered public to that
+   job; log redaction is not treated as a security boundary.
+3. An idle runner still consumes its credential's lifetime. In the first
+   version the credential lifetime covers the instance's maximum lifetime, and
+   an instance left idle too long is reclaimed together with its runner; no
+   fixed remaining time at job start can be promised.
+4. Normal exit, failure, cancellation, timeout and restart recovery all revoke
+   the token, delete the container and network, and clean up the GitHub
+   registration. A new instance gets a new token; old tokens are rejected.
+5. When issuance is uncertain, do not create a container that can reach the
+   model; when revocation fails, stop replenishing that slot and retry, with the
+   absolute expiry as the final backstop. After a management-service or proxy
+   restart, reconcile first: the restart must not lose the revocation state or
+   reset budgets for already-issued tokens.
+
+Any script in the container can use or leak the token and may incur cost before
+revocation; environment variables, container metadata, logs and artifacts are
+all untrusted for that job. Credential capability must therefore be limited,
+and the whole model pool or proxy endpoint must be switchable off immediately.
+
+## Proxy features and budget
+
+Extract a testable shared implementation from the existing forwarding logic and
+preserve the behaviour of the three triage lanes; the new standalone service adds
+credential issuance, revocation, persistent state, statistics and limits
+separately, without changing the permissions or triggers of the existing
+workflows. Allow only configured upstream targets, models and exact API paths;
+requests must not control the upstream URL or pass an upstream authorization
+header. Bound request body size, timeout and concurrency, keep streamed
+disconnect handling, and prevent a client from triggering arbitrary address
+access or unbounded resource use.
+
+Set per-instance request counts, rates, concurrency and input/output limits, plus
+a shared budget across all instances and a hard quota or alert on the upstream
+account. Concurrent debits must be atomic and over-limit requests rejected.
+Request-count and output-parameter caps are not an exact cost cap; a streamed
+response may still incur cost after disconnect. An exact cost cap depends on the
+model provider's metering and quota capability and must be verified once the
+upstream is chosen. Do not log plaintext credentials, complete prompts or
+responses unless a concrete later requirement and data-handling policy exist.
+
+## Implementation and acceptance
+
+1. Determine which model tests actually need external-PR execution, and which
+   upstream model and budget they use; by default open exactly one controlled
+   test workflow and do not change the existing `integration_no_ak` or other CI
+   routing.
+2. Extract and test the HTTP/streaming core of the existing proxy, preserving
+   both triage lanes. Implement the standalone proxy's admin API, separate
+   credentials, permissions, persistence, expiry, revocation and budget.
+3. Add optional model-pool lifecycle to the management service; fail closed when
+   issuance or cleanup is uncertain. Cover token injection, log redaction,
+   abnormal recovery and old-token invalidation in tests.
+4. In an isolated test environment with a fake upstream, one slot and a
+   controlled private repository, verify two consecutive jobs: the two tokens
+   differ; normal and streamed calls succeed; wrong, old, revoked and expired
+   tokens are rejected; over-limit requests, concurrency races, upstream
+   failures, and management-service/proxy restarts do not bypass the limits.
+5. From inside a job, verify that the real AK, the runner-management PAT and the proxy
+   admin credential are unreadable, and that metadata, the Docker socket, other
+   containers and host management services are unreachable; verify that the
+   proxy can reach only the permitted upstream. Both container-cleanup and
+   credential-revocation failures must stop slot replenishment.
+6. Prepare isolated dedicated ECS capacity and proxy services, and measure the
+   network and GitHub authorization, then run a one-slot trial with a low-budget
+   real model credential. Only after it passes does a maintainer decide whether
+   to enable the corresponding model-test routing.
+
+The existing runner design is described in
+[the ephemeral runner design](2026-11-07-ephemeral-container-runners.md).

docs/design/2026-11-07-ephemeral-runner-model-proxy.zh-CN.md

@@ -0,0 +1,116 @@
+# 一次性 runner 与模型代理的结合方案
+
+[English](2026-11-07-ephemeral-runner-model-proxy.md) | [简体中文](2026-11-07-ephemeral-runner-model-proxy.zh-CN.md)
+
+日期：2026-11-07。状态：方案与局部验证完成；尚未实现组合服务、部署或启用。
+
+## 目标与范围
+
+外部 PR 的 CI 需要调用真实模型时，每个一次性 runner 使用独立、短期、受限的
+代理凭证。真实模型 AK 只交给可信模型代理，不进入 runner 容器。runner 执行
+一个 job 后，管理服务回收容器并撤销该凭证。原有无 AK 的 job 不需要改变。
+
+不改变现有 triage/verify 的运行方式，不让所有外部 PR 默认获得无限制模型
+访问，不承诺 Docker 是对恶意代码的虚拟机级隔离。本方案不提供发布、模型输出
+可信性或 prompt 注入防护。
+
+## 当前代码与验证
+
+- `scripts/ci/ephemeral-runner/manager.mjs` 为固定槽位创建单 job JIT runner，
+  每个容器使用单独的 bridge，回收完毕后再补充。当前没有模型代理集成。
+- `.github/workflows/qwen-triage.yml` 的 `triage`、`tmux-testing` 与 `verify` 各有一份
+  内嵌的模型代理：监听本机 loopback，以随机 bearer 校验请求，替换为真实 AK，
+  只转发 `POST .../chat/completions`，支持流式响应与空闲超时。它们是各自
+  workflow 的临时进程，不是独立网关。
+- 使用从现有 `verify` workflow 提取的代理和假上游，验证了错误 token 返回
+  401、其他路径返回 403、正确 token 转发且上游只收到假真实 AK，以及流式
+  数据返回。两个正确请求都能成功：现有实现没有按请求数撤销 token 的行为。
+- 现有 triage 测试中选中的 5 项通过，包含认证，以及 `tmux-testing` 与 `verify`
+  两条 lane 的流式空闲超时；`triage` lane 使用同一套代理形态，也纳入下述提取范围。
+- 在杭州 hz4 通过一个临时 Docker bridge 和单个受限测试容器验证：容器内
+  `127.0.0.1` 不能访问宿主机的 loopback 服务。测试资源已清理。这**不证明**
+  容器不能访问宿主机 bridge 地址或其他宿主机服务。
+- 未使用真实模型 AK；未验证实际代理部署、凭证签发与
+  撤销、配额、外部 PR 的模型 job、或生产网络隔离。
+
+## 信任边界
+
+```text
+GitHub job 队列
+    ↓ 分派到带标签的一次性 runner
+外部 CI 专用 ECS                         可信代理服务环境
+  runner 管理服务 ──受保护的签发/撤销──→ 模型代理管理接口
+       │                                │
+       └─ runner 容器 ──短期 token────→ 模型调用接口 ──真实 AK──→ 上游
+```
+
+模型代理优先部署在独立可信主机或服务，真实 AK 不放在 runner ECS。宿主机
+管理服务保存 runner 管理 PAT 和调用代理管理接口的管理凭证；这些都不挂载、
+注入或暴露给 runner 容器。管理接口与模型调用接口分开授权，并通过 TLS 和
+网络策略限制来源；容器不能访问签发、撤销接口、管理服务、Docker socket、
+敏感内网和 ECS 元数据。runner 仅获准访问模型调用接口及必要 CI 出口。
+
+容器有 sudo，不能依赖容器内 root/node 用户隔离真实 AK。普通 Docker 共享
+宿主机内核；涉及不可信代码时仍需专用主机，按风险考虑虚拟机级隔离。网络
+策略由容器不能修改的宿主机或外部网络设备执行并实际验证。
+
+## 凭证和槽位生命周期
+
+第一版按 **runner 实例**签发，而不是假装按 PR 或 job 签发：管理服务创建
+runner 时还不知道 GitHub 将分配哪个 job。模型能力需使用独立的专用 runner
+标签与 runner group，只供经明确允许的模型测试 workflow 使用；不要把模型
+token 注入默认无 AK 的 runner 池。GitHub 侧 runner group 权限必须经过核验，
+标签或 PR 中的 workflow YAML 不是授权边界。
+
+每个实例的状态机：
+
+1. 生成不可复用的实例 ID。通过受保护的管理接口申请随机 token，设置模型、
+   请求大小、并发、调用速率、总量及绝对过期时间。只在代理端保存验证所需
+   信息；持久状态和日志不记录明文 token。
+2. 将 proxy 地址和 token 交付给对应新容器；管理服务自身不能将管理凭证或
+   上游 AK 交给容器。以 `OPENAI_BASE_URL`、`OPENAI_API_KEY` 供兼容的测试
+   使用，token 对这个 job 视为已公开，不依赖日志脱敏作为安全边界。
+3. runner 空闲时也占用凭证有效期。第一版凭证有效期覆盖实例最长生命周期，
+   空闲太久后连 runner 一起回收；不能承诺任务开始时仍有固定剩余时间。
+4. 正常退出、失败、取消、超时或重启恢复均撤销 token、删除容器和网络、
+   清理 GitHub 注册。新实例获得新 token；旧 token 被拒绝。
+5. 签发结果不确定时，不创建可访问模型的容器；撤销失败时停止对应槽位补充
+   并重试，凭证绝对过期作为最终兜底。管理服务/代理重启后应先对账，不能
+   因重启丢失已签发 token 的撤销状态或重置预算。
+
+容器中任何脚本都能使用或泄露 token，且撤销前可能产生费用；环境变量、
+容器元数据、日志和 artifact 都不能被视作对该 job 保密。必须限制凭证能力，
+并允许立即整体关闭模型池或代理调用接口。
+
+## 代理功能与预算
+
+从现有转发逻辑提取可测试的共享实现，保持已有三条 triage lane 行为；
+新的独立服务另加凭证签发、吊销、持久状态、统计和限额，不直接修改既有
+workflow 的权限或触发方式。只允许配置好的上游目标、模型和确切 API 路径，
+不得让请求控制上游 URL 或传递上游授权头；限制请求体大小、超时和并发，
+保留流式断连处理，避免客户端触发任意地址访问或无限资源消耗。
+
+按实例设置请求数、速率、并发和输入/输出上限，并设置所有实例共享的总预算
+及上游账户侧的硬额度或告警。并发扣额需要原子处理，拒绝超额请求。请求数
+和输出参数上限并不等于精确费用上限；流式响应可能在断连后仍产生费用。
+精确成本上限依赖模型供应方的计量/限额能力，应在选定上游后验证。不要记录
+明文凭证、完整 prompt 或回复，除非后续有明确需求和数据处理策略。
+
+## 实施和验收
+
+1. 确定哪些模型测试确实需要外部 PR 执行、上游模型及预算；默认只开放一条
+   受控测试 workflow，先不要改变现有 `integration_no_ak` 或其他 CI 路由。
+2. 提取并测试现有 proxy 的 HTTP/流式核心，保留三个 triage lane 行为。实现
+   独立代理的管理接口、独立凭证、权限、持久化、过期、撤销和预算。
+3. 为管理服务增加可选的模型池生命周期；签发或清理不确定时 fail closed。
+   明确 token 注入、日志脱敏、异常恢复及旧 token 失效的测试。
+4. 在独立测试环境使用假上游、1 个槽位及受控私有仓库，验证两次连续 job：
+   两个 token 不同；正常和流式调用通过；错误、旧、撤销和过期 token 被拒；
+   超额、并发竞争、上游故障、管理服务/代理重启不绕过限制。
+5. 从 job 内验证不能读到真实 AK、runner 管理 PAT、代理管理凭证，也不能访问元数据、
+   Docker socket、其他容器和宿主机管理服务；验证代理只能访问允许的上游。
+   容器清理和凭证撤销失败均阻止槽位补充。
+6. 准备隔离的专用 ECS 和代理服务、实测网络与 GitHub 授权，再用低预算真实
+   模型凭证进行一槽位试运行。通过后由维护者决定是否启用相应模型测试路由。
+
+现有 runner 方案见 [一次性 runner 设计](2026-11-07-ephemeral-container-runners.zh-CN.md)。

scripts/ci/ephemeral-runner/Dockerfile

@@ -0,0 +1,39 @@
+ARG BASE_IMAGE=node:22-bookworm-slim@sha256:5ec23b7c622b00547753bc6ffd78085dd868a68aeb41c8e0a1418f033e13d96d
+FROM ${BASE_IMAGE}
+
+ARG TARGETARCH
+ARG RUNNER_VERSION=2.338.2
+ARG RUNNER_RELEASE_URL=https://github.com/actions/runner/releases/download
+ARG DEBIAN_MIRROR=deb.debian.org
+ARG PIP_INDEX_URL=https://pypi.org/simple
+
+RUN sed -i "s|deb.debian.org|${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources \
+    && apt-get update -o Acquire::http::Timeout=30 \
+    && apt-get install -y --no-install-recommends \
+      build-essential ca-certificates curl file gh git git-lfs jq libicu72 \
+      libkrb5-3 libssl3 libunwind8 procps python3 python3-pip python3-venv \
+      sudo tmux unzip zip zlib1g \
+    && rm -rf /var/lib/apt/lists/* \
+    && python3 -m venv /opt/ci-tools \
+    && /opt/ci-tools/bin/pip install --no-cache-dir 'yamllint==1.35.1' \
+    && echo 'node ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/runner
+
+ENV PATH="/opt/ci-tools/bin:${PATH}"
+
+WORKDIR /home/node/actions-runner
+RUN case "${TARGETARCH}" in \
+      amd64) runner_arch=x64; runner_sha=baebf204307bac6193480852732f0aba2ae97b6c5597c244f4d5558390147c40 ;; \
+      arm64) runner_arch=arm64; runner_sha=670adb2f268fe21dcbb58b3db0e0aa9d5b3bddfe2c651481336e561ff86de945 ;; \
+      *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
+    esac \
+    && curl -fsSL --retry 3 --connect-timeout 15 --max-time 180 -o /tmp/runner.tar.gz \
+      "${RUNNER_RELEASE_URL}/v${RUNNER_VERSION}/actions-runner-linux-${runner_arch}-${RUNNER_VERSION}.tar.gz" \
+    && echo "${runner_sha}  /tmp/runner.tar.gz" | sha256sum -c - \
+    && tar -xzf /tmp/runner.tar.gz \
+    && rm /tmp/runner.tar.gz \
+    && chown -R node:node /home/node/actions-runner
+
+COPY entrypoint.sh /usr/local/bin/ephemeral-runner
+RUN chmod 755 /usr/local/bin/ephemeral-runner
+USER node
+ENTRYPOINT ["/usr/local/bin/ephemeral-runner"]

scripts/ci/ephemeral-runner/README.md

@@ -0,0 +1,140 @@
+# Ephemeral GitHub Actions runners
+
+Each slot registers one just-in-time runner and runs it in a new container.
+After at most one job, the manager removes the container and runner registration
+before creating the next one. While the manager waits on a container it also
+enforces a per-slot deadline and removes a container whose runner does not exit;
+a manager crash is covered by reconciling leftover resources on the next start.
+
+Each job has a separate Docker bridge with inter-container communication
+disabled. Deleting the network is part of cleanup and a failure blocks slot
+replacement. This does not itself prevent access to the host or private
+networks: configure and verify the host's external egress controls separately.
+
+Use a dedicated host for untrusted pull requests. Docker shares the host kernel;
+jobs must not receive the host Docker socket, host directories, cloud instance
+credentials, internal network access, or long-lived secrets. A runner label is a
+scheduling selector, not an access-control boundary. Before enabling external
+pull requests, verify they cannot select other sensitive runner pools.
+
+## Build the image
+
+The image supports Linux x64 and ARM64. Its Node base is pinned to an OCI index
+digest and the runner downloads are checked against pinned SHA-256 digests.
+Use BuildKit to select both the base image and runner architecture:
+
+```sh
+docker buildx build --platform linux/arm64 --load -t qwen-ephemeral-runner:trial scripts/ci/ephemeral-runner
+```
+
+BuildKit supplies `TARGETARCH`. With Docker's legacy builder, build on the target
+architecture and pass its matching `TARGETARCH` explicitly (`amd64` or `arm64`).
+That argument selects only the runner archive; it does not change the base image
+architecture.
+
+The runner image supports ARM64, but full PR CI on this image has not yet been
+verified. The repository also selects ARM64 linters; verify the complete CI
+toolchain before routing production work to either architecture.
+
+Hosts with restricted upstream connectivity can set `DEBIAN_MIRROR`,
+`PIP_INDEX_URL` or `RUNNER_RELEASE_URL` at build time. The runner archive is
+still verified against its pinned digest when served from a mirror.
+
+For a real deployment, publish the image to a registry and configure its
+immutable `repository@sha256:...` digest. A local `sha256:...` image ID may be
+used for a trial. Runner and base versions need routine security updates.
+
+## Configure the manager
+
+Run one manager on each host, managing only that host's Docker socket. Give each
+host a unique `poolId`; matching runner labels let GitHub share work across the
+hosts without a central scheduler. `slots` is a per-host limit, not a fleet total.
+New runner names start with `ecs-qwen-` so the existing CI worker caps and test
+timeouts apply. Restart recovery also accepts the earlier `qwen-` identities.
+
+Install Docker, Node 22, `gh`, and `flock` on the host. The manager uses a
+dedicated PAT supplied as `GH_TOKEN`. The account must administer the target
+repository. Prefer a fine-grained PAT restricted to that repository with
+Administration: write and a short expiry. GitHub currently requires this broader
+permission for runner registration; it is not a runner-only credential. Some
+organization or outside-collaborator policies prevent fine-grained access. Do
+not silently replace it with a classic `repo` token, which can cover other
+repositories the account can access.
+
+Store `GH_TOKEN=<PAT>` in `/etc/qwen-ephemeral-runner/credentials.env`, owned by
+root with mode `600`, inside the root-only configuration directory. The supplied
+systemd unit loads this file; do not put it in shell history, config JSON, job
+environments or the repository. Replace it and restart the service to rotate or
+renew the token. The service fails startup if `GH_TOKEN` is missing. Docker child
+environments are allowlisted, and only the one-time JIT configuration enters a
+job container. Host root and Docker administrators are trusted and can access
+host credentials; ordinary container isolation does not change that boundary.
+
+Copy `config.example.json` to `/etc/qwen-ephemeral-runner/config.json` and
+replace the placeholders: repository, actual runner group ID, unique host/pool
+ID, immutable image reference, and matching architecture and labels. Use a
+unique test label and a controlled test repository for a trial. The supplied
+manual smoke workflow selects only `ecs-qwen-ephemeral-smoke`; a trial runner
+can register with just that label to avoid matching generic Linux jobs. The example runs
+one slot; increase only after measuring the host. Restrict configuration and
+credential directories to the manager account.
+
+The `ttlSeconds` limit starts when the container starts, including idle time.
+A job assigned near the deadline can be interrupted. The manager removes and
+replaces the container; GitHub remains authoritative for job status. The
+manager logs cleanup but does not distinguish expiry from normal completion.
+A nonzero or unknown container exit status delays replenishment by 60 seconds;
+a successful short job is replenished immediately. GitHub remains authoritative
+for job success. Diagnostic export is best effort: stderr is not archived, and
+output exceeding the capture limits is discarded.
+`minDiskBytes` applies to the Docker data filesystem. Configure and verify
+disk quotas or a dedicated data volume as appropriate for the host.
+
+## Install and operate
+
+Install `manager.mjs` in `/opt/qwen-ephemeral-runner/`, install the supplied
+unit in `/etc/systemd/system/`, and create the configured state/log directories
+with permissions restricted to the manager account. From the trusted checkout:
+
+```sh
+sudo install -d /opt/qwen-ephemeral-runner
+sudo install -d -m 700 /etc/qwen-ephemeral-runner /var/lib/qwen-ephemeral-runner /var/log/qwen-ephemeral-runner
+sudo install -m 644 scripts/ci/ephemeral-runner/manager.mjs /opt/qwen-ephemeral-runner/manager.mjs
+sudo install -m 644 scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service /etc/systemd/system/qwen-ephemeral-runner.service
+```
+
+Finish the configuration and PAT setup above before starting. A Docker-authorized user
+has host-level power; the unit is intended for a trusted dedicated host.
+
+```sh
+sudo systemctl daemon-reload
+sudo systemctl enable --now qwen-ephemeral-runner.service
+sudo systemctl status qwen-ephemeral-runner.service
+sudo journalctl -u qwen-ephemeral-runner.service -f
+```
+
+systemd restarts an unexpectedly exited manager after ten seconds, including a
+clean exit. An explicit `systemctl stop` leaves it stopped. This is process
+supervision, not hang detection; inspect the journal and runner availability if
+slots are not replenishing.
+
+Stopping the service stops replenishment, waits up to the configured grace
+period, and cleans up its containers. The next start reconciles abandoned
+resources first. A cleanup or ambiguous registration error blocks that slot;
+inspect its state and GitHub runner list before manual reconciliation. Never
+remove unrelated runners or run a global Docker prune.
+
+```sh
+sudo systemctl stop qwen-ephemeral-runner.service
+```
+
+After validating a single runner and its recovery paths, set the repository
+variable `EXTERNAL_PR_EPHEMERAL_RUNNER_ENABLED=true` to route untrusted fork PR
+build, test, lint and no-AK integration jobs to this pool. Unset, `false`, or any
+other value keeps those jobs on `ubuntu-latest`. Trusted and same-repository PRs
+retain their existing routing. The global kill-switch and hosted-fleet override
+still win. Leave the switch off until the host and GitHub access policies have
+been verified. Changing it affects newly evaluated jobs; cancel queued work and
+start a new workflow run when rolling back. It does not detect idle runners or
+migrate a queued job automatically.
+Set `EXTERNAL_PR_EPHEMERAL_ARCH=arm64` for an ARM64 pool; the default is x64.

scripts/ci/ephemeral-runner/config.example.json

@@ -0,0 +1,16 @@
+{
+  "repo": "QwenLM/qwen-code",
+  "runnerGroupId": 1,
+  "poolId": "external-ci-host-1",
+  "image": "registry.example.invalid/qwen-runner@sha256:0000000000000000000000000000000000000000000000000000000000000000",
+  "labels": ["self-hosted", "linux", "x64", "ecs-qwen-ephemeral"],
+  "slots": 1,
+  "cpus": 4,
+  "memory": "8g",
+  "pidsLimit": 1024,
+  "ttlSeconds": 10800,
+  "shutdownGraceSeconds": 60,
+  "minDiskBytes": 10737418240,
+  "stateDir": "/var/lib/qwen-ephemeral-runner",
+  "logDir": "/var/log/qwen-ephemeral-runner"
+}

scripts/ci/ephemeral-runner/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RUNNER_JIT_CONFIG:?RUNNER_JIT_CONFIG is required}"
+jit_config="$RUNNER_JIT_CONFIG"
+unset RUNNER_JIT_CONFIG
+cd /home/node/actions-runner
+exec ./run.sh --jitconfig "$jit_config"

scripts/ci/ephemeral-runner/manager.mjs

@@ -0,0 +1,620 @@
+import { randomUUID } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { realpathSync } from 'node:fs';
+import {
+  lstat,
+  mkdir,
+  readFile,
+  readdir,
+  rename,
+  rm,
+  statfs,
+  writeFile,
+} from 'node:fs/promises';
+import { isAbsolute, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const sleep = (ms) =>
+  new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
+
+export function command(
+  program,
+  args,
+  { env = process.env, input, timeout = 30000, binary = false } = {},
+) {
+  return new Promise((resolveCommand, reject) => {
+    const child = spawn(program, args, {
+      env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const output = [];
+    let outputSize = 0;
+    let stderr = '';
+    const timer = setTimeout(() => child.kill('SIGKILL'), timeout);
+    child.stdout.on('data', (chunk) => {
+      outputSize += chunk.length;
+      if (outputSize > 4 * 1024 * 1024) child.kill('SIGKILL');
+      else output.push(chunk);
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+      if (stderr.length > 1024 * 1024) child.kill('SIGKILL');
+    });
+    child.on('error', (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code === 0 && outputSize <= 4 * 1024 * 1024)
+        resolveCommand(
+          binary ? Buffer.concat(output) : Buffer.concat(output).toString(),
+        );
+      else {
+        const status = Number(stderr.match(/HTTP (\d{3})/)?.[1] || 0);
+        const error = new Error(
+          `${program} failed (exit ${code}${status ? `, HTTP ${status}` : ''})`,
+        );
+        error.status = status;
+        error.exitCode = code;
+        reject(error);
+      }
+    });
+    child.stdin.on('error', () => {});
+    child.stdin.end(input);
+  });
+}
+
+function validate(c) {
+  if (
+    !/^[\w.-]+\/[\w.-]+$/.test(c.repo || '') ||
+    !/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,39}$/.test(c.poolId || '') ||
+    !/^(?:[a-zA-Z0-9][a-zA-Z0-9._:/-]*@)?sha256:[a-f0-9]{64}$/.test(
+      c.image || '',
+    ) ||
+    !Number.isInteger(c.slots) ||
+    c.slots < 1 ||
+    c.slots > 10 ||
+    !Number.isInteger(c.runnerGroupId) ||
+    c.runnerGroupId < 1 ||
+    !Array.isArray(c.labels) ||
+    !c.labels.length ||
+    !c.labels.every(
+      (label) =>
+        typeof label === 'string' && /^[a-zA-Z0-9_-]{1,63}$/.test(label),
+    ) ||
+    typeof c.stateDir !== 'string' ||
+    !isAbsolute(c.stateDir) ||
+    typeof c.logDir !== 'string' ||
+    !isAbsolute(c.logDir) ||
+    (c.dockerSocket !== undefined &&
+      (typeof c.dockerSocket !== 'string' ||
+        !isAbsolute(c.dockerSocket) ||
+        c.dockerSocket.includes('\n'))) ||
+    !Number.isFinite(c.cpus) ||
+    !(c.cpus > 0) ||
+    !/^[1-9]\d*[mMgG]$/.test(c.memory || '') ||
+    !Number.isInteger(c.pidsLimit) ||
+    c.pidsLimit < 1 ||
+    !Number.isFinite(c.ttlSeconds) ||
+    !(c.ttlSeconds > 0) ||
+    !Number.isFinite(c.shutdownGraceSeconds) ||
+    !(c.shutdownGraceSeconds >= 0) ||
+    !Number.isSafeInteger(c.minDiskBytes) ||
+    c.minDiskBytes < 0
+  )
+    throw new Error('Invalid manager configuration');
+}
+
+export function createManager(config, dependencies = {}) {
+  validate(config);
+  const execute = dependencies.command || command;
+  const run = (program, args, options = {}) => {
+    if (program !== 'docker') return execute(program, args, options);
+    const source = options.env || process.env;
+    const env = {};
+    for (const key of ['PATH', 'HOME', 'DOCKER_CONFIG', 'XDG_RUNTIME_DIR']) {
+      if (source[key] !== undefined) env[key] = source[key];
+    }
+    env.DOCKER_HOST = `unix://${config.dockerSocket || '/var/run/docker.sock'}`;
+    if (args[0] === 'create' && options.env?.RUNNER_JIT_CONFIG)
+      env.RUNNER_JIT_CONFIG = options.env.RUNNER_JIT_CONFIG;
+    return execute(program, args, { ...options, env });
+  };
+  const wait = dependencies.sleep || sleep;
+  const diskStats = dependencies.statfs || statfs;
+  let stopping = false;
+  let stopAt = Infinity;
+  const stateDir = resolve(config.stateDir);
+  const logDir = resolve(config.logDir);
+  const lockPath = join(stateDir, 'manager.lock');
+  const slotPath = (slot) => join(stateDir, `slot-${slot}.json`);
+  const log = (message) => console.log(`[${config.poolId}] ${message}`);
+  function parseJson(text) {
+    try {
+      return JSON.parse(text);
+    } catch {
+      throw new Error('Invalid JSON response from GitHub');
+    }
+  }
+
+  async function gh(args, input) {
+    return run('gh', ['api', ...args, ...(input ? ['--input', '-'] : [])], {
+      env: process.env,
+      input: input && JSON.stringify(input),
+    });
+  }
+
+  async function save(slot, state) {
+    const path = slotPath(slot);
+    const temp = `${path}.${randomUUID()}.tmp`;
+    await writeFile(temp, JSON.stringify(state), { mode: 0o600 });
+    await rename(temp, path);
+  }
+
+  async function load(slot) {
+    try {
+      const raw = await readFile(slotPath(slot), 'utf8');
+      let state;
+      try {
+        state = JSON.parse(raw);
+      } catch {
+        throw new Error('Invalid slot state; manual reconciliation required');
+      }
+      // Reconcile identities created before the ECS-compatible name prefix.
+      const name =
+        typeof state?.name === 'string' ? state.name.replace(/^ecs-/, '') : '';
+      const prefix = `qwen-${config.poolId}-${slot}-`;
+      if (
+        !name.startsWith(prefix) ||
+        !/^[0-9a-f-]{36}$/.test(name.slice(prefix.length)) ||
+        typeof state.uncertain !== 'boolean' ||
+        (!state.uncertain &&
+          (!Number.isSafeInteger(state.runnerId) || state.runnerId < 1)) ||
+        (state.runnerId !== undefined &&
+          (!Number.isSafeInteger(state.runnerId) || state.runnerId < 1))
+      )
+        throw new Error('Invalid slot state; manual reconciliation required');
+      return state;
+    } catch (error) {
+      if (error.code === 'ENOENT') return null;
+      throw error;
+    }
+  }
+
+  async function listContainers(slot) {
+    const output = await run('docker', [
+      'ps',
+      '-aq',
+      '--no-trunc',
+      '--filter',
+      `label=qwen.pool=${config.poolId}`,
+      '--filter',
+      `label=qwen.slot=${slot}`,
+    ]);
+    return output.trim().split('\n').filter(Boolean);
+  }
+
+  async function listNetworks(slot) {
+    const args = [
+      'network',
+      'ls',
+      '-q',
+      '--no-trunc',
+      '--filter',
+      `label=qwen.pool=${config.poolId}`,
+    ];
+    if (slot !== undefined) args.push('--filter', `label=qwen.slot=${slot}`);
+    return (await run('docker', args)).trim().split('\n').filter(Boolean);
+  }
+
+  async function listRunners() {
+    const output = await gh([
+      '--paginate',
+      '--jq',
+      '.runners | @json',
+      `/repos/${config.repo}/actions/runners?per_page=100`,
+    ]);
+    const pages = output.trim().split('\n').map(parseJson);
+    if (
+      !pages.every(
+        (runners) =>
+          Array.isArray(runners) &&
+          runners.every(
+            (runner) =>
+              typeof runner?.name === 'string' &&
+              Number.isSafeInteger(runner.id) &&
+              runner.id > 0,
+          ),
+      )
+    )
+      throw new Error('Invalid GitHub runner list');
+    return pages.flat();
+  }
+
+  async function pruneLogs() {
+    const files = [];
+    for (const entry of await readdir(logDir, { withFileTypes: true })) {
+      if (
+        !entry.isFile() ||
+        !entry.name.replace(/^ecs-/, '').startsWith(`qwen-${config.poolId}-`) ||
+        !/\.(log|tar)$/.test(entry.name)
+      )
+        continue;
+      const path = join(logDir, entry.name);
+      const info = await lstat(path);
+      files.push({ path, size: info.size, time: info.mtimeMs });
+    }
+    files.sort((a, b) => b.time - a.time);
+    let size = 0;
+    for (const file of files) {
+      size += file.size;
+      if (file.time < Date.now() - 7 * 86400000 || size > 100 * 1024 * 1024)
+        await rm(file.path, { force: true });
+    }
+  }
+
+  async function cleanup(slot, state) {
+    const containers = await listContainers(slot);
+    const networks = await listNetworks(slot);
+    if ((containers.length || networks.length) && !state?.name)
+      throw new Error(
+        'Container has no recorded runner identity; manual reconciliation required',
+      );
+    for (const id of containers) {
+      try {
+        const logs = await run('docker', ['logs', '--tail', '10000', id]);
+        await writeFile(
+          join(
+            logDir,
+            `${state?.name || `slot-${slot}`}-${id.slice(0, 12)}.log`,
+          ),
+          logs.slice(-1024 * 1024),
+          { mode: 0o600 },
+        );
+        await pruneLogs();
+      } catch {
+        log(`slot ${slot}: container logs unavailable`);
+      }
+      try {
+        const archive = await run(
+          'docker',
+          ['cp', `${id}:/home/node/actions-runner/_diag`, '-'],
+          { binary: true },
+        );
+        await writeFile(
+          join(logDir, `${state.name}-${id.slice(0, 12)}.tar`),
+          archive,
+          { mode: 0o600 },
+        );
+        await pruneLogs();
+      } catch {
+        log(
+          `slot ${slot}: runner diagnostics unavailable or larger than 4 MiB`,
+        );
+      }
+      await run('docker', ['rm', '-f', '-v', id]);
+    }
+    if ((await listContainers(slot)).length)
+      throw new Error('Container cleanup failed');
+    for (const id of networks) await run('docker', ['network', 'rm', id]);
+    if ((await listNetworks(slot)).length)
+      throw new Error('Network cleanup failed');
+    if (state?.name) {
+      const all = await listRunners();
+      const runners = all.filter((runner) => runner.name === state.name);
+      if (
+        state.runnerId &&
+        (runners.some((runner) => runner.id !== state.runnerId) ||
+          all.some(
+            (runner) =>
+              runner.id === state.runnerId && runner.name !== state.name,
+          ))
+      )
+        throw new Error(
+          'Runner identity mismatch; manual reconciliation required',
+        );
+      if (
+        runners.length > 1 ||
+        runners.some(
+          (runner) => !Number.isSafeInteger(runner.id) || runner.id < 1,
+        )
+      )
+        throw new Error(
+          'Ambiguous runner identity; manual reconciliation required',
+        );
+      if (state.uncertain && !runners.length)
+        throw new Error(
+          'JIT response was lost; runner not found, manual reconciliation required',
+        );
+      for (const runner of runners) {
+        try {
+          await gh([
+            '-X',
+            'DELETE',
+            `/repos/${config.repo}/actions/runners/${runner.id}`,
+          ]);
+        } catch (error) {
+          if (error.status !== 404) throw error;
+        }
+      }
+    }
+    await rm(slotPath(slot), { force: true });
+    log(`slot ${slot}: cleaned`);
+  }
+
+  async function cycle(slot) {
+    let earlyExit = false;
+    let state = await load(slot);
+    if (
+      state ||
+      (await listContainers(slot)).length ||
+      (await listNetworks(slot)).length
+    )
+      await cleanup(slot, state);
+    if (stopping) return;
+    const dockerRoot = (
+      await run('docker', ['info', '--format', '{{.DockerRootDir}}'])
+    ).trim();
+    if (!isAbsolute(dockerRoot) || dockerRoot.includes('\n'))
+      throw new Error('Invalid Docker Root Dir');
+    const disk = await diskStats(dockerRoot);
+    if (disk.bavail * disk.bsize < config.minDiskBytes)
+      throw new Error(
+        `Insufficient disk capacity on Docker Root Dir ${dockerRoot}`,
+      );
+    state = {
+      name: `ecs-qwen-${config.poolId}-${slot}-${randomUUID()}`,
+      createdAt: new Date().toISOString(),
+      uncertain: true,
+    };
+    await save(slot, state);
+    let result;
+    try {
+      result = parseJson(
+        await gh(
+          [
+            '-X',
+            'POST',
+            `/repos/${config.repo}/actions/runners/generate-jitconfig`,
+          ],
+          {
+            name: state.name,
+            runner_group_id: config.runnerGroupId,
+            labels: config.labels,
+            work_folder: '_work',
+          },
+        ),
+      );
+    } catch (error) {
+      if (error.status >= 400 && error.status < 500 && error.status !== 408) {
+        await rm(slotPath(slot), { force: true });
+      } else {
+        await cleanup(slot, state);
+      }
+      throw error;
+    }
+    try {
+      if (
+        result?.runner?.name !== state.name ||
+        !Number.isSafeInteger(result.runner.id) ||
+        result.runner.id < 1
+      )
+        throw new Error('Invalid JIT runner identity');
+      state.runnerId = result.runner.id;
+      state.uncertain = false;
+      await save(slot, state);
+      if (!result.encoded_jit_config)
+        throw new Error('GitHub returned no JIT configuration');
+      const networkId = (
+        await run('docker', [
+          'network',
+          'create',
+          '--driver',
+          'bridge',
+          '--opt',
+          'com.docker.network.bridge.enable_icc=false',
+          '--label',
+          `qwen.pool=${config.poolId}`,
+          '--label',
+          `qwen.slot=${slot}`,
+          '--label',
+          `qwen.instance=${state.name}`,
+          state.name,
+        ])
+      ).trim();
+      if (!networkId) throw new Error('Docker returned no network ID');
+      state.networkId = networkId;
+      await save(slot, state);
+      const env = {
+        ...process.env,
+        RUNNER_JIT_CONFIG: result.encoded_jit_config,
+      };
+      delete env.GH_TOKEN;
+      delete env.GITHUB_TOKEN;
+      const id = (
+        await run(
+          'docker',
+          [
+            'create',
+            '--init',
+            '--network',
+            networkId,
+            '--label',
+            `qwen.pool=${config.poolId}`,
+            '--label',
+            `qwen.slot=${slot}`,
+            '--label',
+            `qwen.instance=${state.name}`,
+            '--cpus',
+            String(config.cpus),
+            '--memory',
+            config.memory,
+            '--pids-limit',
+            String(config.pidsLimit),
+            '--log-driver',
+            'json-file',
+            '--log-opt',
+            'max-size=10m',
+            '--log-opt',
+            'max-file=3',
+            '--env',
+            'RUNNER_JIT_CONFIG',
+            config.image,
+          ],
+          { env },
+        )
+      ).trim();
+      state.containerId = id;
+      await save(slot, state);
+      if (!id) throw new Error('Docker returned no container ID');
+      if (!stopping) {
+        await run('docker', ['start', id]);
+        log(
+          `slot ${slot}: started runner ${state.name}, id ${state.runnerId}, container ${id}`,
+        );
+        const deadline = Date.now() + config.ttlSeconds * 1000;
+        let exitCode;
+        const watching = run('docker', ['wait', id], {
+          timeout: (config.ttlSeconds + 10) * 1000,
+        }).then(
+          (output) => {
+            const status = output.trim();
+            exitCode =
+              /^\d+$/.test(status) && Number(status) <= 255
+                ? Number(status)
+                : null;
+            return true;
+          },
+          () => {
+            exitCode = null;
+            return true;
+          },
+        );
+        while (Date.now() < Math.min(deadline, stopAt)) {
+          if (await Promise.race([watching, wait(1000).then(() => false)]))
+            break;
+        }
+        if (exitCode !== undefined && exitCode !== 0) {
+          earlyExit = true;
+          log(
+            `slot ${slot}: runner exit status ${exitCode ?? 'unknown'}; next JIT request is delayed`,
+          );
+        }
+      }
+    } finally {
+      await cleanup(slot, state);
+    }
+    return { earlyExit };
+  }
+
+  async function acquireLock() {
+    await mkdir(stateDir, { recursive: true, mode: 0o700 });
+    await mkdir(logDir, { recursive: true, mode: 0o700 });
+    const child = spawn('flock', ['-n', lockPath, 'cat'], {
+      stdio: ['pipe', 'pipe', 'ignore'],
+    });
+    await new Promise((resolveLock, reject) => {
+      child.once('error', reject);
+      child.once('exit', () =>
+        reject(new Error('Could not acquire manager lock')),
+      );
+      child.stdout.once('data', resolveLock);
+      child.stdin.on('error', () => {});
+      child.stdin.write('locked\n');
+    });
+    return child;
+  }
+
+  async function start() {
+    const lock = await acquireLock();
+    try {
+      const slots = (
+        await run('docker', [
+          'ps',
+          '-a',
+          '--filter',
+          `label=qwen.pool=${config.poolId}`,
+          '--format',
+          '{{.Label "qwen.slot"}}',
+        ])
+      )
+        .trim()
+        .split('\n')
+        .filter(Boolean);
+      const states = (await readdir(stateDir))
+        .map((name) => name.match(/^slot-(\d+)\.json$/)?.[1])
+        .filter(Boolean);
+      const networkSlots = await Promise.all(
+        (await listNetworks()).map(async (id) =>
+          (
+            await run('docker', [
+              'network',
+              'inspect',
+              '--format',
+              '{{index .Labels "qwen.slot"}}',
+              id,
+            ])
+          ).trim(),
+        ),
+      );
+      if (
+        [...slots, ...states, ...networkSlots].some(
+          (slot) =>
+            !/^\d+$/.test(slot) ||
+            Number(slot) < 1 ||
+            Number(slot) > config.slots,
+        )
+      ) {
+        throw new Error(
+          'Pool contains resources outside configured slots; reconcile before starting',
+        );
+      }
+      await Promise.all(
+        Array.from({ length: config.slots }, async (_, i) => {
+          const slot = i + 1;
+          while (!stopping) {
+            try {
+              const result = await cycle(slot);
+              if (result?.earlyExit && !stopping) {
+                log(`slot ${slot}: backing off 60 s before replenishing`);
+                await wait(60000);
+              }
+            } catch (error) {
+              log(
+                `slot ${slot}: ${error.message}; retrying without replenishing`,
+              );
+              await wait(10000);
+            }
+          }
+        }),
+      );
+    } finally {
+      lock.stdin.end();
+    }
+  }
+  function stop() {
+    stopping = true;
+    stopAt = Math.min(stopAt, Date.now() + config.shutdownGraceSeconds * 1000);
+  }
+  return { start, stop, cycle, cleanup };
+}
+
+if (
+  process.argv[1] &&
+  realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  try {
+    if (!process.env.GH_TOKEN?.trim()) throw new Error('GH_TOKEN is required');
+    const config = JSON.parse(
+      await readFile(resolve(process.argv[2] || ''), 'utf8'),
+    );
+    const manager = createManager(config);
+    process.on('SIGTERM', manager.stop);
+    process.on('SIGINT', manager.stop);
+    await manager.start();
+  } catch (error) {
+    console.error(`manager: ${error.message}`);
+    process.exitCode = 1;
+  }
+}

scripts/ci/ephemeral-runner/manager.test.mjs

@@ -0,0 +1,822 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import {
+  mkdtemp,
+  mkdir,
+  readFile,
+  rm,
+  symlink,
+  writeFile,
+} from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { command as runCommand, createManager } from './manager.mjs';
+
+async function fixture(t) {
+  const dir = await mkdtemp(join(tmpdir(), 'runner-manager-'));
+  t.after(() => rm(dir, { recursive: true, force: true }));
+  await mkdir(join(dir, 'state'));
+  await mkdir(join(dir, 'logs'));
+  const config = {
+    repo: 'owner/repo',
+    poolId: 'trial',
+    image: `image@sha256:${'a'.repeat(64)}`,
+    slots: 1,
+    runnerGroupId: 1,
+    labels: ['self-hosted', 'linux', 'ARM64', 'trial-only'],
+    cpus: 1,
+    memory: '1g',
+    pidsLimit: 100,
+    ttlSeconds: 2,
+    shutdownGraceSeconds: 0,
+    minDiskBytes: 0,
+    stateDir: join(dir, 'state'),
+    logDir: join(dir, 'logs'),
+  };
+  const calls = [];
+  const containers = new Map();
+  const networks = new Map();
+  let runner;
+  let counter = 0;
+  let foreignCounter = 0;
+  let failRemove = false;
+  let failNetworkRemove = false;
+  let keepContainerAfterRemove = false;
+  let loseJit = false;
+  const labelsFromArgs = (args) => {
+    const labels = {};
+    for (let i = 0; i < args.length; i += 1) {
+      if (args[i] !== '--label' || typeof args[i + 1] !== 'string') continue;
+      const at = args[i + 1].indexOf('=');
+      if (at > 0) labels[args[i + 1].slice(0, at)] = args[i + 1].slice(at + 1);
+    }
+    return labels;
+  };
+  const filtersFromArgs = (args) => {
+    const filters = [];
+    for (let i = 0; i < args.length; i += 1) {
+      if (args[i] === '--filter' && typeof args[i + 1] === 'string')
+        filters.push(args[i + 1].replace(/^label=/, ''));
+    }
+    return filters;
+  };
+  const matchesFilters = (entry, args) =>
+    filtersFromArgs(args).every((filter) => {
+      const at = filter.indexOf('=');
+      if (at < 0) return true;
+      return entry.labels[filter.slice(0, at)] === filter.slice(at + 1);
+    });
+  async function command(program, args, options = {}) {
+    calls.push({ program, args, options });
+    if (program === 'gh') {
+      const path = args.find((v) => v.startsWith('/'));
+      if (path.includes('generate-jitconfig')) {
+        const body = JSON.parse(options.input);
+        runner = { id: ++counter, name: body.name };
+        if (loseJit) throw new Error('network failure');
+        return JSON.stringify({ runner, encoded_jit_config: 'jit-secret' });
+      }
+      if (args.includes('DELETE')) {
+        runner = undefined;
+        return '';
+      }
+      return JSON.stringify(runner ? [runner] : []);
+    }
+    if (args[0] === 'ps') {
+      const matched = [...containers.entries()].filter(([, entry]) =>
+        matchesFilters(entry, args),
+      );
+      if (!matched.length) return '';
+      if (args.includes('--format'))
+        return `${matched.map(([, entry]) => entry.labels['qwen.slot'] ?? '').join('\n')}\n`;
+      return `${matched.map(([id]) => id).join('\n')}\n`;
+    }
+    if (args[0] === 'network' && args[1] === 'ls') {
+      const matched = [...networks.entries()].filter(([, entry]) =>
+        matchesFilters(entry, args),
+      );
+      return matched.length ? `${matched.map(([id]) => id).join('\n')}\n` : '';
+    }
+    if (args[0] === 'network' && args[1] === 'create') {
+      const id = `network-${counter}`;
+      networks.set(id, { labels: labelsFromArgs(args) });
+      return id;
+    }
+    if (args[0] === 'network' && args[1] === 'rm') {
+      if (failNetworkRemove) throw new Error('network removal failed');
+      networks.delete(args[args.length - 1]);
+      return '';
+    }
+    if (args[0] === 'network' && args[1] === 'inspect') {
+      const entry = networks.get(args[args.length - 1]);
+      return `${entry?.labels['qwen.slot'] ?? ''}\n`;
+    }
+    if (args[0] === 'info') return `${dir}\n`;
+    if (args[0] === 'create') {
+      const id = `container-${counter}`;
+      containers.set(id, { labels: labelsFromArgs(args) });
+      return id;
+    }
+    if (args[0] === 'logs') return 'runner log';
+    if (args[0] === 'rm') {
+      if (failRemove) throw new Error('docker unavailable');
+      if (!keepContainerAfterRemove) containers.delete(args[args.length - 1]);
+      return '';
+    }
+    if (args[0] === 'wait') return '0\n';
+    if (args[0] === 'start') return '';
+    return '';
+  }
+  return {
+    config,
+    calls,
+    command,
+    sleep: async () => {},
+    addForeignNetwork(labels) {
+      const id = `network-foreign-${++foreignCounter}`;
+      networks.set(id, { labels });
+      return id;
+    },
+    networkIds: () => [...networks.keys()],
+    setFailRemove: (v) => {
+      failRemove = v;
+    },
+    setLoseJit: (v) => {
+      loseJit = v;
+    },
+    setFailNetworkRemove: (v) => {
+      failNetworkRemove = v;
+    },
+    setKeepContainerAfterRemove: (v) => {
+      keepContainerAfterRemove = v;
+    },
+  };
+}
+
+test('creates a new runner and container per job and keeps GitHub token out of Docker', async (t) => {
+  const previousToken = process.env.GH_TOKEN;
+  process.env.GH_TOKEN = 'host-only-pat';
+  t.after(() => {
+    if (previousToken === undefined) delete process.env.GH_TOKEN;
+    else process.env.GH_TOKEN = previousToken;
+  });
+  const f = await fixture(t);
+  const manager = createManager(f.config, { command: f.command });
+  await manager.cycle(1);
+  await manager.cycle(1);
+  const creates = f.calls.filter(
+    (c) => c.program === 'docker' && c.args[0] === 'create',
+  );
+  const requests = f.calls.filter(
+    (c) =>
+      c.program === 'gh' &&
+      c.args.includes('/repos/owner/repo/actions/runners/generate-jitconfig'),
+  );
+  assert.ok(
+    requests.every((c) =>
+      JSON.parse(c.options.input).name.startsWith('ecs-qwen-'),
+    ),
+  );
+  assert.ok(requests.every((c) => c.options.env.GH_TOKEN === 'host-only-pat'));
+  assert.equal(creates.length, 2);
+  assert.notEqual(
+    JSON.parse(requests[0].options.input).name,
+    JSON.parse(requests[1].options.input).name,
+  );
+  assert.deepEqual(
+    JSON.parse(requests[0].options.input).labels,
+    f.config.labels,
+  );
+  assert.ok(
+    creates.every(
+      (c) =>
+        !c.options.env.GH_TOKEN &&
+        c.options.env.RUNNER_JIT_CONFIG === 'jit-secret',
+    ),
+  );
+  assert.deepEqual(
+    creates.map((c) => c.args[c.args.indexOf('--network') + 1]),
+    ['network-1', 'network-2'],
+  );
+  const networks = f.calls.filter(
+    (c) =>
+      c.program === 'docker' &&
+      c.args[0] === 'network' &&
+      c.args[1] === 'create',
+  );
+  assert.equal(networks.length, 2);
+  assert.ok(
+    networks.every((c) =>
+      c.args.includes('com.docker.network.bridge.enable_icc=false'),
+    ),
+  );
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length,
+    2,
+  );
+});
+
+test('failed cleanup blocks replacement until recovery cleans the old container', async (t) => {
+  const f = await fixture(t);
+  const manager = createManager(f.config, { command: f.command });
+  f.setFailRemove(true);
+  await assert.rejects(manager.cycle(1), /docker unavailable/);
+  await assert.rejects(manager.cycle(1), /docker unavailable/);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    1,
+  );
+  f.setFailRemove(false);
+  await manager.cycle(1);
+  const ops = f.calls
+    .filter((c) => c.program === 'docker')
+    .map((c) => c.args[0]);
+  assert.ok(ops.lastIndexOf('rm') > ops.lastIndexOf('create'));
+  assert.equal(ops.filter((op) => op === 'create').length, 2);
+});
+
+test('lost JIT response reconciles the recorded name before requesting another', async (t) => {
+  const f = await fixture(t);
+  const manager = createManager(f.config, { command: f.command });
+  f.setLoseJit(true);
+  await assert.rejects(manager.cycle(1), /network failure/);
+  await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), {
+    code: 'ENOENT',
+  });
+  assert.ok(
+    f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')),
+  );
+  f.setLoseJit(false);
+  await manager.cycle(1);
+});
+
+test('an unconfirmed JIT request blocks replacement when no runner can be found', async (t) => {
+  const f = await fixture(t);
+  const command = (program, args, options) => {
+    if (
+      program === 'gh' &&
+      args.includes('/repos/owner/repo/actions/runners/generate-jitconfig')
+    ) {
+      return Promise.reject(new Error('connection lost'));
+    }
+    return f.command(program, args, options);
+  };
+  const manager = createManager(f.config, { command });
+  await assert.rejects(manager.cycle(1), /manual reconciliation/);
+  await assert.rejects(manager.cycle(1), /manual reconciliation/);
+  const state = JSON.parse(
+    await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'),
+  );
+  assert.equal(state.uncertain, true);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    0,
+  );
+});
+
+test('a failed container start is cleaned up before another cycle', async (t) => {
+  const f = await fixture(t);
+  let fail = true;
+  const command = (program, args, options) => {
+    if (program === 'docker' && args[0] === 'start' && fail) {
+      fail = false;
+      return Promise.reject(new Error('start failed'));
+    }
+    return f.command(program, args, options);
+  };
+  const manager = createManager(f.config, { command });
+  await assert.rejects(manager.cycle(1), /start failed/);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length,
+    1,
+  );
+  await manager.cycle(1);
+});
+
+test('watchdog removes a container that does not exit', async (t) => {
+  const f = await fixture(t);
+  f.config.ttlSeconds = 0.01;
+  const command = (program, args, options) =>
+    program === 'docker' && args[0] === 'wait'
+      ? new Promise(() => {})
+      : f.command(program, args, options);
+  await createManager(f.config, { command }).cycle(1);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'rm').length,
+    1,
+  );
+});
+
+test('checks Docker root disk rather than the state directory', async (t) => {
+  const f = await fixture(t);
+  let checked;
+  const manager = createManager(f.config, {
+    command: f.command,
+    statfs: async (path) => {
+      checked = path;
+      return { bavail: 0, bsize: 4096 };
+    },
+  });
+  f.config.minDiskBytes = 1;
+  await assert.rejects(manager.cycle(1), /Insufficient disk capacity/);
+  assert.equal(checked, join(f.config.stateDir, '..'));
+  assert.equal(f.calls.filter((c) => c.program === 'gh').length, 0);
+  const info = f.calls.find(
+    (c) => c.program === 'docker' && c.args[0] === 'info',
+  );
+  assert.equal(info.options.env.DOCKER_HOST, 'unix:///var/run/docker.sock');
+});
+
+test('rejects same-name runner with a different ID without deleting it', async (t) => {
+  const f = await fixture(t);
+  const state = {
+    name: 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa',
+    runnerId: 42,
+    uncertain: false,
+  };
+  await writeFile(
+    join(f.config.stateDir, 'slot-1.json'),
+    JSON.stringify(state),
+  );
+  const command = (program, args, options) => {
+    if (program === 'gh' && args.includes('--paginate'))
+      return Promise.resolve(JSON.stringify([{ name: state.name, id: 43 }]));
+    return f.command(program, args, options);
+  };
+  await assert.rejects(
+    createManager(f.config, { command }).cycle(1),
+    /identity mismatch/,
+  );
+  assert.ok(
+    !f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')),
+  );
+  assert.equal(
+    JSON.parse(await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'))
+      .runnerId,
+    42,
+  );
+});
+
+test('reconciles a legacy identity on a later page before registering its replacement', async (t) => {
+  const f = await fixture(t);
+  const name = 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa';
+  await writeFile(
+    join(f.config.stateDir, 'slot-1.json'),
+    JSON.stringify({
+      name,
+      runnerId: 42,
+      uncertain: false,
+    }),
+  );
+  let deleted = false;
+  const command = (program, args, options) => {
+    if (program === 'gh' && args.includes('--paginate') && !deleted) {
+      assert.ok(!args.includes('--slurp'));
+      assert.equal(args[args.indexOf('--jq') + 1], '.runners | @json');
+      return Promise.resolve('[]\n' + JSON.stringify([{ name, id: 42 }]));
+    }
+    if (
+      program === 'gh' &&
+      args.includes('/repos/owner/repo/actions/runners/42')
+    ) {
+      deleted = true;
+    }
+    if (
+      program === 'gh' &&
+      args.includes('/repos/owner/repo/actions/runners/generate-jitconfig')
+    ) {
+      assert.ok(
+        deleted,
+        'legacy registration must be removed before replacement',
+      );
+    }
+    return f.command(program, args, options);
+  };
+  await createManager(f.config, { command }).cycle(1);
+  assert.ok(deleted);
+});
+
+test('startup reclaims a live container before replenishing after a crash', async (t) => {
+  const f = await fixture(t);
+  const previousPath = process.env.PATH;
+  await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', {
+    mode: 0o755,
+  });
+  process.env.PATH = `${f.config.stateDir}:${previousPath}`;
+  t.after(() => {
+    process.env.PATH = previousPath;
+  });
+  await writeFile(
+    join(f.config.stateDir, 'slot-1.json'),
+    JSON.stringify({
+      name: 'ecs-qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa',
+      runnerId: 42,
+      uncertain: false,
+    }),
+  );
+  let oldContainer = true;
+  let manager;
+  const command = async (program, args, options) => {
+    if (program === 'docker' && args[0] === 'ps') {
+      if (args.includes('--format')) {
+        // Docker quiet mode overrides --format, as on the trial host.
+        return oldContainer
+          ? args.includes('-aq')
+            ? 'old-container'
+            : '1'
+          : '';
+      }
+      if (oldContainer) return 'old-container';
+    }
+    if (
+      program === 'docker' &&
+      args[0] === 'rm' &&
+      args.includes('old-container')
+    ) {
+      oldContainer = false;
+      return '';
+    }
+    if (
+      program === 'gh' &&
+      args.includes('/repos/owner/repo/actions/runners/generate-jitconfig')
+    ) {
+      assert.equal(
+        oldContainer,
+        false,
+        'remove old container before registration',
+      );
+    }
+    if (program === 'docker' && args[0] === 'start') manager.stop();
+    return f.command(program, args, options);
+  };
+  manager = createManager(f.config, { command });
+  await manager.start();
+  assert.equal(oldContainer, false);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    1,
+  );
+});
+
+test('accepts immutable local image IDs and rejects mutable tags and invalid configuration', async (t) => {
+  const f = await fixture(t);
+  assert.doesNotThrow(() =>
+    createManager({ ...f.config, image: `sha256:${'b'.repeat(64)}` }),
+  );
+  assert.throws(
+    () => createManager({ ...f.config, image: 'image:latest' }),
+    /Invalid manager configuration/,
+  );
+  assert.throws(
+    () =>
+      createManager({ ...f.config, dockerSocket: 'tcp://example.com:2375' }),
+    /Invalid manager configuration/,
+  );
+  assert.throws(
+    () => createManager({ ...f.config, cpus: Infinity }),
+    /Invalid manager configuration/,
+  );
+});
+
+test('does not expose malformed GitHub responses in errors', async (t) => {
+  const f = await fixture(t);
+  const command = (program, args, options) => {
+    if (
+      program === 'gh' &&
+      args.some((arg) => arg.includes('generate-jitconfig'))
+    ) {
+      return f
+        .command(program, args, options)
+        .then(() => 'malformed-secret-token');
+    }
+    return f.command(program, args, options);
+  };
+  await assert.rejects(
+    createManager(f.config, { command }).cycle(1),
+    (error) => {
+      assert.match(error.message, /Invalid JSON response/);
+      assert.doesNotMatch(error.message, /malformed-secret-token/);
+      return true;
+    },
+  );
+});
+
+test('HTTP 429 rejection does not leave uncertain state and the next cycle can retry', async (t) => {
+  const f = await fixture(t);
+  const fakeGh = join(f.config.stateDir, 'gh');
+  await writeFile(
+    fakeGh,
+    '#!/bin/sh\necho "gh: API rate limit exceeded (HTTP 429)" >&2\nexit 1\n',
+    { mode: 0o755 },
+  );
+  let rejected = false;
+  const command = (program, args, options) => {
+    if (
+      program === 'gh' &&
+      args.includes('/repos/owner/repo/actions/runners/generate-jitconfig') &&
+      !rejected
+    ) {
+      rejected = true;
+      return runCommand(fakeGh, [], { env: {} });
+    }
+    return f.command(program, args, options);
+  };
+  const manager = createManager(f.config, { command });
+  await assert.rejects(manager.cycle(1), (error) => error.status === 429);
+  await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), {
+    code: 'ENOENT',
+  });
+  await manager.cycle(1);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    1,
+  );
+});
+
+test('network cleanup failure prevents new runner and is reconciled on the next cycle', async (t) => {
+  const f = await fixture(t);
+  const manager = createManager(f.config, { command: f.command });
+  f.setFailNetworkRemove(true);
+  await assert.rejects(manager.cycle(1), /network removal failed/);
+  await assert.rejects(manager.cycle(1), /network removal failed/);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    1,
+  );
+  f.setFailNetworkRemove(false);
+  await manager.cycle(1);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    2,
+  );
+});
+
+test('a lost network create response is recovered using labels', async (t) => {
+  const f = await fixture(t);
+  const command = async (program, args, options) => {
+    const result = await f.command(program, args, options);
+    if (program === 'docker' && args[0] === 'network' && args[1] === 'create')
+      throw new Error('network create response lost');
+    return result;
+  };
+  await assert.rejects(
+    createManager(f.config, { command }).cycle(1),
+    /network create response lost/,
+  );
+  assert.equal(
+    f.calls.filter(
+      (c) =>
+        c.program === 'docker' && c.args[0] === 'network' && c.args[1] === 'rm',
+    ).length,
+    1,
+  );
+  await assert.rejects(readFile(join(f.config.stateDir, 'slot-1.json')), {
+    code: 'ENOENT',
+  });
+});
+
+test('network deletion is verified even when docker reports success', async (t) => {
+  const f = await fixture(t);
+  const command = (program, args, options) => {
+    if (program === 'docker' && args[0] === 'network' && args[1] === 'rm')
+      return Promise.resolve('');
+    return f.command(program, args, options);
+  };
+  const manager = createManager(f.config, { command });
+  await assert.rejects(manager.cycle(1), /Network cleanup failed/);
+  await assert.rejects(manager.cycle(1), /Network cleanup failed/);
+  assert.equal(
+    f.calls.filter((c) => c.program === 'docker' && c.args[0] === 'create')
+      .length,
+    1,
+  );
+});
+
+test('passes only the docker environment allowlist and the JIT config by name', async (t) => {
+  const sentinels = {
+    AWS_SECRET_ACCESS_KEY: 'sentinel-secret',
+    SOME_DEPLOY_TOKEN: 'sentinel-token',
+  };
+  const previous = {};
+  for (const key of Object.keys(sentinels)) {
+    previous[key] = process.env[key];
+    process.env[key] = sentinels[key];
+  }
+  t.after(() => {
+    for (const key of Object.keys(sentinels)) {
+      if (previous[key] === undefined) delete process.env[key];
+      else process.env[key] = previous[key];
+    }
+  });
+  const f = await fixture(t);
+  await createManager(f.config, { command: f.command }).cycle(1);
+  const create = f.calls.find(
+    (c) => c.program === 'docker' && c.args[0] === 'create',
+  );
+  const allowed = [
+    'DOCKER_CONFIG',
+    'DOCKER_HOST',
+    'HOME',
+    'PATH',
+    'RUNNER_JIT_CONFIG',
+    'XDG_RUNTIME_DIR',
+  ];
+  assert.deepEqual(
+    Object.keys(create.options.env).filter((key) => !allowed.includes(key)),
+    [],
+  );
+  assert.ok(!JSON.stringify(create.options.env).includes('sentinel'));
+  assert.ok(!JSON.stringify(create.args).includes('sentinel'));
+  assert.equal(
+    create.args[create.args.indexOf('--log-driver') + 1],
+    'json-file',
+  );
+  const at = create.args.indexOf('--env');
+  assert.equal(create.args[at + 1], 'RUNNER_JIT_CONFIG');
+});
+
+test('labels pool and slot on create and honours recovery filters', async (t) => {
+  const f = await fixture(t);
+  const foreign = f.addForeignNetwork({
+    'qwen.pool': 'other',
+    'qwen.slot': '9',
+  });
+  const command = async (program, args, options) => {
+    const result = await f.command(program, args, options);
+    if (program === 'docker' && args[0] === 'network' && args[1] === 'create')
+      throw new Error('network create response lost');
+    return result;
+  };
+  await assert.rejects(
+    createManager(f.config, { command }).cycle(1),
+    /network create response lost/,
+  );
+  const created = f.calls.find(
+    (c) =>
+      c.program === 'docker' &&
+      c.args[0] === 'network' &&
+      c.args[1] === 'create',
+  );
+  assert.ok(created.args.includes('qwen.pool=trial'));
+  assert.ok(created.args.includes('qwen.slot=1'));
+  const removed = f.calls
+    .filter(
+      (c) =>
+        c.program === 'docker' && c.args[0] === 'network' && c.args[1] === 'rm',
+    )
+    .map((c) => c.args[c.args.length - 1]);
+  assert.deepEqual(removed, ['network-1']);
+  assert.ok(f.networkIds().includes(foreign));
+});
+
+test('container cleanup failure blocks replenishment even when docker reports success', async (t) => {
+  const f = await fixture(t);
+  f.setKeepContainerAfterRemove(true);
+  const manager = createManager(f.config, { command: f.command });
+  await assert.rejects(manager.cycle(1), /Container cleanup failed/);
+  const state = JSON.parse(
+    await readFile(join(f.config.stateDir, 'slot-1.json'), 'utf8'),
+  );
+  assert.ok(state.name);
+  assert.equal(state.runnerId, 1);
+});
+
+test('rejects corrupted slot state without deleting a runner', async (t) => {
+  const validName = 'qwen-trial-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa';
+  const cases = [
+    {
+      label: 'foreign prefix',
+      state: {
+        name: 'qwen-other-1-aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa',
+        runnerId: 42,
+        uncertain: false,
+      },
+    },
+    {
+      label: 'non-UUID suffix',
+      state: {
+        name: 'qwen-trial-1-not-a-uuid',
+        runnerId: 42,
+        uncertain: false,
+      },
+    },
+    {
+      label: 'non-boolean uncertain',
+      state: { name: validName, runnerId: 42, uncertain: 'no' },
+    },
+    {
+      label: 'zero runner id',
+      state: { name: validName, runnerId: 0, uncertain: false },
+    },
+  ];
+  for (const item of cases) {
+    const f = await fixture(t);
+    await writeFile(
+      join(f.config.stateDir, 'slot-1.json'),
+      JSON.stringify(item.state),
+    );
+    await assert.rejects(
+      createManager(f.config, { command: f.command }).cycle(1),
+      /Invalid slot state/,
+      item.label,
+    );
+    assert.ok(
+      !f.calls.some((c) => c.program === 'gh' && c.args.includes('DELETE')),
+      item.label,
+    );
+  }
+});
+
+test('backs off on a failed container even after the startup window', async (t) => {
+  const f = await fixture(t);
+  f.config.ttlSeconds = 120;
+  let now = Date.now();
+  t.mock.method(Date, 'now', () => now);
+  const command = (program, args, options) => {
+    if (program === 'docker' && args[0] === 'wait') {
+      now += 31000;
+      return Promise.resolve('126\n');
+    }
+    return f.command(program, args, options);
+  };
+  const result = await createManager(f.config, { command }).cycle(1);
+  assert.equal(result.earlyExit, true);
+});
+
+test('backs off before the next JIT request after a failed container', async (t) => {
+  const f = await fixture(t);
+  const previousPath = process.env.PATH;
+  await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', {
+    mode: 0o755,
+  });
+  process.env.PATH = `${f.config.stateDir}:${previousPath}`;
+  t.after(() => {
+    process.env.PATH = previousPath;
+  });
+  const sleeps = [];
+  let manager;
+  let jitRequests = 0;
+  const command = async (program, args, options) => {
+    if (
+      program === 'gh' &&
+      args.includes('/repos/owner/repo/actions/runners/generate-jitconfig')
+    ) {
+      jitRequests += 1;
+      if (jitRequests > 1) manager.stop();
+    }
+    if (program === 'docker' && args[0] === 'wait') return '127\n';
+    return f.command(program, args, options);
+  };
+  manager = createManager(f.config, {
+    command,
+    sleep: async (ms) => {
+      sleeps.push(ms);
+    },
+  });
+  await manager.start();
+  // The 1 s poll that races `docker wait` also goes through the injected sleep;
+  // the replenishment backoff is the 60 s entry.
+  assert.equal(sleeps.filter((ms) => ms === 60000).length, 1);
+  assert.equal(sleeps.at(-1), 60000);
+  assert.equal(jitRequests, 2);
+});
+
+test('does not penalize a fast successful container', async (t) => {
+  const f = await fixture(t);
+  const result = await createManager(f.config, { command: f.command }).cycle(1);
+  assert.equal(result.earlyExit, false);
+});
+
+test('backs off when docker wait cannot determine container status', async (t) => {
+  const f = await fixture(t);
+  const command = (program, args, options) => {
+    if (program === 'docker' && args[0] === 'wait')
+      return Promise.reject(new Error('Docker disconnected'));
+    return f.command(program, args, options);
+  };
+  const result = await createManager(f.config, { command }).cycle(1);
+  assert.equal(result.earlyExit, true);
+});
+
+test('starts through a symlink and reports invalid configuration', async (t) => {
+  const f = await fixture(t);
+  const link = join(f.config.stateDir, 'manager.mjs');
+  await symlink(new URL('./manager.mjs', import.meta.url), link);
+  const result = spawnSync(
+    process.execPath,
+    [link, join(f.config.stateDir, 'missing.json')],
+    {
+      env: { ...process.env, GH_TOKEN: 'test-only' },
+      encoding: 'utf8',
+    },
+  );
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /^manager: /);
+});

scripts/ci/ephemeral-runner/qwen-ephemeral-runner.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Qwen Code ephemeral GitHub Actions runners
+After=docker.service network-online.target
+Requires=docker.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+EnvironmentFile=/etc/qwen-ephemeral-runner/credentials.env
+WorkingDirectory=/opt/qwen-ephemeral-runner
+ExecStart=/usr/bin/env node /opt/qwen-ephemeral-runner/manager.mjs /etc/qwen-ephemeral-runner/config.json
+Restart=always
+RestartSec=10
+KillMode=mixed
+TimeoutStopSec=300
+
+[Install]
+WantedBy=multi-user.target

scripts/tests/ci-platform-lanes.test.js

@@ -116,8 +116,7 @@ describe('complete macOS test groups', () => {
 // The extended ceilings below, except the test lane's, are scoped to the
 // self-hosted pool: `runs-on` falls back to ubuntu-latest when
 // MAINTAINER_ECS_RUNNER_DISABLED=true (and, for the association-gated lanes,
-// for fork PRs from untrusted authors — lint_and_static reads lint_runner,
-// which routes every PR to the pool), so those hosted paths keep the
+// for untrusted fork PRs with the disposable pool disabled), so hosted paths keep the
 // pre-contention ceiling. The ARM64 fleet renders
 // `["self-hosted", "linux", "arm64"]` — no `ecs-qwen` label — so the scope
 // cannot key on the x64 fleet's name. Evaluate the real timeout expression for
@@ -391,8 +390,11 @@ it('reports the required Linux check only when the build and every shard pass',
   const gate = ci.jobs.test_ubuntu;
   expect(gate.name).toBe('Test (ubuntu-latest, Node 22.x)');
   expect(gate.needs).toEqual(['classify_pr', 'test_build', 'test']);
-  // On the shards' runners, not a hosted one that can queue it for minutes.
-  expect(gate['runs-on']).toBe(ci.jobs.test['runs-on']);
+  // The gate remains on the original association-gated runner, while test
+  // shards can opt into the disposable pool for untrusted forks.
+  expect(gate['runs-on']).toBe(
+    '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || \'["ubuntu-latest"]\') }}',
+  );
   expect(condOf('test_ubuntu')).toBe(condOf('test'));
   expect(gate.steps[0].run).toBe(
     'test "$BUILD_RESULT" = success && test "$SHARDS_RESULT" = success',
@@ -862,15 +864,11 @@ describe('post-merge push lane', () => {
     ).toContain('classify_pr');
   });
 
-  it('keeps test on the association-gated output', () => {
-    // Nothing else reads test's runs-on: switching it to lint_runner would
-    // hand every untrusted fork the pool while the timeout pins above stayed
-    // green — timeoutMinutesOn derives each lane's output from its own
-    // runs-on, so it cannot catch a lane pointed at the wrong output. Pin the
-    // expression, not an equality with lint_and_static — the two are meant
-    // to differ.
+  it('keeps test on the opt-in disposable output', () => {
+    // The classifier defaults this output to ubuntu_runner, but selects the
+    // disposable pool for untrusted forks only when explicitly enabled.
     expect(String(ci.jobs.test['runs-on'])).toBe(
-      '${{ fromJSON(needs.classify_pr.outputs.ubuntu_runner || \'["ubuntu-latest"]\') }}',
+      '${{ fromJSON(needs.classify_pr.outputs.pr_test_runner || \'["ubuntu-latest"]\') }}',
     );
   });
 

scripts/tests/review-worktree-cleanup-workflow.test.js

@@ -55,16 +55,18 @@ const ciYaml = parse(readFileSync('.github/workflows/ci.yml', 'utf8'));
 // possibly dirty workspace. Match the pool itself, not just the output
 // references that usually name it: jobs can also hard-code the shared label
 // array, and a checkout on either form inherits the same leftovers.
-// lint_and_static reads lint_runner (every PR reaches the pool), the other
-// lanes ubuntu_runner — both name the same shared workspace.
+// lint_and_static reads lint_runner (every PR reaches the pool), the test
+// lanes read pr_test_runner, the other lanes ubuntu_runner — all name the same
+// shared workspace. pr_test_runner is listed explicitly because jobs that read
+// it are also kept in scope by their own sweep step, which would otherwise hide
+// a future job that adds the pool and forgets the sweep.
 // Include retained recovery steps on hosted jobs too, so rerouting cannot
 // silently restore an untested sweep. Select by properties, not job name.
+const sharedPoolSelector = /ubuntu_runner|lint_runner|pr_test_runner|ecs-qwen/;
 const ciCleanSteps = Object.entries(ciYaml.jobs)
   .filter(
     ([, job]) =>
-      (/ubuntu_runner|lint_runner|ecs-qwen/.test(
-        JSON.stringify(job['runs-on'] ?? ''),
-      ) ||
+      (sharedPoolSelector.test(JSON.stringify(job['runs-on'] ?? '')) ||
         (job.steps ?? []).some(
           (s) => s.name === 'Clean stale .qwen before checkout',
         )) &&
@@ -381,10 +383,24 @@ describe('review worktree cleanup steps', () => {
     }
   });
 
+  it('matches every pr_test_runner job with the shared-pool selector itself', () => {
+    const poolJobs = Object.entries(ciYaml.jobs).filter(([, job]) =>
+      JSON.stringify(job['runs-on'] ?? '').includes('pr_test_runner'),
+    );
+    expect(poolJobs.length).toBeGreaterThan(0);
+    for (const [id, job] of poolJobs) {
+      expect(
+        sharedPoolSelector.test(JSON.stringify(job['runs-on'] ?? '')),
+        `job "${id}" runs on pr_test_runner and must match the pool selector`,
+      ).toBe(true);
+    }
+  });
+
   it('keeps every shared-pool ci.yml checkout sweep pinned to paths.ts', () => {
     expect(ciCleanSteps.map(({ id }) => id)).toEqual(
       expect.arrayContaining([
         'test',
+        'test_build',
         'web_shell_e2e_smoke',
         'integration_cli',
       ]),