fix(ci): honor container exit status and runner startup contracts

docs/design/2026-11-07-ephemeral-container-runners.md

@@ -2,7 +2,7 @@
 
 [English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md)
 
-Date: 2026-11-07. Status: implemented; a one-slot systemd trial is deployed on
+Date: 2026-11-07. Status: implemented; a five-slot systemd trial is deployed on
 hz4. Production external-PR routing remains disabled.
 
 Ten slots registering and running ten keyless test jobs of a private repository
@@ -55,7 +55,7 @@ Each container
 ```
 
 The manager keeps a configured number of slots, at most one container per slot.
-The Hangzhou trial ran ten slots at once; the hz4 deployment runs one slot. It takes no webhook, schedules no PR
+The Hangzhou trial ran ten slots at once; the hz4 deployment now maintains five slots. It takes no webhook, schedules no PR
 itself, and introduces neither Kubernetes nor a database. GitHub assigns work by
 runner label. Each ECS runs its own manager against its local Docker socket,
 with a unique `poolId` and a per-host `slots` count. Hosts share the scheduling

docs/design/2026-11-07-ephemeral-container-runners.zh-CN.md

@@ -2,7 +2,7 @@
 
 [English](2026-11-07-ephemeral-container-runners.md) | [简体中文](2026-11-07-ephemeral-container-runners.zh-CN.md)
 
-日期：2026-11-07。状态：已实现并在 hz4 部署单槽位 systemd 试运行服务；
+日期：2026-11-07。状态：已实现并在 hz4 部署五槽位 systemd 试运行服务；
 外部 PR 的生产路由仍未启用。
 
 杭州共享测试机上已验证 10 个槽位同时注册，并执行同一个私有测试仓库的 10 个无密钥
@@ -47,7 +47,7 @@
 ```
 
 管理程序按配置维护固定数量的槽位，每个槽位最多一个容器；杭州十槽位试验同时使用
-十个槽位，hz4 部署使用一个槽位。不接 webhook、不自行调度 PR、不引入 Kubernetes 或数据库。
+十个槽位，hz4 部署目前维持五个槽位。不接 webhook、不自行调度 PR、不引入 Kubernetes 或数据库。
 GitHub 根据 runner 标签分配任务。每台 ECS 各运行一个管理程序，只连接本机
 Docker socket，使用独立的 `poolId` 和按主机配置的 `slots`。各主机共用调度
 标签，无需中央管理器。新名称使用 `ecs-qwen-` 前缀以兼容 CI 并发上限和超时；

scripts/ci/ephemeral-runner/README.md

@@ -21,12 +21,17 @@ pull requests, verify they cannot select other sensitive runner pools.
 
 The image supports Linux x64 and ARM64. Its Node base is pinned to an OCI index
 digest and the runner downloads are checked against pinned SHA-256 digests.
-BuildKit supplies `TARGETARCH`; with Docker's legacy builder pass it explicitly:
+Use BuildKit to select both the base image and runner architecture:
 
 ```sh
-docker build --build-arg TARGETARCH=arm64 -t qwen-ephemeral-runner:trial scripts/ci/ephemeral-runner
+docker buildx build --platform linux/arm64 --load -t qwen-ephemeral-runner:trial scripts/ci/ephemeral-runner
 ```
 
+BuildKit supplies `TARGETARCH`. With Docker's legacy builder, build on the target
+architecture and pass its matching `TARGETARCH` explicitly (`amd64` or `arm64`).
+That argument selects only the runner archive; it does not change the base image
+architecture.
+
 The runner image supports ARM64, but full PR CI on this image has not yet been
 verified. The repository also selects ARM64 linters; verify the complete CI
 toolchain before routing production work to either architecture.
@@ -78,6 +83,10 @@ The `ttlSeconds` limit starts when the container starts, including idle time.
 A job assigned near the deadline can be interrupted. The manager removes and
 replaces the container; GitHub remains authoritative for job status. The
 manager logs cleanup but does not distinguish expiry from normal completion.
+A nonzero or unknown container exit status delays replenishment by 60 seconds;
+a successful short job is replenished immediately. GitHub remains authoritative
+for job success. Diagnostic export is best effort: stderr is not archived, and
+output exceeding the capture limits is discarded.
 `minDiskBytes` applies to the Docker data filesystem. Configure and verify
 disk quotas or a dedicated data volume as appropriate for the host.
 

scripts/ci/ephemeral-runner/manager.mjs

@@ -1,5 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { spawn } from 'node:child_process';
+import { realpathSync } from 'node:fs';
 import {
   lstat,
   mkdir,
@@ -451,6 +452,8 @@ export function createManager(config, dependencies = {}) {
             config.memory,
             '--pids-limit',
             String(config.pidsLimit),
+            '--log-driver',
+            'json-file',
             '--log-opt',
             'max-size=10m',
             '--log-opt',
@@ -467,7 +470,6 @@ export function createManager(config, dependencies = {}) {
       if (!id) throw new Error('Docker returned no container ID');
       if (!stopping) {
         await run('docker', ['start', id]);
-        const startedAt = Date.now();
         log(
           `slot ${slot}: started runner ${state.name}, id ${state.runnerId}, container ${id}`,
         );
@@ -476,12 +478,16 @@ export function createManager(config, dependencies = {}) {
         const watching = run('docker', ['wait', id], {
           timeout: (config.ttlSeconds + 10) * 1000,
         }).then(
-          () => {
-            exitCode = 0;
+          (output) => {
+            const status = output.trim();
+            exitCode =
+              /^\d+$/.test(status) && Number(status) <= 255
+                ? Number(status)
+                : null;
             return true;
           },
-          (error) => {
-            exitCode = error.exitCode ?? null;
+          () => {
+            exitCode = null;
             return true;
           },
         );
@@ -489,14 +495,11 @@ export function createManager(config, dependencies = {}) {
           if (await Promise.race([watching, wait(1000).then(() => false)]))
             break;
         }
-        if (exitCode !== undefined) {
-          const uptimeMs = Date.now() - startedAt;
-          if (exitCode !== 0 || uptimeMs < 30000) {
-            earlyExit = true;
-            log(
-              `slot ${slot}: runner exited before serving a job (exit ${exitCode ?? 'unknown'}, uptime ${uptimeMs} ms); next JIT request is delayed`,
-            );
-          }
+        if (exitCode !== undefined && exitCode !== 0) {
+          earlyExit = true;
+          log(
+            `slot ${slot}: runner exit status ${exitCode ?? 'unknown'}; next JIT request is delayed`,
+          );
         }
       }
     } finally {
@@ -599,7 +602,7 @@ export function createManager(config, dependencies = {}) {
 
 if (
   process.argv[1] &&
-  resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+  realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)
 ) {
   try {
     if (!process.env.GH_TOKEN?.trim()) throw new Error('GH_TOKEN is required');

scripts/ci/ephemeral-runner/manager.test.mjs

@@ -1,6 +1,14 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
-import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { spawnSync } from 'node:child_process';
+import {
+  mkdtemp,
+  mkdir,
+  readFile,
+  rm,
+  symlink,
+  writeFile,
+} from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { command as runCommand, createManager } from './manager.mjs';
@@ -116,7 +124,7 @@ async function fixture(t) {
       if (!keepContainerAfterRemove) containers.delete(args[args.length - 1]);
       return '';
     }
-    if (args[0] === 'wait') return '';
+    if (args[0] === 'wait') return '0\n';
     if (args[0] === 'start') return '';
     return '';
   }
@@ -626,6 +634,10 @@ test('passes only the docker environment allowlist and the JIT config by name',
   );
   assert.ok(!JSON.stringify(create.options.env).includes('sentinel'));
   assert.ok(!JSON.stringify(create.args).includes('sentinel'));
+  assert.equal(
+    create.args[create.args.indexOf('--log-driver') + 1],
+    'json-file',
+  );
   const at = create.args.indexOf('--env');
   assert.equal(create.args[at + 1], 'RUNNER_JIT_CONFIG');
 });
@@ -722,20 +734,23 @@ test('rejects corrupted slot state without deleting a runner', async (t) => {
   }
 });
 
-test('marks a cycle where the container exits before serving a job', async (t) => {
+test('backs off on a failed container even after the startup window', async (t) => {
   const f = await fixture(t);
+  f.config.ttlSeconds = 120;
+  let now = Date.now();
+  t.mock.method(Date, 'now', () => now);
   const command = (program, args, options) => {
-    if (program === 'docker' && args[0] === 'wait')
-      return Promise.reject(
-        Object.assign(new Error('docker failed (exit 126)'), { exitCode: 126 }),
-      );
+    if (program === 'docker' && args[0] === 'wait') {
+      now += 31000;
+      return Promise.resolve('126\n');
+    }
     return f.command(program, args, options);
   };
   const result = await createManager(f.config, { command }).cycle(1);
   assert.equal(result.earlyExit, true);
 });
 
-test('backs off before the next JIT request after an early exit', async (t) => {
+test('backs off before the next JIT request after a failed container', async (t) => {
   const f = await fixture(t);
   const previousPath = process.env.PATH;
   await writeFile(join(f.config.stateDir, 'flock'), '#!/bin/sh\nexec cat\n', {
@@ -756,10 +771,7 @@ test('backs off before the next JIT request after an early exit', async (t) => {
       jitRequests += 1;
       if (jitRequests > 1) manager.stop();
     }
-    if (program === 'docker' && args[0] === 'wait')
-      return Promise.reject(
-        Object.assign(new Error('docker failed (exit 127)'), { exitCode: 127 }),
-      );
+    if (program === 'docker' && args[0] === 'wait') return '127\n';
     return f.command(program, args, options);
   };
   manager = createManager(f.config, {
@@ -775,3 +787,36 @@ test('backs off before the next JIT request after an early exit', async (t) => {
   assert.equal(sleeps.at(-1), 60000);
   assert.equal(jitRequests, 2);
 });
+
+test('does not penalize a fast successful container', async (t) => {
+  const f = await fixture(t);
+  const result = await createManager(f.config, { command: f.command }).cycle(1);
+  assert.equal(result.earlyExit, false);
+});
+
+test('backs off when docker wait cannot determine container status', async (t) => {
+  const f = await fixture(t);
+  const command = (program, args, options) => {
+    if (program === 'docker' && args[0] === 'wait')
+      return Promise.reject(new Error('Docker disconnected'));
+    return f.command(program, args, options);
+  };
+  const result = await createManager(f.config, { command }).cycle(1);
+  assert.equal(result.earlyExit, true);
+});
+
+test('starts through a symlink and reports invalid configuration', async (t) => {
+  const f = await fixture(t);
+  const link = join(f.config.stateDir, 'manager.mjs');
+  await symlink(new URL('./manager.mjs', import.meta.url), link);
+  const result = spawnSync(
+    process.execPath,
+    [link, join(f.config.stateDir, 'missing.json')],
+    {
+      env: { ...process.env, GH_TOKEN: 'test-only' },
+      encoding: 'utf8',
+    },
+  );
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /^manager: /);
+});