diff --git a/.gitignore b/.gitignore index afa5f1c5db..d0b463a8b6 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ diagnostics/ tests/e2e/test-results/ tests/e2e/playwright-report/ tests/runner-e2e/results/ +tests/restart-chaos/results/ .env.runner-e2e.local tests/release-smoke/test-results/ tests/release-smoke/playwright-report/ diff --git a/doc/plans/2026-11-06-control-plane-restart-chaos-ledger.md b/doc/plans/2026-11-06-control-plane-restart-chaos-ledger.md new file mode 100644 index 0000000000..43ebbc312f --- /dev/null +++ b/doc/plans/2026-11-06-control-plane-restart-chaos-ledger.md @@ -0,0 +1,55 @@ +# Restart chaos ledger + +Companion to `2026-11-06-control-plane-restart-chaos.md`. One section per wave. +Classes: P product · Q agent quality · T test/harness · E unrelated environment. + +## Phase 0 — 2026-11-06 + +Master: `6cafcaf89b`. Host: macOS, local runner (Daytona image pending a CI build). + +| Check | Result | +| --- | --- | +| `controller-replacement-smoke.ts` (no agent) | PASS, 39 s, Postgres survived replacement | +| Local restart cells, first run | 0/7 — **T**: worktree `node_modules` older than lockfile (UI missing `mdast-util-from-markdown`). Not counted. | +| Local restart cells, rerun | **2/7** (below) | +| Daytona runner image | Blocked. Local build failed: stale Dockerfile lock pin, then the Docker VM ran out of disk. Moved to CI (`runner-full-stack-e2e.yml`, manual dispatch) after PR #19467. | + +### Local baseline rerun (`local-2026-11-06T14-27-42-229Z`) + +| Cell | Result | First error | Fixer | +| --- | --- | --- | --- | +| stock-harness · Codex · continuity-restart | PASS | — | — | +| stock-harness · ACPX Claude · continuity-restart | PASS | — | — | +| everyday-workflows · Codex · recover-controller | FAIL (cleanup_failure) | `kill EPERM` during owned-tree cleanup at the restart | subagent: EPERM | +| everyday-workflows · ACPX Claude · recover-controller | FAIL | Timed out waiting for an active run with saved source | subagent: Claude recover | +| stock-harness · OpenCode · continuity-restart | FAIL | Timed out waiting for the chat turn to settle to waiting | subagent: OpenCode | +| local-session-integrity · OpenCode · question-restart-resume | FAIL | Timed out waiting for the pending user question | subagent: OpenCode | +| completion-updates · Codex · handoff-completion-restart | FAIL | Timed out waiting for the worker to wait while the originating chat is idle | subagent: handoff | + +#### Signatures found + +| ID | Class | Signature | Status | +| --- | --- | --- | --- | +| RC-01 | T | Harness looked for saved source only at the workspace root; ACPX Claude saved `slugify-tool/slugify.py`, so the restart never fired | PR #19471 (open) | +| RC-02 | **P** | ACPX turn active at restart: runner closes it with `acpx_active_turn_recovery_closed` and no error, the control plane reports a generic "Provider turn failed", no continuation, queued follow-up unconsumed (fails R1). Codex has had this handled since #15297; ACPX is excluded by `provider.kind === "codex"` and `driverKind === "codex_app_server"` gates | fixer running | + +| RC-03 | T | macOS `killpg` returns EPERM (not ESRCH) on a zombie-only group; the supervisor rethrew it and aborted the restart. Fixed cell passed 2/2, and native Codex recovery itself works | PR #19470 (open) | +| RC-04 | P (latency) | Graceful shutdown always waits the full 30 s heartbeat finalizer drain when a native run is restart-suspended (`server/src/index.ts` ~L2032, `drainActiveRunExecutions`). Every graceful restart (K2) or deploy takes ≥30 s and, under the harness, escalates to SIGKILL | queued for a fixer | + +| RC-05 | **P** | Dead Retry after restart (fails R3): Retry accepted but no run for >15 min, stuck at `retryWait.reason=execution_recovery`. After the next restart the Retry vanished (`waiting_for_user`, `canRetry=false`). OpenCode t01, Claude t02 | fixer running (with RC-06) | +| RC-06 | **P** | False over-budget stop after restarts: a new issue was blocked "original recovery target is over budget" at 22¢ of 500¢, then a company `hard_stop` with `pendingRunCount=1`, `reservationCents=0`, `unpricedUsagePolicy=block` (`recovery/service.ts` ~L4826). Codex t03 never ran | fixer running (with RC-05) | +| RC-02b | **P** | OpenCode mid-turn kill also ends as a provider failure (t01). Sent to the RC-02 fixer | with RC-02 | + +### Harness smoke `w-20261106T144559-s7` (seed 7, K1 ×3, local, concurrency 2) + +0/3 pass. The harness worked: every failure is product-side (RC-02b, RC-05, RC-06). Branch `worktree-agent-a826af5179b88689b` @ `e864139cec`, not yet a PR. + +Setup note: fresh worktrees need `pnpm install --frozen-lockfile` and a built `paperclip-runnerd` (`pnpm --filter @paperclipai/paperclip-runner build`) before native cells can run. The chaos harness must check this before starting a wave. + +Classification of the remaining cells is pending each fixer's diagnosis. A timeout before the restart point counts against setup or the agent, not restart robustness. + +### Side findings + +- Nightly Runner Full-Stack E2E never builds a Daytona image. Its operating-overhead allowance (`RUNNER_E2E_OPERATING_OVERHEAD_JSON`) expired 2026-11-03, and the nightly sets `needs_daytona=false` anyway. PR #19467 (merged, 7e6d882558) names the error and removes the stale Dockerfile lock default. The nightly stays red until the allowance is renewed (Dotta's call). +- Image build moved to CI: manual dispatch run 42240434011 (`core-compatibility.runner-codex.daytona.message-marker`), 2026-11-06 15:09Z. +- `tests/runner-e2e/workflow-security.test.ts` has 5 local failures in `pr-trusted.yml` and `chat-catalog-testops-case.yml`. Not investigated. diff --git a/doc/plans/2026-11-06-control-plane-restart-chaos.md b/doc/plans/2026-11-06-control-plane-restart-chaos.md new file mode 100644 index 0000000000..42cbc65542 --- /dev/null +++ b/doc/plans/2026-11-06-control-plane-restart-chaos.md @@ -0,0 +1,282 @@ +# Control-plane restart chaos loop: make tasks survive server restarts + +Date: 2026-11-06 +Target: a local, isolated Paperclip control plane with agent runs in Daytona sandboxes +Fix workflow: `fix-failed-paperclip-run` (one session per failure signature) +Predecessor: `2026-11-04-staging-soak-loop.md` (staging soak campaign, PR worktree `paperclip-staging-test-plan-24ca37`) + +## 1. Why this plan exists + +The staging soak campaign (2026-11-04 to 2026-11-06) ran about 430 graded tasks +in waves of about 10. It used a fixed task library (sections A to H), mechanical +oracles (U1 to U9), and a dedupe-by-signature ledger. Each new failure signature +went to its own `fix-failed-paperclip-run` session. The campaign merged 42 fixes, +and the hard-failure rate on test tasks fell from 43% to about 0%. + +The largest remaining source of failures was restarts. Staging restarted up to +about five times an hour. The campaign summary says "frequent restarts stall +in-flight work". Three fixes in this area merged late in the campaign: + +- #19399: lost runner after a restart continues automatically, and Retry is offered. +- #19426: a declined Retry no longer swallows later clicks (the "dead Retry" bug). +- #19431: queued runs start promptly after a restart, and the wait shows a reason. + +The campaign could only observe restarts that staging happened to perform. We +could not choose when the restart came, what kind it was, or what each run was +doing at that moment. This plan reverses that: **we cause the restarts on +purpose**, against a local control plane whose runs execute in Daytona. We run +many tasks at once and repeat until nothing breaks. + +## 2. Success rule (the oracle) + +A run is **restart-crossing** if any part of it overlaps a control-plane fault: +its queued wake, preparation, provider turn, finalization, or a follow-up. Every +restart-crossing task must meet all of the rules below. + +| ID | Rule | Failure means | +| --- | --- | --- | +| **R1** | **Automatic continuation.** If continuing is safe, the task resumes after the server is back, with no user action, and reaches its expected resting state within the task's time budget plus the outage time. | Not robust to restart | +| **R2** | **Durable workspace.** No workspace, durability, or recovery error stays visible at rest. This includes restore, snapshot, finalize, writer-pending, lease mismatch, and stranded lock errors. If such an error appears during recovery, it clears by itself. Files the agent wrote before the fault are present afterwards, and nothing is silently rolled back. | Durability bug | +| **R3** | **Working Retry.** If the task cannot safely continue on its own, it shows exactly one Retry or continue affordance. One click (`POST /api/issues/:id/retry`, and the button in the UI) starts a run that completes the task. "Send a message to continue", a Retry that does nothing, a missing Retry, or a Retry that needs several clicks all fail this rule. | Dead or missing Retry | +| R4 | No duplicate execution. There is at most one active run per issue, no side effect is repeated, and no user message is consumed twice. This is soak oracle U4 and U5. | Replay or duplicate run | +| R5 | No silent stall. A task is never `in_progress` or `queued` with no live run, no queued wake, and no named wait reason for more than 3 minutes after `/api/liveness` reports ready. | Stall | +| R6 | Clean rest. At the end of the wave there is no pending workspace operation, recovery action, orphaned lease, or orphaned Daytona sandbox. | Leak | + +The paid "kill at an arbitrary point" probes (`recover-runner*`, see +`tests/runner-recovery/README.md`) were retired because they demanded +**automatic completion** after any kill. For a turn whose outcome is unknown, +automatic completion is not always safe. R1 and R3 resolve this: when automatic +continuation is not provably safe, the correct result is a Retry that works on +the first click. A random kill therefore always has a defined pass condition. It +is never graded "environment, ignore". + +## 3. Topology + +``` + chaos harness (Node script, no LLM) + ├─ supervisor: owns the server process (start, SIGKILL, SIGTERM, hot restart, outage hold) + ├─ launcher: creates tasks through the board API + ├─ observer: polls issues, runs and run events; grades R1–R6; captures evidence + └─ ledger: one JSONL line per task execution, plus the fault schedule seed + + local control plane (isolated PAPERCLIP_HOME, embedded Postgres, one fixed port) + └─ dials out to runnerd in each Daytona sandbox (runner ingress, port 43127) + + Daytona: one sandbox per run or lease, holding the real runner and provider (Claude, Codex) +``` + +- **Use an isolated instance**, not `~/.paperclip/instances/default`. Chaos kills + and the leftover Postgres state must never touch Dotta's working data. +- **Own the server process.** Reuse the existing supervisor in + `tests/runner-e2e/server.ts` and `controller-outage.ts` + (`replaceController`: stop → wait for unhealthy → hold → start → ready, with a + receipt). It already handles process groups, SIGKILL escalation, and + embedded-Postgres ownership. Do not use `pnpm dev` (watch mode): when its child + dies, the dev runner exits too. +- **Readiness.** Use `/api/liveness` and the startup phases + (`starting → recovering → ready`) to time the outage and the recovery window. + `startup-recovery-timing.ts` logs each phase. +- **Keep across restarts:** the Postgres data and the instance secret key. A + restart that loses either one is a different fault (K11) and is graded on its + own. +- **Concurrency.** Start with 6 concurrent tasks and raise to 12, then 20. The + limits are the Daytona sandbox quota and provider spend. Settle the quota in + Phase 0. + +## 4. Fault menu + +### 4a. What the fault does (K) + +| ID | Fault | How | +| --- | --- | --- | +| K1 | Hard kill, fast return | SIGKILL the server, restart immediately. Hot-restart reconciliation reports `not_requested`. | +| K2 | Graceful stop | SIGTERM (shutdown drain and finalizers run), then restart. | +| K3 | Hot restart | `scripts/request-hot-restart.ts`, then SIGTERM. Runs should be adopted. | +| K4 | Medium outage | Hard kill and hold for 2–5 minutes. This exceeds the 60-second controller lease but stays within the 7-minute runner reconnect grace. | +| K5 | Long outage | Hard kill and hold for 8–20 minutes. This exceeds the reconnect grace, and past 15 minutes Daytona auto-stops idle sandboxes. The runner gives up, so recovery must go through saved work. | +| K6 | Crash loop | Kill again during `recovering`, then a third time just after `ready`. | +| K7 | Kill during the automatic retry | Kill again while the automatic retry or continuation started by R1 is running. | +| K8 | Kill during a user's Retry | Kill between the Retry click (receipt committed) and the start of the new run. | +| K9 | Postgres dies too | SIGKILL embedded Postgres along with the server. | +| K10 | Network partition, no restart | Block the server's outbound traffic to Daytona for 30 seconds to 5 minutes. | +| K11 | Lost local state | Restart with the controller cache and scoped state directory deleted, as on a new container (`remote_reattach_root_missing`). The database is kept. | +| K12 | Rolling deploy | Start a second server on the same database before stopping the first, as in `rolling-deployment-supervisor.ts`. | +| K13 | Restart storm | Restart every 2–4 minutes for an hour with 20 tasks running. | + +### 4b. When the fault hits (B, run boundary) + +Some faults fire at a **random time**: a seeded random generator chooses the +time, so a failure can be replayed. Others are **boundary-triggered**: the +observer watches run events and status and fires the fault at a chosen +transition. Each boundary below lists the run event or state it fires on. + +| ID | Boundary | Fires on | +| --- | --- | --- | +| B1 | Queued, not yet dispatched | Run `queued` | +| B2 | Sandbox acquire or provision | Lease acquire started, sandbox not ready | +| B3 | Workspace restore or prepare | Restore or prepare operation in progress | +| B4 | Runner bootstrap or session open | Native bootstrap events | +| B5 | Mid model turn, streaming | First assistant output event | +| B6 | Mid long tool call | Tool-start event without a tool-end, for example `bench.sh` running about 4 minutes | +| B7 | Waiting on a question or approval | Pending interaction | +| B8 | User message or steering queued during the run | Queued comment exists | +| B9 | Turn done, finalization or sync-out in progress | `workspace_finalize` or sync operation running | +| B10 | Snapshot or archive upload | Snapshot upload begun | +| B11 | Done, before publication | Durable done, reply not yet published (as in the chat-restart cell) | +| B12 | Scheduled-retry backoff | Run `scheduled_retry` | +| B13 | Stop or interrupt settling | Cancel requested, not settled | +| B14 | Parent waiting on delegated children | Parent blocked on children, children running | +| B15 | Routine fire or timer wake | Routine run being created | +| B16 | Reassignment in flight | Assignee changed, old run not ended | + +The `finalization-crash-boundary.ts` and `controller-recovery-observer.ts` +modules in `tests/runner-e2e/` already bind kills to some of these boundaries. +Reuse them; do not write them again. + +### 4c. Workload (W) + +Use a smaller subset of the soak library, chosen so the boundaries above occur +naturally: + +- **A1:** brief written to an issue document. A short run with no repository. +- **A3:** CSV analysis whose answer can be checked exactly. +- **B1:** Python CLI with tests. Workspace files must survive the fault. The + harness runs the tests itself. +- **B2:** fix a failing test, plus a revision follow-up. This exercises workspace + reuse across the fault. +- **B5:** a long `bench.sh` run, which reaches boundary B6. +- **C1:** a comment sent while the run is active (B8). +- **C2:** Stop, then a new direction (B13). +- **A5:** the agent asks a clarifying question (B7). +- **D1:** CEO delegates to children (B14). +- **E1** or **H10:** a routine (B15). +- **C6:** reassignment (B16). + +Each task has a deterministic T-oracle, so a restart cannot hide wrong output. + +### 4d. Adapter (A) + +- **Primary:** native Claude and native Codex on Daytona. +- **Control:** the same tasks with a local runner. This tells apart failures + that are Daytona-specific from failures in control-plane recovery. +- **Later:** other adapters, once the primary pair is green. + +The full cross product is about 13 × 16 × 11 × 2. That is too large to run +exhaustively. Waves therefore sample **pairwise coverage**: every fault paired +with every boundary, and every boundary on every adapter, at least N times. The +remaining slots go to open signatures and random K1 and K13 storms. + +## 5. Harness (`tests/restart-chaos/`) + +The harness is a script, not an LLM, for the same reasons as the soak harness: +deterministic grading and cheap polling. It reuses the runner-e2e process code. +It does not copy that code. + +1. **`config`:** isolated instance path, port, `DAYTONA_API_KEY`, company, + agents, budget, and concurrency. +2. **`seed`:** creates the "Chaos Lab" company, Claude and Codex agents on + Daytona, and the `chaos-sandbox` project with a fixture repository. This + includes `bench.sh` and one failing test. It sets hard budgets. +3. **`supervisor`:** wraps `server.ts` and `controller-outage.ts`, exposing + `kill(kind)`, `hold(ms)`, `start()`, `waitReady()`, `partition(ms)` (for + example a pf or iptables rule, or a proxy in front of runner ingress), and + `wipeLocalState()`. Every action is written to the timeline. +4. **`scheduler`:** a seeded plan of faults and boundaries for each wave. The + seed and the chosen cells are saved, so `--replay /` repeats the + same fault at the same boundary. +5. **`observer`:** polls every 2–5 seconds and evaluates R1–R6 and the + T-oracles. When a task ends in the R3 state, the observer acts as a user + would: it calls Retry once through the API, the same call the UI hook makes + in `useTaskRetry.ts`, and grades the result. +6. **`ui-probe`:** every few waves, a Playwright pass opens one task that is in + the R3 state, checks that the Retry button is visible, clicks it once, and + confirms the task continues. The dead-Retry bug (#19426) was in the UI path, + so checking the API alone is not enough. +7. **`evidence bundle`** (captured at failure time): run IDs and statuses, the + last 200 run events for each run, the server log around the fault, the + startup-phase timings, the recovery claim dispositions + (`claimNativeRestartRecoveries`), workspace operations, leases and their + Daytona sandbox state, the fault timeline, the seed, and the SHA. +8. **`ledger` and summary:** the soak ledger format, plus the fault, boundary + and adapter fields. A small dashboard artifact shows pass rate per cell and + the open signatures. +9. **`sweep`:** at the end of each wave, reconcile Daytona sandboxes against + leases (R6). Then destroy whatever the harness created. + +## 6. Triage and the fix loop + +``` +wave (N tasks × sampled K/B/A cells) + → grade R1–R6 and the T-oracles + → classify failures; signature = component + first causal error + boundary + → new P signature → fixer session running /fix-failed-paperclip-run with the bundle and a replay command + → known signature → add the bundle to the existing record + → fix PR merged → rebase the harness worktree on master, replay that signature 5× plus its neighboring cells +``` + +- **Classes:** the same as the soak campaign. **P** is a product bug, **Q** is + agent quality, **T** is a test or harness defect. **E** (environment) never + applies to the fault we injected. Only an unrelated provider or Daytona outage + counts as E. +- **Fixers start from the replay.** Each fixer gets `--replay`, so the first + step of `fix-failed-paperclip-run` (establish the incident) is already done. + The fixer then writes the smallest red test at the right layer. Usually that + is a server integration test on embedded Postgres, like the six in #19399, or + a `native-runner-restart-*.integration.test.ts`. The fix makes it green. +- **Merge authority:** the same as the soak campaign. Fixer sessions merge their + own PR at Greptile 5/5 with all required checks green, and run `check-pr`. +- **Concurrency:** at most 4 fixer sessions at once. +- **No deploy lag.** Fixes are retested locally as soon as they merge, which is + the main speed advantage over the staging loop. Staging stays the final + confirmation (Phase 4). + +## 7. Phases + +| Phase | Output | Exit | +| --- | --- | --- | +| **0. Baseline** | Isolated instance, Daytona key and quota confirmed, Chaos Lab seeded. Run the existing `everyday-workflows.runner-codex.daytona.recover-controller` cell and `controller-replacement-smoke.ts` as they are today. Also run 10 tasks by hand with a K1 kill each, to measure the starting failure rate. | The baseline failure rate is recorded and the first signatures are filed. | +| **1. Harness MVP** | Supervisor, launcher, observer with R1–R6, ledger and replay. Faults K1, K2 and K4 at random times. Workloads A1, B1, B5 and C1 on Claude and Codex. 6 concurrent tasks. | Two waves run without supervision, and the evidence bundles are good enough for a fixer to start from. | +| **2. Boundary matrix** | Boundary triggers B1–B16 and the full K list except K12 and K13. Pairwise sampling. UI probe. | Every K×B pair has been run at least 3 times. | +| **3. Stress** | Concurrency 12, then 20. K13 storms (an hour of restarts every 2–4 minutes). K12 rolling deploys. Overnight runs. | Finish line (§8). | +| **4. Lock in** | Each fixed signature has a red/green regression test in its PR. Stable cells are promoted to explicit runner-e2e cells (`recover-controller` style) for release qualification. Then one confirmation wave on staging. | Recurring regressions are caught in CI, not by chaos. | + +## 8. Finish line + +At one pinned master SHA: + +1. **500 consecutive restart-crossing task executions** across the sampled + matrix with **zero R1–R6 failures**. At least 20 of them cross K5 (the long + outage) and at least 20 cross K6 or K7 (repeat kills). +2. Every K×B cell has at least 5 passes on both Claude and Codex. +3. Two K13 storm hours at concurrency 20 in which every task reaches its + resting state, and the R3 Retry works on the first click every time. +4. No open P signature remains in the ledger. +5. One staging wave after deploy confirms the result. There, U9 sees no restart + failure across all restarts in a 24-hour window. + +## 9. Cost and safety + +- Hard budget per wave (start at $30) and per night (start at $150). The harness + refuses to start a wave above 80% of the cap. +- The harness signals only processes it started, using the PID and start-time + checks in `process-tree-owner.ts`. It never touches the default instance or + any other dev server. +- The Daytona cleanup sweep runs after every wave and on harness exit. Orphaned + sandboxes are counted as R6 failures before they are deleted. +- No real credentials are placed in fixtures. The canary secret check (soak U7) + stays on, because recovery paths replay logs. + +## 10. Decisions needed from Dotta + +1. **Daytona quota and concurrency ceiling:** is 20 concurrent sandboxes OK, or + is the cap lower? +2. **Budget:** are $30 per wave and $150 per night acceptable caps? +3. **K5 bar (outage longer than the 7-minute reconnect grace):** is a working + Retry an acceptable outcome, or must the task continue automatically from + saved work? The plan assumes automatic continuation is required when the + last provider turn settled, and Retry is acceptable when the turn was in + flight. This matches the #19399 rule. +4. **Adapters for the finish line:** Claude and Codex native only, or also the + ACPX adapters? +5. **K12 rolling deploys:** include them now, or wait until zero-downtime web + deploys (`2026-10-31-zero-downtime-web-deploys.md`) is the production path? diff --git a/scripts/typecheck-test-projects.mjs b/scripts/typecheck-test-projects.mjs index 0c6d537b21..3ef20da796 100644 --- a/scripts/typecheck-test-projects.mjs +++ b/scripts/typecheck-test-projects.mjs @@ -24,6 +24,7 @@ export const PROJECTS = [ "server/src/__tests__/helpers/maintenance", "tests/e2e", "tests/lifecycle-baseline", + "tests/restart-chaos", "tests/runner-e2e", "tests/speko-native", ]; diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md new file mode 100644 index 0000000000..393d5fe984 --- /dev/null +++ b/tests/restart-chaos/README.md @@ -0,0 +1,71 @@ +# Restart chaos harness (Phase 1 MVP) + +Plan: [`doc/plans/2026-11-06-control-plane-restart-chaos.md`](../../doc/plans/2026-11-06-control-plane-restart-chaos.md). + +A script, not an LLM. It starts an isolated Paperclip control plane (temporary +`PAPERCLIP_HOME`, wrapper-owned embedded Postgres, its own port), seeds a +"Chaos Lab" company, launches real agent tasks, kills or restarts the control +plane during those tasks, and grades every task on R1–R6 plus a deterministic +T-oracle. + +## Run + +```sh +pnpm install --frozen-lockfile +pnpm --filter @paperclipai/paperclip-runner build:runner-binaries # native runner (debug) +set -a; source ~/.secrets; set +a # provider keys, process env only +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude,opencode --concurrency 4 --tasks 8 --faults K1,K2,K4 --seed 42 +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --replay w-20261106T101112-s42/t03 +pnpm exec vitest run --config tests/restart-chaos/vitest.config.ts # credential-free unit tests +node scripts/typecheck-test-projects.mjs tests/restart-chaos # typecheck +``` + +Options: `--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap, +default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance). +`--env daytona` is reserved; it fails until the Daytona image and environment +are wired (see the TODO in `seed.ts`). + +## Pieces + +| File | Role | +| --- | --- | +| `supervisor.ts` | Owns the server. Reuses `tests/runner-e2e` process code (config, ports, env isolation, wrapper-owned Postgres, process-tree stopper, `replaceController`). Faults: K1 SIGKILL + immediate restart, K2 SIGTERM + restart, K4 SIGKILL + 2–5 min hold. The controller runtime cache is kept (discarding it is K11). | +| `seed.ts` | Chaos Lab company, encrypted provider secrets, native Codex / ACPX Claude / OpenCode agents from the runner-e2e catalog, and the `chaos-sandbox` project (git repo: python CLI, one failing test, `bench.sh` ≈ 4 min) with isolated git-worktree workspaces. | +| `scheduler.ts` | Seeded plan: workload × adapter × fault cells and a random fire time per task. Saved to `results//plan.json`. `--replay` reruns one task alone with the same choices and fires the recorded sequence of faults that crossed it (other tasks' faults included) at the same offsets from launch. | +| `observer.ts` | Polls each issue and its runs. When a task rests needing Retry it calls `POST /api/issues/:id/retry` once, with the body `useTaskRetry.ts` sends. Collects T-oracle artifacts and the R6 sweep. | +| `grader.ts` | Pure R1–R6 grading over the samples. | +| `workloads.ts` | A1 brief document, B1 python CLI with tests, B5 long bench, C1 comment while running, with their T-oracles. | +| `evidence.ts`, `ledger.ts` | Evidence bundle for each failure and the JSONL ledger plus `SUMMARY.md`. | + +## Output (gitignored) + +`tests/restart-chaos/results/` (override with `RESTART_CHAOS_RESULTS_DIR`): + +- `ledger.jsonl`, `SUMMARY.md` — every graded task, per-cell pass rate, open signatures. +- `/plan.json`, `timeline.jsonl`, `server.log`, `faults.json`, `.grade.json`. +- `/evidence//` on failure — run ids and statuses, last 200 events per + run, workspace operations, server log window around each fault, startup phase + timings, fault timeline, seed, git SHA and the replay command. Known provider + key values are redacted. + +## Grading notes + +- A task is restart-crossing when a run overlaps a fault window or the task was + not yet at rest when the fault started. Non-crossing tasks are still graded. +- R2 also checks that workspace files present before the first fault still exist at rest. +- B5 passes only when `bench.sh` wrote its own completion record showing it + ran, and the harness itself saw at least three `.bench/progress` values + while polling. This is best-effort against a deliberately adversarial agent. +- R4 allows at most one active (queued, running or scheduled-retry) run per + sample. A wake comment may be served again only after a run that did not + succeed. +- T-oracle commands (B1 tests and CLI) run agent-editable code. They run under + a macOS Seatbelt sandbox with a scratch `HOME`, no network and no read access + to the operator's home directory. On other platforms they are refused unless + `RESTART_CHAOS_ALLOW_UNSANDBOXED_ORACLE=1` is set inside a disposable VM. +- R1 allows the workload budget plus outage time. When a Retry was needed R1 is + `n/a` and R3 decides; a task that rests unfinished with no Retry fails both. +- Faults are serialized; each one hits every task running at that moment. +- A `todo` or `in_progress` task with no live run and no named wait is not a rest. The observer keeps watching it until it resumes or passes the R5 stall limit. +- A fault that fails to complete fails the wave as a harness error (exit 2), even when recovery brings the server back. diff --git a/tests/restart-chaos/api.ts b/tests/restart-chaos/api.ts new file mode 100644 index 0000000000..2a37b89d71 --- /dev/null +++ b/tests/restart-chaos/api.ts @@ -0,0 +1,31 @@ +/** Minimal board API client for an isolated local_trusted instance. */ +export class ChaosApiError extends Error { + constructor(readonly status: number | null, message: string) { super(message); this.name = "ChaosApiError"; } +} + +export class ChaosApi { + constructor(readonly baseUrl: string, private readonly timeoutMs = 10_000) {} + + private async request(method: string, route: string, body?: unknown, timeoutMs = this.timeoutMs): Promise { + let response: Response; + try { + response = await fetch(`${this.baseUrl}${route}`, { + method, + headers: { origin: this.baseUrl, ...(body === undefined ? {} : { "content-type": "application/json" }) }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + cache: "no-store", + }); + } catch (error) { + throw new ChaosApiError(null, `${method} ${route} unreachable: ${error instanceof Error ? error.message : String(error)}`); + } + const text = await response.text(); + if (!response.ok) throw new ChaosApiError(response.status, `${method} ${route} returned ${response.status}: ${text.slice(0, 500)}`); + return (text ? JSON.parse(text) : null) as T; + } + + get(route: string, timeoutMs?: number) { return this.request("GET", route, undefined, timeoutMs); } + post(route: string, body?: unknown, timeoutMs?: number) { return this.request("POST", route, body ?? {}, timeoutMs); } + put(route: string, body?: unknown) { return this.request("PUT", route, body ?? {}); } + patch(route: string, body?: unknown) { return this.request("PATCH", route, body ?? {}); } +} diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts new file mode 100644 index 0000000000..05479e6334 --- /dev/null +++ b/tests/restart-chaos/config.ts @@ -0,0 +1,108 @@ +import { parseArgs } from "node:util"; +import { + ADAPTER_IDS, ENVIRONMENT_KINDS, FAULT_KINDS, WORKLOAD_IDS, + type AdapterId, type EnvironmentKind, type FaultKind, type WaveOptions, type WorkloadId, +} from "./types.js"; + +export const CREDENTIAL_FOR_ADAPTER: Record = { + codex: "OPENAI_API_KEY", + claude: "ANTHROPIC_API_KEY", + opencode: "OPENROUTER_API_KEY", +}; + +/** Catalog profile reused from tests/runner-e2e/catalog.ts for each adapter. */ +export const PROFILE_FOR_ADAPTER: Record = { + codex: "runner-codex", + claude: "runner-acpx-claude", + opencode: "runner-opencode", +}; + +export type ParsedCommand = + | { mode: "wave"; options: WaveOptions; keep: boolean } + | { mode: "replay"; ref: ReplayRef; keep: boolean; overrides: Partial> }; + +export interface ReplayRef { waveId: string; taskId: string } + +const WAVE_ID = /^w-[0-9]{8}T[0-9]{6}-s[0-9]+(?:-replay-t[0-9]{2,3}-[0-9]+)?$/; +const TASK_ID = /^t[0-9]{2,3}$/; + +export function parseReplayRef(value: string): ReplayRef { + const trimmed = value.trim().replace(/\/+$/, ""); + const parts = trimmed.split("/"); + if (parts.length !== 2) throw new Error(`--replay expects /, got "${value}"`); + const [waveId, taskId] = parts as [string, string]; + if (!WAVE_ID.test(waveId)) throw new Error(`Invalid wave id "${waveId}"`); + if (!TASK_ID.test(taskId)) throw new Error(`Invalid task id "${taskId}"`); + return { waveId, taskId }; +} + +function list(raw: string | undefined, allowed: readonly T[], name: string, fallback: readonly T[]): T[] { + if (raw === undefined) return [...fallback]; + const values = raw.split(",").map(value => value.trim()).filter(Boolean); + if (!values.length) throw new Error(`--${name} must not be empty`); + for (const value of values) if (!allowed.includes(value as T)) throw new Error(`Unknown ${name} "${value}"; allowed: ${allowed.join(", ")}`); + return [...new Set(values)] as T[]; +} + +function integer(raw: string | undefined, name: string, fallback: number, min: number, max: number): number { + if (raw === undefined) return fallback; + if (!/^[0-9]+$/.test(raw)) throw new Error(`--${name} must be a non-negative integer`); + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < min || value > max) throw new Error(`--${name} must be between ${min} and ${max}`); + return value; +} + +export function parseCommand(argv: readonly string[], now = Date.now()): ParsedCommand { + // Tolerate a forwarded literal `--` (for example from a package runner). + const args = argv[0] === "--" ? argv.slice(1) : [...argv]; + const { values } = parseArgs({ + args, + strict: true, + options: { + env: { type: "string" }, + adapters: { type: "string" }, + workloads: { type: "string" }, + faults: { type: "string" }, + concurrency: { type: "string" }, + tasks: { type: "string" }, + seed: { type: "string" }, + "budget-cents": { type: "string" }, + "poll-ms": { type: "string" }, + replay: { type: "string" }, + keep: { type: "boolean" }, + }, + }); + const keep = values.keep === true; + const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); + const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); + if (values.replay !== undefined) { + for (const key of ["env", "adapters", "workloads", "faults", "concurrency", "tasks", "seed"] as const) { + if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); + } + return { mode: "replay", ref: parseReplayRef(values.replay), keep, + overrides: { ...(values["poll-ms"] ? { pollMs } : {}), ...(values["budget-cents"] ? { budgetCents } : {}) } }; + } + const env = (values.env ?? "local") as EnvironmentKind; + if (!ENVIRONMENT_KINDS.includes(env)) throw new Error(`Unknown env "${env}"; allowed: ${ENVIRONMENT_KINDS.join(", ")}`); + const options: WaveOptions = { + env, + adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), + workloads: list(values.workloads, WORKLOAD_IDS, "workloads", WORKLOAD_IDS), + faults: list(values.faults, FAULT_KINDS, "faults", ["K1", "K2", "K4"]), + concurrency: integer(values.concurrency, "concurrency", 4, 1, 20), + tasks: integer(values.tasks, "tasks", 8, 1, 200), + seed: integer(values.seed, "seed", now % 2_147_483_647, 0, 2_147_483_647), + budgetCents, + pollMs, + }; + return { mode: "wave", options, keep }; +} + +export function waveIdFor(seed: number, at = new Date()): string { + const stamp = at.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, ""); + return `w-${stamp}-s${seed}`; +} + +export function replayCommand(waveId: string, taskId: string): string { + return `node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --replay ${waveId}/${taskId}`; +} diff --git a/tests/restart-chaos/evidence.ts b/tests/restart-chaos/evidence.ts new file mode 100644 index 0000000000..67721b1cc9 --- /dev/null +++ b/tests/restart-chaos/evidence.ts @@ -0,0 +1,104 @@ +import { mkdir, open, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { ChaosApi } from "./api.js"; +import { replayCommand } from "./config.js"; +import type { TimelineEntry } from "./timeline.js"; +import type { FaultReceipt, OracleArtifacts, TaskGrade, TaskRecord, WavePlan } from "./types.js"; + +const LOG_BEFORE_BYTES = 128 * 1024; +const LOG_AFTER_BYTES = 512 * 1024; +const EVENTS_PER_RUN = 200; +const OBSERVER_POLL_LINE = /INFO\S*: \S*(?:GET|HEAD) \/(?:issues\/[0-9a-f-]+(?:\/runs)?|ready|liveness|health) 200/; + +type Json = Record; + +/** Canary (soak U7): never let a provider key reach an evidence file. */ +export function redactKnownSecrets(text: string, secrets: readonly string[]): string { + let out = text; + for (const value of secrets) if (value.length >= 8) out = out.split(value).join("[REDACTED]"); + return out; +} + +export function knownSecretValues(env: NodeJS.ProcessEnv = process.env): string[] { + // Any *KEY, *TOKEN, *SECRET or *PASSWORD name, including PAPERCLIP_AGENT_PRIVATE_KEY and AWS_SECRET_ACCESS_KEY. + return Object.entries(env).filter(([key, value]) => value && /(?:KEY|TOKEN|SECRET|PASSWORD)$/.test(key)).map(([, value]) => value!); +} + +async function lastEvents(api: ChaosApi, runId: string): Promise { + const kept: Json[] = []; + let afterSeq = 0; + for (let page = 0; page < 200; page += 1) { + const rows = await api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=500`); + if (!rows?.length) break; + kept.push(...rows); + if (kept.length > EVENTS_PER_RUN) kept.splice(0, kept.length - EVENTS_PER_RUN); + const next = Number(rows[rows.length - 1]?.seq); + if (!Number.isFinite(next) || next <= afterSeq || rows.length < 500) break; + afterSeq = next; + } + return kept; +} + +async function logWindow(serverLogPath: string, faults: readonly FaultReceipt[]): Promise { + if (!faults.length) return ""; + const handle = await open(serverLogPath, "r").catch(() => null); + if (!handle) return ""; + try { + const size = (await handle.stat()).size; + const parts: string[] = []; + for (const fault of faults) { + const start = Math.max(0, fault.serverLogOffsetAtStart - LOG_BEFORE_BYTES); + const end = Math.min(size, (fault.serverLogOffsetAtReady ?? fault.serverLogOffsetAtStart) + LOG_AFTER_BYTES); + const buffer = Buffer.alloc(Math.max(0, end - start)); + await handle.read(buffer, 0, buffer.length, start); + // Drop the observer's own polling request lines; they bury the recovery log. + const text = buffer.toString("utf8").split("\n").filter(line => !OBSERVER_POLL_LINE.test(line)).join("\n"); + parts.push(`===== ${fault.faultId} ${fault.kind} bytes ${start}-${end} (observer GET lines removed) =====\n${text}`); + } + return parts.join("\n"); + } finally { await handle.close(); } +} + +/** Bundle everything a fix-failed-paperclip-run session needs to start (plan §5.7). */ +export async function writeEvidenceBundle(input: { + directory: string; api: ChaosApi; plan: WavePlan; record: TaskRecord; grade: TaskGrade; artifacts: OracleArtifacts | null; + faults: readonly FaultReceipt[]; timeline: readonly TimelineEntry[]; serverLogPath: string; gitSha: string; gitDirty: boolean; + issue: Json | null; secrets: readonly string[]; +}): Promise { + const { directory, api, record } = input; + await mkdir(directory, { recursive: true, mode: 0o700 }); + const write = (name: string, value: unknown) => writeFile(path.join(directory, name), + redactKnownSecrets(typeof value === "string" ? value : `${JSON.stringify(value, null, 2)}\n`, input.secrets), { mode: 0o600 }); + const errors: string[] = []; + const attempt = async (label: string, work: () => Promise) => { + try { await work(); } catch (error) { errors.push(`${label}: ${error instanceof Error ? error.message : String(error)}`); } + }; + const relevant = input.faults.filter(fault => input.grade.crossingFaults.includes(fault.faultId) || record.faultIds.includes(fault.faultId)); + await write("meta.json", { + schema: "paperclip.restart-chaos.evidence/v1", waveId: input.plan.waveId, seed: input.plan.seed, taskId: record.plan.taskId, + gitSha: input.gitSha, gitDirty: input.gitDirty, signature: input.grade.signature, replay: replayCommand(input.plan.waveId, record.plan.taskId), + issueId: record.issueId, identifier: record.identifier, task: record.plan, options: input.plan.options, + }); + await write("grade.json", input.grade); + await write("task-record.json", record); + await write("faults.json", relevant.length ? relevant : input.faults); + await write("startup-phases.json", (relevant.length ? relevant : input.faults).map(fault => ({ faultId: fault.faultId, kind: fault.kind, + startedAt: fault.startedAt, stoppedAt: fault.stoppedAt, restartedAt: fault.restartedAt, healthyAt: fault.healthyAt, readyAt: fault.readyAt, phases: fault.phases }))); + await write("timeline.json", input.timeline); + if (input.artifacts) await write("oracle-artifacts.json", input.artifacts); + await attempt("server-log", async () => write("server-log-window.txt", await logWindow(input.serverLogPath, relevant.length ? relevant : input.faults))); + if (record.issueId) { + await attempt("issue", async () => write("issue.json", input.issue ?? await api.get(`/api/issues/${record.issueId}`))); + let runs: Json[] = []; + await attempt("runs", async () => { runs = await api.get(`/api/issues/${record.issueId}/runs`); await write("runs.json", runs); }); + for (const run of runs) { + const runId = String(run.runId ?? run.id); + await attempt(`events ${runId}`, async () => write(`run-${runId}-events.json`, await lastEvents(api, runId))); + await attempt(`workspace-operations ${runId}`, async () => write(`run-${runId}-workspace-operations.json`, + await api.get(`/api/heartbeat-runs/${runId}/workspace-operations`))); + } + await attempt("comments", async () => write("comments.json", await api.get(`/api/issues/${record.issueId}/comments`))); + } + if (errors.length) await write("collection-errors.json", errors); + return directory; +} diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts new file mode 100644 index 0000000000..97870f866d --- /dev/null +++ b/tests/restart-chaos/grader.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; +import { crossingFaults, gradeR3, gradeR4, gradeR5, gradeTask, STALL_LIMIT_MS } from "./grader.js"; +import { isResting } from "./observer.js"; +import type { FaultReceipt, OracleArtifacts, RunSample, TaskRecord, TaskSample } from "./types.js"; +import { BENCH_EXPECTED, parseBenchRecord, workloadDefinitions } from "./workloads.js"; + +const T0 = Date.parse("2026-11-06T12:00:00Z"); +const iso = (offsetMs: number) => new Date(T0 + offsetMs).toISOString(); + +function run(id: string, status: string, created: number, finished: number | null = null, extra: Partial = {}): RunSample { + return { id, status, createdAt: iso(created), startedAt: iso(created + 500), finishedAt: finished === null ? null : iso(finished), + errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null, ...extra }; +} + +function sample(at: number, issueStatus: string, runs: RunSample[], extra: Partial = {}): TaskSample { + return { at: T0 + at, reachable: true, serverReady: true, issueStatus, interruption: null, activeRecoveryAction: false, + executionBlocker: false, scheduledRetry: false, runs, ...extra }; +} + +function record(samples: TaskSample[], extra: Partial = {}): TaskRecord { + return { + plan: { taskId: "t01", index: 0, workload: "A1", adapter: "codex", fault: "K1", fireAfterMs: 20_000, holdMs: 0, followUpAfterMs: null, nonce: "abcd1234" }, + issueId: "issue-1", identifier: "CHA-1", createdAt: T0, launchError: null, samples, retry: null, restAt: T0 + 120_000, + finalStatus: "done", deadlineAt: T0 + 600_000, timedOut: false, followUp: null, faultIds: ["f01"], + sweep: { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: null }, + preFaultFiles: null, lostFiles: null, ...extra, + }; +} + +const fault: FaultReceipt = { faultId: "f01", kind: "K1", requestedBy: "t01", signal: "SIGKILL", holdMs: 0, requestedAt: T0 + 20_000, + startedAt: T0 + 20_000, stoppedAt: T0 + 21_000, restartedAt: T0 + 21_000, healthyAt: T0 + 40_000, readyAt: T0 + 45_000, + phases: [{ phase: "unreachable", at: T0 + 21_000 }, { phase: "recovering", at: T0 + 38_000 }, { phase: "ready", at: T0 + 45_000 }], + serverLogOffsetAtStart: 0, serverLogOffsetAtReady: 100, error: null }; + +const passingBrief: OracleArtifacts = { issueStatus: "done", documents: { brief: "CHAOS-abcd1234\n- one\n- two\n- three\n" }, + agentComments: [], files: {}, commands: {}, workspacePath: null }; +const oracle = (artifacts = passingBrief) => workloadDefinitions.A1.grade("abcd1234", artifacts); + +describe("R1–R6 grading on fixture timelines", () => { + it("passes a task that auto-continued across a K1 kill", () => { + const samples = [ + sample(3_000, "in_progress", [run("r1", "running", 1_000)]), + { at: T0 + 25_000, reachable: false, serverReady: false }, + sample(50_000, "in_progress", [run("r1", "failed", 1_000, 30_000), run("r2", "running", 48_000)]), + sample(110_000, "done", [run("r1", "failed", 1_000, 30_000), run("r2", "succeeded", 48_000, 100_000)]), + ]; + const grade = gradeTask(record(samples), [fault], workloadDefinitions.A1.budgetMs, oracle()); + expect(grade.crossed).toBe(true); + expect(grade.crossingFaults).toEqual(["f01"]); + expect(grade.rules.R1.verdict).toBe("pass"); + expect(grade.rules.R3.verdict).toBe("n/a"); + expect(grade.pass).toBe(true); + expect(grade.signature).toBeNull(); + }); + + it("does not count a task that rested before the fault as restart-crossing", () => { + const samples = [sample(3_000, "done", [run("r1", "succeeded", 1_000, 10_000)])]; + expect(crossingFaults(record(samples, { restAt: T0 + 12_000 }), [fault])).toEqual([]); + }); + + it("fails R1 and R3 when a task rests unfinished with no Retry offered", () => { + const samples = [sample(60_000, "in_progress", [run("r1", "failed", 1_000, 30_000)])]; + const grade = gradeTask(record(samples, { finalStatus: "in_progress" }), [fault], workloadDefinitions.A1.budgetMs, + oracle({ ...passingBrief, issueStatus: "in_progress", documents: { brief: null } })); + expect(grade.rules.R1).toMatchObject({ verdict: "fail" }); + expect(grade.rules.R1.detail).toMatch(/rested_not_done/); + expect(grade.rules.R3.detail).toMatch(/missing_retry/); + expect(grade.signature).toBe("R1:rested_not_done:A1:K1"); + }); + + it("passes R3 when one Retry starts a run that completes the task", () => { + const samples = [ + sample(60_000, "blocked", [run("r1", "failed", 1_000, 30_000)], { interruption: { state: "failed", canRetry: true, retryWait: false, workspaceFailure: null, observedRunId: "r1", userAction: null } }), + sample(80_000, "in_progress", [run("r1", "failed", 1_000, 30_000), run("r2", "running", 75_000, null, { retryOfRunId: "r1" })]), + sample(140_000, "done", [run("r1", "failed", 1_000, 30_000), run("r2", "succeeded", 75_000, 130_000)]), + ]; + const retried = record(samples, { retry: { at: T0 + 74_000, observedRunId: "r1", httpStatus: 200, accepted: true, error: null }, restAt: T0 + 140_000 }); + expect(gradeR3(retried)).toMatchObject({ verdict: "pass" }); + const grade = gradeTask(retried, [fault], workloadDefinitions.A1.budgetMs, oracle()); + expect(grade.rules.R1.verdict).toBe("n/a"); + expect(grade.pass).toBe(true); + }); + + it("fails R3 for a dead Retry (accepted but no new run)", () => { + const samples = [sample(60_000, "blocked", [run("r1", "failed", 1_000, 30_000)])]; + const dead = record(samples, { finalStatus: "blocked", retry: { at: T0 + 74_000, observedRunId: "r1", httpStatus: 200, accepted: true, error: null } }); + expect(gradeR3(dead).detail).toBe("retry_no_new_run"); + const rejected = record(samples, { finalStatus: "blocked", retry: { at: T0 + 74_000, observedRunId: "r1", httpStatus: 409, accepted: null, error: "conflict" } }); + expect(gradeR3(rejected).detail).toMatch(/^retry_rejected/); + }); + + it("fails R4 for two concurrent running runs and for a comment consumed twice", () => { + expect(gradeR4(record([sample(10_000, "in_progress", [run("r1", "running", 1_000), run("r2", "running", 5_000)])])).detail).toMatch(/concurrent_runs/); + const twice = [sample(90_000, "done", [ + run("r1", "succeeded", 1_000, 40_000, { wakeCommentIds: ["c1"] }), + run("r2", "succeeded", 50_000, 80_000, { wakeCommentIds: ["c1"] }), + ])]; + expect(gradeR4(record(twice)).detail).toMatch(/comment_consumed_twice: c1/); + // Two active runs that are not both running also fail. + expect(gradeR4(record([sample(10_000, "in_progress", [run("r1", "queued", 1_000), run("r2", "scheduled_retry", 5_000)])])).detail).toMatch(/concurrent_runs/); + // A failed run's comment may be served again by the retry; a successful one may not. + const retried = [sample(90_000, "done", [ + run("r1", "failed", 1_000, 40_000, { wakeCommentIds: ["c1"] }), + run("r2", "succeeded", 50_000, 80_000, { wakeCommentIds: ["c1"], retryOfRunId: "r1" }), + ])]; + expect(gradeR4(record(retried)).verdict).toBe("pass"); + const afterSuccess = [sample(90_000, "done", [ + run("r1", "succeeded", 1_000, 40_000, { wakeCommentIds: ["c1"] }), + run("r2", "failed", 50_000, 80_000, { wakeCommentIds: ["c1"] }), + ])]; + expect(gradeR4(record(afterSuccess)).detail).toMatch(/comment_consumed_twice: c1 by r1,r2/); + }); + + it("fails R5 on a silent stall longer than three minutes after ready, but not during outages or named waits", () => { + const idle = (at: number, extra: Partial = {}) => sample(at, "in_progress", [run("r1", "failed", 1_000, 30_000)], extra); + const stalled = [idle(60_000), idle(60_000 + STALL_LIMIT_MS / 2), idle(60_000 + STALL_LIMIT_MS + 5_000)]; + expect(gradeR5(record(stalled))).toMatchObject({ verdict: "fail" }); + const outage = [idle(60_000), { at: T0 + 120_000, reachable: false, serverReady: false }, idle(200_000), idle(300_000)]; + expect(gradeR5(record(outage))).toMatchObject({ verdict: "pass" }); + const named = stalled.map(entry => ({ ...entry, scheduledRetry: true })); + expect(gradeR5(record(named))).toMatchObject({ verdict: "pass" }); + }); + + it("fails R2 when a workspace failure stays visible and R6 when work is left pending", () => { + const failure = { state: "failed", canRetry: false, retryWait: false, workspaceFailure: "snapshot restore failed", observedRunId: "r1", userAction: null }; + const grade = gradeTask(record([sample(90_000, "done", [run("r1", "succeeded", 1_000, 80_000)], { interruption: failure })]), [fault], 600_000, oracle()); + expect(grade.rules.R2.detail).toMatch(/workspace_failure_visible_at_rest/); + const leaky = record([sample(90_000, "done", [run("r1", "succeeded", 1_000, 80_000)])], + { sweep: { activeRuns: [], runningWorkspaceOperations: ["op1"], activeRecoveryAction: true, error: null } }); + expect(gradeTask(leaky, [fault], 600_000, oracle()).rules.R6.detail).toMatch(/running_workspace_operations=op1; active_recovery_action/); + }); + + it("fails R2 when files written before the fault are gone at rest", () => { + const done = [sample(90_000, "done", [run("r1", "succeeded", 1_000, 80_000)])]; + const lost = record(done, { preFaultFiles: ["NOTES.md", "chaoscli/cli.py"], lostFiles: ["NOTES.md"] }); + expect(gradeTask(lost, [fault], 600_000, oracle()).rules.R2.detail).toBe("pre_fault_files_lost: NOTES.md"); + expect(gradeTask(record(done, { preFaultFiles: ["NOTES.md"], lostFiles: [] }), [fault], 600_000, oracle()).rules.R2.verdict).toBe("pass"); + }); + + it("fails R2 visibly when the pre-fault snapshot could not run", () => { + const done = [sample(90_000, "done", [run("r1", "succeeded", 1_000, 80_000)])]; + const unchecked = record(done, { preFaultSnapshotError: "EACCES" }); + expect(gradeTask(unchecked, [fault], 600_000, oracle()).rules.R2.detail).toMatch(/^pre_fault_snapshot_failed/); + }); + + it("allows outage time on top of the budget", () => { + const k4: FaultReceipt = { ...fault, kind: "K4", holdMs: 240_000, readyAt: T0 + 290_000 }; + const late = record([sample(700_000, "done", [run("r2", "succeeded", 300_000, 690_000)])], { restAt: T0 + 700_000 }); + expect(gradeTask(late, [k4], 600_000, oracle()).rules.R1.verdict).toBe("pass"); + expect(gradeTask(late, [], 600_000, oracle()).rules.R1.detail).toMatch(/done_too_late/); + }); +}); + +describe("T-oracles", () => { + it("grades A1 and C1 markers", () => { + expect(workloadDefinitions.A1.grade("abcd1234", passingBrief).verdict).toBe("pass"); + expect(workloadDefinitions.C1.grade("abcd1234", passingBrief).checks.find(check => check.name.includes("followup"))?.ok).toBe(false); + expect(workloadDefinitions.C1.grade("abcd1234", { ...passingBrief, documents: { brief: `${passingBrief.documents.brief}FOLLOWUP-abcd1234\n` } }).verdict).toBe("pass"); + }); + + it("grades B1 by running the tests and the CLI", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", + files: { "NOTES-abcd1234.md": "B1-abcd1234\n" }, + commands: { unittest: { exitCode: 0, stdout: "", stderr: "OK" }, slugify: { exitCode: 0, stdout: "hello-chaos-world\n", stderr: "" } } }; + expect(workloadDefinitions.B1.grade("abcd1234", base).verdict).toBe("pass"); + expect(workloadDefinitions.B1.grade("abcd1234", { ...base, commands: { ...base.commands, unittest: { exitCode: 1, stdout: "", stderr: "FAILED" } } }).verdict).toBe("fail"); + }); + + it("grades B5 against the deterministic bench result", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", commands: {}, + benchProgressObserved: 6, + files: { "bench-result-abcd1234.txt": `${BENCH_EXPECTED}\n`, ".bench/run-abcd1234.txt": `started=1000 finished=1241 result=${BENCH_EXPECTED}\n` } }; + expect(BENCH_EXPECTED).toBe("BENCH_RESULT=4636840"); + expect(parseBenchRecord(base.files[".bench/run-abcd1234.txt"])).toEqual({ seconds: 241, result: BENCH_EXPECTED }); + expect(workloadDefinitions.B5.grade("abcd1234", base).verdict).toBe("pass"); + // A forged record without harness-observed progress does not pass. + expect(workloadDefinitions.B5.grade("abcd1234", { ...base, benchProgressObserved: 1 }).checks.find(c => c.name === "bench_progress_observed")?.ok).toBe(false); + expect(workloadDefinitions.B5.grade("abcd1234", { ...base, files: { ...base.files, "bench-result-abcd1234.txt": "BENCH_RESULT=1" } }).verdict).toBe("fail"); + // A computed answer without a bench.sh run, or a too-short run, does not pass. + expect(workloadDefinitions.B5.grade("abcd1234", { ...base, files: { "bench-result-abcd1234.txt": BENCH_EXPECTED } }).checks.find(c => c.name === "bench_actually_ran")?.ok).toBe(false); + expect(workloadDefinitions.B5.grade("abcd1234", { ...base, files: { ...base.files, ".bench/run-abcd1234.txt": `started=1000 finished=1010 result=${BENCH_EXPECTED}` } }).verdict).toBe("fail"); + }); +}); + +describe("rest detection", () => { + it("is not resting while a run, scheduled retry or recovery is live", () => { + expect(isResting(sample(0, "in_progress", [run("r1", "queued", 0)]))).toBe(false); + expect(isResting(sample(0, "in_progress", [], { scheduledRetry: true }))).toBe(false); + expect(isResting(sample(0, "done", [run("r1", "succeeded", 0, 10)]))).toBe(true); + // Unexplained idle work is a possible stall, not a rest; a visible Retry is a rest. + expect(isResting(sample(0, "in_progress", [run("r1", "failed", 0, 10)]))).toBe(false); + expect(isResting(sample(0, "in_progress", [run("r1", "failed", 0, 10)], { interruption: { state: "failed", canRetry: true, retryWait: false, workspaceFailure: null, observedRunId: "r1", userAction: null } }))).toBe(true); + expect(isResting({ at: T0, reachable: false, serverReady: false })).toBe(false); + expect(isResting(sample(0, "blocked", [], { activeRecoveryAction: true, recoveryOwner: "agent" }))).toBe(false); + // A board-owned recovery waits on a person, so the task rests (and R6 reports it). + expect(isResting(sample(0, "blocked", [], { activeRecoveryAction: true, recoveryOwner: "board" }))).toBe(true); + }); +}); diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts new file mode 100644 index 0000000000..75eea1f037 --- /dev/null +++ b/tests/restart-chaos/grader.ts @@ -0,0 +1,185 @@ +import type { FaultReceipt, OracleResult, RuleId, RuleResult, TaskGrade, TaskRecord, TaskSample } from "./types.js"; + +/** Pure R1–R6 grading over observer samples (plan §2). No I/O. */ + +export const ACTIVE_RUN_STATUSES = new Set(["queued", "running", "scheduled_retry"]); +export const STALL_LIMIT_MS = 3 * 60_000; +const REST_STATUSES_WITH_WORK = new Set(["todo", "in_progress"]); + +const ms = (iso: string | null | undefined) => (iso ? Date.parse(iso) : NaN); + +function faultWindow(fault: FaultReceipt): [number, number] { + return [fault.startedAt, fault.readyAt ?? fault.healthyAt ?? Number.POSITIVE_INFINITY]; +} + +/** + * Restart-crossing: some run of the task (queued wake through finalization) + * overlaps the fault window, or the task was not yet at rest when it started. + */ +export function crossingFaults(record: TaskRecord, faults: readonly FaultReceipt[]): string[] { + if (record.createdAt === null) return []; + const latest = new Map[number]; at: number }>(); + for (const sample of record.samples) for (const run of sample.runs ?? []) latest.set(run.id, { run, at: sample.at }); + const runs = new Map(); + for (const [id, { run, at }] of latest) { + const start = ms(run.createdAt) || ms(run.startedAt) || at; + const end = ACTIVE_RUN_STATUSES.has(run.status) ? Number.POSITIVE_INFINITY : (ms(run.finishedAt) || at); + runs.set(id, { start, end }); + } + const restAt = record.restAt ?? Number.POSITIVE_INFINITY; + return faults.filter(fault => { + const [start, end] = faultWindow(fault); + if (record.createdAt! <= start && restAt > start) return true; + return [...runs.values()].some(run => run.start <= end && run.end >= start); + }).map(fault => fault.faultId); +} + +function reachable(samples: readonly TaskSample[]) { return samples.filter(sample => sample.reachable); } + +function lastReachable(record: TaskRecord): TaskSample | undefined { + const values = reachable(record.samples); + return values[values.length - 1]; +} + +export function outageOverlapMs(record: TaskRecord, faults: readonly FaultReceipt[]): number { + if (record.createdAt === null) return 0; + const end = record.restAt ?? Number.POSITIVE_INFINITY; + return faults.reduce((total, fault) => { + const [start, stop] = faultWindow(fault); + const overlap = Math.min(stop, end) - Math.max(start, record.createdAt!); + return total + (Number.isFinite(overlap) && overlap > 0 ? overlap : 0); + }, 0); +} + +export function gradeR1(record: TaskRecord, budgetMs: number, faults: readonly FaultReceipt[]): RuleResult { + if (record.launchError) return { verdict: "fail", detail: `launch_failed: ${record.launchError}` }; + if (record.retry) return { verdict: "n/a", detail: "task rested needing Retry; graded under R3" }; + if (record.finalStatus !== "done") { + return { verdict: "fail", detail: record.timedOut ? `no_rest_within_budget (last status ${record.finalStatus ?? "unknown"})` : `rested_not_done (status ${record.finalStatus ?? "unknown"}, no Retry offered)` }; + } + const allowed = budgetMs + outageOverlapMs(record, faults); + const took = (record.restAt ?? Number.POSITIVE_INFINITY) - record.createdAt!; + return took <= allowed + ? { verdict: "pass", detail: `done after ${Math.round(took / 1000)}s (allowed ${Math.round(allowed / 1000)}s)` } + : { verdict: "fail", detail: `done_too_late ${Math.round(took / 1000)}s > ${Math.round(allowed / 1000)}s` }; +} + +export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { + const last = lastReachable(record); + if (!last) return { verdict: "fail", detail: "no_reachable_sample" }; + if (last.interruption?.workspaceFailure) return { verdict: "fail", detail: `workspace_failure_visible_at_rest: ${last.interruption.workspaceFailure}` }; + if (record.preFaultSnapshotError) { + return { verdict: "fail", detail: `pre_fault_snapshot_failed: durability not checked (${record.preFaultSnapshotError})` }; + } + if (record.lostFiles?.length) { + return { verdict: "fail", detail: `pre_fault_files_lost: ${record.lostFiles.slice(0, 10).join(",")}` }; + } + const fileChecks = oracle.checks.filter(check => /file|workspace_located/.test(check.name) && !check.ok); + if (fileChecks.length && record.finalStatus === "done") { + return { verdict: "fail", detail: `workspace_files_missing_after_done: ${fileChecks.map(check => check.name).join(",")}` }; + } + const transient = record.samples.some(sample => sample.interruption?.workspaceFailure); + return { verdict: "pass", detail: transient ? "workspace failure appeared during recovery and cleared" : "no workspace failure at rest" }; +} + +export function gradeR3(record: TaskRecord): RuleResult { + const last = lastReachable(record); + if (!record.retry) { + if (record.finalStatus !== "done" && !record.timedOut && !record.launchError) { + return { verdict: "fail", detail: `missing_retry: rested ${record.finalStatus ?? "unknown"} with canRetry=${String(last?.interruption?.canRetry ?? false)}` }; + } + return { verdict: "n/a", detail: "no Retry needed" }; + } + if (record.retry.error || record.retry.accepted !== true) { + return { verdict: "fail", detail: `retry_rejected: http=${record.retry.httpStatus ?? "none"} accepted=${String(record.retry.accepted)} ${record.retry.error ?? ""}`.trim() }; + } + const retryAt = record.retry.at; + const newRun = record.samples.some(sample => (sample.runs ?? []).some(run => (ms(run.createdAt) || 0) >= retryAt - 1_000)); + if (!newRun) return { verdict: "fail", detail: "retry_no_new_run" }; + if (record.finalStatus !== "done") return { verdict: "fail", detail: `retry_did_not_complete (status ${record.finalStatus ?? "unknown"}${record.timedOut ? ", timed out" : ""})` }; + return { verdict: "pass", detail: "one Retry started a run that completed the task" }; +} + +/** + * R4: at most one active run (queued, running or scheduled_retry) per issue in + * any sample, and no comment consumed twice. A wake comment may be reused only + * by a run that follows a run which did not succeed (an explicit retry or + * continuation after a failure); reuse after a successful run is a replay. + */ +export function gradeR4(record: TaskRecord): RuleResult { + for (const sample of reachable(record.samples)) { + const active = (sample.runs ?? []).filter(run => ACTIVE_RUN_STATUSES.has(run.status)); + if (active.length > 1) return { verdict: "fail", detail: `concurrent_runs: ${active.map(run => `${run.id}(${run.status})`).join(",")}` }; + } + const finalRuns = new Map[number]>(); + for (const sample of reachable(record.samples)) for (const run of sample.runs ?? []) finalRuns.set(run.id, run); + const begun = [...finalRuns.values()].filter(run => run.startedAt) + .sort((a, b) => (ms(a.createdAt) || 0) - (ms(b.createdAt) || 0)); + const consumers = new Map(); + for (const run of begun) for (const commentId of run.wakeCommentIds) consumers.set(commentId, [...(consumers.get(commentId) ?? []), run]); + for (const [commentId, runs] of consumers) { + const replayed = runs.findIndex((_run, index) => index > 0 && runs.slice(0, index).some(prior => prior.status === "succeeded")); + if (replayed > 0) return { verdict: "fail", detail: `comment_consumed_twice: ${commentId} by ${runs.slice(0, replayed + 1).map(run => run.id).join(",")}` }; + } + return { verdict: "pass", detail: "at most one active run per sample; no comment consumed after a successful run" }; +} + +function namedWait(sample: TaskSample): boolean { + const interruption = sample.interruption; + return Boolean(sample.executionBlocker || sample.scheduledRetry || sample.activeRecoveryAction || interruption?.retryWait + || interruption?.state === "retry_scheduled" || interruption?.canRetry + || (interruption && ["failed", "waiting_for_user"].includes(interruption.state))); +} + +export function isStalledSample(sample: TaskSample): boolean { + if (!sample.reachable || !sample.serverReady) return false; + if (!REST_STATUSES_WITH_WORK.has(sample.issueStatus ?? "")) return false; + if ((sample.runs ?? []).some(run => ACTIVE_RUN_STATUSES.has(run.status))) return false; + return !namedWait(sample); +} + +export function gradeR5(record: TaskRecord): RuleResult { + let stallStart: number | null = null; + let worst = 0; + for (const sample of record.samples) { + if (!sample.reachable || !sample.serverReady) { stallStart = null; continue; } + if (isStalledSample(sample)) { + stallStart ??= sample.at; + worst = Math.max(worst, sample.at - stallStart); + if (sample.at - stallStart > STALL_LIMIT_MS) { + return { verdict: "fail", detail: `silent_stall ${Math.round((sample.at - stallStart) / 1000)}s in ${sample.issueStatus} with no live run, wake or wait reason` }; + } + } else stallStart = null; + } + return { verdict: "pass", detail: worst ? `longest unexplained idle ${Math.round(worst / 1000)}s` : "no unexplained idle" }; +} + +export function gradeR6(record: TaskRecord): RuleResult { + const sweep = record.sweep; + if (!sweep) return { verdict: "fail", detail: "rest_sweep_missing" }; + if (sweep.error) return { verdict: "fail", detail: `rest_sweep_error: ${sweep.error}` }; + const leaks = [ + ...(sweep.activeRuns.length ? [`active_runs=${sweep.activeRuns.join(",")}`] : []), + ...(sweep.runningWorkspaceOperations.length ? [`running_workspace_operations=${sweep.runningWorkspaceOperations.join(",")}`] : []), + ...(sweep.activeRecoveryAction ? ["active_recovery_action"] : []), + ]; + return leaks.length ? { verdict: "fail", detail: `leak: ${leaks.join("; ")}` } : { verdict: "pass", detail: "clean rest" }; +} + +export function gradeTask(record: TaskRecord, faults: readonly FaultReceipt[], budgetMs: number, oracle: OracleResult): TaskGrade { + const crossing = crossingFaults(record, faults); + const rules: Record = { + R1: gradeR1(record, budgetMs, faults), + R2: gradeR2(record, oracle), + R3: gradeR3(record), + R4: gradeR4(record), + R5: gradeR5(record), + R6: gradeR6(record), + }; + const failed = (Object.entries(rules) as Array<[RuleId, RuleResult]>).find(([, rule]) => rule.verdict === "fail"); + const pass = !failed && oracle.verdict === "pass"; + let signature: string | null = null; + if (failed) signature = `${failed[0]}:${failed[1].detail.split(/[\s:(]/)[0]}:${record.plan.workload}:${record.plan.fault}`; + else if (oracle.verdict === "fail") signature = `T:${oracle.checks.find(check => !check.ok)?.name ?? "unknown"}:${record.plan.workload}:${record.plan.fault}`; + return { crossed: crossing.length > 0, crossingFaults: crossing, rules, T: oracle, pass, signature }; +} diff --git a/tests/restart-chaos/ledger.ts b/tests/restart-chaos/ledger.ts new file mode 100644 index 0000000000..4569a116c8 --- /dev/null +++ b/tests/restart-chaos/ledger.ts @@ -0,0 +1,91 @@ +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { replayCommand } from "./config.js"; +import type { RuleId, TaskGrade, TaskRecord, WavePlan } from "./types.js"; + +export interface LedgerLine { + schema: "paperclip.restart-chaos.ledger/v1"; + waveId: string; + taskId: string; + seed: number; + gitSha: string; + env: string; + adapter: string; + workload: string; + fault: string; + holdMs: number; + fireAfterMs: number; + issue: string | null; + crossed: boolean; + crossingFaults: string[]; + rules: Record; + ruleDetails: Record; + T: string; + pass: boolean; + signature: string | null; + retried: boolean; + evidence: string | null; + replay: string; + recordedAt: string; +} + +export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, gitSha: string, evidence: string | null): LedgerLine { + const rules = Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])) as Record; + const ruleDetails = Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.detail])) as Record; + return { + schema: "paperclip.restart-chaos.ledger/v1", waveId: plan.waveId, taskId: record.plan.taskId, seed: plan.seed, gitSha, + env: plan.options.env, adapter: record.plan.adapter, workload: record.plan.workload, fault: record.plan.fault, + holdMs: record.plan.holdMs, fireAfterMs: record.plan.fireAfterMs, issue: record.identifier ?? record.issueId, + crossed: grade.crossed, crossingFaults: grade.crossingFaults, rules, ruleDetails, T: grade.T.verdict, pass: grade.pass, + signature: grade.signature, retried: Boolean(record.retry), evidence, replay: replayCommand(plan.waveId, record.plan.taskId), + recordedAt: new Date().toISOString(), + }; +} + +export async function appendLedger(resultsRoot: string, lines: readonly LedgerLine[]) { + await mkdir(resultsRoot, { recursive: true }); + await appendFile(path.join(resultsRoot, "ledger.jsonl"), lines.map(line => `${JSON.stringify(line)}\n`).join(""), { mode: 0o600 }); +} + +export async function readLedger(resultsRoot: string): Promise { + const text = await readFile(path.join(resultsRoot, "ledger.jsonl"), "utf8").catch(() => ""); + return text.split("\n").filter(Boolean).map(line => JSON.parse(line) as LedgerLine); +} + +/** Short summary: per-cell pass rate and open signatures across the ledger. */ +export function renderSummary(lines: readonly LedgerLine[], latestWave?: string): string { + const cells = new Map(); + const signatures = new Map(); + for (const line of lines) { + const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}`; + const cell = cells.get(key) ?? { pass: 0, total: 0, crossed: 0 }; + cell.total += 1; if (line.pass) cell.pass += 1; if (line.crossed) cell.crossed += 1; + cells.set(key, cell); + if (line.signature) { + const entry = signatures.get(line.signature) ?? { count: 0, latest: line }; + entry.count += 1; entry.latest = line; + signatures.set(line.signature, entry); + } + } + const out = ["# Restart chaos summary", "", `Tasks graded: ${lines.length}; passed: ${lines.filter(line => line.pass).length}; restart-crossing: ${lines.filter(line => line.crossed).length}.`, ""]; + if (latestWave) { + out.push(`## Latest wave ${latestWave}`, "", "| Task | Cell | Crossed | R1 | R2 | R3 | R4 | R5 | R6 | T | Result |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"); + for (const line of lines.filter(candidate => candidate.waveId === latestWave)) { + out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`); + } + out.push(""); + } + out.push("## Cells", "", "| Cell | Pass | Total | Crossed |", "| --- | --- | --- | --- |"); + for (const [key, cell] of [...cells].sort()) out.push(`| ${key} | ${cell.pass} | ${cell.total} | ${cell.crossed} |`); + out.push("", "## Open signatures", ""); + if (!signatures.size) out.push("None."); + for (const [signature, entry] of signatures) out.push(`- \`${signature}\` × ${entry.count} — latest ${entry.latest.waveId}/${entry.latest.taskId}; evidence ${entry.latest.evidence ?? "n/a"}; replay \`${entry.latest.replay}\``); + out.push(""); + return out.join("\n"); +} + +export async function writeSummary(resultsRoot: string, latestWave: string) { + const summary = renderSummary(await readLedger(resultsRoot), latestWave); + await writeFile(path.join(resultsRoot, "SUMMARY.md"), summary, { mode: 0o600 }); + return summary; +} diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts new file mode 100644 index 0000000000..7fd6981e02 --- /dev/null +++ b/tests/restart-chaos/main.ts @@ -0,0 +1,188 @@ +/** + * Control-plane restart chaos harness, Phase 1 MVP. + * See doc/plans/2026-11-06-control-plane-restart-chaos.md and ./README.md. + * + * node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local --adapters codex,claude,opencode \ + * --concurrency 4 --tasks 8 --faults K1,K2,K4 --seed 42 + * node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --replay / + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { ChaosApi } from "./api.js"; +import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js"; +import { knownSecretValues, writeEvidenceBundle } from "./evidence.js"; +import { gradeTask } from "./grader.js"; +import { appendLedger, ledgerLine, writeSummary, type LedgerLine } from "./ledger.js"; +import { TaskObserver } from "./observer.js"; +import { observedFaultSchedule, planWave, replayWave } from "./scheduler.js"; +import { seedChaosLab } from "./seed.js"; +import { ChaosSupervisor } from "./supervisor.js"; +import { Timeline } from "./timeline.js"; +import type { FaultReceipt, OracleArtifacts, WavePlan } from "./types.js"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +const resultsRoot = path.resolve(process.env.RESTART_CHAOS_RESULTS_DIR ?? path.join(import.meta.dirname, "results")); +const QUIET_BEFORE_SWEEP_MS = 15_000; + +function git(args: string[]) { + try { return execFileSync("git", args, { cwd: repositoryRoot, encoding: "utf8" }).trim(); } catch { return ""; } +} + +async function loadPlan(): Promise<{ plan: WavePlan; keep: boolean }> { + const command = parseCommand(process.argv.slice(2)); + if (command.mode === "replay") { + const sourceDir = path.join(resultsRoot, command.ref.waveId); + const source = JSON.parse(await readFile(path.join(sourceDir, "plan.json"), "utf8")) as WavePlan; + // Replay the faults that actually crossed the task, including other tasks' faults. + const graded = await readFile(path.join(sourceDir, `${command.ref.taskId}.grade.json`), "utf8") + .then(text => JSON.parse(text) as { record: { createdAt: number | null; faultIds: string[] } }).catch(() => null); + const faults = await readFile(path.join(sourceDir, "faults.json"), "utf8").then(text => JSON.parse(text) as FaultReceipt[]).catch(() => []); + const observed = graded ? observedFaultSchedule(graded.record.createdAt, graded.record.faultIds, faults) : []; + if (!observed.length) console.warn("Replay: no recorded fault sequence for this task; using its planned fault only."); + return { plan: replayWave(source, command.ref, Date.now(), command.overrides, observed), keep: command.keep }; + } + return { plan: planWave(waveIdFor(command.options.seed), command.options), keep: command.keep }; +} + +function preflight(plan: WavePlan) { + const missing = [...new Set(plan.tasks.map(task => CREDENTIAL_FOR_ADAPTER[task.adapter]))].filter(key => !process.env[key]); + if (missing.length) throw new Error(`Missing credentials ${missing.join(", ")}. Run: set -a; source ~/.secrets; set +a`); + const runner = process.env.PAPERCLIP_RUNNER_BINARY + ?? path.join(repositoryRoot, "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"); + if (!existsSync(runner)) throw new Error(`Runner binary missing at ${runner}. Run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries`); + // Same provider bundle preparation as tests/runner-e2e/verified-provider-bundles.setup.mjs. + const build = spawnSync(process.execPath, [path.join(repositoryRoot, "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs")], + { cwd: repositoryRoot, stdio: "inherit" }); + if (build.status !== 0) throw new Error("Verified provider entrypoint build failed"); +} + +async function main(): Promise { + const { plan, keep } = await loadPlan(); + const waveDir = path.join(resultsRoot, plan.waveId); + await mkdir(waveDir, { recursive: true, mode: 0o700 }); + await writeFile(path.join(waveDir, "plan.json"), `${JSON.stringify(plan, null, 2)}\n`, { mode: 0o600 }); + const timeline = new Timeline(path.join(waveDir, "timeline.jsonl")); + const gitSha = git(["rev-parse", "HEAD"]); + const gitDirty = git(["status", "--porcelain"]).length > 0; + timeline.add("wave_planned", { waveId: plan.waveId, seed: plan.seed, gitSha, gitDirty, replayOf: plan.replayOf ?? null, + tasks: plan.tasks.map(task => `${task.taskId}:${task.adapter}/${task.workload}/${task.fault}@${task.fireAfterMs}ms${task.holdMs ? ` hold ${task.holdMs}ms` : ""}`) }); + preflight(plan); + + const supervisor = await ChaosSupervisor.create({ repositoryRoot, waveDir, timeline }); + let interrupted = false; + const onSignal = (signal: NodeJS.Signals) => { if (!interrupted) { interrupted = true; timeline.add("harness_interrupted", { signal }); } }; + process.on("SIGINT", onSignal); process.on("SIGTERM", onSignal); + const faults: FaultReceipt[] = []; + const faultWork: Promise[] = []; + const lines: LedgerLine[] = []; + let anyFailure = false; + let harnessError: unknown; + let faultFailures = 0; + try { + await supervisor.start(); + const api = new ChaosApi(supervisor.baseUrl); + const lab = await seedChaosLab({ api, env: plan.options.env, adapters: [...new Set(plan.tasks.map(task => task.adapter))], + concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, + fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline }); + // The instance root holds the secrets master key and the database: oracle code may not read it. + const protectedRoots = [os.homedir(), supervisor.paths.temporaryRoot, resultsRoot]; + const observers = plan.tasks.map(task => new TaskObserver(task, api, lab, timeline, supervisor, protectedRoots)); + const pending = [...observers]; + const active: TaskObserver[] = []; + const lastPoll = new Map(); + let lastDoneAt = Date.now(); + + while (!interrupted && (pending.length || active.length)) { + const now = Date.now(); + // A failed fault or an unexpected exit leaves the server not ready. + // Recover within a bound, or fail the wave instead of waiting forever. + if (!supervisor.serverReady && !supervisor.faultInProgress) await supervisor.ensureServing(); + while (active.length < plan.options.concurrency && pending.length && supervisor.serverReady && !supervisor.faultInProgress) { + const next = pending.shift()!; + await next.launch(); + if (!next.done) active.push(next); + } + await Promise.all(active.filter(observer => now - (lastPoll.get(observer) ?? 0) >= plan.options.pollMs).map(async observer => { + lastPoll.set(observer, now); + await observer.poll(now); + })); + for (const observer of active) { + const due = observer.faultDue(Date.now()); + if (!due) continue; + if (observer.record.restAt !== null) { timeline.add("fault_skipped_task_resting", { taskId: observer.plan.taskId, fault: due.kind }); continue; } + const exposed = active.filter(candidate => !candidate.done); + await Promise.all(exposed.map(candidate => candidate.snapshotBeforeFault())); + faultWork.push(supervisor.fault(due.kind, due.holdMs, observer.plan.taskId).then(receipt => { + faults.push(receipt); + for (const candidate of exposed) { candidate.noteFault(receipt.faultId); candidate.extendDeadline((receipt.readyAt ?? Date.now()) - receipt.startedAt); } + }, (error: Error & { receipt?: FaultReceipt }) => { + if (error.receipt) faults.push(error.receipt); + faultFailures += 1; + timeline.add("fault_error", { taskId: observer.plan.taskId, error: error.message }); + })); + } + for (let index = active.length - 1; index >= 0; index -= 1) { + if (active[index]!.done) { active.splice(index, 1); lastDoneAt = Date.now(); } + } + await delay(1_000); + } + await Promise.all(faultWork); + // An injected fault that did not complete invalidates the wave, even when + // bounded recovery brought the server back and the tasks passed. + if (faultFailures) harnessError ??= new Error(`${faultFailures} injected fault(s) failed; see fault_error in the timeline`); + if (interrupted) throw new Error("Harness interrupted before the wave finished"); + + // R6: let finalizers settle, then sweep for anything still pending. + const quietFor = Date.now() - lastDoneAt; + if (quietFor < QUIET_BEFORE_SWEEP_MS) await delay(QUIET_BEFORE_SWEEP_MS - quietFor); + timeline.add("wave_sweep", {}); + const secrets = knownSecretValues(); + for (const observer of observers) { + observer.record.sweep = await observer.sweep(); + let artifacts: OracleArtifacts | null = null; + if (observer.record.issueId) artifacts = await observer.collectArtifacts().catch(() => null); + const oracle = artifacts ? observer.definition.grade(observer.plan.nonce, artifacts) + : { verdict: "fail" as const, checks: [{ name: "artifacts_collected", ok: false, detail: observer.record.launchError ?? "collection failed" }] }; + const grade = gradeTask(observer.record, faults, observer.definition.budgetMs, oracle); + let evidence: string | null = null; + if (!grade.pass) { + anyFailure = true; + evidence = await writeEvidenceBundle({ directory: path.join(waveDir, "evidence", observer.plan.taskId), api, plan, + record: observer.record, grade, artifacts, faults, timeline: timeline.entries, serverLogPath: supervisor.paths.serverLogPath, + gitSha, gitDirty, issue: observer.issue, secrets }).catch(error => { timeline.add("evidence_failed", { taskId: observer.plan.taskId, error: String(error) }); return null; }); + } + await writeFile(path.join(waveDir, `${observer.plan.taskId}.grade.json`), `${JSON.stringify({ record: observer.record, grade }, null, 2)}\n`, { mode: 0o600 }); + lines.push(ledgerLine(plan, observer.record, grade, gitSha, evidence)); + timeline.add("task_graded", { taskId: observer.plan.taskId, pass: grade.pass, crossed: grade.crossed, signature: grade.signature, + rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict }); + } + await writeFile(path.join(waveDir, "faults.json"), `${JSON.stringify(faults, null, 2)}\n`, { mode: 0o600 }); + } catch (error) { + harnessError = error; + timeline.add("harness_error", { error: error instanceof Error ? error.stack ?? error.message : String(error) }); + } finally { + try { await supervisor.stop(); } catch (error) { + // Incomplete cleanup is a harness failure; keep the instance for inspection. + harnessError ??= error; + timeline.add("supervisor_stop_failed", { error: String(error) }); + } + if (lines.length) { + await appendLedger(resultsRoot, lines); + const summary = await writeSummary(resultsRoot, plan.waveId); + await writeFile(path.join(waveDir, "SUMMARY.md"), summary, { mode: 0o600 }); + console.info(`\n${summary}`); + } + if (keep || anyFailure || harnessError) timeline.add("instance_kept", { temporaryRoot: supervisor.paths.temporaryRoot }); + else await rm(supervisor.paths.temporaryRoot, { recursive: true, force: true }).catch(() => undefined); + } + console.info(`Wave ${plan.waveId}: results ${waveDir}`); + for (const line of lines.filter(candidate => !candidate.pass)) console.info(` FAIL ${line.taskId} ${line.signature} → ${line.evidence ?? "no evidence"}\n replay: ${replayCommand(plan.waveId, line.taskId)}`); + if (harnessError) { console.error(harnessError); return 2; } + return anyFailure ? 1 : 0; +} + +process.exit(await main().catch(error => { console.error(error); return 2; })); diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts new file mode 100644 index 0000000000..9af1752ed3 --- /dev/null +++ b/tests/restart-chaos/observer.test.ts @@ -0,0 +1,118 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ChaosApi } from "./api.js"; +import { REST_CONFIRM_MS, STALL_GIVE_UP_MS, TaskObserver, listWorkspaceFiles } from "./observer.js"; +import { createFixtureRepository, type ChaosLab } from "./seed.js"; +import { Timeline } from "./timeline.js"; +import type { TaskPlan } from "./types.js"; +import { BENCH_EXPECTED, parseBenchRecord } from "./workloads.js"; + +process.env.RESTART_CHAOS_QUIET = "1"; +const lab: ChaosLab = { companyId: "c1", environmentId: "e1", projectId: "p1", fixtureRepositoryPath: "/tmp/none", + agents: { codex: { id: "a1", name: "Chaos codex" }, claude: { id: "a2", name: "Chaos claude" }, opencode: { id: "a3", name: "Chaos opencode" } } }; +const plan: TaskPlan = { taskId: "t01", index: 0, workload: "C1", adapter: "codex", fault: "K1", fireAfterMs: 999_999, + holdMs: 0, followUpAfterMs: 1_000, nonce: "abcd1234" }; +const ready = { serverReady: true, faultInProgress: false }; +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); + +/** Fake board API: a C1 issue whose first run already finished. */ +function fakeApi(overrides: Partial unknown>> = {}, status = "done") { + const posted: string[] = []; + const api = { + async get(route: string) { + for (const [prefix, handler] of Object.entries(overrides)) if (route.includes(prefix)) return handler!(route); + if (route.endsWith("/runs")) return [{ runId: "r1", status: "succeeded", createdAt: new Date(0).toISOString(), finishedAt: new Date(1).toISOString() }]; + return { id: "i1", status, interruption: null }; + }, + async post(route: string) { posted.push(route); return route.endsWith("/issues") ? { id: "i1", identifier: "CHA-1" } : { id: "comment-1" }; }, + }; + return { api: api as unknown as ChaosApi, posted }; +} + +describe("TaskObserver", () => { + it("does not finish a C1 task on the poll that posts the follow-up", async () => { + const { api, posted } = fakeApi(); + const observer = new TaskObserver(plan, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + // The task rests before the follow-up is due, for longer than the rest window. + await observer.poll(start + 100); + await observer.poll(start + 100 + REST_CONFIRM_MS + 1); + expect(observer.done).toBe(false); + expect(observer.record.followUp).toMatchObject({ commentId: "comment-1" }); + expect(posted.some(route => route.endsWith("/comments"))).toBe(true); + // A fresh rest window must elapse after the comment before the task counts as resting. + await observer.poll(start + 200 + REST_CONFIRM_MS + 2); + expect(observer.done).toBe(false); + await observer.poll(start + 300 + 2 * REST_CONFIRM_MS + 3); + expect(observer.done).toBe(true); + }); + + it("keeps watching unexplained idle work until the stall limit, then records the stall", async () => { + const { api } = fakeApi({}, "in_progress"); + const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + await observer.poll(start + 1_000 + REST_CONFIRM_MS + 1); + expect(observer.done).toBe(false); + await observer.poll(start + 1_000 + STALL_GIVE_UP_MS + 1); + expect(observer.done).toBe(true); + expect(observer.record.finalStatus).toBe("in_progress"); + expect(observer.record.restAt).toBe(start + 1_000); + }); + + it("reports a failed workspace-operation read as a sweep error, not a clean rest", async () => { + const { api } = fakeApi({ "/workspace-operations": () => { throw new Error("timeout"); } }); + const observer = new TaskObserver(plan, api, lab, new Timeline(null), ready); + await observer.launch(); + const sweep = await observer.sweep(); + expect(sweep.error).toMatch(/timeout/); + }); + + it("leaves R2 unchecked instead of failing the wave when the pre-fault snapshot cannot read the workspace", async () => { + const { api } = fakeApi({ "/api/issues/i1": route => route.endsWith("/runs") ? [] + : { id: "i1", status: "in_progress", currentExecutionWorkspace: { cwd: "/nonexistent/restart-chaos-workspace" } } }); + const observer = new TaskObserver({ ...plan, workload: "B1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); + await observer.launch(); + await observer.poll(observer.record.createdAt! + 1_000); + await expect(observer.snapshotBeforeFault()).resolves.toBeUndefined(); + expect(observer.record.preFaultFiles).toBeNull(); + expect(observer.record.preFaultSnapshotError).toMatch(/ENOENT/); + }); + + it("fires the planned fault once, after its offset", async () => { + const { api } = fakeApi(); + const observer = new TaskObserver({ ...plan, fireAfterMs: 5_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + expect(observer.faultDue(start + 4_999)).toBeNull(); + expect(observer.faultDue(start + 5_000)).toEqual({ kind: "K1", holdMs: 0, atMs: 5_000 }); + expect(observer.faultDue(start + 6_000)).toBeNull(); + }); +}); + +describe("fixture repository", () => { + it("lists workspace files without VCS or cache directories", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-fixture-")); + directories.push(root); + await createFixtureRepository(root); + const files = await listWorkspaceFiles(root); + expect(files).toContain("chaoscli/cli.py"); + expect(files.some(file => file.startsWith(".git/"))).toBe(false); + }); + + it("bench.sh writes a tagged completion record with the deterministic result", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-bench-")); + directories.push(root); + await createFixtureRepository(root, 1); + const output = execFileSync("./bench.sh", ["abcd1234"], { cwd: root, encoding: "utf8", timeout: 60_000 }); + expect(output.trim().split("\n").at(-1)).toBe(BENCH_EXPECTED); + const record = parseBenchRecord(await readFile(path.join(root, ".bench", "run-abcd1234.txt"), "utf8")); + expect(record?.result).toBe(BENCH_EXPECTED); + }, 60_000); +}); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts new file mode 100644 index 0000000000..1f0eb32817 --- /dev/null +++ b/tests/restart-chaos/observer.ts @@ -0,0 +1,334 @@ +import { randomUUID } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { ChaosApiError, type ChaosApi } from "./api.js"; +import { ACTIVE_RUN_STATUSES, isStalledSample, STALL_LIMIT_MS } from "./grader.js"; +import type { ChaosLab } from "./seed.js"; +import type { Timeline } from "./timeline.js"; +import type { + InterruptionSample, OracleArtifacts, RestSweep, RetryAttempt, RunSample, ScheduledFault, TaskPlan, TaskRecord, TaskSample, +} from "./types.js"; +import { runOracleCommand } from "./oracle-sandbox.js"; +import { taskFaultSchedule } from "./scheduler.js"; +import { workloadDefinitions } from "./workloads.js"; + +const IGNORED_DIRECTORIES = new Set([".git", ".bench", "__pycache__", "node_modules"]); +const MAX_SNAPSHOT_FILES = 2_000; + +/** Relative paths of workspace files, excluding VCS and cache directories. */ +export async function listWorkspaceFiles(root: string): Promise { + const out: string[] = []; + const walk = async (relative: string) => { + if (out.length >= MAX_SNAPSHOT_FILES) return; + const entries = await readdir(path.join(root, relative), { withFileTypes: true }); + for (const entry of entries) { + const child = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isDirectory()) { if (!IGNORED_DIRECTORIES.has(entry.name)) await walk(child); } + else if (entry.isFile()) out.push(child); + if (out.length >= MAX_SNAPSHOT_FILES) return; + } + }; + await walk(""); + return out.sort(); +} + +/** Rest must hold for this long before the observer acts on it. */ +export const REST_CONFIRM_MS = 12_000; +/** Extra time granted after a Retry click before R3 is graded as timed out. */ +export const RETRY_BUDGET_MS = 15 * 60_000; + +type Json = Record; + +export function toRunSample(run: Json): RunSample { + const list = (value: unknown) => (Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []); + const wake = new Set([...list(run.wakeCommentIds), ...(typeof run.wakeCommentId === "string" ? [run.wakeCommentId] : [])]); + return { + id: String(run.runId ?? run.id), status: String(run.status), createdAt: run.createdAt ?? null, startedAt: run.startedAt ?? null, + finishedAt: run.finishedAt ?? null, errorCode: run.errorCode ?? null, retryOfRunId: run.retryOfRunId ?? null, + wakeCommentIds: [...wake], livenessState: run.livenessState ?? null, + }; +} + +export function toInterruption(value: Json | null | undefined): InterruptionSample | null { + if (!value) return null; + return { state: String(value.state), canRetry: value.canRetry === true, retryWait: Boolean(value.retryWait), + workspaceFailure: value.workspaceFailure?.message ?? null, observedRunId: value.observedRunId ?? null, userAction: value.userAction ?? null }; +} + +/** Rest = no live run and nothing the system itself is still doing. */ +export function isResting(sample: TaskSample): boolean { + if (!sample.reachable || !sample.serverReady) return false; + if ((sample.runs ?? []).some(run => ACTIVE_RUN_STATUSES.has(run.status))) return false; + if (sample.scheduledRetry) return false; + // A board-owned recovery waits on a person: that is a resting state. + if (sample.activeRecoveryAction && sample.recoveryOwner !== "board") return false; + const interruption = sample.interruption; + if (interruption && (interruption.state === "working" || interruption.state === "retry_scheduled" || interruption.retryWait)) return false; + // Unexplained idle work is a possible stall, not a rest: keep watching it so + // the task can still resume and R5 can measure the stall. + if (isStalledSample(sample)) return false; + return true; +} + +/** Stop watching an unexplained idle task once it exceeds the R5 stall limit. */ +export const STALL_GIVE_UP_MS = STALL_LIMIT_MS + 15_000; + +export class TaskObserver { + readonly record: TaskRecord; + issue: Json | null = null; + private restCandidateSince: number | null = null; + private stallSince: number | null = null; + private readonly faultSchedule: ScheduledFault[]; + private nextFault = 0; + done = false; + + constructor(readonly plan: TaskPlan, private readonly api: ChaosApi, private readonly lab: ChaosLab, + private readonly timeline: Timeline, private readonly supervisor: { serverReady: boolean; faultInProgress: boolean }, + /** Paths oracle commands may not read: the operator's home, the instance root (secrets key, database) and results. */ + private readonly protectedRoots: readonly string[] = [os.homedir()]) { + this.record = { plan, issueId: null, identifier: null, createdAt: null, launchError: null, samples: [], retry: null, + restAt: null, finalStatus: null, deadlineAt: null, timedOut: false, followUp: null, faultIds: [], sweep: null, + preFaultFiles: null, lostFiles: null }; + this.faultSchedule = taskFaultSchedule(plan); + } + + get definition() { return workloadDefinitions[this.plan.workload]; } + + async launch(): Promise { + const definition = this.definition; + try { + const issue = await this.api.post(`/api/companies/${this.lab.companyId}/issues`, { + title: definition.title(this.plan.nonce), + description: definition.description(this.plan.nonce), + status: "todo", + priority: "medium", + assigneeAgentId: this.lab.agents[this.plan.adapter].id, + ...(definition.usesRepository ? { projectId: this.lab.projectId } : {}), + }); + this.record.issueId = String(issue.id); + this.record.identifier = issue.identifier ?? null; + this.record.createdAt = Date.now(); + this.record.deadlineAt = this.record.createdAt + definition.budgetMs; + this.timeline.add("task_launched", { taskId: this.plan.taskId, issueId: issue.id, identifier: issue.identifier, + workload: this.plan.workload, adapter: this.plan.adapter, faults: this.faultSchedule }); + } catch (error) { + this.record.launchError = error instanceof Error ? error.message : String(error); + this.done = true; + this.timeline.add("task_launch_failed", { taskId: this.plan.taskId, error: this.record.launchError }); + } + } + + /** The next scheduled fault once it is due and the task has not rested yet. */ + faultDue(now: number): ScheduledFault | null { + if (this.done || this.record.createdAt === null) return null; + const fault = this.faultSchedule[this.nextFault]; + if (!fault || now < this.record.createdAt + fault.atMs) return null; + this.nextFault += 1; + return fault; + } + + /** + * B5: read bench.sh's progress file while the task is live. The harness sees + * the values change over minutes, which a computed answer cannot produce. + */ + private async observeBenchProgress(): Promise { + if (this.plan.workload !== "B5") return; + const cwd = (this.issue?.currentExecutionWorkspace as Json | null | undefined)?.cwd; + if (typeof cwd !== "string") return; + const value = (await readFile(path.join(cwd, ".bench", "progress"), "utf8").catch(() => "")).trim(); + if (!value) return; + const seen = this.record.benchProgressSeen ??= []; + if (!seen.includes(value)) seen.push(value); + } + + /** R2: remember which workspace files existed before the first fault hit this task. */ + async snapshotBeforeFault(): Promise { + if (this.record.preFaultFiles !== null || this.done) return; + // One task's snapshot failure must not end the wave; R2 fails with the reason. + try { + const workspace = await this.workspacePath(); + if (workspace) this.record.preFaultFiles = await listWorkspaceFiles(workspace); + } catch (error) { + this.record.preFaultSnapshotError = error instanceof Error ? error.message : String(error); + this.timeline.add("pre_fault_snapshot_failed", { taskId: this.plan.taskId, error: this.record.preFaultSnapshotError }); + } + } + + noteFault(faultId: string) { if (!this.record.faultIds.includes(faultId)) this.record.faultIds.push(faultId); } + + /** Outages extend the deadline: R1 allows budget plus outage time. */ + extendDeadline(ms: number) { if (this.record.deadlineAt !== null) this.record.deadlineAt += ms; } + + async poll(now = Date.now()): Promise { + if (this.done || !this.record.issueId) return; + const sample = await this.sample(now); + this.record.samples.push(sample); + await this.observeBenchProgress(); + if (await this.maybeFollowUp(sample, now)) { + // The comment response is not the agent's answer: require fresh samples. + this.restCandidateSince = null; + return; + } + this.stallSince = isStalledSample(sample) ? (this.stallSince ?? now) : null; + if (this.stallSince !== null && now - this.stallSince > STALL_GIVE_UP_MS) { + this.record.restAt = this.stallSince; + this.record.finalStatus = sample.issueStatus ?? null; + this.done = true; + this.timeline.add("task_stalled", { taskId: this.plan.taskId, status: sample.issueStatus, idleMs: now - this.stallSince }); + return; + } + if (!isResting(sample) || this.supervisor.faultInProgress) { + this.restCandidateSince = null; + } else { + this.restCandidateSince ??= now; + if (now - this.restCandidateSince >= REST_CONFIRM_MS) { + await this.onRest(sample, now); + return; + } + } + if (this.record.deadlineAt !== null && now > this.record.deadlineAt && !this.supervisor.faultInProgress) { + this.record.timedOut = true; + this.record.finalStatus = sample.issueStatus ?? this.record.finalStatus; + this.done = true; + this.timeline.add("task_timed_out", { taskId: this.plan.taskId, status: sample.issueStatus }); + } + } + + private async sample(now: number): Promise { + const serverReady = this.supervisor.serverReady && !this.supervisor.faultInProgress; + try { + const [issue, runs] = await Promise.all([ + this.api.get(`/api/issues/${this.record.issueId}`, 5_000), + this.api.get(`/api/issues/${this.record.issueId}/runs`, 5_000), + ]); + this.issue = issue; + return { at: now, reachable: true, serverReady, issueStatus: String(issue.status), + interruption: toInterruption(issue.interruption), activeRecoveryAction: Boolean(issue.activeRecoveryAction), + recoveryOwner: issue.activeRecoveryAction?.ownerType ?? null, + executionBlocker: Boolean(issue.executionBlocker), scheduledRetry: Boolean(issue.scheduledRetry), + runs: (runs ?? []).map(toRunSample) }; + } catch { + return { at: now, reachable: false, serverReady: false }; + } + } + + private async maybeFollowUp(sample: TaskSample, now: number): Promise { + const definition = this.definition; + if (!definition.followUp || this.record.followUp || this.plan.followUpAfterMs === null || !sample.reachable) return false; + if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; + // C1 targets a live run (B8); wait for one unless the task already rests. + const running = (sample.runs ?? []).some(run => run.status === "running"); + if (!running && !isResting(sample)) return false; + try { + const comment = await this.api.post(`/api/issues/${this.record.issueId}/comments`, { body: definition.followUp(this.plan.nonce) }); + this.record.followUp = { at: now, commentId: comment?.id ?? null, error: null }; + } catch (error) { + this.record.followUp = { at: now, commentId: null, error: error instanceof Error ? error.message : String(error) }; + } + this.timeline.add("task_follow_up", { taskId: this.plan.taskId, duringRun: running, ...this.record.followUp }); + return true; + } + + private async onRest(sample: TaskSample, now: number) { + const status = sample.issueStatus ?? null; + // C1: a follow-up posted after rest must be served before the task counts as resting. + if (this.definition.followUp && !this.record.followUp) return; + if (status !== "done" && !this.record.retry && sample.interruption?.canRetry) { + this.record.retry = await this.retry(sample, now); + this.restCandidateSince = null; + if (this.record.retry.accepted && this.record.deadlineAt !== null) this.record.deadlineAt = Math.max(this.record.deadlineAt, now + RETRY_BUDGET_MS); + return; + } + this.record.restAt = this.restCandidateSince ?? now; + this.record.finalStatus = status; + this.done = true; + this.timeline.add("task_rested", { taskId: this.plan.taskId, status, interruption: sample.interruption?.state ?? null, + canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) }); + } + + /** Exactly one click, with the same body ui/src/hooks/useTaskRetry.ts sends. */ + private async retry(sample: TaskSample, now: number): Promise { + const observedRunId = sample.interruption?.observedRunId ?? (this.issue?.executionBlocker?.runId as string | undefined) ?? null; + const attempt: RetryAttempt = { at: now, observedRunId, httpStatus: null, accepted: null, error: null }; + try { + const result = await this.api.post<{ accepted?: boolean }>(`/api/issues/${this.record.issueId}/retry`, + { idempotencyKey: randomUUID(), observedRunId }); + attempt.httpStatus = 200; + attempt.accepted = result?.accepted === true; + } catch (error) { + attempt.httpStatus = error instanceof ChaosApiError ? error.status : null; + attempt.error = error instanceof Error ? error.message : String(error); + } + this.timeline.add("task_retry_clicked", { taskId: this.plan.taskId, ...attempt }); + return attempt; + } + + async workspacePath(): Promise { + if (!this.definition.usesRepository) return null; + const current = this.issue?.currentExecutionWorkspace as Json | null | undefined; + if (typeof current?.cwd === "string") return current.cwd; + const workspaceId = this.issue?.executionWorkspaceId; + if (typeof workspaceId === "string") { + const workspace = await this.api.get(`/api/execution-workspaces/${workspaceId}`).catch(() => null); + if (typeof workspace?.cwd === "string") return workspace.cwd; + } + return null; + } + + async collectArtifacts(): Promise { + const definition = this.definition; + const issueId = this.record.issueId!; + const issue = await this.api.get(`/api/issues/${issueId}`).catch(() => this.issue); + if (issue) this.issue = issue; + const documents: Record = {}; + for (const key of definition.collect.documents) { + const document = await this.api.get(`/api/issues/${issueId}/documents/${key}`).catch(() => null); + documents[key] = typeof document?.body === "string" ? document.body : null; + } + const comments = await this.api.get(`/api/issues/${issueId}/comments`).catch(() => []); + const workspacePath = await this.workspacePath(); + if (workspacePath && this.record.preFaultFiles) { + const present = new Set(await listWorkspaceFiles(workspacePath)); + this.record.lostFiles = this.record.preFaultFiles.filter(file => !present.has(file)); + } + const files: Record = {}; + const commands: OracleArtifacts["commands"] = {}; + if (workspacePath) { + for (const relative of definition.collect.files(this.plan.nonce)) { + files[relative] = await readFile(path.join(workspacePath, relative), "utf8").catch(() => null); + } + // Agent-editable code: run it sandboxed, away from the operator's home directory. + for (const command of definition.collect.commands) commands[command.name] = await runOracleCommand(command.argv, workspacePath, { protectedRoots: this.protectedRoots }); + } + return { + issueStatus: issue?.status ?? this.record.finalStatus, + documents, + agentComments: (comments ?? []).filter(comment => comment.authorAgentId).map(comment => String(comment.body ?? "")), + files, commands, workspacePath, + ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), + }; + } + + /** R6 sweep: nothing still pending once the wave is quiet. */ + async sweep(): Promise { + if (!this.record.issueId) return { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: null }; + try { + const [issue, runs] = await Promise.all([ + this.api.get(`/api/issues/${this.record.issueId}`), + this.api.get(`/api/issues/${this.record.issueId}/runs`), + ]); + const samples = runs.map(toRunSample); + const operations: string[] = []; + for (const run of samples) { + // A failed read must not look like a clean rest: let it reach sweep.error. + const rows = await this.api.get(`/api/heartbeat-runs/${run.id}/workspace-operations`); + for (const row of rows ?? []) if (row.status === "running") operations.push(String(row.id)); + } + return { activeRuns: samples.filter(run => ACTIVE_RUN_STATUSES.has(run.status)).map(run => run.id), + runningWorkspaceOperations: operations, activeRecoveryAction: Boolean(issue.activeRecoveryAction), error: null }; + } catch (error) { + return { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: error instanceof Error ? error.message : String(error) }; + } + } +} diff --git a/tests/restart-chaos/oracle-sandbox.test.ts b/tests/restart-chaos/oracle-sandbox.test.ts new file mode 100644 index 0000000000..22c2d09da0 --- /dev/null +++ b/tests/restart-chaos/oracle-sandbox.test.ts @@ -0,0 +1,49 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { oracleSandboxProfile, runOracleCommand } from "./oracle-sandbox.js"; +import { createFixtureRepository } from "./seed.js"; + +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); +const temp = async (prefix: string) => { const directory = await mkdtemp(path.join(os.tmpdir(), prefix)); directories.push(directory); return directory; }; + +describe("oracle sandbox", () => { + it("denies the protected roots and re-allows the workspace", () => { + const profile = oracleSandboxProfile({ workspace: "/w", scratch: "/s", protectedRoots: ["/Users/me"] }); + expect(profile).toContain('(deny file-read* (subpath "/Users/me"))'); + expect(profile).toContain("(deny network*)"); + expect(profile.indexOf('(allow file-read* (subpath "/w")')).toBeGreaterThan(profile.indexOf("(deny file-read*")); + }); + + it("refuses to run unsandboxed on other platforms unless allowed", async () => { + const result = await runOracleCommand(["true"], "/", { platform: "linux" }); + expect(result.exitCode).toBeNull(); + expect(result.stderr).toMatch(/refused/); + }); + + it.runIf(process.platform === "darwin")("protects an instance root that contains the workspace, except the workspace itself", async () => { + const instanceRoot = await temp("restart-chaos-instance-"); + await writeFile(path.join(instanceRoot, "master.key"), "canary-master-key\n"); + const workspace = path.join(instanceRoot, "fixtures", "chaos-sandbox"); + await createFixtureRepository(workspace, 1); + const leak = await runOracleCommand(["/bin/cat", path.join(instanceRoot, "master.key")], workspace, { protectedRoots: [instanceRoot] }); + expect(leak.stdout).not.toContain("canary-master-key"); + const greet = await runOracleCommand(["python3", "-m", "chaoscli", "greet", "chaos"], workspace, { protectedRoots: [instanceRoot] }); + expect(greet.stdout.trim()).toBe("hello, chaos"); + }, 60_000); + + it.runIf(process.platform === "darwin")("keeps agent code away from protected files but runs the fixture tests", async () => { + const secretRoot = await temp("restart-chaos-secret-"); + await writeFile(path.join(secretRoot, ".secrets"), "OPENAI_API_KEY=canary-value\n"); + const workspace = await temp("restart-chaos-ws-"); + await createFixtureRepository(workspace, 1); + const leak = await runOracleCommand(["/bin/cat", path.join(secretRoot, ".secrets")], workspace, { protectedRoots: [secretRoot] }); + expect(leak.exitCode).not.toBe(0); + expect(leak.stdout).not.toContain("canary-value"); + const greet = await runOracleCommand(["python3", "-m", "chaoscli", "greet", "chaos"], workspace, { protectedRoots: [secretRoot] }); + expect(greet).toMatchObject({ exitCode: 0 }); + expect(greet.stdout.trim()).toBe("hello, chaos"); + }, 60_000); +}); diff --git a/tests/restart-chaos/oracle-sandbox.ts b/tests/restart-chaos/oracle-sandbox.ts new file mode 100644 index 0000000000..50ae362b19 --- /dev/null +++ b/tests/restart-chaos/oracle-sandbox.ts @@ -0,0 +1,65 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const OUTPUT_LIMIT = 4_000; + +export interface OracleCommandResult { exitCode: number | null; stdout: string; stderr: string } + +/** Quote a path for an SBPL string literal. */ +function sbplString(value: string) { return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"")}"`; } + +/** + * macOS Seatbelt profile for a T-oracle command. The command runs code the + * agent could edit, so it may not read the operator's home directory (for + * example ~/.secrets or ~/.ssh), write anywhere except a scratch directory, or + * use the network. The workspace stays readable. + */ +export function oracleSandboxProfile(input: { workspace: string; scratch: string; protectedRoots: readonly string[] }): string { + const lines = ["(version 1)", "(allow default)", "(deny network*)", "(deny file-write*)", + `(allow file-write* (subpath ${sbplString(input.scratch)}) (literal "/dev/null") (literal "/dev/tty"))`]; + for (const root of input.protectedRoots) lines.push(`(deny file-read* (subpath ${sbplString(root)}))`); + // Later rules win: re-allow the workspace and scratch even if they sit under a protected root. + lines.push(`(allow file-read* (subpath ${sbplString(input.workspace)}) (subpath ${sbplString(input.scratch)}))`); + return lines.join("\n"); +} + +/** + * Run one oracle command against an agent workspace with a minimal environment, + * a scratch HOME, and (on macOS) a Seatbelt sandbox that hides the operator's + * home directory. Elsewhere it refuses unless explicitly allowed, because a + * temporary HOME alone does not stop absolute-path reads. + */ +export async function runOracleCommand(argv: readonly string[], workspace: string, options: { + protectedRoots?: readonly string[]; timeoutMs?: number; platform?: NodeJS.Platform; allowUnsandboxed?: boolean; +} = {}): Promise { + const platform = options.platform ?? process.platform; + const sandboxed = platform === "darwin"; + if (!sandboxed && !options.allowUnsandboxed && process.env.RESTART_CHAOS_ALLOW_UNSANDBOXED_ORACLE !== "1") { + return { exitCode: null, stdout: "", stderr: "oracle command refused: no sandbox on this platform (set RESTART_CHAOS_ALLOW_UNSANDBOXED_ORACLE=1 inside a disposable VM)" }; + } + // Seatbelt matches resolved paths (/var is /private/var on macOS). + const scratch = await realpath(await mkdtemp(path.join(os.tmpdir(), "restart-chaos-oracle-"))); + const resolve = (value: string) => realpath(value).catch(() => value); + try { + const env = { PATH: "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin", HOME: scratch, TMPDIR: scratch, + PYTHONDONTWRITEBYTECODE: "1", PYTHONNOUSERSITE: "1", LANG: "C.UTF-8" }; + const [command, args] = sandboxed + ? ["/usr/bin/sandbox-exec", ["-p", oracleSandboxProfile({ workspace: await resolve(workspace), scratch, + protectedRoots: await Promise.all((options.protectedRoots ?? [os.homedir()]).map(resolve)) }), ...argv]] + : [argv[0]!, argv.slice(1)]; + try { + const { stdout, stderr } = await execFileAsync(command, args, { cwd: workspace, env, timeout: options.timeoutMs ?? 60_000, maxBuffer: 1024 * 1024 }); + return { exitCode: 0, stdout: stdout.slice(-OUTPUT_LIMIT), stderr: stderr.slice(-OUTPUT_LIMIT) }; + } catch (error) { + const failure = error as { code?: number | string; stdout?: string; stderr?: string }; + return { exitCode: typeof failure.code === "number" ? failure.code : null, stdout: (failure.stdout ?? "").slice(-OUTPUT_LIMIT), + stderr: (failure.stderr ?? String(error)).slice(-OUTPUT_LIMIT) }; + } + } finally { + await rm(scratch, { recursive: true, force: true }); + } +} diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts new file mode 100644 index 0000000000..491d689760 --- /dev/null +++ b/tests/restart-chaos/scheduler.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { parseCommand, parseReplayRef, waveIdFor } from "./config.js"; +import { K4_HOLD_RANGE_MS, createRng, observedFaultSchedule, planTasks, planWave, replayWave, taskFaultSchedule } from "./scheduler.js"; +import type { WaveOptions } from "./types.js"; +import { workloadDefinitions } from "./workloads.js"; + +const options: WaveOptions = { env: "local", adapters: ["codex", "claude", "opencode"], workloads: ["A1", "B1", "B5", "C1"], + faults: ["K1", "K2", "K4"], concurrency: 4, tasks: 8, seed: 42, budgetCents: 3_000, pollMs: 3_000 }; + +describe("scheduler determinism", () => { + it("produces the same plan for the same seed", () => { + expect(planTasks(options)).toEqual(planTasks(options)); + }); + + it("produces a different plan for a different seed", () => { + expect(planTasks({ ...options, seed: 43 })).not.toEqual(planTasks(options)); + }); + + it("keeps fire times, holds and follow-ups inside each workload's window", () => { + for (const task of planTasks({ ...options, tasks: 60 })) { + const definition = workloadDefinitions[task.workload]; + expect(task.fireAfterMs).toBeGreaterThanOrEqual(definition.fireWindowMs[0]); + expect(task.fireAfterMs).toBeLessThanOrEqual(definition.fireWindowMs[1]); + if (task.fault === "K4") { + expect(task.holdMs).toBeGreaterThanOrEqual(K4_HOLD_RANGE_MS[0]); + expect(task.holdMs).toBeLessThanOrEqual(K4_HOLD_RANGE_MS[1]); + } else expect(task.holdMs).toBe(0); + expect(task.followUpAfterMs === null).toBe(task.workload !== "C1"); + expect(task.nonce).toMatch(/^[0-9a-f]{8}$/); + } + }); + + it("covers every workload × adapter × fault cell once per cycle", () => { + const cells = options.workloads.length * options.adapters.length * options.faults.length; + const tasks = planTasks({ ...options, tasks: cells }); + expect(new Set(tasks.map(task => `${task.workload}/${task.adapter}/${task.fault}`)).size).toBe(cells); + expect(tasks.map(task => task.taskId).slice(0, 3)).toEqual(["t01", "t02", "t03"]); + }); + + it("respects narrowed selections", () => { + const tasks = planTasks({ ...options, adapters: ["codex"], workloads: ["A1"], faults: ["K1"], tasks: 3 }); + expect(tasks.every(task => task.adapter === "codex" && task.workload === "A1" && task.fault === "K1")).toBe(true); + }); + + it("has a stable RNG sequence", () => { + const first = createRng(7); + const second = createRng(7); + expect(Array.from({ length: 5 }, () => first.next())).toEqual(Array.from({ length: 5 }, () => second.next())); + }); +}); + +describe("replay", () => { + const plan = planWave(waveIdFor(42, new Date("2026-11-06T10:11:12Z")), options, new Date("2026-11-06T10:11:12Z")); + + it("names waves with the timestamp and seed", () => { + expect(plan.waveId).toBe("w-20261106T101112-s42"); + }); + + it("reproduces exactly one saved task", () => { + const replay = replayWave(plan, { waveId: plan.waveId, taskId: "t03" }, 1_700_000_000_000); + expect(replay.tasks).toEqual([plan.tasks[2]]); + expect(replay.options.concurrency).toBe(1); + expect(replay.replayOf).toEqual({ waveId: plan.waveId, taskId: "t03" }); + expect(replay.waveId).toBe(`${plan.waveId}-replay-t03-1700000000000`); + // A replay of a replay names the original wave, and its id parses again. + const again = replayWave(replay, parseReplayRef(`${replay.waveId}/t03`), 1_700_000_000_001); + expect(again.waveId).toBe(`${plan.waveId}-replay-t03-1700000000001`); + }); + + it("replays the recorded fault sequence that crossed the task, other tasks' faults included", () => { + const createdAt = 1_000_000; + const faults = [ + { faultId: "f01", kind: "K1" as const, holdMs: 0, startedAt: createdAt + 42_000 }, + { faultId: "f02", kind: "K4" as const, holdMs: 150_000, startedAt: createdAt + 60_000 }, + { faultId: "f03", kind: "K2" as const, holdMs: 0, startedAt: createdAt + 900_000 }, + ]; + const observed = observedFaultSchedule(createdAt, ["f02", "f01"], faults); + expect(observed).toEqual([{ kind: "K1", holdMs: 0, atMs: 42_000 }, { kind: "K4", holdMs: 150_000, atMs: 60_000 }]); + const replay = replayWave(plan, { waveId: plan.waveId, taskId: "t02" }, 1, {}, observed); + expect(taskFaultSchedule(replay.tasks[0]!)).toEqual(observed); + // Without a recorded sequence the replay falls back to the planned fault. + const planned = replayWave(plan, { waveId: plan.waveId, taskId: "t02" }, 1); + expect(taskFaultSchedule(planned.tasks[0]!)).toEqual([{ kind: plan.tasks[1]!.fault, holdMs: plan.tasks[1]!.holdMs, atMs: plan.tasks[1]!.fireAfterMs }]); + }); + + it("rejects unknown tasks and mismatched waves", () => { + expect(() => replayWave(plan, { waveId: plan.waveId, taskId: "t99" })).toThrow(/no task t99/); + expect(() => replayWave(plan, { waveId: "w-20261106T101112-s1", taskId: "t01" })).toThrow(/not w-/); + }); +}); + +describe("CLI parsing", () => { + it("parses a wave command including pnpm's forwarded --", () => { + const parsed = parseCommand(["--", "--env", "local", "--adapters", "codex,claude,opencode", "--concurrency", "4", "--tasks", "8", "--faults", "K1,K2,K4", "--seed", "17"]); + expect(parsed).toMatchObject({ mode: "wave", options: { env: "local", adapters: ["codex", "claude", "opencode"], concurrency: 4, tasks: 8, faults: ["K1", "K2", "K4"], seed: 17 } }); + }); + + it("parses --replay", () => { + expect(parseCommand(["--replay", "w-20261106T101112-s42/t03"])).toMatchObject({ mode: "replay", ref: { waveId: "w-20261106T101112-s42", taskId: "t03" } }); + }); + + it("rejects replay mixed with plan options and malformed refs", () => { + expect(() => parseCommand(["--replay", "w-20261106T101112-s42/t03", "--seed", "1"])).toThrow(/reproduces the saved plan/); + expect(() => parseReplayRef("w-20261106T101112-s42")).toThrow(/\//); + expect(() => parseReplayRef("..%2Fetc/t01")).toThrow(/Invalid wave id/); + expect(() => parseReplayRef("../etc/t01")).toThrow(/\//); + expect(() => parseReplayRef("w-20261106T101112-s42/t1")).toThrow(/Invalid task id/); + }); + + it("rejects unknown faults, adapters and out-of-range numbers", () => { + expect(() => parseCommand(["--faults", "K9"])).toThrow(/Unknown faults/); + expect(() => parseCommand(["--adapters", "gemini"])).toThrow(/Unknown adapters/); + expect(() => parseCommand(["--concurrency", "0"])).toThrow(/between 1 and 20/); + expect(() => parseCommand(["--poll-ms", "500"])).toThrow(/between 2000 and 5000/); + }); +}); diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts new file mode 100644 index 0000000000..a4ceab2f5a --- /dev/null +++ b/tests/restart-chaos/scheduler.ts @@ -0,0 +1,100 @@ +import { workloadDefinitions } from "./workloads.js"; +import type { AdapterId, FaultKind, ScheduledFault, TaskPlan, WavePlan, WaveOptions, WorkloadId } from "./types.js"; +import type { ReplayRef } from "./config.js"; + +/** mulberry32: small, fast and fully determined by its 32-bit seed. */ +export function createRng(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; + }; + return { + next, + int(min: number, max: number) { return min + Math.floor(next() * (max - min + 1)); }, + hex(length: number) { let out = ""; while (out.length < length) out += Math.floor(next() * 16).toString(16); return out; }, + shuffle(values: readonly T[]): T[] { + const copy = [...values]; + for (let index = copy.length - 1; index > 0; index -= 1) { + const swap = Math.floor(next() * (index + 1)); + [copy[index], copy[swap]] = [copy[swap]!, copy[index]!]; + } + return copy; + }, + }; +} + +export const K4_HOLD_RANGE_MS: readonly [number, number] = [120_000, 300_000]; + +/** + * Pairwise-friendly sampling: cycle a seeded shuffle of the full + * workload × adapter × fault product so small waves still spread across cells. + */ +export function planTasks(options: Pick): TaskPlan[] { + const rng = createRng(options.seed); + const cells: Array<{ workload: WorkloadId; adapter: AdapterId; fault: FaultKind }> = []; + for (const workload of options.workloads) for (const adapter of options.adapters) for (const fault of options.faults) cells.push({ workload, adapter, fault }); + const tasks: TaskPlan[] = []; + let order = rng.shuffle(cells); + for (let index = 0; index < options.tasks; index += 1) { + if (index > 0 && index % cells.length === 0) order = rng.shuffle(cells); + const cell = order[index % cells.length]!; + const definition = workloadDefinitions[cell.workload]; + const [fireMin, fireMax] = definition.fireWindowMs; + tasks.push({ + taskId: `t${String(index + 1).padStart(2, "0")}`, + index, + ...cell, + fireAfterMs: rng.int(fireMin, fireMax), + holdMs: cell.fault === "K4" ? rng.int(K4_HOLD_RANGE_MS[0], K4_HOLD_RANGE_MS[1]) : 0, + followUpAfterMs: definition.followUpWindowMs ? rng.int(definition.followUpWindowMs[0], definition.followUpWindowMs[1]) : null, + nonce: rng.hex(8), + }); + } + return tasks; +} + +export function planWave(waveId: string, options: WaveOptions, createdAt = new Date()): WavePlan { + return { schema: "paperclip.restart-chaos.wave-plan/v1", waveId, seed: options.seed, + createdAt: createdAt.toISOString(), options, tasks: planTasks(options) }; +} + +/** Faults that actually crossed a task, as offsets from its launch. */ +export function observedFaultSchedule(createdAt: number | null, faultIds: readonly string[], + faults: ReadonlyArray<{ faultId: string; kind: FaultKind; holdMs: number; startedAt: number }>): ScheduledFault[] { + if (createdAt === null) return []; + return faults.filter(fault => faultIds.includes(fault.faultId)) + .map(fault => ({ kind: fault.kind, holdMs: fault.holdMs, atMs: Math.max(0, fault.startedAt - createdAt) })) + .sort((a, b) => a.atMs - b.atMs); +} + +/** The faults a task schedules: its replayed sequence, or its one planned fault. */ +export function taskFaultSchedule(task: TaskPlan): ScheduledFault[] { + return task.replayFaults?.length ? task.replayFaults : [{ kind: task.fault, holdMs: task.holdMs, atMs: task.fireAfterMs }]; +} + +/** + * A replay keeps the saved cell, hold and nonce and runs alone. When the source + * wave recorded which faults crossed the task (including other tasks' faults), + * the replay fires that same sequence at the same offsets from launch. + */ +export function replayWave(source: WavePlan, ref: ReplayRef, now = Date.now(), + overrides: Partial> = {}, observed: ScheduledFault[] = []): WavePlan { + if (source.schema !== "paperclip.restart-chaos.wave-plan/v1") throw new Error("Unsupported wave plan schema"); + if (source.waveId !== ref.waveId) throw new Error(`Plan is for ${source.waveId}, not ${ref.waveId}`); + const task = source.tasks.find(candidate => candidate.taskId === ref.taskId); + if (!task) throw new Error(`Wave ${ref.waveId} has no task ${ref.taskId}`); + const base = source.waveId.replace(/-replay-t[0-9]{2,3}-[0-9]+$/, ""); + return { + schema: source.schema, + waveId: `${base}-replay-${task.taskId}-${now}`, + seed: source.seed, + createdAt: new Date(now).toISOString(), + options: { ...source.options, ...overrides, tasks: 1, concurrency: 1 }, + tasks: [{ ...task, ...(observed.length ? { replayFaults: observed.map(fault => ({ ...fault })) } : {}) }], + replayOf: { ...ref }, + }; +} diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts new file mode 100644 index 0000000000..b9ee5f26c9 --- /dev/null +++ b/tests/restart-chaos/seed.ts @@ -0,0 +1,119 @@ +import { execFileSync } from "node:child_process"; +import { chmod, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { runnerProfiles } from "../runner-e2e/catalog.js"; +import type { SecretReference, SecretReferenceMap } from "../runner-e2e/types.js"; +import type { ChaosApi } from "./api.js"; +import { CREDENTIAL_FOR_ADAPTER, PROFILE_FOR_ADAPTER } from "./config.js"; +import type { Timeline } from "./timeline.js"; +import type { AdapterId, EnvironmentKind } from "./types.js"; +import { fixtureRepositoryFiles } from "./workloads.js"; + +export interface ChaosLab { + companyId: string; + environmentId: string; + projectId: string; + fixtureRepositoryPath: string; + agents: Record; +} + +/** Create the `chaos-sandbox` git repository: python CLI, one failing test, bench.sh. */ +export async function createFixtureRepository(directory: string, benchSeconds?: number): Promise { + await mkdir(directory, { recursive: true }); + for (const [relative, file] of Object.entries(fixtureRepositoryFiles(benchSeconds))) { + const target = path.join(directory, relative); + await mkdir(path.dirname(target), { recursive: true }); + await writeFile(target, file.content); + if (file.mode) await chmod(target, file.mode); + } + const git = (...args: string[]) => execFileSync("git", args, { cwd: directory, stdio: "pipe", + env: { ...process.env, GIT_AUTHOR_NAME: "Chaos Lab", GIT_AUTHOR_EMAIL: "chaos@example.invalid", + GIT_COMMITTER_NAME: "Chaos Lab", GIT_COMMITTER_EMAIL: "chaos@example.invalid" } }); + git("init", "-q", "-b", "main"); + git("add", "-A"); + git("commit", "-q", "-m", "chaos-sandbox fixture"); + return directory; +} + +/** + * Seeds the isolated instance. Credentials cross into the server only as + * encrypted company secrets, as in tests/runner-e2e/live-fixtures.ts, and the + * agent payloads come from the shared runner-e2e catalog profiles. + */ +export async function seedChaosLab(input: { + api: ChaosApi; + env: EnvironmentKind; + adapters: readonly AdapterId[]; + concurrency: number; + budgetCents: number; + fixtureRoot: string; + waveId: string; + timeline: Timeline; +}): Promise { + const { api, timeline } = input; + if (input.env === "daytona") { + // TODO(restart-chaos phase 1b): create the Daytona sandbox environment from + // the prepared image (see tests/runner-e2e/catalog.ts runnerEnvironments + // and live-fixtures.ts installFixtureDaytonaProvider) and add the R6 + // sandbox sweep. The image and env are being prepared separately. + throw new Error("--env daytona is not wired yet; use --env local"); + } + const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); + const company = await api.post<{ id: string }>("/api/companies", { + name: "Chaos Lab", + description: `Restart chaos harness wave ${input.waveId}`, + budgetMonthlyCents: input.budgetCents, + }); + timeline.add("seed_company", { companyId: company.id }); + + const secretRefs: SecretReferenceMap = {}; + for (const adapter of input.adapters) { + const key = CREDENTIAL_FOR_ADAPTER[adapter]; + if (secretRefs[key]) continue; + const value = process.env[key]; + if (!value) throw new Error(`Missing ${key} for adapter ${adapter}; load ~/.secrets into the environment first`); + const created = await api.post<{ id: string }>(`/api/companies/${company.id}/secrets`, { + name: `Chaos ${key}`, key, value, description: "Restart chaos harness provider credential", + }); + secretRefs[key] = { type: "secret_ref", secretId: created.id, version: "latest" } satisfies SecretReference; + } + timeline.add("seed_secrets", { keys: Object.keys(secretRefs) }); + + const environments = await api.get>(`/api/companies/${company.id}/environments?driver=local`); + const environment = environments.find(candidate => candidate.driver === "local"); + if (!environment) throw new Error("Isolated instance has no managed local environment"); + + const agents = {} as ChaosLab["agents"]; + for (const adapter of input.adapters) { + const profile = runnerProfiles.find(candidate => candidate.id === PROFILE_FOR_ADAPTER[adapter]); + if (!profile) throw new Error(`runner-e2e catalog has no profile ${PROFILE_FOR_ADAPTER[adapter]}`); + const payload = profile.buildAgent({ environmentId: environment.id, environmentFixtureId: "local", + workspacePath: fixtureRepositoryPath, secretRefs, executionId: input.waveId }); + const name = `Chaos ${adapter}`; + const runtimeConfig = (payload.runtimeConfig ?? {}) as Record; + const created = await api.post<{ id: string; name: string }>(`/api/companies/${company.id}/agents`, { + ...payload, + name, + title: "Restart chaos worker", + runtimeConfig: { ...runtimeConfig, heartbeat: { ...((runtimeConfig.heartbeat ?? {}) as Record), + enabled: false, maxConcurrentRuns: Math.max(1, input.concurrency) } }, + }); + agents[adapter] = { id: created.id, name }; + timeline.add("seed_agent", { adapter, agentId: created.id, profile: profile.id, model: profile.model }); + } + + const project = await api.post<{ id: string }>(`/api/companies/${company.id}/projects`, { + name: "chaos-sandbox", + description: "Fixture repository for restart chaos workloads", + executionWorkspacePolicy: { + enabled: true, + defaultMode: "isolated_workspace", + allowIssueOverride: false, + environmentId: environment.id, + workspaceStrategy: { type: "git_worktree" }, + }, + workspace: { name: "Primary", sourceType: "local_path", cwd: fixtureRepositoryPath, isPrimary: true }, + }); + timeline.add("seed_project", { projectId: project.id, fixtureRepositoryPath }); + return { companyId: company.id, environmentId: environment.id, projectId: project.id, fixtureRepositoryPath, agents }; +} diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts new file mode 100644 index 0000000000..2cb1ccd5d5 --- /dev/null +++ b/tests/restart-chaos/supervisor.ts @@ -0,0 +1,299 @@ +/** + * Owns one isolated Paperclip control plane and injects K faults. + * + * Reuses the runner-e2e process code instead of copying it: the config seed + * (server-config.ts), port reservation (ports.ts), environment isolation + * (harness-env.ts), wrapper-owned Postgres (controller-database.ts), the + * process-tree stopper (server-stop.ts + process-tree-owner.ts), the source + * server entry (server-entry.ts) and the stop → unhealthy → hold → start → + * ready receipt in controller-outage.ts `replaceController`. + * + * Postgres runs beside, not under, the server process tree, so K1 kills only + * the controller and its owned descendants (local runners included), never the + * database (that is K9). The controller runtime cache is kept: discarding it is + * the separate K11 fault. + */ +import { spawn, type ChildProcess } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { createWriteStream, statSync, type WriteStream } from "node:fs"; +import { mkdir, readFile, symlink } from "node:fs/promises"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { prepareControllerDatabase } from "../runner-e2e/controller-database.js"; +import { replaceController, type ControllerOutageReceipt } from "../runner-e2e/controller-outage.js"; +import { + assertIsolatedServerEnvironment, buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment, + resolvePaperclipRunnerBinaryForHarness, +} from "../runner-e2e/harness-env.js"; +import { reserveRunnerE2EServerPort } from "../runner-e2e/ports.js"; +import { createRunnerE2ETemporaryRoot, prepareRunnerE2EServerConfig } from "../runner-e2e/server-config.js"; +import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "../runner-e2e/server-stop.js"; +import { prepareSourceServerDependencies } from "../runner-e2e/source-startup.js"; +import { runnerE2ETypeScriptProcessArgs } from "../runner-e2e/web-server-command.js"; +import type { MatrixExecution } from "../runner-e2e/types.js"; +import type { Timeline } from "./timeline.js"; +import type { FaultKind, FaultReceipt } from "./types.js"; + +const HEALTH_TIMEOUT_MS = 180_000; +const READY_TIMEOUT_MS = 300_000; +const STOP_TIMEOUT_MS = 30_000; + +export const FAULT_SIGNAL: Record = { K1: "SIGKILL", K2: "SIGTERM", K4: "SIGKILL" }; + +export interface SupervisorPaths { temporaryRoot: string; home: string; configPath: string; serverLogPath: string } + +const secret = () => randomBytes(48).toString("base64url"); + +/** Same provider PATH shape as the runner-e2e launcher: pinned Codex first. */ +async function prepareProviderPath(repositoryRoot: string, temporaryRoot: string, inherited: string | undefined) { + const toolBin = path.join(temporaryRoot, "provider-bin"); + await mkdir(toolBin, { recursive: true }); + const runnerRequire = createRequire(path.join(repositoryRoot, "packages/paperclip-runner/package.json")); + const codexAcp = runnerRequire.resolve("@agentclientprotocol/codex-acp/package.json"); + const codexPackage = createRequire(codexAcp).resolve("@openai/codex/package.json"); + const manifest = JSON.parse(await readFile(codexPackage, "utf8")) as { bin?: string | Record }; + const codexBin = typeof manifest.bin === "string" ? manifest.bin : manifest.bin?.codex; + if (!codexBin) throw new Error("Pinned Codex executable not found"); + await symlink(path.resolve(path.dirname(codexPackage), codexBin), path.join(toolBin, "codex")); + return [toolBin, path.join(repositoryRoot, "packages/paperclip-runner/node_modules/.bin"), inherited].filter(Boolean).join(path.delimiter); +} + +export class ChaosSupervisor { + readonly baseUrl: string; + private child: ChildProcess | null = null; + private readonly expectedStops = new WeakSet(); + private readonly spawnErrors = new WeakMap(); + private readonly log: WriteStream; + private readonly stopper: ReturnType; + private database: { cleanup(): Promise } | undefined; + private environment: Record = {}; + private faultChain: Promise = Promise.resolve(); + private faultCount = 0; + private shuttingDown = false; + /** False from fault start until /api/ready reports `ready` again. */ + serverReady = false; + faultInProgress = false; + unexpectedExits = 0; + + private constructor(readonly repositoryRoot: string, readonly port: number, readonly paths: SupervisorPaths, + private readonly timeline: Timeline) { + this.baseUrl = `http://127.0.0.1:${port}`; + this.log = createWriteStream(paths.serverLogPath, { flags: "a", mode: 0o600 }); + this.stopper = createRunnerE2EServerStopper({ + gracefulTimeoutMs: STOP_TIMEOUT_MS, forcedTimeoutMs: 5_000, + hasSpawnError: candidate => this.spawnErrors.has(candidate), + markExpectedStop: candidate => { this.expectedStops.add(candidate); }, + log: message => { this.log.write(message); }, + }); + } + + static async create(input: { repositoryRoot: string; waveDir: string; timeline: Timeline }): Promise { + const temporaryRoot = await createRunnerE2ETemporaryRoot(); + const home = path.join(temporaryRoot, "home"); + const paths: SupervisorPaths = { temporaryRoot, home, configPath: path.join(home, "instances", "default", "config.json"), + serverLogPath: path.join(input.waveDir, "server.log") }; + const port = await reserveRunnerE2EServerPort(); + const supervisor = new ChaosSupervisor(input.repositoryRoot, port, paths, input.timeline); + input.timeline.add("supervisor_created", { temporaryRoot, port }); + return supervisor; + } + + serverLogOffset(): number { + try { return statSync(this.paths.serverLogPath).size + this.log.writableLength; } catch { return 0; } + } + + async start(): Promise { + const { temporaryRoot, home, configPath } = this.paths; + const reservation = await prepareRunnerE2EServerConfig({ temporaryRoot, configPath, serverPort: this.port }); + await reservation?.close(); + // Wrapper-owned Postgres: survives K1/K2/K4 like a managed database would. + this.database = await prepareControllerDatabase(configPath, temporaryRoot); + this.timeline.add("database_started", {}); + const runnerBinary = resolvePaperclipRunnerBinaryForHarness( + [{ profile: { generation: "native" } } as unknown as MatrixExecution], this.repositoryRoot); + const source = { + ...buildRunnerE2EProcessEnvironment(process.env, []), + PATH: await prepareProviderPath(this.repositoryRoot, temporaryRoot, process.env.PATH), + PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: "[]", + PAPERCLIP_RUNNER_BINARY: runnerBinary, + PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"), + }; + const overrides = { + NODE_ENV: "test", PORT: String(this.port), XDG_CACHE_HOME: path.join(temporaryRoot, "xdg-cache"), + PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "default", + PAPERCLIP_AGENT_JWT_SECRET: secret(), PAPERCLIP_DECISION_SIGNING_SECRET: secret(), + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: secret(), BETTER_AUTH_SECRET: secret(), + PAPERCLIP_BIND: "loopback", PAPERCLIP_BIND_HOST: "127.0.0.1", PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", + PAPERCLIP_DEPLOYMENT_EXPOSURE: "private", SERVE_UI: "true", PAPERCLIP_STORAGE_PROVIDER: "local_disk", + PAPERCLIP_STORAGE_LOCAL_DIR: path.join(temporaryRoot, "storage"), PAPERCLIP_SECRETS_PROVIDER: "local_encrypted", + PAPERCLIP_SECRETS_STRICT_MODE: "true", PAPERCLIP_DB_BACKUP_ENABLED: "false", + PAPERCLIP_DB_BACKUP_DIR: path.join(temporaryRoot, "backups"), PAPERCLIP_OPEN_ON_LISTEN: "false", + }; + const environment = buildPaperclipServerEnvironment(source, overrides); + assertIsolatedServerEnvironment(environment, { temporaryRoot, paperclipHome: home, configPath }); + this.environment = Object.fromEntries(Object.entries(environment).filter((entry): entry is [string, string] => entry[1] !== undefined)); + await prepareSourceServerDependencies(this.repositoryRoot, this.environment, false); + const child = await this.spawnServer(); + await this.waitForHealth(child); + await this.waitForReadyPhase([]); + this.timeline.add("server_ready", { pid: child.pid }); + } + + private async spawnServer(): Promise { + if (this.shuttingDown) throw new Error("Refusing to start Paperclip during harness shutdown"); + const entry = path.join(this.repositoryRoot, "tests/runner-e2e/server-entry.ts"); + const child = spawn(process.execPath, runnerE2ETypeScriptProcessArgs(this.repositoryRoot, entry, ["onboard", "--yes", "--run"]), + { cwd: this.repositoryRoot, env: this.environment, stdio: ["ignore", "pipe", "pipe"], detached: runnerE2EServerDetached }); + this.child = child; + child.stdout?.on("data", chunk => this.log.write(chunk)); + child.stderr?.on("data", chunk => this.log.write(chunk)); + child.once("error", error => { this.spawnErrors.set(child, error); }); + child.once("exit", (code, signal) => { + this.log.write(`\n[restart-chaos] server pid=${child.pid} exited code=${code} signal=${signal}\n`); + if (!this.expectedStops.has(child) && !this.shuttingDown) { + this.unexpectedExits += 1; + this.serverReady = false; + this.timeline.add("server_exited_unexpectedly", { pid: child.pid, code, signal }); + } + }); + await this.stopper.watch(child); + this.timeline.add("server_spawned", { pid: child.pid }); + return child; + } + + private exited(child: ChildProcess) { return child.exitCode !== null || child.signalCode !== null || this.spawnErrors.has(child); } + + private async readPhase(): Promise { + try { + const response = await fetch(`${this.baseUrl}/api/ready`, { signal: AbortSignal.timeout(1_000), cache: "no-store" }); + return String(((await response.json()) as { status?: string }).status ?? response.status); + } catch { return "unreachable"; } + } + + private async waitForHealth(child: ChildProcess, phases?: FaultReceipt["phases"]): Promise { + const deadline = Date.now() + HEALTH_TIMEOUT_MS; + while (Date.now() < deadline) { + if (this.shuttingDown) throw new Error("Harness shutdown interrupted startup"); + if (this.exited(child)) throw new Error(`Paperclip server exited during startup (code=${child.exitCode} signal=${child.signalCode})`); + if (phases) { + const phase = await this.readPhase(); + if (phase !== phases[phases.length - 1]?.phase) phases.push({ phase, at: Date.now() }); + } + try { + const response = await fetch(`${this.baseUrl}/api/health`, { signal: AbortSignal.timeout(1_000) }); + await response.body?.cancel(); + if (response.ok) return Date.now(); + } catch { /* still booting */ } + await delay(250); + } + throw new Error(`Paperclip server did not become healthy within ${HEALTH_TIMEOUT_MS}ms`); + } + + private async waitForHealthToStop(): Promise { + const deadline = Date.now() + STOP_TIMEOUT_MS; + while (Date.now() < deadline) { + try { const response = await fetch(`${this.baseUrl}/api/health`, { signal: AbortSignal.timeout(500) }); await response.body?.cancel(); } + catch { return; } + await delay(100); + } + throw new Error("Old Paperclip server stayed healthy after stop"); + } + + /** Records starting → recovering → ready from /api/ready (liveness gate). */ + private async waitForReadyPhase(phases: FaultReceipt["phases"]): Promise { + const deadline = Date.now() + READY_TIMEOUT_MS; + let last: string | null = phases[phases.length - 1]?.phase ?? null; + while (Date.now() < deadline) { + if (this.shuttingDown) throw new Error("Harness shutdown interrupted readiness wait"); + const phase = await this.readPhase(); + if (phase !== last) { phases.push({ phase, at: Date.now() }); last = phase; } + if (phase === "ready") { + const liveness = await fetch(`${this.baseUrl}/api/liveness`, { signal: AbortSignal.timeout(1_000) }).catch(() => null); + await liveness?.body?.cancel(); + if (liveness?.ok) { this.serverReady = true; return Date.now(); } + } + await delay(250); + } + throw new Error(`Paperclip did not report ready within ${READY_TIMEOUT_MS}ms`); + } + + /** Faults are serialized: one control plane, one outage at a time. */ + fault(kind: FaultKind, holdMs: number, requestedBy: string): Promise { + const requestedAt = Date.now(); + const run = this.faultChain.then(() => this.executeFault(kind, holdMs, requestedBy, requestedAt)); + this.faultChain = run.catch(() => undefined); + return run; + } + + private async executeFault(kind: FaultKind, holdMs: number, requestedBy: string, requestedAt: number): Promise { + const previous = this.child; + this.faultCount += 1; + const receipt: FaultReceipt = { + faultId: `f${String(this.faultCount).padStart(2, "0")}`, kind, requestedBy, signal: FAULT_SIGNAL[kind], holdMs, + requestedAt, startedAt: Date.now(), stoppedAt: null, restartedAt: null, healthyAt: null, readyAt: null, + phases: [], serverLogOffsetAtStart: this.serverLogOffset(), serverLogOffsetAtReady: null, error: null, + }; + this.faultInProgress = true; + this.serverReady = false; + this.timeline.add("fault_started", { faultId: receipt.faultId, kind, signal: receipt.signal, holdMs, requestedBy, previousPid: previous?.pid }); + this.log.write(`\n[restart-chaos] ${receipt.faultId} ${kind} ${receipt.signal} hold=${holdMs}ms requestedBy=${requestedBy} at ${new Date().toISOString()}\n`); + try { + if (!previous?.pid || this.exited(previous)) throw new Error("No live Paperclip server to fault"); + let outage: ControllerOutageReceipt | undefined; + outage = await replaceController({ + requestId: `chaos-${receipt.faultId}-${requestedBy}`, + previousPid: previous.pid, + outage: { signal: receipt.signal, holdMs }, + stop: async signal => { + if (signal === "SIGKILL") await this.stopper.crashForRestart(previous); + else await this.stopper(previous, "SIGTERM"); + if (this.child === previous) this.child = null; + }, + waitForUnhealthy: async () => { await this.waitForHealthToStop(); receipt.stoppedAt = Date.now(); this.timeline.add("fault_server_down", { faultId: receipt.faultId }); }, + // K1/K2/K4 keep the controller runtime cache; K11 will discard it. + discardCache: async () => {}, + start: async () => { receipt.restartedAt = Date.now(); this.timeline.add("fault_restarting", { faultId: receipt.faultId }); return this.spawnServer(); }, + waitForReady: async replacement => { receipt.healthyAt = await this.waitForHealth(replacement, receipt.phases); }, + cancelled: () => this.shuttingDown, + record: async progress => { outage = progress; }, + }); + receipt.readyAt = await this.waitForReadyPhase(receipt.phases); + receipt.serverLogOffsetAtReady = this.serverLogOffset(); + this.timeline.add("fault_ready", { faultId: receipt.faultId, replacementPid: outage?.replacementPid, + downMs: (receipt.restartedAt ?? 0) - (receipt.stoppedAt ?? 0), readyAfterMs: receipt.readyAt - receipt.startedAt, phases: receipt.phases }); + } catch (error) { + receipt.error = error instanceof Error ? error.message : String(error); + this.timeline.add("fault_failed", { faultId: receipt.faultId, error: receipt.error }); + throw Object.assign(new Error(`Fault ${receipt.faultId} failed: ${receipt.error}`), { receipt }); + } finally { + this.faultInProgress = false; + } + return receipt; + } + + /** + * Bounded recovery after a failed fault or a crash the harness did not cause: + * wait for a live server to become ready, or start a new one. Throws when the + * control plane cannot be brought back, so the wave fails instead of hanging. + */ + async ensureServing(): Promise { + if (this.serverReady || this.faultInProgress) return; + const live = this.child && !this.exited(this.child) ? this.child : null; + this.timeline.add("server_recovery_attempt", { livePid: live?.pid ?? null }); + const child = live ?? await this.spawnServer(); + await this.waitForHealth(child); + await this.waitForReadyPhase([]); + this.timeline.add("server_recovered", { pid: child.pid }); + } + + async stop(): Promise { + this.shuttingDown = true; + const errors: string[] = []; + try { await this.stopper.stopAll(); } catch (error) { errors.push(error instanceof Error ? error.message : String(error)); } + try { await this.database?.cleanup(); } catch (error) { errors.push(error instanceof Error ? error.message : String(error)); } + await new Promise(resolve => this.log.end(resolve)); + this.timeline.add("supervisor_stopped", { errors }); + if (errors.length) throw new Error(`Supervisor cleanup incomplete: ${errors.join("; ")}`); + } +} diff --git a/tests/restart-chaos/timeline.ts b/tests/restart-chaos/timeline.ts new file mode 100644 index 0000000000..95d73d1ead --- /dev/null +++ b/tests/restart-chaos/timeline.ts @@ -0,0 +1,18 @@ +import { appendFileSync } from "node:fs"; + +export interface TimelineEntry { at: string; t: number; kind: string; [key: string]: unknown } + +/** Append-only action timeline. Every supervisor and launcher action lands here. */ +export class Timeline { + readonly entries: TimelineEntry[] = []; + constructor(private readonly path: string | null, readonly origin = Date.now()) {} + + add(kind: string, data: Record = {}): TimelineEntry { + const now = Date.now(); + const entry: TimelineEntry = { at: new Date(now).toISOString(), t: now - this.origin, kind, ...data }; + this.entries.push(entry); + if (this.path) appendFileSync(this.path, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); + if (process.env.RESTART_CHAOS_QUIET !== "1") console.info(`[chaos +${Math.round(entry.t / 1000)}s] ${kind} ${JSON.stringify(data)}`); + return entry; + } +} diff --git a/tests/restart-chaos/tsconfig.json b/tests/restart-chaos/tsconfig.json new file mode 100644 index 0000000000..03eb032646 --- /dev/null +++ b/tests/restart-chaos/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "../..", + "types": ["node"], + "typeRoots": [ + "../../cli/node_modules/@types", + "../../server/node_modules/@types" + ] + }, + "include": ["./**/*.ts", "../../server/src/types/express.d.ts"] +} diff --git a/tests/restart-chaos/tsconfig.layers.json b/tests/restart-chaos/tsconfig.layers.json new file mode 100644 index 0000000000..f1c6863228 --- /dev/null +++ b/tests/restart-chaos/tsconfig.layers.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "declarationMap": false + }, + "references": [{ "path": "../../server/node_modules/.cache/paperclip-typecheck/tsconfig.declarations.json" }] +} diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts new file mode 100644 index 0000000000..ce2203f878 --- /dev/null +++ b/tests/restart-chaos/types.ts @@ -0,0 +1,182 @@ +/** Shared types for the control-plane restart chaos harness (plan §5). */ + +export const FAULT_KINDS = ["K1", "K2", "K4"] as const; +export type FaultKind = (typeof FAULT_KINDS)[number]; + +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1"] as const; +export type WorkloadId = (typeof WORKLOAD_IDS)[number]; + +export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; +export type AdapterId = (typeof ADAPTER_IDS)[number]; + +export const ENVIRONMENT_KINDS = ["local", "daytona"] as const; +export type EnvironmentKind = (typeof ENVIRONMENT_KINDS)[number]; + +export interface WaveOptions { + env: EnvironmentKind; + adapters: AdapterId[]; + workloads: WorkloadId[]; + faults: FaultKind[]; + concurrency: number; + tasks: number; + seed: number; + /** Company budget hard cap in cents; the product hard-stops at this value. */ + budgetCents: number; + /** Poll interval for the observer, 2–5 s per plan §5. */ + pollMs: number; +} + +export interface TaskPlan { + taskId: string; + index: number; + workload: WorkloadId; + adapter: AdapterId; + fault: FaultKind; + /** Fire the fault this long after the task's issue is created. */ + fireAfterMs: number; + /** Outage hold between stop and restart. 0 for K1/K2. */ + holdMs: number; + /** C1 only: post the follow-up comment this long after creation. */ + followUpAfterMs: number | null; + /** Unique marker used by the deterministic T-oracle. */ + nonce: string; + /** + * Replay only: every fault that crossed this task in the source wave, with + * its offset from the task's launch. Replaces the single planned fault. + */ + replayFaults?: ScheduledFault[]; +} + +export interface ScheduledFault { kind: FaultKind; holdMs: number; atMs: number } + +export interface WavePlan { + schema: "paperclip.restart-chaos.wave-plan/v1"; + waveId: string; + seed: number; + createdAt: string; + options: WaveOptions; + tasks: TaskPlan[]; + /** Present when this wave replays one task of an earlier wave. */ + replayOf?: { waveId: string; taskId: string }; +} + +export type Verdict = "pass" | "fail" | "n/a"; +export interface RuleResult { verdict: Verdict; detail: string } +export type RuleId = "R1" | "R2" | "R3" | "R4" | "R5" | "R6"; + +export interface RunSample { + id: string; + status: string; + createdAt: string | null; + startedAt: string | null; + finishedAt: string | null; + errorCode: string | null; + retryOfRunId: string | null; + wakeCommentIds: string[]; + livenessState: string | null; +} + +export interface InterruptionSample { + state: string; + canRetry: boolean; + retryWait: boolean; + workspaceFailure: string | null; + observedRunId: string | null; + userAction: string | null; +} + +/** One observer poll. Unreachable polls keep only the timestamp. */ +export interface TaskSample { + at: number; + reachable: boolean; + serverReady: boolean; + issueStatus?: string; + interruption?: InterruptionSample | null; + activeRecoveryAction?: boolean; + /** Owner of the active recovery action; "board" means it waits on a person. */ + recoveryOwner?: string | null; + executionBlocker?: boolean; + scheduledRetry?: boolean; + runs?: RunSample[]; +} + +export interface RetryAttempt { + at: number; + observedRunId: string | null; + httpStatus: number | null; + accepted: boolean | null; + error: string | null; +} + +export interface FaultReceipt { + faultId: string; + kind: FaultKind; + /** Task that scheduled the fault; every other running task is also exposed. */ + requestedBy: string; + signal: "SIGKILL" | "SIGTERM"; + holdMs: number; + requestedAt: number; + startedAt: number; + stoppedAt: number | null; + restartedAt: number | null; + healthyAt: number | null; + readyAt: number | null; + phases: Array<{ phase: string; at: number }>; + serverLogOffsetAtStart: number; + serverLogOffsetAtReady: number | null; + error: string | null; +} + +export interface OracleArtifacts { + issueStatus: string | null; + documents: Record; + agentComments: string[]; + files: Record; + commands: Record; + workspacePath: string | null; + /** B5: distinct `.bench/progress` values the harness itself saw while the task was live. */ + benchProgressObserved?: number; +} + +export interface OracleResult { verdict: Exclude; checks: Array<{ name: string; ok: boolean; detail?: string }> } + +export interface RestSweep { + activeRuns: string[]; + runningWorkspaceOperations: string[]; + activeRecoveryAction: boolean; + error: string | null; +} + +export interface TaskRecord { + plan: TaskPlan; + issueId: string | null; + identifier: string | null; + createdAt: number | null; + launchError: string | null; + samples: TaskSample[]; + retry: RetryAttempt | null; + restAt: number | null; + finalStatus: string | null; + deadlineAt: number | null; + timedOut: boolean; + followUp: { at: number; commentId: string | null; error: string | null } | null; + faultIds: string[]; + sweep: RestSweep | null; + /** Workspace files present just before the first fault that hit this task (R2). */ + preFaultFiles: string[] | null; + /** Set when the pre-fault snapshot could not read the workspace; R2 then fails visibly. */ + preFaultSnapshotError?: string | null; + /** Pre-fault files missing at rest; null when not checked. */ + lostFiles: string[] | null; + /** B5: distinct bench progress values the observer read during polling. */ + benchProgressSeen?: string[]; +} + +export interface TaskGrade { + crossed: boolean; + crossingFaults: string[]; + rules: Record; + T: OracleResult; + pass: boolean; + signature: string | null; +} diff --git a/tests/restart-chaos/vitest.config.ts b/tests/restart-chaos/vitest.config.ts new file mode 100644 index 0000000000..aeafb2963b --- /dev/null +++ b/tests/restart-chaos/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +/** Credential-free unit tests only; the harness itself is run by main.ts. */ +export default defineConfig({ + root: import.meta.dirname, + test: { + name: "restart-chaos-unit", + include: ["**/*.test.ts"], + exclude: ["results/**", "node_modules/**"], + testTimeout: 30_000, + }, +}); diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts new file mode 100644 index 0000000000..a9f85a26fb --- /dev/null +++ b/tests/restart-chaos/workloads.ts @@ -0,0 +1,241 @@ +import type { OracleArtifacts, OracleResult, WorkloadId } from "./types.js"; + +/** Deterministic bench output: sum of squares 1..240. */ +export const BENCH_STEPS = 240; +export const BENCH_EXPECTED = `BENCH_RESULT=${(BENCH_STEPS * (BENCH_STEPS + 1) * (2 * BENCH_STEPS + 1)) / 6}`; + +export interface WorkloadDefinition { + id: WorkloadId; + label: string; + usesRepository: boolean; + /** Fault fire offset from issue creation (seeded uniform). */ + fireWindowMs: readonly [number, number]; + followUpWindowMs?: readonly [number, number]; + /** Time budget for the task excluding outage time (R1). */ + budgetMs: number; + title(nonce: string): string; + description(nonce: string): string; + followUp?(nonce: string): string; + /** Files and commands the observer must collect for the T-oracle. */ + collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] }> }; + grade(nonce: string, artifacts: OracleArtifacts): OracleResult; +} + +const briefMarker = (nonce: string) => `CHAOS-${nonce}`; +const followUpMarker = (nonce: string) => `FOLLOWUP-${nonce}`; +const notesFile = (nonce: string) => `NOTES-${nonce}.md`; +const benchFile = (nonce: string) => `bench-result-${nonce}.txt`; +const benchRecord = (nonce: string) => `.bench/run-${nonce}.txt`; +/** bench.sh must have actually run: at least this long between start and finish. */ +export const BENCH_MIN_SECONDS = 200; +/** B5: distinct progress values the harness must have observed itself while polling. */ +export const BENCH_MIN_OBSERVED_PROGRESS = 3; + +/** Parse bench.sh's own completion record: `started= finished= result=`. */ +export function parseBenchRecord(text: string | null | undefined) { + const fields = Object.fromEntries((text ?? "").trim().split(/\s+/).map(part => part.split("=")).filter(pair => pair.length >= 2) + .map(([key, ...rest]) => [key!, rest.join("=")])); + const started = Number(fields.started), finished = Number(fields.finished); + return Number.isFinite(started) && Number.isFinite(finished) && fields.result + ? { seconds: finished - started, result: fields.result } : null; +} + +function result(checks: OracleResult["checks"]): OracleResult { + return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; +} + +function statusCheck(artifacts: OracleArtifacts) { + return { name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` }; +} + +function briefChecks(body: string | null | undefined, markers: string[]) { + const text = body ?? ""; + const bullets = text.split(/\r?\n/).filter(line => /^\s*[-*] \S/.test(line)).length; + return [ + { name: "brief_document_present", ok: body != null && text.trim().length > 0 }, + ...markers.map(marker => ({ name: `brief_contains_${marker.split("-")[0]!.toLowerCase()}_marker`, ok: text.includes(marker), detail: marker })), + { name: "brief_has_three_bullets", ok: bullets >= 3, detail: `bullets=${bullets}` }, + ]; +} + +const briefDescription = (nonce: string) => [ + "Write a short brief on why idempotent retries matter for background job queues.", + "Save it as this issue's document with key `brief` (title \"Brief\"), in markdown.", + `The document must contain the line \`${briefMarker(nonce)}\` exactly, on its own line,`, + "and at least three bullet points that start with \"- \".", + "Do not create files or other tasks. When the document is saved, mark this task done.", +].join("\n"); + +export const workloadDefinitions: Record = { + A1: { + id: "A1", + label: "Brief to issue document", + usesRepository: false, + fireWindowMs: [10_000, 45_000], + budgetMs: 10 * 60_000, + title: nonce => `Chaos A1 brief ${nonce}`, + description: briefDescription, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade(nonce, artifacts) { + return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce)])]); + }, + }, + C1: { + id: "C1", + label: "Comment while running", + usesRepository: false, + fireWindowMs: [20_000, 60_000], + followUpWindowMs: [8_000, 25_000], + budgetMs: 12 * 60_000, + title: nonce => `Chaos C1 brief with follow-up ${nonce}`, + description: briefDescription, + followUp: nonce => `Additional requirement for this task: also include the line \`${followUpMarker(nonce)}\` on its own line in the \`brief\` document before you mark the task done.`, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade(nonce, artifacts) { + return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce), followUpMarker(nonce)])]); + }, + }, + B1: { + id: "B1", + label: "Python CLI with tests", + usesRepository: true, + fireWindowMs: [20_000, 90_000], + budgetMs: 15 * 60_000, + title: nonce => `Chaos B1 slugify command ${nonce}`, + description: nonce => [ + "Work in this task's repository workspace (a small Python CLI called chaoscli).", + "Implement the `slugify` subcommand in `chaoscli/cli.py` so that", + "`python3 -m chaoscli slugify \"Hello Chaos World\"` prints `hello-chaos-world`", + "(lowercase, runs of non-alphanumeric characters become a single `-`, no leading or trailing `-`).", + "The currently failing test must pass: run `python3 -m unittest discover -s tests -v`.", + `Also create \`${notesFile(nonce)}\` in the repository root containing exactly the line \`B1-${nonce}\`.`, + "Use only the Python standard library. Do not run ./bench.sh. When the tests pass, mark this task done.", + ].join("\n"), + collect: { + documents: [], + files: nonce => [notesFile(nonce), "chaoscli/cli.py"], + commands: [ + { name: "unittest", argv: ["python3", "-m", "unittest", "discover", "-s", "tests"] }, + { name: "slugify", argv: ["python3", "-m", "chaoscli", "slugify", "Hello Chaos World"] }, + ], + }, + grade(nonce, artifacts) { + const tests = artifacts.commands.unittest; + const slug = artifacts.commands.slugify; + return result([ + statusCheck(artifacts), + { name: "workspace_located", ok: artifacts.workspacePath !== null }, + { name: "unittest_passes", ok: tests?.exitCode === 0, detail: `exit=${tests?.exitCode ?? "not run"}` }, + { name: "slugify_output", ok: slug?.exitCode === 0 && slug.stdout.trim() === "hello-chaos-world", detail: JSON.stringify(slug?.stdout.trim() ?? null) }, + { name: "notes_file", ok: artifacts.files[notesFile(nonce)]?.trim() === `B1-${nonce}` }, + ]); + }, + }, + B5: { + id: "B5", + label: "Long bench.sh run", + usesRepository: true, + fireWindowMs: [60_000, 180_000], + budgetMs: 20 * 60_000, + title: nonce => `Chaos B5 bench ${nonce}`, + description: nonce => [ + "Work in this task's repository workspace.", + `Run \`./bench.sh ${nonce}\` from the repository root in the foreground and wait for it to finish.`, + "It takes about four minutes; if your shell tool has a timeout, set it to at least 10 minutes. Do not background or interrupt it.", + "It prints one line of the form `BENCH_RESULT=`.", + `Write exactly that line into a new file \`${benchFile(nonce)}\` in the repository root.`, + "Do not compute the result yourself and do not edit bench.sh. Then mark this task done.", + ].join("\n"), + collect: { documents: [], files: nonce => [benchFile(nonce), benchRecord(nonce)], commands: [] }, + grade(nonce, artifacts) { + const content = artifacts.files[benchFile(nonce)]?.trim() ?? null; + const record = parseBenchRecord(artifacts.files[benchRecord(nonce)]); + return result([ + statusCheck(artifacts), + { name: "workspace_located", ok: artifacts.workspacePath !== null }, + { name: "bench_result_file", ok: content === BENCH_EXPECTED, detail: JSON.stringify(content) }, + { name: "bench_actually_ran", ok: record !== null && record.result === BENCH_EXPECTED && record.seconds >= BENCH_MIN_SECONDS, + detail: record ? `ran ${record.seconds}s, ${record.result}` : "no completion record from bench.sh" }, + // The record is agent-writable; the harness's own progress observations are not. + { name: "bench_progress_observed", ok: (artifacts.benchProgressObserved ?? 0) >= BENCH_MIN_OBSERVED_PROGRESS, + detail: `observed ${artifacts.benchProgressObserved ?? 0} progress values` }, + ]); + }, + }, +}; + +/** Files committed into the `chaos-sandbox` fixture repository. */ +export function fixtureRepositoryFiles(benchSeconds = 240): Record { + return { + "README.md": { content: "# chaos-sandbox\n\nFixture repository for the Paperclip restart chaos harness.\n" }, + "chaoscli/__init__.py": { content: "" }, + "chaoscli/__main__.py": { content: "from chaoscli.cli import main\n\nraise SystemExit(main())\n" }, + "chaoscli/cli.py": { content: [ + "import argparse", + "import sys", + "", + "", + "def greet(name: str) -> str:", + " return f\"hello, {name}\"", + "", + "", + "def slugify(text: str) -> str:", + " raise NotImplementedError(\"slugify is not implemented yet\")", + "", + "", + "def main(argv=None) -> int:", + " parser = argparse.ArgumentParser(prog=\"chaoscli\")", + " commands = parser.add_subparsers(dest=\"command\", required=True)", + " greet_parser = commands.add_parser(\"greet\")", + " greet_parser.add_argument(\"name\")", + " args = parser.parse_args(argv)", + " if args.command == \"greet\":", + " print(greet(args.name))", + " return 0", + " return 2", + "", + ].join("\n") }, + "tests/__init__.py": { content: "" }, + "tests/test_cli.py": { content: [ + "import unittest", + "", + "from chaoscli.cli import greet, slugify", + "", + "", + "class CliTest(unittest.TestCase):", + " def test_greet(self):", + " self.assertEqual(greet(\"chaos\"), \"hello, chaos\")", + "", + " def test_slugify(self):", + " self.assertEqual(slugify(\"Hello Chaos World\"), \"hello-chaos-world\")", + " self.assertEqual(slugify(\" A--b__C!! \"), \"a-b-c\")", + "", + "", + "if __name__ == \"__main__\":", + " unittest.main()", + "", + ].join("\n") }, + "bench.sh": { mode: 0o755, content: [ + "#!/bin/sh", + "# Deterministic long-running benchmark for restart chaos boundary B6.", + "set -eu", + `steps=${BENCH_STEPS}`, + `seconds="\${BENCH_SECONDS:-${benchSeconds}}"`, + "tag=\"${1:-untagged}\"", + "mkdir -p .bench", + "started=$(date +%s)", + "total=0", + "i=1", + "while [ \"$i\" -le \"$steps\" ]; do", + " total=$((total + i * i))", + " if [ $((i % 24)) -eq 0 ]; then echo \"bench progress $i/$steps\"; echo \"$i\" > .bench/progress; fi", + " sleep \"$(awk -v s=\"$seconds\" -v n=\"$steps\" 'BEGIN { printf \"%.3f\", s / n }')\"", + " i=$((i + 1))", + "done", + "echo \"started=$started finished=$(date +%s) result=BENCH_RESULT=$total\" > \".bench/run-$tag.txt\"", + "echo \"BENCH_RESULT=$total\"", + "", + ].join("\n") }, + ".gitignore": { content: ".bench/\n__pycache__/\n" }, + }; +}