diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 348747af49..caa29c30eb 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -21,19 +21,41 @@ pnpm exec vitest run --config tests/restart-chaos/vitest.config.ts # credential node scripts/typecheck-test-projects.mjs tests/restart-chaos # typecheck ``` -Options: `--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap, +Options: `--triggers time,comment,retry,queued,tool` (fault timing, below), +`--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap, default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance). `--env daytona` is reserved; it fails until the Daytona image and environment are wired (see the TODO in `seed.ts`). +## Fault timing (`--triggers`) + +Each task samples one trigger with its cell. `time` (default) fires at the +seeded offset. The others fire relative to a user action or run state seen by +the observer, after a seeded delay: + +| Trigger | Fires | Plan | +| --- | --- | --- | +| `comment` | 0–5 s after the C1 follow-up was posted to a live run (C1 only) | C1 + K1, B8 | +| `retry` | 0–2 s after an accepted Retry click, as an extra K1 after the planned fault | K8 | +| `queued` | on the first poll whose run is still `queued` with no start | B1 | +| `tool` | 15–60 s after `bench.sh` progress first appears (B5 only) | B6 | + +A trigger whose condition can no longer hold (comment posted at rest, run +already started, no Retry offered) is recorded as `triggerMissed` in the +ledger; the task is still graded. `--faults K6` is the crash loop: after the +first kill, the replacement is killed again as soon as `/api/ready` reports +`recovering` (or `ready`, if recovery was too quick to see), and once more just +after `ready`. The receipt's `crashLoop` lists the phase each kill hit. A replay +re-arms the task's own triggered faults instead of using their offsets. + ## Pieces | File | Role | | --- | --- | -| `supervisor.ts` | Owns the server. Reuses `tests/runner-e2e` process code (config, ports, env isolation, wrapper-owned Postgres, process-tree stopper, `replaceController`). Faults: K1 SIGKILL + immediate restart, K2 SIGTERM + restart, K4 SIGKILL + 2–5 min hold. The controller runtime cache is kept (discarding it is K11). | +| `supervisor.ts` | Owns the server. Reuses `tests/runner-e2e` process code (config, ports, env isolation, wrapper-owned Postgres, process-tree stopper, `replaceController`). Faults: K1 SIGKILL + immediate restart, K2 SIGTERM + restart, K4 SIGKILL + 2–5 min hold, K6 crash loop. The controller runtime cache is kept (discarding it is K11). | | `seed.ts` | Chaos Lab company, encrypted provider secrets, native Codex / ACPX Claude / OpenCode agents from the runner-e2e catalog, and the `chaos-sandbox` project (git repo: python CLI, one failing test, `bench.sh` ≈ 4 min) with isolated git-worktree workspaces. It first turns on the instance `enableIsolatedWorkspaces` setting; without it the server ignores the project policy and every repo task shares the primary checkout. | | `scheduler.ts` | Seeded plan: workload × adapter × fault cells and a random fire time per task. Saved to `results//plan.json`. `--replay` reruns one task alone with the same choices and fires the recorded sequence of faults that crossed it (other tasks' faults included) at the same offsets from launch. | -| `observer.ts` | Polls each issue and its runs. When a task rests needing Retry it calls `POST /api/issues/:id/retry` once, with the body `useTaskRetry.ts` sends. Collects T-oracle artifacts and the R6 sweep. | +| `observer.ts` | Polls each issue and its runs. When a task rests needing Retry it calls `POST /api/issues/:id/retry` once (once more only if a later fault started after that click), with the body `useTaskRetry.ts` sends. Collects T-oracle artifacts and the R6 sweep. | | `grader.ts` | Pure R1–R6 grading over the samples. | | `workloads.ts` | A1 brief document, B1 python CLI with tests, B5 long bench, C1 comment while running, with their T-oracles. | | `evidence.ts`, `ledger.ts` | Evidence bundle for each failure and the JSONL ledger plus `SUMMARY.md`. | @@ -68,4 +90,5 @@ are wired (see the TODO in `seed.ts`). `n/a` and R3 decides; a task that rests unfinished with no Retry fails both. - Faults are serialized; each one hits every task running at that moment. - A `todo` or `in_progress` task with no live run and no named wait is not a rest. The observer keeps watching it until it resumes or passes the R5 stall limit. +- R5 also fails a run that stays `running` with no live evidence (no new run event, `lastUsefulActionAt` or `execution.lastConfirmedActivityAt`, and no start) for more than 3 minutes after the server is ready. A `running` status alone is not proof of life. - A fault that fails to complete fails the wave as a harness error (exit 2), even when recovery brings the server back. diff --git a/tests/restart-chaos/api.ts b/tests/restart-chaos/api.ts index 2a37b89d71..b72d5b900b 100644 --- a/tests/restart-chaos/api.ts +++ b/tests/restart-chaos/api.ts @@ -1,4 +1,9 @@ /** Minimal board API client for an isolated local_trusted instance. */ +export function isSocketTypeOfServiceRace(error: unknown): boolean { + const candidate = error as { code?: unknown; syscall?: unknown } | null; + return candidate?.code === "EINVAL" && candidate.syscall === "setTypeOfService"; +} + export class ChaosApiError extends Error { constructor(readonly status: number | null, message: string) { super(message); this.name = "ChaosApiError"; } } diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 05479e6334..a8e675feff 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -1,7 +1,7 @@ import { parseArgs } from "node:util"; import { - ADAPTER_IDS, ENVIRONMENT_KINDS, FAULT_KINDS, WORKLOAD_IDS, - type AdapterId, type EnvironmentKind, type FaultKind, type WaveOptions, type WorkloadId, + ADAPTER_IDS, ENVIRONMENT_KINDS, FAULT_KINDS, TRIGGER_IDS, WORKLOAD_IDS, + type AdapterId, type EnvironmentKind, type FaultKind, type TriggerId, type WaveOptions, type WorkloadId, } from "./types.js"; export const CREDENTIAL_FOR_ADAPTER: Record = { @@ -63,6 +63,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC adapters: { type: "string" }, workloads: { type: "string" }, faults: { type: "string" }, + triggers: { type: "string" }, concurrency: { type: "string" }, tasks: { type: "string" }, seed: { type: "string" }, @@ -76,7 +77,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "concurrency", "tasks", "seed"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -89,6 +90,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), workloads: list(values.workloads, WORKLOAD_IDS, "workloads", WORKLOAD_IDS), faults: list(values.faults, FAULT_KINDS, "faults", ["K1", "K2", "K4"]), + triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), concurrency: integer(values.concurrency, "concurrency", 4, 1, 20), tasks: integer(values.tasks, "tasks", 8, 1, 200), seed: integer(values.seed, "seed", now % 2_147_483_647, 0, 2_147_483_647), diff --git a/tests/restart-chaos/fixtures/rc30-terminal-then-k2.json b/tests/restart-chaos/fixtures/rc30-terminal-then-k2.json new file mode 100644 index 0000000000..a5ca200dca --- /dev/null +++ b/tests/restart-chaos/fixtures/rc30-terminal-then-k2.json @@ -0,0 +1 @@ +{"source":"w-20261106T190257-s707/t05 (RC-30): K2 0.75 s after run.terminal; the run stayed running with no events","samples":[{"at":1793992258523,"reachable":true,"serverReady":true,"issueStatus":"todo","interruption":null,"runs":[]},{"at":1793992261614,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:01.068Z"}]},{"at":1793992264726,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:01.068Z"}]},{"at":1793992267930,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:01.068Z"}]},{"at":1793992271070,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:01.068Z"}]},{"at":1793992274197,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:01.068Z"}]},{"at":1793992277589,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:15.726Z"}]},{"at":1793992280735,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:20.713Z"}]},{"at":1793992283920,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:21.933Z"}]},{"at":1793992287074,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:26.235Z"}]},{"at":1793992290368,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:29.076Z"}]},{"at":1793992293657,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:33.461Z"}]},{"at":1793992297031,"reachable":false,"serverReady":false},{"at":1793992300033,"reachable":false,"serverReady":false},{"at":1793992303040,"reachable":false,"serverReady":false},{"at":1793992306044,"reachable":false,"serverReady":false},{"at":1793992309051,"reachable":false,"serverReady":false},{"at":1793992312059,"reachable":false,"serverReady":false},{"at":1793992315064,"reachable":false,"serverReady":false},{"at":1793992318068,"reachable":false,"serverReady":false},{"at":1793992321072,"reachable":false,"serverReady":false},{"at":1793992324101,"reachable":false,"serverReady":false},{"at":1793992327106,"reachable":false,"serverReady":false},{"at":1793992330110,"reachable":false,"serverReady":false},{"at":1793992333114,"reachable":false,"serverReady":false},{"at":1793992336119,"reachable":false,"serverReady":false},{"at":1793992339651,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992342924,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992346089,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992349185,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992352285,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992355364,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992358444,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992361525,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992364599,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992367717,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992370846,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992373975,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992377081,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992380290,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992383408,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992386887,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992390135,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992393243,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992396348,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992399444,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992402543,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992405647,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992408794,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992411931,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992415146,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992418320,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992421709,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992424933,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992428060,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992431157,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992434246,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992437398,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992440504,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992443612,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992446716,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992449793,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992452877,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992455966,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992459037,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992462088,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992465128,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992468152,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992471191,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992474216,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992477238,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992480266,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992483293,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992486325,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992489353,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992492386,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992495428,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992498465,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992501497,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992504527,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992507553,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992510578,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992513601,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992516625,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992519649,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992522674,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992525702,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992528732,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992531754,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992534800,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992537848,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992540872,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992543901,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992546927,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992549955,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992552976,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992556004,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992559032,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992562052,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992565072,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992568098,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992571119,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992574139,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992577170,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992580191,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992583219,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992586240,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992589261,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992592281,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992595310,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992598332,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992601356,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992604378,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992607400,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992610423,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992613448,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992616473,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992619496,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992622515,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992625542,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992628759,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992631941,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992634963,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992637985,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992641010,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992644036,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992647063,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992650085,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992653109,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992656139,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992659162,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992662187,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992665208,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992668229,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992671253,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992674275,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992677301,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992680325,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992683346,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992686371,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992689394,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992692418,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992695440,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992698465,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992701491,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992704517,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992707546,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992710568,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992713588,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992716613,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992719640,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992722663,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992725687,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992728708,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992731730,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992734754,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992737775,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992740801,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992743823,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992746845,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992749871,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992752892,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992755914,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992758935,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992761960,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992764979,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992768002,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992771028,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992774052,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992777078,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992780102,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992783130,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992786151,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992789174,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992792201,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992795228,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992798253,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992801281,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992804308,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992807411,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992810442,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992813466,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992816492,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992819517,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992822539,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992825559,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992828588,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992831609,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992834633,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992837661,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992840767,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992843794,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992846817,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992849840,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992852862,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992855883,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992858908,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992861930,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992864963,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992868008,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992871032,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992874054,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992877085,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992880109,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992883135,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992886167,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992889220,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992892259,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992895285,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992898314,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992901340,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]},{"at":1793992904367,"reachable":true,"serverReady":true,"issueStatus":"in_progress","interruption":{"state":"working","canRetry":false,"retryWait":false,"workspaceFailure":null,"observedRunId":"b57890ae-139d-4863-85e7-348651ce2e92","userAction":null},"runs":[{"id":"b57890ae-139d-4863-85e7-348651ce2e92","status":"running","createdAt":"2026-11-06T19:10:58.591Z","startedAt":"2026-11-06T19:10:58.693Z","finishedAt":null,"errorCode":null,"retryOfRunId":null,"wakeCommentIds":[],"livenessState":null,"lastEvidenceAt":"2026-11-06T19:11:34.923Z"}]}]} \ No newline at end of file diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 75eea1f037..51b29bbcdb 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -71,6 +71,9 @@ export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { if (record.preFaultSnapshotError) { return { verdict: "fail", detail: `pre_fault_snapshot_failed: durability not checked (${record.preFaultSnapshotError})` }; } + if (record.restSnapshotError) { + return { verdict: "fail", detail: `rest_snapshot_failed: durability not checked (${record.restSnapshotError})` }; + } if (record.lostFiles?.length) { return { verdict: "fail", detail: `pre_fault_files_lost: ${record.lostFiles.slice(0, 10).join(",")}` }; } @@ -138,11 +141,25 @@ export function isStalledSample(sample: TaskSample): boolean { return !namedWait(sample); } +export function deadRunningRun(sample: TaskSample, readySince: number): { id: string; idleMs: number } | null { + if (!sample.reachable || !sample.serverReady || sample.at - readySince <= STALL_LIMIT_MS) return null; + for (const run of sample.runs ?? []) { + if (run.status !== "running") continue; + const evidence = Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0, readySince); + if (sample.at - evidence > STALL_LIMIT_MS) return { id: run.id, idleMs: sample.at - Math.max(ms(run.lastEvidenceAt) || 0, ms(run.startedAt) || 0) }; + } + return null; +} + export function gradeR5(record: TaskRecord): RuleResult { let stallStart: number | null = null; let worst = 0; + let readySince: number | null = null; for (const sample of record.samples) { - if (!sample.reachable || !sample.serverReady) { stallStart = null; continue; } + if (!sample.reachable || !sample.serverReady) { stallStart = null; readySince = null; continue; } + readySince ??= sample.at; + const dead = deadRunningRun(sample, readySince); + if (dead) return { verdict: "fail", detail: `silent_dead_run ${dead.id} running with no event, activity or start evidence for ${Math.round(dead.idleMs / 1000)}s` }; if (isStalledSample(sample)) { stallStart ??= sample.at; worst = Math.max(worst, sample.at - stallStart); diff --git a/tests/restart-chaos/ledger.ts b/tests/restart-chaos/ledger.ts index 4569a116c8..b08c053e2a 100644 --- a/tests/restart-chaos/ledger.ts +++ b/tests/restart-chaos/ledger.ts @@ -13,6 +13,8 @@ export interface LedgerLine { adapter: string; workload: string; fault: string; + trigger?: string; + triggerMissed?: string[]; holdMs: number; fireAfterMs: number; issue: string | null; @@ -24,19 +26,24 @@ export interface LedgerLine { pass: boolean; signature: string | null; retried: boolean; + maxLoadAvg1m?: number; evidence: string | null; replay: string; recordedAt: string; } -export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, gitSha: string, evidence: string | null): LedgerLine { +export function ledgerLine(plan: WavePlan, record: TaskRecord, grade: TaskGrade, gitSha: string, evidence: string | null, + faults: ReadonlyArray<{ faultId: string; loadAvg1m?: number }> = []): LedgerLine { + const loads = faults.filter(fault => grade.crossingFaults.includes(fault.faultId) && fault.loadAvg1m !== undefined).map(fault => fault.loadAvg1m!); const rules = Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])) as Record; const ruleDetails = Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.detail])) as Record; return { schema: "paperclip.restart-chaos.ledger/v1", waveId: plan.waveId, taskId: record.plan.taskId, seed: plan.seed, gitSha, env: plan.options.env, adapter: record.plan.adapter, workload: record.plan.workload, fault: record.plan.fault, + ...(record.plan.trigger ? { trigger: record.plan.trigger, triggerMissed: record.triggerMissed ?? [] } : {}), holdMs: record.plan.holdMs, fireAfterMs: record.plan.fireAfterMs, issue: record.identifier ?? record.issueId, crossed: grade.crossed, crossingFaults: grade.crossingFaults, rules, ruleDetails, T: grade.T.verdict, pass: grade.pass, + ...(loads.length ? { maxLoadAvg1m: Math.max(...loads) } : {}), signature: grade.signature, retried: Boolean(record.retry), evidence, replay: replayCommand(plan.waveId, record.plan.taskId), recordedAt: new Date().toISOString(), }; @@ -57,7 +64,7 @@ export function renderSummary(lines: readonly LedgerLine[], latestWave?: string) const cells = new Map(); const signatures = new Map(); for (const line of lines) { - const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}`; + const key = `${line.env} · ${line.adapter} · ${line.workload} · ${line.fault}${line.trigger ? ` ^${line.trigger}` : ""}`; const cell = cells.get(key) ?? { pass: 0, total: 0, crossed: 0 }; cell.total += 1; if (line.pass) cell.pass += 1; if (line.crossed) cell.crossed += 1; cells.set(key, cell); @@ -71,7 +78,7 @@ export function renderSummary(lines: readonly LedgerLine[], latestWave?: string) if (latestWave) { out.push(`## Latest wave ${latestWave}`, "", "| Task | Cell | Crossed | R1 | R2 | R3 | R4 | R5 | R6 | T | Result |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"); for (const line of lines.filter(candidate => candidate.waveId === latestWave)) { - out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`); + out.push(`| ${line.taskId} ${line.issue ?? ""} | ${line.adapter} ${line.workload} ${line.fault}${line.trigger ? ` ^${line.trigger}${line.triggerMissed?.length ? " (missed)" : ""}` : ""} | ${line.crossed ? "yes" : "no"} | ${line.rules.R1} | ${line.rules.R2} | ${line.rules.R3} | ${line.rules.R4} | ${line.rules.R5} | ${line.rules.R6} | ${line.T} | ${line.pass ? "PASS" : `FAIL ${line.signature}`} |`); } out.push(""); } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index c17c08d6c0..89623d437c 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -12,7 +12,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; -import { ChaosApi } from "./api.js"; +import { ChaosApi, isSocketTypeOfServiceRace } from "./api.js"; import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js"; import { knownSecretValues, writeEvidenceBundle } from "./evidence.js"; import { gradeTask } from "./grader.js"; @@ -41,7 +41,7 @@ async function loadPlan(): Promise<{ plan: WavePlan; keep: boolean }> { const graded = await readFile(path.join(sourceDir, `${command.ref.taskId}.grade.json`), "utf8") .then(text => JSON.parse(text) as { record: { createdAt: number | null; faultIds: string[] } }).catch(() => null); const faults = await readFile(path.join(sourceDir, "faults.json"), "utf8").then(text => JSON.parse(text) as FaultReceipt[]).catch(() => []); - const observed = graded ? observedFaultSchedule(graded.record.createdAt, graded.record.faultIds, faults) : []; + const observed = graded ? observedFaultSchedule(graded.record.createdAt, graded.record.faultIds, faults, command.ref.taskId) : []; if (!observed.length) console.warn("Replay: no recorded fault sequence for this task; using its planned fault only."); return { plan: replayWave(source, command.ref, Date.now(), command.overrides, observed), keep: command.keep }; } @@ -69,7 +69,7 @@ async function main(): Promise { const gitSha = git(["rev-parse", "HEAD"]); const gitDirty = git(["status", "--porcelain"]).length > 0; timeline.add("wave_planned", { waveId: plan.waveId, seed: plan.seed, gitSha, gitDirty, replayOf: plan.replayOf ?? null, - tasks: plan.tasks.map(task => `${task.taskId}:${task.adapter}/${task.workload}/${task.fault}@${task.fireAfterMs}ms${task.holdMs ? ` hold ${task.holdMs}ms` : ""}`) }); + tasks: plan.tasks.map(task => `${task.taskId}:${task.adapter}/${task.workload}/${task.fault}${task.trigger ? `^${task.trigger}+${task.triggerDelayMs ?? 0}ms` : ""}@${task.fireAfterMs}ms${task.holdMs ? ` hold ${task.holdMs}ms` : ""}`) }); preflight(plan); const supervisor = await ChaosSupervisor.create({ repositoryRoot, waveDir, timeline }); @@ -81,7 +81,14 @@ async function main(): Promise { const lines: LedgerLine[] = []; let anyFailure = false; let harnessError: unknown; + process.on("uncaughtException", error => { + if (isSocketTypeOfServiceRace(error)) { timeline.add("harness_socket_race_ignored", { error: String(error) }); return; } + timeline.add("harness_uncaught_exception", { error: error.stack ?? String(error) }); + harnessError ??= error; + interrupted = true; + }); let faultFailures = 0; + let faultsQueued = 0; try { await supervisor.start(); const api = new ChaosApi(supervisor.baseUrl); @@ -111,14 +118,19 @@ async function main(): Promise { await observer.poll(now); })); for (const observer of active) { + if (faultsQueued || supervisor.faultInProgress || !supervisor.serverReady) break; const due = observer.faultDue(Date.now()); if (!due) continue; if (observer.record.restAt !== null) { timeline.add("fault_skipped_task_resting", { taskId: observer.plan.taskId, fault: due.kind }); continue; } const exposed = active.filter(candidate => !candidate.done); await Promise.all(exposed.map(candidate => candidate.snapshotBeforeFault())); - faultWork.push(supervisor.fault(due.kind, due.holdMs, observer.plan.taskId).then(receipt => { + if (!(await observer.confirmTrigger(due, Date.now()))) continue; + const trigger = due.trigger && due.trigger !== "time" ? { trigger: due.trigger, delayMs: due.delayMs ?? 0 } : undefined; + if (trigger) timeline.add("trigger_fired", { taskId: observer.plan.taskId, fault: due.kind, ...trigger }); + faultsQueued += 1; + faultWork.push(supervisor.fault(due.kind, due.holdMs, observer.plan.taskId, trigger).finally(() => { faultsQueued -= 1; }).then(receipt => { faults.push(receipt); - for (const candidate of exposed) { candidate.noteFault(receipt.faultId); candidate.extendDeadline((receipt.readyAt ?? Date.now()) - receipt.startedAt); } + for (const candidate of exposed) { candidate.noteFault(receipt.faultId, receipt.startedAt); candidate.extendDeadline((receipt.readyAt ?? Date.now()) - receipt.startedAt); } }, (error: Error & { receipt?: FaultReceipt }) => { if (error.receipt) faults.push(error.receipt); faultFailures += 1; @@ -156,7 +168,7 @@ async function main(): Promise { gitSha, gitDirty, issue: observer.issue, secrets }).catch(error => { timeline.add("evidence_failed", { taskId: observer.plan.taskId, error: String(error) }); return null; }); } await writeFile(path.join(waveDir, `${observer.plan.taskId}.grade.json`), `${JSON.stringify({ record: observer.record, grade }, null, 2)}\n`, { mode: 0o600 }); - lines.push(ledgerLine(plan, observer.record, grade, gitSha, evidence)); + lines.push(ledgerLine(plan, observer.record, grade, gitSha, evidence, faults)); timeline.add("task_graded", { taskId: observer.plan.taskId, pass: grade.pass, crossed: grade.crossed, signature: grade.signature, rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict }); } diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 1f0eb32817..a33b334429 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -33,6 +33,13 @@ export async function listWorkspaceFiles(root: string): Promise { return out.sort(); } +export const QUEUED_TRIGGER_MAX_EMPTY_SAMPLES = 3; + +function stillQueued(runs: readonly RunSample[]): boolean { + return runs.some(run => run.status === "queued" && !run.startedAt) && !runs.some(run => run.startedAt || run.status === "running"); +} +export const TRIGGER_LATE_MS = 10_000; + /** Rest must hold for this long before the observer acts on it. */ export const REST_CONFIRM_MS = 12_000; /** Extra time granted after a Retry click before R3 is graded as timed out. */ @@ -47,9 +54,21 @@ export function toRunSample(run: Json): RunSample { id: String(run.runId ?? run.id), status: String(run.status), createdAt: run.createdAt ?? null, startedAt: run.startedAt ?? null, finishedAt: run.finishedAt ?? null, errorCode: run.errorCode ?? null, retryOfRunId: run.retryOfRunId ?? null, wakeCommentIds: [...wake], livenessState: run.livenessState ?? null, + lastEvidenceAt: latestIso([run.startedAt, run.lastUsefulActionAt, run.execution?.lastConfirmedActivityAt]), }; } +export function latestIso(values: ReadonlyArray): string | null { + let best: number | null = null; + for (const value of values) { + const at = typeof value === "string" ? Date.parse(value) : NaN; + if (Number.isFinite(at) && (best === null || at > best)) best = at; + } + return best === null ? null : new Date(best).toISOString(); +} + +export const RUN_EVENT_PAGE = 500; + export function toInterruption(value: Json | null | undefined): InterruptionSample | null { if (!value) return null; return { state: String(value.state), canRetry: value.canRetry === true, retryWait: Boolean(value.retryWait), @@ -80,7 +99,7 @@ export class TaskObserver { private restCandidateSince: number | null = null; private stallSince: number | null = null; private readonly faultSchedule: ScheduledFault[]; - private nextFault = 0; + private readonly pendingFaults: ScheduledFault[]; done = false; constructor(readonly plan: TaskPlan, private readonly api: ChaosApi, private readonly lab: ChaosLab, @@ -91,6 +110,7 @@ export class TaskObserver { restAt: null, finalStatus: null, deadlineAt: null, timedOut: false, followUp: null, faultIds: [], sweep: null, preFaultFiles: null, lostFiles: null }; this.faultSchedule = taskFaultSchedule(plan); + this.pendingFaults = [...this.faultSchedule]; } get definition() { return workloadDefinitions[this.plan.workload]; } @@ -122,10 +142,75 @@ export class TaskObserver { /** The next scheduled fault once it is due and the task has not rested yet. */ faultDue(now: number): ScheduledFault | null { if (this.done || this.record.createdAt === null) return null; - const fault = this.faultSchedule[this.nextFault]; - if (!fault || now < this.record.createdAt + fault.atMs) return null; - this.nextFault += 1; - return fault; + for (const fault of [...this.pendingFaults]) { + const armedAt = this.triggerArmedAt(fault); + if (armedAt === null) continue; + const dueAt = armedAt === "missed" ? 0 : armedAt + (fault.delayMs ?? 0); + if (armedAt !== "missed" && now < dueAt) continue; + this.pendingFaults.splice(this.pendingFaults.indexOf(fault), 1); + const triggered = fault.trigger !== undefined && fault.trigger !== "time"; + if (armedAt === "missed" || (triggered && !this.triggerWindowOpen(fault, now, dueAt))) { + this.noteTriggerMissed(fault); + continue; + } + return fault; + } + return null; + } + + private triggerWindowOpen(fault: ScheduledFault, now: number, dueAt: number): boolean { + if (now - dueAt > TRIGGER_LATE_MS) return false; + if (fault.trigger !== "queued") return true; + const latest = [...this.record.samples].reverse().find(sample => sample.reachable); + return stillQueued(latest?.runs ?? []); + } + + async confirmTrigger(fault: ScheduledFault, now: number): Promise { + if (!fault.trigger || fault.trigger === "time") return true; + const armedAt = this.triggerArmedAt(fault); + let open = typeof armedAt === "number" && now - (armedAt + (fault.delayMs ?? 0)) <= TRIGGER_LATE_MS; + if (open && fault.trigger === "queued") { + const runs = await this.api.get(`/api/issues/${this.record.issueId}/runs`, 5_000) + .then(rows => (rows ?? []).map(toRunSample)).catch(() => null); + open = runs !== null && stillQueued(runs); + } + if (!open) this.noteTriggerMissed(fault); + return open; + } + + private noteTriggerMissed(fault: ScheduledFault) { + const trigger = fault.trigger ?? "time"; + (this.record.triggerMissed ??= []).push(trigger); + this.timeline.add("trigger_missed", { taskId: this.plan.taskId, trigger, fault: fault.kind }); + } + + private triggerArmedAt(fault: ScheduledFault): number | null | "missed" { + const createdAt = this.record.createdAt!; + switch (fault.trigger ?? "time") { + case "time": return createdAt + fault.atMs; + case "comment": { + const followUp = this.record.followUp; + if (!followUp) return null; + return followUp.commentId && followUp.duringRun ? followUp.postedAt ?? followUp.at : "missed"; + } + case "retry": { + const retry = this.record.retry; + if (!retry) return null; + return retry.accepted ? retry.acceptedAt ?? retry.at : "missed"; + } + case "queued": { + const first = this.record.samples.find(sample => sample.reachable && (sample.runs ?? []).length > 0); + if (!first) return this.record.samples.filter(sample => sample.reachable).length >= QUEUED_TRIGGER_MAX_EMPTY_SAMPLES ? "missed" : null; + const queued = (first.runs ?? []).some(run => run.status === "queued" && !run.startedAt); + const started = (first.runs ?? []).some(run => run.startedAt || run.status === "running"); + return queued && !started ? first.at : "missed"; + } + case "tool": return this.record.benchFirstSeenAt ?? null; + } + } + + private closeUnfiredTriggers() { + for (const fault of this.pendingFaults.splice(0)) if (fault.trigger && fault.trigger !== "time") this.noteTriggerMissed(fault); } /** @@ -140,6 +225,7 @@ export class TaskObserver { if (!value) return; const seen = this.record.benchProgressSeen ??= []; if (!seen.includes(value)) seen.push(value); + this.record.benchFirstSeenAt ??= Date.now(); } /** R2: remember which workspace files existed before the first fault hit this task. */ @@ -148,14 +234,17 @@ export class TaskObserver { // One task's snapshot failure must not end the wave; R2 fails with the reason. try { const workspace = await this.workspacePath(); - if (workspace) this.record.preFaultFiles = await listWorkspaceFiles(workspace); + if (workspace) { this.record.preFaultFiles = await listWorkspaceFiles(workspace); this.record.preFaultWorkspacePath = workspace; } } catch (error) { this.record.preFaultSnapshotError = error instanceof Error ? error.message : String(error); this.timeline.add("pre_fault_snapshot_failed", { taskId: this.plan.taskId, error: this.record.preFaultSnapshotError }); } } - noteFault(faultId: string) { if (!this.record.faultIds.includes(faultId)) this.record.faultIds.push(faultId); } + noteFault(faultId: string, startedAt?: number) { + if (!this.record.faultIds.includes(faultId)) this.record.faultIds.push(faultId); + if (startedAt !== undefined) this.record.lastFaultAt = Math.max(this.record.lastFaultAt ?? 0, startedAt); + } /** Outages extend the deadline: R1 allows budget plus outage time. */ extendDeadline(ms: number) { if (this.record.deadlineAt !== null) this.record.deadlineAt += ms; } @@ -195,6 +284,24 @@ export class TaskObserver { } } + private readonly runEventCursors = new Map(); + + private async runEvidence(run: RunSample): Promise { + if (run.status !== "running") return run; + const cursor = this.runEventCursors.get(run.id) ?? { seq: 0, lastAt: null }; + for (let page = 0; page < 3; page += 1) { + const events = await this.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${cursor.seq}&limit=${RUN_EVENT_PAGE}`, 5_000).catch(() => null); + if (!events?.length) break; + for (const event of events) { + if (typeof event.seq === "number" && event.seq > cursor.seq) cursor.seq = event.seq; + cursor.lastAt = latestIso([cursor.lastAt, event.createdAt]); + } + if (events.length < RUN_EVENT_PAGE) break; + } + this.runEventCursors.set(run.id, cursor); + return { ...run, lastEvidenceAt: latestIso([run.lastEvidenceAt, cursor.lastAt]) }; + } + private async sample(now: number): Promise { const serverReady = this.supervisor.serverReady && !this.supervisor.faultInProgress; try { @@ -207,7 +314,7 @@ export class TaskObserver { interruption: toInterruption(issue.interruption), activeRecoveryAction: Boolean(issue.activeRecoveryAction), recoveryOwner: issue.activeRecoveryAction?.ownerType ?? null, executionBlocker: Boolean(issue.executionBlocker), scheduledRetry: Boolean(issue.scheduledRetry), - runs: (runs ?? []).map(toRunSample) }; + runs: await Promise.all((runs ?? []).map(toRunSample).map(run => this.runEvidence(run))) }; } catch { return { at: now, reachable: false, serverReady: false }; } @@ -222,11 +329,11 @@ export class TaskObserver { if (!running && !isResting(sample)) return false; try { const comment = await this.api.post(`/api/issues/${this.record.issueId}/comments`, { body: definition.followUp(this.plan.nonce) }); - this.record.followUp = { at: now, commentId: comment?.id ?? null, error: null }; + this.record.followUp = { at: now, postedAt: Math.max(now, Date.now()), commentId: comment?.id ?? null, error: null, duringRun: running }; } catch (error) { - this.record.followUp = { at: now, commentId: null, error: error instanceof Error ? error.message : String(error) }; + this.record.followUp = { at: now, commentId: null, error: error instanceof Error ? error.message : String(error), duringRun: running }; } - this.timeline.add("task_follow_up", { taskId: this.plan.taskId, duringRun: running, ...this.record.followUp }); + this.timeline.add("task_follow_up", { taskId: this.plan.taskId, ...this.record.followUp }); return true; } @@ -234,7 +341,9 @@ export class TaskObserver { const status = sample.issueStatus ?? null; // C1: a follow-up posted after rest must be served before the task counts as resting. if (this.definition.followUp && !this.record.followUp) return; - if (status !== "done" && !this.record.retry && sample.interruption?.canRetry) { + const faultSinceRetry = this.record.retry !== null && (this.record.lastFaultAt ?? 0) > this.record.retry.at; + if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) { + if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry); this.record.retry = await this.retry(sample, now); this.restCandidateSince = null; if (this.record.retry.accepted && this.record.deadlineAt !== null) this.record.deadlineAt = Math.max(this.record.deadlineAt, now + RETRY_BUDGET_MS); @@ -256,6 +365,7 @@ export class TaskObserver { { idempotencyKey: randomUUID(), observedRunId }); attempt.httpStatus = 200; attempt.accepted = result?.accepted === true; + attempt.acceptedAt = Math.max(now, Date.now()); } catch (error) { attempt.httpStatus = error instanceof ChaosApiError ? error.status : null; attempt.error = error instanceof Error ? error.message : String(error); @@ -288,9 +398,15 @@ export class TaskObserver { } const comments = await this.api.get(`/api/issues/${issueId}/comments`).catch(() => []); const workspacePath = await this.workspacePath(); - if (workspacePath && this.record.preFaultFiles) { - const present = new Set(await listWorkspaceFiles(workspacePath)); - this.record.lostFiles = this.record.preFaultFiles.filter(file => !present.has(file)); + const snapshotPath = this.record.preFaultWorkspacePath ?? workspacePath; + if (snapshotPath && this.record.preFaultFiles) { + if (workspacePath && snapshotPath !== workspacePath) this.record.workspaceMovedTo = workspacePath; + try { + const present = new Set(await listWorkspaceFiles(snapshotPath)); + this.record.lostFiles = this.record.preFaultFiles.filter(file => !present.has(file)); + } catch (error) { + this.record.restSnapshotError = error instanceof Error ? error.message : String(error); + } } const files: Record = {}; const commands: OracleArtifacts["commands"] = {}; @@ -312,6 +428,7 @@ export class TaskObserver { /** R6 sweep: nothing still pending once the wave is quiet. */ async sweep(): Promise { + this.closeUnfiredTriggers(); if (!this.record.issueId) return { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: null }; try { const [issue, runs] = await Promise.all([ diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index a4ceab2f5a..6d2fff21f2 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -1,5 +1,5 @@ import { workloadDefinitions } from "./workloads.js"; -import type { AdapterId, FaultKind, ScheduledFault, TaskPlan, WavePlan, WaveOptions, WorkloadId } from "./types.js"; +import type { AdapterId, FaultKind, ScheduledFault, TaskPlan, TriggerId, WavePlan, WaveOptions, WorkloadId } from "./types.js"; import type { ReplayRef } from "./config.js"; /** mulberry32: small, fast and fully determined by its 32-bit seed. */ @@ -29,14 +29,35 @@ export function createRng(seed: number) { export const K4_HOLD_RANGE_MS: readonly [number, number] = [120_000, 300_000]; +export const TRIGGER_DELAY_RANGE_MS: Record, readonly [number, number]> = { + comment: [0, 5_000], + retry: [0, 2_000], + queued: [0, 0], + tool: [15_000, 60_000], +}; + +export function triggerFitsWorkload(trigger: TriggerId, workload: WorkloadId): boolean { + if (trigger === "comment") return workload === "C1"; + if (trigger === "tool") return workload === "B5"; + return true; +} + /** * Pairwise-friendly sampling: cycle a seeded shuffle of the full * workload × adapter × fault product so small waves still spread across cells. */ -export function planTasks(options: Pick): TaskPlan[] { +export function planTasks(options: Pick): TaskPlan[] { const rng = createRng(options.seed); - const cells: Array<{ workload: WorkloadId; adapter: AdapterId; fault: FaultKind }> = []; - for (const workload of options.workloads) for (const adapter of options.adapters) for (const fault of options.faults) cells.push({ workload, adapter, fault }); + const triggers = options.triggers?.length ? options.triggers : ["time" as const]; + for (const trigger of triggers) { + if (!options.workloads.some(workload => triggerFitsWorkload(trigger, workload))) { + throw new Error(`Trigger "${trigger}" needs workload ${trigger === "comment" ? "C1" : "B5"} in --workloads`); + } + } + const cells: Array<{ workload: WorkloadId; adapter: AdapterId; fault: FaultKind; trigger: TriggerId }> = []; + for (const workload of options.workloads) for (const adapter of options.adapters) for (const fault of options.faults) { + for (const trigger of triggers) if (triggerFitsWorkload(trigger, workload)) cells.push({ workload, adapter, fault, trigger }); + } const tasks: TaskPlan[] = []; let order = rng.shuffle(cells); for (let index = 0; index < options.tasks; index += 1) { @@ -44,10 +65,12 @@ export function planTasks(options: Pick): ScheduledFault[] { + faults: ReadonlyArray<{ faultId: string; kind: FaultKind; holdMs: number; startedAt: number; requestedBy?: string; + trigger?: TriggerId; triggerDelayMs?: number }>, taskId?: string): ScheduledFault[] { if (createdAt === null) return []; return faults.filter(fault => faultIds.includes(fault.faultId)) - .map(fault => ({ kind: fault.kind, holdMs: fault.holdMs, atMs: Math.max(0, fault.startedAt - createdAt) })) + .map((fault): ScheduledFault => ({ kind: fault.kind, holdMs: fault.holdMs, atMs: Math.max(0, fault.startedAt - createdAt), + ...(fault.trigger && fault.trigger !== "time" && fault.requestedBy === taskId + ? { trigger: fault.trigger, delayMs: fault.triggerDelayMs ?? 0 } : {}) })) .sort((a, b) => a.atMs - b.atMs); } /** The faults a task schedules: its replayed sequence, or its one planned fault. */ export function taskFaultSchedule(task: TaskPlan): ScheduledFault[] { - return task.replayFaults?.length ? task.replayFaults : [{ kind: task.fault, holdMs: task.holdMs, atMs: task.fireAfterMs }]; + if (task.replayFaults?.length) return task.replayFaults; + const planned: ScheduledFault = { kind: task.fault, holdMs: task.holdMs, atMs: task.fireAfterMs }; + const trigger = task.trigger ?? "time"; + if (trigger === "time") return [planned]; + const delayMs = task.triggerDelayMs ?? 0; + if (trigger === "retry") return [planned, { kind: "K1", holdMs: 0, atMs: 0, trigger, delayMs }]; + return [{ ...planned, trigger, delayMs }]; } /** diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index 2cb1ccd5d5..e309a16a34 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -18,6 +18,7 @@ import { randomBytes } from "node:crypto"; import { createWriteStream, statSync, type WriteStream } from "node:fs"; import { mkdir, readFile, symlink } from "node:fs/promises"; import { createRequire } from "node:module"; +import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { prepareControllerDatabase } from "../runner-e2e/controller-database.js"; @@ -33,13 +34,21 @@ import { prepareSourceServerDependencies } from "../runner-e2e/source-startup.js import { runnerE2ETypeScriptProcessArgs } from "../runner-e2e/web-server-command.js"; import type { MatrixExecution } from "../runner-e2e/types.js"; import type { Timeline } from "./timeline.js"; -import type { FaultKind, FaultReceipt } from "./types.js"; +import type { FaultKind, FaultReceipt, TriggerId } from "./types.js"; const HEALTH_TIMEOUT_MS = 180_000; const READY_TIMEOUT_MS = 300_000; const STOP_TIMEOUT_MS = 30_000; -export const FAULT_SIGNAL: Record = { K1: "SIGKILL", K2: "SIGTERM", K4: "SIGKILL" }; +export const FAULT_SIGNAL: Record = { K1: "SIGKILL", K2: "SIGTERM", K4: "SIGKILL", K6: "SIGKILL" }; + +export const CRASH_LOOP_TARGETS = ["recovering", "ready"] as const; +const PHASE_POLL_MS = 50; + +export function crashLoopShouldFire(target: (typeof CRASH_LOOP_TARGETS)[number], phase: string): boolean { + if (target === "recovering") return phase === "recovering" || phase === "ready"; + return phase === "ready"; +} export interface SupervisorPaths { temporaryRoot: string; home: string; configPath: string; serverLogPath: string } @@ -219,24 +228,27 @@ export class ChaosSupervisor { } /** Faults are serialized: one control plane, one outage at a time. */ - fault(kind: FaultKind, holdMs: number, requestedBy: string): Promise { + fault(kind: FaultKind, holdMs: number, requestedBy: string, trigger?: { trigger: TriggerId; delayMs: number }): Promise { const requestedAt = Date.now(); - const run = this.faultChain.then(() => this.executeFault(kind, holdMs, requestedBy, requestedAt)); + const run = this.faultChain.then(() => this.executeFault(kind, holdMs, requestedBy, requestedAt, trigger)); this.faultChain = run.catch(() => undefined); return run; } - private async executeFault(kind: FaultKind, holdMs: number, requestedBy: string, requestedAt: number): Promise { + private async executeFault(kind: FaultKind, holdMs: number, requestedBy: string, requestedAt: number, + trigger?: { trigger: TriggerId; delayMs: number }): Promise { const previous = this.child; this.faultCount += 1; const receipt: FaultReceipt = { faultId: `f${String(this.faultCount).padStart(2, "0")}`, kind, requestedBy, signal: FAULT_SIGNAL[kind], holdMs, + ...(trigger ? { trigger: trigger.trigger, triggerDelayMs: trigger.delayMs } : {}), + loadAvg1m: Math.round(os.loadavg()[0]! * 100) / 100, requestedAt, startedAt: Date.now(), stoppedAt: null, restartedAt: null, healthyAt: null, readyAt: null, phases: [], serverLogOffsetAtStart: this.serverLogOffset(), serverLogOffsetAtReady: null, error: null, }; this.faultInProgress = true; this.serverReady = false; - this.timeline.add("fault_started", { faultId: receipt.faultId, kind, signal: receipt.signal, holdMs, requestedBy, previousPid: previous?.pid }); + this.timeline.add("fault_started", { faultId: receipt.faultId, fault: kind, signal: receipt.signal, loadAvg1m: receipt.loadAvg1m, holdMs, requestedBy, previousPid: previous?.pid }); this.log.write(`\n[restart-chaos] ${receipt.faultId} ${kind} ${receipt.signal} hold=${holdMs}ms requestedBy=${requestedBy} at ${new Date().toISOString()}\n`); try { if (!previous?.pid || this.exited(previous)) throw new Error("No live Paperclip server to fault"); @@ -258,9 +270,13 @@ export class ChaosSupervisor { cancelled: () => this.shuttingDown, record: async progress => { outage = progress; }, }); + if (kind === "K6") { + if (!this.child) throw new Error("K6 crash loop has no replacement server to kill"); + receipt.healthyAt = await this.waitForHealth(await this.crashLoop(this.child, receipt), receipt.phases); + } receipt.readyAt = await this.waitForReadyPhase(receipt.phases); receipt.serverLogOffsetAtReady = this.serverLogOffset(); - this.timeline.add("fault_ready", { faultId: receipt.faultId, replacementPid: outage?.replacementPid, + this.timeline.add("fault_ready", { faultId: receipt.faultId, replacementPid: this.child?.pid ?? outage?.replacementPid, downMs: (receipt.restartedAt ?? 0) - (receipt.stoppedAt ?? 0), readyAfterMs: receipt.readyAt - receipt.startedAt, phases: receipt.phases }); } catch (error) { receipt.error = error instanceof Error ? error.message : String(error); @@ -272,6 +288,36 @@ export class ChaosSupervisor { return receipt; } + private async waitForCrashLoopPhase(child: ChildProcess, target: (typeof CRASH_LOOP_TARGETS)[number], + phases: FaultReceipt["phases"]): Promise { + const deadline = Date.now() + READY_TIMEOUT_MS; + while (Date.now() < deadline) { + if (this.shuttingDown) throw new Error("Harness shutdown interrupted the crash loop"); + if (this.exited(child)) throw new Error(`Paperclip server exited during the crash loop (code=${child.exitCode} signal=${child.signalCode})`); + const phase = await this.readPhase(); + if (phase !== phases[phases.length - 1]?.phase) phases.push({ phase, at: Date.now() }); + if (crashLoopShouldFire(target, phase)) return phase; + await delay(PHASE_POLL_MS); + } + throw new Error(`Crash loop: phase ${target} not reached within ${READY_TIMEOUT_MS}ms`); + } + + private async crashLoop(first: ChildProcess, receipt: FaultReceipt): Promise { + let current = first; + receipt.crashLoop = []; + for (const target of CRASH_LOOP_TARGETS) { + const phase = await this.waitForCrashLoopPhase(current, target, receipt.phases); + receipt.crashLoop.push({ pid: current.pid ?? null, target, phase, at: Date.now() }); + this.timeline.add("fault_crash_loop_kill", { faultId: receipt.faultId, pid: current.pid, target, phase }); + this.log.write(`\n[restart-chaos] ${receipt.faultId} K6 crash-loop SIGKILL pid=${current.pid} target=${target} phase=${phase}\n`); + await this.stopper.crashForRestart(current); + if (this.child === current) this.child = null; + await this.waitForHealthToStop(); + current = await this.spawnServer(); + } + return current; + } + /** * Bounded recovery after a failed fault or a crash the harness did not cause: * wait for a live server to become ready, or start a new one. Throws when the diff --git a/tests/restart-chaos/timeline.ts b/tests/restart-chaos/timeline.ts index 95d73d1ead..6bc7536818 100644 --- a/tests/restart-chaos/timeline.ts +++ b/tests/restart-chaos/timeline.ts @@ -9,7 +9,7 @@ export class Timeline { add(kind: string, data: Record = {}): TimelineEntry { const now = Date.now(); - const entry: TimelineEntry = { at: new Date(now).toISOString(), t: now - this.origin, kind, ...data }; + const entry: TimelineEntry = { ...data, at: new Date(now).toISOString(), t: now - this.origin, kind }; this.entries.push(entry); if (this.path) appendFileSync(this.path, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); if (process.env.RESTART_CHAOS_QUIET !== "1") console.info(`[chaos +${Math.round(entry.t / 1000)}s] ${kind} ${JSON.stringify(data)}`); diff --git a/tests/restart-chaos/triggers.test.ts b/tests/restart-chaos/triggers.test.ts new file mode 100644 index 0000000000..8a8c4e2978 --- /dev/null +++ b/tests/restart-chaos/triggers.test.ts @@ -0,0 +1,354 @@ +import { readFileSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { isSocketTypeOfServiceRace, type ChaosApi } from "./api.js"; +import { parseCommand } from "./config.js"; +import { gradeR5 } from "./grader.js"; +import { latestIso, QUEUED_TRIGGER_MAX_EMPTY_SAMPLES, TaskObserver, TRIGGER_LATE_MS } from "./observer.js"; +import { observedFaultSchedule, planTasks, taskFaultSchedule, TRIGGER_DELAY_RANGE_MS } from "./scheduler.js"; +import type { ChaosLab } from "./seed.js"; +import { crashLoopShouldFire } from "./supervisor.js"; +import { Timeline } from "./timeline.js"; +import { ledgerLine } from "./ledger.js"; +import type { TaskGrade, TaskPlan, TaskRecord, TaskSample, WaveOptions, WavePlan } from "./types.js"; + +process.env.RESTART_CHAOS_QUIET = "1"; +const lab: ChaosLab = { companyId: "c1", environmentId: "e1", projectId: "p1", fixtureRepositoryPath: "/tmp/none", + agents: { codex: { id: "a1", name: "Chaos codex" }, claude: { id: "a2", name: "Chaos claude" }, opencode: { id: "a3", name: "Chaos opencode" } } }; +const ready = { serverReady: true, faultInProgress: false }; +const base: TaskPlan = { taskId: "t01", index: 0, workload: "A1", adapter: "codex", fault: "K1", fireAfterMs: 999_999, + holdMs: 0, followUpAfterMs: null, nonce: "abcd1234" }; +const options: WaveOptions = { env: "local", adapters: ["codex", "claude", "opencode"], workloads: ["A1", "B1", "B5", "C1"], + faults: ["K1", "K2", "K4", "K6"], concurrency: 3, tasks: 60, seed: 9, budgetCents: 3_000, pollMs: 3_000, + triggers: ["time", "comment", "retry", "queued", "tool"] }; + +type Json = Record; +function scriptedApi(state: { runs: Json[]; issue: Json; retry?: Json }) { + const api = { + async get(route: string) { return route.endsWith("/runs") ? state.runs : route.endsWith("/comments") ? [] : state.issue; }, + async post(route: string) { + if (route.endsWith("/issues")) return { id: "i1", identifier: "CHA-1" }; + if (route.endsWith("/retry")) return state.retry ?? { accepted: true }; + return { id: "comment-1" }; + }, + }; + return api as unknown as ChaosApi; +} + +describe("trigger planning", () => { + it("pairs comment only with C1 and tool only with B5, with delays inside each range", () => { + const tasks = planTasks(options); + for (const task of tasks) { + if (task.trigger === "comment") expect(task.workload).toBe("C1"); + if (task.trigger === "tool") expect(task.workload).toBe("B5"); + if (task.trigger && task.trigger !== "time") { + const [min, max] = TRIGGER_DELAY_RANGE_MS[task.trigger]; + expect(task.triggerDelayMs).toBeGreaterThanOrEqual(min); + expect(task.triggerDelayMs).toBeLessThanOrEqual(max); + } else expect(task.triggerDelayMs).toBeUndefined(); + } + expect(new Set(tasks.map(task => task.trigger ?? "time")).size).toBe(5); + expect(planTasks(options)).toEqual(tasks); + }); + + it("keeps time-only plans free of trigger fields and rejects a trigger with no fitting workload", () => { + expect(planTasks({ ...options, triggers: undefined }).every(task => task.trigger === undefined)).toBe(true); + expect(() => planTasks({ ...options, workloads: ["A1"], triggers: ["tool"] })).toThrow(/needs workload B5/); + }); + + it("schedules the K8 retry fault as an extra K1 after the planned fault", () => { + expect(taskFaultSchedule({ ...base, fireAfterMs: 30_000, trigger: "retry", triggerDelayMs: 500 })).toEqual([ + { kind: "K1", holdMs: 0, atMs: 30_000 }, { kind: "K1", holdMs: 0, atMs: 0, trigger: "retry", delayMs: 500 }]); + expect(taskFaultSchedule({ ...base, fault: "K6", trigger: "queued", triggerDelayMs: 0 })) + .toEqual([{ kind: "K6", holdMs: 0, atMs: 999_999, trigger: "queued", delayMs: 0 }]); + }); + + it("re-arms only the replayed task's own triggered faults", () => { + const faults = [ + { faultId: "f01", kind: "K1" as const, holdMs: 0, startedAt: 2_000, requestedBy: "t01", trigger: "queued" as const, triggerDelayMs: 0 }, + { faultId: "f02", kind: "K1" as const, holdMs: 0, startedAt: 9_000, requestedBy: "t02", trigger: "comment" as const, triggerDelayMs: 3_000 }, + ]; + expect(observedFaultSchedule(1_000, ["f01", "f02"], faults, "t01")).toEqual([ + { kind: "K1", holdMs: 0, atMs: 1_000, trigger: "queued", delayMs: 0 }, { kind: "K1", holdMs: 0, atMs: 8_000 }]); + }); + + it("parses --triggers and refuses it with --replay", () => { + expect(parseCommand(["--triggers", "comment,queued"])).toMatchObject({ options: { triggers: ["comment", "queued"] } }); + expect(parseCommand([])).toMatchObject({ options: { triggers: ["time"] } }); + expect(() => parseCommand(["--triggers", "later"])).toThrow(/Unknown triggers/); + expect(() => parseCommand(["--replay", "w-20261106T101112-s42/t03", "--triggers", "comment"])).toThrow(/reproduces the saved plan/); + }); +}); + +describe("trigger firing", () => { + it("queued: fires on the first run that is still queued, and misses once a run has started", async () => { + const state = { runs: [{ runId: "r1", status: "queued", createdAt: new Date().toISOString() }] as Json[], issue: { id: "i1", status: "todo" } }; + const observer = new TaskObserver({ ...base, trigger: "queued", triggerDelayMs: 0 }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + expect(observer.faultDue(start)).toBeNull(); + await observer.poll(start + 10); + expect(observer.faultDue(start + 10)).toMatchObject({ trigger: "queued", kind: "K1" }); + + const late = { runs: [{ runId: "r1", status: "running", startedAt: new Date().toISOString() }] as Json[], issue: { id: "i1", status: "in_progress" } }; + const missed = new TaskObserver({ ...base, trigger: "queued", triggerDelayMs: 0 }, scriptedApi(late), lab, new Timeline(null), ready); + await missed.launch(); + await missed.poll(missed.record.createdAt! + 10); + expect(missed.faultDue(missed.record.createdAt! + 10)).toBeNull(); + expect(missed.record.triggerMissed).toEqual(["queued"]); + }); + + it("queued: gives up after several polls with no run at all", async () => { + const state = { runs: [] as Json[], issue: { id: "i1", status: "todo" } }; + const observer = new TaskObserver({ ...base, trigger: "queued", triggerDelayMs: 0 }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + for (let index = 1; index < QUEUED_TRIGGER_MAX_EMPTY_SAMPLES; index += 1) { + await observer.poll(start + index); + expect(observer.faultDue(start + index)).toBeNull(); + } + expect(observer.record.triggerMissed).toBeUndefined(); + await observer.poll(start + 100); + observer.faultDue(start + 100); + expect(observer.record.triggerMissed).toEqual(["queued"]); + }); + + it("comment: fires its delay after the follow-up reaches a live run", async () => { + const state = { runs: [{ runId: "r1", status: "running", startedAt: new Date().toISOString() }] as Json[], issue: { id: "i1", status: "in_progress" } }; + const observer = new TaskObserver({ ...base, workload: "C1", followUpAfterMs: 1_000, trigger: "comment", triggerDelayMs: 2_000 }, + scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 500); + expect(observer.faultDue(start + 500)).toBeNull(); + await observer.poll(start + 1_500); + expect(observer.record.followUp).toMatchObject({ commentId: "comment-1", duringRun: true }); + expect(observer.faultDue(start + 3_499)).toBeNull(); + expect(observer.faultDue(start + 3_500)).toMatchObject({ trigger: "comment" }); + }); + + it("comment: a follow-up posted to a resting task does not fire", async () => { + const state = { runs: [{ runId: "r1", status: "succeeded", finishedAt: new Date().toISOString() }] as Json[], issue: { id: "i1", status: "done" } }; + const observer = new TaskObserver({ ...base, workload: "C1", followUpAfterMs: 0, trigger: "comment", triggerDelayMs: 0 }, + scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + await observer.poll(observer.record.createdAt! + 10); + expect(observer.faultDue(observer.record.createdAt! + 10)).toBeNull(); + expect(observer.record.triggerMissed).toEqual(["comment"]); + }); + + it("retry: fires the extra K1 right after an accepted Retry click, and closes unfired triggers at sweep", async () => { + const state = { runs: [{ runId: "r1", status: "failed", finishedAt: new Date().toISOString() }] as Json[], + issue: { id: "i1", status: "blocked", interruption: { state: "needs_retry", canRetry: true, observedRunId: "r1" } } }; + const observer = new TaskObserver({ ...base, fireAfterMs: 1_000, trigger: "retry", triggerDelayMs: 0 }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + expect(observer.faultDue(start + 1_000)).toEqual({ kind: "K1", holdMs: 0, atMs: 1_000 }); + await observer.poll(start + 2_000); + expect(observer.faultDue(start + 2_000)).toBeNull(); + await observer.poll(start + 2_000 + 12_001); + expect(observer.record.retry).toMatchObject({ accepted: true }); + expect(observer.faultDue(start + 14_002)).toMatchObject({ kind: "K1", trigger: "retry" }); + + const unclicked = new TaskObserver({ ...base, trigger: "retry", triggerDelayMs: 0 }, scriptedApi(state), lab, new Timeline(null), ready); + await unclicked.launch(); + await unclicked.sweep(); + expect(unclicked.record.triggerMissed).toEqual(["retry"]); + }); + + it("clicks Retry once per fault: again only after a fault that started after the last click", async () => { + const state = { runs: [{ runId: "r1", status: "failed", finishedAt: new Date().toISOString() }] as Json[], + issue: { id: "i1", status: "blocked", interruption: { state: "failed", canRetry: true, observedRunId: "r1" } } }; + const observer = new TaskObserver({ ...base }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + await observer.poll(start + 1_000 + 12_001); + const first = observer.record.retry!; + expect(first.accepted).toBe(true); + await observer.poll(start + 20_000); + await observer.poll(start + 20_000 + 12_001); + expect(observer.done).toBe(true); + expect(observer.record.retry).toBe(first); + + const again = new TaskObserver({ ...base }, scriptedApi(state), lab, new Timeline(null), ready); + await again.launch(); + const begin = again.record.createdAt!; + await again.poll(begin + 1_000); + await again.poll(begin + 1_000 + 12_001); + const clicked = again.record.retry!; + again.noteFault("f01", clicked.at + 1); + await again.poll(begin + 20_000); + await again.poll(begin + 20_000 + 12_001); + expect(again.done).toBe(false); + expect(again.record.priorRetries).toEqual([clicked]); + expect(again.record.retry).not.toBe(clicked); + }); + + it("tool: fires its delay after bench.sh progress is first seen", async () => { + const observer = new TaskObserver({ ...base, workload: "B5", trigger: "tool", triggerDelayMs: 20_000 }, + scriptedApi({ runs: [], issue: { id: "i1", status: "in_progress" } }), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + expect(observer.faultDue(start + 60_000)).toBeNull(); + observer.record.benchFirstSeenAt = start + 30_000; + expect(observer.faultDue(start + 49_999)).toBeNull(); + expect(observer.faultDue(start + 50_000)).toMatchObject({ trigger: "tool" }); + }); + + it("records a miss instead of firing a triggered fault after its window has passed", async () => { + const observer = new TaskObserver({ ...base, workload: "B5", trigger: "tool", triggerDelayMs: 1_000 }, + scriptedApi({ runs: [], issue: { id: "i1", status: "in_progress" } }), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + observer.record.benchFirstSeenAt = start; + expect(observer.faultDue(start + 1_000 + TRIGGER_LATE_MS + 1)).toBeNull(); + expect(observer.record.triggerMissed).toEqual(["tool"]); + }); + + it("queued: misses when the run started before the fault could fire", async () => { + const state = { runs: [{ runId: "r1", status: "queued" }] as Json[], issue: { id: "i1", status: "todo" } }; + const observer = new TaskObserver({ ...base, trigger: "queued", triggerDelayMs: 0 }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 10); + state.runs = [{ runId: "r1", status: "running", startedAt: new Date().toISOString() }]; + await observer.poll(start + 20); + expect(observer.faultDue(start + 20)).toBeNull(); + expect(observer.record.triggerMissed).toEqual(["queued"]); + }); + + it("keeps replayed fixed-offset faults on time while a re-armed trigger waits", async () => { + const observer = new TaskObserver({ ...base, workload: "B5", replayFaults: [ + { kind: "K1", holdMs: 0, atMs: 0, trigger: "tool", delayMs: 0 }, { kind: "K2", holdMs: 0, atMs: 5_000 }] }, + scriptedApi({ runs: [], issue: { id: "i1", status: "in_progress" } }), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + expect(observer.faultDue(start + 5_000)).toEqual({ kind: "K2", holdMs: 0, atMs: 5_000 }); + observer.record.benchFirstSeenAt = start + 6_000; + expect(observer.faultDue(start + 6_000)).toMatchObject({ kind: "K1", trigger: "tool" }); + }); +}); + +describe("ledger load", () => { + it("records the highest load among the task's crossing faults", () => { + const plan = { waveId: "w", seed: 1, options: { env: "local" } } as unknown as WavePlan; + const record = { plan: base, identifier: "CHA-1", issueId: "i1", retry: null } as unknown as TaskRecord; + const grade = { rules: {}, T: { verdict: "pass" }, crossed: true, crossingFaults: ["f01", "f02"], pass: true, signature: null } as unknown as TaskGrade; + const line = ledgerLine(plan, record, grade, "sha", null, + [{ faultId: "f01", loadAvg1m: 12.5 }, { faultId: "f02", loadAvg1m: 21 }, { faultId: "f03", loadAvg1m: 40 }]); + expect(line.maxLoadAvg1m).toBe(21); + expect(ledgerLine(plan, record, grade, "sha", null).maxLoadAvg1m).toBeUndefined(); + }); +}); + +describe("pre-fault files", () => { + it("checks lost files in the workspace the snapshot was taken from, and records a move", async () => { + const before = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-before-")); + const after = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-after-")); + try { + await writeFile(path.join(before, "kept.txt"), "x"); + await writeFile(path.join(before, "other-task.txt"), "x"); + const state = { runs: [] as Json[], issue: { id: "i1", status: "in_progress", currentExecutionWorkspace: { cwd: before } } as Json }; + const observer = new TaskObserver({ ...base, workload: "B5" }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + await observer.poll(observer.record.createdAt! + 10); + await observer.snapshotBeforeFault(); + await rm(path.join(before, "other-task.txt")); + state.issue = { id: "i1", status: "done", currentExecutionWorkspace: { cwd: after } }; + await observer.collectArtifacts(); + expect(observer.record.lostFiles).toEqual(["other-task.txt"]); + expect(observer.record.workspaceMovedTo).toBe(after); + } finally { + await rm(before, { recursive: true, force: true }); + await rm(after, { recursive: true, force: true }); + } + }); +}); + +describe("dispatch checks", () => { + it("confirmTrigger rechecks a queued run right before the kill", async () => { + const state = { runs: [{ runId: "r1", status: "queued" }] as Json[], issue: { id: "i1", status: "todo" } }; + const observer = new TaskObserver({ ...base, trigger: "queued", triggerDelayMs: 0 }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 10); + const due = observer.faultDue(start + 10)!; + expect(await observer.confirmTrigger(due, start + 20)).toBe(true); + state.runs = [{ runId: "r1", status: "running", startedAt: new Date().toISOString() }]; + expect(await observer.confirmTrigger(due, start + 30)).toBe(false); + expect(observer.record.triggerMissed).toEqual(["queued"]); + expect(await observer.confirmTrigger({ kind: "K1", holdMs: 0, atMs: 0 }, start)).toBe(true); + }); + + it("records an unreadable pre-fault workspace at rest instead of reporting every file lost", async () => { + const before = await mkdtemp(path.join(os.tmpdir(), "restart-chaos-gone-")); + await writeFile(path.join(before, "a.txt"), "x"); + const state = { runs: [] as Json[], issue: { id: "i1", status: "in_progress", currentExecutionWorkspace: { cwd: before } } as Json }; + const observer = new TaskObserver({ ...base, workload: "B5" }, scriptedApi(state), lab, new Timeline(null), ready); + await observer.launch(); + await observer.poll(observer.record.createdAt! + 10); + await observer.snapshotBeforeFault(); + await rm(before, { recursive: true, force: true }); + state.issue = { id: "i1", status: "done" }; + await observer.collectArtifacts(); + expect(observer.record.lostFiles).toBeNull(); + expect(observer.record.restSnapshotError).toMatch(/ENOENT/); + }); +}); + +describe("R5 live evidence", () => { + const fixture = JSON.parse(readFileSync(path.join(import.meta.dirname, "fixtures/rc30-terminal-then-k2.json"), "utf8")) as { samples: TaskSample[] }; + const record = (samples: TaskSample[]) => ({ samples } as unknown as TaskRecord); + + it("fails a run that stays running with no event evidence after the server is ready (RC-30, w-20261106T190257-s707/t05)", () => { + expect(gradeR5(record(fixture.samples))).toMatchObject({ verdict: "fail", detail: expect.stringMatching(/^silent_dead_run b57890ae/) }); + }); + + it("passes the same timeline while the run keeps producing events", () => { + const live = fixture.samples.map(sample => ({ ...sample, + runs: sample.runs?.map(run => ({ ...run, lastEvidenceAt: new Date(sample.at - 30_000).toISOString() })) })); + expect(gradeR5(record(live)).verdict).toBe("pass"); + }); + + it("counts only time since the server became ready", () => { + const start = Date.parse("2026-11-06T10:00:00Z"); + const run = { id: "r1", status: "running", createdAt: null, startedAt: new Date(start - 600_000).toISOString(), finishedAt: null, + errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null, lastEvidenceAt: null }; + const sample = (at: number, ready = true): TaskSample => ({ at, reachable: ready, serverReady: ready, issueStatus: "in_progress", runs: [run] }); + expect(gradeR5(record([sample(start, false), sample(start + 1_000), sample(start + 120_000)])).verdict).toBe("pass"); + expect(gradeR5(record([sample(start, false), sample(start + 1_000), sample(start + 200_000)])).verdict).toBe("fail"); + }); + + it("keeps the latest evidence timestamp", () => { + expect(latestIso([null, "2026-11-06T10:00:00Z", "2026-11-06T09:00:00Z", "bad"])).toBe("2026-11-06T10:00:00.000Z"); + expect(latestIso([undefined])).toBeNull(); + }); +}); + +describe("socket race", () => { + it("matches only the undici setTypeOfService EINVAL race", () => { + expect(isSocketTypeOfServiceRace(Object.assign(new Error("setTypeOfService EINVAL"), { code: "EINVAL", syscall: "setTypeOfService" }))).toBe(true); + expect(isSocketTypeOfServiceRace(Object.assign(new Error("x"), { code: "EINVAL", syscall: "write" }))).toBe(false); + expect(isSocketTypeOfServiceRace(null)).toBe(false); + }); +}); + +describe("timeline", () => { + it("keeps the event kind when event data has a kind field", () => { + expect(new Timeline(null).add("fault_started", { kind: "K1" }).kind).toBe("fault_started"); + }); +}); + +describe("K6 crash loop", () => { + it("kills in recovering (or ready when recovering was skipped), then just after ready", () => { + expect(crashLoopShouldFire("recovering", "starting")).toBe(false); + expect(crashLoopShouldFire("recovering", "unreachable")).toBe(false); + expect(crashLoopShouldFire("recovering", "recovering")).toBe(true); + expect(crashLoopShouldFire("recovering", "ready")).toBe(true); + expect(crashLoopShouldFire("ready", "recovering")).toBe(false); + expect(crashLoopShouldFire("ready", "ready")).toBe(true); + }); +}); diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index ce2203f878..451e044ca9 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -1,6 +1,6 @@ /** Shared types for the control-plane restart chaos harness (plan §5). */ -export const FAULT_KINDS = ["K1", "K2", "K4"] as const; +export const FAULT_KINDS = ["K1", "K2", "K4", "K6"] as const; export type FaultKind = (typeof FAULT_KINDS)[number]; export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1"] as const; @@ -9,6 +9,9 @@ export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; export type AdapterId = (typeof ADAPTER_IDS)[number]; +export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool"] as const; +export type TriggerId = (typeof TRIGGER_IDS)[number]; + export const ENVIRONMENT_KINDS = ["local", "daytona"] as const; export type EnvironmentKind = (typeof ENVIRONMENT_KINDS)[number]; @@ -17,6 +20,7 @@ export interface WaveOptions { adapters: AdapterId[]; workloads: WorkloadId[]; faults: FaultKind[]; + triggers?: TriggerId[]; concurrency: number; tasks: number; seed: number; @@ -36,6 +40,8 @@ export interface TaskPlan { fireAfterMs: number; /** Outage hold between stop and restart. 0 for K1/K2. */ holdMs: number; + trigger?: TriggerId; + triggerDelayMs?: number; /** C1 only: post the follow-up comment this long after creation. */ followUpAfterMs: number | null; /** Unique marker used by the deterministic T-oracle. */ @@ -47,7 +53,12 @@ export interface TaskPlan { replayFaults?: ScheduledFault[]; } -export interface ScheduledFault { kind: FaultKind; holdMs: number; atMs: number } +export interface ScheduledFault { + kind: FaultKind; holdMs: number; + atMs: number; + trigger?: TriggerId; + delayMs?: number; +} export interface WavePlan { schema: "paperclip.restart-chaos.wave-plan/v1"; @@ -74,6 +85,7 @@ export interface RunSample { retryOfRunId: string | null; wakeCommentIds: string[]; livenessState: string | null; + lastEvidenceAt?: string | null; } export interface InterruptionSample { @@ -102,6 +114,7 @@ export interface TaskSample { export interface RetryAttempt { at: number; + acceptedAt?: number; observedRunId: string | null; httpStatus: number | null; accepted: boolean | null; @@ -122,6 +135,10 @@ export interface FaultReceipt { healthyAt: number | null; readyAt: number | null; phases: Array<{ phase: string; at: number }>; + trigger?: TriggerId; + triggerDelayMs?: number; + loadAvg1m?: number; + crashLoop?: Array<{ pid: number | null; target: string; phase: string; at: number }>; serverLogOffsetAtStart: number; serverLogOffsetAtReady: number | null; error: string | null; @@ -159,17 +176,24 @@ export interface TaskRecord { finalStatus: string | null; deadlineAt: number | null; timedOut: boolean; - followUp: { at: number; commentId: string | null; error: string | null } | null; + followUp: { at: number; postedAt?: number; commentId: string | null; error: string | null; duringRun?: boolean } | null; faultIds: string[]; sweep: RestSweep | null; /** Workspace files present just before the first fault that hit this task (R2). */ preFaultFiles: string[] | null; + preFaultWorkspacePath?: string; + lastFaultAt?: number; + restSnapshotError?: string; + priorRetries?: RetryAttempt[]; + workspaceMovedTo?: string; /** Set when the pre-fault snapshot could not read the workspace; R2 then fails visibly. */ preFaultSnapshotError?: string | null; /** Pre-fault files missing at rest; null when not checked. */ lostFiles: string[] | null; /** B5: distinct bench progress values the observer read during polling. */ benchProgressSeen?: string[]; + benchFirstSeenAt?: number | null; + triggerMissed?: TriggerId[]; } export interface TaskGrade {