feat(daemon): codex resume verifies pane via native probe

Closes the false-positive `resumed` shape that fire-and-forget left open.
Codex resume now polls getPaneCommand + capturePaneContent through
assessNativeResumeProbe (using the existing Codex shape — no new probe
patterns) and returns:

- ok: true            when probe reports `resumed` (codex foreground or TUI)
- code: retry_fresh   when probe reports `failed: no_saved_session` or pane
                      returned to shell
- code: resume_failed when probe stays inconclusive past timeout

Mirrors ClaudeResumeAdapter.verifyResume; restore-orchestrator.ts:944-947
keeps the existing Codex mapping unchanged (`ok:true → resumed`,
otherwise `failed`).

Lands as part of the lifecycle reboot/recovery scenario matrix slice
(primitive-hardening #2 rig lifecycle, bedrock). Driver matrix audit:
5 rows PASS at HEAD, 1 NEEDS-HONEST-PATCH (this commit), 1 NEEDS-DEFERRAL
(Codex `attention_required` for auth refusal — multi-surface feature
requiring probe extension; deferred to a follow-up provider-auth-attention
slice).

Tests:
- packages/daemon/test/codex-resume.test.ts +6 verifyResume cases
- packages/daemon/test/lifecycle-scenario-matrix.test.ts (new file, 23
  Tier 1 in-process tests + 1 documented `it.skip` deferral covering all
  six scenarios: clean start, warm resume, host reboot/tmux absence,
  provider auth loss, partial boot, operator recovery)
- Daemon suite: 2176 passed | 1 skipped (was 2146 baseline at 578bd5c)

Slice rails: substrate/shared-docs/openrig-work/missions/recursive-self-
improvement-v2/primitive-hardening/slices/lifecycle-reboot-recovery-
scenario-matrix/

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
mvschwarzandClaude Opus 4.7 committed 2026-04-30 08:22:38 -07:00
1 parent 6b7043a04d
commit ab01331f50
3 files changed
+995 -5

No files matched your search

+79 -5
View File
@@ -1,13 +1,25 @@
import { setTimeout as sleep } from "node:timers/promises";
import type { TmuxAdapter } from "./tmux.js";
import type { ResumeResult } from "./claude-resume.js";
import { shellQuote } from "./shell-quote.js";
import { assessNativeResumeProbe } from "../domain/native-resume-probe.js";
const CODEX_TYPES = new Set(["codex_id", "codex_last"]);
const SHELL_COMMANDS = new Set(["bash", "fish", "nu", "sh", "tmux", "zsh"]);
export { type ResumeResult };
interface CodexResumeOptions {
pollMs?: number;
maxWaitMs?: number;
sleep?: (ms: number) => Promise<void>;
}
export class CodexResumeAdapter {
constructor(private tmux: TmuxAdapter) {}
constructor(
private tmux: TmuxAdapter,
private options: CodexResumeOptions = {}
) {}
canResume(resumeType: string | null, resumeToken: string | null): boolean {
if (!resumeType || !CODEX_TYPES.has(resumeType)) return false;
@@ -18,9 +30,6 @@ export class CodexResumeAdapter {
return true;
}
// NOTE: Resume success is fire-and-forget. sendText succeeding means the
// command was typed into the tmux pane, NOT that the harness actually resumed.
// TODO(Phase 3): Add resume verification (e.g., poll harness status endpoint).
async resume(
tmuxSessionName: string,
resumeType: string | null,
@@ -49,6 +58,71 @@ export class CodexResumeAdapter {
return { ok: false, code: "resume_failed", message: keyResult.message };
}
return { ok: true };
return this.verifyResume(tmuxSessionName);
}
// Mirrors ClaudeResumeAdapter.verifyResume: poll the pane, run the native
// probe, return resumed / retry_fresh / resume_failed based on observable
// runtime state. Closes the false-positive `resumed` shape that fire-and-
// forget left open. Does NOT introduce a Codex `attention_required` outcome
// — that requires new probe patterns for Codex auth refusal and is deferred
// to a follow-up provider-auth-attention slice.
private async verifyResume(tmuxSessionName: string): Promise<ResumeResult> {
const pollMs = this.options.pollMs ?? 200;
const maxWaitMs = this.options.maxWaitMs ?? 5_000;
const sleepFn = this.options.sleep ?? sleep;
const attempts = Math.max(1, Math.floor(maxWaitMs / Math.max(pollMs, 1)) + 1);
for (let attempt = 0; attempt < attempts; attempt++) {
const paneCommand = await this.tmux.getPaneCommand(tmuxSessionName);
const paneContent = (await this.tmux.capturePaneContent(tmuxSessionName, 40)) ?? "";
const probe = assessNativeResumeProbe({
runtime: "codex",
paneCommand,
paneContent,
});
if (probe.code === "no_saved_session") {
return {
ok: false,
code: "retry_fresh",
message: "Codex resume failed: no saved session found for the requested token",
};
}
if (probe.status === "resumed") {
return { ok: true };
}
if (attempt < attempts - 1) {
await sleepFn(pollMs);
}
}
const finalCommand = await this.tmux.getPaneCommand(tmuxSessionName);
const finalContent = (await this.tmux.capturePaneContent(tmuxSessionName, 40)) ?? "";
const finalProbe = assessNativeResumeProbe({
runtime: "codex",
paneCommand: finalCommand,
paneContent: finalContent,
});
if (finalProbe.status === "resumed") {
return { ok: true };
}
if (finalCommand && SHELL_COMMANDS.has(finalCommand)) {
return {
ok: false,
code: "retry_fresh",
message: "Codex resume failed: pane returned to shell instead of entering Codex",
};
}
return {
ok: false,
code: "resume_failed",
message: "Codex resume failed: timed out waiting for Codex to become active",
};
}
}
+113
View File
@@ -5,10 +5,16 @@ import type { TmuxAdapter, TmuxResult } from "../src/adapters/tmux.js";
function mockTmux(overrides?: {
sendText?: (target: string, text: string) => Promise<TmuxResult>;
sendKeys?: (target: string, keys: string[]) => Promise<TmuxResult>;
getPaneCommand?: (target: string) => Promise<string | null>;
capturePaneContent?: (target: string, lines?: number) => Promise<string | null>;
}) {
return {
sendText: overrides?.sendText ?? vi.fn(async () => ({ ok: true as const })),
sendKeys: overrides?.sendKeys ?? vi.fn(async () => ({ ok: true as const })),
// Default verifyResume probe: pane shows codex foreground process so
// assessNativeResumeProbe returns resumed/active_runtime on first attempt.
getPaneCommand: overrides?.getPaneCommand ?? vi.fn(async () => "codex"),
capturePaneContent: overrides?.capturePaneContent ?? vi.fn(async () => ""),
createSession: async () => ({ ok: true as const }),
killSession: async () => ({ ok: true as const }),
listSessions: async () => [],
@@ -134,4 +140,111 @@ describe("CodexResumeAdapter", () => {
expect(sendKeys).not.toHaveBeenCalled();
});
});
// verifyResume — closes the false-positive `resumed` gap that fire-and-forget
// left open. Mirrors ClaudeResumeAdapter.verifyResume; uses the existing
// Codex shape in assessNativeResumeProbe (no new probe patterns).
describe("verifyResume", () => {
const fastOptions = { pollMs: 1, maxWaitMs: 5, sleep: async () => {} };
it("probe returns resumed (codex foreground) -> { ok: true }", async () => {
const getPaneCommand = vi.fn(async () => "codex");
const capturePaneContent = vi.fn(async () => "");
const adapter = new CodexResumeAdapter(
mockTmux({ getPaneCommand, capturePaneContent }),
fastOptions,
);
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
expect(result).toEqual({ ok: true });
expect(getPaneCommand).toHaveBeenCalled();
});
it("probe sees Codex TUI banner (paneContent) -> { ok: true }", async () => {
const adapter = new CodexResumeAdapter(
mockTmux({
getPaneCommand: async () => "node",
capturePaneContent: async () => "OpenAI Codex (v0.42.0)\n › ready\n gpt-5 · context",
}),
fastOptions,
);
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
expect(result).toEqual({ ok: true });
});
it("probe sees `No saved session found` -> { ok: false, code: 'retry_fresh' }", async () => {
const adapter = new CodexResumeAdapter(
mockTmux({
getPaneCommand: async () => "codex",
capturePaneContent: async () => "Error: No saved session found for that token.\n",
}),
fastOptions,
);
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.code).toBe("retry_fresh");
expect(result.message).toContain("no saved session");
}
});
it("pane falls back to shell after timeout -> { ok: false, code: 'retry_fresh' }", async () => {
const adapter = new CodexResumeAdapter(
mockTmux({
// Inconclusive during polls (unknown command, empty content), then
// shell on the final assessment.
getPaneCommand: async () => "zsh",
capturePaneContent: async () => "",
}),
fastOptions,
);
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.code).toBe("retry_fresh");
expect(result.message).toContain("returned to shell");
}
});
it("probe stays inconclusive (unknown pane) past timeout -> { ok: false, code: 'resume_failed' }", async () => {
const adapter = new CodexResumeAdapter(
mockTmux({
getPaneCommand: async () => "unknown-binary",
capturePaneContent: async () => "still booting...",
}),
fastOptions,
);
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.code).toBe("resume_failed");
expect(result.message).toContain("timed out");
}
});
it("polls until resumed: first inconclusive, then codex foreground -> { ok: true }", async () => {
let attempt = 0;
const adapter = new CodexResumeAdapter(
mockTmux({
getPaneCommand: async () => (attempt++ === 0 ? "node" : "codex"),
capturePaneContent: async () => "",
}),
{ pollMs: 1, maxWaitMs: 50, sleep: async () => {} },
);
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
expect(result).toEqual({ ok: true });
expect(attempt).toBeGreaterThanOrEqual(2);
});
});
});
@@ -0,0 +1,803 @@
// Tier 1 in-process foundation proof for the lifecycle reboot/recovery
// scenario matrix. One suite per scenario. Mocks tmuxExec/cmuxExec at the
// adapter boundary; in-memory SQLite via createDaemon/createFullTestDb.
//
// Slice packet:
// substrate/shared-docs/openrig-work/missions/recursive-self-improvement-v2/
// primitive-hardening/slices/lifecycle-reboot-recovery-scenario-matrix/
import { describe, it, expect, vi } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type Database from "better-sqlite3";
import { createDaemon } from "../src/startup.js";
import { createDb } from "../src/db/connection.js";
import { migrate } from "../src/db/migrate.js";
import { coreSchema } from "../src/db/migrations/001_core_schema.js";
import { bindingsSessionsSchema } from "../src/db/migrations/002_bindings_sessions.js";
import { eventsSchema } from "../src/db/migrations/003_events.js";
import { nodeSpecFieldsSchema } from "../src/db/migrations/007_node_spec_fields.js";
import { checkpointsSchema } from "../src/db/migrations/005_checkpoints.js";
import { agentspecRebootSchema } from "../src/db/migrations/014_agentspec_reboot.js";
import { RigRepository } from "../src/domain/rig-repository.js";
import { SessionRegistry } from "../src/domain/session-registry.js";
import { EventBus } from "../src/domain/event-bus.js";
import { SnapshotRepository } from "../src/domain/snapshot-repository.js";
import { CheckpointStore } from "../src/domain/checkpoint-store.js";
import { SnapshotCapture } from "../src/domain/snapshot-capture.js";
import { NodeLauncher } from "../src/domain/node-launcher.js";
import { RestoreOrchestrator } from "../src/domain/restore-orchestrator.js";
import { ClaudeResumeAdapter } from "../src/adapters/claude-resume.js";
import { CodexResumeAdapter } from "../src/adapters/codex-resume.js";
import { TmuxAdapter, type ExecFn } from "../src/adapters/tmux.js";
import type { ResumeResult } from "../src/adapters/claude-resume.js";
import { createFullTestDb } from "./helpers/test-app.js";
// ---------------------------------------------------------------------------
// Shared helpers
// ---------------------------------------------------------------------------
function seedDbWithStaleSessions(
dbPath: string,
rigs: { rigName: string; logicalId: string; sessionName: string }[],
): void {
const db = createDb(dbPath);
migrate(db, [
coreSchema,
bindingsSessionsSchema,
eventsSchema,
nodeSpecFieldsSchema,
checkpointsSchema,
agentspecRebootSchema,
]);
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
for (const r of rigs) {
const rig = rigRepo.createRig(r.rigName);
const node = rigRepo.addNode(rig.id, r.logicalId);
const session = sessionRegistry.registerSession(node.id, r.sessionName);
sessionRegistry.updateStatus(session.id, "running");
}
db.close();
}
function unavailableCmuxExec(): ExecFn {
return async () => {
throw Object.assign(new Error(""), { code: "ENOENT" });
};
}
function captureLog<T>(fn: () => Promise<T>): Promise<{ value: T; lines: string[] }> {
const lines: string[] = [];
const logSpy = vi.spyOn(console, "log").mockImplementation((...args: unknown[]) => {
lines.push(args.map(String).join(" "));
});
return (async () => {
try {
const value = await fn();
return { value, lines };
} finally {
logSpy.mockRestore();
}
})();
}
function mockTmuxForRestore(overrides?: Partial<{
hasSession: boolean | (() => Promise<boolean>);
paneCommand: string;
paneContent: string;
}>): TmuxAdapter {
const hasSessionVal = overrides?.hasSession ?? true;
const hasSessionFn = typeof hasSessionVal === "function"
? hasSessionVal
: async () => hasSessionVal;
return {
createSession: vi.fn(async () => ({ ok: true as const })),
killSession: vi.fn(async () => ({ ok: true as const })),
sendText: vi.fn(async () => ({ ok: true as const })),
sendKeys: vi.fn(async () => ({ ok: true as const })),
getPaneCommand: vi.fn(async () => overrides?.paneCommand ?? "claude"),
capturePaneContent: vi.fn(async () => overrides?.paneContent ?? ""),
hasSession: vi.fn(hasSessionFn),
listSessions: async () => [],
listWindows: async () => [],
listPanes: async () => [],
} as unknown as TmuxAdapter;
}
function mockClaudeResumeReturning(result: ResumeResult): ClaudeResumeAdapter {
return {
canResume: vi.fn((type: string | null) => type === "claude_name" || type === "claude_id"),
resume: vi.fn(async () => result),
} as unknown as ClaudeResumeAdapter;
}
function mockCodexResumeReturning(result: ResumeResult): CodexResumeAdapter {
return {
canResume: vi.fn((type: string | null) => type === "codex_id" || type === "codex_last"),
resume: vi.fn(async () => result),
} as unknown as CodexResumeAdapter;
}
// ---------------------------------------------------------------------------
// Scenario suites
// ---------------------------------------------------------------------------
describe("Lifecycle reboot/recovery scenario matrix (Tier 1)", () => {
describe("Scenario 1: Clean start (empty DB, no rigs)", () => {
it("daemon comes up; reconciliation summary line reads rigs=0 checked=0 detached=0 errors=0", async () => {
const tmuxExec: ExecFn = async () => "";
const cmuxExec = unavailableCmuxExec();
const { value, lines } = await captureLog(async () =>
createDaemon({ tmuxExec, cmuxExec }),
);
const summary = lines.find((line) => line.startsWith("startup reconcile:"));
expect(summary).toBeDefined();
expect(summary).toMatch(/rigs=0\b/);
expect(summary).toMatch(/checked=0\b/);
expect(summary).toMatch(/detached=0\b/);
expect(summary).toMatch(/errors=0\b/);
// No spurious session.detached events on a clean start.
const detachedEvents = value.db
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
.all();
expect(detachedEvents).toHaveLength(0);
value.db.close();
});
});
describe("Scenario 2: Warm resume (daemon restart, tmux still alive)", () => {
it("live tmux session is NOT marked detached; summary shows detached=0", async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-matrix-warm-"));
const dbPath = path.join(tmpDir, "warm.sqlite");
seedDbWithStaleSessions(dbPath, [
{ rigName: "r01", logicalId: "dev1-impl", sessionName: "r01-dev1-impl" },
]);
// tmux is alive; has-session returns success (stdout empty, no throw).
const tmuxExec: ExecFn = async () => "";
const cmuxExec = unavailableCmuxExec();
const { value, lines } = await captureLog(async () =>
createDaemon({ dbPath, tmuxExec, cmuxExec }),
);
const sessions = value.db
.prepare("SELECT status FROM sessions")
.all() as { status: string }[];
expect(sessions).toHaveLength(1);
expect(sessions[0]!.status).toBe("running");
const detachedEvents = value.db
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
.all();
expect(detachedEvents).toHaveLength(0);
const summary = lines.find((line) => line.startsWith("startup reconcile:"));
expect(summary).toMatch(/detached=0\b/);
expect(summary).toMatch(/errors=0\b/);
value.db.close();
fs.rmSync(tmpDir, { recursive: true });
});
});
describe("Scenario 3: Host reboot / tmux socket absence (postmortem fixes #1, #2)", () => {
// Each absence-class error must classify as missing session, NOT crash and
// NOT silently leave the row marked running.
it.each([
["error connecting to /private/tmp/tmux-501/default (No such file or directory)"],
["error connecting to /private/tmp/tmux-501/default (Connection refused)"],
["no server running on /private/tmp/tmux-501/default"],
["can't find session: r01-dev1-impl"],
["session not found"],
])("classifies %s as absence; session marked detached and event emitted", async (errMsg) => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-matrix-reboot-"));
const dbPath = path.join(tmpDir, "reboot.sqlite");
seedDbWithStaleSessions(dbPath, [
{ rigName: "r01", logicalId: "dev1-impl", sessionName: "r01-dev1-impl" },
]);
const tmuxExec: ExecFn = async (cmd: string) => {
if (cmd.includes("has-session")) throw new Error(errMsg);
return "";
};
const cmuxExec = unavailableCmuxExec();
const { db } = await createDaemon({ dbPath, tmuxExec, cmuxExec });
const sessions = db.prepare("SELECT status FROM sessions").all() as { status: string }[];
expect(sessions).toHaveLength(1);
expect(sessions[0]!.status).toBe("detached");
const detachedEvents = db
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
.all();
expect(detachedEvents).toHaveLength(1);
db.close();
fs.rmSync(tmpDir, { recursive: true });
});
// Negative case: permission-denied tmux probe must NOT be classified as
// absence. Session stays running (fail-closed) and the unexpected probe
// error is recorded in the summary's errors=N count plus a warning line.
it("permission-denied tmux probe stays NOT-detached (fail-closed) and surfaces warning", async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-matrix-permerr-"));
const dbPath = path.join(tmpDir, "permerr.sqlite");
seedDbWithStaleSessions(dbPath, [
{ rigName: "r02", logicalId: "dev2-impl", sessionName: "r02-dev2-impl" },
]);
const tmuxExec: ExecFn = async (cmd: string) => {
if (cmd.includes("has-session")) {
throw new Error("error: permission denied (EACCES)");
}
return "";
};
const cmuxExec = unavailableCmuxExec();
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
const { value, lines } = await captureLog(async () =>
createDaemon({ dbPath, tmuxExec, cmuxExec }),
);
try {
// Session must NOT be marked detached on ambiguous probe failure.
const sessions = value.db
.prepare("SELECT status FROM sessions")
.all() as { status: string }[];
expect(sessions).toHaveLength(1);
expect(sessions[0]!.status).toBe("running");
// No detached event.
const detached = value.db
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
.all();
expect(detached).toHaveLength(0);
// Summary records the error count.
const summary = lines.find((line) => line.startsWith("startup reconcile:"));
expect(summary).toMatch(/errors=1\b/);
expect(summary).toMatch(/detached=0\b/);
// Per-session warning line emitted.
const warnCalls = warnSpy.mock.calls.map((c) => String(c[0] ?? ""));
const sessWarn = warnCalls.find((line) =>
line.startsWith("startup reconcile warning:") && line.includes("session="),
);
expect(sessWarn).toBeDefined();
expect(sessWarn).toContain("permission denied");
value.db.close();
} finally {
warnSpy.mockRestore();
fs.rmSync(tmpDir, { recursive: true });
}
});
});
describe("Scenario 4: Provider auth loss", () => {
// Helper: build orchestrator + seed a snapshot with a Claude node that
// requests resume via claude_name (legacy resume path).
function setupClaudeAttentionRequiredScenario(opts: {
claudeResult: ResumeResult;
}) {
const db = createFullTestDb();
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const snapshotRepo = new SnapshotRepository(db);
const checkpointStore = new CheckpointStore(db);
const snapshotCapture = new SnapshotCapture({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
});
const tmux = mockTmuxForRestore({ hasSession: false }); // session does not exist yet
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
const orchestrator = new RestoreOrchestrator({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
claudeResume: mockClaudeResumeReturning(opts.claudeResult),
codexResume: mockCodexResumeReturning({ ok: true }),
});
// Seed rig + node + snapshot with a resumable session.
const rig = rigRepo.createRig("r88");
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
// Patch the persisted snapshot to embed a resume_token so the orchestrator
// exercises the legacy resume path with non-pod-aware nodes.
const fullSnap = snapshotRepo.getSnapshot(snap.id)!;
const data = JSON.parse(JSON.stringify(fullSnap.data));
data.sessions = [{
id: "sess-1",
nodeId: node.id,
sessionName: "r88-worker",
status: "running",
resumeType: "claude_name",
resumeToken: "tok-abc",
restorePolicy: "resume_if_possible",
}];
db.prepare("UPDATE snapshots SET data = ? WHERE id = ?")
.run(JSON.stringify(data), snap.id);
return { db, rigRepo, sessionRegistry, eventBus, snapshotRepo, orchestrator, rig, node, snapshotId: snap.id };
}
it("Claude resume returning attention_required → node status=attention_required with prompt evidence", async () => {
const ctx = setupClaudeAttentionRequiredScenario({
claudeResult: {
ok: false,
code: "attention_required",
message: "Claude is at a resume-selection prompt; an operator must choose the conversation to continue.",
evidence: "Choose a conversation to resume:\n 1. project-foo\n 2. project-bar",
},
});
const outcome = await ctx.orchestrator.restore(ctx.snapshotId);
expect(outcome.ok).toBe(true);
if (!outcome.ok) throw new Error(`restore failed: ${outcome.code}`);
// Per-node status carries attention_required (NOT failed).
const workerNode = outcome.result.nodes.find((n) => n.nodeId === ctx.node.id);
expect(workerNode?.status).toBe("attention_required");
// Evidence is preserved on the node result.
expect(workerNode?.attentionEvidence).toBeDefined();
expect(workerNode?.attentionEvidence).toContain("Choose a conversation");
// Rig-level rollup: any attention_required (with at least one
// not-failed) yields partially_restored, NOT failed.
expect(outcome.result.rigResult).toBe("partially_restored");
ctx.db.close();
});
// Codex side — closes the false-positive `resumed` shape that fire-and-
// forget left open. Driver patch (this slice) added Codex verifyResume
// mirroring Claude using existing native-resume-probe Codex outcomes.
describe("Codex verifyResume (driver patch in this slice)", () => {
// Use the real CodexResumeAdapter against a controlled tmux mock so we
// verify the patch wires probe + tmux honestly. No new probe patterns.
const fastOptions = { pollMs: 1, maxWaitMs: 5, sleep: async () => {} };
it("probe sees codex foreground process → {ok: true}", async () => {
const tmux = mockTmuxForRestore({ paneCommand: "codex", paneContent: "" });
const adapter = new CodexResumeAdapter(tmux, fastOptions);
const r = await adapter.resume("r99-worker", "codex_id", "tok-abc", "/tmp");
expect(r).toEqual({ ok: true });
});
it("probe sees `No saved session found` → {ok:false, code:'retry_fresh'} (NOT silent ok:true)", async () => {
const tmux = mockTmuxForRestore({
paneCommand: "codex",
paneContent: "Error: No saved session found for that token.",
});
const adapter = new CodexResumeAdapter(tmux, fastOptions);
const r = await adapter.resume("r99-worker", "codex_id", "tok-abc", "/tmp");
expect(r.ok).toBe(false);
if (!r.ok) expect(r.code).toBe("retry_fresh");
});
it("pane returns to shell → {ok:false, code:'retry_fresh'} (NOT silent ok:true)", async () => {
const tmux = mockTmuxForRestore({ paneCommand: "zsh", paneContent: "" });
const adapter = new CodexResumeAdapter(tmux, fastOptions);
const r = await adapter.resume("r99-worker", "codex_id", "tok-abc", "/tmp");
expect(r.ok).toBe(false);
if (!r.ok) expect(r.code).toBe("retry_fresh");
});
// DOCUMENTED DEFERRAL (slice closeout):
// Codex auth-refusal → attention_required is NOT a current outcome.
// Adding it requires (a) Codex auth-refusal pane patterns researched
// and added to native-resume-probe.ts, (b) a Codex `attention_required`
// outcome added to the probe, (c) a verifyResume branch returning that
// outcome with evidence, (d) restore-orchestrator wiring to map it the
// same way Claude is mapped at lines 725-735. That is a multi-surface
// feature that grows beyond smallest-honest-patch in this matrix slice.
// Tier 2 (real Codex auth refusal in disposable VM) is human-gated per
// disposable-vm-proof-default rule. Defer to a follow-up provider-auth-
// attention slice.
it.skip("DEFERRED: Codex auth-refusal → attention_required (probe extension required)", () => {
// Intentionally skipped; documents the deferred gap in-tree so a
// future driver picks it up alongside the smallest-honest-patch.
});
});
});
describe("Scenario 5: Partial boot", () => {
function buildOrchestratorWithMixedNodes() {
const db = createFullTestDb();
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const snapshotRepo = new SnapshotRepository(db);
const checkpointStore = new CheckpointStore(db);
const snapshotCapture = new SnapshotCapture({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
});
// Mocked Claude returns attention_required for one node, success for
// another. Codex returns failed for the third. We seed three nodes
// sharing one snapshot.
const tmux = mockTmuxForRestore({ hasSession: false });
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
// Dispatch by sessionName so test doesn't depend on iteration order.
const claudeStub = {
canResume: vi.fn((type: string | null) => type === "claude_name" || type === "claude_id"),
resume: vi.fn(async (sessionName: string) => {
if (sessionName.includes("claude-ok")) return { ok: true as const };
return {
ok: false as const,
code: "attention_required",
message: "Claude resume-selection prompt",
evidence: "Choose a conversation:\n 1. foo",
};
}),
} as unknown as ClaudeResumeAdapter;
const codexStub = {
canResume: vi.fn((type: string | null) => type === "codex_id" || type === "codex_last"),
resume: vi.fn(async () => ({
ok: false,
code: "resume_failed",
message: "Codex resume failed: timed out",
})),
} as unknown as CodexResumeAdapter;
const orchestrator = new RestoreOrchestrator({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
claudeResume: claudeStub, codexResume: codexStub,
});
const rig = rigRepo.createRig("r77");
const claudeOk = rigRepo.addNode(rig.id, "claude-ok", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
const claudeAtt = rigRepo.addNode(rig.id, "claude-att", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
const codexFail = rigRepo.addNode(rig.id, "codex-fail", { role: "worker", runtime: "codex", cwd: "/tmp" });
const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
const fullSnap = snapshotRepo.getSnapshot(snap.id)!;
const data = JSON.parse(JSON.stringify(fullSnap.data));
data.sessions = [
{ id: "s-ok", nodeId: claudeOk.id, sessionName: "r77-claude-ok",
status: "running", resumeType: "claude_name", resumeToken: "t1",
restorePolicy: "resume_if_possible" },
{ id: "s-att", nodeId: claudeAtt.id, sessionName: "r77-claude-att",
status: "running", resumeType: "claude_name", resumeToken: "t2",
restorePolicy: "resume_if_possible" },
{ id: "s-fail", nodeId: codexFail.id, sessionName: "r77-codex-fail",
status: "running", resumeType: "codex_id", resumeToken: "t3",
restorePolicy: "resume_if_possible" },
];
db.prepare("UPDATE snapshots SET data = ? WHERE id = ?")
.run(JSON.stringify(data), snap.id);
return { db, orchestrator, rig, snapshotId: snap.id, nodeIds: { claudeOk: claudeOk.id, claudeAtt: claudeAtt.id, codexFail: codexFail.id } };
}
it("mixed resumed + attention_required + failed → rigResult=partially_restored", async () => {
const ctx = buildOrchestratorWithMixedNodes();
const outcome = await ctx.orchestrator.restore(ctx.snapshotId);
expect(outcome.ok).toBe(true);
if (!outcome.ok) throw new Error(`restore failed: ${outcome.code}`);
expect(outcome.result.rigResult).toBe("partially_restored");
// Per-node statuses preserved in the restore.completed payload.
const map = Object.fromEntries(outcome.result.nodes.map((n) => [n.nodeId, n.status]));
expect(map[ctx.nodeIds.claudeOk]).toBe("resumed");
expect(map[ctx.nodeIds.claudeAtt]).toBe("attention_required");
expect(map[ctx.nodeIds.codexFail]).toBe("failed");
// restore.completed event payload preserves per-node discrimination.
const completedRow = ctx.db
.prepare("SELECT payload FROM events WHERE type = 'restore.completed' ORDER BY seq DESC LIMIT 1")
.get() as { payload: string };
const parsed = JSON.parse(completedRow.payload) as { result: { rigResult: string; nodes: Array<{ nodeId: string; status: string }> } };
expect(parsed.result.rigResult).toBe("partially_restored");
expect(parsed.result.nodes.find((n) => n.nodeId === ctx.nodeIds.claudeAtt)?.status).toBe("attention_required");
ctx.db.close();
});
it("all-failed → rigResult=failed (NOT partially_restored)", async () => {
const db = createFullTestDb();
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const snapshotRepo = new SnapshotRepository(db);
const checkpointStore = new CheckpointStore(db);
const snapshotCapture = new SnapshotCapture({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
});
const tmux = mockTmuxForRestore({ hasSession: false });
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
const orchestrator = new RestoreOrchestrator({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
claudeResume: mockClaudeResumeReturning({ ok: false, code: "resume_failed", message: "boom" }),
codexResume: mockCodexResumeReturning({ ok: true }),
});
const rig = rigRepo.createRig("r66");
const a = rigRepo.addNode(rig.id, "a", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
const b = rigRepo.addNode(rig.id, "b", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
const fullSnap = snapshotRepo.getSnapshot(snap.id)!;
const data = JSON.parse(JSON.stringify(fullSnap.data));
data.sessions = [
{ id: "s-a", nodeId: a.id, sessionName: "r66-a", status: "running",
resumeType: "claude_name", resumeToken: "ta", restorePolicy: "resume_if_possible" },
{ id: "s-b", nodeId: b.id, sessionName: "r66-b", status: "running",
resumeType: "claude_name", resumeToken: "tb", restorePolicy: "resume_if_possible" },
];
db.prepare("UPDATE snapshots SET data = ? WHERE id = ?")
.run(JSON.stringify(data), snap.id);
const outcome = await orchestrator.restore(snap.id);
expect(outcome.ok).toBe(true);
if (!outcome.ok) throw new Error(`restore failed: ${outcome.code}`);
expect(outcome.result.rigResult).toBe("failed");
db.close();
});
});
describe("Scenario 6: Operator recovery (reconcileNodeRuntimeTruth)", () => {
function setupForReconcile(opts: {
runtime: "claude-code" | "codex";
restoreOutcome: "failed" | "attention_required";
withResumeToken?: boolean;
paneCommand?: string;
paneContent?: string;
hasSession?: boolean;
}) {
const db = createFullTestDb();
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const snapshotRepo = new SnapshotRepository(db);
const checkpointStore = new CheckpointStore(db);
const snapshotCapture = new SnapshotCapture({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
});
const tmux = mockTmuxForRestore({
hasSession: opts.hasSession ?? true,
paneCommand: opts.paneCommand ?? (opts.runtime === "codex" ? "codex" : "claude"),
paneContent: opts.paneContent ?? "",
});
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
const orchestrator = new RestoreOrchestrator({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
claudeResume: mockClaudeResumeReturning({ ok: true }),
codexResume: mockCodexResumeReturning({ ok: true }),
});
const rig = rigRepo.createRig(`r${Math.floor(Math.random() * 90) + 10}`);
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: opts.runtime });
sessionRegistry.updateBinding(node.id, { tmuxSession: `${rig.name}-worker` });
const sess = sessionRegistry.registerSession(node.id, `${rig.name}-worker`);
if (opts.withResumeToken ?? true) {
db.prepare("UPDATE sessions SET resume_type = ?, resume_token = ? WHERE id = ?").run(
opts.runtime === "codex" ? "codex_id" : "claude_id",
"tok-abc",
sess.id,
);
}
eventBus.emit({ type: "restore.started", rigId: rig.id, snapshotId: "snap-recon" });
eventBus.emit({
type: "restore.completed",
rigId: rig.id,
snapshotId: "snap-recon",
result: {
snapshotId: "snap-recon",
preRestoreSnapshotId: null,
rigResult: "partially_restored",
nodes: [{ nodeId: node.id, logicalId: "worker", status: opts.restoreOutcome }],
warnings: [],
},
});
return { db, orchestrator, rig, nodeId: node.id };
}
it("upgrades failed → operator_recovered when ALL four preconditions hold; emits restore.outcome_reconciled", async () => {
const ctx = setupForReconcile({
runtime: "claude-code",
restoreOutcome: "failed",
paneContent: "Claude Code v2.1.89\n ❯ accept edits on",
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
expect(result.ok).toBe(true);
if (result.ok) {
expect(result.from).toBe("failed");
expect(result.to).toBe("operator_recovered");
expect(result.evidence).toEqual({
tmux: true, fgProcess: "claude", resumeTokenUsed: true, paneState: "usable",
});
}
const reconciled = ctx.db
.prepare("SELECT type FROM events WHERE rig_id = ? AND type = 'restore.outcome_reconciled'")
.all(ctx.rig.id);
expect(reconciled).toHaveLength(1);
// Original failure event NOT mutated/deleted (load-bearing invariant).
const completed = ctx.db
.prepare("SELECT payload FROM events WHERE rig_id = ? AND type = 'restore.completed'")
.all(ctx.rig.id) as { payload: string }[];
expect(completed).toHaveLength(1);
const parsed = JSON.parse(completed[0]!.payload) as { result: { nodes: Array<{ status: string }> } };
expect(parsed.result.nodes[0]!.status).toBe("failed");
ctx.db.close();
});
// Codex foreground recognition is load-bearing per planner research gap
// (`paneCommand.startsWith("codex")` at restore-orchestrator.ts:1038).
it("recognizes Codex foreground process: paneCommand='codex' → operator_recovered with fgProcess='codex'", async () => {
const ctx = setupForReconcile({
runtime: "codex",
restoreOutcome: "failed",
paneCommand: "codex",
paneContent: "OpenAI Codex (v0.42.0)\n › ready\n gpt-5 · context",
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
expect(result.ok).toBe(true);
if (result.ok) {
expect(result.evidence.fgProcess).toBe("codex");
}
ctx.db.close();
});
it("NEVER produces 'ready' as terminal outcome; only operator_recovered", async () => {
const ctx = setupForReconcile({
runtime: "claude-code",
restoreOutcome: "failed",
paneContent: "Claude Code v2.1.89\n ❯ accept edits on",
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
if (result.ok) {
expect(result.to).toBe("operator_recovered");
expect((result.to as string)).not.toBe("ready");
}
const rows = ctx.db
.prepare("SELECT payload FROM events WHERE type = 'restore.outcome_reconciled'")
.all() as { payload: string }[];
for (const row of rows) {
const parsed = JSON.parse(row.payload) as { to: string };
expect(parsed.to).toBe("operator_recovered");
}
ctx.db.close();
});
// Per-precondition refusal codes (load-bearing for honest UX).
it("refuses upgrade with code=tmux_session_missing when tmux probe says false", async () => {
const ctx = setupForReconcile({
runtime: "claude-code", restoreOutcome: "failed", hasSession: false,
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.code).toBe("tmux_session_missing");
ctx.db.close();
});
it("refuses upgrade with code=fg_process_not_runtime when pane is in shell", async () => {
const ctx = setupForReconcile({
runtime: "claude-code", restoreOutcome: "failed",
paneCommand: "zsh", paneContent: "$ ",
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.code).toBe("fg_process_not_runtime");
ctx.db.close();
});
it("refuses upgrade with code=resume_token_not_used when no token recorded", async () => {
const ctx = setupForReconcile({
runtime: "claude-code", restoreOutcome: "failed",
withResumeToken: false,
paneContent: "Claude Code v2.1.89\n ❯ accept edits on",
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.code).toBe("resume_token_not_used");
ctx.db.close();
});
it("refuses upgrade with code=pane_not_usable when pane is at Claude resume-selection prompt", async () => {
const ctx = setupForReconcile({
runtime: "claude-code", restoreOutcome: "attention_required",
paneContent: "Choose a conversation to resume:\n 1. project-foo\n 2. project-bar",
});
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.code).toBe("pane_not_usable");
ctx.db.close();
});
it("refuses upgrade with code=outcome_not_upgradable for an already-resumed outcome", async () => {
const db = createFullTestDb();
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const snapshotRepo = new SnapshotRepository(db);
const checkpointStore = new CheckpointStore(db);
const snapshotCapture = new SnapshotCapture({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
});
const tmux = mockTmuxForRestore();
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
const orchestrator = new RestoreOrchestrator({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
claudeResume: mockClaudeResumeReturning({ ok: true }),
codexResume: mockCodexResumeReturning({ ok: true }),
});
const rig = rigRepo.createRig("r55");
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
sessionRegistry.updateBinding(node.id, { tmuxSession: "r55-worker" });
eventBus.emit({ type: "restore.started", rigId: rig.id, snapshotId: "snap-x" });
eventBus.emit({
type: "restore.completed",
rigId: rig.id,
snapshotId: "snap-x",
result: {
snapshotId: "snap-x",
preRestoreSnapshotId: null,
rigResult: "fully_restored",
nodes: [{ nodeId: node.id, logicalId: "worker", status: "resumed" }],
warnings: [],
},
});
const result = await orchestrator.reconcileNodeRuntimeTruth(rig.id, node.id);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.code).toBe("outcome_not_upgradable");
db.close();
});
it("refuses upgrade with code=no_attempt when no restore.started exists for the rig", async () => {
const db = createFullTestDb();
const rigRepo = new RigRepository(db);
const sessionRegistry = new SessionRegistry(db);
const eventBus = new EventBus(db);
const snapshotRepo = new SnapshotRepository(db);
const checkpointStore = new CheckpointStore(db);
const snapshotCapture = new SnapshotCapture({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
});
const tmux = mockTmuxForRestore();
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
const orchestrator = new RestoreOrchestrator({
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
claudeResume: mockClaudeResumeReturning({ ok: true }),
codexResume: mockCodexResumeReturning({ ok: true }),
});
const rig = rigRepo.createRig("r44");
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
const result = await orchestrator.reconcileNodeRuntimeTruth(rig.id, node.id);
expect(result.ok).toBe(false);
if (!result.ok) expect(result.code).toBe("no_attempt");
db.close();
});
});
});