mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-08 08:11:17 -10:00
feat(daemon): codex resume verifies pane via native probe
Closes the false-positive `resumed` shape that fire-and-forget left open.
Codex resume now polls getPaneCommand + capturePaneContent through
assessNativeResumeProbe (using the existing Codex shape — no new probe
patterns) and returns:
- ok: true when probe reports `resumed` (codex foreground or TUI)
- code: retry_fresh when probe reports `failed: no_saved_session` or pane
returned to shell
- code: resume_failed when probe stays inconclusive past timeout
Mirrors ClaudeResumeAdapter.verifyResume; restore-orchestrator.ts:944-947
keeps the existing Codex mapping unchanged (`ok:true → resumed`,
otherwise `failed`).
Lands as part of the lifecycle reboot/recovery scenario matrix slice
(primitive-hardening #2 rig lifecycle, bedrock). Driver matrix audit:
5 rows PASS at HEAD, 1 NEEDS-HONEST-PATCH (this commit), 1 NEEDS-DEFERRAL
(Codex `attention_required` for auth refusal — multi-surface feature
requiring probe extension; deferred to a follow-up provider-auth-attention
slice).
Tests:
- packages/daemon/test/codex-resume.test.ts +6 verifyResume cases
- packages/daemon/test/lifecycle-scenario-matrix.test.ts (new file, 23
Tier 1 in-process tests + 1 documented `it.skip` deferral covering all
six scenarios: clean start, warm resume, host reboot/tmux absence,
provider auth loss, partial boot, operator recovery)
- Daemon suite: 2176 passed | 1 skipped (was 2146 baseline at 578bd5c)
Slice rails: substrate/shared-docs/openrig-work/missions/recursive-self-
improvement-v2/primitive-hardening/slices/lifecycle-reboot-recovery-
scenario-matrix/
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
1 parent
6b7043a04d
commit
ab01331f50
3 files changed
+995
-5
No files matched your search
@@ -1,13 +1,25 @@
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { TmuxAdapter } from "./tmux.js";
|
||||
import type { ResumeResult } from "./claude-resume.js";
|
||||
import { shellQuote } from "./shell-quote.js";
|
||||
import { assessNativeResumeProbe } from "../domain/native-resume-probe.js";
|
||||
|
||||
const CODEX_TYPES = new Set(["codex_id", "codex_last"]);
|
||||
const SHELL_COMMANDS = new Set(["bash", "fish", "nu", "sh", "tmux", "zsh"]);
|
||||
|
||||
export { type ResumeResult };
|
||||
|
||||
interface CodexResumeOptions {
|
||||
pollMs?: number;
|
||||
maxWaitMs?: number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export class CodexResumeAdapter {
|
||||
constructor(private tmux: TmuxAdapter) {}
|
||||
constructor(
|
||||
private tmux: TmuxAdapter,
|
||||
private options: CodexResumeOptions = {}
|
||||
) {}
|
||||
|
||||
canResume(resumeType: string | null, resumeToken: string | null): boolean {
|
||||
if (!resumeType || !CODEX_TYPES.has(resumeType)) return false;
|
||||
@@ -18,9 +30,6 @@ export class CodexResumeAdapter {
|
||||
return true;
|
||||
}
|
||||
|
||||
// NOTE: Resume success is fire-and-forget. sendText succeeding means the
|
||||
// command was typed into the tmux pane, NOT that the harness actually resumed.
|
||||
// TODO(Phase 3): Add resume verification (e.g., poll harness status endpoint).
|
||||
async resume(
|
||||
tmuxSessionName: string,
|
||||
resumeType: string | null,
|
||||
@@ -49,6 +58,71 @@ export class CodexResumeAdapter {
|
||||
return { ok: false, code: "resume_failed", message: keyResult.message };
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
return this.verifyResume(tmuxSessionName);
|
||||
}
|
||||
|
||||
// Mirrors ClaudeResumeAdapter.verifyResume: poll the pane, run the native
|
||||
// probe, return resumed / retry_fresh / resume_failed based on observable
|
||||
// runtime state. Closes the false-positive `resumed` shape that fire-and-
|
||||
// forget left open. Does NOT introduce a Codex `attention_required` outcome
|
||||
// — that requires new probe patterns for Codex auth refusal and is deferred
|
||||
// to a follow-up provider-auth-attention slice.
|
||||
private async verifyResume(tmuxSessionName: string): Promise<ResumeResult> {
|
||||
const pollMs = this.options.pollMs ?? 200;
|
||||
const maxWaitMs = this.options.maxWaitMs ?? 5_000;
|
||||
const sleepFn = this.options.sleep ?? sleep;
|
||||
const attempts = Math.max(1, Math.floor(maxWaitMs / Math.max(pollMs, 1)) + 1);
|
||||
|
||||
for (let attempt = 0; attempt < attempts; attempt++) {
|
||||
const paneCommand = await this.tmux.getPaneCommand(tmuxSessionName);
|
||||
const paneContent = (await this.tmux.capturePaneContent(tmuxSessionName, 40)) ?? "";
|
||||
const probe = assessNativeResumeProbe({
|
||||
runtime: "codex",
|
||||
paneCommand,
|
||||
paneContent,
|
||||
});
|
||||
|
||||
if (probe.code === "no_saved_session") {
|
||||
return {
|
||||
ok: false,
|
||||
code: "retry_fresh",
|
||||
message: "Codex resume failed: no saved session found for the requested token",
|
||||
};
|
||||
}
|
||||
|
||||
if (probe.status === "resumed") {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
if (attempt < attempts - 1) {
|
||||
await sleepFn(pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
const finalCommand = await this.tmux.getPaneCommand(tmuxSessionName);
|
||||
const finalContent = (await this.tmux.capturePaneContent(tmuxSessionName, 40)) ?? "";
|
||||
const finalProbe = assessNativeResumeProbe({
|
||||
runtime: "codex",
|
||||
paneCommand: finalCommand,
|
||||
paneContent: finalContent,
|
||||
});
|
||||
|
||||
if (finalProbe.status === "resumed") {
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
if (finalCommand && SHELL_COMMANDS.has(finalCommand)) {
|
||||
return {
|
||||
ok: false,
|
||||
code: "retry_fresh",
|
||||
message: "Codex resume failed: pane returned to shell instead of entering Codex",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
code: "resume_failed",
|
||||
message: "Codex resume failed: timed out waiting for Codex to become active",
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,16 @@ import type { TmuxAdapter, TmuxResult } from "../src/adapters/tmux.js";
|
||||
function mockTmux(overrides?: {
|
||||
sendText?: (target: string, text: string) => Promise<TmuxResult>;
|
||||
sendKeys?: (target: string, keys: string[]) => Promise<TmuxResult>;
|
||||
getPaneCommand?: (target: string) => Promise<string | null>;
|
||||
capturePaneContent?: (target: string, lines?: number) => Promise<string | null>;
|
||||
}) {
|
||||
return {
|
||||
sendText: overrides?.sendText ?? vi.fn(async () => ({ ok: true as const })),
|
||||
sendKeys: overrides?.sendKeys ?? vi.fn(async () => ({ ok: true as const })),
|
||||
// Default verifyResume probe: pane shows codex foreground process so
|
||||
// assessNativeResumeProbe returns resumed/active_runtime on first attempt.
|
||||
getPaneCommand: overrides?.getPaneCommand ?? vi.fn(async () => "codex"),
|
||||
capturePaneContent: overrides?.capturePaneContent ?? vi.fn(async () => ""),
|
||||
createSession: async () => ({ ok: true as const }),
|
||||
killSession: async () => ({ ok: true as const }),
|
||||
listSessions: async () => [],
|
||||
@@ -134,4 +140,111 @@ describe("CodexResumeAdapter", () => {
|
||||
expect(sendKeys).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// verifyResume — closes the false-positive `resumed` gap that fire-and-forget
|
||||
// left open. Mirrors ClaudeResumeAdapter.verifyResume; uses the existing
|
||||
// Codex shape in assessNativeResumeProbe (no new probe patterns).
|
||||
describe("verifyResume", () => {
|
||||
const fastOptions = { pollMs: 1, maxWaitMs: 5, sleep: async () => {} };
|
||||
|
||||
it("probe returns resumed (codex foreground) -> { ok: true }", async () => {
|
||||
const getPaneCommand = vi.fn(async () => "codex");
|
||||
const capturePaneContent = vi.fn(async () => "");
|
||||
const adapter = new CodexResumeAdapter(
|
||||
mockTmux({ getPaneCommand, capturePaneContent }),
|
||||
fastOptions,
|
||||
);
|
||||
|
||||
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
|
||||
|
||||
expect(result).toEqual({ ok: true });
|
||||
expect(getPaneCommand).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("probe sees Codex TUI banner (paneContent) -> { ok: true }", async () => {
|
||||
const adapter = new CodexResumeAdapter(
|
||||
mockTmux({
|
||||
getPaneCommand: async () => "node",
|
||||
capturePaneContent: async () => "OpenAI Codex (v0.42.0)\n › ready\n gpt-5 · context",
|
||||
}),
|
||||
fastOptions,
|
||||
);
|
||||
|
||||
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
|
||||
|
||||
expect(result).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it("probe sees `No saved session found` -> { ok: false, code: 'retry_fresh' }", async () => {
|
||||
const adapter = new CodexResumeAdapter(
|
||||
mockTmux({
|
||||
getPaneCommand: async () => "codex",
|
||||
capturePaneContent: async () => "Error: No saved session found for that token.\n",
|
||||
}),
|
||||
fastOptions,
|
||||
);
|
||||
|
||||
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) {
|
||||
expect(result.code).toBe("retry_fresh");
|
||||
expect(result.message).toContain("no saved session");
|
||||
}
|
||||
});
|
||||
|
||||
it("pane falls back to shell after timeout -> { ok: false, code: 'retry_fresh' }", async () => {
|
||||
const adapter = new CodexResumeAdapter(
|
||||
mockTmux({
|
||||
// Inconclusive during polls (unknown command, empty content), then
|
||||
// shell on the final assessment.
|
||||
getPaneCommand: async () => "zsh",
|
||||
capturePaneContent: async () => "",
|
||||
}),
|
||||
fastOptions,
|
||||
);
|
||||
|
||||
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) {
|
||||
expect(result.code).toBe("retry_fresh");
|
||||
expect(result.message).toContain("returned to shell");
|
||||
}
|
||||
});
|
||||
|
||||
it("probe stays inconclusive (unknown pane) past timeout -> { ok: false, code: 'resume_failed' }", async () => {
|
||||
const adapter = new CodexResumeAdapter(
|
||||
mockTmux({
|
||||
getPaneCommand: async () => "unknown-binary",
|
||||
capturePaneContent: async () => "still booting...",
|
||||
}),
|
||||
fastOptions,
|
||||
);
|
||||
|
||||
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) {
|
||||
expect(result.code).toBe("resume_failed");
|
||||
expect(result.message).toContain("timed out");
|
||||
}
|
||||
});
|
||||
|
||||
it("polls until resumed: first inconclusive, then codex foreground -> { ok: true }", async () => {
|
||||
let attempt = 0;
|
||||
const adapter = new CodexResumeAdapter(
|
||||
mockTmux({
|
||||
getPaneCommand: async () => (attempt++ === 0 ? "node" : "codex"),
|
||||
capturePaneContent: async () => "",
|
||||
}),
|
||||
{ pollMs: 1, maxWaitMs: 50, sleep: async () => {} },
|
||||
);
|
||||
|
||||
const result = await adapter.resume("r99-demo1-impl", "codex_id", "uuid-123", "/repo");
|
||||
|
||||
expect(result).toEqual({ ok: true });
|
||||
expect(attempt).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,803 @@
|
||||
// Tier 1 in-process foundation proof for the lifecycle reboot/recovery
|
||||
// scenario matrix. One suite per scenario. Mocks tmuxExec/cmuxExec at the
|
||||
// adapter boundary; in-memory SQLite via createDaemon/createFullTestDb.
|
||||
//
|
||||
// Slice packet:
|
||||
// substrate/shared-docs/openrig-work/missions/recursive-self-improvement-v2/
|
||||
// primitive-hardening/slices/lifecycle-reboot-recovery-scenario-matrix/
|
||||
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type Database from "better-sqlite3";
|
||||
import { createDaemon } from "../src/startup.js";
|
||||
import { createDb } from "../src/db/connection.js";
|
||||
import { migrate } from "../src/db/migrate.js";
|
||||
import { coreSchema } from "../src/db/migrations/001_core_schema.js";
|
||||
import { bindingsSessionsSchema } from "../src/db/migrations/002_bindings_sessions.js";
|
||||
import { eventsSchema } from "../src/db/migrations/003_events.js";
|
||||
import { nodeSpecFieldsSchema } from "../src/db/migrations/007_node_spec_fields.js";
|
||||
import { checkpointsSchema } from "../src/db/migrations/005_checkpoints.js";
|
||||
import { agentspecRebootSchema } from "../src/db/migrations/014_agentspec_reboot.js";
|
||||
import { RigRepository } from "../src/domain/rig-repository.js";
|
||||
import { SessionRegistry } from "../src/domain/session-registry.js";
|
||||
import { EventBus } from "../src/domain/event-bus.js";
|
||||
import { SnapshotRepository } from "../src/domain/snapshot-repository.js";
|
||||
import { CheckpointStore } from "../src/domain/checkpoint-store.js";
|
||||
import { SnapshotCapture } from "../src/domain/snapshot-capture.js";
|
||||
import { NodeLauncher } from "../src/domain/node-launcher.js";
|
||||
import { RestoreOrchestrator } from "../src/domain/restore-orchestrator.js";
|
||||
import { ClaudeResumeAdapter } from "../src/adapters/claude-resume.js";
|
||||
import { CodexResumeAdapter } from "../src/adapters/codex-resume.js";
|
||||
import { TmuxAdapter, type ExecFn } from "../src/adapters/tmux.js";
|
||||
import type { ResumeResult } from "../src/adapters/claude-resume.js";
|
||||
import { createFullTestDb } from "./helpers/test-app.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function seedDbWithStaleSessions(
|
||||
dbPath: string,
|
||||
rigs: { rigName: string; logicalId: string; sessionName: string }[],
|
||||
): void {
|
||||
const db = createDb(dbPath);
|
||||
migrate(db, [
|
||||
coreSchema,
|
||||
bindingsSessionsSchema,
|
||||
eventsSchema,
|
||||
nodeSpecFieldsSchema,
|
||||
checkpointsSchema,
|
||||
agentspecRebootSchema,
|
||||
]);
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
for (const r of rigs) {
|
||||
const rig = rigRepo.createRig(r.rigName);
|
||||
const node = rigRepo.addNode(rig.id, r.logicalId);
|
||||
const session = sessionRegistry.registerSession(node.id, r.sessionName);
|
||||
sessionRegistry.updateStatus(session.id, "running");
|
||||
}
|
||||
db.close();
|
||||
}
|
||||
|
||||
function unavailableCmuxExec(): ExecFn {
|
||||
return async () => {
|
||||
throw Object.assign(new Error(""), { code: "ENOENT" });
|
||||
};
|
||||
}
|
||||
|
||||
function captureLog<T>(fn: () => Promise<T>): Promise<{ value: T; lines: string[] }> {
|
||||
const lines: string[] = [];
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation((...args: unknown[]) => {
|
||||
lines.push(args.map(String).join(" "));
|
||||
});
|
||||
return (async () => {
|
||||
try {
|
||||
const value = await fn();
|
||||
return { value, lines };
|
||||
} finally {
|
||||
logSpy.mockRestore();
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
function mockTmuxForRestore(overrides?: Partial<{
|
||||
hasSession: boolean | (() => Promise<boolean>);
|
||||
paneCommand: string;
|
||||
paneContent: string;
|
||||
}>): TmuxAdapter {
|
||||
const hasSessionVal = overrides?.hasSession ?? true;
|
||||
const hasSessionFn = typeof hasSessionVal === "function"
|
||||
? hasSessionVal
|
||||
: async () => hasSessionVal;
|
||||
return {
|
||||
createSession: vi.fn(async () => ({ ok: true as const })),
|
||||
killSession: vi.fn(async () => ({ ok: true as const })),
|
||||
sendText: vi.fn(async () => ({ ok: true as const })),
|
||||
sendKeys: vi.fn(async () => ({ ok: true as const })),
|
||||
getPaneCommand: vi.fn(async () => overrides?.paneCommand ?? "claude"),
|
||||
capturePaneContent: vi.fn(async () => overrides?.paneContent ?? ""),
|
||||
hasSession: vi.fn(hasSessionFn),
|
||||
listSessions: async () => [],
|
||||
listWindows: async () => [],
|
||||
listPanes: async () => [],
|
||||
} as unknown as TmuxAdapter;
|
||||
}
|
||||
|
||||
function mockClaudeResumeReturning(result: ResumeResult): ClaudeResumeAdapter {
|
||||
return {
|
||||
canResume: vi.fn((type: string | null) => type === "claude_name" || type === "claude_id"),
|
||||
resume: vi.fn(async () => result),
|
||||
} as unknown as ClaudeResumeAdapter;
|
||||
}
|
||||
|
||||
function mockCodexResumeReturning(result: ResumeResult): CodexResumeAdapter {
|
||||
return {
|
||||
canResume: vi.fn((type: string | null) => type === "codex_id" || type === "codex_last"),
|
||||
resume: vi.fn(async () => result),
|
||||
} as unknown as CodexResumeAdapter;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario suites
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("Lifecycle reboot/recovery scenario matrix (Tier 1)", () => {
|
||||
describe("Scenario 1: Clean start (empty DB, no rigs)", () => {
|
||||
it("daemon comes up; reconciliation summary line reads rigs=0 checked=0 detached=0 errors=0", async () => {
|
||||
const tmuxExec: ExecFn = async () => "";
|
||||
const cmuxExec = unavailableCmuxExec();
|
||||
|
||||
const { value, lines } = await captureLog(async () =>
|
||||
createDaemon({ tmuxExec, cmuxExec }),
|
||||
);
|
||||
|
||||
const summary = lines.find((line) => line.startsWith("startup reconcile:"));
|
||||
expect(summary).toBeDefined();
|
||||
expect(summary).toMatch(/rigs=0\b/);
|
||||
expect(summary).toMatch(/checked=0\b/);
|
||||
expect(summary).toMatch(/detached=0\b/);
|
||||
expect(summary).toMatch(/errors=0\b/);
|
||||
|
||||
// No spurious session.detached events on a clean start.
|
||||
const detachedEvents = value.db
|
||||
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
|
||||
.all();
|
||||
expect(detachedEvents).toHaveLength(0);
|
||||
|
||||
value.db.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Scenario 2: Warm resume (daemon restart, tmux still alive)", () => {
|
||||
it("live tmux session is NOT marked detached; summary shows detached=0", async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-matrix-warm-"));
|
||||
const dbPath = path.join(tmpDir, "warm.sqlite");
|
||||
seedDbWithStaleSessions(dbPath, [
|
||||
{ rigName: "r01", logicalId: "dev1-impl", sessionName: "r01-dev1-impl" },
|
||||
]);
|
||||
|
||||
// tmux is alive; has-session returns success (stdout empty, no throw).
|
||||
const tmuxExec: ExecFn = async () => "";
|
||||
const cmuxExec = unavailableCmuxExec();
|
||||
|
||||
const { value, lines } = await captureLog(async () =>
|
||||
createDaemon({ dbPath, tmuxExec, cmuxExec }),
|
||||
);
|
||||
|
||||
const sessions = value.db
|
||||
.prepare("SELECT status FROM sessions")
|
||||
.all() as { status: string }[];
|
||||
expect(sessions).toHaveLength(1);
|
||||
expect(sessions[0]!.status).toBe("running");
|
||||
|
||||
const detachedEvents = value.db
|
||||
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
|
||||
.all();
|
||||
expect(detachedEvents).toHaveLength(0);
|
||||
|
||||
const summary = lines.find((line) => line.startsWith("startup reconcile:"));
|
||||
expect(summary).toMatch(/detached=0\b/);
|
||||
expect(summary).toMatch(/errors=0\b/);
|
||||
|
||||
value.db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("Scenario 3: Host reboot / tmux socket absence (postmortem fixes #1, #2)", () => {
|
||||
// Each absence-class error must classify as missing session, NOT crash and
|
||||
// NOT silently leave the row marked running.
|
||||
it.each([
|
||||
["error connecting to /private/tmp/tmux-501/default (No such file or directory)"],
|
||||
["error connecting to /private/tmp/tmux-501/default (Connection refused)"],
|
||||
["no server running on /private/tmp/tmux-501/default"],
|
||||
["can't find session: r01-dev1-impl"],
|
||||
["session not found"],
|
||||
])("classifies %s as absence; session marked detached and event emitted", async (errMsg) => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-matrix-reboot-"));
|
||||
const dbPath = path.join(tmpDir, "reboot.sqlite");
|
||||
seedDbWithStaleSessions(dbPath, [
|
||||
{ rigName: "r01", logicalId: "dev1-impl", sessionName: "r01-dev1-impl" },
|
||||
]);
|
||||
|
||||
const tmuxExec: ExecFn = async (cmd: string) => {
|
||||
if (cmd.includes("has-session")) throw new Error(errMsg);
|
||||
return "";
|
||||
};
|
||||
const cmuxExec = unavailableCmuxExec();
|
||||
|
||||
const { db } = await createDaemon({ dbPath, tmuxExec, cmuxExec });
|
||||
|
||||
const sessions = db.prepare("SELECT status FROM sessions").all() as { status: string }[];
|
||||
expect(sessions).toHaveLength(1);
|
||||
expect(sessions[0]!.status).toBe("detached");
|
||||
|
||||
const detachedEvents = db
|
||||
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
|
||||
.all();
|
||||
expect(detachedEvents).toHaveLength(1);
|
||||
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true });
|
||||
});
|
||||
|
||||
// Negative case: permission-denied tmux probe must NOT be classified as
|
||||
// absence. Session stays running (fail-closed) and the unexpected probe
|
||||
// error is recorded in the summary's errors=N count plus a warning line.
|
||||
it("permission-denied tmux probe stays NOT-detached (fail-closed) and surfaces warning", async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-matrix-permerr-"));
|
||||
const dbPath = path.join(tmpDir, "permerr.sqlite");
|
||||
seedDbWithStaleSessions(dbPath, [
|
||||
{ rigName: "r02", logicalId: "dev2-impl", sessionName: "r02-dev2-impl" },
|
||||
]);
|
||||
|
||||
const tmuxExec: ExecFn = async (cmd: string) => {
|
||||
if (cmd.includes("has-session")) {
|
||||
throw new Error("error: permission denied (EACCES)");
|
||||
}
|
||||
return "";
|
||||
};
|
||||
const cmuxExec = unavailableCmuxExec();
|
||||
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const { value, lines } = await captureLog(async () =>
|
||||
createDaemon({ dbPath, tmuxExec, cmuxExec }),
|
||||
);
|
||||
|
||||
try {
|
||||
// Session must NOT be marked detached on ambiguous probe failure.
|
||||
const sessions = value.db
|
||||
.prepare("SELECT status FROM sessions")
|
||||
.all() as { status: string }[];
|
||||
expect(sessions).toHaveLength(1);
|
||||
expect(sessions[0]!.status).toBe("running");
|
||||
|
||||
// No detached event.
|
||||
const detached = value.db
|
||||
.prepare("SELECT type FROM events WHERE type = 'session.detached'")
|
||||
.all();
|
||||
expect(detached).toHaveLength(0);
|
||||
|
||||
// Summary records the error count.
|
||||
const summary = lines.find((line) => line.startsWith("startup reconcile:"));
|
||||
expect(summary).toMatch(/errors=1\b/);
|
||||
expect(summary).toMatch(/detached=0\b/);
|
||||
|
||||
// Per-session warning line emitted.
|
||||
const warnCalls = warnSpy.mock.calls.map((c) => String(c[0] ?? ""));
|
||||
const sessWarn = warnCalls.find((line) =>
|
||||
line.startsWith("startup reconcile warning:") && line.includes("session="),
|
||||
);
|
||||
expect(sessWarn).toBeDefined();
|
||||
expect(sessWarn).toContain("permission denied");
|
||||
|
||||
value.db.close();
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
fs.rmSync(tmpDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("Scenario 4: Provider auth loss", () => {
|
||||
// Helper: build orchestrator + seed a snapshot with a Claude node that
|
||||
// requests resume via claude_name (legacy resume path).
|
||||
function setupClaudeAttentionRequiredScenario(opts: {
|
||||
claudeResult: ResumeResult;
|
||||
}) {
|
||||
const db = createFullTestDb();
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const snapshotRepo = new SnapshotRepository(db);
|
||||
const checkpointStore = new CheckpointStore(db);
|
||||
const snapshotCapture = new SnapshotCapture({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
|
||||
});
|
||||
const tmux = mockTmuxForRestore({ hasSession: false }); // session does not exist yet
|
||||
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
|
||||
const orchestrator = new RestoreOrchestrator({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
|
||||
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
|
||||
claudeResume: mockClaudeResumeReturning(opts.claudeResult),
|
||||
codexResume: mockCodexResumeReturning({ ok: true }),
|
||||
});
|
||||
|
||||
// Seed rig + node + snapshot with a resumable session.
|
||||
const rig = rigRepo.createRig("r88");
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
|
||||
const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
|
||||
// Patch the persisted snapshot to embed a resume_token so the orchestrator
|
||||
// exercises the legacy resume path with non-pod-aware nodes.
|
||||
const fullSnap = snapshotRepo.getSnapshot(snap.id)!;
|
||||
const data = JSON.parse(JSON.stringify(fullSnap.data));
|
||||
data.sessions = [{
|
||||
id: "sess-1",
|
||||
nodeId: node.id,
|
||||
sessionName: "r88-worker",
|
||||
status: "running",
|
||||
resumeType: "claude_name",
|
||||
resumeToken: "tok-abc",
|
||||
restorePolicy: "resume_if_possible",
|
||||
}];
|
||||
db.prepare("UPDATE snapshots SET data = ? WHERE id = ?")
|
||||
.run(JSON.stringify(data), snap.id);
|
||||
|
||||
return { db, rigRepo, sessionRegistry, eventBus, snapshotRepo, orchestrator, rig, node, snapshotId: snap.id };
|
||||
}
|
||||
|
||||
it("Claude resume returning attention_required → node status=attention_required with prompt evidence", async () => {
|
||||
const ctx = setupClaudeAttentionRequiredScenario({
|
||||
claudeResult: {
|
||||
ok: false,
|
||||
code: "attention_required",
|
||||
message: "Claude is at a resume-selection prompt; an operator must choose the conversation to continue.",
|
||||
evidence: "Choose a conversation to resume:\n 1. project-foo\n 2. project-bar",
|
||||
},
|
||||
});
|
||||
|
||||
const outcome = await ctx.orchestrator.restore(ctx.snapshotId);
|
||||
expect(outcome.ok).toBe(true);
|
||||
if (!outcome.ok) throw new Error(`restore failed: ${outcome.code}`);
|
||||
|
||||
// Per-node status carries attention_required (NOT failed).
|
||||
const workerNode = outcome.result.nodes.find((n) => n.nodeId === ctx.node.id);
|
||||
expect(workerNode?.status).toBe("attention_required");
|
||||
// Evidence is preserved on the node result.
|
||||
expect(workerNode?.attentionEvidence).toBeDefined();
|
||||
expect(workerNode?.attentionEvidence).toContain("Choose a conversation");
|
||||
|
||||
// Rig-level rollup: any attention_required (with at least one
|
||||
// not-failed) yields partially_restored, NOT failed.
|
||||
expect(outcome.result.rigResult).toBe("partially_restored");
|
||||
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
// Codex side — closes the false-positive `resumed` shape that fire-and-
|
||||
// forget left open. Driver patch (this slice) added Codex verifyResume
|
||||
// mirroring Claude using existing native-resume-probe Codex outcomes.
|
||||
describe("Codex verifyResume (driver patch in this slice)", () => {
|
||||
// Use the real CodexResumeAdapter against a controlled tmux mock so we
|
||||
// verify the patch wires probe + tmux honestly. No new probe patterns.
|
||||
const fastOptions = { pollMs: 1, maxWaitMs: 5, sleep: async () => {} };
|
||||
|
||||
it("probe sees codex foreground process → {ok: true}", async () => {
|
||||
const tmux = mockTmuxForRestore({ paneCommand: "codex", paneContent: "" });
|
||||
const adapter = new CodexResumeAdapter(tmux, fastOptions);
|
||||
|
||||
const r = await adapter.resume("r99-worker", "codex_id", "tok-abc", "/tmp");
|
||||
|
||||
expect(r).toEqual({ ok: true });
|
||||
});
|
||||
|
||||
it("probe sees `No saved session found` → {ok:false, code:'retry_fresh'} (NOT silent ok:true)", async () => {
|
||||
const tmux = mockTmuxForRestore({
|
||||
paneCommand: "codex",
|
||||
paneContent: "Error: No saved session found for that token.",
|
||||
});
|
||||
const adapter = new CodexResumeAdapter(tmux, fastOptions);
|
||||
|
||||
const r = await adapter.resume("r99-worker", "codex_id", "tok-abc", "/tmp");
|
||||
|
||||
expect(r.ok).toBe(false);
|
||||
if (!r.ok) expect(r.code).toBe("retry_fresh");
|
||||
});
|
||||
|
||||
it("pane returns to shell → {ok:false, code:'retry_fresh'} (NOT silent ok:true)", async () => {
|
||||
const tmux = mockTmuxForRestore({ paneCommand: "zsh", paneContent: "" });
|
||||
const adapter = new CodexResumeAdapter(tmux, fastOptions);
|
||||
|
||||
const r = await adapter.resume("r99-worker", "codex_id", "tok-abc", "/tmp");
|
||||
|
||||
expect(r.ok).toBe(false);
|
||||
if (!r.ok) expect(r.code).toBe("retry_fresh");
|
||||
});
|
||||
|
||||
// DOCUMENTED DEFERRAL (slice closeout):
|
||||
// Codex auth-refusal → attention_required is NOT a current outcome.
|
||||
// Adding it requires (a) Codex auth-refusal pane patterns researched
|
||||
// and added to native-resume-probe.ts, (b) a Codex `attention_required`
|
||||
// outcome added to the probe, (c) a verifyResume branch returning that
|
||||
// outcome with evidence, (d) restore-orchestrator wiring to map it the
|
||||
// same way Claude is mapped at lines 725-735. That is a multi-surface
|
||||
// feature that grows beyond smallest-honest-patch in this matrix slice.
|
||||
// Tier 2 (real Codex auth refusal in disposable VM) is human-gated per
|
||||
// disposable-vm-proof-default rule. Defer to a follow-up provider-auth-
|
||||
// attention slice.
|
||||
it.skip("DEFERRED: Codex auth-refusal → attention_required (probe extension required)", () => {
|
||||
// Intentionally skipped; documents the deferred gap in-tree so a
|
||||
// future driver picks it up alongside the smallest-honest-patch.
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Scenario 5: Partial boot", () => {
|
||||
function buildOrchestratorWithMixedNodes() {
|
||||
const db = createFullTestDb();
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const snapshotRepo = new SnapshotRepository(db);
|
||||
const checkpointStore = new CheckpointStore(db);
|
||||
const snapshotCapture = new SnapshotCapture({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
|
||||
});
|
||||
|
||||
// Mocked Claude returns attention_required for one node, success for
|
||||
// another. Codex returns failed for the third. We seed three nodes
|
||||
// sharing one snapshot.
|
||||
const tmux = mockTmuxForRestore({ hasSession: false });
|
||||
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
|
||||
// Dispatch by sessionName so test doesn't depend on iteration order.
|
||||
const claudeStub = {
|
||||
canResume: vi.fn((type: string | null) => type === "claude_name" || type === "claude_id"),
|
||||
resume: vi.fn(async (sessionName: string) => {
|
||||
if (sessionName.includes("claude-ok")) return { ok: true as const };
|
||||
return {
|
||||
ok: false as const,
|
||||
code: "attention_required",
|
||||
message: "Claude resume-selection prompt",
|
||||
evidence: "Choose a conversation:\n 1. foo",
|
||||
};
|
||||
}),
|
||||
} as unknown as ClaudeResumeAdapter;
|
||||
const codexStub = {
|
||||
canResume: vi.fn((type: string | null) => type === "codex_id" || type === "codex_last"),
|
||||
resume: vi.fn(async () => ({
|
||||
ok: false,
|
||||
code: "resume_failed",
|
||||
message: "Codex resume failed: timed out",
|
||||
})),
|
||||
} as unknown as CodexResumeAdapter;
|
||||
|
||||
const orchestrator = new RestoreOrchestrator({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
|
||||
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
|
||||
claudeResume: claudeStub, codexResume: codexStub,
|
||||
});
|
||||
|
||||
const rig = rigRepo.createRig("r77");
|
||||
const claudeOk = rigRepo.addNode(rig.id, "claude-ok", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
|
||||
const claudeAtt = rigRepo.addNode(rig.id, "claude-att", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
|
||||
const codexFail = rigRepo.addNode(rig.id, "codex-fail", { role: "worker", runtime: "codex", cwd: "/tmp" });
|
||||
const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
|
||||
const fullSnap = snapshotRepo.getSnapshot(snap.id)!;
|
||||
const data = JSON.parse(JSON.stringify(fullSnap.data));
|
||||
data.sessions = [
|
||||
{ id: "s-ok", nodeId: claudeOk.id, sessionName: "r77-claude-ok",
|
||||
status: "running", resumeType: "claude_name", resumeToken: "t1",
|
||||
restorePolicy: "resume_if_possible" },
|
||||
{ id: "s-att", nodeId: claudeAtt.id, sessionName: "r77-claude-att",
|
||||
status: "running", resumeType: "claude_name", resumeToken: "t2",
|
||||
restorePolicy: "resume_if_possible" },
|
||||
{ id: "s-fail", nodeId: codexFail.id, sessionName: "r77-codex-fail",
|
||||
status: "running", resumeType: "codex_id", resumeToken: "t3",
|
||||
restorePolicy: "resume_if_possible" },
|
||||
];
|
||||
db.prepare("UPDATE snapshots SET data = ? WHERE id = ?")
|
||||
.run(JSON.stringify(data), snap.id);
|
||||
|
||||
return { db, orchestrator, rig, snapshotId: snap.id, nodeIds: { claudeOk: claudeOk.id, claudeAtt: claudeAtt.id, codexFail: codexFail.id } };
|
||||
}
|
||||
|
||||
it("mixed resumed + attention_required + failed → rigResult=partially_restored", async () => {
|
||||
const ctx = buildOrchestratorWithMixedNodes();
|
||||
|
||||
const outcome = await ctx.orchestrator.restore(ctx.snapshotId);
|
||||
expect(outcome.ok).toBe(true);
|
||||
if (!outcome.ok) throw new Error(`restore failed: ${outcome.code}`);
|
||||
|
||||
expect(outcome.result.rigResult).toBe("partially_restored");
|
||||
|
||||
// Per-node statuses preserved in the restore.completed payload.
|
||||
const map = Object.fromEntries(outcome.result.nodes.map((n) => [n.nodeId, n.status]));
|
||||
expect(map[ctx.nodeIds.claudeOk]).toBe("resumed");
|
||||
expect(map[ctx.nodeIds.claudeAtt]).toBe("attention_required");
|
||||
expect(map[ctx.nodeIds.codexFail]).toBe("failed");
|
||||
|
||||
// restore.completed event payload preserves per-node discrimination.
|
||||
const completedRow = ctx.db
|
||||
.prepare("SELECT payload FROM events WHERE type = 'restore.completed' ORDER BY seq DESC LIMIT 1")
|
||||
.get() as { payload: string };
|
||||
const parsed = JSON.parse(completedRow.payload) as { result: { rigResult: string; nodes: Array<{ nodeId: string; status: string }> } };
|
||||
expect(parsed.result.rigResult).toBe("partially_restored");
|
||||
expect(parsed.result.nodes.find((n) => n.nodeId === ctx.nodeIds.claudeAtt)?.status).toBe("attention_required");
|
||||
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
it("all-failed → rigResult=failed (NOT partially_restored)", async () => {
|
||||
const db = createFullTestDb();
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const snapshotRepo = new SnapshotRepository(db);
|
||||
const checkpointStore = new CheckpointStore(db);
|
||||
const snapshotCapture = new SnapshotCapture({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
|
||||
});
|
||||
const tmux = mockTmuxForRestore({ hasSession: false });
|
||||
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
|
||||
const orchestrator = new RestoreOrchestrator({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
|
||||
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
|
||||
claudeResume: mockClaudeResumeReturning({ ok: false, code: "resume_failed", message: "boom" }),
|
||||
codexResume: mockCodexResumeReturning({ ok: true }),
|
||||
});
|
||||
|
||||
const rig = rigRepo.createRig("r66");
|
||||
const a = rigRepo.addNode(rig.id, "a", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
|
||||
const b = rigRepo.addNode(rig.id, "b", { role: "worker", runtime: "claude-code", cwd: "/tmp" });
|
||||
const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
|
||||
const fullSnap = snapshotRepo.getSnapshot(snap.id)!;
|
||||
const data = JSON.parse(JSON.stringify(fullSnap.data));
|
||||
data.sessions = [
|
||||
{ id: "s-a", nodeId: a.id, sessionName: "r66-a", status: "running",
|
||||
resumeType: "claude_name", resumeToken: "ta", restorePolicy: "resume_if_possible" },
|
||||
{ id: "s-b", nodeId: b.id, sessionName: "r66-b", status: "running",
|
||||
resumeType: "claude_name", resumeToken: "tb", restorePolicy: "resume_if_possible" },
|
||||
];
|
||||
db.prepare("UPDATE snapshots SET data = ? WHERE id = ?")
|
||||
.run(JSON.stringify(data), snap.id);
|
||||
|
||||
const outcome = await orchestrator.restore(snap.id);
|
||||
expect(outcome.ok).toBe(true);
|
||||
if (!outcome.ok) throw new Error(`restore failed: ${outcome.code}`);
|
||||
expect(outcome.result.rigResult).toBe("failed");
|
||||
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Scenario 6: Operator recovery (reconcileNodeRuntimeTruth)", () => {
|
||||
function setupForReconcile(opts: {
|
||||
runtime: "claude-code" | "codex";
|
||||
restoreOutcome: "failed" | "attention_required";
|
||||
withResumeToken?: boolean;
|
||||
paneCommand?: string;
|
||||
paneContent?: string;
|
||||
hasSession?: boolean;
|
||||
}) {
|
||||
const db = createFullTestDb();
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const snapshotRepo = new SnapshotRepository(db);
|
||||
const checkpointStore = new CheckpointStore(db);
|
||||
const snapshotCapture = new SnapshotCapture({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
|
||||
});
|
||||
const tmux = mockTmuxForRestore({
|
||||
hasSession: opts.hasSession ?? true,
|
||||
paneCommand: opts.paneCommand ?? (opts.runtime === "codex" ? "codex" : "claude"),
|
||||
paneContent: opts.paneContent ?? "",
|
||||
});
|
||||
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
|
||||
const orchestrator = new RestoreOrchestrator({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
|
||||
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
|
||||
claudeResume: mockClaudeResumeReturning({ ok: true }),
|
||||
codexResume: mockCodexResumeReturning({ ok: true }),
|
||||
});
|
||||
|
||||
const rig = rigRepo.createRig(`r${Math.floor(Math.random() * 90) + 10}`);
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: opts.runtime });
|
||||
sessionRegistry.updateBinding(node.id, { tmuxSession: `${rig.name}-worker` });
|
||||
const sess = sessionRegistry.registerSession(node.id, `${rig.name}-worker`);
|
||||
if (opts.withResumeToken ?? true) {
|
||||
db.prepare("UPDATE sessions SET resume_type = ?, resume_token = ? WHERE id = ?").run(
|
||||
opts.runtime === "codex" ? "codex_id" : "claude_id",
|
||||
"tok-abc",
|
||||
sess.id,
|
||||
);
|
||||
}
|
||||
eventBus.emit({ type: "restore.started", rigId: rig.id, snapshotId: "snap-recon" });
|
||||
eventBus.emit({
|
||||
type: "restore.completed",
|
||||
rigId: rig.id,
|
||||
snapshotId: "snap-recon",
|
||||
result: {
|
||||
snapshotId: "snap-recon",
|
||||
preRestoreSnapshotId: null,
|
||||
rigResult: "partially_restored",
|
||||
nodes: [{ nodeId: node.id, logicalId: "worker", status: opts.restoreOutcome }],
|
||||
warnings: [],
|
||||
},
|
||||
});
|
||||
|
||||
return { db, orchestrator, rig, nodeId: node.id };
|
||||
}
|
||||
|
||||
it("upgrades failed → operator_recovered when ALL four preconditions hold; emits restore.outcome_reconciled", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "claude-code",
|
||||
restoreOutcome: "failed",
|
||||
paneContent: "Claude Code v2.1.89\n ❯ accept edits on",
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
expect(result.ok).toBe(true);
|
||||
if (result.ok) {
|
||||
expect(result.from).toBe("failed");
|
||||
expect(result.to).toBe("operator_recovered");
|
||||
expect(result.evidence).toEqual({
|
||||
tmux: true, fgProcess: "claude", resumeTokenUsed: true, paneState: "usable",
|
||||
});
|
||||
}
|
||||
|
||||
const reconciled = ctx.db
|
||||
.prepare("SELECT type FROM events WHERE rig_id = ? AND type = 'restore.outcome_reconciled'")
|
||||
.all(ctx.rig.id);
|
||||
expect(reconciled).toHaveLength(1);
|
||||
|
||||
// Original failure event NOT mutated/deleted (load-bearing invariant).
|
||||
const completed = ctx.db
|
||||
.prepare("SELECT payload FROM events WHERE rig_id = ? AND type = 'restore.completed'")
|
||||
.all(ctx.rig.id) as { payload: string }[];
|
||||
expect(completed).toHaveLength(1);
|
||||
const parsed = JSON.parse(completed[0]!.payload) as { result: { nodes: Array<{ status: string }> } };
|
||||
expect(parsed.result.nodes[0]!.status).toBe("failed");
|
||||
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
// Codex foreground recognition is load-bearing per planner research gap
|
||||
// (`paneCommand.startsWith("codex")` at restore-orchestrator.ts:1038).
|
||||
it("recognizes Codex foreground process: paneCommand='codex' → operator_recovered with fgProcess='codex'", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "codex",
|
||||
restoreOutcome: "failed",
|
||||
paneCommand: "codex",
|
||||
paneContent: "OpenAI Codex (v0.42.0)\n › ready\n gpt-5 · context",
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
expect(result.ok).toBe(true);
|
||||
if (result.ok) {
|
||||
expect(result.evidence.fgProcess).toBe("codex");
|
||||
}
|
||||
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
it("NEVER produces 'ready' as terminal outcome; only operator_recovered", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "claude-code",
|
||||
restoreOutcome: "failed",
|
||||
paneContent: "Claude Code v2.1.89\n ❯ accept edits on",
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
if (result.ok) {
|
||||
expect(result.to).toBe("operator_recovered");
|
||||
expect((result.to as string)).not.toBe("ready");
|
||||
}
|
||||
const rows = ctx.db
|
||||
.prepare("SELECT payload FROM events WHERE type = 'restore.outcome_reconciled'")
|
||||
.all() as { payload: string }[];
|
||||
for (const row of rows) {
|
||||
const parsed = JSON.parse(row.payload) as { to: string };
|
||||
expect(parsed.to).toBe("operator_recovered");
|
||||
}
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
// Per-precondition refusal codes (load-bearing for honest UX).
|
||||
it("refuses upgrade with code=tmux_session_missing when tmux probe says false", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "claude-code", restoreOutcome: "failed", hasSession: false,
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.code).toBe("tmux_session_missing");
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
it("refuses upgrade with code=fg_process_not_runtime when pane is in shell", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "claude-code", restoreOutcome: "failed",
|
||||
paneCommand: "zsh", paneContent: "$ ",
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.code).toBe("fg_process_not_runtime");
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
it("refuses upgrade with code=resume_token_not_used when no token recorded", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "claude-code", restoreOutcome: "failed",
|
||||
withResumeToken: false,
|
||||
paneContent: "Claude Code v2.1.89\n ❯ accept edits on",
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.code).toBe("resume_token_not_used");
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
it("refuses upgrade with code=pane_not_usable when pane is at Claude resume-selection prompt", async () => {
|
||||
const ctx = setupForReconcile({
|
||||
runtime: "claude-code", restoreOutcome: "attention_required",
|
||||
paneContent: "Choose a conversation to resume:\n 1. project-foo\n 2. project-bar",
|
||||
});
|
||||
const result = await ctx.orchestrator.reconcileNodeRuntimeTruth(ctx.rig.id, ctx.nodeId);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.code).toBe("pane_not_usable");
|
||||
ctx.db.close();
|
||||
});
|
||||
|
||||
it("refuses upgrade with code=outcome_not_upgradable for an already-resumed outcome", async () => {
|
||||
const db = createFullTestDb();
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const snapshotRepo = new SnapshotRepository(db);
|
||||
const checkpointStore = new CheckpointStore(db);
|
||||
const snapshotCapture = new SnapshotCapture({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
|
||||
});
|
||||
const tmux = mockTmuxForRestore();
|
||||
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
|
||||
const orchestrator = new RestoreOrchestrator({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
|
||||
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
|
||||
claudeResume: mockClaudeResumeReturning({ ok: true }),
|
||||
codexResume: mockCodexResumeReturning({ ok: true }),
|
||||
});
|
||||
|
||||
const rig = rigRepo.createRig("r55");
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
|
||||
sessionRegistry.updateBinding(node.id, { tmuxSession: "r55-worker" });
|
||||
eventBus.emit({ type: "restore.started", rigId: rig.id, snapshotId: "snap-x" });
|
||||
eventBus.emit({
|
||||
type: "restore.completed",
|
||||
rigId: rig.id,
|
||||
snapshotId: "snap-x",
|
||||
result: {
|
||||
snapshotId: "snap-x",
|
||||
preRestoreSnapshotId: null,
|
||||
rigResult: "fully_restored",
|
||||
nodes: [{ nodeId: node.id, logicalId: "worker", status: "resumed" }],
|
||||
warnings: [],
|
||||
},
|
||||
});
|
||||
|
||||
const result = await orchestrator.reconcileNodeRuntimeTruth(rig.id, node.id);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.code).toBe("outcome_not_upgradable");
|
||||
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("refuses upgrade with code=no_attempt when no restore.started exists for the rig", async () => {
|
||||
const db = createFullTestDb();
|
||||
const rigRepo = new RigRepository(db);
|
||||
const sessionRegistry = new SessionRegistry(db);
|
||||
const eventBus = new EventBus(db);
|
||||
const snapshotRepo = new SnapshotRepository(db);
|
||||
const checkpointStore = new CheckpointStore(db);
|
||||
const snapshotCapture = new SnapshotCapture({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, checkpointStore,
|
||||
});
|
||||
const tmux = mockTmuxForRestore();
|
||||
const nodeLauncher = new NodeLauncher({ db, rigRepo, sessionRegistry, eventBus, tmuxAdapter: tmux });
|
||||
const orchestrator = new RestoreOrchestrator({
|
||||
db, rigRepo, sessionRegistry, eventBus, snapshotRepo, snapshotCapture,
|
||||
checkpointStore, nodeLauncher, tmuxAdapter: tmux,
|
||||
claudeResume: mockClaudeResumeReturning({ ok: true }),
|
||||
codexResume: mockCodexResumeReturning({ ok: true }),
|
||||
});
|
||||
|
||||
const rig = rigRepo.createRig("r44");
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
|
||||
|
||||
const result = await orchestrator.reconcileNodeRuntimeTruth(rig.id, node.id);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.code).toBe("no_attempt");
|
||||
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user