mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-13 17:48:02 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fb1b7b74e |
No files matched your search
@@ -4747,6 +4747,63 @@ describe("RestoreOrchestrator", () => {
|
||||
expect(db.prepare("SELECT * FROM events WHERE type = 'restore.outcome_reconciled'").all()).toHaveLength(1);
|
||||
});
|
||||
|
||||
// A subset launch records no restore.started: its completion is its attempt.
|
||||
function seedSubsetAttempt(restoreOutcome: "failed" | "attention_required") {
|
||||
const rigName = `r${nextSeed++}`;
|
||||
const rig = rigRepo.createRig(rigName);
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "codex" });
|
||||
const sessionName = `test-worker@${rigName}`;
|
||||
sessionRegistry.updateBinding(node.id, { tmuxSession: sessionName });
|
||||
const sess = sessionRegistry.registerSession(node.id, sessionName);
|
||||
db.prepare("UPDATE sessions SET resume_type = 'codex_id', resume_token = 'tok-abc-123' WHERE id = ?").run(sess.id);
|
||||
const completed = eventBus.emit({ type: "restore.subset_completed", rigId: rig.id, snapshotId: "snap-subset", result: {
|
||||
snapshotId: "snap-subset", preRestoreSnapshotId: null, rigResult: "partially_restored",
|
||||
nodes: [{ nodeId: node.id, logicalId: "worker", status: restoreOutcome }], warnings: [],
|
||||
} } as any);
|
||||
return { rig, nodeId: node.id, attemptId: completed.seq };
|
||||
}
|
||||
const resumedCodexPane = (token: string) => {
|
||||
const tmux = mockTmuxForReconciler();
|
||||
vi.mocked(tmux.hasSession).mockResolvedValue(true);
|
||||
vi.mocked(tmux.getPaneCommand).mockResolvedValue("bash");
|
||||
vi.mocked(tmux.capturePaneContent).mockResolvedValue("OpenAI Codex (v0.155.1)\n› continue");
|
||||
const listProcesses = async () => [
|
||||
{ pid: 1234, ppid: 1, pgid: 1234, tpgid: 1235, command: "bash", executableName: "bash", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 1235, ppid: 1234, pgid: 1235, tpgid: 1235, command: `codex resume ${token}`,
|
||||
argv: ["codex", "resume", token], executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
];
|
||||
return { tmux, listProcesses };
|
||||
};
|
||||
|
||||
it.each(["attention_required", "failed"] as const)("reconciles a subset-launched Codex seat (%s) from the same exact lineage, keeping its outcome and appending the recovery", async (outcome) => {
|
||||
const seeded = seedSubsetAttempt(outcome);
|
||||
const original = db.prepare("SELECT payload FROM events WHERE type = 'restore.subset_completed'").get();
|
||||
const { tmux, listProcesses } = resumedCodexPane("tok-abc-123");
|
||||
|
||||
const result = await createOrchestrator({ tmux, listProcesses }).reconcileNodeRuntimeTruth(seeded.rig.id, seeded.nodeId);
|
||||
|
||||
expect(result).toMatchObject({ ok: true, attemptId: seeded.attemptId, from: outcome, to: "operator_recovered" });
|
||||
expect(tmux.sendKeys).not.toHaveBeenCalled();
|
||||
expect(tmux.sendText).not.toHaveBeenCalled();
|
||||
expect(db.prepare("SELECT payload FROM events WHERE type = 'restore.subset_completed'").get()).toEqual(original);
|
||||
const reconciled = db.prepare("SELECT payload FROM events WHERE type = 'restore.outcome_reconciled'").all() as { payload: string }[];
|
||||
expect(reconciled.map((row) => JSON.parse(row.payload))).toEqual([expect.objectContaining({
|
||||
nodeId: seeded.nodeId, attemptId: seeded.attemptId, from: outcome, to: "operator_recovered" })]);
|
||||
// One receipt per attempt: a repeated check reports it without appending another.
|
||||
await createOrchestrator({ tmux, listProcesses }).reconcileNodeRuntimeTruth(seeded.rig.id, seeded.nodeId);
|
||||
expect(db.prepare("SELECT * FROM events WHERE type = 'restore.outcome_reconciled'").all()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("a subset-launched seat running another conversation keeps its outcome, and nothing is appended", async () => {
|
||||
const seeded = seedSubsetAttempt("attention_required");
|
||||
const { tmux, listProcesses } = resumedCodexPane("tok-other-999");
|
||||
|
||||
const result = await createOrchestrator({ tmux, listProcesses }).reconcileNodeRuntimeTruth(seeded.rig.id, seeded.nodeId);
|
||||
|
||||
expect(result).toMatchObject({ ok: false, code: "process_lineage_mismatch" });
|
||||
expect(db.prepare("SELECT * FROM events WHERE type = 'restore.outcome_reconciled'").all()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["Claude Code v2.1.220\n ❯ accept edits on", true],
|
||||
["Choose a conversation to resume:\n 1. project-foo", false],
|
||||
|
||||
@@ -1082,5 +1082,75 @@ describe("SeatAttentionReconciler", () => {
|
||||
expect(attention(rig.id, waiting).needsAttention).toBe(true);
|
||||
expect(attention(other.id, elsewhere).needsAttention).toBe(true);
|
||||
});
|
||||
|
||||
// A subset launch (`rig launch --seats`) records only its completion; that completion is the attempt.
|
||||
function seedSubsetAtPrompt(rigName: string, sessionName: string): { rigId: string; nodeId: string; attemptId: number } {
|
||||
const rig = rigRepo.createRig(rigName);
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
|
||||
const session = sessionRegistry.registerSession(node.id, sessionName);
|
||||
sessionRegistry.updateStartupStatus(session.id, "ready");
|
||||
sessionRegistry.updateStatus(session.id, "running");
|
||||
const completed = eventBus.emit({ type: "restore.subset_completed", rigId: rig.id, snapshotId: "snap-subset", result: {
|
||||
snapshotId: "snap-subset", preRestoreSnapshotId: null, rigResult: "partially_restored",
|
||||
nodes: [{ nodeId: node.id, logicalId: "worker", status: "attention_required" }], warnings: [],
|
||||
} } as any);
|
||||
return { rigId: rig.id, nodeId: node.id, attemptId: completed.seq };
|
||||
}
|
||||
// Stands in for the restore owner's proof: it appends the attempt-scoped recovery when the live seat proves out.
|
||||
const subsetProof = (proves: boolean) => vi.fn(async (rigId: string, nodeId: string) => {
|
||||
if (!proves) return { ok: false as const, code: "process_lineage_mismatch", detail: "The pane runs another conversation." };
|
||||
const attemptId = (db.prepare("SELECT seq FROM events WHERE rig_id = ? AND type = 'restore.subset_completed' ORDER BY seq DESC LIMIT 1")
|
||||
.get(rigId) as { seq: number }).seq;
|
||||
const evidence = { tmux: true, fgProcess: "claude" as const, resumeTokenUsed: true, paneState: "usable" as const };
|
||||
eventBus.emit({ type: "restore.outcome_reconciled", rigId, nodeId, attemptId, from: "attention_required", to: "operator_recovered", evidence });
|
||||
return { ok: true as const, attemptId, from: "attention_required" as const, to: "operator_recovered" as const, evidence };
|
||||
});
|
||||
|
||||
it("a subset-launched seat's first positive activity runs the strict reconciler, and a proven resume stops needing attention", async () => {
|
||||
const { rigId, nodeId, attemptId } = seedSubsetAtPrompt("r-subset-resumed", "worker@r-subset-resumed");
|
||||
const strict = subsetProof(true);
|
||||
const checked = new SeatAttentionReconciler({ db, sessionRegistry, eventBus, agentActivityStore: activityStore, reconcileRestoreOutcome: strict });
|
||||
expect(attention(rigId, nodeId)).toEqual({ lifecycleState: "attention_required", needsAttention: true });
|
||||
|
||||
expect(await checked.onSeatActivity("worker@r-subset-resumed", "running")).toBe(true);
|
||||
|
||||
expect(strict).toHaveBeenCalledWith(rigId, nodeId);
|
||||
const subset = db.prepare("SELECT seq FROM events WHERE type = 'restore.subset_completed'").all() as { seq: number }[];
|
||||
expect(subset.map((row) => row.seq)).toEqual([attemptId]);
|
||||
expect(attention(rigId, nodeId).needsAttention).toBe(false);
|
||||
});
|
||||
|
||||
it("a subset-launched seat whose proof fails keeps its outcome, with nothing appended or refused", async () => {
|
||||
const { rigId, nodeId } = seedSubsetAtPrompt("r-subset-other", "worker@r-subset-other");
|
||||
const strict = subsetProof(false);
|
||||
const checked = new SeatAttentionReconciler({ db, sessionRegistry, eventBus, agentActivityStore: activityStore, reconcileRestoreOutcome: strict });
|
||||
const before = eventCount();
|
||||
|
||||
expect(await checked.onSeatActivity("worker@r-subset-other", "idle")).toBe(false);
|
||||
|
||||
expect(strict).toHaveBeenCalledTimes(1);
|
||||
expect(eventCount()).toBe(before);
|
||||
expect(attention(rigId, nodeId)).toEqual({ lifecycleState: "attention_required", needsAttention: true });
|
||||
});
|
||||
|
||||
it("a seat that settled before its subset launch completed is checked once at that completion", async () => {
|
||||
const rig = rigRepo.createRig("r-subset-early");
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
|
||||
const session = sessionRegistry.registerSession(node.id, "worker@r-subset-early");
|
||||
sessionRegistry.updateStartupStatus(session.id, "ready");
|
||||
sessionRegistry.updateStatus(session.id, "running");
|
||||
const strict = subsetProof(true);
|
||||
const checked = new SeatAttentionReconciler({ db, sessionRegistry, eventBus, agentActivityStore: activityStore, reconcileRestoreOutcome: strict });
|
||||
expect(await checked.onSeatActivity("worker@r-subset-early", "idle")).toBe(false);
|
||||
eventBus.emit({ type: "restore.subset_completed", rigId: rig.id, snapshotId: "snap-subset", result: {
|
||||
snapshotId: "snap-subset", preRestoreSnapshotId: null, rigResult: "partially_restored",
|
||||
nodes: [{ nodeId: node.id, logicalId: "worker", status: "attention_required" }], warnings: [],
|
||||
} } as any);
|
||||
|
||||
expect(await checked.onRestoreCompleted(rig.id)).toBe(1);
|
||||
expect(await checked.onRestoreCompleted(rig.id)).toBe(0);
|
||||
expect(strict).toHaveBeenCalledTimes(1);
|
||||
expect(attention(rig.id, node.id).needsAttention).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user