fix(restore): reconcile stale DB sessions before blocking on running check

After a tmux crash, DB sessions still show status='running' even
though tmux sessions are dead. RestoreOrchestrator.restore()
checked hasRunningSessions() against DB state alone, refusing
restore with 'rig_not_stopped' when the rig was actually stopped.

Add reconcileStaleRunningSessions() before the running-sessions
check: verifies each DB-running session against tmux reality via
hasSession(); marks dead sessions as 'detached'. Genuinely live
tmux sessions still block restore (inverse invariant preserved).
Fails closed: if tmux check errors, session stays running (honest
about uncertainty).

Error message updated from generic 'must be stopped' to 'has live
sessions' with concrete recovery guidance.

Tests added (3):
- Post-crash: stale DB running + tmux dead → restore proceeds
- Inverse: live tmux sessions → restore still blocks (409)
- Partial failure: one node launch fails → all snapshot nodes
  present in result with logicalIds (no silent pod drops)

Route test (snapshots-routes) updated: 409 test now uses
hasSession=true to simulate genuinely live sessions.

Gates: restore-orchestrator 51/51; snapshots-routes 20/20; full
daemon 1805/1805 (128 files); tsc + build clean.

Restore semantics materially changed (reconcile-before-check):
recommend r1/r2 adversarial review + qa_mentor VM dogfood.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

packages/daemon/src/domain/restore-orchestrator.ts

@@ -110,8 +110,14 @@ export class RestoreOrchestrator {
       return { ok: false, code: "rig_not_found", message: `Rig ${rigId} not found` };
     }
 
+    // Reconcile stale session state before checking — after a tmux crash, DB
+    // sessions may still show 'running' even though tmux is dead. Verify each
+    // DB-running session against tmux reality; mark dead sessions as detached
+    // so restore can proceed. Sessions that ARE genuinely alive still block.
+    await this.reconcileStaleRunningSessions(rigId);
+
     if (this.hasRunningSessions(rigId)) {
-      return { ok: false, code: "rig_not_stopped", message: `Rig ${rigId} must be stopped before restore` };
+      return { ok: false, code: "rig_not_stopped", message: `Rig ${rigId} has live sessions. Stop the rig with 'rig down' before restoring, or use the latest auto-pre-down snapshot.` };
     }
 
     // Per-rig concurrency lock
@@ -180,6 +186,22 @@ export class RestoreOrchestrator {
     return { binding, sessions };
   }
 
+  private async reconcileStaleRunningSessions(rigId: string): Promise<void> {
+    const sessions = this.sessionRegistry.getSessionsForRig(rigId);
+    for (const session of sessions) {
+      if (session.status !== "running") continue;
+      try {
+        const alive = await this.tmuxAdapter.hasSession(session.sessionName);
+        if (!alive) {
+          this.sessionRegistry.markDetached(session.id);
+        }
+      } catch {
+        // tmux check failed — fail closed: leave status as-is so
+        // hasRunningSessions still blocks (honest about uncertainty)
+      }
+    }
+  }
+
   private hasRunningSessions(rigId: string): boolean {
     const latestByNode = new Map<string, Session>();
     for (const session of this.sessionRegistry.getSessionsForRig(rigId)) {

packages/daemon/test/restore-orchestrator.test.ts

@@ -229,14 +229,16 @@ describe("RestoreOrchestrator", () => {
     if (!result.ok) expect(result.code).toBe("snapshot_not_found");
   });
 
-  it("running rig -> { ok: false, code: 'rig_not_stopped' }", async () => {
+  it("running rig with live tmux sessions -> { ok: false, code: 'rig_not_stopped' }", async () => {
     const rig = rigRepo.createRig("r99");
     const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
     const session = sessionRegistry.registerSession(node.id, "r99-worker");
     sessionRegistry.updateStatus(session.id, "running");
     const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
 
-    const orch = createOrchestrator();
+    // tmux session IS alive — restore should block
+    const tmux = { ...mockTmux(), hasSession: vi.fn(async () => true) } as unknown as TmuxAdapter;
+    const orch = createOrchestrator({ tmux });
     const result = await orch.restore(snap.id);
 
     expect(result.ok).toBe(false);
@@ -244,6 +246,91 @@ describe("RestoreOrchestrator", () => {
     expect(snapshotRepo.listSnapshots(rig.id)).toHaveLength(1);
   });
 
+  // --- Scenario A: stale DB sessions after tmux crash ---
+
+  it("restores successfully when DB shows sessions running but tmux sessions are dead (post-crash)", async () => {
+    // After a tmux crash, DB sessions are still status='running' but tmux
+    // has no matching sessions. Restore should reconcile stale state and
+    // proceed, not refuse with 'rig_not_stopped'.
+    const rig = rigRepo.createRig("crash-rig");
+    const node1 = rigRepo.addNode(rig.id, "dev.impl", { role: "worker", runtime: "claude-code" });
+    const node2 = rigRepo.addNode(rig.id, "dev.qa", { role: "worker", runtime: "codex" });
+    const sess1 = sessionRegistry.registerSession(node1.id, "dev-impl@crash-rig");
+    const sess2 = sessionRegistry.registerSession(node2.id, "dev-qa@crash-rig");
+    sessionRegistry.updateStatus(sess1.id, "running");
+    sessionRegistry.updateStatus(sess2.id, "running");
+    const snap = snapshotCapture.captureSnapshot(rig.id, "pre-crash");
+
+    // Simulate post-crash: tmux sessions are gone
+    const tmux = { ...mockTmux(), hasSession: vi.fn(async () => false) } as unknown as TmuxAdapter;
+    const orch = createOrchestrator({ tmux });
+
+    const result = await orch.restore(snap.id);
+
+    // Should proceed with restore, not refuse
+    expect(result.ok).toBe(true);
+    if (result.ok) {
+      expect(result.result.nodes).toHaveLength(2);
+      const ids = result.result.nodes.map((n) => n.logicalId).sort();
+      expect(ids).toEqual(["dev.impl", "dev.qa"]);
+    }
+  });
+
+  it("still blocks restore when tmux sessions are genuinely alive (inverse invariant)", async () => {
+    const rig = rigRepo.createRig("live-rig");
+    const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
+    const sess = sessionRegistry.registerSession(node.id, "worker@live-rig");
+    sessionRegistry.updateStatus(sess.id, "running");
+    const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
+
+    // tmux session IS alive
+    const tmux = { ...mockTmux(), hasSession: vi.fn(async () => true) } as unknown as TmuxAdapter;
+    const orch = createOrchestrator({ tmux });
+
+    const result = await orch.restore(snap.id);
+
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.code).toBe("rig_not_stopped");
+  });
+
+  // --- Scenario B: partial failure doesn't drop pods ---
+
+  it("partial node launch failure includes all snapshot nodes in result (no silent pod drops)", async () => {
+    const snap = seedRigAndSnapshot({
+      nodes: [
+        { logicalId: "orch.lead", role: "orch", runtime: "claude-code" },
+        { logicalId: "dev.impl", role: "impl", runtime: "claude-code" },
+        { logicalId: "dev.qa", role: "qa", runtime: "codex" },
+        { logicalId: "rev.r1", role: "reviewer", runtime: "claude-code" },
+      ],
+      edges: [
+        { sourceLogical: "orch.lead", targetLogical: "dev.impl", kind: "delegates_to" },
+        { sourceLogical: "orch.lead", targetLogical: "rev.r1", kind: "delegates_to" },
+      ],
+    });
+
+    // Make createSession fail for one specific node
+    const tmux = mockTmux();
+    (tmux.createSession as ReturnType<typeof vi.fn>).mockImplementation(async (name: string) => {
+      if (name.includes("dev_qa")) return { ok: false as const, message: "tmux error" };
+      return { ok: true as const };
+    });
+    const orch = createOrchestrator({ tmux });
+
+    const result = await orch.restore(snap.id);
+
+    expect(result.ok).toBe(true);
+    if (result.ok) {
+      const logicalIds = new Set(result.result.nodes.map((n) => n.logicalId));
+      // ALL 4 snapshot nodes present in result — none silently dropped
+      expect(logicalIds).toEqual(new Set(["orch.lead", "dev.impl", "dev.qa", "rev.r1"]));
+      // The failed node should be reported as failed, not absent
+      const qaResult = result.result.nodes.find((n) => n.logicalId === "dev.qa");
+      expect(qaResult).toBeDefined();
+      expect(qaResult!.status).toBe("failed");
+    }
+  });
+
   it("topological order: delegates_to (exact order)", async () => {
     const snap = seedRigAndSnapshot();
     const tmux = mockTmux();

packages/daemon/test/snapshots-routes.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
@@ -8,7 +8,7 @@ import type { RigRepository } from "../src/domain/rig-repository.js";
 import type { SessionRegistry } from "../src/domain/session-registry.js";
 import type { SnapshotCapture } from "../src/domain/snapshot-capture.js";
 import type { SnapshotRepository } from "../src/domain/snapshot-repository.js";
-import { createFullTestDb, createTestApp } from "./helpers/test-app.js";
+import { createFullTestDb, createTestApp, mockTmuxAdapter } from "./helpers/test-app.js";
 import { createDaemon } from "../src/startup.js";
 import type { ExecFn } from "../src/adapters/tmux.js";
 
@@ -208,17 +208,24 @@ describe("Restore routes", () => {
     expect(res.status).toBe(500);
   });
 
-  it("POST running rig restore -> 409", async () => {
-    const rig = rigRepo.createRig("r99");
-    const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
-    const session = sessionRegistry.registerSession(node.id, "r99-worker");
-    sessionRegistry.updateStatus(session.id, "running");
-    const snap = snapshotCapture.captureSnapshot(rig.id, "manual");
+  it("POST running rig restore with live tmux sessions -> 409", async () => {
+    // Use a custom app with hasSession=true to simulate genuinely live tmux
+    const db2 = createFullTestDb();
+    const tmux = mockTmuxAdapter();
+    (tmux.hasSession as ReturnType<typeof vi.fn>).mockResolvedValue(true);
+    const setup = createTestApp(db2, { tmux });
 
-    const res = await app.request(`/api/rigs/${rig.id}/restore/${snap.id}`, { method: "POST" });
+    const rig = setup.rigRepo.createRig("r99");
+    const node = setup.rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
+    const session = setup.sessionRegistry.registerSession(node.id, "r99-worker");
+    setup.sessionRegistry.updateStatus(session.id, "running");
+    const snap = setup.snapshotCapture.captureSnapshot(rig.id, "manual");
+
+    const res = await setup.app.request(`/api/rigs/${rig.id}/restore/${snap.id}`, { method: "POST" });
     expect(res.status).toBe(409);
     const body = await res.json();
     expect(body.code).toBe("rig_not_stopped");
+    db2.close();
   });
 
   it("restore response includes nodes array with status per node", async () => {