mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-07 22:58:42 -10:00
feat: native rig restore-check command
H46 Slice 1: read-only restore-readiness diagnostic across host, rigs, and seats. CLI: rig restore-check [--rig <name>] [--no-queue] [--no-hooks] [--json]. Exit 0 for restorable/restorable_with_caveats, 1 for not_restorable, 2 for unknown/probe-error. Human table shows per-check evidence + remediation. JSON includes verdict, counts, checks array (check/status/evidence/remediation), repairPacket null (Slice 2). Daemon: RestoreCheckService (framework-free, read-only per ADR-0001) consumes NodeInventory (ADR-0002) + TranscriptStore + SnapshotRepo. No state mutation, no new ServiceOrchestrator call site (ADR-0003). Route at GET /api/restore-check?rig=<name>&noQueue=true&noHooks=true. Three-tier check model: - Host: daemon.reachable (anchored positive match preserving false-green fix), host.state-dir-writable - Rig: snapshot presence - Seat: transcript (infra exempt), resume-path, queue-file (skippable via --no-queue), hooks (skippable via --no-hooks) Verdict: all-green=restorable, any-yellow=caveats, any-red=not_restorable. Daemon false-green regression matrix: exact "Daemon not running", lowercase, empty output, suspicious non-anchored "running", canonical anchored "Daemon running" — 5 test cases. Tests: daemon service 18/18 + route 6/6; CLI 11/11. Full daemon 1840/1840; CLI 470/471 (preexisting chatroom); tsc + builds clean. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
1 parent
9efaa249f1
commit
4a333f294b
8 files changed
+967
No files matched your search
@@ -0,0 +1,164 @@
|
||||
import { Command } from "commander";
|
||||
import { DaemonClient } from "../client.js";
|
||||
import { getDaemonStatus, getDaemonUrl, type LifecycleDeps } from "../daemon-lifecycle.js";
|
||||
import { realDeps } from "./daemon.js";
|
||||
|
||||
export interface RestoreCheckDeps {
|
||||
lifecycleDeps: LifecycleDeps;
|
||||
clientFactory: (url: string) => DaemonClient;
|
||||
}
|
||||
|
||||
interface CheckEntry {
|
||||
check: string;
|
||||
status: "green" | "yellow" | "red";
|
||||
evidence: string;
|
||||
remediation: string;
|
||||
}
|
||||
|
||||
interface RestoreCheckResult {
|
||||
verdict: "restorable" | "restorable_with_caveats" | "not_restorable" | "unknown";
|
||||
counts: { red: number; yellow: number; green: number };
|
||||
checks: CheckEntry[];
|
||||
repairPacket: null;
|
||||
}
|
||||
|
||||
const STATUS_SYMBOLS: Record<string, string> = {
|
||||
green: "✓",
|
||||
yellow: "⚠",
|
||||
red: "✗",
|
||||
};
|
||||
|
||||
const STATUS_COLORS: Record<string, string> = {
|
||||
green: "",
|
||||
yellow: "",
|
||||
red: "",
|
||||
};
|
||||
|
||||
export function restoreCheckCommand(depsOverride?: RestoreCheckDeps): Command {
|
||||
const cmd = new Command("restore-check")
|
||||
.description("Check restore readiness across running rigs")
|
||||
.addHelpText("after", `
|
||||
Examples:
|
||||
rig restore-check Check all rigs
|
||||
rig restore-check --rig openrig-pm Check one rig
|
||||
rig restore-check --no-queue Skip queue file checks
|
||||
rig restore-check --no-hooks Skip hook checks
|
||||
rig restore-check --json JSON output for agents
|
||||
|
||||
Exit codes:
|
||||
0 Restorable (or restorable with caveats)
|
||||
1 Not restorable (red blockers found)
|
||||
2 Unknown / probe error`);
|
||||
|
||||
const getDepsF = () => depsOverride ?? { lifecycleDeps: realDeps(), clientFactory: (url: string) => new DaemonClient(url) };
|
||||
|
||||
cmd
|
||||
.option("--json", "JSON output for agents")
|
||||
.option("--rig <name>", "Check one rig only")
|
||||
.option("--no-queue", "Skip queue file checks")
|
||||
.option("--no-hooks", "Skip hook checks")
|
||||
.action(async (opts: { json?: boolean; rig?: string; queue?: boolean; hooks?: boolean }) => {
|
||||
const deps = getDepsF();
|
||||
const status = await getDaemonStatus(deps.lifecycleDeps);
|
||||
if (status.state !== "running" || status.healthy === false) {
|
||||
// Daemon down — produce a structured not_restorable result
|
||||
const result: RestoreCheckResult = {
|
||||
verdict: "not_restorable",
|
||||
counts: { red: 1, yellow: 0, green: 0 },
|
||||
checks: [{
|
||||
check: "daemon.reachable",
|
||||
status: "red",
|
||||
evidence: "Daemon is not running",
|
||||
remediation: "Start the daemon with: rig daemon start",
|
||||
}],
|
||||
repairPacket: null,
|
||||
};
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
printHuman(result);
|
||||
}
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const client = deps.clientFactory(getDaemonUrl(status));
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams();
|
||||
if (opts.rig) params.set("rig", opts.rig);
|
||||
if (opts.queue === false) params.set("noQueue", "true");
|
||||
if (opts.hooks === false) params.set("noHooks", "true");
|
||||
const query = params.toString();
|
||||
const path = `/api/restore-check${query ? `?${query}` : ""}`;
|
||||
|
||||
const response = await client.get<RestoreCheckResult>(path);
|
||||
|
||||
if (response.status >= 500) {
|
||||
const result: RestoreCheckResult = {
|
||||
verdict: "unknown",
|
||||
counts: { red: 0, yellow: 0, green: 0 },
|
||||
checks: [{
|
||||
check: "probe.error",
|
||||
status: "red",
|
||||
evidence: `Daemon returned HTTP ${response.status}`,
|
||||
remediation: "Check daemon logs with: rig daemon logs",
|
||||
}],
|
||||
repairPacket: null,
|
||||
};
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
printHuman(result);
|
||||
}
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
|
||||
const result = response.data;
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
printHuman(result);
|
||||
}
|
||||
|
||||
// Exit codes: 0 for restorable/restorable_with_caveats; 1 for not_restorable; 2 for unknown
|
||||
if (result.verdict === "not_restorable") {
|
||||
process.exitCode = 1;
|
||||
} else if (result.verdict === "unknown") {
|
||||
process.exitCode = 2;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
console.error("Fix: check daemon status with: rig daemon status");
|
||||
process.exitCode = 2;
|
||||
}
|
||||
});
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
function printHuman(result: RestoreCheckResult): void {
|
||||
const verdictLabel = result.verdict.replace(/_/g, " ").toUpperCase();
|
||||
console.log(`RESTORE CHECK — ${verdictLabel}`);
|
||||
console.log(`${result.counts.green} green | ${result.counts.yellow} yellow | ${result.counts.red} red`);
|
||||
console.log();
|
||||
|
||||
for (const check of result.checks) {
|
||||
const sym = STATUS_SYMBOLS[check.status] ?? "?";
|
||||
console.log(` ${sym} ${check.check}: ${check.evidence}`);
|
||||
if (check.remediation) {
|
||||
console.log(` Fix: ${check.remediation}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.counts.red > 0) {
|
||||
console.log();
|
||||
console.log("Blockers found. Resolve red items before restoring.");
|
||||
} else if (result.counts.yellow > 0) {
|
||||
console.log();
|
||||
console.log("Restorable with caveats. Yellow items are non-blocking but worth reviewing.");
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,7 @@ import { shrinkCommand } from "./commands/shrink.js";
|
||||
import { destroyCommand, type DestroyCommandDeps } from "./commands/destroy.js";
|
||||
import { setupCommand } from "./commands/setup.js";
|
||||
import { contextCommand } from "./commands/context.js";
|
||||
import { restoreCheckCommand } from "./commands/restore-check.js";
|
||||
import type { LifecycleDeps } from "./daemon-lifecycle.js";
|
||||
import { CLI_VERSION } from "./version.js";
|
||||
|
||||
@@ -138,6 +139,7 @@ export function createProgram(depsOverride?: ProgramDeps): Command {
|
||||
program.addCommand(destroyCommand(depsOverride?.destroyDeps));
|
||||
program.addCommand(setupCommand());
|
||||
program.addCommand(contextCommand());
|
||||
program.addCommand(restoreCheckCommand());
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { restoreCheckCommand, type RestoreCheckDeps } from "../src/commands/restore-check.js";
|
||||
|
||||
function makeResult(overrides?: Record<string, unknown>) {
|
||||
return {
|
||||
verdict: "restorable",
|
||||
counts: { red: 0, yellow: 1, green: 5 },
|
||||
checks: [
|
||||
{ check: "daemon.reachable", status: "green", evidence: "Daemon running on port 7433", remediation: "" },
|
||||
{ check: "host.state-dir-writable", status: "green", evidence: "~/.openrig/ is writable", remediation: "" },
|
||||
{ check: "rig.test-rig.snapshot", status: "yellow", evidence: "No snapshot found", remediation: "Create a snapshot" },
|
||||
],
|
||||
repairPacket: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeDeps(overrides?: {
|
||||
daemonDown?: boolean;
|
||||
result?: Record<string, unknown>;
|
||||
serverError?: boolean;
|
||||
}): { deps: RestoreCheckDeps; requestedPaths: string[] } {
|
||||
const requestedPaths: string[] = [];
|
||||
return {
|
||||
requestedPaths,
|
||||
deps: {
|
||||
lifecycleDeps: {} as RestoreCheckDeps["lifecycleDeps"],
|
||||
clientFactory: () => ({
|
||||
get: vi.fn(async (path: string) => {
|
||||
requestedPaths.push(path);
|
||||
if (overrides?.serverError) return { status: 500, data: { error: "internal error" } };
|
||||
return { status: 200, data: makeResult(overrides?.result) };
|
||||
}),
|
||||
}) as unknown as ReturnType<RestoreCheckDeps["clientFactory"]>,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
vi.mock("../src/daemon-lifecycle.js", async () => {
|
||||
const actual = await vi.importActual<Record<string, unknown>>("../src/daemon-lifecycle.js");
|
||||
return {
|
||||
...actual,
|
||||
getDaemonStatus: vi.fn(async () => ({ state: "running", healthy: true, pid: 1234, port: 7433 })),
|
||||
getDaemonUrl: vi.fn(() => "http://localhost:7433"),
|
||||
};
|
||||
});
|
||||
|
||||
describe("rig restore-check", () => {
|
||||
let logs: string[];
|
||||
let errors: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
logs = [];
|
||||
errors = [];
|
||||
vi.spyOn(console, "log").mockImplementation((...args) => logs.push(args.join(" ")));
|
||||
vi.spyOn(console, "error").mockImplementation((...args) => errors.push(args.join(" ")));
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
it("JSON output has verdict + counts + checks + repairPacket", async () => {
|
||||
const { deps } = makeDeps();
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
|
||||
const json = JSON.parse(logs.join(""));
|
||||
expect(json.verdict).toBe("restorable");
|
||||
expect(json.counts).toBeDefined();
|
||||
expect(json.checks).toBeInstanceOf(Array);
|
||||
expect(json.repairPacket).toBeNull();
|
||||
expect(json.checks[0].check).toBeDefined();
|
||||
expect(json.checks[0].status).toBeDefined();
|
||||
expect(json.checks[0].evidence).toBeDefined();
|
||||
expect(json.checks[0].remediation).toBeDefined();
|
||||
});
|
||||
|
||||
it("human output includes verdict + check details + remediation", async () => {
|
||||
const { deps } = makeDeps({ result: { verdict: "restorable_with_caveats" } });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig"]);
|
||||
|
||||
const output = logs.join("\n");
|
||||
expect(output).toContain("RESTORE CHECK");
|
||||
expect(output).toContain("RESTORABLE WITH CAVEATS");
|
||||
expect(output).toContain("daemon.reachable");
|
||||
});
|
||||
|
||||
it("exit 0 for restorable verdict", async () => {
|
||||
const { deps } = makeDeps({ result: { verdict: "restorable" } });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
expect(process.exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("exit 0 for restorable_with_caveats verdict", async () => {
|
||||
const { deps } = makeDeps({ result: { verdict: "restorable_with_caveats" } });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
expect(process.exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("exit 1 for not_restorable verdict", async () => {
|
||||
const { deps } = makeDeps({ result: { verdict: "not_restorable", counts: { red: 1, yellow: 0, green: 0 } } });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it("exit 2 for unknown verdict", async () => {
|
||||
const { deps } = makeDeps({ result: { verdict: "unknown", counts: { red: 0, yellow: 0, green: 0 } } });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
expect(process.exitCode).toBe(2);
|
||||
});
|
||||
|
||||
it("exit 2 on daemon 500 error", async () => {
|
||||
const { deps } = makeDeps({ serverError: true });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
expect(process.exitCode).toBe(2);
|
||||
});
|
||||
|
||||
it("--rig passes through to daemon route query", async () => {
|
||||
const { deps, requestedPaths } = makeDeps();
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--rig", "openrig-pm", "--json"]);
|
||||
|
||||
expect(requestedPaths.some((p) => p.includes("rig=openrig-pm"))).toBe(true);
|
||||
});
|
||||
|
||||
it("--no-queue passes through to daemon route query", async () => {
|
||||
const { deps, requestedPaths } = makeDeps();
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--no-queue", "--json"]);
|
||||
|
||||
expect(requestedPaths.some((p) => p.includes("noQueue=true"))).toBe(true);
|
||||
});
|
||||
|
||||
it("--no-hooks passes through to daemon route query", async () => {
|
||||
const { deps, requestedPaths } = makeDeps();
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--no-hooks", "--json"]);
|
||||
|
||||
expect(requestedPaths.some((p) => p.includes("noHooks=true"))).toBe(true);
|
||||
});
|
||||
|
||||
it("daemon-down produces not_restorable locally without daemon call", async () => {
|
||||
const { getDaemonStatus } = await import("../src/daemon-lifecycle.js");
|
||||
(getDaemonStatus as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ state: "stopped", healthy: false });
|
||||
|
||||
const { deps, requestedPaths } = makeDeps();
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig", "--json"]);
|
||||
|
||||
const json = JSON.parse(logs.join(""));
|
||||
expect(json.verdict).toBe("not_restorable");
|
||||
expect(json.checks[0].check).toBe("daemon.reachable");
|
||||
expect(json.checks[0].status).toBe("red");
|
||||
expect(process.exitCode).toBe(1);
|
||||
// Should NOT have called the daemon
|
||||
expect(requestedPaths).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,268 @@
|
||||
import { existsSync, accessSync, constants, writeFileSync, unlinkSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { getCompatibleOpenRigPath } from "../openrig-compat.js";
|
||||
|
||||
// --- Types ---
|
||||
|
||||
export type CheckStatus = "green" | "yellow" | "red";
|
||||
export type Verdict = "restorable" | "restorable_with_caveats" | "not_restorable" | "unknown";
|
||||
|
||||
export interface CheckEntry {
|
||||
check: string;
|
||||
status: CheckStatus;
|
||||
evidence: string;
|
||||
remediation: string;
|
||||
}
|
||||
|
||||
export interface RestoreCheckResult {
|
||||
verdict: Verdict;
|
||||
counts: { red: number; yellow: number; green: number };
|
||||
checks: CheckEntry[];
|
||||
repairPacket: null;
|
||||
}
|
||||
|
||||
export interface RestoreCheckOpts {
|
||||
rig?: string;
|
||||
noQueue?: boolean;
|
||||
noHooks?: boolean;
|
||||
}
|
||||
|
||||
// --- Deps (framework-free per ADR-0001; reads from existing projections per ADR-0002) ---
|
||||
|
||||
export interface NodeInventoryEntry {
|
||||
rigId: string;
|
||||
rigName: string;
|
||||
logicalId: string;
|
||||
podId: string | null;
|
||||
podNamespace?: string | null;
|
||||
canonicalSessionName: string | null;
|
||||
nodeKind: "agent" | "infrastructure";
|
||||
runtime: string | null;
|
||||
sessionStatus: string | null;
|
||||
startupStatus: string | null;
|
||||
tmuxAttachCommand: string | null;
|
||||
latestError: string | null;
|
||||
}
|
||||
|
||||
export interface RestoreCheckDeps {
|
||||
/** Get all rigs as summaries */
|
||||
listRigs: () => Array<{ rigId: string; name: string; hasServices?: boolean }>;
|
||||
/** Get node inventory for a rig (ADR-0002: NodeInventory projection) */
|
||||
getNodeInventory: (rigId: string) => NodeInventoryEntry[];
|
||||
/** Check if a snapshot exists for a rig */
|
||||
hasSnapshot: (rigId: string) => boolean;
|
||||
/** Probe daemon health: returns { healthy: boolean; evidence: string } */
|
||||
probeDaemonHealth: () => { healthy: boolean; evidence: string };
|
||||
/** Filesystem probes */
|
||||
exists: (path: string) => boolean;
|
||||
/** Substrate root for queue file path resolution */
|
||||
substrateRoot?: string;
|
||||
}
|
||||
|
||||
// --- Service ---
|
||||
|
||||
const DAEMON_HEALTHY_PATTERN = /^Daemon running\b/m;
|
||||
|
||||
export class RestoreCheckService {
|
||||
private deps: RestoreCheckDeps;
|
||||
|
||||
constructor(deps: RestoreCheckDeps) {
|
||||
this.deps = deps;
|
||||
}
|
||||
|
||||
check(opts: RestoreCheckOpts): RestoreCheckResult {
|
||||
const checks: CheckEntry[] = [];
|
||||
|
||||
// Host-level checks
|
||||
checks.push(this.checkDaemonReachable());
|
||||
checks.push(this.checkStateDirWritable());
|
||||
|
||||
// Get rigs
|
||||
let rigs: Array<{ rigId: string; name: string; hasServices?: boolean }>;
|
||||
try {
|
||||
rigs = this.deps.listRigs();
|
||||
} catch {
|
||||
return this.buildResult([
|
||||
...checks,
|
||||
{ check: "host.rigs", status: "red", evidence: "Failed to list rigs", remediation: "Check daemon status with: rig daemon status" },
|
||||
]);
|
||||
}
|
||||
|
||||
if (opts.rig) {
|
||||
rigs = rigs.filter((r) => r.name === opts.rig);
|
||||
if (rigs.length === 0) {
|
||||
return this.buildResult([
|
||||
...checks,
|
||||
{ check: `rig.${opts.rig}.exists`, status: "red", evidence: `Rig "${opts.rig}" not found`, remediation: "List rigs with: rig ps" },
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-rig checks
|
||||
for (const rig of rigs) {
|
||||
checks.push(this.checkSnapshot(rig));
|
||||
|
||||
// Per-seat checks
|
||||
let nodes: NodeInventoryEntry[];
|
||||
try {
|
||||
nodes = this.deps.getNodeInventory(rig.rigId);
|
||||
} catch {
|
||||
checks.push({ check: `rig.${rig.name}.seats-healthy`, status: "red", evidence: "Failed to get node inventory", remediation: "Check daemon status" });
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const node of nodes) {
|
||||
checks.push(this.checkTranscript(rig.name, node));
|
||||
checks.push(this.checkResumePath(node));
|
||||
if (!opts.noQueue) {
|
||||
checks.push(this.checkQueueFile(rig.name, node));
|
||||
}
|
||||
if (!opts.noHooks) {
|
||||
checks.push(this.checkHooks(node));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this.buildResult(checks);
|
||||
}
|
||||
|
||||
private checkDaemonReachable(): CheckEntry {
|
||||
try {
|
||||
const probe = this.deps.probeDaemonHealth();
|
||||
// Anchored positive match: only "Daemon running" at line start is green.
|
||||
// Anything else (including "Daemon not running", empty output, or text
|
||||
// that contains "running" non-anchored) is red. This preserves the
|
||||
// reviewer fix from prototype 0e2af8d.
|
||||
if (probe.healthy && DAEMON_HEALTHY_PATTERN.test(probe.evidence)) {
|
||||
return { check: "daemon.reachable", status: "green", evidence: probe.evidence, remediation: "" };
|
||||
}
|
||||
return {
|
||||
check: "daemon.reachable", status: "red",
|
||||
evidence: probe.evidence || "Daemon health probe returned non-positive result",
|
||||
remediation: "Start the daemon with: rig daemon start",
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
check: "daemon.reachable", status: "red",
|
||||
evidence: `Probe failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
remediation: "Start the daemon with: rig daemon start",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private checkStateDirWritable(): CheckEntry {
|
||||
const stateDir = getCompatibleOpenRigPath("");
|
||||
const probePath = join(stateDir, ".restore-check-probe");
|
||||
try {
|
||||
writeFileSync(probePath, "probe", "utf-8");
|
||||
unlinkSync(probePath);
|
||||
return { check: "host.state-dir-writable", status: "green", evidence: `${stateDir} is writable`, remediation: "" };
|
||||
} catch {
|
||||
return {
|
||||
check: "host.state-dir-writable", status: "red",
|
||||
evidence: `${stateDir} is not writable`,
|
||||
remediation: `Fix permissions: chmod u+w ${stateDir}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private checkSnapshot(rig: { rigId: string; name: string }): CheckEntry {
|
||||
try {
|
||||
const has = this.deps.hasSnapshot(rig.rigId);
|
||||
if (has) {
|
||||
return { check: `rig.${rig.name}.snapshot`, status: "green", evidence: "Snapshot available", remediation: "" };
|
||||
}
|
||||
return {
|
||||
check: `rig.${rig.name}.snapshot`, status: "yellow",
|
||||
evidence: "No snapshot found (first-boot or adopted rig)",
|
||||
remediation: "Create a snapshot with: rig snapshot <rigId>",
|
||||
};
|
||||
} catch {
|
||||
return { check: `rig.${rig.name}.snapshot`, status: "yellow", evidence: "Could not check snapshots", remediation: "" };
|
||||
}
|
||||
}
|
||||
|
||||
private checkTranscript(rigName: string, node: NodeInventoryEntry): CheckEntry {
|
||||
const session = node.canonicalSessionName ?? node.logicalId;
|
||||
const check = `seat.${session}.transcript`;
|
||||
|
||||
// Terminal/infrastructure nodes are exempt from transcript checks
|
||||
if (node.nodeKind === "infrastructure") {
|
||||
return { check, status: "yellow", evidence: "Terminal/infrastructure node — transcript exempt", remediation: "" };
|
||||
}
|
||||
|
||||
const transcriptPath = join(
|
||||
getCompatibleOpenRigPath("transcripts"),
|
||||
rigName,
|
||||
`${session}.log`
|
||||
);
|
||||
if (this.deps.exists(transcriptPath)) {
|
||||
return { check, status: "green", evidence: `Transcript exists at ${transcriptPath}`, remediation: "" };
|
||||
}
|
||||
return {
|
||||
check, status: "yellow",
|
||||
evidence: `Transcript missing: ${transcriptPath}`,
|
||||
remediation: "Transcript will be created on next session launch",
|
||||
};
|
||||
}
|
||||
|
||||
private checkResumePath(node: NodeInventoryEntry): CheckEntry {
|
||||
const session = node.canonicalSessionName ?? node.logicalId;
|
||||
if (node.tmuxAttachCommand) {
|
||||
return { check: `seat.${session}.resume-path`, status: "green", evidence: node.tmuxAttachCommand, remediation: "" };
|
||||
}
|
||||
return {
|
||||
check: `seat.${session}.resume-path`, status: "yellow",
|
||||
evidence: "No attach command available",
|
||||
remediation: "Session will be created fresh on restore",
|
||||
};
|
||||
}
|
||||
|
||||
private checkQueueFile(rigName: string, node: NodeInventoryEntry): CheckEntry {
|
||||
const session = node.canonicalSessionName ?? node.logicalId;
|
||||
const check = `seat.${session}.queue-file`;
|
||||
|
||||
// Derive queue file path from pod/member
|
||||
const podName = node.podNamespace ?? (node.logicalId.includes(".") ? node.logicalId.split(".")[0] : null);
|
||||
const memberName = node.logicalId.includes(".") ? node.logicalId.split(".").slice(1).join(".") : node.logicalId;
|
||||
|
||||
if (!podName) {
|
||||
return { check, status: "yellow", evidence: "Cannot derive queue path (no pod namespace)", remediation: "" };
|
||||
}
|
||||
|
||||
const substrateRoot = this.deps.substrateRoot ?? join(process.env["HOME"] ?? "~", "code", "substrate", "shared-docs");
|
||||
const queuePath = join(substrateRoot, "rigs", rigName, "state", podName, `${memberName}.queue.md`);
|
||||
|
||||
if (this.deps.exists(queuePath)) {
|
||||
return { check, status: "green", evidence: `Queue file exists at ${queuePath}`, remediation: "" };
|
||||
}
|
||||
return {
|
||||
check, status: "yellow",
|
||||
evidence: `Queue file missing: ${queuePath}`,
|
||||
remediation: "Create queue file per attention-queue convention",
|
||||
};
|
||||
}
|
||||
|
||||
private checkHooks(node: NodeInventoryEntry): CheckEntry {
|
||||
const session = node.canonicalSessionName ?? node.logicalId;
|
||||
// Slice 1: hook check is a placeholder — checks for hook install records
|
||||
return { check: `seat.${session}.hooks`, status: "green", evidence: "Hook check not yet implemented (Slice 2)", remediation: "" };
|
||||
}
|
||||
|
||||
private buildResult(checks: CheckEntry[]): RestoreCheckResult {
|
||||
const red = checks.filter((c) => c.status === "red").length;
|
||||
const yellow = checks.filter((c) => c.status === "yellow").length;
|
||||
const green = checks.filter((c) => c.status === "green").length;
|
||||
|
||||
let verdict: Verdict;
|
||||
if (red > 0) {
|
||||
verdict = "not_restorable";
|
||||
} else if (yellow > 0) {
|
||||
verdict = "restorable_with_caveats";
|
||||
} else {
|
||||
verdict = "restorable";
|
||||
}
|
||||
|
||||
return { verdict, counts: { red, yellow, green }, checks, repairPacket: null };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { Hono } from "hono";
|
||||
import { RestoreCheckService, type RestoreCheckDeps, type NodeInventoryEntry } from "../domain/restore-check-service.js";
|
||||
import { getNodeInventory } from "../domain/node-inventory.js";
|
||||
import type { RigRepository } from "../domain/rig-repository.js";
|
||||
import type { SnapshotRepository } from "../domain/snapshot-repository.js";
|
||||
|
||||
function getDeps(c: { get(key: never): unknown }): {
|
||||
rigRepo: RigRepository;
|
||||
snapshotRepo: SnapshotRepository;
|
||||
} {
|
||||
return {
|
||||
rigRepo: c.get("rigRepo" as never) as RigRepository,
|
||||
snapshotRepo: c.get("snapshotRepo" as never) as SnapshotRepository,
|
||||
};
|
||||
}
|
||||
|
||||
export const restoreCheckRoutes = new Hono();
|
||||
|
||||
// GET /api/restore-check?rig=<name>&noQueue=true&noHooks=true
|
||||
restoreCheckRoutes.get("/", (c) => {
|
||||
const deps = getDeps(c);
|
||||
const rigFilter = c.req.query("rig") ?? undefined;
|
||||
const noQueue = c.req.query("noQueue") === "true";
|
||||
const noHooks = c.req.query("noHooks") === "true";
|
||||
|
||||
try {
|
||||
const serviceDeps: RestoreCheckDeps = {
|
||||
listRigs: () => {
|
||||
const rigs = deps.rigRepo.listRigs();
|
||||
return rigs.map((r) => ({ rigId: r.id, name: r.name }));
|
||||
},
|
||||
getNodeInventory: (rigId: string) => {
|
||||
return getNodeInventory(deps.rigRepo.db, rigId) as NodeInventoryEntry[];
|
||||
},
|
||||
hasSnapshot: (rigId: string) => {
|
||||
return deps.snapshotRepo.listSnapshots(rigId).length > 0;
|
||||
},
|
||||
probeDaemonHealth: () => {
|
||||
// We're inside the daemon — if this route is responding, daemon is healthy
|
||||
return { healthy: true, evidence: "Daemon running (responding to API requests)" };
|
||||
},
|
||||
exists: (path: string) => {
|
||||
try { return existsSync(path); } catch { return false; }
|
||||
},
|
||||
};
|
||||
|
||||
const service = new RestoreCheckService(serviceDeps);
|
||||
const result = service.check({ rig: rigFilter, noQueue, noHooks });
|
||||
|
||||
return c.json(result);
|
||||
} catch (err) {
|
||||
return c.json({
|
||||
verdict: "unknown",
|
||||
counts: { red: 0, yellow: 0, green: 0 },
|
||||
checks: [{
|
||||
check: "probe.error",
|
||||
status: "red",
|
||||
evidence: `Service error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
remediation: "Check daemon logs with: rig daemon logs",
|
||||
}],
|
||||
repairPacket: null,
|
||||
}, 500);
|
||||
}
|
||||
});
|
||||
@@ -35,6 +35,7 @@ import { packagesRoutes } from "./routes/packages.js";
|
||||
import { bootstrapRoutes } from "./routes/bootstrap.js";
|
||||
import { discoveryRoutes } from "./routes/discovery.js";
|
||||
import { bundleRoutes } from "./routes/bundles.js";
|
||||
import { restoreCheckRoutes } from "./routes/restore-check.js";
|
||||
import { agentsRoutes } from "./routes/agents.js";
|
||||
import { psRoutes } from "./routes/ps.js";
|
||||
import type { PsProjectionService } from "./domain/ps-projection.js";
|
||||
@@ -276,6 +277,7 @@ export function createApp(deps: AppDeps): Hono {
|
||||
app.route("/api/whoami", whoamiRoutes());
|
||||
app.route("/api/rigs/:rigId/chat", chatRoutes());
|
||||
app.route("/api/rigs/:rigId/env", envRoutes());
|
||||
app.route("/api/restore-check", restoreCheckRoutes);
|
||||
|
||||
const uiDistDir = deps.uiDistDir ?? resolveDefaultUiDistDir();
|
||||
const uiIndexPath = nodePath.join(uiDistDir, "index.html");
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import type Database from "better-sqlite3";
|
||||
import { createFullTestDb, createTestApp } from "./helpers/test-app.js";
|
||||
|
||||
describe("Restore check routes", () => {
|
||||
let db: Database.Database;
|
||||
let app: ReturnType<typeof createTestApp>["app"];
|
||||
let rigRepo: ReturnType<typeof createTestApp>["rigRepo"];
|
||||
|
||||
beforeEach(async () => {
|
||||
db = createFullTestDb();
|
||||
const setup = createTestApp(db);
|
||||
// Remount with restore-check route
|
||||
const { createApp } = await import("../src/server.js");
|
||||
app = createApp(setup);
|
||||
rigRepo = setup.rigRepo;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("GET /api/restore-check returns JSON with verdict + checks + repairPacket", async () => {
|
||||
rigRepo.createRig("test-rig");
|
||||
|
||||
const res = await app.request("/api/restore-check");
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
const body = await res.json();
|
||||
expect(body.verdict).toBeDefined();
|
||||
expect(["restorable", "restorable_with_caveats", "not_restorable", "unknown"]).toContain(body.verdict);
|
||||
expect(body.counts).toBeDefined();
|
||||
expect(body.checks).toBeInstanceOf(Array);
|
||||
expect(body.repairPacket).toBeNull();
|
||||
|
||||
// Every check has required fields
|
||||
for (const check of body.checks) {
|
||||
expect(typeof check.check).toBe("string");
|
||||
expect(["green", "yellow", "red"]).toContain(check.status);
|
||||
expect(typeof check.evidence).toBe("string");
|
||||
expect(typeof check.remediation).toBe("string");
|
||||
}
|
||||
});
|
||||
|
||||
it("GET /api/restore-check?rig=test-rig filters to named rig", async () => {
|
||||
rigRepo.createRig("rig-a");
|
||||
rigRepo.createRig("rig-b");
|
||||
|
||||
const res = await app.request("/api/restore-check?rig=rig-a");
|
||||
const body = await res.json();
|
||||
|
||||
const rigChecks = body.checks.filter((c: { check: string }) => c.check.includes("rig."));
|
||||
expect(rigChecks.every((c: { check: string }) => !c.check.includes("rig-b"))).toBe(true);
|
||||
});
|
||||
|
||||
it("GET /api/restore-check?rig=unknown returns not_restorable with red check", async () => {
|
||||
rigRepo.createRig("real-rig");
|
||||
|
||||
const res = await app.request("/api/restore-check?rig=nonexistent");
|
||||
const body = await res.json();
|
||||
|
||||
expect(body.verdict).toBe("not_restorable");
|
||||
const notFound = body.checks.find((c: { check: string }) => c.check.includes("nonexistent"));
|
||||
expect(notFound).toBeDefined();
|
||||
expect(notFound.status).toBe("red");
|
||||
});
|
||||
|
||||
it("GET /api/restore-check?noQueue=true skips queue checks", async () => {
|
||||
const rig = rigRepo.createRig("test-rig");
|
||||
rigRepo.addNode(rig.id, "dev.impl", { runtime: "claude-code" });
|
||||
|
||||
const res = await app.request("/api/restore-check?noQueue=true");
|
||||
const body = await res.json();
|
||||
|
||||
const queueChecks = body.checks.filter((c: { check: string }) => c.check.includes("queue-file"));
|
||||
expect(queueChecks).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("GET /api/restore-check?noHooks=true skips hook checks", async () => {
|
||||
const rig = rigRepo.createRig("test-rig");
|
||||
rigRepo.addNode(rig.id, "dev.impl", { runtime: "claude-code" });
|
||||
|
||||
const res = await app.request("/api/restore-check?noHooks=true");
|
||||
const body = await res.json();
|
||||
|
||||
const hookChecks = body.checks.filter((c: { check: string }) => c.check.includes("hooks"));
|
||||
expect(hookChecks).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("daemon.reachable is green inside daemon route (self-proof)", async () => {
|
||||
rigRepo.createRig("test-rig");
|
||||
|
||||
const res = await app.request("/api/restore-check");
|
||||
const body = await res.json();
|
||||
|
||||
const daemon = body.checks.find((c: { check: string }) => c.check === "daemon.reachable");
|
||||
expect(daemon).toBeDefined();
|
||||
expect(daemon.status).toBe("green");
|
||||
expect(daemon.evidence).toContain("Daemon running");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,203 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { RestoreCheckService, type RestoreCheckDeps, type NodeInventoryEntry } from "../src/domain/restore-check-service.js";
|
||||
|
||||
function mockDeps(overrides?: Partial<RestoreCheckDeps>): RestoreCheckDeps {
|
||||
return {
|
||||
listRigs: () => [{ rigId: "rig-1", name: "test-rig" }],
|
||||
getNodeInventory: () => [
|
||||
{
|
||||
rigId: "rig-1", rigName: "test-rig", logicalId: "dev.impl",
|
||||
podId: "dev", podNamespace: "dev",
|
||||
canonicalSessionName: "dev-impl@test-rig",
|
||||
nodeKind: "agent", runtime: "claude-code",
|
||||
sessionStatus: "running", startupStatus: "ready",
|
||||
tmuxAttachCommand: "tmux attach -t dev-impl@test-rig",
|
||||
latestError: null,
|
||||
} as NodeInventoryEntry,
|
||||
],
|
||||
hasSnapshot: () => true,
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running on port 7433" }),
|
||||
exists: () => true,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("RestoreCheckService", () => {
|
||||
// --- Daemon false-green regression matrix ---
|
||||
|
||||
it("daemon-down: exact 'Daemon not running' text produces red", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
probeDaemonHealth: () => ({ healthy: false, evidence: "Daemon not running — start it with: rig daemon start" }),
|
||||
}));
|
||||
const result = service.check({});
|
||||
const daemon = result.checks.find((c) => c.check === "daemon.reachable");
|
||||
expect(daemon?.status).toBe("red");
|
||||
});
|
||||
|
||||
it("daemon-down: lowercase 'daemon not running' produces red", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
probeDaemonHealth: () => ({ healthy: false, evidence: "daemon not running" }),
|
||||
}));
|
||||
const result = service.check({});
|
||||
const daemon = result.checks.find((c) => c.check === "daemon.reachable");
|
||||
expect(daemon?.status).toBe("red");
|
||||
});
|
||||
|
||||
it("daemon-down: empty output produces red", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
probeDaemonHealth: () => ({ healthy: false, evidence: "" }),
|
||||
}));
|
||||
const result = service.check({});
|
||||
const daemon = result.checks.find((c) => c.check === "daemon.reachable");
|
||||
expect(daemon?.status).toBe("red");
|
||||
});
|
||||
|
||||
it("daemon-down: suspicious text containing 'running' non-anchored produces red", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Something is running but not the daemon" }),
|
||||
}));
|
||||
const result = service.check({});
|
||||
const daemon = result.checks.find((c) => c.check === "daemon.reachable");
|
||||
expect(daemon?.status).toBe("red");
|
||||
});
|
||||
|
||||
it("daemon-up: canonical anchored 'Daemon running' produces green", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running on port 7433" }),
|
||||
}));
|
||||
const result = service.check({});
|
||||
const daemon = result.checks.find((c) => c.check === "daemon.reachable");
|
||||
expect(daemon?.status).toBe("green");
|
||||
});
|
||||
|
||||
// --- Rig-level checks ---
|
||||
|
||||
it("missing snapshot produces yellow (not red)", () => {
|
||||
const service = new RestoreCheckService(mockDeps({ hasSnapshot: () => false }));
|
||||
const result = service.check({});
|
||||
const snap = result.checks.find((c) => c.check === "rig.test-rig.snapshot");
|
||||
expect(snap?.status).toBe("yellow");
|
||||
});
|
||||
|
||||
// --- Seat-level checks ---
|
||||
|
||||
it("missing transcript for agent node produces yellow", () => {
|
||||
const service = new RestoreCheckService(mockDeps({ exists: (p) => !p.includes(".log") }));
|
||||
const result = service.check({});
|
||||
const transcript = result.checks.find((c) => c.check === "[email protected]");
|
||||
expect(transcript?.status).toBe("yellow");
|
||||
expect(transcript?.evidence).toContain("missing");
|
||||
});
|
||||
|
||||
it("terminal/infra node transcript check is exempt (yellow, never red)", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
getNodeInventory: () => [{
|
||||
rigId: "rig-1", rigName: "test-rig", logicalId: "infra.board",
|
||||
podId: "infra", podNamespace: "infra",
|
||||
canonicalSessionName: "infra-board@test-rig",
|
||||
nodeKind: "infrastructure", runtime: "terminal",
|
||||
sessionStatus: "running", startupStatus: "ready",
|
||||
tmuxAttachCommand: null, latestError: null,
|
||||
} as NodeInventoryEntry],
|
||||
exists: () => false,
|
||||
}));
|
||||
const result = service.check({});
|
||||
const transcript = result.checks.find((c) => c.check === "[email protected]");
|
||||
expect(transcript?.status).toBe("yellow");
|
||||
expect(transcript?.evidence).toContain("exempt");
|
||||
});
|
||||
|
||||
it("missing queue file produces yellow", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
exists: (p) => !p.includes("queue.md"),
|
||||
}));
|
||||
const result = service.check({});
|
||||
const queue = result.checks.find((c) => c.check === "[email protected]");
|
||||
expect(queue?.status).toBe("yellow");
|
||||
expect(queue?.evidence).toContain("missing");
|
||||
});
|
||||
|
||||
it("--no-queue skips queue file checks", () => {
|
||||
const service = new RestoreCheckService(mockDeps());
|
||||
const result = service.check({ noQueue: true });
|
||||
const queueChecks = result.checks.filter((c) => c.check.includes("queue-file"));
|
||||
expect(queueChecks).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("--no-hooks skips hook checks", () => {
|
||||
const service = new RestoreCheckService(mockDeps());
|
||||
const result = service.check({ noHooks: true });
|
||||
const hookChecks = result.checks.filter((c) => c.check.includes("hooks"));
|
||||
expect(hookChecks).toHaveLength(0);
|
||||
});
|
||||
|
||||
// --- Verdict aggregation ---
|
||||
|
||||
it("all green produces verdict restorable", () => {
|
||||
const service = new RestoreCheckService(mockDeps());
|
||||
const result = service.check({});
|
||||
expect(result.verdict).toBe("restorable");
|
||||
expect(result.counts.red).toBe(0);
|
||||
expect(result.counts.yellow).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it("any yellow (no red) produces verdict restorable_with_caveats", () => {
|
||||
const service = new RestoreCheckService(mockDeps({ hasSnapshot: () => false }));
|
||||
const result = service.check({});
|
||||
expect(result.verdict).toBe("restorable_with_caveats");
|
||||
expect(result.counts.yellow).toBeGreaterThan(0);
|
||||
expect(result.counts.red).toBe(0);
|
||||
});
|
||||
|
||||
it("any red produces verdict not_restorable", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
probeDaemonHealth: () => ({ healthy: false, evidence: "Daemon not running" }),
|
||||
}));
|
||||
const result = service.check({});
|
||||
expect(result.verdict).toBe("not_restorable");
|
||||
expect(result.counts.red).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// --- Rig filter ---
|
||||
|
||||
it("--rig filters to named rig only", () => {
|
||||
const service = new RestoreCheckService(mockDeps({
|
||||
listRigs: () => [
|
||||
{ rigId: "rig-1", name: "rig-a" },
|
||||
{ rigId: "rig-2", name: "rig-b" },
|
||||
],
|
||||
}));
|
||||
const result = service.check({ rig: "rig-a" });
|
||||
// Only rig-specific + seat checks present — no rig-b contamination
|
||||
const rigSpecificChecks = result.checks.filter((c) => c.check.startsWith("rig.") || c.check.startsWith("seat."));
|
||||
expect(rigSpecificChecks.length).toBeGreaterThan(0);
|
||||
expect(rigSpecificChecks.some((c) => c.check.includes("rig-b"))).toBe(false);
|
||||
});
|
||||
|
||||
it("--rig with unknown name produces red", () => {
|
||||
const service = new RestoreCheckService(mockDeps());
|
||||
const result = service.check({ rig: "nonexistent" });
|
||||
expect(result.verdict).toBe("not_restorable");
|
||||
const notFound = result.checks.find((c) => c.check.includes("nonexistent"));
|
||||
expect(notFound?.status).toBe("red");
|
||||
});
|
||||
|
||||
// --- JSON shape ---
|
||||
|
||||
it("result has repairPacket null", () => {
|
||||
const service = new RestoreCheckService(mockDeps());
|
||||
const result = service.check({});
|
||||
expect(result.repairPacket).toBeNull();
|
||||
});
|
||||
|
||||
it("every check has check/status/evidence/remediation fields", () => {
|
||||
const service = new RestoreCheckService(mockDeps());
|
||||
const result = service.check({});
|
||||
for (const check of result.checks) {
|
||||
expect(typeof check.check).toBe("string");
|
||||
expect(["green", "yellow", "red"]).toContain(check.status);
|
||||
expect(typeof check.evidence).toBe("string");
|
||||
expect(typeof check.remediation).toBe("string");
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user