mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-07 22:58:42 -10:00
Merge OPR.0.4.0.29 restore-check compact.
This commit is contained in:
commit
8d55ea6058
6 files changed
+592
-18
No files matched your search
@@ -90,6 +90,14 @@ interface RestoreCheckResult {
|
||||
hostInfra: HostInfraAssertion;
|
||||
recovery: RecoveryPlan;
|
||||
counts: { red: number; yellow: number; green: number };
|
||||
// OPR.0.4.0.29 FR-8 — ready-confidence breakdown by the 5 real-enum classes.
|
||||
classCounts?: {
|
||||
ready: number;
|
||||
ready_with_caveats: number;
|
||||
not_ready: number;
|
||||
attention_required: number;
|
||||
unknown: number;
|
||||
};
|
||||
checks: CheckEntry[];
|
||||
repairPacket: RepairStep[] | null;
|
||||
}
|
||||
@@ -125,11 +133,30 @@ Exit codes:
|
||||
const getDepsF = () => depsOverride ?? { lifecycleDeps: realDeps(), clientFactory: (url: string) => new DaemonClient(url) };
|
||||
|
||||
cmd
|
||||
.option("--json", "JSON output for agents")
|
||||
.option("--json", "JSON output (compact summary by default; use --full --json for complete detail)")
|
||||
.option("--full", "Show complete per-seat detail (today's default)")
|
||||
.option("--ready", "Show ready-seat detail in addition to not-ready")
|
||||
.option("--rig <name>", "Check one rig only")
|
||||
.option("--no-queue", "Skip queue file checks")
|
||||
.option("--no-hooks", "Skip hook checks")
|
||||
.action(async (opts: { json?: boolean; rig?: string; queue?: boolean; hooks?: boolean }) => {
|
||||
.addHelpText("after", `
|
||||
Default: compact summary with per-rig readiness counts and NOT-READY seats only.
|
||||
Ready seats are counted but their detail is omitted to save tokens.
|
||||
|
||||
Use --full for today's complete per-seat detail (all checks, repair packet, etc.).
|
||||
Use --ready to include ready-seat detail alongside not-ready.
|
||||
Use --full --json for the complete JSON firehose.
|
||||
|
||||
Readiness classes: ready, ready_with_caveats, not_ready, attention_required, unknown.
|
||||
|
||||
Examples:
|
||||
rig restore-check Compact summary (counts + not-ready)
|
||||
rig restore-check --json Compact JSON summary
|
||||
rig restore-check --full Complete per-seat detail
|
||||
rig restore-check --full --json Complete JSON (the firehose)
|
||||
rig restore-check --ready Include ready seats in output
|
||||
rig restore-check --rig my-rig Check one rig only`)
|
||||
.action(async (opts: { json?: boolean; full?: boolean; ready?: boolean; rig?: string; queue?: boolean; hooks?: boolean }) => {
|
||||
const deps = getDepsF();
|
||||
const status = await getDaemonStatus(deps.lifecycleDeps);
|
||||
if (status.state !== "running" || status.healthy === false) {
|
||||
@@ -174,6 +201,11 @@ Exit codes:
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams();
|
||||
// Compact by default; --full opts out. --ready stays COMPACT and adds
|
||||
// ready-seat detail (ready=1) IN ADDITION to the not-ready rows — it is
|
||||
// NOT the full firehose (OPR.0.4.0.29 FR-2 / forward-fix corrective).
|
||||
if (!opts.full) params.set("compact", "1");
|
||||
if (opts.ready) params.set("ready", "1");
|
||||
if (opts.rig) params.set("rig", opts.rig);
|
||||
if (opts.queue === false) params.set("noQueue", "true");
|
||||
if (opts.hooks === false) params.set("noHooks", "true");
|
||||
@@ -222,8 +254,10 @@ Exit codes:
|
||||
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
} else if (opts.full) {
|
||||
printHuman(result);
|
||||
} else {
|
||||
printCompact(result);
|
||||
}
|
||||
|
||||
// Exit codes: 0 for restorable/restorable_with_caveats; 1 for not_restorable; 2 for unknown
|
||||
@@ -301,6 +335,64 @@ function printHuman(result: RestoreCheckResult): void {
|
||||
}
|
||||
}
|
||||
|
||||
function printCompact(result: RestoreCheckResult): void {
|
||||
const verdictLabel = result.verdict.replace(/_/g, " ").toUpperCase();
|
||||
console.log(`RESTORE CHECK — ${verdictLabel}`);
|
||||
console.log(`${result.counts.green} green | ${result.counts.yellow} yellow | ${result.counts.red} red`);
|
||||
const readinessLabel = result.readiness.status.replace(/_/g, " ");
|
||||
console.log(`READINESS: ${readinessLabel}`);
|
||||
// FR-8: ready-confidence breakdown by class (the 5 real-enum classes).
|
||||
if (result.classCounts) {
|
||||
const cc = result.classCounts;
|
||||
console.log(
|
||||
`CLASSES: ${cc.ready} ready | ${cc.ready_with_caveats} ready_with_caveats | ` +
|
||||
`${cc.not_ready} not_ready | ${cc.attention_required} attention_required | ${cc.unknown} unknown`
|
||||
);
|
||||
}
|
||||
|
||||
if (result.rigs && result.rigs.length > 0) {
|
||||
console.log();
|
||||
console.log(`${"RIG".padEnd(24)} ${"STATUS".padEnd(18)} ${"READY".padEnd(9)} ${"BLOCKED".padEnd(8)} CAVEATS`);
|
||||
for (const rig of result.rigs) {
|
||||
console.log(
|
||||
`${rig.rigName.slice(0, 24).padEnd(24)} ${rig.status.padEnd(18)} ` +
|
||||
`${`${rig.runningReadyNodes}/${rig.expectedNodes}`.padEnd(9)} ${String(rig.blockedNodes).padEnd(8)} ${rig.caveatNodes}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// The route already scopes `checks` to the right set: not-ready only by
|
||||
// default, or all seats (including ready detail) when --ready (ready=1).
|
||||
// Render exactly what it returned — do not re-filter here.
|
||||
if (result.checks.length > 0) {
|
||||
console.log();
|
||||
for (const check of result.checks) {
|
||||
const sym = STATUS_SYMBOLS[check.status] ?? "?";
|
||||
console.log(` ${sym} ${check.check}: ${check.evidence}`);
|
||||
if (check.remediation) {
|
||||
console.log(` Fix: ${check.remediation}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (result.recovery && result.recovery.status !== "not_needed") {
|
||||
console.log();
|
||||
console.log(`RECOVERY: ${result.recovery.status.replace(/_/g, " ").toUpperCase()}`);
|
||||
console.log(result.recovery.summary);
|
||||
for (const action of result.recovery.actions) {
|
||||
console.log(` Action: ${action.command}`);
|
||||
}
|
||||
for (const issue of result.recovery.blocked) {
|
||||
console.log(` Blocked: ${issue.reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.counts.red > 0) {
|
||||
console.log();
|
||||
console.log("Use --full for complete detail. Use --ready to include ready-seat info.");
|
||||
}
|
||||
}
|
||||
|
||||
function localRestoreResult(input: {
|
||||
verdict: RestoreCheckResult["verdict"];
|
||||
checks: CheckEntry[];
|
||||
|
||||
@@ -140,7 +140,7 @@ describe("rig restore-check", () => {
|
||||
it("human output includes verdict + check details + remediation", async () => {
|
||||
const { deps } = makeDeps({ result: { verdict: "restorable_with_caveats" } });
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig"]);
|
||||
await cmd.parseAsync(["node", "rig", "--full"]);
|
||||
|
||||
const output = logs.join("\n");
|
||||
expect(output).toContain("RESTORE CHECK");
|
||||
@@ -210,7 +210,7 @@ describe("rig restore-check", () => {
|
||||
},
|
||||
});
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig"]);
|
||||
await cmd.parseAsync(["node", "rig", "--full"]);
|
||||
|
||||
const output = logs.join("\n");
|
||||
expect(output).toContain("READINESS: ready (all_observable_checks_green_host_infra_declared_not_verified)");
|
||||
@@ -267,7 +267,7 @@ describe("rig restore-check", () => {
|
||||
},
|
||||
});
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig"]);
|
||||
await cmd.parseAsync(["node", "rig", "--full"]);
|
||||
|
||||
const output = logs.join("\n");
|
||||
expect(output).toContain("RESTORABLE WITH CAVEATS");
|
||||
@@ -326,7 +326,7 @@ describe("rig restore-check", () => {
|
||||
],
|
||||
}});
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig"]);
|
||||
await cmd.parseAsync(["node", "rig", "--full"]);
|
||||
|
||||
const output = logs.join("\n");
|
||||
expect(output).toContain("Repair steps: 2");
|
||||
@@ -440,7 +440,7 @@ describe("rig restore-check", () => {
|
||||
it("human output has no FULLY BACK line", async () => {
|
||||
const { deps } = makeDeps({});
|
||||
const cmd = restoreCheckCommand(deps);
|
||||
await cmd.parseAsync(["node", "rig"]);
|
||||
await cmd.parseAsync(["node", "rig", "--full"]);
|
||||
|
||||
const output = logs.join("\n");
|
||||
expect(output).not.toContain("FULLY BACK");
|
||||
@@ -477,3 +477,50 @@ describe("rig restore-check", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("OPR.0.4.0.29 — --ready is compact + ready detail (not a no-op, not the firehose)", () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
it("--ready keeps compact AND adds ready=1 (regression: FAILS if --ready is a no-op or drops compact)", async () => {
|
||||
const { deps, requestedPaths } = makeDeps({});
|
||||
await restoreCheckCommand(deps).parseAsync(["node", "rig", "--ready"]);
|
||||
expect(requestedPaths[0]).toContain("compact=1");
|
||||
expect(requestedPaths[0]).toContain("ready=1");
|
||||
});
|
||||
|
||||
it("default is compact WITHOUT ready", async () => {
|
||||
const { deps, requestedPaths } = makeDeps({});
|
||||
await restoreCheckCommand(deps).parseAsync(["node", "rig"]);
|
||||
expect(requestedPaths[0]).toContain("compact=1");
|
||||
expect(requestedPaths[0]).not.toContain("ready=1");
|
||||
});
|
||||
|
||||
it("--full drops compact (the firehose); --ready does NOT", async () => {
|
||||
const full = makeDeps({});
|
||||
await restoreCheckCommand(full.deps).parseAsync(["node", "rig", "--full"]);
|
||||
expect(full.requestedPaths[0]).not.toContain("compact=1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("OPR.0.4.0.29 FR-8 — class breakdown render", () => {
|
||||
let logs: string[];
|
||||
beforeEach(() => {
|
||||
logs = [];
|
||||
vi.spyOn(console, "log").mockImplementation((...a) => logs.push(a.join(" ")));
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
it("compact human output renders the 5-class CLASSES line when classCounts is present", async () => {
|
||||
const { deps } = makeDeps({ result: { classCounts: { ready: 3, ready_with_caveats: 1, not_ready: 2, attention_required: 0, unknown: 0 } } });
|
||||
await restoreCheckCommand(deps).parseAsync(["node", "rig"]);
|
||||
const output = logs.join("\n");
|
||||
expect(output).toContain("CLASSES:");
|
||||
expect(output).toContain("3 ready");
|
||||
expect(output).toContain("2 not_ready");
|
||||
});
|
||||
});
|
||||
@@ -46,6 +46,20 @@ export interface ContinuityAssertion {
|
||||
unprovenCapabilities: string[];
|
||||
}
|
||||
|
||||
// OPR.0.4.0.29 FR-8 — ready-confidence breakdown by the 5 REAL-enum seat
|
||||
// classes. Each derives from a real primitive (no invented status): ready /
|
||||
// ready_with_caveats / not_ready come from the seat readiness checks;
|
||||
// attention_required from node.startupStatus; unknown is the indeterminate
|
||||
// remainder. (The fresh-primed/awaiting-decision SPLIT of ready_with_caveats is
|
||||
// the deferred/escalated follow-on — NOT emitted here.)
|
||||
export interface ReadinessClassCounts {
|
||||
ready: number;
|
||||
ready_with_caveats: number;
|
||||
not_ready: number;
|
||||
attention_required: number;
|
||||
unknown: number;
|
||||
}
|
||||
|
||||
export interface RigRestoreRollup {
|
||||
rigId: string;
|
||||
rigName: string;
|
||||
@@ -55,6 +69,7 @@ export interface RigRestoreRollup {
|
||||
runningReadyNodes: number;
|
||||
blockedNodes: number;
|
||||
caveatNodes: number;
|
||||
classCounts: ReadinessClassCounts;
|
||||
blockingChecks: CheckEntry[];
|
||||
caveatChecks: CheckEntry[];
|
||||
}
|
||||
@@ -125,6 +140,8 @@ export interface RestoreCheckResult {
|
||||
hostInfra: HostInfraAssertion;
|
||||
recovery: RecoveryPlan;
|
||||
counts: { red: number; yellow: number; green: number };
|
||||
/** OPR.0.4.0.29 FR-8 — fleet-wide ready-confidence breakdown by class. */
|
||||
classCounts: ReadinessClassCounts;
|
||||
checks: CheckEntry[];
|
||||
repairPacket: RepairStep[] | null;
|
||||
}
|
||||
@@ -133,6 +150,10 @@ export interface RestoreCheckOpts {
|
||||
rig?: string;
|
||||
noQueue?: boolean;
|
||||
noHooks?: boolean;
|
||||
compact?: boolean;
|
||||
/** OPR.0.4.0.29 FR-2: in compact mode, still assemble ready-seat detail so
|
||||
* `--ready` shows ready seats without dropping to the full firehose. */
|
||||
includeReady?: boolean;
|
||||
}
|
||||
|
||||
// --- Deps (framework-free per ADR-0001; reads from existing projections per ADR-0002) ---
|
||||
@@ -250,6 +271,11 @@ export class RestoreCheckService {
|
||||
|
||||
check(opts: RestoreCheckOpts): RestoreCheckResult {
|
||||
const checks: CheckEntry[] = [];
|
||||
// rev1-r2 no-false-ready: ready-seat caveat signals that default compact
|
||||
// COMPUTES but does NOT emit (AC-4 token-safe) still count toward the
|
||||
// top-level verdict/readiness/counts, so the headline never reports a false
|
||||
// ready while a rig rollup is ready_with_caveats.
|
||||
const deferredAssessmentChecks: CheckEntry[] = [];
|
||||
const rigRollupInputs: RigRollupInput[] = [];
|
||||
const recoveryRigInputs: RecoveryRigInput[] = [];
|
||||
|
||||
@@ -319,6 +345,16 @@ export class RestoreCheckService {
|
||||
checks.push(readinessCheck);
|
||||
rigChecks.push(readinessCheck);
|
||||
|
||||
// OPR.0.4.0.29: in default compact, ready (green) seats SKIP full per-seat
|
||||
// detail assembly — the FR-3/AC-4 compute "look-above" win (transcript,
|
||||
// resume, queue, hooks are NOT probed). But the FR-8 summary still needs
|
||||
// the restore-readiness caveat signal: a running/ready seat whose startup
|
||||
// context is missing/unrestorable is ready_with_caveats, not ready. So we
|
||||
// compute ONLY checkStartupContext for every seat and feed it to the rig
|
||||
// rollup (rigChecks); it is emitted to the top-level `checks` only when we
|
||||
// are not skipping (full, --ready/includeReady, or a non-ready seat).
|
||||
const omitReadyDetail = opts.compact && !opts.includeReady && readinessCheck.status === "green";
|
||||
|
||||
const startupContextCheck = this.checkStartupContext(node);
|
||||
if ("unknownChecks" in startupContextCheck) {
|
||||
return this.buildUnknown([
|
||||
@@ -326,8 +362,20 @@ export class RestoreCheckService {
|
||||
...startupContextCheck.unknownChecks,
|
||||
]);
|
||||
}
|
||||
checks.push(startupContextCheck.check);
|
||||
rigChecks.push(startupContextCheck.check);
|
||||
if (omitReadyDetail) {
|
||||
// Not emitted (AC-4 token-safe), but counts toward the top-level
|
||||
// verdict/readiness/counts (rev1-r2 no-false-ready).
|
||||
deferredAssessmentChecks.push(startupContextCheck.check);
|
||||
} else {
|
||||
checks.push(startupContextCheck.check);
|
||||
}
|
||||
|
||||
// AC-4 / FR-3: default compact does NOT assemble the rest of the
|
||||
// ready-seat detail (the genuine compute skip, not a post-assembly hide).
|
||||
if (omitReadyDetail) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const transcriptCheck = this.checkTranscript(rig.name, node);
|
||||
checks.push(transcriptCheck);
|
||||
@@ -366,7 +414,7 @@ export class RestoreCheckService {
|
||||
});
|
||||
}
|
||||
|
||||
return this.buildResult(checks, rigRollups, hostInfraCheck.hostInfra, recoveryRigInputs);
|
||||
return this.buildResult(checks, rigRollups, hostInfraCheck.hostInfra, recoveryRigInputs, deferredAssessmentChecks);
|
||||
}
|
||||
|
||||
/** Returns CheckEntry on success/definite-down; null on probe exception
|
||||
@@ -1075,10 +1123,15 @@ export class RestoreCheckService {
|
||||
rigs: RigRestoreRollup[],
|
||||
hostInfra?: HostInfraAssertion,
|
||||
recoveryInputs: RecoveryRigInput[] = [],
|
||||
assessmentExtra: CheckEntry[] = [],
|
||||
): RestoreCheckResult {
|
||||
const red = checks.filter((c) => c.status === "red").length;
|
||||
const yellow = checks.filter((c) => c.status === "yellow").length;
|
||||
const green = checks.filter((c) => c.status === "green").length;
|
||||
// counts + verdict assess the FULL signal (emitted checks + any computed-but-
|
||||
// omitted ready-seat caveats) so default compact never reports a false ready;
|
||||
// repairPacket/recovery still operate on the EMITTED checks (token-safe).
|
||||
const assessed = assessmentExtra.length > 0 ? [...checks, ...assessmentExtra] : checks;
|
||||
const red = assessed.filter((c) => c.status === "red").length;
|
||||
const yellow = assessed.filter((c) => c.status === "yellow").length;
|
||||
const green = assessed.filter((c) => c.status === "green").length;
|
||||
|
||||
let verdict: Verdict;
|
||||
if (red > 0) {
|
||||
@@ -1168,6 +1221,14 @@ export class RestoreCheckService {
|
||||
},
|
||||
continuity,
|
||||
rigs,
|
||||
// FR-8: fleet-wide ready-confidence breakdown = sum of the per-rig class counts.
|
||||
classCounts: rigs.reduce<ReadinessClassCounts>((acc, r) => ({
|
||||
ready: acc.ready + r.classCounts.ready,
|
||||
ready_with_caveats: acc.ready_with_caveats + r.classCounts.ready_with_caveats,
|
||||
not_ready: acc.not_ready + r.classCounts.not_ready,
|
||||
attention_required: acc.attention_required + r.classCounts.attention_required,
|
||||
unknown: acc.unknown + r.classCounts.unknown,
|
||||
}), { ready: 0, ready_with_caveats: 0, not_ready: 0, attention_required: 0, unknown: 0 }),
|
||||
hostInfra: result.verdict === "unknown"
|
||||
? {
|
||||
status: "unknown",
|
||||
@@ -1355,17 +1416,44 @@ export class RestoreCheckService {
|
||||
let runningReadyNodes = 0;
|
||||
let blockedNodes = 0;
|
||||
let caveatNodes = 0;
|
||||
// FR-8: per-seat class breakdown (each seat counts toward exactly one class).
|
||||
const classCounts: ReadinessClassCounts = { ready: 0, ready_with_caveats: 0, not_ready: 0, attention_required: 0, unknown: 0 };
|
||||
// unknown/no-snapshot derives from the REAL snapshot primitive: the
|
||||
// rig-level `rig.<name>.snapshot` yellow check (no snapshot found / could
|
||||
// not check). A rig with no snapshot cannot be restored, so its seats'
|
||||
// restore-readiness is unknown (not "ready"), per FR-8/AC-7.
|
||||
const rigNoSnapshot = input.checks.some(
|
||||
(check) => check.check === `rig.${input.rig.name}.snapshot` && check.status === "yellow",
|
||||
);
|
||||
|
||||
for (const node of input.nodes) {
|
||||
const session = node.canonicalSessionName ?? node.logicalId;
|
||||
const nodeChecks = input.checks.filter((check) => check.check.startsWith(`seat.${session}.`));
|
||||
const hasBlocking = nodeChecks.some((check) => check.status === "red");
|
||||
const hasCaveat = nodeChecks.some((check) => check.status === "yellow");
|
||||
if (node.canonicalSessionName && node.sessionStatus === "running" && node.startupStatus === "ready") {
|
||||
runningReadyNodes += 1;
|
||||
}
|
||||
const runningReady = Boolean(node.canonicalSessionName) && node.sessionStatus === "running" && node.startupStatus === "ready";
|
||||
if (runningReady) runningReadyNodes += 1;
|
||||
if (hasBlocking) blockedNodes += 1;
|
||||
else if (hasCaveat) caveatNodes += 1;
|
||||
|
||||
// FR-8 class derivation (precedence; each from a real primitive).
|
||||
// attention/not_ready WIN FIRST (a failed/attention seat is surfaced even
|
||||
// in a no-snapshot rig); no-snapshot then overrides ONLY ready/caveat (a
|
||||
// clean seat in an unrestorable rig is unknown); a real yellow caveat then
|
||||
// wins over plain ready so the class count matches the per-rig status +
|
||||
// caveatNodes (a running/ready seat with a yellow check is NOT plain ready):
|
||||
// attention_required ← node.startupStatus
|
||||
// not_ready ← a red seat check (failed/down)
|
||||
// unknown ← no-snapshot (rig can't restore)
|
||||
// ready_with_caveats ← a yellow seat check
|
||||
// ready ← running/ready, no caveat
|
||||
// unknown ← indeterminate remainder
|
||||
if (node.startupStatus === "attention_required") classCounts.attention_required += 1;
|
||||
else if (hasBlocking) classCounts.not_ready += 1;
|
||||
else if (rigNoSnapshot) classCounts.unknown += 1;
|
||||
else if (hasCaveat) classCounts.ready_with_caveats += 1;
|
||||
else if (runningReady) classCounts.ready += 1;
|
||||
else classCounts.unknown += 1;
|
||||
}
|
||||
|
||||
let verdict: Verdict;
|
||||
@@ -1390,6 +1478,7 @@ export class RestoreCheckService {
|
||||
runningReadyNodes,
|
||||
blockedNodes,
|
||||
caveatNodes,
|
||||
classCounts,
|
||||
blockingChecks,
|
||||
caveatChecks,
|
||||
};
|
||||
|
||||
@@ -127,12 +127,15 @@ function getStartupContext(db: Database.Database, nodeId: string): StartupContex
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/restore-check?rig=<name>&noQueue=true&noHooks=true
|
||||
// GET /api/restore-check?rig=<name>&noQueue=true&noHooks=true&compact=1
|
||||
restoreCheckRoutes.get("/", (c) => {
|
||||
const deps = getDeps(c);
|
||||
const rigFilter = c.req.query("rig") ?? undefined;
|
||||
const noQueue = c.req.query("noQueue") === "true";
|
||||
const noHooks = c.req.query("noHooks") === "true";
|
||||
const compact = c.req.query("compact") === "1";
|
||||
// OPR.0.4.0.29 FR-2: --ready stays compact but INCLUDES ready-seat detail.
|
||||
const includeReady = c.req.query("ready") === "1";
|
||||
|
||||
try {
|
||||
const serviceDeps: RestoreCheckDeps = {
|
||||
@@ -168,7 +171,36 @@ restoreCheckRoutes.get("/", (c) => {
|
||||
};
|
||||
|
||||
const service = new RestoreCheckService(serviceDeps);
|
||||
const result = service.check({ rig: rigFilter, noQueue, noHooks });
|
||||
const result = service.check({ rig: rigFilter, noQueue, noHooks, compact, includeReady });
|
||||
|
||||
if (compact) {
|
||||
const compactResult = {
|
||||
verdict: result.verdict,
|
||||
readiness: result.readiness,
|
||||
counts: result.counts,
|
||||
classCounts: result.classCounts,
|
||||
rigs: result.rigs.map((r) => ({
|
||||
rigId: r.rigId,
|
||||
rigName: r.rigName,
|
||||
status: r.status,
|
||||
verdict: r.verdict,
|
||||
expectedNodes: r.expectedNodes,
|
||||
runningReadyNodes: r.runningReadyNodes,
|
||||
blockedNodes: r.blockedNodes,
|
||||
caveatNodes: r.caveatNodes,
|
||||
})),
|
||||
// --ready (ready=1) keeps the ready-seat checks in compact mode;
|
||||
// default compact drops the green (ready) checks for token safety.
|
||||
checks: includeReady ? result.checks : result.checks.filter((ch) => ch.status !== "green"),
|
||||
recovery: {
|
||||
status: result.recovery.status,
|
||||
summary: result.recovery.summary,
|
||||
actions: result.recovery.actions,
|
||||
blocked: result.recovery.blocked,
|
||||
},
|
||||
};
|
||||
return c.json(compactResult);
|
||||
}
|
||||
|
||||
return c.json(result);
|
||||
} catch (err) {
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { RestoreCheckService, type RestoreCheckDeps, type NodeInventoryEntry, type RestoreCheckResult } from "../src/domain/restore-check-service.js";
|
||||
|
||||
function makeReadyNode(logicalId: string): NodeInventoryEntry {
|
||||
return {
|
||||
logicalId,
|
||||
canonicalSessionName: `${logicalId.replace(".", "-")}@test-rig`,
|
||||
sessionStatus: "running",
|
||||
startupStatus: "ready",
|
||||
cwd: "/project",
|
||||
latestError: null,
|
||||
tmuxAttachCommand: `tmux attach -t ${logicalId.replace(".", "-")}@test-rig`,
|
||||
} as NodeInventoryEntry;
|
||||
}
|
||||
|
||||
function makeNotReadyNode(logicalId: string): NodeInventoryEntry {
|
||||
return {
|
||||
logicalId,
|
||||
canonicalSessionName: `${logicalId.replace(".", "-")}@test-rig`,
|
||||
sessionStatus: "exited",
|
||||
startupStatus: "failed",
|
||||
cwd: "/project",
|
||||
latestError: "Startup failed",
|
||||
} as NodeInventoryEntry;
|
||||
}
|
||||
|
||||
function makeDeps(nodes: NodeInventoryEntry[]): RestoreCheckDeps {
|
||||
return {
|
||||
listRigs: () => [{ rigId: "rig-1", name: "test-rig" }],
|
||||
getNodeInventory: () => nodes,
|
||||
getStartupContext: () => ({ status: "missing" as const, evidence: "no context" }),
|
||||
hasSnapshot: () => true,
|
||||
getLatestSnapshot: () => ({ id: "snap-1", kind: "full" }),
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "OK" }),
|
||||
exists: () => false,
|
||||
readFile: () => "",
|
||||
};
|
||||
}
|
||||
|
||||
describe("OPR.0.4.0.29 — restore-check compact via service", () => {
|
||||
it("AC-1: compact produces fewer checks than full", () => {
|
||||
const nodes = [
|
||||
makeReadyNode("dev.impl"),
|
||||
makeReadyNode("dev.qa"),
|
||||
makeReadyNode("dev.guard"),
|
||||
makeNotReadyNode("dev.design"),
|
||||
];
|
||||
|
||||
const service = new RestoreCheckService(makeDeps(nodes));
|
||||
const full = service.check({ compact: false });
|
||||
const compact = service.check({ compact: true });
|
||||
|
||||
expect(compact.checks.length).toBeLessThan(full.checks.length);
|
||||
expect(compact.rigs.length).toBe(full.rigs.length);
|
||||
});
|
||||
|
||||
it("AC-3: compact skips per-seat detail for ready seats (FR-3/AC-4)", () => {
|
||||
const nodes = [
|
||||
makeReadyNode("dev.impl"),
|
||||
makeReadyNode("dev.qa"),
|
||||
makeNotReadyNode("dev.design"),
|
||||
];
|
||||
|
||||
const service = new RestoreCheckService(makeDeps(nodes));
|
||||
const compact = service.check({ compact: true });
|
||||
const full = service.check({ compact: false });
|
||||
|
||||
const compactSeatChecks = compact.checks.filter((c) => c.check.startsWith("seat."));
|
||||
const fullSeatChecks = full.checks.filter((c) => c.check.startsWith("seat."));
|
||||
expect(compactSeatChecks.length).toBeLessThan(fullSeatChecks.length);
|
||||
|
||||
const notReadyCheck = compact.checks.find((c) => c.status === "red" && c.check.includes("readiness"));
|
||||
expect(notReadyCheck).toBeDefined();
|
||||
expect(notReadyCheck!.evidence).toContain("not running/ready");
|
||||
});
|
||||
|
||||
it("AC-7: readiness classes derive from real enums", () => {
|
||||
const nodes = [makeReadyNode("dev.impl"), makeNotReadyNode("dev.qa")];
|
||||
const service = new RestoreCheckService(makeDeps(nodes));
|
||||
const result = service.check({});
|
||||
|
||||
const validStatuses = new Set(["ready", "ready_with_caveats", "not_ready", "unknown"]);
|
||||
expect(validStatuses.has(result.readiness.status)).toBe(true);
|
||||
for (const rig of result.rigs) {
|
||||
expect(validStatuses.has(rig.status)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("AC-5: no compact option = full result (back-compat)", () => {
|
||||
const nodes = [makeReadyNode("dev.impl")];
|
||||
const service = new RestoreCheckService(makeDeps(nodes));
|
||||
const result = service.check({});
|
||||
|
||||
const seatChecks = result.checks.filter((c) => c.check.startsWith("seat.") || c.check.includes("startup_context") || c.check.includes("transcript") || c.check.includes("resume"));
|
||||
expect(seatChecks.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("AC-8: per-rig grouping shows rig rollup", () => {
|
||||
const nodes = [makeReadyNode("dev.impl"), makeNotReadyNode("dev.qa")];
|
||||
const service = new RestoreCheckService(makeDeps(nodes));
|
||||
const result = service.check({});
|
||||
|
||||
expect(result.rigs.length).toBe(1);
|
||||
expect(result.rigs[0]!.rigName).toBe("test-rig");
|
||||
expect(result.rigs[0]!.expectedNodes).toBe(2);
|
||||
});
|
||||
|
||||
// OPR.0.4.0.29 FR-8 / AC-7 — ready-confidence breakdown by the 5 real-enum classes.
|
||||
it("AC-7: classCounts breaks seats into the 5 real-enum classes (no invented status)", () => {
|
||||
const makeAttentionNode = (logicalId: string): NodeInventoryEntry => ({
|
||||
logicalId,
|
||||
canonicalSessionName: `${logicalId.replace(".", "-")}@test-rig`,
|
||||
sessionStatus: "running",
|
||||
startupStatus: "attention_required",
|
||||
cwd: "/project",
|
||||
latestError: "Awaiting operator",
|
||||
} as NodeInventoryEntry);
|
||||
|
||||
// The two ready seats are GENUINELY clean (nodeId + ok startup context +
|
||||
// present files via exists:true + healthy daemon) so they count as plain
|
||||
// `ready` — keeping real coverage of the ready class. (Caveat detection in
|
||||
// default compact is covered by the two dedicated tests below; a bare
|
||||
// makeReadyNode with a missing startup context is ready_with_caveats, not
|
||||
// ready, and default compact now detects that.)
|
||||
const nodes = [
|
||||
{ ...makeReadyNode("dev.impl"), nodeId: "node-impl" } as NodeInventoryEntry,
|
||||
{ ...makeReadyNode("dev.qa"), nodeId: "node-qa" } as NodeInventoryEntry,
|
||||
makeNotReadyNode("dev.design"),
|
||||
makeAttentionNode("dev.synth"),
|
||||
];
|
||||
const deps: RestoreCheckDeps = {
|
||||
...makeDeps(nodes),
|
||||
exists: () => true,
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running" }),
|
||||
getStartupContext: () => ({ status: "ok" as const, runtime: null, resolvedStartupFiles: [], projectionEntries: [] }),
|
||||
};
|
||||
const result = new RestoreCheckService(deps).check({ compact: true });
|
||||
|
||||
// Exactly the 5 real-enum class keys — no fresh-primed/awaiting-decision invented status.
|
||||
expect(Object.keys(result.classCounts).sort()).toEqual(
|
||||
["attention_required", "not_ready", "ready", "ready_with_caveats", "unknown"],
|
||||
);
|
||||
expect(result.classCounts.ready).toBe(2);
|
||||
expect(result.classCounts.attention_required).toBe(1);
|
||||
expect(result.classCounts.not_ready).toBe(1);
|
||||
// The breakdown accounts for every seat.
|
||||
const total = Object.values(result.classCounts).reduce((a, b) => a + b, 0);
|
||||
expect(total).toBe(4);
|
||||
});
|
||||
|
||||
it("AC-7: per-rig classCounts sum to the fleet-wide classCounts", () => {
|
||||
const result = new RestoreCheckService(
|
||||
makeDeps([makeReadyNode("dev.impl"), makeNotReadyNode("dev.qa")]),
|
||||
).check({});
|
||||
const rigSum = result.rigs.reduce((a, r) => a + Object.values(r.classCounts).reduce((x, y) => x + y, 0), 0);
|
||||
const fleetSum = Object.values(result.classCounts).reduce((a, b) => a + b, 0);
|
||||
expect(rigSum).toBe(fleetSum);
|
||||
});
|
||||
|
||||
it("AC-7: a no-snapshot rig's running/ready seats count as unknown, not ready (real snapshot primitive)", () => {
|
||||
const deps: RestoreCheckDeps = { ...makeDeps([makeReadyNode("dev.impl"), makeReadyNode("dev.qa")]), hasSnapshot: () => false };
|
||||
const result = new RestoreCheckService(deps).check({ compact: true });
|
||||
// Without a snapshot the rig cannot be restored -> its ready seats are unknown.
|
||||
expect(result.classCounts.ready).toBe(0);
|
||||
expect(result.classCounts.unknown).toBe(2);
|
||||
});
|
||||
|
||||
it("AC-7: no-snapshot does NOT hide failed seats (not_ready wins over no-snapshot)", () => {
|
||||
const deps: RestoreCheckDeps = { ...makeDeps([makeReadyNode("dev.impl"), makeNotReadyNode("dev.qa")]), hasSnapshot: () => false };
|
||||
const result = new RestoreCheckService(deps).check({ compact: true });
|
||||
// The failed seat stays not_ready (surfaced); only the clean seat -> unknown.
|
||||
expect(result.classCounts.not_ready).toBe(1);
|
||||
expect(result.classCounts.unknown).toBe(1);
|
||||
expect(result.classCounts.ready).toBe(0);
|
||||
});
|
||||
|
||||
// OPR.0.4.0.29 QA-blocking forward-fix (qa-blocking-1f7b1282): the
|
||||
// restore-proof-caveat repro. A snapshot-backed, running/ready seat whose
|
||||
// startup context is MISSING is a real yellow caveat. The per-rig status +
|
||||
// caveatNodes already report it; classCounts must follow the caveat too —
|
||||
// ready_with_caveats wins over plain ready (but still loses to
|
||||
// attention/not_ready/no-snapshot, which are asserted above).
|
||||
it("AC-7: a running/ready seat with a real yellow caveat counts as ready_with_caveats, not ready", () => {
|
||||
// makeReadyNode is running+ready; with a missing startup context its
|
||||
// seat.<session>.startup-context check is yellow (buildStartupContextAvailabilityCheck).
|
||||
// includeReady forces the ready-seat detail to assemble (the --ready path the QA proof used).
|
||||
// exists:true + a healthy daemon clears the unrelated rig/host red checks
|
||||
// (spec-present, daemon.reachable, transcript/queue files) so the ONLY
|
||||
// non-green signal is the real startup-context caveat — isolating the bug.
|
||||
const deps: RestoreCheckDeps = {
|
||||
...makeDeps([makeReadyNode("dev.impl")]),
|
||||
hasSnapshot: () => true,
|
||||
exists: () => true,
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running" }),
|
||||
getStartupContext: () => ({ status: "missing" as const, evidence: "Persisted startup context missing" }),
|
||||
};
|
||||
const result = new RestoreCheckService(deps).check({ compact: true, includeReady: true });
|
||||
|
||||
const rig = result.rigs.find((r) => r.rigName === "test-rig")!;
|
||||
expect(rig.status).toBe("ready_with_caveats");
|
||||
expect(rig.caveatNodes).toBe(1);
|
||||
expect(result.classCounts.ready_with_caveats).toBe(1);
|
||||
expect(result.classCounts.ready).toBe(0);
|
||||
});
|
||||
|
||||
// OPR.0.4.0.29 code-review BLOCKING forward-fix (qitem-...52809188): the DEFAULT
|
||||
// compact path the QA proof actually ran (rig restore-check, NOT --ready). Default
|
||||
// compact omits ready-seat DETAIL from the emitted output (token-safe), but the
|
||||
// summary must STILL detect the caveat -- a snapshot-backed running/ready seat with
|
||||
// a missing startup context is ready_with_caveats, not ready. The includeReady test
|
||||
// above is necessary but insufficient: it masks the default-compact detail skip.
|
||||
it("AC-7: DEFAULT compact (no includeReady) still counts a running/ready seat's yellow caveat as ready_with_caveats", () => {
|
||||
// nodeId set so the real getStartupContext-missing path produces the yellow
|
||||
// startup-context caveat. Clean deps so it is the ONLY non-green seat signal.
|
||||
const caveatNode = { ...makeReadyNode("dev.impl"), nodeId: "node-caveat" } as NodeInventoryEntry;
|
||||
const deps: RestoreCheckDeps = {
|
||||
...makeDeps([caveatNode]),
|
||||
hasSnapshot: () => true,
|
||||
exists: () => true,
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running" }),
|
||||
getStartupContext: () => ({ status: "missing" as const, evidence: "Persisted startup context missing" }),
|
||||
};
|
||||
const result = new RestoreCheckService(deps).check({ compact: true });
|
||||
|
||||
const rig = result.rigs.find((r) => r.rigName === "test-rig")!;
|
||||
expect(rig.status).toBe("ready_with_caveats");
|
||||
expect(rig.caveatNodes).toBe(1);
|
||||
expect(result.classCounts.ready_with_caveats).toBe(1);
|
||||
expect(result.classCounts.ready).toBe(0);
|
||||
// Token-safe: default compact does NOT dump the ready seat's detail rows.
|
||||
expect(result.checks.some((c) => c.check.endsWith(".startup-context"))).toBe(false);
|
||||
});
|
||||
|
||||
// OPR.0.4.0.29 code-review BLOCKING (qitem-...4f06e820): AC-4/FR-3 require the
|
||||
// daemon to SKIP full ready-seat detail assembly in default compact, not just
|
||||
// hide it after assembling. Default compact computes ONLY the startup-context
|
||||
// caveat signal the FR-8 summary needs; transcript/resume/queue/hooks are NOT
|
||||
// assembled for a green ready seat. Behavioral no-call proof: a green ready
|
||||
// seat with a CLEAN startup context but NO attach command. If default compact
|
||||
// assembled detail it would run checkResumePath -> a yellow resume-path caveat
|
||||
// -> ready_with_caveats. Skipping it keeps the seat plain ready.
|
||||
it("AC-4: default compact does NOT assemble full ready-seat detail (resume/transcript/queue/hooks) for a green ready seat", () => {
|
||||
const cleanReadyNoAttach = { ...makeReadyNode("dev.impl"), nodeId: "node-clean", tmuxAttachCommand: undefined } as NodeInventoryEntry;
|
||||
const deps: RestoreCheckDeps = {
|
||||
...makeDeps([cleanReadyNoAttach]),
|
||||
hasSnapshot: () => true,
|
||||
exists: () => true,
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running" }),
|
||||
// ok startup context -> startup-context is GREEN, so it is NOT the caveat.
|
||||
getStartupContext: () => ({ status: "ok" as const, runtime: null, resolvedStartupFiles: [], projectionEntries: [] }),
|
||||
};
|
||||
|
||||
// Default compact: resume-path (and the rest of the detail) is NOT assembled,
|
||||
// so the missing attach command produces no caveat -> plain ready.
|
||||
const compact = new RestoreCheckService(deps).check({ compact: true });
|
||||
expect(compact.classCounts.ready).toBe(1);
|
||||
expect(compact.classCounts.ready_with_caveats).toBe(0);
|
||||
expect(compact.rigs.find((r) => r.rigName === "test-rig")!.status).toBe("ready");
|
||||
|
||||
// --ready/includeReady DOES assemble detail -> the missing attach command is
|
||||
// a yellow resume-path caveat -> ready_with_caveats. Proves the detail set is
|
||||
// real and only the default-compact path skips it.
|
||||
const ready = new RestoreCheckService(deps).check({ compact: true, includeReady: true });
|
||||
expect(ready.classCounts.ready_with_caveats).toBe(1);
|
||||
expect(ready.classCounts.ready).toBe(0);
|
||||
});
|
||||
|
||||
// OPR.0.4.0.29 rev1-r2 BLOCKING (qitem-...f13fc5b4): no-false-ready. A skipped
|
||||
// (computed-but-not-emitted) ready-seat startup-context caveat must STILL drive
|
||||
// the top-level verdict/readiness/counts, or default compact reports a false
|
||||
// top-level ready/restorable while the same payload admits a ready_with_caveats
|
||||
// rig. Clean host (green daemon + green host-infra declaration) so the ONLY
|
||||
// non-green signal is the hidden startup-context caveat.
|
||||
it("AC-4 / no-false-ready: a hidden ready-seat caveat drives the top-level verdict/readiness in default compact", () => {
|
||||
const caveatNode = { ...makeReadyNode("dev.impl"), nodeId: "node-caveat" } as NodeInventoryEntry;
|
||||
const hostInfra = JSON.stringify({ schemaVersion: 1, daemonBootstrap: { mechanism: "launchd", declared: true }, supportingInfra: [] });
|
||||
const deps: RestoreCheckDeps = {
|
||||
...makeDeps([caveatNode]),
|
||||
hasSnapshot: () => true,
|
||||
exists: () => true,
|
||||
readFile: () => hostInfra,
|
||||
probeDaemonHealth: () => ({ healthy: true, evidence: "Daemon running" }),
|
||||
getStartupContext: () => ({ status: "missing" as const, evidence: "Persisted startup context missing" }),
|
||||
};
|
||||
const result = new RestoreCheckService(deps).check({ compact: true });
|
||||
|
||||
// Top-level must NOT be a false ready/restorable.
|
||||
expect(result.verdict).toBe("restorable_with_caveats");
|
||||
expect(result.readiness.status).toBe("ready_with_caveats");
|
||||
expect(result.readiness.caveatRigCount).toBe(1);
|
||||
expect(result.counts.yellow).toBeGreaterThanOrEqual(1);
|
||||
// Rollup + classCounts agree.
|
||||
expect(result.classCounts.ready_with_caveats).toBe(1);
|
||||
expect(result.classCounts.ready).toBe(0);
|
||||
// AC-4 preserved: the startup-context detail ROW is still not emitted.
|
||||
expect(result.checks.some((c) => c.check.endsWith(".startup-context"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -628,4 +628,20 @@ describe("Restore check routes", () => {
|
||||
expect(body.continuity.evidence).toBeTruthy();
|
||||
expect(body.continuity.unprovenCapabilities).toContain("provider_session_resume");
|
||||
});
|
||||
|
||||
// OPR.0.4.0.29 — compact vs --ready (ready=1) at the PRODUCTION route (finding 3:
|
||||
// exercise restoreCheckRoutes, not a duplicate test-only path).
|
||||
it("?compact=1 drops green checks (token win); ?compact=1&ready=1 keeps the ready detail", async () => {
|
||||
rigRepo.createRig("test-rig");
|
||||
|
||||
const compact = await (await app.request("/api/restore-check?compact=1")).json();
|
||||
// Green checks (e.g. daemon.reachable) are dropped in compact mode.
|
||||
expect(compact.checks.some((c: { check: string }) => c.check === "daemon.reachable")).toBe(false);
|
||||
|
||||
const compactReady = await (await app.request("/api/restore-check?compact=1&ready=1")).json();
|
||||
// --ready (ready=1) keeps the ready-seat (green) detail while staying compact,
|
||||
// NOT dropping to the full firehose (FR-2 corrective).
|
||||
expect(compactReady.checks.some((c: { check: string }) => c.check === "daemon.reachable")).toBe(true);
|
||||
expect(compactReady.checks.length).toBeGreaterThan(compact.checks.length);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user