Preserve shutdown receipt failures across status and repeated stops

This commit is contained in:
v-openrig-build committed 2026-09-07 04:13:44 +00:00
1 parent fd7e3d4ede
commit 7bc90ac4e3
6 files changed
+172 -64

No files matched your search

+13 -4
View File
@@ -40,7 +40,8 @@ control. See [the first-use journey](../reference/getting-started.md).
## Daemon shutdown
`rig daemon stop` sends one SIGTERM to the daemon identified by its local state.
`rig daemon stop` sends at most one SIGTERM to a live daemon identified by its
local state.
An explicit `OPENRIG_URL` that disagrees with that target refuses before signaling.
The daemon has one referenced **10-second** budget covering asynchronous service
shutdown, connections and recorder drain; repeated SIGINT/SIGTERM joins that stop.
@@ -51,9 +52,17 @@ listener and an unavailable probe are different outcomes.
`$OPENRIG_HOME/daemon-shutdown.json` records the PID, shutdown start/completion,
phase, failures and `clean|failed|timed-out` outcome. Only successful drains mark
the lifecycle record clean. A failed/timed-out drain returns nonzero, including
after the process exits; missing or stale receipt evidence is unverified and also
returns nonzero. Older daemons without the receipt can therefore be proven stopped
without their graceful drain being certified. Inspect the receipt and `daemon.log`;
after the process exits. For a recorded target, missing or stale receipt evidence
is unverified and also returns nonzero. Failed/unverified stops retain target
state for retries; status reads only clean it after a matching clean receipt.
An already-exited target uses the same receipt judgment without another signal.
With no recorded target and a refused listener, the CLI reports a distinct
no-target no-op, not a clean-drain verdict. Unbound incomplete or unreadable
local shutdown evidence remains nonzero/unverified rather than being attributed
to an unrelated listener.
Older daemons without the receipt can therefore be proven stopped without their
graceful drain being certified. Inspect the receipt and `daemon.log`;
do not infer that pending external work completed or was rolled back, or retry it
blindly. The bound covers asynchronous waits, not a synchronous event-loop wedge.
+4 -4
View File
@@ -215,11 +215,11 @@ export function daemonCommand(depsOverride?: LifecycleDeps): Command {
cmd
.command("stop")
.description("Stop the daemon (10s shutdown budget; 12s process wait; incomplete drain exits nonzero)")
.addHelpText("after", "\nSends one SIGTERM and verifies the original PID and listener. Repeated signals join shutdown.\nA missing/stale receipt is unverified; failed or timed-out drain exits nonzero even after the PID exits.\nInspect OPENRIG_HOME/daemon-shutdown.json and daemon.log for the phase and outcome.\nThe bound covers asynchronous shutdown; an event-loop wedge still requires operator recovery.\n")
.addHelpText("after", "\nSends at most one SIGTERM to a live target and verifies the original PID and listener. Repeated signals join shutdown.\nFor a recorded target, missing/stale receipts and incomplete drains exit nonzero, including retries.\nTarget state is retained until a matching clean receipt; status reads preserve unverified state.\nNo target is a distinct no-op, never clean-drain certification; unbound incomplete evidence stays unverified.\nInspect OPENRIG_HOME/daemon-shutdown.json and daemon.log for the phase and outcome.\nThe bound covers asynchronous shutdown; an event-loop wedge still requires operator recovery.\n")
.action(async () => {
try {
await stopDaemon(getDeps());
console.log("Daemon stopped");
const outcome = await stopDaemon(getDeps());
console.log(outcome === "stopped" ? "Daemon stopped" : "No daemon target recorded; listener refused. Nothing to stop; prior drain not certified.");
} catch (err) {
console.error(err instanceof Error ? err.message : String(err));
process.exitCode = 1;
@@ -260,7 +260,7 @@ export function daemonCommand(depsOverride?: LifecycleDeps): Command {
console.log("Daemon stopped");
break;
case "stale":
console.log("Daemon stale (cleaned up)");
console.log("Daemon PID is absent (stale state)");
break;
case "unverified":
// 1ae863d2 — C3 semantics: we could NOT confirm up or down; never claim stopped.
+1 -1
View File
@@ -70,7 +70,7 @@ function resolveRuntimeConfig(configStore: Pick<ConfigStore, "resolve">): Resolv
function realDestroyDeps(): DestroyDeps {
const lifecycleDeps = realDeps();
return {
stopDaemon: async () => stopDaemon(lifecycleDeps),
stopDaemon: async () => { await stopDaemon(lifecycleDeps); },
inspectListener: async (host: string, port: number): Promise<ListenerInspection> => {
try {
const res = await fetch(`http://${host}:${port}/healthz`);
+63 -39
View File
@@ -59,7 +59,7 @@ export interface DaemonStatus {
}
export interface GetDaemonStatusOptions {
/** Preserve stale daemon state for observational callers. Lifecycle callers clean it by default. */
/** Observers may disable cleanup; only a matching clean shutdown permits removal. */
cleanupStaleState?: boolean;
}
@@ -343,7 +343,7 @@ function readState(deps: LifecycleDeps): DaemonState | null {
return JSON.parse(raw) as DaemonState;
} catch {
// Malformed daemon.json — treat as no state
console.error("Warning: malformed daemon.json, treating as stopped");
console.error("Warning: malformed daemon.json; target identity unavailable");
return null;
}
}
@@ -721,7 +721,34 @@ export async function startDaemon(opts: StartOptions, deps: LifecycleDeps): Prom
return state;
}
export async function stopDaemon(deps: LifecycleDeps): Promise<void> {
function readShutdownReceipt(deps: LifecycleDeps, stateFile: string) {
const receiptPath = path.join(path.dirname(stateFile), DAEMON_SHUTDOWN_RECEIPT);
const raw = deps.readFile(receiptPath);
let receipt: DaemonShutdownReceipt | undefined;
try { receipt = JSON.parse(raw ?? "null") ?? undefined; } catch { /* unavailable */ }
if (receipt?.schema !== "openrig.daemon-shutdown/v1" || !Number.isSafeInteger(receipt.pid) || receipt.pid <= 0
|| !Number.isFinite(Date.parse(receipt.startedAt)) || !(Date.parse(receipt.completedAt) >= Date.parse(receipt.startedAt))
|| !(Date.parse(receipt.completedAt) <= Date.now()) || typeof receipt.phase !== "string"
|| !Array.isArray(receipt.failures) || !["clean", "failed", "timed-out"].includes(receipt.outcome)) receipt = undefined;
return { receipt, present: raw !== null || deps.exists(receiptPath), receiptPath };
}
function receiptMatchesState(receipt: DaemonShutdownReceipt | undefined, state: DaemonState, notBefore = Date.parse(state.startedAt)): boolean {
return receipt?.pid === state.pid && Date.parse(receipt.startedAt) >= notBefore;
}
function isCleanShutdown(receipt: DaemonShutdownReceipt | undefined): boolean {
return receipt?.outcome === "clean" && receipt.phase === "complete" && receipt.failures.length === 0;
}
function removeMatchingState(deps: LifecycleDeps, stateFile: string, state: DaemonState): void {
// A concurrent start/rebind must not have its state removed by an old stop/status.
const current = readState(deps);
if (current?.pid === state.pid && current.startedAt === state.startedAt && current.port === state.port
&& current.host === state.host && current.db === state.db) deps.removeFile(stateFile);
}
export async function stopDaemon(deps: LifecycleDeps): Promise<"stopped" | "no-target"> {
const state = readState(deps);
const configured = resolveConfiguredDaemonTarget();
const explicitUrl = readOpenRigEnv("OPENRIG_URL", "RIGGED_URL");
@@ -746,35 +773,33 @@ export async function stopDaemon(deps: LifecycleDeps): Promise<void> {
throw new Error(`Daemon state is missing — cannot stop safely. Checked ${check}; listener ${effect}.` +
(sibling ? ` Resolved home ${sibling.resolvedHome}; live sibling ${sibling.siblingHome}.` : ""));
}
return;
const stateFile = resolveLifecycleFile(deps, "daemon.json");
if (deps.exists(stateFile)) {
throw new Error(`Daemon target state is unreadable; drain completion unverified. Checked ${check}; listener refused; no signal sent. Inspect ${stateFile}.`);
}
const prior = readShutdownReceipt(deps, stateFile);
if (prior.present && !isCleanShutdown(prior.receipt)) {
throw new Error(`Drain completion unverified: local shutdown evidence exists but no target state is recorded; cannot attribute it to ${target}. Checked ${check}; listener refused. Inspect ${prior.receiptPath}.`);
}
// No target is not a clean-drain verdict, including after an earlier clean stop.
return "no-target";
}
const stateFile = resolveLifecycleFile(deps, "daemon.json");
const removeMatchingState = () => {
// A concurrent start/rebind must not have its state removed by this stop.
const current = readState(deps);
if (current?.pid === state.pid && current.startedAt === state.startedAt && current.port === state.port) {
deps.removeFile(stateFile);
}
};
const pidState = await checkPid(state, deps);
if (pidState === "dead") {
removeMatchingState();
const effect = await listener();
if (effect !== "refused") throw new Error(`Daemon PID ${state.pid} is absent; checked ${check}; listener ${effect}. Stop is unverified.`);
return;
}
if (pidState === "not_openrig") {
removeMatchingState();
throw new Error(`Cannot stop safely: PID ${state.pid} is present but identity is not confirmed at ${check}; no signal sent, stale state removed.`);
throw new Error(`Cannot stop safely: PID ${state.pid} is present but identity is not confirmed at ${check}; no signal sent, state preserved.`);
}
const requestedAt = Date.now();
deps.kill(state.pid, "SIGTERM");
// One signal. The daemon owns its bound; allow its entire budget plus exit slack.
const deadline = Date.now() + DAEMON_STOP_WAIT_MS;
while (deps.isProcessAlive(state.pid) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 100));
let notBefore = Date.parse(state.startedAt);
if (pidState !== "dead") {
notBefore = Date.now();
deps.kill(state.pid, "SIGTERM");
// One signal. An already-exited target goes straight to the same judgment.
const deadline = Date.now() + DAEMON_STOP_WAIT_MS;
while (deps.isProcessAlive(state.pid) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 100));
}
}
const present = deps.isProcessAlive(state.pid);
const effect = await listener();
@@ -783,20 +808,17 @@ export async function stopDaemon(deps: LifecycleDeps): Promise<void> {
throw new Error(`Daemon did not exit after SIGTERM within ${DAEMON_STOP_WAIT_MS}ms; ${observation}` +
(effect === "responding" ? " (still listening)" : "") + "; state file preserved.");
}
removeMatchingState();
if (effect !== "refused") throw new Error(`Daemon stop verification is unverified: ${observation}.`);
if (effect !== "refused") throw new Error(`Daemon stop verification is unverified: ${observation}; state preserved.`);
let receipt: DaemonShutdownReceipt | undefined;
try { receipt = JSON.parse(deps.readFile(path.join(path.dirname(stateFile), DAEMON_SHUTDOWN_RECEIPT)) ?? "null") ?? undefined; } catch { /* unavailable */ }
if (receipt?.schema !== "openrig.daemon-shutdown/v1" || receipt.pid !== state.pid
|| !(Date.parse(receipt.startedAt) >= requestedAt) || !(Date.parse(receipt.completedAt) >= Date.parse(receipt.startedAt))
|| !(Date.parse(receipt.completedAt) <= Date.now()) || typeof receipt.phase !== "string"
|| !Array.isArray(receipt.failures) || !["clean", "failed", "timed-out"].includes(receipt.outcome)) {
throw new Error(`Daemon process stopped; ${observation}; drain completion unverified (no matching shutdown receipt). Inspect ${path.join(path.dirname(stateFile), "daemon.log")}.`);
const { receipt, receiptPath } = readShutdownReceipt(deps, stateFile);
if (!receiptMatchesState(receipt, state, notBefore)) {
throw new Error(`Daemon process stopped; ${observation}; drain completion unverified (no matching shutdown receipt); state preserved. Inspect ${path.join(path.dirname(stateFile), "daemon.log")}.`);
}
if (receipt.outcome !== "clean" || receipt.failures.length || receipt.phase !== "complete") {
throw new Error(`Daemon stopped with incomplete shutdown: ${receipt.outcome}; phase=${receipt.phase}; ${observation}. Pending effects are unverified; inspect ${path.join(path.dirname(stateFile), DAEMON_SHUTDOWN_RECEIPT)}.`);
if (!isCleanShutdown(receipt)) {
throw new Error(`Daemon stopped with incomplete shutdown: ${receipt!.outcome}; phase=${receipt!.phase}; ${observation}. Pending effects are unverified; state preserved; inspect ${receiptPath}.`);
}
removeMatchingState(deps, stateFile, state);
return "stopped";
}
/** RULING 1ae863d2 — positive-down evidence classifier (lockstep with the daemon's
@@ -887,9 +909,11 @@ export async function getDaemonStatus(
}
if (!deps.isProcessAlive(state.pid)) {
// Process dead — stale state
if (options.cleanupStaleState !== false) {
deps.removeFile(resolveLifecycleFile(deps, "daemon.json"));
// A status read must not erase the identity needed to judge a failed stop.
const stateFile = resolveLifecycleFile(deps, "daemon.json");
const { receipt } = readShutdownReceipt(deps, stateFile);
if (options.cleanupStaleState !== false && receiptMatchesState(receipt, state) && isCleanShutdown(receipt)) {
removeMatchingState(deps, stateFile, state);
}
return { state: "stale" };
}
+8 -8
View File
@@ -228,7 +228,7 @@ describe("Daemon Lifecycle", () => {
});
// Should not throw
await expect(stopDaemon(deps)).resolves.toBeUndefined();
await expect(stopDaemon(deps)).resolves.toBe("no-target");
});
it("stop: recovered running daemon without daemon.json -> throws honest error", async () => {
@@ -365,8 +365,8 @@ describe("Daemon Lifecycle", () => {
}
});
// Test 9: status stale (daemon.json exists, process dead) -> reports stale, cleans up
it("status: stale (daemon.json exists, pid dead) -> { state: 'stale' }, cleans up", async () => {
// A status read must preserve target identity when its shutdown is unverified.
it("status: stale with no shutdown receipt -> preserves unverified target", async () => {
const state: DaemonState = { pid: 888, port: 7433, db: "x.db", startedAt: "2026-01-01T00:00:00Z" };
const deps = mockDeps({
exists: vi.fn((p: string) => p === STATE_FILE),
@@ -379,7 +379,7 @@ describe("Daemon Lifecycle", () => {
const status = await getDaemonStatus(deps);
expect(status.state).toBe("stale");
expect(deps.removeFile).toHaveBeenCalledWith(STATE_FILE);
expect(deps.removeFile).not.toHaveBeenCalled();
});
// Test 10: start --port flag stored in daemon.json and forwarded to env
@@ -775,8 +775,8 @@ describe("Daemon Lifecycle", () => {
vi.useRealTimers();
});
// Test 20: stop with stale PID (alive but not rig) -> cleans up state, no SIGTERM
it("stop: stale PID (alive but not rig) -> removes state, does not kill", async () => {
// A reused PID must receive no signal and must not erase unresolved stop identity.
it("stop: stale PID (alive but not rig) -> preserves state, does not kill", async () => {
const state: DaemonState = { pid: 999, port: 7433, db: "x.db", startedAt: "2026-01-01T00:00:00Z" };
const deps = mockDeps({
exists: vi.fn((p: string) => p === STATE_FILE),
@@ -790,7 +790,7 @@ describe("Daemon Lifecycle", () => {
await expect(stopDaemon(deps)).rejects.toThrow(/identity is not confirmed/);
expect(deps.kill).not.toHaveBeenCalled();
expect(deps.removeFile).toHaveBeenCalledWith(STATE_FILE);
expect(deps.removeFile).not.toHaveBeenCalled();
});
it("stop: hanging healthz probe still issues SIGTERM and reports the unavailable final probe", async () => {
@@ -813,7 +813,7 @@ describe("Daemon Lifecycle", () => {
await stopPromise;
expect(deps.kill).toHaveBeenCalledWith(999, "SIGTERM");
expect(deps.removeFile).toHaveBeenCalledWith(STATE_FILE);
expect(deps.removeFile).not.toHaveBeenCalled();
vi.useRealTimers();
});
+83 -8
View File
@@ -2,17 +2,17 @@ import path from "node:path";
import { Command } from "commander";
import { afterEach, expect, it, vi } from "vitest";
import { daemonCommand } from "../src/commands/daemon.js";
import { STATE_FILE, type LifecycleDeps } from "../src/daemon-lifecycle.js";
import { STATE_FILE, getDaemonStatus, type LifecycleDeps } from "../src/daemon-lifecycle.js";
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); process.exitCode = undefined; });
function fixture(opts: { survives?: boolean; unknownProbe?: boolean; receipt?: string } = {}) {
let killed = false;
let removed = false;
afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.unstubAllEnvs(); process.exitCode = undefined; });
function fixture(opts: { survives?: boolean; unknownProbe?: boolean; receipt?: string | null; exited?: boolean; noState?: boolean } = {}) {
let killed = opts.exited === true;
let removed = opts.noState === true;
const state = { pid: 987, host: "127.0.0.1", port: 17433, db: "private.db", startedAt: "2026-01-01T00:00:00Z" };
const deps = {
exists: (p: string) => p === STATE_FILE && !removed,
readFile: (p: string) => p === STATE_FILE ? (removed ? null : JSON.stringify(state))
: p === path.join(path.dirname(STATE_FILE), "daemon-shutdown.json") ? opts.receipt ?? JSON.stringify({
: p === path.join(path.dirname(STATE_FILE), "daemon-shutdown.json") ? opts.receipt !== undefined ? opts.receipt : JSON.stringify({
schema: "openrig.daemon-shutdown/v1", pid: 987, startedAt: new Date().toISOString(),
completedAt: new Date().toISOString(), outcome: "clean", phase: "complete", failures: [],
}) : null,
@@ -29,8 +29,9 @@ function fixture(opts: { survives?: boolean; unknownProbe?: boolean; receipt?: s
return deps;
}
async function stop(deps: LifecycleDeps) {
const out = vi.spyOn(console, "log").mockImplementation(() => {});
const err = vi.spyOn(console, "error").mockImplementation(() => {});
process.exitCode = undefined;
const out = vi.spyOn(console, "log").mockImplementation(() => {}).mockClear();
const err = vi.spyOn(console, "error").mockImplementation(() => {}).mockClear();
const p = new Command().addCommand(daemonCommand(deps)).parseAsync(["node", "rig", "daemon", "stop"]);
await vi.runAllTimersAsync(); await p;
return { out: out.mock.calls.flat().join(" "), err: err.mock.calls.flat().join(" "), code: process.exitCode };
@@ -64,3 +65,77 @@ it.each([null, { pid: 123 }, { pid: 987, startedAt: "2020-01-01T00:00:00Z" }])("
const result = await stop(fixture({ receipt: JSON.stringify(receipt) }));
expect(result.code).toBe(1); expect(result.err).toMatch(/unverified.*receipt/i);
});
it.each(["failed", "timed-out"])("retains an already-exited %s outcome and identity on repeated stop", async outcome => {
vi.useFakeTimers(); const now = new Date().toISOString();
const deps = fixture({ exited: true, receipt: JSON.stringify({ schema: "openrig.daemon-shutdown/v1", pid: 987,
startedAt: now, completedAt: now, outcome, phase: "health-diagnosis", failures: [{ phase: "health-diagnosis", error: "incomplete" }] }) });
for (let i = 0; i < 2; i++) {
const result = await stop(deps);
expect(result.code).toBe(1); expect(result.err).toContain(outcome);
expect(result.err).toContain("phase=health-diagnosis"); expect(deps.exists(STATE_FILE)).toBe(true);
}
expect(deps.kill).not.toHaveBeenCalled();
});
it("a missing receipt stays unverified across status and retry without another signal", async () => {
vi.useFakeTimers(); const deps = fixture({ receipt: null });
const first = await stop(deps); expect(first.code).toBe(1);
expect(deps.exists(STATE_FILE)).toBe(true);
expect((await getDaemonStatus(deps)).state).toBe("stale");
expect(deps.exists(STATE_FILE)).toBe(true);
const retry = await stop(deps); expect(retry.code).toBe(1); expect(retry.err).toMatch(/unverified/);
expect(deps.kill).toHaveBeenCalledTimes(1);
});
it("clean completion permits a truthful no-target repeat", async () => {
vi.useFakeTimers(); vi.stubEnv("OPENRIG_URL", "http://127.0.0.1:17433");
const deps = fixture(); const first = await stop(deps);
expect(first.code).not.toBe(1); expect(first.out).toContain("Daemon stopped");
expect(deps.exists(STATE_FILE)).toBe(false);
const again = await stop(deps); expect(again.code).not.toBe(1);
expect(again.out).toContain("No daemon target recorded"); expect(again.out).not.toContain("Daemon stopped");
expect(deps.kill).toHaveBeenCalledTimes(1);
});
it("no recorded target or receipt is a useful no-op, not clean-drain certification", async () => {
vi.useFakeTimers(); vi.stubEnv("OPENRIG_URL", "http://127.0.0.1:17433");
const deps = fixture({ noState: true, exited: true, receipt: null });
const result = await stop(deps);
expect(result.code).not.toBe(1); expect(result.out).toContain("No daemon target recorded");
expect(result.out).toContain("prior drain not certified"); expect(deps.kill).not.toHaveBeenCalled();
});
it.each(["malformed", JSON.stringify({ outcome: "failed", pid: 123 })])("no-state does not silently discard unbound incomplete evidence: %s", async receipt => {
vi.useFakeTimers(); vi.stubEnv("OPENRIG_URL", "http://127.0.0.1:17433");
const deps = fixture({ noState: true, exited: true, receipt }); const result = await stop(deps);
expect(result.code).toBe(1); expect(result.err).toContain("unverified");
expect(result.err).toContain("cannot attribute"); expect(deps.kill).not.toHaveBeenCalled();
});
it.each([{pid: 123}, {startedAt: "2020-01-01T00:00:00Z"}])("an exited target cannot borrow unrelated clean evidence: %s", async override => {
vi.useFakeTimers(); const now = new Date().toISOString();
const deps = fixture({ exited: true, receipt: JSON.stringify({ schema: "openrig.daemon-shutdown/v1", pid: 987,
startedAt: now, completedAt: now, outcome: "clean", phase: "complete", failures: [], ...override }) });
const result = await stop(deps); expect(result.code).toBe(1); expect(result.err).toContain("unverified");
expect(deps.exists(STATE_FILE)).toBe(true); expect(deps.kill).not.toHaveBeenCalled();
});
it("status may clean an exited target only with its matching clean receipt", async () => {
vi.useFakeTimers(); const deps = fixture({ exited: true });
expect((await getDaemonStatus(deps)).state).toBe("stale"); expect(deps.exists(STATE_FILE)).toBe(false);
});
it.each(["stop", "status"])("%s cannot remove a concurrently rebound state file", async operation => {
vi.useFakeTimers(); const deps = fixture({ exited: true }); const read = deps.readFile; let stateReads = 0;
deps.readFile = p => {
const raw = read(p);
return p === STATE_FILE && raw && stateReads++ > 0 ? JSON.stringify({ ...JSON.parse(raw), db: "replacement.db" }) : raw;
};
if (operation === "stop") expect((await stop(deps)).code).not.toBe(1);
else expect((await getDaemonStatus(deps)).state).toBe("stale");
expect(deps.exists(STATE_FILE)).toBe(true);
});
it("an unreadable target file is not a never-recorded no-target control", async () => {
vi.useFakeTimers(); vi.stubEnv("OPENRIG_URL", "http://127.0.0.1:17433");
const deps = fixture({ exited: true, receipt: null }); const read = deps.readFile;
deps.readFile = p => p === STATE_FILE ? "{malformed" : read(p);
const result = await stop(deps);
expect(result.code).toBe(1); expect(result.err).toContain("unverified");
expect(result.out).not.toContain("No daemon target recorded"); expect(deps.kill).not.toHaveBeenCalled();
});