mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-11 17:32:22 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a594650892 |
No files matched your search
@@ -702,7 +702,7 @@ rig seat handover, THEN retarget the view. Examples:
|
||||
cmd
|
||||
.command("clear-attention")
|
||||
.argument("<session>", "Canonical session name (e.g. dev-impl@my-rig)")
|
||||
.option("--reason <text>", "Attest startup or subset-restore attention, or that a prompt the latest hook names is gone; cannot bypass full-restore or pane-identity checks")
|
||||
.option("--reason <text>", "Attest startup or subset-restore attention, acknowledge a full restore's attention its checks can't settle (recovery stays unproven), or say a prompt the latest hook names is gone; cannot bypass pane-identity checks")
|
||||
.option("--json", "JSON output for agents")
|
||||
.description("Reconcile seat attention using the checks for its attention class")
|
||||
.addHelpText("after", `
|
||||
@@ -715,9 +715,13 @@ Examples:
|
||||
also say a prompt is gone: when the seat's latest hook still says it waits on a
|
||||
person (a permission prompt declined or answered with no hook after) and sends
|
||||
are refused, --reason "<what you saw>" records the prompt cleared. Each send
|
||||
still checks the pane for a prompt it recognizes. It does
|
||||
not bypass full-restore continuity or pane-identity checks. Acknowledgment alone
|
||||
does not prove that the original conversation resumed.
|
||||
still checks the pane for a prompt it recognizes. A full restore's attention is
|
||||
first checked against the live seat; when that check can't settle it, --reason
|
||||
acknowledges it and prints what stays unproven. The outcome stays as it was:
|
||||
acknowledgment never records the conversation as resumed. It covers only the
|
||||
attempt and occupant you addressed: a newer occupant, a changed binding or
|
||||
token, or a restore that includes the seat while it was checked is not
|
||||
acknowledged. It does not bypass pane-identity checks.
|
||||
If the recorded native token needs correction and you know the actual token:
|
||||
printf '%s' "$TOKEN" | rig seat set-resume-token dev-impl@my-rig --token-stdin --reason "verified native token"
|
||||
Then rerun clear-attention to check the live evidence. Setting the token alone,
|
||||
@@ -753,6 +757,13 @@ identity remains unverified; a version number alone cannot clear it.
|
||||
}
|
||||
const clearedBy = res.data["clearedBy"] as string | undefined;
|
||||
const from = res.data["from"] as string | undefined;
|
||||
const ack = res.data["acknowledgment"] as { attemptId?: number; from?: string; unproven?: { code?: string; detail?: string };
|
||||
paneIdentity?: string } | undefined;
|
||||
if (ack) {
|
||||
console.log(`Acknowledged ${session}'s full-restore attention (attempt ${ack.attemptId}, ${ack.from}): recovery not proven (${ack.unproven?.code}: ${ack.unproven?.detail})`
|
||||
+ (ack.paneIdentity === "unverified" ? ". Its pane identity is still unverified; only live evidence clears that." : ""));
|
||||
return;
|
||||
}
|
||||
console.log(`Cleared ${session}: ${from} -> ready (${clearedBy})`);
|
||||
});
|
||||
|
||||
|
||||
@@ -106,5 +106,21 @@ it.each(["valid", "unknown path", "wrong token", "no token", "PID reused", "logi
|
||||
}
|
||||
expect(db.prepare("SELECT * FROM events WHERE type IN ('restore.started', 'restore.completed')").all()).toEqual(historical);
|
||||
expect(sendText).not.toHaveBeenCalled(); expect(sendKeys).not.toHaveBeenCalled();
|
||||
if (mode !== "valid") {
|
||||
// #2473: --reason acknowledges what the live check couldn't settle, and nothing records a resume. An identity it
|
||||
// couldn't prove keeps its own class.
|
||||
const again = new Command().exitOverride().addCommand(seatCommand({ lifecycleDeps, clientFactory }));
|
||||
await again.parseAsync(["node", "rig", "seat", "clear-attention", name, "--reason", "checked the pane myself"]);
|
||||
const line = String(output.mock.calls.at(-1)?.[0]);
|
||||
expect(line.startsWith(`Acknowledged ${name}'s full-restore attention (attempt ${attempt.seq}, failed): recovery not proven (`)).toBe(true);
|
||||
const unverified = ["unknown path", "wrong token", "PID reused"].includes(mode);
|
||||
expect(line.endsWith("only live evidence clears that.")).toBe(unverified);
|
||||
expect(getNodeInventory(db, rig.id)[0]).toMatchObject({ restoreOutcome: "failed", continuityOutcome: "failed",
|
||||
restoreAcknowledgment: { attemptId: attempt.seq, reason: "checked the pane myself" } });
|
||||
if (!unverified) expect(getNodeInventory(db, rig.id)[0]?.lifecycleState).toBe("running");
|
||||
expect(db.prepare("SELECT * FROM events WHERE type = 'restore.outcome_reconciled'").all()).toHaveLength(0);
|
||||
expect(db.prepare("SELECT count(*) AS n FROM events WHERE type = 'restore.outcome_acknowledged'").get()).toEqual({ n: 1 });
|
||||
expect(sendText).not.toHaveBeenCalled(); expect(sendKeys).not.toHaveBeenCalled();
|
||||
}
|
||||
} finally { db.close(); }
|
||||
});
|
||||
@@ -465,8 +465,14 @@ rig seat clear-attention <session> --json
|
||||
```
|
||||
|
||||
`--reason <text>` can acknowledge startup-status and subset-restore attention.
|
||||
It does not bypass full-restore continuity checks or an active pane-identity
|
||||
mismatch/missing-pane check; those run first. Successful identity re-verification
|
||||
Full-restore continuity checks and an active pane-identity mismatch/missing-pane
|
||||
check run first. When the full-restore check can't settle the outcome (an
|
||||
unreadable process list, an older runner, a timeout), `--reason` acknowledges
|
||||
that attempt's attention for the current occupant and prints what stays
|
||||
unproven; the outcome is unchanged and nothing records the conversation as
|
||||
resumed. It does not bypass a pane-identity mismatch, and a newer occupant,
|
||||
changed binding or token, or a restore that includes the seat while it was
|
||||
checked is not acknowledged. Successful identity re-verification
|
||||
can also clear coexisting startup or subset-restore attention. If neither earlier
|
||||
path clears attention, omitting `--reason` requires activity or send evidence.
|
||||
Startup and restore clears write their corresponding audit events; an identity-only
|
||||
|
||||
@@ -222,6 +222,8 @@ export function deriveNodeLifecycleState(input: {
|
||||
/** Startup attention is live seat state, independent of restore history. */
|
||||
startupStatus?: string | null;
|
||||
restoreOutcome: NodeRestoreOutcome;
|
||||
/** A person acknowledged this occupant's full-restore outcome: it no longer asks for attention (still unrecovered). */
|
||||
restoreAcknowledged?: boolean;
|
||||
nodeId: string;
|
||||
usableSnapshot: Snapshot | null;
|
||||
/** OPR.0.4.3.19 — the persisted liveness identity verdict for this node.
|
||||
@@ -235,8 +237,9 @@ export function deriveNodeLifecycleState(input: {
|
||||
// lifecycleState=attention_required.
|
||||
if (
|
||||
input.startupStatus === "attention_required"
|
||||
|| input.restoreOutcome === "attention_required"
|
||||
|| (input.restoreOutcome === "failed" && input.sessionStatus === "running")
|
||||
|| (!input.restoreAcknowledged && (
|
||||
input.restoreOutcome === "attention_required"
|
||||
|| (input.restoreOutcome === "failed" && input.sessionStatus === "running")))
|
||||
) {
|
||||
return "attention_required";
|
||||
}
|
||||
@@ -318,6 +321,8 @@ function deriveContinuityOutcome(
|
||||
interface RestoreProjection {
|
||||
outcome: NodeRestoreOutcome;
|
||||
resumedRuntime?: "claude-code" | "codex";
|
||||
/** A person's acknowledgement of this full-restore attempt's outcome, for the occupant session it names. */
|
||||
acknowledgment?: { sessionId: string; attemptId: number; reason: string; unproven: { code: string; detail: string } };
|
||||
}
|
||||
|
||||
function restoreKey(rigId: string, nodeId: string): string {
|
||||
@@ -350,8 +355,8 @@ function buildRestoreOutcomeMap(db: Database.Database, rigId?: string): Map<stri
|
||||
type: string; payload: string; seq: number; rig_id: string; node_id: string | null;
|
||||
}[];
|
||||
const map = new Map<string, RestoreProjection>();
|
||||
// Each entry's attempt, by its start seq; a subset restore records no start.
|
||||
const attemptOf = new Map<string, { nodeId: string; seq: number }>();
|
||||
// Each entry's attempt, by its start seq; a subset restore records no start. `full` marks a full restore's outcome.
|
||||
const attemptOf = new Map<string, { nodeId: string; seq: number; full?: true }>();
|
||||
const attempts = new Map<string, {
|
||||
seq: number;
|
||||
start?: Record<string, any>;
|
||||
@@ -412,7 +417,7 @@ function buildRestoreOutcomeMap(db: Database.Database, rigId?: string): Map<stri
|
||||
if (typeof node?.nodeId === "string") {
|
||||
const key = restoreKey(row.rig_id, node.nodeId);
|
||||
map.set(key, { outcome: mapStatus(node.status) });
|
||||
attemptOf.set(key, { nodeId: node.nodeId, seq: attemptSeq });
|
||||
attemptOf.set(key, { nodeId: node.nodeId, seq: attemptSeq, ...(row.type === "restore.completed" ? { full: true as const } : {}) });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -424,6 +429,25 @@ function buildRestoreOutcomeMap(db: Database.Database, rigId?: string): Map<stri
|
||||
const fence = fences.get(entry.nodeId);
|
||||
if (fence !== undefined && entry.seq < fence) map.delete(key);
|
||||
}
|
||||
// A person's acknowledgement of a full restore's attention (clear-attention --reason, when the strict reconciler
|
||||
// couldn't settle it) applies to that attempt's outcome only. Only those outcomes look one up: the node's latest,
|
||||
// through the node/type index.
|
||||
const latestAck = db.prepare(
|
||||
"SELECT payload FROM events WHERE node_id = ? AND type = 'restore.outcome_acknowledged' ORDER BY seq DESC LIMIT 1",
|
||||
);
|
||||
for (const [key, entry] of attemptOf) {
|
||||
const projection = map.get(key);
|
||||
if (!projection || !entry.full || (projection.outcome !== "failed" && projection.outcome !== "attention_required")) continue;
|
||||
const row = latestAck.get(entry.nodeId) as { payload: string } | undefined;
|
||||
let ack: Record<string, any> | undefined;
|
||||
try { ack = row ? eventObject(JSON.parse(row.payload)) : undefined; } catch { /* Unknown evidence: no acknowledgement. */ }
|
||||
if (ack && JSON.stringify([ack.rigId, ack.nodeId]) === key && ack.attemptId === entry.seq && ack.from === projection.outcome
|
||||
&& typeof ack.sessionId === "string" && typeof ack.reason === "string"
|
||||
&& typeof ack.unproven?.code === "string" && typeof ack.unproven?.detail === "string") {
|
||||
projection.acknowledgment = { sessionId: ack.sessionId, attemptId: entry.seq, reason: ack.reason,
|
||||
unproven: { code: ack.unproven.code, detail: ack.unproven.detail } };
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
@@ -754,6 +778,9 @@ function buildInventoryEntry(
|
||||
// is referenced by no restore event, matching the prior fall-through).
|
||||
const restore = restoreOutcomes.get(restoreKey(row.rig_id, row.node_id));
|
||||
const restoreOutcome = restore?.outcome ?? "n-a";
|
||||
// An acknowledgement covers the occupant session it named; a newer occupant's attention stands.
|
||||
const restoreAcknowledgment = restore?.acknowledgment && restore.acknowledgment.sessionId === row.session_id
|
||||
? restore.acknowledgment : undefined;
|
||||
// OPR.0.4.3.19 rev1-r2 B1 — a durable verdict is keyed only by node_id, so
|
||||
// after a rebind/relaunch (same node, NEW session + NEW pane) a stale
|
||||
// `verified` verdict for the OLD pane would otherwise be served for the new
|
||||
@@ -777,6 +804,7 @@ function buildInventoryEntry(
|
||||
sessionStatus,
|
||||
startupStatus,
|
||||
restoreOutcome,
|
||||
restoreAcknowledged: restoreAcknowledgment !== undefined,
|
||||
nodeId: row.node_id,
|
||||
usableSnapshot,
|
||||
identityVerdict: identityVerdict?.verdict ?? null,
|
||||
@@ -808,6 +836,8 @@ function buildInventoryEntry(
|
||||
storedStartupStatus: row.startup_status as NodeInventoryEntry["startupStatus"],
|
||||
startupStatus,
|
||||
restoreOutcome,
|
||||
...(restoreAcknowledgment ? { restoreAcknowledgment: { attemptId: restoreAcknowledgment.attemptId,
|
||||
reason: restoreAcknowledgment.reason, unproven: restoreAcknowledgment.unproven } } : {}),
|
||||
// FS-1 W1.3 S2 — O(1) lookup into the fleet-batched oriented map
|
||||
// (byte-identical to the prior per-node deriveOriented; "n-a" when a node has
|
||||
// no proof events, matching deriveOriented's no-challenge branch).
|
||||
|
||||
@@ -334,6 +334,10 @@ export interface ClearAttentionResult {
|
||||
detail?: string;
|
||||
previousError?: string | null;
|
||||
clearedClasses?: ("startup_status" | "restore_outcome" | "pane_identity" | "needs_input_hook")[];
|
||||
/** A full restore's attention acknowledged on the person's word: recovery stays unproven, for the reason named. */
|
||||
acknowledgment?: { attemptId: number; from: "failed" | "attention_required"; unproven: AcknowledgedUnproven;
|
||||
/** The pane identity couldn't be proven either: that class stays, and only live evidence clears it. */
|
||||
paneIdentity?: "unverified" };
|
||||
derivedEvidence?: {
|
||||
source: string;
|
||||
kind?: string;
|
||||
@@ -348,6 +352,10 @@ export interface ClearAttentionResult {
|
||||
};
|
||||
}
|
||||
|
||||
/** What the strict reconciler couldn't show when a person acknowledged a full restore's attention: its code and detail,
|
||||
* and the native screen it read when it read one. */
|
||||
type AcknowledgedUnproven = { code: string; detail: string; probe?: import("./native-resume-probe-evidence.js").NativeResumeProbeEvidence };
|
||||
|
||||
export interface SendVerifyFn {
|
||||
(sessionName: string, text: string, opts?: { verify?: boolean }): Promise<{ ok: boolean; outcome?: string; verified?: boolean }>;
|
||||
}
|
||||
@@ -394,13 +402,25 @@ interface ClearAttentionDeps {
|
||||
>;
|
||||
}
|
||||
|
||||
const IDENTITY_RECOVERY_GUIDANCE = "--reason cannot bypass this attention class. If the recorded native token needs correction and you know the actual token, use rig seat set-resume-token <session> --token-stdin --reason <explanation>, then rerun rig seat clear-attention <session> to check the live evidence. Setting a token or stopping/relaunching the seat alone does not prove continuity.";
|
||||
const TOKEN_CORRECTION_GUIDANCE = "If the recorded native token needs correction and you know the actual token, use rig seat set-resume-token <session> --token-stdin --reason <explanation>, then rerun rig seat clear-attention <session> to check the live evidence. Setting a token or stopping/relaunching the seat alone does not prove continuity.";
|
||||
const IDENTITY_RECOVERY_GUIDANCE = `--reason cannot bypass this attention class. ${TOKEN_CORRECTION_GUIDANCE}`;
|
||||
const FULL_RESTORE_GUIDANCE = `If you've checked the seat yourself, --reason "<what you saw>" acknowledges this attention without claiming the conversation resumed, and reports what stays unproven. ${TOKEN_CORRECTION_GUIDANCE}`;
|
||||
|
||||
/** A pane-identity verdict that positively shows another process or no pane, as opposed to one that couldn't prove which. */
|
||||
function positiveIdentityFailure(verdict: SeatIdentityVerdict | null): boolean {
|
||||
return verdict !== null && !(verdict.verdict === "mismatch" && verdict.reason === "process_identity_ambiguous");
|
||||
}
|
||||
|
||||
const POSITIVE_STATES = new Set(["running", "idle"]);
|
||||
type DerivedAttentionOutcome = {
|
||||
status: "failed" | "attention_required";
|
||||
source: "restore.completed" | "restore.subset_completed";
|
||||
/** The full restore's attempt (its restore.started seq), or the subset completion's own seq. */
|
||||
attemptSeq: number;
|
||||
assistedStartupAttemptId?: number;
|
||||
/** A person acknowledged this full-restore attempt's attention for this session: it no longer asks for attention,
|
||||
* but its recovery is still unproven, so a later strict proof can still settle it. */
|
||||
acknowledged?: true;
|
||||
};
|
||||
|
||||
export class SeatAttentionReconciler {
|
||||
@@ -459,7 +479,9 @@ export class SeatAttentionReconciler {
|
||||
}
|
||||
|
||||
const startupClassActive = session.startupStatus === "attention_required" || session.startupStatus === "failed";
|
||||
const derivedOutcome = this.getDerivedAttentionOutcome(session);
|
||||
const derived = this.getDerivedAttentionOutcome(session);
|
||||
// An acknowledged full-restore outcome no longer asks for attention; its recovery stays unproven.
|
||||
const derivedOutcome = derived?.acknowledged ? null : derived;
|
||||
const identityClassActive = this.getActiveIdentityVerdict(session, sessionName);
|
||||
|
||||
// The latest runtime hook still says the seat waits on a person (an approval, a picker, an elicitation) and nothing
|
||||
@@ -494,8 +516,20 @@ export class SeatAttentionReconciler {
|
||||
// and appends the attempt-scoped reconciliation event. Subset restores do
|
||||
// not have a restore.started receipt and retain their established path.
|
||||
if (derivedOutcome?.source === "restore.completed" && assistedRestore === undefined) {
|
||||
// With --reason, an outcome the reconciler can't settle is acknowledged for the subject the person addressed.
|
||||
// The check below awaits native reads, so record that subject first and compare it after.
|
||||
const subject = opts?.reason ? this.restoreAckSubject(sessionName) : null;
|
||||
const acknowledge = (unproven: AcknowledgedUnproven) => opts?.reason
|
||||
? this.acknowledgeFullRestore({ session, sessionName, outcome: derivedOutcome, reason: opts.reason, subject, unproven,
|
||||
startupClassActive, hook, hookClassActive, previousError })
|
||||
: null;
|
||||
const unacknowledged = (why: string | null) => why
|
||||
? `Not acknowledged: ${why}.`
|
||||
: positiveIdentityFailure(identityClassActive) ? IDENTITY_RECOVERY_GUIDANCE : FULL_RESTORE_GUIDANCE;
|
||||
const reconcile = this.deps.reconcileRestoreOutcome;
|
||||
if (!reconcile) {
|
||||
const acknowledged = acknowledge({ code: "strict_reconciler_unavailable", detail: "the strict restore reconciler is unavailable" });
|
||||
if (acknowledged && typeof acknowledged !== "string") return acknowledged;
|
||||
eventBus.emit({
|
||||
type: "seat.attention_clear_refused",
|
||||
rigId: session.rigId,
|
||||
@@ -508,11 +542,13 @@ export class SeatAttentionReconciler {
|
||||
return {
|
||||
ok: false,
|
||||
code: "not_demonstrably_responsive",
|
||||
detail: `Uncleared attention class restore_outcome (full restore): strict restore reconciler is unavailable. ${IDENTITY_RECOVERY_GUIDANCE}`,
|
||||
detail: `Uncleared attention class restore_outcome (full restore): strict restore reconciler is unavailable. ${unacknowledged(acknowledged)}`,
|
||||
};
|
||||
}
|
||||
const restored = await reconcile(session.rigId, session.nodeId);
|
||||
if (!restored.ok) {
|
||||
const acknowledged = acknowledge({ code: restored.code, detail: restored.detail, ...(restored.probe ? { probe: restored.probe } : {}) });
|
||||
if (acknowledged && typeof acknowledged !== "string") return acknowledged;
|
||||
eventBus.emit({
|
||||
type: "seat.attention_clear_refused",
|
||||
rigId: session.rigId,
|
||||
@@ -527,7 +563,7 @@ export class SeatAttentionReconciler {
|
||||
return {
|
||||
ok: false,
|
||||
code: "not_demonstrably_responsive",
|
||||
detail: `Uncleared attention class restore_outcome (full restore): ${restored.code}: ${restored.detail}. ${IDENTITY_RECOVERY_GUIDANCE}`,
|
||||
detail: `Uncleared attention class restore_outcome (full restore): ${restored.code}: ${restored.detail}. ${unacknowledged(acknowledged)}`,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -722,6 +758,83 @@ export class SeatAttentionReconciler {
|
||||
};
|
||||
}
|
||||
|
||||
/** Root's fail-open acknowledgement (#2473): the person's --reason acknowledges a full restore's attention that the
|
||||
* strict reconciler couldn't settle, and never records a recovery. It covers only the subject the person addressed,
|
||||
* so a positive mismatch (the pane identity check, a binding that disagrees with the latest session) or a subject
|
||||
* that changed while the reconciler checked is not acknowledged. Returns the result, or why it wasn't acknowledged. */
|
||||
private acknowledgeFullRestore(input: {
|
||||
session: { id: string; nodeId: string; rigId: string; runtime: string | null; startupStatus: string; latestError: string | null };
|
||||
sessionName: string;
|
||||
outcome: DerivedAttentionOutcome;
|
||||
reason: string;
|
||||
subject: string | null;
|
||||
unproven: AcknowledgedUnproven;
|
||||
startupClassActive: boolean;
|
||||
hook: AgentActivity | null;
|
||||
hookClassActive: boolean;
|
||||
previousError: string | null;
|
||||
}): ClearAttentionResult | string {
|
||||
const { session, sessionName, outcome, reason } = input;
|
||||
// Read after the check, which records the identity evidence it reads. An identity it couldn't prove is inability to
|
||||
// inspect, not a replacement: that class stays as it is, and the restore's attention is still acknowledged.
|
||||
const current = this.findLatestSessionByName(sessionName);
|
||||
const identity = current ? this.getActiveIdentityVerdict(current, sessionName) : null;
|
||||
if (positiveIdentityFailure(identity)) return `the pane identity is positively mismatched or missing. ${IDENTITY_RECOVERY_GUIDANCE}`;
|
||||
if (input.unproven.code === "binding_mismatch") {
|
||||
return "the seat's binding disagrees with its latest session, so OpenRig can't tell which occupant this acknowledges";
|
||||
}
|
||||
if (input.subject === null || this.restoreAckSubject(sessionName) !== input.subject) {
|
||||
return "the seat changed while it was checked (a newer occupant, binding or token, or a restore that includes it); rerun clear-attention for its current attention";
|
||||
}
|
||||
const { sessionRegistry, eventBus } = this.deps;
|
||||
eventBus.emit({ type: "restore.outcome_acknowledged", rigId: session.rigId, nodeId: session.nodeId, attemptId: outcome.attemptSeq,
|
||||
from: outcome.status, sessionId: session.id, reason, unproven: input.unproven });
|
||||
const clearedClasses: NonNullable<ClearAttentionResult["clearedClasses"]> = ["restore_outcome"];
|
||||
const evidence = { kind: "restore_outcome_acknowledged", state: outcome.status, reason };
|
||||
if (input.hookClassActive && input.hook) {
|
||||
this.clearWaitingHook(session, sessionName, input.hook, reason);
|
||||
clearedClasses.push("needs_input_hook");
|
||||
}
|
||||
if (input.startupClassActive) {
|
||||
sessionRegistry.updateStartupStatus(session.id, "ready", new Date().toISOString());
|
||||
eventBus.emit({ type: "seat.attention_cleared", rigId: session.rigId, nodeId: session.nodeId, sessionName,
|
||||
from: session.startupStatus, to: "ready", clearedBy: "operator_attestation", evidence, reason, previousError: input.previousError });
|
||||
clearedClasses.push("startup_status");
|
||||
}
|
||||
return { ok: true, code: "cleared", from: session.startupStatus, to: "ready", clearedBy: "operator_attestation", evidence, reason,
|
||||
previousError: input.previousError, clearedClasses,
|
||||
acknowledgment: { attemptId: outcome.attemptSeq, from: outcome.status, unproven: input.unproven,
|
||||
...(identity ? { paneIdentity: "unverified" as const } : {}) } };
|
||||
}
|
||||
|
||||
/** What an acknowledgement addresses, read fresh: the latest session (a newer occupant is a new row), its token, the
|
||||
* node's bound session, its full-restore outcome and attempt, and the newest later restore start that may include
|
||||
* the node: the strict reconciler's own current-subject recheck, with the start narrowed to this node. A start whose
|
||||
* recorded roster leaves the node out doesn't change its subject; every current start records its roster, and one
|
||||
* that can't be read counts as including it. The pane isn't compared: the strict check's identity step can rebind
|
||||
* it, and a mismatched pane is the identity class's to judge. Null when there's no full-restore outcome. */
|
||||
private restoreAckSubject(sessionName: string): string | null {
|
||||
const session = this.findLatestSessionByName(sessionName);
|
||||
if (!session || !this.deps.db) return null;
|
||||
const outcome = this.getDerivedAttentionOutcome(session);
|
||||
if (outcome?.source !== "restore.completed") return null;
|
||||
const db = this.deps.db;
|
||||
const binding = db.prepare("SELECT tmux_session FROM bindings WHERE node_id = ?").get(session.nodeId) as
|
||||
{ tmux_session: string | null } | undefined;
|
||||
// Starts carry no node: read them as the outcome query does, through the node/type index.
|
||||
const starts = db.prepare(
|
||||
"SELECT seq, payload FROM events WHERE node_id IS NULL AND type = 'restore.started' AND +rig_id = ? AND seq > ? ORDER BY seq DESC",
|
||||
).all(session.rigId, outcome.attemptSeq) as { seq: number; payload: string }[];
|
||||
const including = starts.find((start) => {
|
||||
try {
|
||||
const roster = JSON.parse(start.payload)?.intendedRoster;
|
||||
return !Array.isArray(roster) || roster.some((node: { nodeId?: unknown } | null) => node?.nodeId === session.nodeId);
|
||||
} catch { return true; }
|
||||
});
|
||||
return JSON.stringify([session.id, session.resumeToken, binding?.tmux_session ?? null, outcome.status, outcome.attemptSeq,
|
||||
including?.seq ?? null]);
|
||||
}
|
||||
|
||||
/** Records the operator's word that the prompt the latest hook named is gone, as the "prompt cleared" hook the activity
|
||||
* poller writes when it sees one go, and settles the poller's pending prompt, so the send gate, the wake ladder and the
|
||||
* inventory stop reading the seat as waiting on a person. */
|
||||
@@ -918,15 +1031,34 @@ export class SeatAttentionReconciler {
|
||||
// Native identity failures and unrelated/stale community failures still require strict reconciliation.
|
||||
const assisted = source === "restore.completed" && communityHarnessOf(session.communityHarness, session.runtime)
|
||||
&& this.assistedStartupInAttempt(session, attemptSeq, row.seq);
|
||||
return { status: "attention_required", source, ...(assisted ? { assistedStartupAttemptId: attemptSeq } : {}) };
|
||||
if (assisted) return { status: "attention_required", source, attemptSeq, assistedStartupAttemptId: attemptSeq };
|
||||
return { status: "attention_required", source, attemptSeq,
|
||||
...(source === "restore.completed" && this.acknowledged(session, attemptSeq, "attention_required") ? { acknowledged: true as const } : {}) };
|
||||
}
|
||||
if (n.status === "failed" && session.sessionStatus === "running") {
|
||||
return { status: "failed", source, attemptSeq,
|
||||
...(source === "restore.completed" && this.acknowledged(session, attemptSeq, "failed") ? { acknowledged: true as const } : {}) };
|
||||
}
|
||||
if (n.status === "failed" && session.sessionStatus === "running") return { status: "failed", source };
|
||||
return null;
|
||||
} catch { continue; }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Whether this session's person acknowledged this full-restore attempt's outcome. The node's latest
|
||||
* acknowledgement, through the node/type index. */
|
||||
private acknowledged(session: { id: string; nodeId: string; rigId: string }, attemptSeq: number,
|
||||
status: "failed" | "attention_required"): boolean {
|
||||
const row = this.deps.db!.prepare(
|
||||
"SELECT payload FROM events WHERE node_id = ? AND type = 'restore.outcome_acknowledged' ORDER BY seq DESC LIMIT 1",
|
||||
).get(session.nodeId) as { payload: string } | undefined;
|
||||
if (!row) return false;
|
||||
try {
|
||||
const ack = JSON.parse(row.payload);
|
||||
return ack.rigId === session.rigId && ack.attemptId === attemptSeq && ack.from === status && ack.sessionId === session.id;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
private assistedStartupInAttempt(session: { id: string; nodeId: string }, attemptId: number, completedSeq: number): boolean {
|
||||
const latest = this.deps.db!.prepare(
|
||||
"SELECT seq, payload FROM events WHERE node_id = ? AND type IN ('node.startup_pending','node.startup_ready','node.startup_failed') ORDER BY seq DESC LIMIT 1",
|
||||
|
||||
@@ -186,6 +186,10 @@ export type RigEvent =
|
||||
// visible-runtime-evidence preconditions hold. The original failure event
|
||||
// is preserved in the log; this event records the audit trail of the upgrade.
|
||||
| { type: "restore.outcome_reconciled"; rigId: string; nodeId: string; attemptId: number; from: "failed" | "attention_required"; to: "operator_recovered"; evidence: { tmux: boolean; fgProcess: "claude" | "codex" | string; resumeTokenUsed: boolean; paneState: "usable" } | { source: string; reason?: string; kind?: string; state?: string; runtimeCwdVerified?: boolean } }
|
||||
// A person acknowledged one full-restore attempt's attention for one occupant session when the strict reconciler
|
||||
// couldn't prove recovery. Not a recovery: the outcome stays, and `unproven` names what the reconciler couldn't show.
|
||||
// Its own type, so no reader of `restore.outcome_reconciled` takes it for one.
|
||||
| { type: "restore.outcome_acknowledged"; rigId: string; nodeId: string; attemptId: number; from: "failed" | "attention_required"; sessionId: string; reason: string; unproven: { code: string; detail: string; probe?: import("./native-resume-probe-evidence.js").NativeResumeProbeEvidence } }
|
||||
| { type: "agent.activity"; rigId: string; nodeId: string; sessionName: string; runtime: string | null; activity: AgentActivity }
|
||||
/** What one send's own result says was admitted, written and seen on screen, for the seat and
|
||||
* occupant it was sent to. Additive: it sits beside the send's result and never rewrites it. `written` is what the
|
||||
@@ -792,6 +796,9 @@ export interface NodeInventoryEntry {
|
||||
/** Persisted startup outcome, before current identity projection. */
|
||||
storedStartupStatus?: "pending" | "ready" | "attention_required" | "failed" | null;
|
||||
restoreOutcome: NodeRestoreOutcome;
|
||||
/** Present when a person acknowledged this occupant's current full-restore attention without proof of recovery:
|
||||
* `restoreOutcome` and `continuityOutcome` are unchanged, and `unproven` says what the reconciler couldn't show. */
|
||||
restoreAcknowledgment?: { attemptId: number; reason: string; unproven: { code: string; detail: string } };
|
||||
// OPR.0.4.3.06 — challenge-verified orientation, surfaced beside (never
|
||||
// folded into) startupStatus.
|
||||
oriented: NodeOriented;
|
||||
|
||||
@@ -13,7 +13,8 @@ import { sessionAdminRoutes } from '../src/routes/sessions.js';
|
||||
import { SeatStatusService } from '../src/domain/seat-status-service.js';
|
||||
const databases: ReturnType<typeof createFullTestDb>[] = [];
|
||||
afterEach(() => { for (const db of databases.splice(0)) db.close(); });
|
||||
function fixture(kind: 'subset' | 'full' | 'startup' | 'identity', explicit: string | null = null) {
|
||||
type Strict = NonNullable<ConstructorParameters<typeof SeatAttentionReconciler>[0]['reconcileRestoreOutcome']>;
|
||||
function fixture(kind: 'subset' | 'full' | 'startup' | 'identity', explicit: string | null = null, strict?: Strict) {
|
||||
const db = createFullTestDb(); databases.push(db);
|
||||
const repo = new RigRepository(db), registry = new SessionRegistry(db), bus = new EventBus(db);
|
||||
const rig = repo.createRig('attention-specimen');
|
||||
@@ -31,7 +32,8 @@ function fixture(kind: 'subset' | 'full' | 'startup' | 'identity', explicit: str
|
||||
const identities = new SeatIdentityStore(db);
|
||||
if (kind === 'identity') identities.upsert({nodeId:node.id,verdict:'mismatch',evidenceSource:'pane_process',reason:'process_identity_mismatch',evidence:{registeredPane:'%991',observedPid:991,observedCommand:'bash',matchedLayer:null},sessionName:name,observedAt:new Date().toISOString()});
|
||||
const sendVerify=vi.fn(async (): Promise<{ok:boolean; verified?:boolean}> => ({ok:false}));
|
||||
const clear = new SeatAttentionReconciler({db,sessionRegistry:registry,eventBus:bus,agentActivityStore:new AgentActivityStore({db,eventBus:bus}),sendVerify});
|
||||
const clear = new SeatAttentionReconciler({db,sessionRegistry:registry,eventBus:bus,agentActivityStore:new AgentActivityStore({db,eventBus:bus}),sendVerify,
|
||||
...(strict ? {reconcileRestoreOutcome:strict} : {})});
|
||||
const app = new Hono();
|
||||
app.use('*',async(c,next)=>{c.set('seatAttentionReconciler' as never,clear as never);c.set('terminalBearerToken' as never,null as never);await next();});
|
||||
app.route('/api/sessions',sessionAdminRoutes);
|
||||
@@ -44,7 +46,7 @@ function fixture(kind: 'subset' | 'full' | 'startup' | 'identity', explicit: str
|
||||
return {status:r.status,body:await r.json() as any};
|
||||
};
|
||||
const events=()=>db.prepare("SELECT type,payload FROM events WHERE type IN ('seat.attention_cleared','restore.outcome_reconciled') ORDER BY seq").all().map((r:any)=>({type:r.type,payload:JSON.parse(r.payload)}));
|
||||
return {db,node,identities,name,inventory,post,events,sendVerify,repo,registry,bus,rig};
|
||||
return {db,node,identities,name,inventory,post,events,sendVerify,repo,registry,bus,rig,clear};
|
||||
}
|
||||
describe('bounded truthful-attention specimen projection',()=>{
|
||||
it.each([null,'failed','fresh'])('subset acknowledgment with explicit continuity %s',async explicit=>{
|
||||
@@ -70,11 +72,12 @@ describe('bounded truthful-attention specimen projection',()=>{
|
||||
expect(f.inventory()).toMatchObject({restoreOutcome:'operator_recovered',continuityOutcome:null,lifecycleState:'running'});
|
||||
expect(f.events()[0].payload.evidence).toMatchObject({source:'clear_attention_evidence',runtimeCwdVerified:false});
|
||||
});
|
||||
it.each(['full','identity'] as const)('%s class is not bypassed by reason',async kind=>{
|
||||
// A full restore's attention is acknowledged on --reason instead (below); its recovery is never claimed.
|
||||
it.each(['identity'] as const)('%s class is not bypassed by reason',async kind=>{
|
||||
const f=fixture(kind),before=f.inventory(),result=await f.post('operator reports this seat is working');
|
||||
expect(result.status).toBe(422);
|
||||
expect(result.body.code).toBe('not_demonstrably_responsive');
|
||||
expect(result.body.detail).toContain(kind==='full'?'restore_outcome':'pane_identity');
|
||||
expect(result.body.detail).toContain('pane_identity');
|
||||
expect(f.events()).toEqual([]);
|
||||
expect(f.sendVerify).not.toHaveBeenCalled();
|
||||
expect(f.inventory()).toEqual(before);
|
||||
@@ -174,3 +177,175 @@ describe('restore proof binding in the inventory batch', () => {
|
||||
expect(both(f).lifecycleState).toBe('attention_required');
|
||||
});
|
||||
});
|
||||
|
||||
// #2473 (Root's disposition, section 3): when the strict reconciler can't settle a full restore's attention, the
|
||||
// person's --reason acknowledges that attempt's attention for that occupant, never as a recovery.
|
||||
describe('full-restore acknowledgement when recovery cannot be proven', () => {
|
||||
type F = ReturnType<typeof fixture>;
|
||||
const unproven = {code:'process_lineage_mismatch', detail:'the process list could not be read'};
|
||||
const reason = 'checked the pane: the conversation is there';
|
||||
const proof = {tmux:true, fgProcess:'claude', resumeTokenUsed:true, paneState:'usable' as const};
|
||||
const strictSays = (result: Awaited<ReturnType<Strict>>, during?: () => void) => vi.fn(async () => { during?.(); return result; });
|
||||
const unsure = (during?: () => void) => strictSays({ok:false, ...unproven}, during);
|
||||
const attempt = (f: F) => (f.db.prepare("SELECT seq FROM events WHERE type='restore.started' ORDER BY seq LIMIT 1").get() as {seq:number}).seq;
|
||||
const acks = (f: F) => f.db.prepare("SELECT payload FROM events WHERE type='restore.outcome_acknowledged' ORDER BY seq").all()
|
||||
.map((r:any)=>JSON.parse(r.payload));
|
||||
const entry = (f: F) => {
|
||||
const local = getNodeInventory(f.db, f.rig.id).find(e => e.nodeId === f.node.id)!;
|
||||
expect(getNodeInventoryForRigs(f.db, new Set([f.rig.id])).get(f.rig.id)!.find(e => e.nodeId === f.node.id)).toEqual(local);
|
||||
return local;
|
||||
};
|
||||
const restore = (f: F, nodeId: string, status = 'attention_required') => {
|
||||
f.bus.emit({type:'restore.started', rigId:f.rig.id, snapshotId:'s2', intendedRoster:[{nodeId, logicalId:'worker'}]});
|
||||
f.bus.emit({type:'restore.completed', rigId:f.rig.id, snapshotId:'s2', result:{snapshotId:'s2', preRestoreSnapshotId:null,
|
||||
rigResult:'partially_restored', nodes:[{nodeId, logicalId:'worker', status}], warnings:[]}} as never);
|
||||
};
|
||||
|
||||
it.each([['an unreadable process list', () => unsure()], ['no strict reconciler', () => undefined]] as const)(
|
||||
'without --reason, %s leaves the attention and says how to acknowledge it', async (_case, strict) => {
|
||||
const f = fixture('full', null, strict()), before = f.inventory();
|
||||
const result = await f.post();
|
||||
expect(result.status).toBe(422);
|
||||
expect(result.body.detail).toContain('--reason "<what you saw>" acknowledges this attention without claiming the conversation resumed');
|
||||
expect(acks(f)).toEqual([]);
|
||||
expect(f.inventory()).toEqual(before);
|
||||
expect(before.lifecycleState).toBe('attention_required');
|
||||
});
|
||||
|
||||
it.each([['an unreadable process list', () => unsure(), unproven],
|
||||
['no strict reconciler', () => undefined, {code:'strict_reconciler_unavailable', detail:'the strict restore reconciler is unavailable'}]] as const)(
|
||||
'with --reason, %s is acknowledged as unproven and never recorded as a recovery', async (_case, strict, expected) => {
|
||||
const f = fixture('full', null, strict()), sessionId = f.registry.getSessionsForRig(f.rig.id)[0]!.id;
|
||||
const result = await f.post(reason);
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.body).toMatchObject({ok:true, clearedBy:'operator_attestation', clearedClasses:['restore_outcome'],
|
||||
acknowledgment:{attemptId:attempt(f), from:'attention_required', unproven:expected}});
|
||||
expect(acks(f)).toEqual([{type:'restore.outcome_acknowledged', rigId:f.rig.id, nodeId:f.node.id, attemptId:attempt(f),
|
||||
from:'attention_required', sessionId, reason, unproven:expected}]);
|
||||
expect(f.events()).toEqual([]);
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'attention_required', continuityOutcome:null, lifecycleState:'running',
|
||||
restoreAcknowledgment:{attemptId:attempt(f), reason, unproven:expected}});
|
||||
expect(new SeatStatusService({rigRepo:f.repo}).getStatus(f.name)).toMatchObject({ok:true,
|
||||
status:{restore_outcome:'attention_required', continuity_outcome:null}});
|
||||
expect(f.sendVerify).not.toHaveBeenCalled();
|
||||
expect((await f.post(reason)).body.code).toBe('not_in_attention');
|
||||
});
|
||||
|
||||
it('a successful native recovery is unchanged by --reason', async () => {
|
||||
let f!: F;
|
||||
f = fixture('full', null, vi.fn(async () => {
|
||||
f.bus.emit({type:'restore.outcome_reconciled', rigId:f.rig.id, nodeId:f.node.id, attemptId:attempt(f), from:'attention_required',
|
||||
to:'operator_recovered', evidence:proof});
|
||||
return {ok:true as const, attemptId:attempt(f), from:'attention_required' as const, to:'operator_recovered' as const, evidence:proof};
|
||||
}));
|
||||
const result = await f.post(reason);
|
||||
expect(result.body).toMatchObject({ok:true, clearedBy:'evidence', derivedEvidence:{source:'restore_runtime_truth'}});
|
||||
expect(result.body.acknowledgment).toBeUndefined();
|
||||
expect(acks(f)).toEqual([]);
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'operator_recovered', continuityOutcome:'resumed', lifecycleState:'running'});
|
||||
expect(entry(f).restoreAcknowledgment).toBeUndefined();
|
||||
});
|
||||
|
||||
it('the strict check still runs on activity after an acknowledgement, and its proof settles the attempt', async () => {
|
||||
const strict = unsure(), f = fixture('full', null, strict);
|
||||
await f.post(reason);
|
||||
expect(await f.clear.onSeatActivity(f.name, 'running')).toBe(false);
|
||||
expect(strict).toHaveBeenCalledTimes(2);
|
||||
f.bus.emit({type:'restore.outcome_reconciled', rigId:f.rig.id, nodeId:f.node.id, attemptId:attempt(f), from:'attention_required',
|
||||
to:'operator_recovered', evidence:proof});
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'operator_recovered', continuityOutcome:'resumed', lifecycleState:'running'});
|
||||
expect(entry(f).restoreAcknowledgment).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a newer occupant', (f: F) => { f.registry.registerSession(f.node.id, f.name); }],
|
||||
['a changed binding', (f: F) => { f.registry.updateBinding(f.node.id, {tmuxSession:'another@attention-specimen', tmuxPane:'%991'}); }],
|
||||
['a changed token', (f: F) => { f.db.prepare("UPDATE sessions SET resume_token='another-token' WHERE node_id=?").run(f.node.id); }],
|
||||
['a restore that includes this seat', (f: F) => { f.bus.emit({type:'restore.started', rigId:f.rig.id, snapshotId:'s2',
|
||||
intendedRoster:[{nodeId:f.node.id, logicalId:'worker'}]}); }],
|
||||
['a restore whose roster is not recorded', (f: F) => { f.bus.emit({type:'restore.started', rigId:f.rig.id, snapshotId:'s2'}); }],
|
||||
] as const)('a change while the check ran is not acknowledged: %s', async (_case, change) => {
|
||||
let f!: F;
|
||||
f = fixture('full', null, unsure(() => change(f)));
|
||||
const result = await f.post(reason);
|
||||
expect(result.status).toBe(422);
|
||||
expect(result.body.detail).toContain('Not acknowledged: the seat changed while it was checked');
|
||||
expect(acks(f)).toEqual([]);
|
||||
});
|
||||
|
||||
it('a pane the check rebinds itself does not block the acknowledgement', async () => {
|
||||
let f!: F;
|
||||
f = fixture('full', null, unsure(() => { f.registry.updateBinding(f.node.id, {tmuxSession:f.name, tmuxPane:'%992'}); }));
|
||||
expect((await f.post(reason)).status).toBe(200);
|
||||
expect(acks(f)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('a positive binding mismatch is not acknowledged', async () => {
|
||||
const f = fixture('full', null, strictSays({ok:false, code:'binding_mismatch', detail:'Canonical binding does not match the latest session row.'}));
|
||||
const result = await f.post(reason);
|
||||
expect(result.status).toBe(422);
|
||||
expect(result.body.detail).toContain("Not acknowledged: the seat's binding disagrees with its latest session");
|
||||
expect(acks(f)).toEqual([]);
|
||||
});
|
||||
|
||||
const identity = (f: F, reasonCode: 'process_identity_mismatch' | 'process_identity_ambiguous') => f.identities.upsert({nodeId:f.node.id,
|
||||
verdict:'mismatch', evidenceSource:'pane_process', reason:reasonCode,
|
||||
evidence:{registeredPane:'%991', observedPid:991, observedCommand:'bash', matchedLayer:null}, sessionName:f.name, observedAt:new Date().toISOString()});
|
||||
|
||||
it('a positive pane-identity mismatch is not acknowledged', async () => {
|
||||
const f = fixture('full', null, unsure());
|
||||
identity(f, 'process_identity_mismatch');
|
||||
const result = await f.post(reason);
|
||||
expect(result.status).toBe(422);
|
||||
expect(result.body.detail).toContain('Not acknowledged: the pane identity is positively mismatched or missing. --reason cannot bypass this attention class');
|
||||
expect(acks(f)).toEqual([]);
|
||||
});
|
||||
|
||||
it('a pane identity that could not be proven keeps its own class; the restore is still acknowledged', async () => {
|
||||
const f = fixture('full', null, unsure());
|
||||
identity(f, 'process_identity_ambiguous');
|
||||
const result = await f.post(reason);
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.body).toMatchObject({clearedClasses:['restore_outcome'], acknowledgment:{paneIdentity:'unverified', unproven}});
|
||||
expect(acks(f)).toHaveLength(1);
|
||||
expect(f.identities.getForNode(f.node.id)).toMatchObject({verdict:'mismatch', reason:'process_identity_ambiguous'});
|
||||
});
|
||||
|
||||
it('a restore that leaves this seat out, started while the check ran, does not change what is acknowledged', async () => {
|
||||
let f!: F;
|
||||
f = fixture('full', null, unsure(() => { f.bus.emit({type:'restore.started', rigId:f.rig.id, snapshotId:'s2',
|
||||
intendedRoster:[{nodeId:'another-node', logicalId:'other'}]}); }));
|
||||
const result = await f.post(reason);
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.body.acknowledgment).toMatchObject({attemptId:attempt(f)});
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'attention_required', lifecycleState:'running', restoreAcknowledgment:{attemptId:attempt(f)}});
|
||||
});
|
||||
|
||||
it('an earlier attempt, after a newer restore left this seat out, is acknowledged for that attempt', async () => {
|
||||
const f = fixture('full', null, strictSays({ok:false, code:'node_not_found', detail:'Node has no record in the latest restore.completed event.'}));
|
||||
const first = attempt(f), other = f.repo.addNode(f.rig.id, 'other', {role:'worker', runtime:'claude-code'});
|
||||
restore(f, other.id, 'resumed');
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'attention_required', lifecycleState:'attention_required'});
|
||||
const result = await f.post(reason);
|
||||
expect(result.body).toMatchObject({ok:true, acknowledgment:{attemptId:first, unproven:{code:'node_not_found'}}});
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'attention_required', lifecycleState:'running', restoreAcknowledgment:{attemptId:first}});
|
||||
});
|
||||
|
||||
it('a newer restore that includes this seat replaces the acknowledged outcome', async () => {
|
||||
const strict = unsure(), f = fixture('full', null, strict);
|
||||
await f.post(reason);
|
||||
restore(f, f.node.id);
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'attention_required', lifecycleState:'attention_required'});
|
||||
expect(entry(f).restoreAcknowledgment).toBeUndefined();
|
||||
expect((await f.post()).status).toBe(422);
|
||||
expect(strict).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('a newer occupant after the acknowledgement keeps its own attention', async () => {
|
||||
const f = fixture('full', null, unsure());
|
||||
await f.post(reason);
|
||||
f.registry.updateStatus(f.registry.registerSession(f.node.id, f.name).id, 'running');
|
||||
expect(entry(f)).toMatchObject({restoreOutcome:'attention_required', lifecycleState:'attention_required'});
|
||||
expect(entry(f).restoreAcknowledgment).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -4792,7 +4792,7 @@ describe("RestoreOrchestrator", () => {
|
||||
agentActivityStore: new AgentActivityStore({ db, eventBus }),
|
||||
reconcileRestoreOutcome: (rigId, nodeId) => restore.reconcileNodeRuntimeTruth(rigId, nodeId),
|
||||
});
|
||||
const result = await reconciler.clearAttention(seeded.sessionName, { reason: "operator explanation" });
|
||||
const result = await reconciler.clearAttention(seeded.sessionName);
|
||||
expect(result).toMatchObject({ ok: false, code: "not_demonstrably_responsive",
|
||||
detail: expect.stringContaining("pane_not_usable: Pane state is failed (login_required)") });
|
||||
const events = db.prepare("SELECT payload FROM events WHERE type = 'seat.attention_clear_refused'").all() as { payload: string }[];
|
||||
@@ -4808,6 +4808,15 @@ describe("RestoreOrchestrator", () => {
|
||||
expect(events[0]!.payload).not.toContain("fixture-private");
|
||||
expect(db.prepare("SELECT payload FROM events WHERE type = 'restore.completed'").get()).toEqual(before);
|
||||
expect(db.prepare("SELECT * FROM events WHERE type IN ('restore.outcome_reconciled', 'seat.attention_cleared')").all()).toHaveLength(0);
|
||||
// #2473: with the person's reason the attention is acknowledged, and the same native screen evidence is kept on it.
|
||||
captured = false;
|
||||
const acknowledged = await reconciler.clearAttention(seeded.sessionName, { reason: "operator explanation" });
|
||||
expect(acknowledged).toMatchObject({ ok: true, acknowledgment: { attemptId: attempt.seq, from: "attention_required",
|
||||
unproven: { code: "pane_not_usable", probe: JSON.parse(events[0]!.payload).probe } } });
|
||||
const acks = db.prepare("SELECT payload FROM events WHERE type = 'restore.outcome_acknowledged'").all() as { payload: string }[];
|
||||
expect(acks.map((row) => JSON.parse(row.payload).unproven.probe)).toEqual([JSON.parse(events[0]!.payload).probe]);
|
||||
expect(acks[0]!.payload).not.toContain("fixture-private");
|
||||
expect(db.prepare("SELECT * FROM events WHERE type IN ('restore.outcome_reconciled', 'seat.attention_cleared')").all()).toHaveLength(0);
|
||||
expect(tmux.sendText).not.toHaveBeenCalled();
|
||||
expect(tmux.sendKeys).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -665,7 +665,7 @@ describe("SeatAttentionReconciler", () => {
|
||||
["a previous restore attempt", "assisted_startup", true, false],
|
||||
["another startup failure", "readiness_check", false, false],
|
||||
["another occupant", "assisted_startup", false, true],
|
||||
] as const)("doesn't acknowledge full-restore attention from %s as this seat's assisted startup", async (_case, stage, oldAttempt, otherOccupant) => {
|
||||
] as const)("doesn't settle full-restore attention from %s as this seat's assisted startup; --reason only acknowledges it", async (_case, stage, oldAttempt, otherOccupant) => {
|
||||
const { rigId, nodeId, sessionId } = seedAttentionSeat("assisted-control", "worker@assisted-control");
|
||||
sessionRegistry.updateStatus(sessionId, "running");
|
||||
db.prepare("UPDATE nodes SET runtime='synth' WHERE id=?").run(nodeId);
|
||||
@@ -681,14 +681,19 @@ describe("SeatAttentionReconciler", () => {
|
||||
} });
|
||||
const strict = vi.fn(async () => ({ ok: false as const, code: "runtime_unsupported", detail: "Not an assisted startup acknowledgement." }));
|
||||
const checked = new SeatAttentionReconciler({ db, sessionRegistry, eventBus, agentActivityStore: activityStore, reconcileRestoreOutcome: strict });
|
||||
expect(await checked.clearAttention("worker@assisted-control", { reason: "Checked startup" })).toMatchObject({ ok: false });
|
||||
expect(await checked.clearAttention("worker@assisted-control", { reason: "Checked startup" })).toMatchObject({ ok: true,
|
||||
clearedBy: "operator_attestation", clearedClasses: ["restore_outcome", "startup_status"],
|
||||
acknowledgment: { from: "attention_required", unproven: { code: "runtime_unsupported" } } });
|
||||
expect(strict).toHaveBeenCalledTimes(1);
|
||||
expect(db.prepare("SELECT startup_status FROM sessions WHERE id=?").get(sessionId)).toEqual({ startup_status: "attention_required" });
|
||||
// Acknowledged on the person's word, never recorded as a recovery.
|
||||
expect(db.prepare("SELECT seq FROM events WHERE type='restore.outcome_reconciled'").all()).toEqual([]);
|
||||
expect(db.prepare("SELECT count(*) AS n FROM events WHERE type='restore.outcome_acknowledged'").get()).toEqual({ n: 1 });
|
||||
expect(JSON.parse((db.prepare("SELECT payload FROM events WHERE type='seat.attention_cleared'").get() as { payload: string }).payload))
|
||||
.toMatchObject({ clearedBy: "operator_attestation", evidence: { kind: "restore_outcome_acknowledged" }, reason: "Checked startup" });
|
||||
});
|
||||
|
||||
it("OPR.0.5.9.14: refuses derived-class clear without strict evidence even with operator attestation", async () => {
|
||||
const { rigId, nodeId, sessionId } = seedDerivedAttentionSeat("r-derived-refuse", "worker@r-derived-refuse");
|
||||
it("acknowledges, never recovers, a full restore when the strict reconciler is unavailable (#2473)", async () => {
|
||||
const { rigId, nodeId, sessionId, attemptId } = seedDerivedAttentionSeat("r-derived-refuse", "worker@r-derived-refuse");
|
||||
const before = db.prepare("SELECT payload FROM events WHERE type = 'restore.completed'").all();
|
||||
const sendVerify = vi.fn(async () => ({ ok: true }));
|
||||
const capture = vi.fn(async (sessionName: string) => ({ ok: true, sessionName, content: "fixture" }));
|
||||
@@ -696,17 +701,28 @@ describe("SeatAttentionReconciler", () => {
|
||||
sessionRegistry, eventBus, agentActivityStore: activityStore, db, sendVerify, capture,
|
||||
});
|
||||
|
||||
const result = await derivedReconciler.clearAttention("worker@r-derived-refuse", { reason: "operator verified" });
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.code).toBe("not_demonstrably_responsive");
|
||||
expect(result.detail).toContain("strict restore reconciler is unavailable");
|
||||
const unproven = { code: "strict_reconciler_unavailable", detail: "the strict restore reconciler is unavailable" };
|
||||
const refused = await derivedReconciler.clearAttention("worker@r-derived-refuse");
|
||||
expect(refused).toMatchObject({ ok: false, code: "not_demonstrably_responsive" });
|
||||
expect(refused.detail).toContain("strict restore reconciler is unavailable");
|
||||
const refusals = db.prepare("SELECT payload FROM events WHERE type = 'seat.attention_clear_refused'").all() as { payload: string }[];
|
||||
expect(refusals).toHaveLength(1);
|
||||
expect(JSON.parse(refusals[0]!.payload)).toEqual({
|
||||
type: "seat.attention_clear_refused", rigId, nodeId, sessionId,
|
||||
sessionName: "worker@r-derived-refuse", attentionClass: "restore_outcome", code: "strict_reconciler_unavailable",
|
||||
});
|
||||
|
||||
const result = await derivedReconciler.clearAttention("worker@r-derived-refuse", { reason: "operator verified" });
|
||||
|
||||
expect(result).toMatchObject({ ok: true, code: "cleared", clearedBy: "operator_attestation", clearedClasses: ["restore_outcome"],
|
||||
acknowledgment: { attemptId, from: "failed", unproven } });
|
||||
expect(db.prepare("SELECT payload FROM events WHERE type = 'seat.attention_clear_refused'").all()).toHaveLength(1);
|
||||
expect((db.prepare("SELECT payload FROM events WHERE type = 'restore.outcome_acknowledged'").all() as { payload: string }[])
|
||||
.map((row) => JSON.parse(row.payload))).toEqual([{ type: "restore.outcome_acknowledged", rigId, nodeId, attemptId, from: "failed",
|
||||
sessionId, reason: "operator verified", unproven }]);
|
||||
// The failed outcome and its continuity stand; only the attention is acknowledged.
|
||||
expect(getNodeInventory(db, rigId)[0]).toMatchObject({ restoreOutcome: "failed", continuityOutcome: "failed", lifecycleState: "running",
|
||||
restoreAcknowledgment: { attemptId, reason: "operator verified", unproven } });
|
||||
expect(db.prepare("SELECT payload FROM events WHERE type = 'restore.completed'").all()).toEqual(before);
|
||||
expect(db.prepare("SELECT payload FROM events WHERE type = 'seat.attention_cleared'").all()).toHaveLength(0);
|
||||
expect(sendVerify).not.toHaveBeenCalled();
|
||||
@@ -767,17 +783,21 @@ describe("SeatAttentionReconciler", () => {
|
||||
expect(getNodeInventory(db, rigId)[0]).toMatchObject({ restoreOutcome: "n-a", lifecycleState: "running" });
|
||||
});
|
||||
|
||||
it("a fresh launch that needed attention still needs the strict reconciler", async () => {
|
||||
it("a fresh launch that needed attention still needs the strict reconciler; --reason then acknowledges it", async () => {
|
||||
const { rigId, nodeId } = seedDerivedAttentionSeat("r-116-unready", "worker@r-116-unready");
|
||||
launchFreshOccupant(rigId, nodeId, "worker@r-116-unready", "attention_required");
|
||||
const { strict, fenced } = refusingStrictReconciler();
|
||||
|
||||
expect(await fenced.clearAttention("worker@r-116-unready")).toMatchObject({ ok: false, code: "not_demonstrably_responsive" });
|
||||
expect(strict).toHaveBeenCalledWith(rigId, nodeId);
|
||||
expect(getNodeInventory(db, rigId)[0]).toMatchObject({ restoreOutcome: "failed", lifecycleState: "attention_required" });
|
||||
|
||||
const result = await fenced.clearAttention("worker@r-116-unready", { reason: "operator verified" });
|
||||
|
||||
expect(result).toMatchObject({ ok: false, code: "not_demonstrably_responsive" });
|
||||
expect(strict).toHaveBeenCalledWith(rigId, nodeId);
|
||||
expect(result).toMatchObject({ ok: true, acknowledgment: { from: "failed", unproven: { code: "no_attempt_proof" } } });
|
||||
expect(strict).toHaveBeenCalledTimes(2);
|
||||
expect(reconciledEvents()).toHaveLength(0);
|
||||
expect(getNodeInventory(db, rigId)[0]).toMatchObject({ restoreOutcome: "failed", lifecycleState: "attention_required" });
|
||||
expect(getNodeInventory(db, rigId)[0]).toMatchObject({ restoreOutcome: "failed", lifecycleState: "running" });
|
||||
});
|
||||
|
||||
it("a restore attempt started after the fresh launch still needs the strict reconciler", async () => {
|
||||
@@ -839,10 +859,12 @@ describe("SeatAttentionReconciler", () => {
|
||||
evidence: { source: "operator_attestation" } });
|
||||
const { strict, fenced } = refusingStrictReconciler();
|
||||
|
||||
const result = await fenced.clearAttention("worker@r-116-zero-ack", { reason: "operator verified" });
|
||||
|
||||
expect(result).toMatchObject({ ok: false, code: "not_demonstrably_responsive" });
|
||||
expect(await fenced.clearAttention("worker@r-116-zero-ack")).toMatchObject({ ok: false, code: "not_demonstrably_responsive" });
|
||||
expect(strict).toHaveBeenCalledWith(rigId, nodeId);
|
||||
// --reason now acknowledges it as unproven: still no recovery beside the legacy zero-ID row.
|
||||
expect(await fenced.clearAttention("worker@r-116-zero-ack", { reason: "operator verified" }))
|
||||
.toMatchObject({ ok: true, acknowledgment: { from: "attention_required", unproven: { code: "no_attempt_proof" } } });
|
||||
expect(reconciledEvents()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -917,7 +939,7 @@ describe("SeatAttentionReconciler", () => {
|
||||
expect(result.derivedEvidence).toMatchObject({ source: "restore_runtime_truth", attemptId });
|
||||
});
|
||||
|
||||
it("OPR.0.5.9.14: refuses a non-running derived outcome when strict live lineage is absent", async () => {
|
||||
it("OPR.0.5.9.14: a non-running derived outcome without strict live lineage is only acknowledged, never recovered", async () => {
|
||||
const rig = rigRepo.createRig("r-nonrun-attn");
|
||||
const node = rigRepo.addNode(rig.id, "worker", { role: "worker", runtime: "claude-code" });
|
||||
const session = sessionRegistry.registerSession(node.id, "worker@r-nonrun-attn");
|
||||
@@ -943,10 +965,13 @@ describe("SeatAttentionReconciler", () => {
|
||||
}),
|
||||
});
|
||||
|
||||
const refused = await nonrunReconciler.clearAttention("worker@r-nonrun-attn");
|
||||
expect(refused.ok).toBe(false);
|
||||
expect(refused.detail).toContain("tmux_session_missing");
|
||||
|
||||
const result = await nonrunReconciler.clearAttention("worker@r-nonrun-attn", { reason: "manually verified" });
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.detail).toContain("tmux_session_missing");
|
||||
expect(result).toMatchObject({ ok: true, acknowledgment: { from: "attention_required", unproven: { code: "tmux_session_missing" } } });
|
||||
const events = db.prepare("SELECT payload FROM events WHERE type = 'restore.outcome_reconciled'").all() as { payload: string }[];
|
||||
expect(events).toHaveLength(0);
|
||||
});
|
||||
|
||||
@@ -93,7 +93,7 @@
|
||||
"openrig-operating-model/templates/SOP.md": "1eaf0daf4f9bbf0715e6a5aaf0f29f030af5fe8b8a10232488ee97c507afa6dc",
|
||||
"openrig-operating-model/templates/SPEC.md": "f126536daa59cea3f1cfb0898d83c7cdf99f6392f6f155ea64fe55187dd0f5f1",
|
||||
"openrig-skills/SKILL.md": "6d7bf933ac40531f11757a31a93d26dcbda617f1b85f48a21865b3a5c5e2ddaa",
|
||||
"openrig-user/SKILL.md": "6176761014c9ecd61675cb47d4de991de6dd126a92be7e06c3ef31aa0573f82e",
|
||||
"openrig-user/SKILL.md": "34997376d4aa921b7ac597e442cd157575b62864023220de25d835cc45226c67",
|
||||
"orienting-to-an-inherited-seat/CHANGELOG.md": "4d0970e75e5c15b00a48d6a7150376d646257777264fd1b9a191165a8d5a6e84",
|
||||
"orienting-to-an-inherited-seat/SKILL.md": "252fed402f19027056bbba241df3305be88de3175b67dc8fe051143bb0d4412b",
|
||||
"queue-handoff/SKILL.md": "96251f6830006bb9d8ede2e5826ab936df4a9fb91090187a23ccaf860ab76336",
|
||||
|
||||
Reference in new issue
Block a user