mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-09 03:39:51 -10:00
fix(conveyor): complete trust recovery readiness after approval
HG-8 adversarial dogfood found residual trust-recovery defects after the route-level 409 discriminator: - The 409 operator action advertised `rig setup --cwd`, but `rig setup` has no `--cwd` flag. Remove that stale guidance so the preserved state points only to the shipped in-pane trust prompt path. - The multi-node 409 operator action could say "each parked session" while abbreviating the attach hint list. Keep the short hint list, but point operators to attentionNodes when hints are abbreviated. - Codex sessions could remain stuck at startup_status=attention_required after the prompt was approved because ContextMonitor only normalized Claude readiness and the Codex native probe prioritized old trust-prompt scrollback over the current active TUI. Wire runtime-specific readiness checkers into ContextMonitor, include stale Codex attention_required sessions in monitor eligibility, skip context-usage parsing for Codex sessions without resume tokens, and classify active Codex TUI content before old trust-prompt scrollback. Verification: - npm run build --workspace @openrig/daemon - npm run build --workspace @openrig/cli - npm run test --workspace @openrig/daemon -- test/native-resume-probe.test.ts test/context-monitor.test.ts test/up-route.test.ts test/pod-rigspec-instantiator.test.ts test/rigspec-instantiator.test.ts test/bootstrap-orchestrator.test.ts test/rig-expansion-service.test.ts - Live isolated daemon HG-8 dogfood: first-run conveyor launch parks/listable with 409 attention_required, prompt approval transitions all four Claude/Codex sessions to ready.
This commit is contained in:
1 parent
33547403bf
commit
5264b19ae4
8 files changed
+147
-33
No files matched your search
@@ -26,6 +26,10 @@ interface ClaudeContextProvisioner {
|
||||
checkReady?(binding: NodeBinding): Promise<ReadinessResult>;
|
||||
}
|
||||
|
||||
interface RuntimeReadinessChecker {
|
||||
checkReady?(binding: NodeBinding): Promise<ReadinessResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls known managed runtime context sources and persists the latest
|
||||
* normalized telemetry. Scheduler-only: no queries, no response shaping,
|
||||
@@ -41,6 +45,7 @@ export class ContextMonitor {
|
||||
private store: ContextUsageStore;
|
||||
private claudeContextProvisioner: ClaudeContextProvisioner | null;
|
||||
private compactionEnforcer: ClaudeCompactionEnforcer | null;
|
||||
private readinessCheckers: Record<string, RuntimeReadinessChecker | undefined>;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(
|
||||
@@ -48,11 +53,16 @@ export class ContextMonitor {
|
||||
store: ContextUsageStore,
|
||||
claudeContextProvisioner?: ClaudeContextProvisioner,
|
||||
compactionEnforcer?: ClaudeCompactionEnforcer,
|
||||
readinessCheckers?: Record<string, RuntimeReadinessChecker | undefined>,
|
||||
) {
|
||||
this.db = db;
|
||||
this.store = store;
|
||||
this.claudeContextProvisioner = claudeContextProvisioner ?? null;
|
||||
this.compactionEnforcer = compactionEnforcer ?? null;
|
||||
this.readinessCheckers = readinessCheckers ?? {};
|
||||
if (claudeContextProvisioner && !this.readinessCheckers["claude-code"]) {
|
||||
this.readinessCheckers["claude-code"] = claudeContextProvisioner;
|
||||
}
|
||||
}
|
||||
|
||||
/** Discover active managed Claude sessions and poll their sidecar files. */
|
||||
@@ -60,15 +70,18 @@ export class ContextMonitor {
|
||||
const sessions = this.getEligibleSessions();
|
||||
for (const session of sessions) {
|
||||
let observed: ContextUsage | null = null;
|
||||
try {
|
||||
observed = this.readContextUsage(session);
|
||||
this.store.persist(session.node_id, observed);
|
||||
} catch {
|
||||
observed = null;
|
||||
// One bad session must not crash polling for others
|
||||
const canReadContextUsage = session.runtime !== "codex" || !!session.resume_token;
|
||||
if (canReadContextUsage) {
|
||||
try {
|
||||
this.store.persist(session.node_id, this.store.unknownUsage("parse_error"));
|
||||
} catch { /* give up on this session */ }
|
||||
observed = this.readContextUsage(session);
|
||||
this.store.persist(session.node_id, observed);
|
||||
} catch {
|
||||
observed = null;
|
||||
// One bad session must not crash polling for others
|
||||
try {
|
||||
this.store.persist(session.node_id, this.store.unknownUsage("parse_error"));
|
||||
} catch { /* give up on this session */ }
|
||||
}
|
||||
}
|
||||
|
||||
await this.normalizeStartupStatus(session);
|
||||
@@ -139,7 +152,13 @@ export class ContextMonitor {
|
||||
LEFT JOIN bindings b ON b.node_id = n.id
|
||||
WHERE (
|
||||
(n.runtime = 'claude-code' AND s.status = 'running')
|
||||
OR (n.runtime = 'codex' AND s.resume_token IS NOT NULL)
|
||||
OR (
|
||||
n.runtime = 'codex'
|
||||
AND (
|
||||
s.resume_token IS NOT NULL
|
||||
OR s.startup_status IN ('failed', 'attention_required')
|
||||
)
|
||||
)
|
||||
)
|
||||
AND COALESCE(b.attachment_type, 'tmux') = 'tmux'
|
||||
AND COALESCE(b.tmux_session, s.session_name) IS NOT NULL
|
||||
@@ -162,12 +181,12 @@ export class ContextMonitor {
|
||||
}
|
||||
|
||||
private async normalizeStartupStatus(session: EligibleSession): Promise<void> {
|
||||
if (session.runtime !== "claude-code") return;
|
||||
if (!this.claudeContextProvisioner?.checkReady) return;
|
||||
const checker = session.runtime ? this.readinessCheckers[session.runtime] : undefined;
|
||||
if (!checker?.checkReady) return;
|
||||
if (session.startup_status !== "failed" && session.startup_status !== "attention_required") return;
|
||||
|
||||
try {
|
||||
const readiness = await this.claudeContextProvisioner.checkReady({
|
||||
const readiness = await checker.checkReady({
|
||||
id: `context-monitor:${session.session_id}`,
|
||||
nodeId: session.node_id,
|
||||
attachmentType: "tmux",
|
||||
|
||||
@@ -126,13 +126,6 @@ export function assessNativeResumeProbe(
|
||||
detail: "Codex could not refresh the stored access token; an operator must sign in again before the session can resume.",
|
||||
};
|
||||
}
|
||||
if (looksLikeCodexTrustPrompt(paneContent)) {
|
||||
return {
|
||||
status: "inconclusive",
|
||||
code: "trust_gate",
|
||||
detail: "Codex is waiting for workspace trust approval before the session can become interactive.",
|
||||
};
|
||||
}
|
||||
if (looksLikeCodexModelSelectionPrompt(paneContent)) {
|
||||
return {
|
||||
status: "inconclusive",
|
||||
@@ -147,6 +140,13 @@ export function assessNativeResumeProbe(
|
||||
detail: "Codex is running with an active interactive TUI in the probe pane.",
|
||||
};
|
||||
}
|
||||
if (looksLikeCodexTrustPrompt(paneContent)) {
|
||||
return {
|
||||
status: "inconclusive",
|
||||
code: "trust_gate",
|
||||
detail: "Codex is waiting for workspace trust approval before the session can become interactive.",
|
||||
};
|
||||
}
|
||||
if (paneContent.includes("Update available!") || paneContent.includes("Updating Codex")) {
|
||||
return {
|
||||
status: "inconclusive",
|
||||
|
||||
@@ -855,12 +855,10 @@ export class PodRigInstantiator {
|
||||
if (hasAttention && !hasLaunched) {
|
||||
// All-attention_required path — rig + sessions PRESERVED; no
|
||||
// tear-down. The operator's path: attach to a session via
|
||||
// `tmux attach -t <session>` and answer the runtime prompt, OR
|
||||
// pre-trust the workspace (e.g., `rig setup --cwd <cwd>`) and
|
||||
// restore (`rig restore <rigId>`). NO new trust primitive
|
||||
// introduced (HG-5); reuses the shipped trust-grant + resume
|
||||
// primitives already exercised by `rig setup` + the existing
|
||||
// restore path.
|
||||
// `tmux attach -t <session>` and answer the runtime prompt. NO
|
||||
// new trust primitive
|
||||
// introduced (HG-5); reuses the runtime's shipped in-pane
|
||||
// trust-grant prompt.
|
||||
const attentionNodes = nodeResults
|
||||
.filter((n) => n.status === "attention_required")
|
||||
.map((n) => ({
|
||||
|
||||
@@ -41,19 +41,22 @@ export function buildAttentionResponse(result: {
|
||||
attentionNodes: import("../domain/types.js").AttentionNode[];
|
||||
};
|
||||
const nodeCount = detail.attentionNodes.length;
|
||||
const attachHints = detail.attentionNodes
|
||||
const sessionAttachHints = detail.attentionNodes
|
||||
.filter((n) => n.sessionName)
|
||||
.map((n) => `tmux attach -t ${n.sessionName}`)
|
||||
.slice(0, 3)
|
||||
.join(" ; ");
|
||||
const visibleAttachHints = sessionAttachHints.slice(0, 3).join(" ; ");
|
||||
const remainingAttachHintCount = Math.max(0, sessionAttachHints.length - 3);
|
||||
const attachHintText = visibleAttachHints
|
||||
? `${visibleAttachHints}${remainingAttachHintCount > 0 ? ` ; plus ${remainingAttachHintCount} more listed in attentionNodes` : ""}`
|
||||
: "see `rig ps`";
|
||||
const rigIdDisplay = result.rigId ?? "(rigId unavailable)";
|
||||
return {
|
||||
error: {
|
||||
fact: detail.message,
|
||||
consequence: `Rig ${rigIdDisplay} is created and listable via \`rig ps\`. Sessions are running with startup_status='attention_required'. Tmux panes show the runtime's trust/approval prompt — answering it in-pane completes the launch.`,
|
||||
action: nodeCount === 1
|
||||
? `Attach to the session and answer the prompt: ${attachHints || "see `rig ps`"}. Alternatively, pre-trust the workspace (e.g., \`rig setup --cwd <workspace>\`) and re-run the launch.`
|
||||
: `Attach to each parked session and answer its prompt: ${attachHints || "see `rig ps`"}. Alternatively, pre-trust the workspace (e.g., \`rig setup --cwd <workspace>\`) and re-run the launch.`,
|
||||
? `Attach to the session and answer the prompt: ${attachHintText}.`
|
||||
: `Attach to each parked session listed in attentionNodes and answer its prompt: ${attachHintText}.`,
|
||||
},
|
||||
attentionNodes: detail.attentionNodes,
|
||||
};
|
||||
|
||||
@@ -1218,7 +1218,10 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
|
||||
deps.sessionTransport,
|
||||
)
|
||||
: undefined;
|
||||
const contextMonitor = new ContextMonitor(db, contextUsageStore, claudeAdapter, compactionEnforcer);
|
||||
const contextMonitor = new ContextMonitor(db, contextUsageStore, claudeAdapter, compactionEnforcer, {
|
||||
"claude-code": claudeAdapter,
|
||||
codex: codexAdapter,
|
||||
});
|
||||
deps.contextMonitor = contextMonitor;
|
||||
|
||||
const app = createApp(deps);
|
||||
|
||||
@@ -355,6 +355,37 @@ describe("ContextMonitor", () => {
|
||||
expect(refreshed.startup_status).toBe("failed");
|
||||
});
|
||||
|
||||
it("pollOnce normalizes stale Codex attention_required state after the trust prompt is cleared", async () => {
|
||||
const rig = rigRepo.createRig("test-rig-codex-trust");
|
||||
const node = rigRepo.addNode(rig.id, "dev.qa", { runtime: "codex", cwd: "/project" });
|
||||
const session = sessionRegistry.registerSession(node.id, "dev-qa-trust@test");
|
||||
db.prepare("UPDATE sessions SET status = 'running', startup_status = 'attention_required', resume_token = NULL WHERE id = ?")
|
||||
.run(session.id);
|
||||
|
||||
const codexReadySpy = vi.fn(async (): Promise<ReadinessResult> => ({ ready: true }));
|
||||
monitor = new ContextMonitor(db, store, {
|
||||
ensureContextCollector: ensureContextCollectorSpy,
|
||||
checkReady: checkReadySpy,
|
||||
}, undefined, {
|
||||
"claude-code": { checkReady: checkReadySpy },
|
||||
codex: { checkReady: codexReadySpy },
|
||||
});
|
||||
|
||||
await monitor.pollOnce();
|
||||
|
||||
const refreshed = db.prepare("SELECT startup_status, startup_completed_at FROM sessions WHERE id = ?").get(session.id) as {
|
||||
startup_status: string;
|
||||
startup_completed_at: string | null;
|
||||
};
|
||||
expect(refreshed.startup_status).toBe("ready");
|
||||
expect(refreshed.startup_completed_at).toBeTruthy();
|
||||
expect(codexReadySpy).toHaveBeenCalledWith(expect.objectContaining({
|
||||
nodeId: session.nodeId,
|
||||
tmuxSession: "dev-qa-trust@test",
|
||||
cwd: "/project",
|
||||
}));
|
||||
});
|
||||
|
||||
it("pollOnce does not overwrite pending Claude startup state", async () => {
|
||||
const { sessionName, session } = (() => {
|
||||
const rig = rigRepo.createRig("test-rig-6");
|
||||
|
||||
@@ -221,6 +221,35 @@ describe("native resume probe", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies Codex as active when the current TUI appears below an old trust prompt in scrollback", () => {
|
||||
expect(
|
||||
assessNativeResumeProbe({
|
||||
runtime: "codex",
|
||||
paneCommand: "codex-aarch64-a",
|
||||
paneContent: [
|
||||
"> You are in /some/workspace",
|
||||
"",
|
||||
" Do you trust the contents of this directory? Working with untrusted contents",
|
||||
"",
|
||||
"› 1. Yes, continue",
|
||||
" 2. No, quit",
|
||||
"",
|
||||
"╭────────────────────────────────────────────────────╮",
|
||||
"│ >_ OpenAI Codex (v0.130.0) │",
|
||||
"╰────────────────────────────────────────────────────╯",
|
||||
"",
|
||||
"› Write tests for @filename",
|
||||
"",
|
||||
" gpt-5.5 xhigh fast · Context 0% used · Fast on",
|
||||
].join("\n"),
|
||||
})
|
||||
).toEqual({
|
||||
status: "resumed",
|
||||
code: "active_runtime",
|
||||
detail: "Codex is running with an active interactive TUI in the probe pane.",
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies Codex update prompts as inconclusive", () => {
|
||||
expect(
|
||||
assessNativeResumeProbe({
|
||||
|
||||
@@ -554,7 +554,7 @@ edges: []
|
||||
expect(r!.error.consequence).toMatch(/rig ps/);
|
||||
expect(r!.error.consequence).toMatch(/attention_required/);
|
||||
expect(r!.error.action).toMatch(/tmux attach -t dev-impl@conveyor/);
|
||||
expect(r!.error.action).toMatch(/rig setup --cwd/);
|
||||
expect(r!.error.action).not.toMatch(/rig setup --cwd/);
|
||||
// Singular phrasing for a 1-node case
|
||||
expect(r!.error.action).toMatch(/^Attach to the session/);
|
||||
expect(r!.attentionNodes).toHaveLength(1);
|
||||
@@ -591,6 +591,37 @@ edges: []
|
||||
expect(r!.attentionNodes).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("HG-4: multi-node action message points to attentionNodes when attach hints are abbreviated", () => {
|
||||
const result = {
|
||||
rigId: "rig-conveyor-4",
|
||||
stages: [
|
||||
{
|
||||
stage: "import_rig",
|
||||
status: "blocked",
|
||||
detail: {
|
||||
code: "attention_required",
|
||||
message: "4 nodes require attention before becoming interactive.",
|
||||
attentionNodes: [
|
||||
{ logicalId: "intake.lead", sessionName: "intake-lead@conveyor", reason: "trust_gate" },
|
||||
{ logicalId: "plan.planner", sessionName: "plan-planner@conveyor", reason: "trust_gate" },
|
||||
{ logicalId: "build.builder", sessionName: "build-builder@conveyor", reason: "trust_gate" },
|
||||
{ logicalId: "review.reviewer", sessionName: "review-reviewer@conveyor", reason: "trust_gate" },
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const r = buildAttentionResponse(result);
|
||||
expect(r).not.toBeNull();
|
||||
expect(r!.error.action).toMatch(/^Attach to each parked session listed in attentionNodes/);
|
||||
expect(r!.error.action).toContain("tmux attach -t intake-lead@conveyor");
|
||||
expect(r!.error.action).toContain("tmux attach -t plan-planner@conveyor");
|
||||
expect(r!.error.action).toContain("tmux attach -t build-builder@conveyor");
|
||||
expect(r!.error.action).toContain("plus 1 more listed in attentionNodes");
|
||||
expect(r!.error.action).not.toContain("review-reviewer@conveyor");
|
||||
expect(r!.attentionNodes).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("HG-4: when no sessionName is present on any node, action falls back to 'see rig ps' (defense)", () => {
|
||||
const result = {
|
||||
rigId: "rig-conveyor-3",
|
||||
@@ -741,7 +772,7 @@ edges: []
|
||||
expect(body.error.consequence).toMatch(/rig-stub-1/);
|
||||
expect(body.error.consequence).toMatch(/rig ps/);
|
||||
expect(body.error.action).toMatch(/tmux attach -t dev-impl@conveyor-attention-route-test/);
|
||||
expect(body.error.action).toMatch(/rig setup --cwd/);
|
||||
expect(body.error.action).not.toMatch(/rig setup --cwd/);
|
||||
// attentionNodes array carried through to operator
|
||||
expect(body.attentionNodes).toBeInstanceOf(Array);
|
||||
expect(body.attentionNodes).toHaveLength(1);
|
||||
|
||||
Reference in new issue
Block a user