mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-07 22:58:42 -10:00
fix: improve Codex startup and restore reporting
This commit is contained in:
OpenRig Contributors committed
1 parent
9c9e518d29
commit
6553aaef3a
22 files changed
+873
-163
No files matched your search
@@ -8,6 +8,7 @@ import { getDaemonStatus, getDaemonUrl, startDaemon, type LifecycleDeps, daemonS
|
||||
import type { RiggedConfig } from "../config-store.js";
|
||||
import { realDeps } from "./daemon.js";
|
||||
import type { StatusDeps } from "./status.js";
|
||||
import { formatThreePart, type ThreePartRejection } from "./workflow-errors.js";
|
||||
|
||||
const LONG_RUNNING_UP_TIMEOUT_MS = 120_000;
|
||||
|
||||
@@ -295,7 +296,7 @@ Examples:
|
||||
console.error(`'${source}' is ambiguous — it matches both an existing rig restore target and a library spec.`);
|
||||
console.error(` To launch the library spec: rig up ${entry.sourcePath}`);
|
||||
console.error(` The rig-name match refers to a stopped rig / snapshot-backed restore path.`);
|
||||
console.error(` To power on the existing rig: rename or remove the library spec first, then retry.`);
|
||||
console.error(` To recover the existing rig instead of importing a starter: rig up ${source} --existing`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -442,7 +443,15 @@ Examples:
|
||||
|
||||
if (res.status >= 400) {
|
||||
const code = res.data["code"] as string | undefined;
|
||||
if (code === "cycle_error") {
|
||||
const error = res.data["error"] as Partial<ThreePartRejection> | null | undefined;
|
||||
if (error && typeof error === "object" && typeof error.fact === "string"
|
||||
&& typeof error.consequence === "string" && typeof error.action === "string") {
|
||||
for (const line of formatThreePart(error as ThreePartRejection)) console.error(line);
|
||||
const nodes = res.data["attentionNodes"] as Array<{ logicalId: string; sessionName?: string; reason: string }> | undefined;
|
||||
for (const node of nodes ?? []) {
|
||||
console.error(` ${node.logicalId}${node.sessionName ? ` (${node.sessionName})` : ""}: ${node.reason}`);
|
||||
}
|
||||
} else if (code === "cycle_error") {
|
||||
console.error("Cycle detected in rig topology. Check edge definitions for circular dependencies.");
|
||||
} else if (code === "validation_failed") {
|
||||
const errors = (res.data["errors"] as string[]) ?? [];
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { Command } from "commander";
|
||||
import { upCommand } from "../src/commands/up.js";
|
||||
import { STATE_FILE, type LifecycleDeps } from "../src/daemon-lifecycle.js";
|
||||
import type { DaemonClient } from "../src/client.js";
|
||||
import { buildAttentionResponse } from "../../daemon/src/routes/up.js";
|
||||
|
||||
const attentionNodes = [
|
||||
{ logicalId: "dev.owner", sessionName: "dev-owner@first", reason: "update_gate", evidence: "Update available!" },
|
||||
{ logicalId: "dev.checker", sessionName: "dev-checker@first", reason: "returned_to_shell", evidence: "$" },
|
||||
];
|
||||
function attentionBody() {
|
||||
const result = { rigId: "first", stages: [{ stage: "import_rig", status: "blocked",
|
||||
detail: { code: "attention_required", message: "2 members need attention.", attentionNodes } }] };
|
||||
return { ...result, ...buildAttentionResponse(result)! };
|
||||
}
|
||||
|
||||
// Execute the actual command and daemon response builder; transport/lifecycle
|
||||
// are injected. No listener, daemon, tmux or provider process is started.
|
||||
async function render(data: Record<string, unknown>, status = 409, json = false) {
|
||||
const oldExit = process.exitCode;
|
||||
const out: string[] = [], err: string[] = [];
|
||||
const log = vi.spyOn(console, "log").mockImplementation((...s) => { out.push(s.join(" ")); });
|
||||
const error = vi.spyOn(console, "error").mockImplementation((...s) => { err.push(s.join(" ")); });
|
||||
const forbidden = () => { throw new Error("unexpected lifecycle operation"); };
|
||||
const lifecycleDeps = {
|
||||
spawn: forbidden, kill: forbidden, writeFile: forbidden, removeFile: forbidden,
|
||||
mkdirp: forbidden, openForAppend: forbidden,
|
||||
fetch: async () => ({ ok: true }), isProcessAlive: () => true,
|
||||
exists: (p: string) => p === STATE_FILE,
|
||||
readFile: (p: string) => p === STATE_FILE ? JSON.stringify({ pid: 123, port: 12345, db: "mock", startedAt: "2000-01-01" }) : null,
|
||||
} as LifecycleDeps;
|
||||
const client = { get: vi.fn(async () => ({ status: 200, data: [] })),
|
||||
post: vi.fn(async () => ({ status, data })) };
|
||||
process.exitCode = undefined;
|
||||
try {
|
||||
const cmd = new Command().exitOverride().addCommand(upCommand({
|
||||
lifecycleDeps, clientFactory: () => client as unknown as DaemonClient,
|
||||
preflightExec: forbidden,
|
||||
}));
|
||||
await cmd.parseAsync(["node", "rig", "up", "first-project", ...(json ? ["--json"] : [])]);
|
||||
expect(client.post).toHaveBeenCalledTimes(1);
|
||||
return { out: out.join("\n"), err: err.join("\n"), exit: process.exitCode };
|
||||
} finally { process.exitCode = oldExit; log.mockRestore(); error.mockRestore(); }
|
||||
}
|
||||
|
||||
describe("startup attention guidance", () => {
|
||||
it("renders actual structured response and all affected members without spec advice", async () => {
|
||||
const body = attentionBody();
|
||||
const r = await render(body);
|
||||
expect(r.err).toContain(`Error: ${body.error.fact}`);
|
||||
expect(r.err).toContain(body.error.consequence);
|
||||
expect(r.err).toContain(body.error.action);
|
||||
for (const n of attentionNodes) {
|
||||
expect(r.err).toContain(n.logicalId);
|
||||
expect(r.err).toContain(n.sessionName);
|
||||
expect(r.err).toContain(n.reason);
|
||||
}
|
||||
expect(r.err).not.toMatch(/\[object Object\]|validate your spec|Sessions are running|answering it.*completes/);
|
||||
expect(r.out).toContain("import_rig: blocked");
|
||||
expect(r.exit).toBe(1);
|
||||
});
|
||||
it("daemon guidance does not assume a trust prompt or a live runtime", () => {
|
||||
const { error } = attentionBody();
|
||||
expect(error.consequence).toContain("attention_required");
|
||||
expect(error.consequence).toMatch(/not.*proven|not.*confirmed/i);
|
||||
expect(error.action).toMatch(/inspect/i);
|
||||
expect(error.action).not.toMatch(/answer.*prompt|parked session/i);
|
||||
});
|
||||
it.each([409, 500])("preserves exact JSON body and exit for HTTP %i", async (status) => {
|
||||
const data = attentionBody();
|
||||
const r = await render(data, status, true);
|
||||
expect(r.out).toBe(JSON.stringify(data));
|
||||
expect(r.err).toBe("");
|
||||
expect(r.exit).toBe(status === 409 ? 1 : 2);
|
||||
});
|
||||
it.each([409, 500])("preserves string fallback and exit for HTTP %i", async (status) => {
|
||||
const r = await render({ error: "agent_ref resolution failed", stages: [] }, status);
|
||||
expect(r.err).toContain(`Up failed: agent_ref resolution failed (HTTP ${status})`);
|
||||
expect(r.err).toContain("local: agent_ref paths resolve relative");
|
||||
expect(r.exit).toBe(status === 409 ? 1 : 2);
|
||||
});
|
||||
});
|
||||
@@ -877,10 +877,12 @@ describe("Up CLI", () => {
|
||||
|
||||
it("up with library name matching existing rig shows ambiguity error", async () => {
|
||||
// Mock server that has both a library spec and an existing rig named "my-rig"
|
||||
let startupRequests = 0;
|
||||
const origListeners = server.listeners("request");
|
||||
server.removeAllListeners("request");
|
||||
server.on("request", (req: http.IncomingMessage, res: http.ServerResponse) => {
|
||||
const url = decodeURIComponent(req.url ?? "");
|
||||
if (url === "/api/up") startupRequests++;
|
||||
if (url.startsWith("/api/specs/library")) {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify([{ id: "lib1", name: "alpha", sourcePath: "/specs/alpha.yaml" }]));
|
||||
@@ -903,6 +905,9 @@ describe("Up CLI", () => {
|
||||
expect(logs.join("\n")).toContain("ambiguous");
|
||||
expect(logs.join("\n")).toContain("existing rig restore target");
|
||||
expect(logs.join("\n")).toContain("/specs/alpha.yaml");
|
||||
expect(logs.join("\n")).toContain("rig up alpha --existing");
|
||||
expect(logs.join("\n")).not.toContain("rename or remove");
|
||||
expect(startupRequests).toBe(0);
|
||||
expect(exitCode).toBe(1);
|
||||
});
|
||||
|
||||
@@ -942,7 +947,7 @@ describe("Up CLI", () => {
|
||||
expect(exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("up --existing bypasses library-name ambiguity and posts the rig name", async () => {
|
||||
it.each([false, true])("up --existing bypasses library-name ambiguity with plan=%s", async (plan) => {
|
||||
const origListeners = server.listeners("request");
|
||||
let lastBody: Record<string, unknown> = {};
|
||||
server.removeAllListeners("request");
|
||||
@@ -963,13 +968,14 @@ describe("Up CLI", () => {
|
||||
});
|
||||
|
||||
const { logs, exitCode } = await captureLogs(async () => {
|
||||
await makeCmd().parseAsync(["node", "rig", "up", "alpha", "--existing"]);
|
||||
await makeCmd().parseAsync(["node", "rig", "up", "alpha", "--existing", ...(plan ? ["--plan"] : [])]);
|
||||
});
|
||||
|
||||
server.removeAllListeners("request");
|
||||
for (const l of origListeners) server.on("request", l as (...args: unknown[]) => void);
|
||||
|
||||
expect(lastBody.sourceRef).toBe("alpha");
|
||||
expect(lastBody.plan).toBe(plan);
|
||||
expect(logs.join("\n")).toContain('Recovering rig "alpha" from latest snapshot or current DB state');
|
||||
expect(exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -2,8 +2,6 @@ import nodePath from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import os from "node:os";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import Database from "better-sqlite3";
|
||||
import { parse as parseToml } from "smol-toml";
|
||||
import type { TmuxAdapter } from "./tmux.js";
|
||||
@@ -28,9 +26,18 @@ import { assessNativeResumeProbe, buildCodexResumeCore, type NativeResumeProbeRe
|
||||
import { mergeManagedBlock } from "../domain/managed-blocks.js";
|
||||
import { parseSessionName } from "../domain/session-name.js";
|
||||
import { shellQuote } from "./shell-quote.js";
|
||||
import { runSyncSite, runAsyncSite } from "../domain/sync-site-wrap.js";
|
||||
import { runSyncSite } from "../domain/sync-site-wrap.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
import { listNativeProcesses, observeCodexPaneProcess, type NativeProcessRow } from "../domain/native-process-lineage.js";
|
||||
|
||||
// Shared by all probes of ONE launch, never reset by a delayed screen or an
|
||||
// ambiguous transport result. A separately requested launch gets a new attempt.
|
||||
interface UpdatePromptAttempt {
|
||||
handled: boolean;
|
||||
failure?: Extract<HarnessLaunchResult, { ok: false }>;
|
||||
}
|
||||
|
||||
type CodexProcess = NativeProcessRow;
|
||||
|
||||
export interface CodexAdapterFsOps {
|
||||
readFile(path: string): string;
|
||||
@@ -53,7 +60,7 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
readonly runtime = "codex";
|
||||
private tmux: TmuxAdapter;
|
||||
private fs: CodexAdapterFsOps;
|
||||
private listProcesses: () => Array<{ pid: number; ppid: number; command: string }> | Promise<Array<{ pid: number; ppid: number; command: string }>>;
|
||||
private listProcesses: () => CodexProcess[] | Promise<CodexProcess[]>;
|
||||
private readThreadIdByPid: (pid: number) => Promise<string | undefined> | string | undefined;
|
||||
private sleep: (ms: number) => Promise<void>;
|
||||
private resolveHomeDirByPid: ResolveHomeDirByPid;
|
||||
@@ -75,7 +82,7 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
constructor(deps: {
|
||||
tmux: TmuxAdapter;
|
||||
fsOps: CodexAdapterFsOps;
|
||||
listProcesses?: () => Array<{ pid: number; ppid: number; command: string }> | Promise<Array<{ pid: number; ppid: number; command: string }>>;
|
||||
listProcesses?: () => CodexProcess[] | Promise<CodexProcess[]>;
|
||||
readThreadIdByPid?: (pid: number) => Promise<string | undefined> | string | undefined;
|
||||
resolveHomeDirByPid?: ResolveHomeDirByPid;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
@@ -313,6 +320,7 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
return { ok: false, error: "resumeToken and forkSource are mutually exclusive — pick one" };
|
||||
}
|
||||
|
||||
const updatePrompt: UpdatePromptAttempt = { handled: false };
|
||||
const model = binding.model?.trim();
|
||||
const modelArg = model ? ` -m ${shellQuote(model)}` : "";
|
||||
const profile = binding.codexConfigProfile?.trim();
|
||||
@@ -359,8 +367,10 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
if (!textResult.ok) {
|
||||
return { ok: false, error: `Failed to send launch command: ${textResult.message}` };
|
||||
}
|
||||
await this.dismissCodexInteractiveGates(binding.tmuxSession);
|
||||
const threadId = await this.captureFreshThreadId(binding);
|
||||
await this.dismissSkippableCodexUpdatePrompt(binding.tmuxSession, updatePrompt, 8);
|
||||
if (updatePrompt.failure) return updatePrompt.failure;
|
||||
const threadId = await this.captureFreshThreadId(binding, updatePrompt);
|
||||
if (updatePrompt.failure) return updatePrompt.failure;
|
||||
if (!threadId) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -384,15 +394,17 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
return { ok: false, error: `Failed to send launch command: ${textResult.message}` };
|
||||
}
|
||||
|
||||
await this.dismissSkippableCodexUpdatePrompt(binding.tmuxSession);
|
||||
await this.dismissSkippableCodexUpdatePrompt(binding.tmuxSession, updatePrompt);
|
||||
if (updatePrompt.failure) return updatePrompt.failure;
|
||||
|
||||
if (opts.resumeToken) {
|
||||
const verification = await this.verifyResumeLaunch(binding.tmuxSession, { resumeToken: opts.resumeToken });
|
||||
const verification = await this.verifyResumeLaunch(binding.tmuxSession, updatePrompt, { resumeToken: opts.resumeToken });
|
||||
if (!verification.ok) return verification;
|
||||
return { ok: true, resumeToken: opts.resumeToken, resumeType: "codex_id", appliedLaunch };
|
||||
}
|
||||
|
||||
const threadId = await this.captureFreshThreadId(binding);
|
||||
const threadId = await this.captureFreshThreadId(binding, updatePrompt);
|
||||
if (updatePrompt.failure) return updatePrompt.failure;
|
||||
if (threadId) {
|
||||
return { ok: true, resumeToken: threadId, resumeType: "codex_id", appliedLaunch };
|
||||
}
|
||||
@@ -439,72 +451,66 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
return { ready: false, reason: probe.detail, code: probe.code };
|
||||
}
|
||||
|
||||
private async dismissSkippableCodexUpdatePrompt(tmuxSession: string, attempts = 6): Promise<boolean> {
|
||||
private async dismissSkippableCodexUpdatePrompt(
|
||||
tmuxSession: string, updatePrompt: UpdatePromptAttempt, attempts = 6,
|
||||
): Promise<boolean> {
|
||||
for (let attempt = 0; attempt < attempts; attempt++) {
|
||||
const paneCommand = await this.tmux.getPaneCommand(tmuxSession);
|
||||
const paneContent = await this.captureProbeScreen(tmuxSession);
|
||||
const probe = assessNativeResumeProbe({
|
||||
runtime: "codex",
|
||||
paneCommand,
|
||||
paneContent,
|
||||
});
|
||||
const probe = assessNativeResumeProbe({ runtime: "codex", paneCommand, paneContent });
|
||||
|
||||
if (probe.code === "update_gate") {
|
||||
if (!isSkippableCodexUpdatePrompt(paneContent)) return false;
|
||||
if (updatePrompt.handled || !isSkippableCodexUpdatePrompt(paneContent)) return false;
|
||||
const identity = await this.observeMenuProcess(tmuxSession, paneCommand);
|
||||
if (!identity) return false;
|
||||
// The npm launcher may be foreground Node with a native Codex child.
|
||||
// Recheck identity and CURRENT screen after sampling, never scrollback.
|
||||
const currentCommand = await this.tmux.getPaneCommand(tmuxSession);
|
||||
if (await this.observeMenuProcess(tmuxSession, currentCommand) !== identity) {
|
||||
updatePrompt.handled = true;
|
||||
return false;
|
||||
}
|
||||
const currentScreen = await this.tmux.capturePaneScreen?.(tmuxSession);
|
||||
if (!currentScreen || !isSkippableCodexUpdatePrompt(currentScreen)
|
||||
|| assessNativeResumeProbe({ runtime: "codex", paneCommand: currentCommand, paneContent: currentScreen }).code !== "update_gate") {
|
||||
updatePrompt.handled = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const textResult = await this.tmux.sendText(tmuxSession, "3");
|
||||
if (!textResult.ok) return false;
|
||||
const enterResult = await this.tmux.sendKeys(tmuxSession, ["Enter"]);
|
||||
if (!enterResult.ok) return false;
|
||||
// Codex 0.155.1 ignores Paste; Key3 selects AND submits DontRemind
|
||||
// (including its version-cache write). An Enter would hit the next
|
||||
// screen. Consume the attempt before sending, even if delivery fails.
|
||||
updatePrompt.handled = true;
|
||||
const result = await this.tmux.sendKeys(tmuxSession, ["3"]);
|
||||
if (!result.ok) {
|
||||
updatePrompt.failure = {
|
||||
ok: false, recovery: "attention_required",
|
||||
error: `Could not skip the Codex update prompt: ${result.message}. Inspect the session before retrying.`,
|
||||
evidence: paneContent.split("\n").slice(-12).join("\n"),
|
||||
};
|
||||
return false;
|
||||
}
|
||||
await this.sleep(500);
|
||||
return true;
|
||||
continue; // Observe transition; never send a second choice on this launch.
|
||||
}
|
||||
|
||||
if (probe.status === "resumed" || probe.code === "trust_gate") {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (attempt < attempts - 1) {
|
||||
await this.sleep(200);
|
||||
// Readiness or another native decision closes update automation. A later
|
||||
// stale capture must not reopen it. Trust/auth decisions remain in-pane.
|
||||
if (probe.status === "resumed" || probe.status === "attention_required"
|
||||
|| probe.code === "trust_gate" || probe.code === "hook_trust_gate") {
|
||||
updatePrompt.handled = true;
|
||||
return probe.status === "resumed";
|
||||
}
|
||||
if (attempt < attempts - 1) await this.sleep(200);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async dismissCodexInteractiveGates(tmuxSession: string, attempts = 8): Promise<void> {
|
||||
for (let attempt = 0; attempt < attempts; attempt++) {
|
||||
const paneCommand = await this.tmux.getPaneCommand(tmuxSession);
|
||||
const paneContent = await this.captureProbeScreen(tmuxSession);
|
||||
const probe = assessNativeResumeProbe({
|
||||
runtime: "codex",
|
||||
paneCommand,
|
||||
paneContent,
|
||||
});
|
||||
|
||||
if (probe.code === "update_gate") {
|
||||
if (!isSkippableCodexUpdatePrompt(paneContent)) return;
|
||||
const textResult = await this.tmux.sendText(tmuxSession, "3");
|
||||
if (!textResult.ok) return;
|
||||
const enterResult = await this.tmux.sendKeys(tmuxSession, ["Enter"]);
|
||||
if (!enterResult.ok) return;
|
||||
await this.sleep(500);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Activity hooks are provisioned by exact authored hash above. A remaining
|
||||
// review can include unrelated hooks or a changed native UI; never type
|
||||
// a blanket trust choice into it. The TUI exposes the native decision.
|
||||
if (probe.code === "hook_trust_gate") return;
|
||||
|
||||
if (probe.status === "resumed" || probe.code === "trust_gate") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (attempt < attempts - 1) {
|
||||
await this.sleep(200);
|
||||
}
|
||||
}
|
||||
private async observeMenuProcess(target: string, paneCommand: string | null): Promise<string | null> {
|
||||
// tmux may name the shell wrapper; native ancestry and foreground group decide identity.
|
||||
if (!paneCommand) return null;
|
||||
const observation = await observeCodexPaneProcess({ target, tmux: this.tmux, listProcesses: this.listProcesses });
|
||||
return observation ? JSON.stringify([paneCommand, observation.fingerprint]) : null;
|
||||
}
|
||||
|
||||
ensureManagedBootstrap(binding: { cwd?: string | null }): void {
|
||||
@@ -700,7 +706,7 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
return Array.from(keys);
|
||||
}
|
||||
|
||||
private async captureFreshThreadId(binding: NodeBinding): Promise<string | undefined> {
|
||||
private async captureFreshThreadId(binding: NodeBinding, updatePrompt: UpdatePromptAttempt): Promise<string | undefined> {
|
||||
const target = binding.tmuxPane ?? binding.tmuxSession;
|
||||
if (!target || !this.tmux.getPanePid) return undefined;
|
||||
|
||||
@@ -714,7 +720,8 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
}
|
||||
}
|
||||
if (binding.tmuxSession) {
|
||||
await this.dismissCodexInteractiveGates(binding.tmuxSession, 1);
|
||||
await this.dismissSkippableCodexUpdatePrompt(binding.tmuxSession, updatePrompt, 1);
|
||||
if (updatePrompt.failure) return undefined;
|
||||
}
|
||||
await this.sleep(250);
|
||||
}
|
||||
@@ -722,7 +729,7 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private async verifyResumeLaunch(tmuxSession: string, opts?: { resumeToken?: string }): Promise<HarnessLaunchResult> {
|
||||
private async verifyResumeLaunch(tmuxSession: string, updatePrompt: UpdatePromptAttempt, opts?: { resumeToken?: string }): Promise<HarnessLaunchResult> {
|
||||
const quickAttempts = 6;
|
||||
const extendedAttempts = 24;
|
||||
const quickSleepMs = 200;
|
||||
@@ -794,7 +801,8 @@ export class CodexRuntimeAdapter implements RuntimeAdapter {
|
||||
}
|
||||
|
||||
if (probe.code === "update_gate") {
|
||||
const dismissed = await this.dismissSkippableCodexUpdatePrompt(tmuxSession, 1);
|
||||
const dismissed = await this.dismissSkippableCodexUpdatePrompt(tmuxSession, updatePrompt, 1);
|
||||
if (updatePrompt.failure) return updatePrompt.failure;
|
||||
if (dismissed) {
|
||||
lastUnresolved = null;
|
||||
sawRealGate = false;
|
||||
@@ -1434,30 +1442,8 @@ async function defaultProfilePreflight(profile: string): Promise<CodexProfilePro
|
||||
// Exported for unit test (B12-T): the REAL async sampling path — the anti-vacuity test drives
|
||||
// this default directly (every other suite injects sync stubs) and asserts the non-blocking
|
||||
// property that the pre-B12 sync implementation violated.
|
||||
export async function defaultListProcesses(): Promise<Array<{ pid: number; ppid: number; command: string }>> {
|
||||
try {
|
||||
const output = await runAsyncSite("codex.runtime.list_processes", async () => {
|
||||
const { stdout } = await execFileAsync("ps", ["-Ao", "pid,ppid,command"], { encoding: "utf-8", maxBuffer: 8 * 1024 * 1024 });
|
||||
return stdout;
|
||||
});
|
||||
return output
|
||||
.split("\n")
|
||||
.slice(1)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const match = line.match(/^(\d+)\s+(\d+)\s+(.*)$/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
pid: Number(match[1]),
|
||||
ppid: Number(match[2]),
|
||||
command: match[3] ?? "",
|
||||
};
|
||||
})
|
||||
.filter((row): row is { pid: number; ppid: number; command: string } => row !== null);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
export async function defaultListProcesses(): Promise<CodexProcess[]> {
|
||||
return listNativeProcesses();
|
||||
}
|
||||
|
||||
function findCodexDescendantPids(
|
||||
@@ -1496,5 +1482,5 @@ function commandLooksLikeCodex(command: string): boolean {
|
||||
|
||||
function isSkippableCodexUpdatePrompt(paneContent: string): boolean {
|
||||
return paneContent.includes("Update available!")
|
||||
&& paneContent.includes("Skip until next version");
|
||||
&& /^\s*[›>]?\s*3\. Skip until next version\s*$/m.test(paneContent);
|
||||
}
|
||||
@@ -680,7 +680,7 @@ export class BootstrapOrchestrator {
|
||||
|
||||
// OPR.0.3.2.CT (guard verdict qitem-20260518082933 BLOCKER 1):
|
||||
// attention_required nodes are recoverable but NOT done — the
|
||||
// launch is partial and the operator needs the same approve→resume
|
||||
// launch is partial and the operator needs the same inspection
|
||||
// surface the all-attention path already gets. Treating mixed
|
||||
// launched+attention as "completed" would hide the parked seat
|
||||
// behind a 201 success response. finalStatus is partial whenever
|
||||
@@ -699,7 +699,7 @@ export class BootstrapOrchestrator {
|
||||
evidence: n.evidence,
|
||||
reason: n.error ?? "node awaiting attention",
|
||||
}));
|
||||
const message = `${attentionNodes.length} node${attentionNodes.length === 1 ? "" : "s"} require attention before becoming interactive (rig parked, NOT failed; approve and resume to proceed).`;
|
||||
const message = `${attentionNodes.length} node${attentionNodes.length === 1 ? " requires" : "s require"} attention before becoming interactive. Inspect the affected sessions and reasons before choosing recovery.`;
|
||||
stages.push({
|
||||
stage: "import_rig",
|
||||
status: "blocked",
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { runAsyncSite } from "./sync-site-wrap.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface NativeProcessRow {
|
||||
pid: number;
|
||||
ppid: number;
|
||||
command: string;
|
||||
pgid?: number;
|
||||
tpgid?: number;
|
||||
executableName?: string;
|
||||
startedAt?: string;
|
||||
}
|
||||
|
||||
export type NativeRuntime = "claude-code" | "codex";
|
||||
@@ -29,6 +39,11 @@ function commandUsesExpectedToken(command: string, runtime: NativeRuntime, expec
|
||||
return false;
|
||||
}
|
||||
|
||||
return codexResumeToken(args) === expectedToken;
|
||||
}
|
||||
|
||||
// undefined is a fresh command; null is a resume command without an exact token.
|
||||
function codexResumeToken(args: string[]): string | null | undefined {
|
||||
const topLevelOptionsWithValues = new Set([
|
||||
"-a", "--ask-for-approval", "-c", "--config", "-m", "--model",
|
||||
"-p", "--profile", "-s", "--sandbox",
|
||||
@@ -41,16 +56,16 @@ function commandUsesExpectedToken(command: string, runtime: NativeRuntime, expec
|
||||
if (arg === "resume") resumeIndex = index;
|
||||
break;
|
||||
}
|
||||
if (resumeIndex < 0) return false;
|
||||
if (resumeIndex < 0) return undefined;
|
||||
const resumeArgs = args.slice(resumeIndex + 1);
|
||||
let index = 0;
|
||||
while (index < resumeArgs.length) {
|
||||
const arg = resumeArgs[index]!;
|
||||
if (arg === "--add-dir") { index += 2; continue; }
|
||||
if (arg.startsWith("-")) { index += 1; continue; }
|
||||
return arg === expectedToken;
|
||||
return arg;
|
||||
}
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Require a live process in the pane's own lineage whose argv names both the
|
||||
@@ -61,7 +76,8 @@ export function findExactNativeResumeProcess(
|
||||
runtime: string | null,
|
||||
expectedToken: string,
|
||||
): NativeProcessRow | null {
|
||||
if (runtime !== "claude-code" && runtime !== "codex") return null;
|
||||
if (runtime === "codex") return selectCodexProcess(processes, panePid, expectedToken, true)?.process ?? null;
|
||||
if (runtime !== "claude-code") return null;
|
||||
const byParent = new Map<number, NativeProcessRow[]>();
|
||||
for (const process of processes) {
|
||||
const children = byParent.get(process.ppid) ?? [];
|
||||
@@ -81,3 +97,70 @@ export function findExactNativeResumeProcess(
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The same OS observation serves menu input, restore proof and periodic identity.
|
||||
* Older callers may carry only pid/ppid/command; that is insufficient positive Codex proof. */
|
||||
export async function listNativeProcesses(): Promise<NativeProcessRow[]> {
|
||||
try {
|
||||
const output = await runAsyncSite("codex.runtime.list_processes", async () => {
|
||||
const { stdout } = await execFileAsync("ps", ["-Ao", "pid,ppid,pgid,tpgid,ucomm,lstart,command"], { encoding: "utf-8", maxBuffer: 8 * 1024 * 1024 });
|
||||
return stdout;
|
||||
});
|
||||
return output.split("\n").slice(1).flatMap((line) => {
|
||||
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(-?\d+)\s+(-?\d+)\s+(\S+)\s+(\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.+)$/);
|
||||
return match ? [{ pid: Number(match[1]), ppid: Number(match[2]), pgid: Number(match[3]), tpgid: Number(match[4]), executableName: match[5]!, startedAt: match[6]!, command: match[7]! }] : [];
|
||||
});
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
export type NativeProcessLister = () => NativeProcessRow[] | Promise<NativeProcessRow[]>;
|
||||
export type CodexProcessObservation = { panePid: number; process: NativeProcessRow; fingerprint: string };
|
||||
|
||||
function selectCodexProcess(rows: NativeProcessRow[], panePid: number, expectedToken?: string | null, requireResume = false): CodexProcessObservation | null {
|
||||
const byPid = new Map(rows.map((row) => [row.pid, row]));
|
||||
const root = byPid.get(panePid);
|
||||
if (byPid.size !== rows.length || !root?.startedAt || !root.tpgid || root.tpgid <= 0) return null;
|
||||
const matches: { process: NativeProcessRow; chain: NativeProcessRow[] }[] = [];
|
||||
for (const row of rows) {
|
||||
if (row.executableName !== "codex" || executableName(tokens(row.command)[0] ?? "") !== "codex"
|
||||
|| row.pgid !== root.tpgid || row.tpgid !== root.tpgid) continue;
|
||||
const chain: NativeProcessRow[] = [];
|
||||
const visited = new Set<number>();
|
||||
let current: NativeProcessRow | undefined = row;
|
||||
while (current && !visited.has(current.pid) && current.startedAt) {
|
||||
visited.add(current.pid);
|
||||
chain.push(current);
|
||||
if (current.pid === panePid) { matches.push({ process: row, chain }); break; }
|
||||
current = byPid.get(current.ppid);
|
||||
}
|
||||
}
|
||||
if (matches.length !== 1) return null;
|
||||
const { process, chain } = matches[0]!;
|
||||
const resumeToken = codexResumeToken(tokens(process.command).slice(1));
|
||||
if (requireResume && !expectedToken) return null;
|
||||
if ((requireResume || (expectedToken !== undefined && resumeToken !== undefined))
|
||||
&& (!expectedToken || resumeToken !== expectedToken)) return null;
|
||||
return { panePid, process, fingerprint: JSON.stringify(chain.map((row) => [row.pid, row.ppid, row.startedAt, row.pgid, row.tpgid, row.executableName, row.command])) };
|
||||
}
|
||||
|
||||
export async function observeCodexPaneProcess(input: {
|
||||
target: string;
|
||||
tmux: { getPanePid(target: string): Promise<number | null> };
|
||||
listProcesses?: NativeProcessLister;
|
||||
expectedToken?: string | null;
|
||||
requireResume?: boolean;
|
||||
}): Promise<CodexProcessObservation | null> {
|
||||
try {
|
||||
const pid = await input.tmux.getPanePid(input.target);
|
||||
if (!pid) return null;
|
||||
const rows = await (input.listProcesses ?? listNativeProcesses)();
|
||||
return selectCodexProcess(rows, pid, input.expectedToken, input.requireResume);
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
export async function verifyCodexPaneProcess(input: Parameters<typeof observeCodexPaneProcess>[0]): Promise<CodexProcessObservation | null> {
|
||||
const first = await observeCodexPaneProcess(input);
|
||||
if (!first) return null;
|
||||
const second = await observeCodexPaneProcess(input);
|
||||
return second?.fingerprint === first.fingerprint ? second : null;
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import type {
|
||||
} from "./types.js";
|
||||
import { AppliedLaunchObservationStore } from "./applied-launch-observation-store.js";
|
||||
import { rebindAndVerifyPaneIdentity } from "./seat-attention-reconciler.js";
|
||||
import { SeatIdentityStore } from "./seat-identity-store.js";
|
||||
import { resolveSnapshotRestoreTopology } from "./restore-topology.js";
|
||||
|
||||
// L3: result shape for runtime-truth reconciliation. A reconciliation that
|
||||
@@ -1375,7 +1376,7 @@ export class RestoreOrchestrator {
|
||||
? ((startupResult.continuityOutcome === "resumed" || nativeContinuityProved) ? "resumed" : baseStatus)
|
||||
: baseStatus;
|
||||
if (finalStatus === "resumed") {
|
||||
return this.finishJoinedResume(node, sessionName, resumeToken);
|
||||
return this.finishJoinedResume(node, sessionName, resumeToken, launchResult?.session.id);
|
||||
}
|
||||
return { nodeId: node.id, logicalId: node.logicalId, status: finalStatus };
|
||||
}
|
||||
@@ -1441,7 +1442,7 @@ export class RestoreOrchestrator {
|
||||
}
|
||||
|
||||
if (baseStatus === "resumed") {
|
||||
return this.finishJoinedResume(node, sessionName, resumeToken);
|
||||
return this.finishJoinedResume(node, sessionName, resumeToken, launchResult?.session.id);
|
||||
}
|
||||
return { nodeId: node.id, logicalId: node.logicalId, status: baseStatus };
|
||||
}
|
||||
@@ -1452,6 +1453,7 @@ export class RestoreOrchestrator {
|
||||
node: SnapshotData["nodes"][number],
|
||||
sessionName: string,
|
||||
resumeToken: string | null,
|
||||
sessionId?: string,
|
||||
): Promise<RestoreNodeResult> {
|
||||
const identity = await rebindAndVerifyPaneIdentity({
|
||||
db: this.db,
|
||||
@@ -1472,6 +1474,22 @@ export class RestoreOrchestrator {
|
||||
error: `Exact native session resumed, but joined restore proof is incomplete: ${identity.detail}. The resumed session was preserved; no replacement was started.`,
|
||||
};
|
||||
}
|
||||
// Legacy resume adapters do not write native metadata. Fill only the
|
||||
// launched row's empty token after proof; never overwrite a hook/operator.
|
||||
if (node.runtime === "codex" && sessionId && resumeToken) {
|
||||
const current = this.db.prepare("SELECT node_id, session_name, status, resume_token FROM sessions WHERE id = ?").get(sessionId) as
|
||||
{ node_id: string; session_name: string; status: string; resume_token: string | null } | undefined;
|
||||
const sameSession = current?.node_id === node.id && current.session_name === sessionName && current.status === "running";
|
||||
const retained = sameSession && (current.resume_token === resumeToken
|
||||
|| (!current.resume_token && this.sessionRegistry.updateResumeToken(sessionId, "codex_id", resumeToken, "scrape")));
|
||||
if (!retained) {
|
||||
const store = new SeatIdentityStore(this.db);
|
||||
const proof = store.getForNode(node.id);
|
||||
if (proof) store.upsert({ ...proof, verdict: "mismatch", reason: "process_identity_mismatch" });
|
||||
return { nodeId: node.id, logicalId: node.logicalId, status: "attention_required",
|
||||
error: "Native resume was observed but its current session metadata conflicts or could not be retained; session preserved." };
|
||||
}
|
||||
}
|
||||
return { nodeId: node.id, logicalId: node.logicalId, status: "resumed" };
|
||||
}
|
||||
|
||||
|
||||
@@ -1441,11 +1441,8 @@ export class PodRigInstantiator {
|
||||
|
||||
if (hasAttention && !hasLaunched) {
|
||||
// All-attention_required path — rig + sessions PRESERVED; no
|
||||
// tear-down. The operator's path: attach to a session via
|
||||
// `tmux attach -t <session>` and answer the runtime prompt. NO
|
||||
// new trust primitive
|
||||
// introduced (HG-5); reuses the runtime's shipped in-pane
|
||||
// trust-grant prompt.
|
||||
// tear-down. Inspect the session and reason to distinguish native
|
||||
// decisions from failed/exited runtimes before choosing recovery.
|
||||
const attentionNodes = nodeResults
|
||||
.filter((n) => n.status === "attention_required")
|
||||
.map((n) => ({
|
||||
@@ -1469,7 +1466,7 @@ export class PodRigInstantiator {
|
||||
return {
|
||||
ok: false,
|
||||
code: "attention_required",
|
||||
message: `${attentionNodes.length} node${attentionNodes.length === 1 ? "" : "s"} require attention before becoming interactive (rig parked, NOT failed; approve and resume to proceed).`,
|
||||
message: `${attentionNodes.length} node${attentionNodes.length === 1 ? " requires" : "s require"} attention before becoming interactive. Inspect the affected sessions and reasons before choosing recovery.`,
|
||||
rigId,
|
||||
attentionNodes,
|
||||
};
|
||||
|
||||
@@ -10,10 +10,10 @@ import type { TmuxAdapter } from "../adapters/tmux.js";
|
||||
import { classifyPaneRuntimeMatch } from "./seat-identity-reconciler.js";
|
||||
import { SeatIdentityStore } from "./seat-identity-store.js";
|
||||
import { defaultListProcesses } from "./resume-metadata-refresher.js";
|
||||
import { findExactNativeResumeProcess } from "./native-process-lineage.js";
|
||||
import { verifyCodexPaneProcess, type NativeProcessRow, type NativeProcessLister, findExactNativeResumeProcess } from "./native-process-lineage.js";
|
||||
|
||||
type PaneIdentityTmux = Pick<TmuxAdapter, "listPanes" | "getPanePid" | "getPaneCommand">;
|
||||
type ProcessRow = { pid: number; ppid: number; command: string };
|
||||
type ProcessRow = NativeProcessRow;
|
||||
|
||||
export type PaneIdentityReconcileResult =
|
||||
| { ok: true; pane: string; pid: number; command: string | null }
|
||||
@@ -30,7 +30,7 @@ export async function rebindAndVerifyPaneIdentity(input: {
|
||||
runtime: string | null;
|
||||
expectedResumeToken?: string | null;
|
||||
requireExactResumeLineage?: boolean;
|
||||
listProcesses?: () => Promise<ProcessRow[]>;
|
||||
listProcesses?: NativeProcessLister;
|
||||
now?: () => Date;
|
||||
}): Promise<PaneIdentityReconcileResult> {
|
||||
const observedAt = (input.now ?? (() => new Date()))().toISOString();
|
||||
@@ -69,14 +69,22 @@ export async function rebindAndVerifyPaneIdentity(input: {
|
||||
} catch (error) {
|
||||
return { ok: false, detail: `tmux pane identity lookup failed: ${(error as Error).message}` };
|
||||
}
|
||||
const runtimeMatch = classifyPaneRuntimeMatch(command, input.runtime);
|
||||
let runtimeMatch = classifyPaneRuntimeMatch(command, input.runtime);
|
||||
const normalizedCommand = command?.trim().toLowerCase() ?? "";
|
||||
let lineageMatch: ProcessRow | null = null;
|
||||
const expectedResumeToken = input.expectedResumeToken ?? null;
|
||||
const strictNativeLineage = input.requireExactResumeLineage === true
|
||||
&& expectedResumeToken !== null
|
||||
&& (input.runtime === "claude-code" || input.runtime === "codex");
|
||||
if (pid !== null && runtimeMatch === "match" && strictNativeLineage) {
|
||||
if (input.runtime === "codex") {
|
||||
// A shell/Node label describes the wrapper, not the native occupant.
|
||||
runtimeMatch = "match";
|
||||
const native = await verifyCodexPaneProcess({ target: pane.id, tmux: input.tmux,
|
||||
listProcesses: input.listProcesses, expectedToken: expectedResumeToken,
|
||||
requireResume: input.requireExactResumeLineage === true });
|
||||
const currentPanes = await input.tmux.listPanes(input.sessionName).catch(() => []);
|
||||
if (native?.panePid === pid && currentPanes.length === 1 && currentPanes[0]?.id === pane.id) lineageMatch = native.process;
|
||||
} else if (pid !== null && runtimeMatch === "match" && strictNativeLineage) {
|
||||
try {
|
||||
lineageMatch = findExactNativeResumeProcess(
|
||||
await (input.listProcesses ?? defaultListProcesses)(),
|
||||
@@ -88,10 +96,9 @@ export async function rebindAndVerifyPaneIdentity(input: {
|
||||
// Missing process evidence is ambiguity, never positive identity.
|
||||
}
|
||||
}
|
||||
const runtimeAmbiguous = runtimeMatch === "match" && (strictNativeLineage
|
||||
const runtimeAmbiguous = input.runtime === "codex" ? lineageMatch === null : runtimeMatch === "match" && (strictNativeLineage
|
||||
? lineageMatch === null
|
||||
: (input.runtime === "claude-code" && !normalizedCommand.includes("claude"))
|
||||
|| (input.runtime === "codex" && !normalizedCommand.includes("codex")));
|
||||
: input.runtime === "claude-code" && !normalizedCommand.includes("claude"));
|
||||
const verdict: SeatIdentityVerdict = {
|
||||
nodeId: input.nodeId,
|
||||
verdict: pid === null
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { observeCodexPaneProcess, listNativeProcesses, type NativeProcessLister, type CodexProcessObservation } from "./native-process-lineage.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type Database from "better-sqlite3";
|
||||
import type { TmuxAdapter } from "../adapters/tmux.js";
|
||||
@@ -21,7 +22,8 @@ const SHELL_COMMANDS = new Set([
|
||||
]);
|
||||
|
||||
/**
|
||||
* Classify the pane's foreground command against the registered runtime.
|
||||
* Legacy short-command classifier. Codex callers additionally require the
|
||||
* shared native-process proof; this label alone is not positive Codex identity.
|
||||
*
|
||||
* Deliberately LENIENT to preserve the slice-15 no-false-green-for-live-seats
|
||||
* invariant: a genuinely-live claude/codex TUI usually reports its host
|
||||
@@ -66,12 +68,14 @@ interface RunningSeatRow {
|
||||
runtime: string | null;
|
||||
session_name: string;
|
||||
tmux_pane: string | null;
|
||||
resume_token?: string | null;
|
||||
}
|
||||
|
||||
export interface SeatIdentityReconcilerDeps {
|
||||
db: Database.Database;
|
||||
tmux: Pick<TmuxAdapter, "listSessions" | "getPanePid" | "getPaneCommand">;
|
||||
now?: () => Date;
|
||||
listProcesses?: NativeProcessLister;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -90,6 +94,7 @@ export class SeatIdentityReconciler {
|
||||
private readonly tmux: SeatIdentityReconcilerDeps["tmux"];
|
||||
private readonly now: () => Date;
|
||||
private readonly store: SeatIdentityStore;
|
||||
private readonly listProcesses: NativeProcessLister;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(deps: SeatIdentityReconcilerDeps) {
|
||||
@@ -97,12 +102,13 @@ export class SeatIdentityReconciler {
|
||||
this.tmux = deps.tmux;
|
||||
this.now = deps.now ?? (() => new Date());
|
||||
this.store = new SeatIdentityStore(deps.db);
|
||||
this.listProcesses = deps.listProcesses ?? listNativeProcesses;
|
||||
}
|
||||
|
||||
private runningSeats(): RunningSeatRow[] {
|
||||
return this.db.prepare(`
|
||||
SELECT n.id as node_id, n.runtime as runtime,
|
||||
s.session_name as session_name, b.tmux_pane as tmux_pane
|
||||
s.session_name as session_name, b.tmux_pane as tmux_pane, s.resume_token as resume_token
|
||||
FROM nodes n
|
||||
JOIN sessions s ON s.node_id = n.id
|
||||
AND s.id = (SELECT s2.id FROM sessions s2 WHERE s2.node_id = n.id ORDER BY s2.id DESC LIMIT 1)
|
||||
@@ -125,7 +131,7 @@ export class SeatIdentityReconciler {
|
||||
// One tmux availability probe per poll. If tmux is entirely unreachable
|
||||
// (throws) OR reports zero live sessions while we have running seats in
|
||||
// the DB, treat it as a transient tmux blip and record `tmux_unavailable`
|
||||
// for every seat — NEVER down-rank a live fleet on an infra hiccup.
|
||||
// for legacy runtimes. Codex remains non-green without native evidence.
|
||||
let liveSessions: Set<string> | null = null;
|
||||
try {
|
||||
const sessions = await this.tmux.listSessions();
|
||||
@@ -140,12 +146,26 @@ export class SeatIdentityReconciler {
|
||||
return;
|
||||
}
|
||||
|
||||
// Two fresh process snapshots per sweep, not two ps calls per seat. Each
|
||||
// phase observes every bound pane; the second begins after the first ends.
|
||||
const codexSeats = seats.filter((seat) => seat.runtime === "codex" && seat.tmux_pane && liveSessions.has(seat.session_name));
|
||||
const sample = async () => {
|
||||
let snapshot: ReturnType<NativeProcessLister> | undefined;
|
||||
return Promise.all(codexSeats.map((seat) => observeCodexPaneProcess({
|
||||
target: seat.tmux_pane!, tmux: this.tmux, expectedToken: seat.resume_token,
|
||||
listProcesses: () => snapshot ??= this.listProcesses(),
|
||||
})));
|
||||
};
|
||||
const first = await sample();
|
||||
const second = await sample();
|
||||
const codexProofs = new Map(codexSeats.map((seat, index) => [seat.node_id,
|
||||
first[index] && first[index]?.fingerprint === second[index]?.fingerprint ? second[index]! : null]));
|
||||
for (const seat of seats) {
|
||||
try {
|
||||
this.store.upsert(await this.computeVerdict(seat, liveSessions, observedAt));
|
||||
this.store.upsert(await this.computeVerdict(seat, liveSessions, observedAt, codexProofs.get(seat.node_id) ?? null));
|
||||
} catch {
|
||||
// A single seat's tmux failure must not crash the loop; record it as
|
||||
// an unknown (non-down-ranking) observation.
|
||||
// unavailable observation (non-green for Codex).
|
||||
this.store.upsert(this.tmuxUnavailableVerdict(seat, observedAt));
|
||||
}
|
||||
}
|
||||
@@ -154,7 +174,7 @@ export class SeatIdentityReconciler {
|
||||
private tmuxUnavailableVerdict(seat: RunningSeatRow, observedAt: string): SeatIdentityVerdict {
|
||||
return {
|
||||
nodeId: seat.node_id,
|
||||
verdict: "tmux_unavailable",
|
||||
verdict: seat.runtime === "codex" ? "mismatch" : "tmux_unavailable",
|
||||
evidenceSource: null,
|
||||
reason: "tmux_unavailable",
|
||||
evidence: { registeredPane: seat.tmux_pane, observedPid: null, observedCommand: null, matchedLayer: null },
|
||||
@@ -167,6 +187,7 @@ export class SeatIdentityReconciler {
|
||||
seat: RunningSeatRow,
|
||||
liveSessions: Set<string>,
|
||||
observedAt: string,
|
||||
native: CodexProcessObservation | null,
|
||||
): Promise<SeatIdentityVerdict> {
|
||||
const base = {
|
||||
nodeId: seat.node_id,
|
||||
@@ -176,7 +197,7 @@ export class SeatIdentityReconciler {
|
||||
|
||||
// A null binding pane has two distinct causes. When the target session is
|
||||
// absent, that is a down-ranking missing-session fact. When it is live,
|
||||
// only the binding is absent: named, but non-down-ranking.
|
||||
// only the binding is absent: named, and non-green for Codex.
|
||||
if (!seat.tmux_pane) {
|
||||
if (!liveSessions.has(seat.session_name)) {
|
||||
return {
|
||||
@@ -189,7 +210,7 @@ export class SeatIdentityReconciler {
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
verdict: "binding_absent",
|
||||
verdict: seat.runtime === "codex" ? "mismatch" : "binding_absent",
|
||||
evidenceSource: "tmux_session",
|
||||
reason: "binding_pane_missing",
|
||||
evidence: { registeredPane: null, observedPid: null, observedCommand: null, matchedLayer: null },
|
||||
@@ -211,6 +232,14 @@ export class SeatIdentityReconciler {
|
||||
}
|
||||
|
||||
const command = await this.tmux.getPaneCommand(seat.tmux_pane);
|
||||
if (seat.runtime === "codex") {
|
||||
return {
|
||||
...base, verdict: native?.panePid === pid ? "verified" : "mismatch",
|
||||
evidenceSource: "pane_process", reason: native?.panePid === pid ? null : "process_identity_ambiguous",
|
||||
evidence: { registeredPane: seat.tmux_pane, observedPid: native?.process.pid ?? pid,
|
||||
observedCommand: native?.process.command ?? command, matchedLayer: native?.panePid === pid ? 1 : null },
|
||||
};
|
||||
}
|
||||
const match = classifyPaneRuntimeMatch(command, seat.runtime);
|
||||
if (match === "mismatch") {
|
||||
return {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { NativeProcessLister } from "./native-process-lineage.js";
|
||||
import type Database from "better-sqlite3";
|
||||
import type { RigRepository } from "./rig-repository.js";
|
||||
import type { SessionRegistry } from "./session-registry.js";
|
||||
@@ -43,6 +44,7 @@ export interface SeatLifecycleDeps {
|
||||
sessionRegistry: SessionRegistry;
|
||||
eventBus: EventBus;
|
||||
tmuxAdapter: TmuxAdapter;
|
||||
listProcesses?: NativeProcessLister;
|
||||
nodeLauncher?: NodeLauncher;
|
||||
startupOrchestrator?: StartupOrchestrator;
|
||||
runtimeAdapters?: Record<string, RuntimeAdapter>;
|
||||
@@ -151,6 +153,7 @@ export class SeatLifecycleService {
|
||||
private readonly sessionRegistry: SessionRegistry;
|
||||
private readonly eventBus: EventBus;
|
||||
private readonly tmuxAdapter: TmuxAdapter;
|
||||
private readonly listProcesses?: NativeProcessLister;
|
||||
private readonly nodeLauncher: NodeLauncher | null;
|
||||
private readonly startupOrchestrator: StartupOrchestrator | null;
|
||||
private readonly runtimeAdapters: Record<string, RuntimeAdapter>;
|
||||
@@ -166,6 +169,7 @@ export class SeatLifecycleService {
|
||||
this.sessionRegistry = deps.sessionRegistry;
|
||||
this.eventBus = deps.eventBus;
|
||||
this.tmuxAdapter = deps.tmuxAdapter;
|
||||
this.listProcesses = deps.listProcesses;
|
||||
this.nodeLauncher = deps.nodeLauncher ?? null;
|
||||
this.startupOrchestrator = deps.startupOrchestrator ?? null;
|
||||
this.runtimeAdapters = deps.runtimeAdapters ?? {};
|
||||
@@ -610,6 +614,7 @@ export class SeatLifecycleService {
|
||||
sessionName: canonicalSessionName,
|
||||
runtime: node.runtime,
|
||||
expectedResumeToken: this.sessionResumeToken(launch.session.id),
|
||||
listProcesses: this.listProcesses,
|
||||
});
|
||||
const attentionRequired = !startupResult.ok || !identity.ok;
|
||||
if (!identity.ok) this.sessionRegistry.updateStartupStatus(launch.session.id, "attention_required");
|
||||
|
||||
@@ -60,10 +60,10 @@ export function buildAttentionResponse(result: {
|
||||
return {
|
||||
error: {
|
||||
fact: detail.message,
|
||||
consequence: `Rig ${rigIdDisplay} is created and listable via \`rig ps\`. Sessions are running with startup_status='attention_required'. Tmux panes show the runtime's trust/approval prompt — answering it in-pane completes the launch.`,
|
||||
consequence: `Rig ${rigIdDisplay} is created and listable via \`rig ps\`. Members marked attention_required have not been proven interactive; a runtime may be waiting for input or may have exited.`,
|
||||
action: nodeCount === 1
|
||||
? `Attach to the session and answer the prompt: ${attachHintText}.`
|
||||
: `Attach to each parked session listed in attentionNodes and answer its prompt: ${attachHintText}.`,
|
||||
? `Inspect the affected session and its reported reason before choosing recovery: ${attachHintText}.`
|
||||
: `Inspect each affected session listed in attentionNodes and its reported reason before choosing recovery: ${attachHintText}.`,
|
||||
},
|
||||
attentionNodes: detail.attentionNodes,
|
||||
};
|
||||
@@ -414,7 +414,7 @@ upRoutes.post("/", async (c) => {
|
||||
// path from the instantiator's new outcome variant OR the
|
||||
// mixed launched+attention path the orchestrator routes the
|
||||
// same way), surface a 3-part error so the operator sees the
|
||||
// actionable approve→resume path. The rig + sessions are
|
||||
// affected sessions and inspect their actual state. The rig + sessions are
|
||||
// PRESERVED on disk; `rig ps` lists them. PRD HG-4.
|
||||
const attentionResponse = buildAttentionResponse(result);
|
||||
if (attentionResponse) {
|
||||
|
||||
@@ -1384,8 +1384,10 @@ edges: []
|
||||
const importStage = result.stages.find((s) => s.stage === "import_rig");
|
||||
expect(importStage).toBeDefined();
|
||||
expect(importStage!.status).toBe("blocked");
|
||||
const detail = importStage!.detail as { code: string; attentionNodes: Array<{ logicalId: string; sessionName: string; evidence?: string }> };
|
||||
const detail = importStage!.detail as { code: string; message: string; attentionNodes: Array<{ logicalId: string; sessionName: string; evidence?: string }> };
|
||||
expect(detail.code).toBe("attention_required");
|
||||
expect(detail.message).toMatch(/inspect/i);
|
||||
expect(detail.message).not.toMatch(/approve and resume|NOT failed/);
|
||||
expect(detail.attentionNodes.length).toBe(1);
|
||||
expect(detail.attentionNodes[0]!.logicalId).toBe("dev.qa");
|
||||
expect(detail.attentionNodes[0]!.sessionName).toBe("dev-qa@pod-mixed-rig");
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { defaultListProcesses } from "../src/adapters/codex-runtime-adapter.js";
|
||||
|
||||
let scratch: string | undefined;
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
if (scratch) fs.rmSync(scratch, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("reads foreground identity through the real async executable and parser path", async () => {
|
||||
scratch = fs.mkdtempSync(path.join(os.tmpdir(), "codex-process-read-"));
|
||||
// A hermetic ps executable; no provider, tmux or daemon is launched.
|
||||
const output = [
|
||||
"PID PPID PGID TPGID UCOMM LSTART COMMAND",
|
||||
"101 1 101 102 zsh Sat Jan 1 12:00:00 2000 -zsh",
|
||||
"102 101 102 102 node Sat Jan 1 12:00:00 2000 node /opt/bin/codex -C /project",
|
||||
"103 102 102 102 codex Sat Jan 1 12:00:00 2000 /opt/vendor/bin/codex -C /project",
|
||||
"104 1 104 -1 node Sat Jan 1 12:00:00 2000 node /tmp/other.js --label codex",
|
||||
"broken row",
|
||||
"105 1 105 0",
|
||||
];
|
||||
const quote = (s: string) => `'${s.replaceAll("'", "'\\''")}'`;
|
||||
fs.writeFileSync(path.join(scratch, "ps"), "#!/bin/sh\n"
|
||||
+ `printf '%s\\n' "$@" > ${quote(path.join(scratch, "args"))}\n`
|
||||
+ `printf '%s\\n' ${output.map(quote).join(" ")}\n`, { mode: 0o755 });
|
||||
vi.stubEnv("PATH", scratch);
|
||||
const rows = await defaultListProcesses();
|
||||
expect(fs.readFileSync(path.join(scratch, "args"), "utf8")).toBe("-Ao\npid,ppid,pgid,tpgid,ucomm,lstart,command\n");
|
||||
expect(rows).toEqual([
|
||||
{ pid: 101, ppid: 1, pgid: 101, tpgid: 102, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000", command: "-zsh" },
|
||||
{ pid: 102, ppid: 101, pgid: 102, tpgid: 102, executableName: "node", startedAt: "Sat Jan 1 12:00:00 2000", command: "node /opt/bin/codex -C /project" },
|
||||
{ pid: 103, ppid: 102, pgid: 102, tpgid: 102, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000", command: "/opt/vendor/bin/codex -C /project" },
|
||||
{ pid: 104, ppid: 1, pgid: 104, tpgid: -1, executableName: "node", startedAt: "Sat Jan 1 12:00:00 2000", command: "node /tmp/other.js --label codex" },
|
||||
]);
|
||||
fs.writeFileSync(path.join(scratch, "ps"), "#!/bin/sh\nexit 1\n", { mode: 0o755 });
|
||||
expect(await defaultListProcesses()).toEqual([]);
|
||||
});
|
||||
|
||||
it("reads this test process with the installed ps columns, without starting a provider", async () => {
|
||||
const rows = await defaultListProcesses();
|
||||
const self = rows.find((r) => r.pid === process.pid);
|
||||
expect(self).toBeDefined();
|
||||
expect(self!.ppid).toBe(process.ppid);
|
||||
expect(self!.pgid).toBeGreaterThan(0);
|
||||
expect(Number.isInteger(self!.tpgid)).toBe(true);
|
||||
expect(self!.executableName).toBeTruthy();
|
||||
expect(self!.command).toBeTruthy();
|
||||
expect(self!.startedAt).toMatch(/\d{2}:\d{2}:\d{2}.*\d{4}/);
|
||||
});
|
||||
@@ -554,7 +554,7 @@ describe("Codex runtime adapter", () => {
|
||||
expect(sendText).toHaveBeenCalledWith("r01-qa", expectedFreshLaunchCommand({ queueRoot: null }));
|
||||
});
|
||||
|
||||
it("launchHarness skips the non-mutating Codex update prompt before capturing a fresh thread id", async () => {
|
||||
it("launchHarness skips the Codex update prompt with one control key before capturing a fresh thread id", async () => {
|
||||
const initialShell = [
|
||||
expectedFreshLaunchCommand(),
|
||||
"admin@host project %",
|
||||
@@ -571,9 +571,10 @@ describe("Codex runtime adapter", () => {
|
||||
getPaneCommand: vi.fn()
|
||||
.mockResolvedValueOnce("zsh")
|
||||
.mockResolvedValue("codex"),
|
||||
capturePaneContent: vi.fn()
|
||||
capturePaneScreen: vi.fn()
|
||||
.mockResolvedValueOnce(initialShell)
|
||||
.mockResolvedValueOnce(updatePrompt)
|
||||
.mockResolvedValueOnce(updatePrompt)
|
||||
.mockResolvedValue("OpenAI Codex (v0.120.0)\n› Ask Codex to do anything"),
|
||||
getPanePid: vi.fn(async () => 900),
|
||||
});
|
||||
@@ -581,8 +582,8 @@ describe("Codex runtime adapter", () => {
|
||||
tmux,
|
||||
fsOps: mockFs(),
|
||||
listProcesses: () => [
|
||||
{ pid: 900, ppid: 1, command: "-zsh" },
|
||||
{ pid: 901, ppid: 900, command: "codex" },
|
||||
{ pid: 900, ppid: 1, command: "-zsh", pgid: 900, tpgid: 901, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 901, ppid: 900, command: "codex", pgid: 901, tpgid: 901, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
],
|
||||
readThreadIdByPid: (pid) => pid === 901 ? "019d45bc-117d-78a3-a4ad-6fb186e5a86d" : undefined,
|
||||
sleep: async () => {},
|
||||
@@ -599,12 +600,11 @@ describe("Codex runtime adapter", () => {
|
||||
const sendText = tmux.sendText as ReturnType<typeof vi.fn>;
|
||||
expect(sendText.mock.calls).toEqual([
|
||||
["r01-qa", expectedFreshLaunchCommand()],
|
||||
["r01-qa", "3"],
|
||||
]);
|
||||
const sendKeys = tmux.sendKeys as ReturnType<typeof vi.fn>;
|
||||
expect(sendKeys.mock.calls).toEqual([
|
||||
["r01-qa", ["Enter"]],
|
||||
["r01-qa", ["Enter"]],
|
||||
["r01-qa", ["3"]],
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -644,8 +644,12 @@ describe("Codex runtime adapter", () => {
|
||||
"Press enter to continue",
|
||||
].join("\n");
|
||||
const tmux = mockTmux({
|
||||
getPaneCommand: vi.fn(async () => "zsh"),
|
||||
capturePaneContent: vi.fn()
|
||||
getPaneCommand: vi.fn()
|
||||
.mockResolvedValueOnce("zsh").mockResolvedValueOnce("zsh")
|
||||
.mockResolvedValueOnce("zsh").mockResolvedValueOnce("zsh")
|
||||
.mockResolvedValueOnce("zsh").mockResolvedValueOnce("zsh")
|
||||
.mockResolvedValue("codex"),
|
||||
capturePaneScreen: vi.fn()
|
||||
.mockResolvedValueOnce(initialShell)
|
||||
.mockResolvedValueOnce(initialShell)
|
||||
.mockResolvedValueOnce(initialShell)
|
||||
@@ -653,6 +657,7 @@ describe("Codex runtime adapter", () => {
|
||||
.mockResolvedValueOnce(initialShell)
|
||||
.mockResolvedValueOnce(initialShell)
|
||||
.mockResolvedValueOnce(updatePrompt)
|
||||
.mockResolvedValueOnce(updatePrompt)
|
||||
.mockResolvedValue("OpenAI Codex (v0.120.0)\n› Ask Codex to do anything"),
|
||||
getPanePid: vi.fn()
|
||||
.mockResolvedValueOnce(null)
|
||||
@@ -662,8 +667,8 @@ describe("Codex runtime adapter", () => {
|
||||
tmux,
|
||||
fsOps: mockFs(),
|
||||
listProcesses: () => [
|
||||
{ pid: 900, ppid: 1, command: "-zsh" },
|
||||
{ pid: 901, ppid: 900, command: "codex" },
|
||||
{ pid: 900, ppid: 1, command: "-zsh", pgid: 900, tpgid: 901, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 901, ppid: 900, command: "codex", pgid: 901, tpgid: 901, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
],
|
||||
readThreadIdByPid: (pid) => pid === 901 ? "019d45bc-117d-78a3-a4ad-6fb186e5a86d" : undefined,
|
||||
sleep: async () => {},
|
||||
@@ -680,8 +685,8 @@ describe("Codex runtime adapter", () => {
|
||||
const sendText = tmux.sendText as ReturnType<typeof vi.fn>;
|
||||
expect(sendText.mock.calls).toEqual([
|
||||
["r01-qa", expectedFreshLaunchCommand()],
|
||||
["r01-qa", "3"],
|
||||
]);
|
||||
expect(tmux.sendKeys).toHaveBeenLastCalledWith("r01-qa", ["3"]);
|
||||
});
|
||||
|
||||
it("uses the visible conversation rather than dismissed loading/review scrollback", async () => {
|
||||
@@ -930,7 +935,7 @@ describe("Codex runtime adapter", () => {
|
||||
// ensureCodexFeatureFlag) + activity-hook-rip-proof.test.ts (negative
|
||||
// assertions on adapter symbol absence + endpoint-stays).
|
||||
|
||||
it("launchHarness skips the non-mutating Codex update prompt during resume verification", async () => {
|
||||
it("launchHarness skips the Codex update prompt with one control key during resume verification", async () => {
|
||||
const updatePrompt = [
|
||||
"✨ Update available! 0.120.0 -> 0.121.0",
|
||||
"› 1. Update now (runs `npm install -g @openai/codex`)",
|
||||
@@ -939,11 +944,15 @@ describe("Codex runtime adapter", () => {
|
||||
"Press enter to continue",
|
||||
].join("\n");
|
||||
const tmux = mockTmux({
|
||||
capturePaneContent: vi.fn()
|
||||
getPanePid: vi.fn(async () => 901),
|
||||
capturePaneScreen: vi.fn()
|
||||
.mockResolvedValueOnce(updatePrompt)
|
||||
.mockResolvedValueOnce(updatePrompt)
|
||||
.mockResolvedValue("OpenAI Codex (v0.120.0)\n› Ask Codex to do anything"),
|
||||
});
|
||||
const adapter = new CodexRuntimeAdapter({ tmux, fsOps: mockFs(), sleep: async () => {} });
|
||||
const adapter = new CodexRuntimeAdapter({ tmux, fsOps: mockFs(), sleep: async () => {},
|
||||
listProcesses: () => [{ pid: 901, ppid: 1, pgid: 901, tpgid: 901, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000", command: "codex" }],
|
||||
});
|
||||
|
||||
const result = await adapter.launchHarness(makeBinding(), { name: "dev-qa@test-rig", resumeToken: "sess-456" });
|
||||
|
||||
@@ -951,12 +960,11 @@ describe("Codex runtime adapter", () => {
|
||||
const sendText = tmux.sendText as ReturnType<typeof vi.fn>;
|
||||
expect(sendText.mock.calls).toEqual([
|
||||
["r01-qa", expectedResumeCommand()],
|
||||
["r01-qa", "3"],
|
||||
]);
|
||||
const sendKeys = tmux.sendKeys as ReturnType<typeof vi.fn>;
|
||||
expect(sendKeys.mock.calls).toEqual([
|
||||
["r01-qa", ["Enter"]],
|
||||
["r01-qa", ["Enter"]],
|
||||
["r01-qa", ["3"]],
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -1186,7 +1194,7 @@ describe("Codex runtime adapter", () => {
|
||||
|
||||
// Guard against breaking the working auto-dismiss: a skippable update gate
|
||||
// still auto-dismisses and continues to success (covered end-to-end by
|
||||
// "launchHarness skips the non-mutating Codex update prompt during resume
|
||||
// "launchHarness skips the Codex update prompt with one control key during resume
|
||||
// verification" above) — only UNRESOLVED gates fail loudly.
|
||||
|
||||
it("deliverStartup pre-seeds Codex trust for the managed project", async () => {
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CodexRuntimeAdapter, type CodexAdapterFsOps } from "../src/adapters/codex-runtime-adapter.js";
|
||||
import { TmuxAdapter } from "../src/adapters/tmux.js";
|
||||
import type { NodeBinding } from "../src/domain/runtime-adapter.js";
|
||||
|
||||
const MENU = [
|
||||
"✨ Update available! 0.155.1 -> 0.156.1",
|
||||
"› 1. Update now (runs `npm install -g @openai/codex`)",
|
||||
" 2. Skip", " 3. Skip until next version", "Press enter to continue",
|
||||
].join("\n");
|
||||
const READY = "OpenAI Codex (v0.155.1)\n› Ask Codex to do anything";
|
||||
const binding = {
|
||||
id: "b", nodeId: "n", tmuxSession: "checker@test", tmuxWindow: null, tmuxPane: null,
|
||||
cmuxWorkspace: null, cmuxSurface: null, updatedAt: "", cwd: "/private-test",
|
||||
} satisfies NodeBinding;
|
||||
|
||||
// Modeled receiver, NOT native execution. Pinned upstream be2951ea:
|
||||
// update_prompt.rs:57-68 ignores Paste; :121-158 Key3 selects/submits
|
||||
// DontRemind immediately; :74-85 persists dismissal or returns RunUpdate.
|
||||
// Drive real TmuxAdapter serialization into this model, not a sequence of
|
||||
// ready screenshots that would pass even if the wrong input were sent.
|
||||
type ProcessRow = { pid: number; ppid: number; command: string; pgid?: number; tpgid?: number; executableName?: string; startedAt?: string };
|
||||
const native = "/opt/codex/vendor/aarch64-apple-darwin/bin/codex";
|
||||
// Synthetic shell -> Node -> native ancestry, including a background helper.
|
||||
// Paths, PIDs and start times are invented; only the process relationships matter.
|
||||
// The receiver below models the update menu; this is not native execution.
|
||||
const ordinaryRows: ProcessRow[] = [
|
||||
{ pid: 2001, ppid: 2000, pgid: 2001, tpgid: 2002, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000", command: "-zsh" },
|
||||
{ pid: 2002, ppid: 2001, pgid: 2002, tpgid: 2002, executableName: "bash", startedAt: "Sat Jan 1 12:00:00 2000", command: "/bin/sh /tmp/test-launch/openrig-tmux-send.txt" },
|
||||
{ pid: 2003, ppid: 2002, pgid: 2002, tpgid: 2002, executableName: "node", startedAt: "Sat Jan 1 12:00:00 2000", command: "node /opt/test-provider/bin/codex -s workspace-write -C /workspace/sample-project --add-dir /workspace/sample-project/.git --add-dir /tmp/test-instance/shared-docs/rigs/first-project/state/dev -m fixture-model" },
|
||||
{ pid: 2004, ppid: 2003, pgid: 2002, tpgid: 2002, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000", command: "/opt/test-provider/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin/codex -s workspace-write -C /workspace/sample-project --add-dir /workspace/sample-project/.git --add-dir /tmp/test-instance/shared-docs/rigs/first-project/state/dev -m fixture-model" },
|
||||
{ pid: 2005, ppid: 2004, pgid: 2005, tpgid: 2002, executableName: "codex-code-mode-", startedAt: "Sat Jan 1 12:00:00 2000", command: "/opt/test-provider/lib/node_modules/@openai/codex/node_modules/@openai/codex-darwin-arm64/vendor/aarch64-apple-darwin/bin/codex-code-mode-host" },
|
||||
];
|
||||
const processRows = (shape: "wrapper" | "exec-wrapper" | "native"): ProcessRow[] => {
|
||||
if (shape === "native") return [{ pid: 101, ppid: 1, command: native, pgid: 101, tpgid: 101, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" }];
|
||||
if (shape === "exec-wrapper") return [
|
||||
{ pid: 101, ppid: 1, command: "node /opt/bin/codex", pgid: 101, tpgid: 101, executableName: "node", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 102, ppid: 101, command: native, pgid: 101, tpgid: 101, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
];
|
||||
return [
|
||||
{ pid: 101, ppid: 1, command: "-zsh", pgid: 101, tpgid: 102, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 102, ppid: 101, command: "node /opt/bin/codex", pgid: 102, tpgid: 102, executableName: "node", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 103, ppid: 102, command: native, pgid: 102, tpgid: 102, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
];
|
||||
};
|
||||
function fixture(options: { delay?: number; beforeMenu?: number; failInput?: boolean; screen?: string; command?: string; panePid?: number; processes?: ProcessRow[]; shape?: "wrapper" | "exec-wrapper" | "native" } = {}) {
|
||||
let selected = false;
|
||||
let ticks = 0;
|
||||
let updates = 0;
|
||||
let dismissals = 0;
|
||||
let paste = "";
|
||||
const commands: string[] = [];
|
||||
const leaked: string[] = [];
|
||||
const ready = () => selected && ticks >= (options.delay ?? 0);
|
||||
const screen = () => options.screen ?? (ticks < (options.beforeMenu ?? 0) ? "Starting Codex..." : ready() ? READY : MENU);
|
||||
const tmux = new TmuxAdapter(async (cmd) => {
|
||||
commands.push(cmd);
|
||||
if (cmd.includes("paste-buffer")) {
|
||||
expect(cmd).toContain("-p");
|
||||
// Native update menu ignores bracketed Paste, including Paste("3").
|
||||
if (ready()) leaked.push(paste);
|
||||
}
|
||||
if (cmd.startsWith("tmux send-keys")) {
|
||||
if (options.failInput) throw new Error("input transport failed");
|
||||
if (selected) leaked.push(cmd);
|
||||
else if (cmd.endsWith(" '3'")) { selected = true; dismissals++; }
|
||||
else if (cmd.endsWith(" 'Enter'")) updates++;
|
||||
}
|
||||
return "";
|
||||
}, {
|
||||
writeFile: async (_path, content) => { paste = content; },
|
||||
unlink: async () => {}, tmpName: () => "/mock/input", bufferName: () => "input",
|
||||
});
|
||||
vi.spyOn(tmux, "sendShellCommand").mockResolvedValue({ ok: true });
|
||||
vi.spyOn(tmux, "getPaneCommand").mockImplementation(async () => options.command ?? "codex");
|
||||
vi.spyOn(tmux, "capturePaneScreen").mockImplementation(async () => screen());
|
||||
vi.spyOn(tmux, "capturePaneContent").mockResolvedValue(MENU); // stale scrollback
|
||||
vi.spyOn(tmux, "hasSession").mockResolvedValue(true);
|
||||
vi.spyOn(tmux, "getPanePid").mockResolvedValue(options.panePid ?? 101);
|
||||
const fsOps: CodexAdapterFsOps = {
|
||||
readFile: () => { throw new Error("absent"); }, writeFile: () => {},
|
||||
exists: () => false, mkdirp: () => {}, listFiles: () => [], homedir: "/mock/home",
|
||||
};
|
||||
const listProcesses = vi.fn(() => options.processes ?? processRows(options.shape ?? "wrapper"));
|
||||
const adapter = new CodexRuntimeAdapter({
|
||||
tmux, fsOps, sleep: async () => { ticks++; },
|
||||
listProcesses,
|
||||
readThreadIdByPid: () => ready() ? "new-thread" : undefined,
|
||||
});
|
||||
return { adapter, tmux, listProcesses, commands, leaked, ready, counts: () => ({ updates, dismissals }),
|
||||
reset: () => { selected = false; ticks = 0; } };
|
||||
}
|
||||
|
||||
const paths = [
|
||||
{ name: "fresh", opts: { name: "checker@test" } },
|
||||
{ name: "resume", opts: { name: "checker@test", resumeToken: "original-thread" } },
|
||||
{ name: "fork", opts: { name: "checker@test", forkSource: { kind: "native_id" as const, value: "parent-thread" } } },
|
||||
];
|
||||
|
||||
describe.each(paths)("Codex update input: $name", ({ opts }) => {
|
||||
it.each([0, 4, 100])("sends one real key, no Enter or retry, with %i delayed ticks", async (delay) => {
|
||||
const f = fixture({ delay });
|
||||
const result = await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual(["tmux send-keys -t 'checker@test' '3'"]);
|
||||
expect(f.counts()).toEqual({ updates: 0, dismissals: 1 });
|
||||
expect(f.leaked).toEqual([]);
|
||||
expect((await f.adapter.checkReady(binding)).ready).toBe(f.ready());
|
||||
if (f.ready()) expect(result.ok).toBe(true);
|
||||
else if (opts.resumeToken) expect(result).toMatchObject({ ok: false, recovery: "attention_required" });
|
||||
});
|
||||
|
||||
it.each([0, 9])("reports input failure without retry when the menu arrives after %i ticks", async (beforeMenu) => {
|
||||
const f = fixture({ failInput: true, beforeMenu });
|
||||
const result = await f.adapter.launchHarness(binding, opts);
|
||||
expect(result).toMatchObject({ ok: false, recovery: "attention_required" });
|
||||
if (!result.ok) expect(result.error).toContain("input transport failed");
|
||||
expect(f.commands).toEqual(["tmux send-keys -t 'checker@test' '3'"]);
|
||||
expect(f.counts()).toEqual({ updates: 0, dismissals: 0 });
|
||||
expect((await f.adapter.checkReady(binding)).ready).toBe(false);
|
||||
});
|
||||
|
||||
it("handles a menu first appearing during later thread/resume polls", async () => {
|
||||
const f = fixture({ beforeMenu: 9 });
|
||||
const result = await f.adapter.launchHarness(binding, opts);
|
||||
expect(result.ok).toBe(true);
|
||||
expect(f.commands).toEqual(["tmux send-keys -t 'checker@test' '3'"]);
|
||||
expect(f.counts()).toEqual({ updates: 0, dismissals: 1 });
|
||||
expect(f.leaked).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reopen update automation after seeing the conversation", async () => {
|
||||
const f = fixture();
|
||||
vi.mocked(f.tmux.capturePaneScreen).mockResolvedValueOnce(READY).mockResolvedValue(MENU);
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ screen: READY },
|
||||
{ screen: MENU.replace("3. Skip", "4. Skip") },
|
||||
{ screen: "Update available! Updating Codex..." },
|
||||
{ screen: MENU, command: "zsh", processes: [{ pid: 101, ppid: 1, command: "-zsh", pgid: 101, tpgid: 101, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" }] },
|
||||
{ screen: "Unknown screen" },
|
||||
])("leaves unknown/changed/exited or already-ready screens alone: %j", async (options) => {
|
||||
const f = fixture(options);
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["wrapper", "exec-wrapper", "native"] as const)("recognizes the live foreground %s identity", async (shape) => {
|
||||
const f = fixture({ shape, command: shape === "native" ? "codex" : "node" });
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual(["tmux send-keys -t 'checker@test' '3'"]);
|
||||
expect(f.counts()).toEqual({ updates: 0, dismissals: 1 });
|
||||
});
|
||||
|
||||
it("checks synthetic shell-owned ancestry before selecting one key", async () => {
|
||||
const f = fixture({ command: "bash", panePid: 2001, processes: ordinaryRows });
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual(["tmux send-keys -t 'checker@test' '3'"]);
|
||||
expect(f.listProcesses.mock.calls.length).toBeGreaterThanOrEqual(2);
|
||||
expect(f.counts()).toEqual({ updates: 0, dismissals: 1 });
|
||||
expect(f.leaked).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "background native", rows: ordinaryRows.map(r => r.pid === 2004 ? { ...r, pgid: 9999 } : r) },
|
||||
{ name: "unrelated native", rows: ordinaryRows.map(r => r.pid === 2004 ? { ...r, ppid: 1 } : r) },
|
||||
{ name: "exited native with stale menu", rows: ordinaryRows.filter(r => r.pid !== 2004) },
|
||||
])("rejects synthetic shell ancestry with $name", async ({ rows }) => {
|
||||
const f = fixture({ command: "bash", panePid: 2001, processes: rows });
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "no process rows", rows: [] },
|
||||
{ name: "native name with unrelated executable argument zero", rows: [{pid:101,ppid:1,command:"/bin/echo codex",pgid:101,tpgid:101,executableName:"codex", startedAt: "Sat Jan 1 12:00:00 2000"}] },
|
||||
{ name: "argument named codex", rows: [{pid:101,ppid:1,command:"node other.js --label codex",pgid:101,tpgid:101,executableName:"node", startedAt: "Sat Jan 1 12:00:00 2000"}] },
|
||||
{ name: "native name only in arguments", rows: [{pid:101,ppid:1,command:"/bin/echo codex",pgid:101,tpgid:101,executableName:"echo", startedAt: "Sat Jan 1 12:00:00 2000"}] },
|
||||
{ name: "background Codex", rows: processRows("wrapper").map(r=>({...r,tpgid:500})) },
|
||||
{ name: "missing group", rows: processRows("wrapper").map(({pgid,...r})=>r) },
|
||||
{ name: "no terminal foreground", rows: processRows("wrapper").map(r=>({...r,tpgid:-1})) },
|
||||
{ name: "conflicting terminal group", rows: processRows("wrapper").map(r=>r.pid===103?{...r,tpgid:500}:r) },
|
||||
{ name: "another pane", rows: processRows("wrapper").map(r=>r.pid===103?{...r,ppid:999}:r) },
|
||||
{ name: "kernel name contradicts argv", rows: processRows("wrapper").map(r=>r.pid===103?{...r,executableName:"node", startedAt: "Sat Jan 1 12:00:00 2000"}:r) },
|
||||
{ name: "ambiguous native processes", rows: [...processRows("wrapper"),{pid:104,ppid:102,command:native,pgid:102,tpgid:102,executableName:"codex", startedAt: "Sat Jan 1 12:00:00 2000"}] },
|
||||
])("does not choose Skip with $name", async ({rows}) => {
|
||||
for (const command of ["node", "codex", "bash"]) {
|
||||
const f=fixture({command,processes:rows});
|
||||
await f.adapter.launchHarness(binding,opts);
|
||||
expect(f.commands).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["missing pane", "missing pane command", "failed process read", "missing current screen"])("refuses %s", async (failure) => {
|
||||
const f = fixture({ command: "node" });
|
||||
if (failure === "missing pane") vi.mocked(f.tmux.getPanePid).mockResolvedValue(null);
|
||||
if (failure === "missing pane command") vi.mocked(f.tmux.getPaneCommand).mockResolvedValue(null);
|
||||
if (failure === "failed process read") f.listProcesses.mockImplementation(() => { throw Error("ps unavailable"); });
|
||||
if (failure === "missing current screen") vi.mocked(f.tmux.capturePaneScreen).mockResolvedValue(null);
|
||||
await (f.adapter as any).dismissSkippableCodexUpdatePrompt(binding.tmuxSession, {handled:false}, 1);
|
||||
expect(f.commands).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["pane", "group", "process", "command", "menu"])("rechecks %s identity before choosing", async (changed) => {
|
||||
const f=fixture({command:"node"});
|
||||
// Directly bound one observation isolates the before-send recheck;
|
||||
// later launch polling is covered separately above.
|
||||
if(changed==="pane") vi.mocked(f.tmux.getPanePid).mockResolvedValueOnce(101).mockResolvedValue(200);
|
||||
if(changed==="group") f.listProcesses.mockReturnValueOnce(processRows("wrapper")).mockReturnValue(processRows("wrapper").map(r=>({...r,tpgid:500})));
|
||||
if(changed==="process") f.listProcesses.mockReturnValueOnce(processRows("wrapper")).mockReturnValue(processRows("wrapper").map(r=>r.pid===103?{...r,pid:104}:r));
|
||||
if(changed==="command") vi.mocked(f.tmux.getPaneCommand).mockResolvedValueOnce("node").mockResolvedValue("zsh");
|
||||
if(changed==="menu") vi.mocked(f.tmux.capturePaneScreen).mockResolvedValueOnce(MENU).mockResolvedValue(READY);
|
||||
const attempt = {handled:false};
|
||||
await (f.adapter as any).dismissSkippableCodexUpdatePrompt(binding.tmuxSession,attempt,1);
|
||||
expect(attempt.handled).toBe(true);
|
||||
vi.mocked(f.tmux.getPanePid).mockResolvedValue(101);
|
||||
vi.mocked(f.tmux.getPaneCommand).mockResolvedValue("node");
|
||||
vi.mocked(f.tmux.capturePaneScreen).mockResolvedValue(MENU);
|
||||
f.listProcesses.mockReturnValue(processRows("wrapper"));
|
||||
await (f.adapter as any).dismissSkippableCodexUpdatePrompt(binding.tmuxSession,attempt,1);
|
||||
expect(f.commands).toEqual([]);
|
||||
});
|
||||
|
||||
it("allows one choice on a separately requested new launch", async () => {
|
||||
const f = fixture();
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
f.reset();
|
||||
await f.adapter.launchHarness(binding, opts);
|
||||
expect(f.commands).toEqual(Array(2).fill("tmux send-keys -t 'checker@test' '3'"));
|
||||
expect(f.counts()).toEqual({ updates: 0, dismissals: 2 });
|
||||
expect(f.leaked).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
it("negative control: real bracketed Paste3 then Enter selects UpdateNow in the pinned model", async () => {
|
||||
const f = fixture();
|
||||
await f.tmux.sendText(binding.tmuxSession!, "3");
|
||||
await f.tmux.sendKeys(binding.tmuxSession!, ["Enter"]);
|
||||
expect(f.counts()).toEqual({ updates: 1, dismissals: 0 });
|
||||
expect(f.ready()).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { findExactNativeResumeProcess, verifyCodexPaneProcess, type NativeProcessRow } from "../src/domain/native-process-lineage.js";
|
||||
|
||||
const token = "00000000-0000-7000-8000-000000000001";
|
||||
const startedAt = "Sat Jan 1 12:00:00 2000";
|
||||
function rows(): NativeProcessRow[] {
|
||||
return [
|
||||
{ pid: 10, ppid: 1, pgid: 10, tpgid: 11, executableName: "zsh", command: "-zsh", startedAt },
|
||||
{ pid: 11, ppid: 10, pgid: 11, tpgid: 11, executableName: "bash", command: "/bin/sh /tmp/openrig-tmux-send.txt", startedAt },
|
||||
{ pid: 12, ppid: 11, pgid: 11, tpgid: 11, executableName: "node", command: `node /opt/bin/codex resume ${token}`, startedAt },
|
||||
{ pid: 13, ppid: 12, pgid: 11, tpgid: 11, executableName: "codex", command: `/opt/native/codex -p resume resume --add-dir /tmp/state ${token}`, startedAt },
|
||||
];
|
||||
}
|
||||
const check = (listProcesses: () => NativeProcessRow[] | Promise<NativeProcessRow[]>, overrides = {}) => verifyCodexPaneProcess({
|
||||
target: "%1", tmux: { getPanePid: async () => 10 }, listProcesses, expectedToken: token, requireResume: true, ...overrides,
|
||||
});
|
||||
|
||||
describe("joined native Codex identity", () => {
|
||||
it("selects the unique native process, not its Node wrapper", async () => {
|
||||
expect((await check(rows))?.process.pid).toBe(13);
|
||||
expect(findExactNativeResumeProcess(rows(), 10, "codex", token)?.pid).toBe(13);
|
||||
});
|
||||
it("proves direct-native resume", async () => {
|
||||
expect((await check(() => [{ ...rows()[3]!, pid: 10, ppid: 1 }]))?.process.pid).toBe(10);
|
||||
});
|
||||
it("distinguishes fresh/non-strict runtime proof from exact resume", async () => {
|
||||
const fresh = () => rows().map(r => r.pid === 13 ? { ...r, command: "/opt/native/codex -m model" } : r);
|
||||
expect(await check(fresh)).toBeNull();
|
||||
expect(await check(fresh, { requireResume: false })).not.toBeNull();
|
||||
expect(await check(rows, { requireResume: false, expectedToken: "different" })).toBeNull();
|
||||
expect(await check(rows, { requireResume: true, expectedToken: null })).toBeNull();
|
||||
expect(await check(rows, { requireResume: false, expectedToken: null })).toBeNull();
|
||||
expect(await check(() => rows().map(r => r.pid === 13 ? { ...r, command: "/opt/native/codex resume --last" } : r), { requireResume: false, expectedToken: null })).toBeNull();
|
||||
});
|
||||
const controls: [string, (r: NativeProcessRow[]) => NativeProcessRow[]][] = [
|
||||
["wrong UUID", r => r.map(x => x.pid === 13 ? { ...x, command: "/opt/native/codex resume other" } : x)],
|
||||
["missing UUID", r => r.map(x => x.pid === 13 ? { ...x, command: "/opt/native/codex resume" } : x)],
|
||||
["token only in prompt", r => r.map(x => x.pid === 13 ? { ...x, command: `/opt/native/codex resume other ${token}` } : x)],
|
||||
["wrong OS executable", r => r.map(x => x.pid === 13 ? { ...x, executableName: "printf" } : x)],
|
||||
["argv-only executable", r => r.map(x => x.pid === 13 ? { ...x, command: `/bin/echo codex resume ${token}` } : x)],
|
||||
["unrelated descendant", r => r.map(x => x.pid === 13 ? { ...x, ppid: 999 } : x)],
|
||||
["background native", r => r.map(x => x.pid === 13 ? { ...x, pgid: 99 } : x)],
|
||||
["conflicting foreground", r => r.map(x => x.pid === 13 ? { ...x, tpgid: 99 } : x)],
|
||||
["missing root", r => r.slice(1)],
|
||||
["missing ancestry", r => r.filter(x => x.pid !== 11)],
|
||||
["cyclic ancestry", r => r.map(x => x.pid === 11 ? { ...x, ppid: 13 } : x)],
|
||||
["multiple native candidates", r => [...r, { ...r[3]!, pid: 14 }]],
|
||||
["duplicate PID", r => [...r, r[3]!]],
|
||||
["missing start time", r => r.map(x => ({ ...x, startedAt: undefined }))],
|
||||
["missing group", r => r.map(x => ({ ...x, tpgid: undefined }))],
|
||||
["exited native", r => r.filter(x => x.pid !== 13)],
|
||||
];
|
||||
it.each(controls)("refuses %s", async (_name, mutate) => {
|
||||
expect(await check(() => mutate(rows()))).toBeNull();
|
||||
});
|
||||
it.each(["startedAt", "command", "ppid", "pgid"] as const)("refuses a changed native %s between observations", async (field) => {
|
||||
const changed = rows().map(r => r.pid === 13 ? { ...r, [field]: field === "startedAt" ? "Sat Jan 1 12:00:01 2000" : field === "command" ? r.command + " --verbose" : 99 } : r);
|
||||
expect(await check(vi.fn().mockResolvedValueOnce(rows()).mockResolvedValueOnce(changed))).toBeNull();
|
||||
});
|
||||
it("refuses a reused pane PID even when the native PID is unchanged", async () => {
|
||||
const changed = rows().map(r => r.pid === 10 ? { ...r, startedAt: "Sat Jan 1 12:00:01 2000" } : r);
|
||||
expect(await check(vi.fn().mockResolvedValueOnce(rows()).mockResolvedValueOnce(changed))).toBeNull();
|
||||
});
|
||||
it("refuses a changed or missing pane and process observation failures", async () => {
|
||||
expect(await check(rows, { tmux: { getPanePid: vi.fn().mockResolvedValueOnce(10).mockResolvedValueOnce(11) } })).toBeNull();
|
||||
expect(await check(rows, { tmux: { getPanePid: async () => null } })).toBeNull();
|
||||
expect(await check(async () => { throw new Error("ps failed"); })).toBeNull();
|
||||
});
|
||||
it("retains the existing Claude exact-token contract", () => {
|
||||
expect(findExactNativeResumeProcess([{ pid: 10, ppid: 1, command: `claude --resume ${token}` }], 10, "claude-code", token)?.pid).toBe(10);
|
||||
expect(findExactNativeResumeProcess([{ pid: 10, ppid: 1, command: "claude --resume wrong" }], 10, "claude-code", token)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1262,6 +1262,8 @@ state: 2-named
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok && result.code === "attention_required") {
|
||||
expect(result.message).toMatch(/inspect/i);
|
||||
expect(result.message).not.toMatch(/approve and resume|NOT failed/);
|
||||
expect(result.rigId).toBeDefined();
|
||||
expect(result.attentionNodes).toBeInstanceOf(Array);
|
||||
expect(result.attentionNodes!.length).toBe(1);
|
||||
|
||||
@@ -33,6 +33,7 @@ import { createFullTestDb } from "./helpers/test-app.js";
|
||||
import { AppliedLaunchObservationStore } from "../src/domain/applied-launch-observation-store.js";
|
||||
import { observeClaudePermission, observeCodexSandbox, observePiResourceTrust, type AppliedLaunchObservation } from "../src/domain/permission-drift.js";
|
||||
import { buildCodexResumeCore } from "../src/domain/native-resume-probe.js";
|
||||
import { SeatIdentityReconciler } from "../src/domain/seat-identity-reconciler.js";
|
||||
|
||||
function setupDb(): Database.Database {
|
||||
return createFullTestDb();
|
||||
@@ -884,7 +885,41 @@ describe("RestoreOrchestrator", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("reports a managed Codex node wrapper resumed when its child lineage carries the saved native session", async () => {
|
||||
it.each(["full", "subset", "existing-hook", "conflicting-hook"])("proves synthetic shell-wrapped Codex through %s recovery and the next identity poll", async (mode) => {
|
||||
const token = "00000000-0000-7000-8000-000000000001";
|
||||
const snap = seedRigAndSnapshot({ nodes: [{ logicalId: "worker", role: "worker", runtime: "codex" }], edges: [], resumeType: "codex_id", resumeToken: token });
|
||||
// Synthetic IDs, paths and times preserve shell/Node/native foreground relationships.
|
||||
const rows = [
|
||||
{ pid: 3001, ppid: 3000, pgid: 3001, tpgid: 3002, executableName: "zsh", command: "-zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 3002, ppid: 3001, pgid: 3002, tpgid: 3002, executableName: "bash", command: "/bin/sh /tmp/openrig-tmux-send.txt", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 3003, ppid: 3002, pgid: 3002, tpgid: 3002, executableName: "node", command: `node /opt/bin/codex -s workspace-write -m fixture-model resume --add-dir /tmp/state ${token}`, startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 3004, ppid: 3003, pgid: 3002, tpgid: 3002, executableName: "codex", command: `/opt/native/codex -s workspace-write -m fixture-model resume --add-dir /tmp/state ${token}`, startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
];
|
||||
const tmux = { ...mockTmux(), getPanePid: vi.fn(async () => 3001), getPaneCommand: vi.fn(async () => "bash") } as unknown as TmuxAdapter;
|
||||
const listProcesses = async () => {
|
||||
if (mode.endsWith("hook")) db.prepare("UPDATE sessions SET resume_token = ?, resume_provenance = 'hook' WHERE id = (SELECT id FROM sessions ORDER BY id DESC LIMIT 1)").run(mode === "existing-hook" ? token : "other-thread");
|
||||
return rows;
|
||||
};
|
||||
const orch = createOrchestrator({ tmux, listProcesses });
|
||||
const result = mode !== "subset" ? await orch.restore(snap.id) : await orch.launchSingleNode(snap.rigId, "worker", { snapshotId: snap.id });
|
||||
expect(result.ok).toBe(true);
|
||||
const nodes = "result" in result ? result.result?.nodes : result.launched;
|
||||
if (mode === "conflicting-hook") {
|
||||
expect(nodes?.[0]?.status).toBe("attention_required");
|
||||
expect(db.prepare("SELECT verdict FROM seat_identity_verdicts WHERE node_id = ?").get(nodes![0]!.nodeId)).toEqual({ verdict: "mismatch" });
|
||||
expect(db.prepare("SELECT resume_token, resume_provenance FROM sessions ORDER BY id DESC LIMIT 1").get()).toEqual({ resume_token: "other-thread", resume_provenance: "hook" });
|
||||
return;
|
||||
}
|
||||
expect(nodes?.[0]?.status).toBe("resumed");
|
||||
if (mode === "existing-hook") expect(db.prepare("SELECT resume_provenance FROM sessions ORDER BY id DESC LIMIT 1").get()).toEqual({ resume_provenance: "hook" });
|
||||
const nodeId = nodes![0]!.nodeId;
|
||||
const name = sessionRegistry.getBindingForNode(nodeId)!.tmuxSession!;
|
||||
tmux.listSessions = vi.fn(async () => [{ name }] as never);
|
||||
await new SeatIdentityReconciler({ db, tmux, listProcesses } as ConstructorParameters<typeof SeatIdentityReconciler>[0]).reconcileAll();
|
||||
expect(db.prepare("SELECT verdict, observed_pid FROM seat_identity_verdicts WHERE node_id = ?").get(nodeId)).toEqual({ verdict: "verified", observed_pid: 3004 });
|
||||
});
|
||||
|
||||
it("reports a managed Codex wrapper resumed when its native child carries the saved session", async () => {
|
||||
const resumeToken = "01a05645-37a2-7dd0-970c-031d2f2510cb";
|
||||
const snap = seedRigAndSnapshot({
|
||||
nodes: [{ logicalId: "worker", role: "worker", runtime: "codex" }],
|
||||
@@ -893,13 +928,13 @@ describe("RestoreOrchestrator", () => {
|
||||
resumeToken,
|
||||
});
|
||||
const tmux = { ...mockTmux(), getPaneCommand: vi.fn(async () => "node") } as unknown as TmuxAdapter;
|
||||
const codexCommand = `node /opt/openrig/lib/node_modules/@openai/codex/bin/codex --model gpt-5.6 resume --add-dir /tmp/openrig-state ${resumeToken}`;
|
||||
const codexCommand = `/opt/openrig/vendor/bin/codex --model gpt-5.6 resume --add-dir /tmp/openrig-state ${resumeToken}`;
|
||||
const result = await createOrchestrator({
|
||||
tmux,
|
||||
codex: mockCodexResume({ ok: true }),
|
||||
listProcesses: async () => [
|
||||
{ pid: 1234, ppid: 1, command: "-zsh" },
|
||||
{ pid: 54776, ppid: 1234, command: codexCommand },
|
||||
{ pid: 1234, ppid: 1, command: "-zsh", pgid: 1234, tpgid: 54776, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 54776, ppid: 1234, command: codexCommand, pgid: 54776, tpgid: 54776, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
],
|
||||
}).restore(snap.id);
|
||||
|
||||
@@ -929,8 +964,8 @@ describe("RestoreOrchestrator", () => {
|
||||
tmux,
|
||||
codex: mockCodexResume({ ok: true }),
|
||||
listProcesses: async () => [
|
||||
{ pid: 1234, ppid: 1, command: "-zsh" },
|
||||
{ pid: 54776, ppid: 1234, command: codexCommand },
|
||||
{ pid: 1234, ppid: 1, command: "-zsh", pgid: 1234, tpgid: 54776, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 54776, ppid: 1234, command: codexCommand, pgid: 54776, tpgid: 54776, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
],
|
||||
}).restore(snap.id);
|
||||
|
||||
@@ -960,8 +995,8 @@ describe("RestoreOrchestrator", () => {
|
||||
tmux,
|
||||
codex: mockCodexResume({ ok: true }),
|
||||
listProcesses: async () => [
|
||||
{ pid: 1234, ppid: 1, command: "-zsh" },
|
||||
{ pid: 54776, ppid: 1234, command: codexCommand },
|
||||
{ pid: 1234, ppid: 1, command: "-zsh", pgid: 1234, tpgid: 54776, executableName: "zsh", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 54776, ppid: 1234, command: codexCommand, pgid: 54776, tpgid: 54776, executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
],
|
||||
}).restore(snap.id);
|
||||
|
||||
@@ -2698,9 +2733,8 @@ describe("RestoreOrchestrator", () => {
|
||||
withResumeToken?: boolean;
|
||||
withBinding?: boolean;
|
||||
}) {
|
||||
// Session name validator requires legacy `r{NN}-{suffix}` or canonical
|
||||
// `{pod}-{member}@{rig}`. Each call gets a unique numeric rig id so the
|
||||
// legacy pattern matches and rig names don't collide.
|
||||
// Use canonical names so adding cases cannot overflow the legacy
|
||||
// two-digit rig-name range.
|
||||
const rigName = opts.rigName ?? `r${nextSeed++}`;
|
||||
const logicalId = opts.logicalId ?? "worker";
|
||||
const runtime = opts.runtime ?? "claude-code";
|
||||
@@ -2708,7 +2742,7 @@ describe("RestoreOrchestrator", () => {
|
||||
const node = rigRepo.addNode(rig.id, logicalId, { role: "worker", runtime });
|
||||
|
||||
// Bind a tmux session name so the reconciler can probe.
|
||||
const sessionName = `${rigName}-${logicalId}`;
|
||||
const sessionName = `test-${logicalId}@${rigName}`;
|
||||
if (opts.withBinding ?? true) {
|
||||
sessionRegistry.updateBinding(node.id, { tmuxSession: sessionName });
|
||||
}
|
||||
@@ -2740,6 +2774,30 @@ describe("RestoreOrchestrator", () => {
|
||||
return { rig, nodeId: node.id, sessionName };
|
||||
}
|
||||
|
||||
it.each([
|
||||
["OpenAI Codex (v0.155.1)\n› continue", true],
|
||||
["Update available!", false],
|
||||
["Your access token could not be refreshed because your refresh token was already used. Please log out and sign in again.", false],
|
||||
["Do you trust the contents of this directory?", false],
|
||||
])("reconciles wrapped Codex without input only with usable content: %s", async (content, usable) => {
|
||||
const tmux = mockTmuxForReconciler();
|
||||
vi.mocked(tmux.hasSession).mockResolvedValue(true);
|
||||
vi.mocked(tmux.getPaneCommand).mockResolvedValue("bash");
|
||||
vi.mocked(tmux.capturePaneContent).mockResolvedValue(String(content));
|
||||
const seeded = seedFailedAttempt({ runtime: "codex", restoreOutcome: "attention_required", withResumeToken: true });
|
||||
const oldEvent = db.prepare("SELECT payload FROM events WHERE type = 'restore.completed'").get();
|
||||
const listProcesses = async () => [
|
||||
{ pid: 1234, ppid: 1, pgid: 1234, tpgid: 1235, command: "bash", executableName: "bash", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: 1235, ppid: 1234, pgid: 1235, tpgid: 1235, command: "codex resume tok-abc-123", executableName: "codex", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
];
|
||||
const result = await createOrchestrator({ tmux, listProcesses }).reconcileNodeRuntimeTruth(seeded.rig.id, seeded.nodeId);
|
||||
expect(result.ok).toBe(usable);
|
||||
expect(tmux.sendKeys).not.toHaveBeenCalled();
|
||||
expect(tmux.sendText).not.toHaveBeenCalled();
|
||||
expect(db.prepare("SELECT payload FROM events WHERE type = 'restore.completed'").get()).toEqual(oldEvent);
|
||||
expect(db.prepare("SELECT * FROM events WHERE type = 'restore.outcome_reconciled'").all()).toHaveLength(usable ? 1 : 0);
|
||||
});
|
||||
|
||||
it("upgrades failed -> operator_recovered when ALL four preconditions hold; emits audit event", async () => {
|
||||
const tmux = mockTmuxForReconciler();
|
||||
(tmux.hasSession as ReturnType<typeof vi.fn>).mockResolvedValue(true);
|
||||
|
||||
@@ -120,6 +120,8 @@ describe("SeatLifecycleService.launchFresh", () => {
|
||||
sessionRegistry,
|
||||
eventBus,
|
||||
tmuxAdapter: tmux,
|
||||
listProcesses: async () => [{ pid: 4242, ppid: 1, pgid: 4242, tpgid: 4242,
|
||||
executableName: "codex", command: "/opt/native/codex -m model", startedAt: "Sat Jan 1 12:00:00 2000" }],
|
||||
nodeLauncher,
|
||||
startupOrchestrator,
|
||||
runtimeAdapters: { "claude-code": adapter, codex: { ...adapter, runtime: "codex" } },
|
||||
|
||||
@@ -105,6 +105,53 @@ describe("classifyPaneRuntimeMatch", () => {
|
||||
});
|
||||
|
||||
describe("SeatIdentityReconciler.reconcileAll", () => {
|
||||
it("shares two native observations across wrapped Codex seats and rejects later stale or missing proof", async () => {
|
||||
const db = createFullTestDb();
|
||||
for (const n of [1, 2]) {
|
||||
seedSeat(db, { nodeId: `c${n}`, sessionName: `c${n}@rig`, pane: `%${n}`, runtime: "codex" });
|
||||
db.prepare("UPDATE sessions SET resume_token = ? WHERE node_id = ?").run(`thread-${n}`, `c${n}`);
|
||||
}
|
||||
const tmux = makeTmux({ sessions: ["c1@rig", "c2@rig"], panePid: { "%1": 10, "%2": 20 }, paneCommand: { "%1": "bash", "%2": "node" } });
|
||||
const rows = [1, 2].flatMap(n => [
|
||||
{ pid: n * 10, ppid: 1, pgid: n * 10, tpgid: n * 10 + 1, executableName: "bash", command: "bash", startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
{ pid: n * 10 + 1, ppid: n * 10, pgid: n * 10 + 1, tpgid: n * 10 + 1, executableName: "codex", command: `codex resume thread-${n}`, startedAt: "Sat Jan 1 12:00:00 2000" },
|
||||
]);
|
||||
const listProcesses = vi.fn(async () => rows);
|
||||
const rec = new SeatIdentityReconciler({ db, tmux, listProcesses, now: NOW });
|
||||
const store = new SeatIdentityStore(db);
|
||||
await rec.reconcileAll();
|
||||
expect(listProcesses).toHaveBeenCalledTimes(2);
|
||||
for (const n of [1, 2]) expect(store.getForNode(`c${n}`)).toMatchObject({ verdict: "verified", evidence: { observedPid: n * 10 + 1 } });
|
||||
|
||||
// A reused native PID during the next sweep cannot retain the old green.
|
||||
listProcesses.mockResolvedValueOnce(rows).mockResolvedValueOnce(rows.map(r => r.pid === 11 ? { ...r, startedAt: "Sat Jan 1 12:00:01 2000" } : r));
|
||||
await rec.reconcileAll();
|
||||
expect(store.getForNode("c1")?.verdict).toBe("mismatch");
|
||||
expect(store.getForNode("c2")?.verdict).toBe("verified");
|
||||
|
||||
db.prepare("UPDATE sessions SET resume_token = 'wrong' WHERE node_id = 'c1'").run();
|
||||
await rec.reconcileAll();
|
||||
expect(store.getForNode("c1")?.verdict).toBe("mismatch");
|
||||
listProcesses.mockResolvedValue([]);
|
||||
await rec.reconcileAll();
|
||||
for (const n of [1, 2]) expect(store.getForNode(`c${n}`)?.verdict).toBe("mismatch");
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("keeps fresh Codex distinct from resume, and cannot verify an unavailable pane", async () => {
|
||||
const db = createFullTestDb();
|
||||
seedSeat(db, { nodeId: "c1", sessionName: "c1@rig", pane: "%1", runtime: "codex" });
|
||||
const tmux = makeTmux({ sessions: ["c1@rig"], panePid: { "%1": 10 }, paneCommand: { "%1": "codex" } });
|
||||
const listProcesses = vi.fn(async () => [{ pid: 10, ppid: 1, pgid: 10, tpgid: 10, executableName: "codex", command: "codex -m configured-model", startedAt: "Sat Jan 1 12:00:00 2000" }]);
|
||||
const rec = new SeatIdentityReconciler({ db, tmux, listProcesses, now: NOW });
|
||||
await rec.reconcileAll();
|
||||
expect(new SeatIdentityStore(db).getForNode("c1")?.verdict).toBe("verified");
|
||||
vi.mocked(tmux.listSessions).mockRejectedValue(new Error("unavailable"));
|
||||
await rec.reconcileAll();
|
||||
expect(new SeatIdentityStore(db).getForNode("c1")?.verdict).toBe("mismatch");
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("VERIFIED — matching pane pid + command persists a verified verdict", async () => {
|
||||
const db = createFullTestDb();
|
||||
seedSeat(db, { nodeId: "n1", sessionName: "s1@rig", pane: "%1", runtime: "claude-code" });
|
||||
|
||||
@@ -858,7 +858,7 @@ edges: []
|
||||
expect(r!.error.action).toMatch(/tmux attach -t dev-impl@conveyor/);
|
||||
expect(r!.error.action).not.toMatch(/rig setup --cwd/);
|
||||
// Singular phrasing for a 1-node case
|
||||
expect(r!.error.action).toMatch(/^Attach to the session/);
|
||||
expect(r!.error.action).toMatch(/^Inspect the affected session/);
|
||||
expect(r!.attentionNodes).toHaveLength(1);
|
||||
expect(r!.attentionNodes[0]!.logicalId).toBe("dev.impl");
|
||||
});
|
||||
@@ -885,7 +885,7 @@ edges: []
|
||||
const r = buildAttentionResponse(result);
|
||||
expect(r).not.toBeNull();
|
||||
// Plural phrasing
|
||||
expect(r!.error.action).toMatch(/^Attach to each parked session/);
|
||||
expect(r!.error.action).toMatch(/^Inspect each affected session/);
|
||||
// First 3 hints listed
|
||||
expect(r!.error.action).toContain("tmux attach -t dev-impl@conveyor");
|
||||
expect(r!.error.action).toContain("tmux attach -t dev-qa@conveyor");
|
||||
@@ -915,7 +915,7 @@ edges: []
|
||||
};
|
||||
const r = buildAttentionResponse(result);
|
||||
expect(r).not.toBeNull();
|
||||
expect(r!.error.action).toMatch(/^Attach to each parked session listed in attentionNodes/);
|
||||
expect(r!.error.action).toMatch(/^Inspect each affected session listed in attentionNodes/);
|
||||
expect(r!.error.action).toContain("tmux attach -t intake-lead@conveyor");
|
||||
expect(r!.error.action).toContain("tmux attach -t plan-planner@conveyor");
|
||||
expect(r!.error.action).toContain("tmux attach -t build-builder@conveyor");
|
||||
|
||||
Reference in new issue
Block a user