mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-11 17:32:22 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
07bbf5d473 | ||
|
|
4f244bb3c1 | ||
|
|
4827dbe70c |
No files matched your search
@@ -92,7 +92,7 @@ export class Reconciler {
|
||||
// A session this server doesn't show isn't gone while its seat's recorded process still runs: the daemon is
|
||||
// looking at a different tmux server. Detaching it would have the kernel relaunch duplicates and supersede the
|
||||
// live seats. Without that evidence (after a reboot nothing recorded is running), it detaches as before.
|
||||
const elsewhere = await this.recordedProcessStillRunning(session, processes);
|
||||
const elsewhere = await recordedProcessStillRunning(this.db, session, processes);
|
||||
if (elsewhere) {
|
||||
unseen.push({ sessionId: session.id, sessionName: session.sessionName, ...elsewhere });
|
||||
continue;
|
||||
@@ -123,31 +123,32 @@ export class Reconciler {
|
||||
|
||||
return { checked, detached, detachedSessionIds, unseen, errors };
|
||||
}
|
||||
}
|
||||
|
||||
/** The seat's last observed process, as a conservative hint that it still runs, not proof of identity: the last pid
|
||||
* evidence recorded for this very session (kept with its own time, so a later poll that couldn't see the seat doesn't
|
||||
* erase it), and a process with that pid whose start isn't observably later than that observation. Start times are
|
||||
* whole seconds, so a pid reused within the observation's second still passes. An unreadable table or record is no
|
||||
* evidence. */
|
||||
private async recordedProcessStillRunning(
|
||||
session: { nodeId: string; sessionName: string },
|
||||
processes: () => Promise<ProcessTableRow[] | null>,
|
||||
): Promise<{ pid: number; parent: { pid: number; command: string } | null } | null> {
|
||||
const evidence = new SeatIdentityStore(this.db).getProcessEvidence(session.nodeId);
|
||||
if (!evidence || evidence.sessionName !== session.sessionName) return null;
|
||||
const observedAt = Date.parse(evidence.observedAt) / 1000;
|
||||
const pids = [evidence.panePid, evidence.pid]
|
||||
.filter((pid): pid is number => typeof pid === "number" && pid > 0);
|
||||
if (!pids.length || Number.isNaN(observedAt)) return null;
|
||||
const rows = await processes();
|
||||
if (!rows) return null;
|
||||
for (const pid of pids) {
|
||||
const row = rows.find((r) => r.pid === pid);
|
||||
const started = row ? lstartToMinTs(row.startedAt) : undefined;
|
||||
if (!row || started === undefined || started > observedAt) continue;
|
||||
const parent = rows.find((r) => r.pid === row.ppid);
|
||||
return { pid, parent: parent ? { pid: parent.pid, command: parent.command } : null };
|
||||
}
|
||||
return null;
|
||||
/** The seat's last observed process, as a conservative hint that it still runs, not proof of identity: the last pid
|
||||
* evidence recorded for this very session (kept with its own time, so a later poll that couldn't see the seat doesn't
|
||||
* erase it), and a process with that pid whose start isn't observably later than that observation. Start times are
|
||||
* whole seconds, so a pid reused within the observation's second still passes. An unreadable table or record is no
|
||||
* evidence. */
|
||||
export async function recordedProcessStillRunning(
|
||||
db: Database.Database,
|
||||
session: { nodeId: string; sessionName: string },
|
||||
processes: () => Promise<ProcessTableRow[] | null>,
|
||||
): Promise<{ pid: number; parent: { pid: number; command: string } | null } | null> {
|
||||
const evidence = new SeatIdentityStore(db).getProcessEvidence(session.nodeId);
|
||||
if (!evidence || evidence.sessionName !== session.sessionName) return null;
|
||||
const observedAt = Date.parse(evidence.observedAt) / 1000;
|
||||
const pids = [evidence.panePid, evidence.pid]
|
||||
.filter((pid): pid is number => typeof pid === "number" && pid > 0);
|
||||
if (!pids.length || Number.isNaN(observedAt)) return null;
|
||||
const rows = await processes();
|
||||
if (!rows) return null;
|
||||
for (const pid of pids) {
|
||||
const row = rows.find((r) => r.pid === pid);
|
||||
const started = row ? lstartToMinTs(row.startedAt) : undefined;
|
||||
if (!row || started === undefined || started > observedAt) continue;
|
||||
const parent = rows.find((r) => r.pid === row.ppid);
|
||||
return { pid, parent: parent ? { pid: parent.pid, command: parent.command } : null };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -141,6 +141,22 @@ export class SeatIdentityReconciler {
|
||||
`).all() as RunningSeatRow[];
|
||||
}
|
||||
|
||||
/** Nodes whose latest session `rig unclaim` released (detached, origin claimed). Unclaim leaves the process running,
|
||||
* so the node's last recorded process stays on record: a fresh launch reads it before reopening a tmux server, and
|
||||
* must not start a second occupant beside a process that still runs. An older schema without `origin` keeps none. */
|
||||
private releasedSeats(): string[] {
|
||||
try {
|
||||
return (this.db.prepare(`
|
||||
SELECT n.id as node_id FROM nodes n
|
||||
JOIN sessions s ON s.node_id = n.id
|
||||
AND s.id = (SELECT s2.id FROM sessions s2 WHERE s2.node_id = n.id ORDER BY s2.id DESC LIMIT 1)
|
||||
WHERE s.status = 'detached' AND s.origin = 'claimed'
|
||||
`).all() as Array<{ node_id: string }>).map((r) => r.node_id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Reconcile every running tmux-bound seat once and persist the verdicts. */
|
||||
async reconcileAll(): Promise<void> {
|
||||
// Skip ticks while actual reads are pending; never release on a deadline
|
||||
@@ -156,8 +172,8 @@ export class SeatIdentityReconciler {
|
||||
|
||||
private async reconcileSweep(generation: number): Promise<void> {
|
||||
const seats = this.runningSeats();
|
||||
// Prune verdicts for nodes no longer running (keep the table bounded).
|
||||
this.store.pruneExcept(seats.map((s) => s.node_id));
|
||||
// Prune verdicts for nodes no longer running (keep the table bounded), except a released seat's.
|
||||
this.store.pruneExcept([...seats.map((s) => s.node_id), ...this.releasedSeats()]);
|
||||
if (seats.length === 0) return;
|
||||
|
||||
const observedAt = this.now().toISOString();
|
||||
|
||||
@@ -55,6 +55,8 @@ import { sessionCommunityHarnessSelect } from "./session-registry.js";
|
||||
import { seatLaunchAdditionsReport } from "./seat-launch-additions.js";
|
||||
import { seatSkillPaths } from "./seat-skill-storage.js";
|
||||
import { resolveSeatKind, seatProcessOwner } from "./seat-kinds.js";
|
||||
import { recordedProcessStillRunning } from "./reconciler.js";
|
||||
import { readProcessTable, type ProcessTableRow } from "./process-table.js";
|
||||
|
||||
/**
|
||||
* S5 (OPR.0.5.4.7) — the seat-lifecycle verb surface: set-model, single-seat stop,
|
||||
@@ -105,6 +107,8 @@ export interface SeatLifecycleDeps {
|
||||
eventBus: EventBus;
|
||||
tmuxAdapter: TmuxAdapter;
|
||||
listProcesses?: NativeProcessLister;
|
||||
/** The OS process table, read before reopening a released seat's tmux server; injectable for tests. */
|
||||
readProcesses?: () => Promise<ProcessTableRow[]>;
|
||||
nodeLauncher?: NodeLauncher;
|
||||
startupOrchestrator?: StartupOrchestrator;
|
||||
runtimeAdapters?: Record<string, RuntimeAdapter>;
|
||||
@@ -333,6 +337,7 @@ export class SeatLifecycleService {
|
||||
private readonly eventBus: EventBus;
|
||||
private readonly tmuxAdapter: TmuxAdapter;
|
||||
private readonly listProcesses?: NativeProcessLister;
|
||||
private readonly readProcesses: () => Promise<ProcessTableRow[]>;
|
||||
private readonly nodeLauncher: NodeLauncher | null;
|
||||
private readonly startupOrchestrator: StartupOrchestrator | null;
|
||||
private readonly runtimeAdapters: Record<string, RuntimeAdapter>;
|
||||
@@ -355,6 +360,7 @@ export class SeatLifecycleService {
|
||||
this.eventBus = deps.eventBus;
|
||||
this.tmuxAdapter = deps.tmuxAdapter;
|
||||
this.listProcesses = deps.listProcesses;
|
||||
this.readProcesses = deps.readProcesses ?? readProcessTable;
|
||||
this.nodeLauncher = deps.nodeLauncher ?? null;
|
||||
this.startupOrchestrator = deps.startupOrchestrator ?? null;
|
||||
this.runtimeAdapters = deps.runtimeAdapters ?? {};
|
||||
@@ -1566,17 +1572,34 @@ export class SeatLifecycleService {
|
||||
// probe sees unavailable transport instead of positive absence. Reopen an
|
||||
// empty server only when the persisted state proves this seat's managed
|
||||
// occupant was deliberately stopped and no other managed row remains live.
|
||||
// `rig unclaim` is as deliberate for an adopted session: it marks the row
|
||||
// detached and clears the binding, and the person ends the process and its
|
||||
// tmux session, which can be the server's last session too.
|
||||
// The probe below still decides absence; a recreated session or failed
|
||||
// transport remains a refusal.
|
||||
if ("code" in canonicalProbe) {
|
||||
const latest = this.latestSession(node.id);
|
||||
const hasCurrentBinding = this.sessionRegistry.getBindingForNode(node.id) !== null;
|
||||
const released = latest?.status === "detached" && latest.origin === "claimed";
|
||||
const stoppedManagedOccupant = latest !== null
|
||||
&& latest.session_name === canonicalSessionName
|
||||
&& latest.status === "exited"
|
||||
&& latest.origin !== "claimed"
|
||||
&& ((latest.status === "exited" && latest.origin !== "claimed") || released)
|
||||
&& !hasCurrentBinding
|
||||
&& this.nonTerminalSessions(node.id).length === 0;
|
||||
// Unclaim doesn't end the process. While the released session's recorded process still runs, its server may
|
||||
// only have lost its socket: a server started there would read the session absent and launch a second occupant
|
||||
// beside it. So keep the refusal, as the reconciler keeps such a session (the same evidence).
|
||||
const stillRuns = stoppedManagedOccupant && released
|
||||
? await recordedProcessStillRunning(this.db, { nodeId: node.id, sessionName: canonicalSessionName },
|
||||
() => this.readProcesses().catch(() => null))
|
||||
: null;
|
||||
if (stillRuns) {
|
||||
return {
|
||||
...canonicalProbe,
|
||||
guidance: `This seat's released session last ran as process ${stillRuns.pid}, which is still running, so it may be `
|
||||
+ "on a tmux server this daemon can't reach. End that process, then launch again.",
|
||||
};
|
||||
}
|
||||
if (stoppedManagedOccupant) {
|
||||
try {
|
||||
const restored = await this.tmuxAdapter.startServer();
|
||||
|
||||
@@ -16,6 +16,7 @@ import { SeatLifecycleService } from "../src/domain/seat-lifecycle-service.js";
|
||||
import { getNodeInventory } from "../src/domain/node-inventory.js";
|
||||
import { PsProjectionService } from "../src/domain/ps-projection.js";
|
||||
import { SeatIdentityStore } from "../src/domain/seat-identity-store.js";
|
||||
import { SeatIdentityReconciler } from "../src/domain/seat-identity-reconciler.js";
|
||||
import type { RuntimeAdapter } from "../src/domain/runtime-adapter.js";
|
||||
import type { ProjectionPlan } from "../src/domain/projection-planner.js";
|
||||
import type { TmuxAdapter, TmuxResult } from "../src/adapters/tmux.js";
|
||||
@@ -24,6 +25,7 @@ import { deriveOriented } from "../src/domain/startup-proof.js";
|
||||
import { deriveRehydrateSessionIdByNode } from "../src/domain/active-occupant.js";
|
||||
import { readFreshOccupantRelations } from "../src/domain/fresh-occupant-relation.js";
|
||||
import type { NativeProcessLister, NativeProcessRow } from "../src/domain/native-process-lineage.js";
|
||||
import type { ProcessTableRow } from "../src/domain/process-table.js";
|
||||
|
||||
import { ClaudeCodeAdapter } from "../src/adapters/claude-code-adapter.js";
|
||||
import { CLAUDE_BYPASS_CONSENT } from "./fixtures/claude-bypass-consent.js";
|
||||
@@ -59,6 +61,7 @@ describe("SeatLifecycleService.launchFresh", () => {
|
||||
let activitySwaps: Array<{ nodeId: string; generation: string }>;
|
||||
let service: SeatLifecycleService;
|
||||
let listProcesses: NativeProcessLister;
|
||||
let readProcesses: () => Promise<ProcessTableRow[]>;
|
||||
|
||||
beforeEach(() => {
|
||||
db = createFullTestDb();
|
||||
@@ -131,6 +134,7 @@ describe("SeatLifecycleService.launchFresh", () => {
|
||||
listProcesses = async () => [{ pid: 4242, ppid: 1, pgid: 4242, tpgid: 4242,
|
||||
executableName: "codex", command: "/opt/native/codex -m model", argv: ["/opt/native/codex", "-m", "model"],
|
||||
startedAt: "Sat Jan 1 12:00:00 2000" }];
|
||||
readProcesses = async () => [];
|
||||
runtimeAdapters = { "claude-code": adapter, codex: { ...adapter, runtime: "codex" } };
|
||||
service = new SeatLifecycleService({
|
||||
db,
|
||||
@@ -139,6 +143,7 @@ describe("SeatLifecycleService.launchFresh", () => {
|
||||
eventBus,
|
||||
tmuxAdapter: tmux,
|
||||
listProcesses: () => listProcesses(),
|
||||
readProcesses: () => readProcesses(),
|
||||
nodeLauncher,
|
||||
startupOrchestrator,
|
||||
runtimeAdapters,
|
||||
@@ -837,6 +842,105 @@ describe("SeatLifecycleService.launchFresh", () => {
|
||||
expect(alive.has(seat.sessionName)).toBe(true);
|
||||
});
|
||||
|
||||
// `rig unclaim` releases an adopted session (its row detached, its binding cleared) and leaves its process running;
|
||||
// the person then ends it and its tmux session. When that was the server's last session, tmux exits with it.
|
||||
const releaseAdopted = (seat: ReturnType<typeof seedSeat>) => {
|
||||
sessionRegistry.markDetached(seat.session!.id);
|
||||
sessionRegistry.clearBinding(seat.node.id);
|
||||
};
|
||||
|
||||
it("reopens an empty tmux server after rig unclaim released the seat's last session", async () => {
|
||||
const seat = seedSeat({ adopted: true });
|
||||
const server = modelServerLifetime();
|
||||
releaseAdopted(seat);
|
||||
alive.delete(seat.sessionName);
|
||||
livePanes.delete(seat.sessionName);
|
||||
server.up = false;
|
||||
|
||||
const result = await service.launchFresh({ seatRef: seat.sessionName, fresh: true, reason: "start a new occupant" });
|
||||
|
||||
expect(result).toMatchObject({ ok: true, status: "ready", sessionName: seat.sessionName });
|
||||
expect(server).toEqual({ up: true, starts: 1 });
|
||||
expect(tmux.createSession).toHaveBeenCalledTimes(1);
|
||||
expect(killed).toEqual([]);
|
||||
});
|
||||
|
||||
it("still refuses a released seat's fresh launch when its session is live behind a transport blip", async () => {
|
||||
const seat = seedSeat({ adopted: true });
|
||||
const server = modelServerLifetime();
|
||||
releaseAdopted(seat);
|
||||
// The released process is still running in its session; only the first probe fails.
|
||||
vi.mocked(tmux.probeSession).mockResolvedValueOnce({ state: "transport_unavailable", cause: "probe blip" });
|
||||
|
||||
const result = await service.launchFresh({ seatRef: seat.sessionName, fresh: true, reason: "start a new occupant" });
|
||||
|
||||
expect(result).toMatchObject({ ok: false, code: "unmanaged_session_collision" });
|
||||
expect(server.starts).toBe(0);
|
||||
expect(tmux.createSession).not.toHaveBeenCalled();
|
||||
expect(killed).toEqual([]);
|
||||
expect(alive.has(seat.sessionName)).toBe(true);
|
||||
});
|
||||
|
||||
// Unclaim doesn't end the process, and tmux can lose its socket while its server keeps running: the probe then
|
||||
// can't reach the server, and a server started at that path is a new, empty one. So the model keeps the two apart:
|
||||
// `alive` and `server.up` are what the socket answers, and the process table is what still runs.
|
||||
const DAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
const lstart = (d: Date) => `${DAYS[d.getDay()]} ${MONTHS[d.getMonth()]} ${String(d.getDate()).padStart(2, " ")} `
|
||||
+ `${[d.getHours(), d.getMinutes(), d.getSeconds()].map((n) => String(n).padStart(2, "0")).join(":")} ${d.getFullYear()}`;
|
||||
const processRow = (pid: number, ppid: number, startedAt: Date, command: string): ProcessTableRow =>
|
||||
({ pid, ppid, pgid: pid, tpgid: -1, executableName: command.split(" ")[0]!, startedAt: lstart(startedAt), command });
|
||||
const observed = new Date(Date.now() - 60_000);
|
||||
const releasedWithLostSocket = async () => {
|
||||
const seat = seedSeat({ adopted: true });
|
||||
const server = modelServerLifetime();
|
||||
// Recorded while the adopted seat ran; then released, and the daemon's identity sweep (every 5 s) runs.
|
||||
new SeatIdentityStore(db).upsert({
|
||||
nodeId: seat.node.id, verdict: "verified", evidenceSource: null, reason: null,
|
||||
evidence: { registeredPane: "%old", observedPid: 5151, observedPanePid: 5150, observedCommand: "claude", matchedLayer: 1 },
|
||||
sessionName: seat.sessionName, observedAt: observed.toISOString(),
|
||||
});
|
||||
releaseAdopted(seat);
|
||||
const blindPoll = { listSessions: async () => [] } as unknown as ConstructorParameters<typeof SeatIdentityReconciler>[0]["tmux"];
|
||||
await new SeatIdentityReconciler({ db, tmux: blindPoll }).reconcileAll();
|
||||
alive.delete(seat.sessionName);
|
||||
livePanes.delete(seat.sessionName);
|
||||
server.up = false;
|
||||
return { seat, server };
|
||||
};
|
||||
|
||||
it("keeps refusing a released seat's fresh launch while its recorded process still runs behind a lost socket", async () => {
|
||||
const { seat } = await releasedWithLostSocket();
|
||||
const before = new Date(observed.getTime() - 10 * 60_000);
|
||||
readProcesses = async () => [processRow(5150, 900, before, "zsh"), processRow(900, 1, before, "tmux new-session")];
|
||||
|
||||
const result = await service.launchFresh({ seatRef: seat.sessionName, fresh: true, reason: "start a new occupant" });
|
||||
|
||||
expect(result).toMatchObject({ ok: false, code: "tmux_probe_failed",
|
||||
guidance: expect.stringContaining("last ran as process 5150, which is still running") });
|
||||
expect(tmux.startServer).not.toHaveBeenCalled();
|
||||
expect(tmux.createSession).not.toHaveBeenCalled();
|
||||
expect(killed).toEqual([]);
|
||||
expect(sessionRegistry.getSessionsForRig(seat.rig.id).map((s) => s.status)).toEqual(["detached"]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "its recorded process has ended", rows: async () => [] as ProcessTableRow[] },
|
||||
{ name: "its pid now belongs to a later process",
|
||||
rows: async () => [processRow(5150, 1, new Date(observed.getTime() + 30_000), "vim")] },
|
||||
{ name: "the process table can't be read", rows: async (): Promise<ProcessTableRow[]> => { throw new Error("ps failed"); } },
|
||||
])("reopens the server for a released seat when $name", async ({ rows }) => {
|
||||
const { seat, server } = await releasedWithLostSocket();
|
||||
readProcesses = rows;
|
||||
|
||||
const result = await service.launchFresh({ seatRef: seat.sessionName, fresh: true, reason: "start a new occupant" });
|
||||
|
||||
expect(result).toMatchObject({ ok: true, status: "ready", sessionName: seat.sessionName });
|
||||
expect(server).toEqual({ up: true, starts: 1 });
|
||||
expect(tmux.createSession).toHaveBeenCalledTimes(1);
|
||||
expect(killed).toEqual([]);
|
||||
});
|
||||
|
||||
it("proves older non-terminal rows absent on the restored server before superseding them", async () => {
|
||||
const seat = seedSeat({ clean: true });
|
||||
const older = sessionRegistry.registerSession(seat.node.id, "r00-dev-impl@fresh-rig");
|
||||
|
||||
@@ -440,6 +440,28 @@ describe("SeatIdentityReconciler.reconcileAll", () => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("keeps a released seat's recorded process through the prune, and still prunes a stopped one", async () => {
|
||||
// rig unclaim leaves the latest row detached with origin claimed, and the process running. A fresh launch reads
|
||||
// that process's record before reopening a tmux server, so the 5 s sweep mustn't drop it.
|
||||
const db = createFullTestDb();
|
||||
seedSeat(db, { nodeId: "n1", sessionName: "s1@rig", pane: "%1", status: "detached" });
|
||||
db.prepare("UPDATE sessions SET origin = 'claimed' WHERE node_id = 'n1'").run();
|
||||
seedSeat(db, { nodeId: "n2", sessionName: "s2@rig", pane: "%2", status: "exited" });
|
||||
const store = new SeatIdentityStore(db);
|
||||
for (const [nodeId, sessionName, pid] of [["n1", "s1@rig", 5151], ["n2", "s2@rig", 6161]] as const) {
|
||||
store.upsert({ nodeId, verdict: "verified", evidenceSource: null, reason: null,
|
||||
evidence: { registeredPane: "%1", observedPid: pid, observedPanePid: pid - 1, observedCommand: "claude", matchedLayer: 1 },
|
||||
sessionName, observedAt: NOW().toISOString() });
|
||||
}
|
||||
const rec = new SeatIdentityReconciler({ db, tmux: makeTmux({ sessions: [] }), now: NOW });
|
||||
|
||||
await rec.reconcileAll();
|
||||
|
||||
expect(store.getProcessEvidence("n1")).toEqual({ sessionName: "s1@rig", pid: 5151, panePid: 5150, observedAt: NOW().toISOString() });
|
||||
expect(store.getForNode("n2")).toBeNull();
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("liveness is NOT derived from heartbeats — a dead pane stays non-green regardless of any heartbeat", async () => {
|
||||
// The reconciler never reads queue/classifier/hook state; a seat whose pane
|
||||
// is gone yields pane_missing even if (hypothetically) work/heartbeats exist.
|
||||
|
||||
Reference in new issue
Block a user