mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-07 22:58:42 -10:00
fix(daemon): keep a seat's last observed process across polls that can't see it
After a restart without the seats' TMUX, the identity poll sees no sessions and records a verdict with no pid. That verdict is honest, but it erased the only evidence startup reconcile had, so a second restart in the same environment detached the seats as before. Keep the last observed process with its own observation time beside the current verdict (migration 135: evidence_pid, evidence_pane_pid, evidence_at, backfilled from verdicts that carry a pid). The store refreshes it from any verdict that observes a process, keeps it through verdicts without one for the same session, and clears it for another session, so every verdict writer behaves the same. Startup reconcile reads that evidence; the process-table check still decides. The unseen record now names the process's immediate parent, which may be a shell or wrapper rather than the tmux server, and the startup warning says "parent process".
This commit is contained in:
1 parent
3fd58712f9
commit
01d6752c84
11 files changed
+192
-29
No files matched your search
@@ -8,6 +8,7 @@ import { terminalPolicySchema } from "./migrations/128_terminal_policy.js";
|
||||
import { streamDispositionsSchema } from "./migrations/129_stream_dispositions.js";
|
||||
import { streamSubscriptionsSchema } from "./migrations/130_stream_subscriptions.js";
|
||||
import { streamRetentionFloorSchema } from "./migrations/131_stream_retention_floor.js";
|
||||
import { seatIdentityProcessEvidenceSchema } from "./migrations/135_seat_identity_process_evidence.js";
|
||||
import { bindingsSessionsSchema } from "./migrations/002_bindings_sessions.js";
|
||||
import { eventsSchema } from "./migrations/003_events.js";
|
||||
import { snapshotsSchema } from "./migrations/004_snapshots.js";
|
||||
@@ -261,4 +262,5 @@ export const ALL_MIGRATIONS: Migration[] = [
|
||||
streamDispositionsSchema,
|
||||
streamSubscriptionsSchema,
|
||||
streamRetentionFloorSchema,
|
||||
seatIdentityProcessEvidenceSchema,
|
||||
];
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { Migration } from "../migrate.js";
|
||||
|
||||
// The last process the identity poll observed for a seat, kept with its own observation time apart from the current
|
||||
// verdict. A verdict that can't see the seat (tmux unavailable, session missing) honestly records no pid; startup
|
||||
// reconcile reads this evidence instead, so a seat whose last observed process still runs isn't read as gone on a
|
||||
// second restart either. Backfilled from current verdicts that carry a pid.
|
||||
export const seatIdentityProcessEvidenceSchema: Migration = {
|
||||
name: "135_seat_identity_process_evidence.sql",
|
||||
sql: `
|
||||
ALTER TABLE seat_identity_verdicts ADD COLUMN evidence_pid INTEGER;
|
||||
ALTER TABLE seat_identity_verdicts ADD COLUMN evidence_pane_pid INTEGER;
|
||||
ALTER TABLE seat_identity_verdicts ADD COLUMN evidence_at TEXT;
|
||||
UPDATE seat_identity_verdicts
|
||||
SET evidence_pid = observed_pid, evidence_pane_pid = observed_pane_pid, evidence_at = observed_at
|
||||
WHERE observed_pid IS NOT NULL OR observed_pane_pid IS NOT NULL;
|
||||
`,
|
||||
};
|
||||
@@ -9,10 +9,11 @@ import { lstartToMinTs } from "./codex-thread-id.js";
|
||||
export interface ReconcileResult {
|
||||
checked: number;
|
||||
detached: number;
|
||||
/** Not on this daemon's tmux server, yet the seat's recorded process is still running: most likely on a tmux server
|
||||
* this daemon can't see (it didn't inherit the seats' TMUX). Left as they were, never detached, so nothing relaunches
|
||||
* them. `server` is that process's parent, its tmux server, when the process table shows it. */
|
||||
unseen: { sessionId: string; sessionName: string; pid: number; server: { pid: number; command: string } | null }[];
|
||||
/** Not on this daemon's tmux server, yet the seat's last observed process is still running: most likely on a tmux
|
||||
* server this daemon can't see (it didn't inherit the seats' TMUX). Left as they were, never detached, so nothing
|
||||
* relaunches them. `parent` is that process's immediate parent when the process table shows it; it may be the tmux
|
||||
* server, or a shell or wrapper in between. */
|
||||
unseen: { sessionId: string; sessionName: string; pid: number; parent: { pid: number; command: string } | null }[];
|
||||
errors: { sessionId: string; error: string }[];
|
||||
}
|
||||
|
||||
@@ -118,18 +119,19 @@ export class Reconciler {
|
||||
return { checked, detached, unseen, errors };
|
||||
}
|
||||
|
||||
/** The seat's recorded process, as a conservative hint that it still runs, not proof of identity: the identity
|
||||
* reconciler's last observation of this very session, and a process with that pid whose start isn't observably later
|
||||
* than the observation. Start times are whole seconds, so a pid reused within the observation's second still passes.
|
||||
* An unreadable table or record is no evidence. */
|
||||
/** The seat's last observed process, as a conservative hint that it still runs, not proof of identity: the last pid
|
||||
* evidence recorded for this very session (kept with its own time, so a later poll that couldn't see the seat doesn't
|
||||
* erase it), and a process with that pid whose start isn't observably later than that observation. Start times are
|
||||
* whole seconds, so a pid reused within the observation's second still passes. An unreadable table or record is no
|
||||
* evidence. */
|
||||
private async recordedProcessStillRunning(
|
||||
session: { nodeId: string; sessionName: string },
|
||||
processes: () => Promise<ProcessTableRow[] | null>,
|
||||
): Promise<{ pid: number; server: { pid: number; command: string } | null } | null> {
|
||||
const verdict = new SeatIdentityStore(this.db).getForNode(session.nodeId);
|
||||
if (!verdict || verdict.sessionName !== session.sessionName) return null;
|
||||
const observedAt = Date.parse(verdict.observedAt) / 1000;
|
||||
const pids = [verdict.evidence.observedPanePid, verdict.evidence.observedPid]
|
||||
): Promise<{ pid: number; parent: { pid: number; command: string } | null } | null> {
|
||||
const evidence = new SeatIdentityStore(this.db).getProcessEvidence(session.nodeId);
|
||||
if (!evidence || evidence.sessionName !== session.sessionName) return null;
|
||||
const observedAt = Date.parse(evidence.observedAt) / 1000;
|
||||
const pids = [evidence.panePid, evidence.pid]
|
||||
.filter((pid): pid is number => typeof pid === "number" && pid > 0);
|
||||
if (!pids.length || Number.isNaN(observedAt)) return null;
|
||||
const rows = await processes();
|
||||
@@ -139,7 +141,7 @@ export class Reconciler {
|
||||
const started = row ? lstartToMinTs(row.startedAt) : undefined;
|
||||
if (!row || started === undefined || started > observedAt) continue;
|
||||
const parent = rows.find((r) => r.pid === row.ppid);
|
||||
return { pid, server: parent ? { pid: parent.pid, command: parent.command } : null };
|
||||
return { pid, parent: parent ? { pid: parent.pid, command: parent.command } : null };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -48,12 +48,17 @@ function rowToVerdict(row: VerdictRow): SeatIdentityVerdict {
|
||||
export class SeatIdentityStore {
|
||||
constructor(private readonly db: Database.Database) {}
|
||||
|
||||
/** Upsert one node's verdict (last-writer-wins per node). */
|
||||
/** Upsert one node's verdict (last-writer-wins per node). The current verdict is always replaced, pids included. The
|
||||
* last observed process (`evidence_*`, with its own time) is refreshed only by a verdict that carries a pid; one
|
||||
* without a pid keeps it for the same session and clears it for another. */
|
||||
upsert(v: SeatIdentityVerdict): void {
|
||||
const observedProcess = v.evidence.observedPid != null || v.evidence.observedPanePid != null;
|
||||
this.db.prepare(`
|
||||
INSERT INTO seat_identity_verdicts
|
||||
(node_id, verdict, evidence_source, reason, registered_pane, observed_pid, observed_pane_pid, observed_command, matched_layer, session_name, observed_at)
|
||||
VALUES (@node_id, @verdict, @evidence_source, @reason, @registered_pane, @observed_pid, @observed_pane_pid, @observed_command, @matched_layer, @session_name, @observed_at)
|
||||
(node_id, verdict, evidence_source, reason, registered_pane, observed_pid, observed_pane_pid, observed_command, matched_layer, session_name, observed_at,
|
||||
evidence_pid, evidence_pane_pid, evidence_at)
|
||||
VALUES (@node_id, @verdict, @evidence_source, @reason, @registered_pane, @observed_pid, @observed_pane_pid, @observed_command, @matched_layer, @session_name, @observed_at,
|
||||
@evidence_pid, @evidence_pane_pid, @evidence_at)
|
||||
ON CONFLICT(node_id) DO UPDATE SET
|
||||
verdict = excluded.verdict,
|
||||
evidence_source = excluded.evidence_source,
|
||||
@@ -64,7 +69,13 @@ export class SeatIdentityStore {
|
||||
observed_command = excluded.observed_command,
|
||||
matched_layer = excluded.matched_layer,
|
||||
session_name = excluded.session_name,
|
||||
observed_at = excluded.observed_at
|
||||
observed_at = excluded.observed_at,
|
||||
evidence_pid = CASE WHEN excluded.evidence_at IS NOT NULL THEN excluded.evidence_pid
|
||||
WHEN excluded.session_name IS seat_identity_verdicts.session_name THEN seat_identity_verdicts.evidence_pid END,
|
||||
evidence_pane_pid = CASE WHEN excluded.evidence_at IS NOT NULL THEN excluded.evidence_pane_pid
|
||||
WHEN excluded.session_name IS seat_identity_verdicts.session_name THEN seat_identity_verdicts.evidence_pane_pid END,
|
||||
evidence_at = CASE WHEN excluded.evidence_at IS NOT NULL THEN excluded.evidence_at
|
||||
WHEN excluded.session_name IS seat_identity_verdicts.session_name THEN seat_identity_verdicts.evidence_at END
|
||||
`).run({
|
||||
node_id: v.nodeId,
|
||||
verdict: v.verdict,
|
||||
@@ -77,6 +88,9 @@ export class SeatIdentityStore {
|
||||
matched_layer: v.evidence.matchedLayer,
|
||||
session_name: v.sessionName,
|
||||
observed_at: v.observedAt,
|
||||
evidence_pid: observedProcess ? v.evidence.observedPid : null,
|
||||
evidence_pane_pid: observedProcess ? v.evidence.observedPanePid ?? null : null,
|
||||
evidence_at: observedProcess ? v.observedAt : null,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -120,6 +134,19 @@ export class SeatIdentityStore {
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The last process observed for a node, with that observation's own time and the session the row belongs to, or
|
||||
* null when none is recorded. Defensive. */
|
||||
getProcessEvidence(nodeId: string): { sessionName: string | null; pid: number | null; panePid: number | null; observedAt: string } | null {
|
||||
try {
|
||||
const row = this.db.prepare(
|
||||
"SELECT session_name, evidence_pid, evidence_pane_pid, evidence_at FROM seat_identity_verdicts WHERE node_id = ?",
|
||||
).get(nodeId) as { session_name: string | null; evidence_pid: number | null; evidence_pane_pid: number | null; evidence_at: string | null } | undefined;
|
||||
return row?.evidence_at ? { sessionName: row.session_name, pid: row.evidence_pid, panePid: row.evidence_pane_pid, observedAt: row.evidence_at } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Read a single node's verdict, or null. Defensive. */
|
||||
getForNode(nodeId: string): SeatIdentityVerdict | null {
|
||||
try {
|
||||
|
||||
@@ -748,8 +748,8 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
|
||||
for (const u of result.unseen) {
|
||||
try {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`startup reconcile: ${u.sessionName} isn't on this daemon's tmux server, but its process ${u.pid} is still running`
|
||||
+ `${u.server ? ` under tmux server process ${u.server.pid} (${u.server.command.slice(0, 160)})` : ""}, so it was left as it was: not marked`
|
||||
console.warn(`startup reconcile: ${u.sessionName} isn't on this daemon's tmux server, but its last observed process ${u.pid} is still running`
|
||||
+ `${u.parent ? ` (parent process ${u.parent.pid}: ${u.parent.command.slice(0, 160)})` : ""}, so it was left as it was: not marked`
|
||||
+ ` detached and not relaunched. The daemon is probably using a different tmux server from its seats (it didn't inherit their`
|
||||
+ ` TMUX). Restart the daemon from inside the seats' tmux server, or with TMUX set to it.`);
|
||||
} catch { /* logging must never throw */ }
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { seatIdentityPanePidSchema } from "../../src/db/migrations/119_seat_identity_pane_pid.js";
|
||||
import { seatIdentityProcessEvidenceSchema } from "../../src/db/migrations/135_seat_identity_process_evidence.js";
|
||||
// Host-shaped synthetic seeder for the /api/ps + /api/rigs/summary event-loop
|
||||
// regression asset (qitem-20260721000001-ps-stall-driver, slice-04).
|
||||
//
|
||||
@@ -62,7 +63,7 @@ import { sessionStopIntentSchema } from "../../src/db/migrations/103_session_sto
|
||||
// index-independent).
|
||||
export function createMigratedDb(): Database.Database {
|
||||
const db = createDb();
|
||||
migrate(db, [coreSchema, bindingsSessionsSchema, eventsSchema, snapshotsSchema, checkpointsSchema, resumeMetadataSchema, nodeSpecFieldsSchema, packagesSchema, installJournalSchema, journalSeqSchema, bootstrapSchema, discoverySchema, discoveryFkFix, agentspecRebootSchema, startupContextSchema, chatMessagesSchema, podNamespaceSchema, contextUsageSchema, externalCliAttachmentSchema, rigServicesSchema, seatHandoverObservabilitySchema, nodeCodexConfigProfileSchema, nodePermissionPolicySchema, rigPermissionPolicySchema, nodePolicyProvenanceSchema, rigPolicyProvenanceSchema, streamItemsSchema, queueItemsSchema, queueTransitionsSchema, rigPolicySchema, rigArchiveSchema, resumeProvenanceSchema, resumeVerificationSchema, seatIdentityVerdictsSchema, eventsNodeTypeIndexSchema, sessionsNodeIdIndexSchema, occupantTenuresSchema, sessionStopIntentSchema, seatIdentityPanePidSchema]);
|
||||
migrate(db, [coreSchema, bindingsSessionsSchema, eventsSchema, snapshotsSchema, checkpointsSchema, resumeMetadataSchema, nodeSpecFieldsSchema, packagesSchema, installJournalSchema, journalSeqSchema, bootstrapSchema, discoverySchema, discoveryFkFix, agentspecRebootSchema, startupContextSchema, chatMessagesSchema, podNamespaceSchema, contextUsageSchema, externalCliAttachmentSchema, rigServicesSchema, seatHandoverObservabilitySchema, nodeCodexConfigProfileSchema, nodePermissionPolicySchema, rigPermissionPolicySchema, nodePolicyProvenanceSchema, rigPolicyProvenanceSchema, streamItemsSchema, queueItemsSchema, queueTransitionsSchema, rigPolicySchema, rigArchiveSchema, resumeProvenanceSchema, resumeVerificationSchema, seatIdentityVerdictsSchema, eventsNodeTypeIndexSchema, sessionsNodeIdIndexSchema, occupantTenuresSchema, sessionStopIntentSchema, seatIdentityPanePidSchema, seatIdentityProcessEvidenceSchema]);
|
||||
return db;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { terminalPolicySchema } from "../../src/db/migrations/128_terminal_policy.js";
|
||||
import { seatIdentityPanePidSchema } from "../../src/db/migrations/119_seat_identity_pane_pid.js";
|
||||
import { seatIdentityProcessEvidenceSchema } from "../../src/db/migrations/135_seat_identity_process_evidence.js";
|
||||
import { rigStartupLayersSchema } from "../../src/db/migrations/121_rig_startup_layers.js";
|
||||
import { humanOperatingModesSchema } from "../../src/db/migrations/122_human_operating_modes.js";
|
||||
import { queueDelegatedDecisionSchema } from "../../src/db/migrations/123_queue_delegated_decision.js";
|
||||
@@ -142,7 +143,7 @@ import fs from "node:fs";
|
||||
|
||||
/** Seam B R6: the canonical full-fixture migration list, exported so file-backed
|
||||
* DB-reopen tests migrate IDENTICALLY to createFullTestDb. */
|
||||
export const migrationsForFullTestDb = [coreSchema, bindingsSessionsSchema, eventsSchema, snapshotsSchema, checkpointsSchema, resumeMetadataSchema, nodeSpecFieldsSchema, packagesSchema, installJournalSchema, journalSeqSchema, bootstrapSchema, discoverySchema, discoveryFkFix, agentspecRebootSchema, startupContextSchema, chatMessagesSchema, podNamespaceSchema, contextUsageSchema, externalCliAttachmentSchema, rigServicesSchema, seatHandoverObservabilitySchema, nodeCodexConfigProfileSchema, nodePermissionPolicySchema, rigPermissionPolicySchema, nodePolicyProvenanceSchema, rigPolicyProvenanceSchema, streamItemsSchema, queueItemsSchema, queueTransitionsSchema, rigPolicySchema, rigArchiveSchema, resumeProvenanceSchema, resumeVerificationSchema, seatIdentityVerdictsSchema, selfHostIdentitySchema, occupantTenuresSchema, daemonLifecycleSchema, watchdogJobsSchema, occupantGenerationStampsSchema, projectionManifestSchema, watchdogTargetGenerationSchema, appliedLaunchObservationsSchema, appliedLaunchObservationInvalidationsSchema, threadSeatMapSchema, queueTransitionWakesSchema, nodeSessionSourceSchema, scopedOperatingPostureSchema, humanNotificationIntentSchema, reviewReadIndexesSchema, inventoryEventIndexesSchema, rigClaudeManagedBlockFileSchema, nodePermissionSelectionsSchema, humanReplyToSchema, humanQuestionsSchema, nodeEffortSchema, rigNonInterruptiveSchema, rigInstallRootSchema, threadPartMapSchema, queueTransitionsQitemIdOrderSchema, queueItemsRecoveryTagsSchema, resumeRotationSchema, sessionStopIntentSchema, nodeEffortChoiceSchema, modePostureCoexistSchema, seatEnvSchema, appliedLaunchDaemonOptOutSchema, sessionCommunityHarnessSchema, threadSeatMapChannelKeySchema, queueItemWakeInputBoxSchema, providerAccountReadingsSchema, humanQuietWindowsSchema, seatIdentityPanePidSchema, rigStartupLayersSchema, humanOperatingModesSchema, queueDelegatedDecisionSchema, nodeAdvisorSchema, operationCorrelationSchema, terminalPolicySchema];
|
||||
export const migrationsForFullTestDb = [coreSchema, bindingsSessionsSchema, eventsSchema, snapshotsSchema, checkpointsSchema, resumeMetadataSchema, nodeSpecFieldsSchema, packagesSchema, installJournalSchema, journalSeqSchema, bootstrapSchema, discoverySchema, discoveryFkFix, agentspecRebootSchema, startupContextSchema, chatMessagesSchema, podNamespaceSchema, contextUsageSchema, externalCliAttachmentSchema, rigServicesSchema, seatHandoverObservabilitySchema, nodeCodexConfigProfileSchema, nodePermissionPolicySchema, rigPermissionPolicySchema, nodePolicyProvenanceSchema, rigPolicyProvenanceSchema, streamItemsSchema, queueItemsSchema, queueTransitionsSchema, rigPolicySchema, rigArchiveSchema, resumeProvenanceSchema, resumeVerificationSchema, seatIdentityVerdictsSchema, selfHostIdentitySchema, occupantTenuresSchema, daemonLifecycleSchema, watchdogJobsSchema, occupantGenerationStampsSchema, projectionManifestSchema, watchdogTargetGenerationSchema, appliedLaunchObservationsSchema, appliedLaunchObservationInvalidationsSchema, threadSeatMapSchema, queueTransitionWakesSchema, nodeSessionSourceSchema, scopedOperatingPostureSchema, humanNotificationIntentSchema, reviewReadIndexesSchema, inventoryEventIndexesSchema, rigClaudeManagedBlockFileSchema, nodePermissionSelectionsSchema, humanReplyToSchema, humanQuestionsSchema, nodeEffortSchema, rigNonInterruptiveSchema, rigInstallRootSchema, threadPartMapSchema, queueTransitionsQitemIdOrderSchema, queueItemsRecoveryTagsSchema, resumeRotationSchema, sessionStopIntentSchema, nodeEffortChoiceSchema, modePostureCoexistSchema, seatEnvSchema, appliedLaunchDaemonOptOutSchema, sessionCommunityHarnessSchema, threadSeatMapChannelKeySchema, queueItemWakeInputBoxSchema, providerAccountReadingsSchema, humanQuietWindowsSchema, seatIdentityPanePidSchema, seatIdentityProcessEvidenceSchema, rigStartupLayersSchema, humanOperatingModesSchema, queueDelegatedDecisionSchema, nodeAdvisorSchema, operationCorrelationSchema, terminalPolicySchema];
|
||||
|
||||
/**
|
||||
* P24 — the DECLARED exclusions for {@link migrationsForFullTestDb}. That list is deliberately a
|
||||
|
||||
@@ -11,11 +11,13 @@ import { SessionRegistry } from "../src/domain/session-registry.js";
|
||||
import { EventBus } from "../src/domain/event-bus.js";
|
||||
import { Reconciler } from "../src/domain/reconciler.js";
|
||||
import { SeatIdentityStore } from "../src/domain/seat-identity-store.js";
|
||||
import { SeatIdentityReconciler } from "../src/domain/seat-identity-reconciler.js";
|
||||
import { createFullTestDb } from "./helpers/test-app.js";
|
||||
|
||||
// The incident, with a real tmux: the seats live on a tmux server named by a socket the daemon didn't inherit (no TMUX
|
||||
// in its environment), so the daemon's own tmux reaches an empty default server. Its startup reconcile must leave the
|
||||
// seat as it was while the seat's process runs, and detach it once that server is really gone.
|
||||
// seat as it was while the seat's process runs, also after its own identity poll saw nothing, find it again once TMUX
|
||||
// names the seats' socket, and detach it once that server is really gone.
|
||||
|
||||
let nativeAvailable = process.platform !== "win32";
|
||||
try { execFileSync("tmux", ["-V"], { timeout: 2000, stdio: "ignore" }); } catch { nativeAvailable = false; }
|
||||
@@ -29,7 +31,7 @@ afterAll(async () => {
|
||||
});
|
||||
|
||||
describe.skipIf(!nativeAvailable)("startup reconcile against a real tmux server it can't see", () => {
|
||||
it("leaves a running seat as it was, names its server, and detaches it once that server is gone", async () => {
|
||||
it("leaves a running seat as it was across two blind restarts, finds it on its own server, and detaches it once that server is gone", async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "openrig-unseen-"));
|
||||
roots.push(root);
|
||||
const sock = path.join(root, "seats.sock");
|
||||
@@ -66,9 +68,20 @@ describe.skipIf(!nativeAvailable)("startup reconcile against a real tmux server
|
||||
|
||||
const unseen = await reconciler.reconcile(rig.id);
|
||||
expect(unseen).toMatchObject({ detached: 0, unseen: [{ sessionName: "advisor-lead@demo", pid: panePid }] });
|
||||
expect(unseen.unseen[0]!.server?.command).toContain(sock);
|
||||
expect(unseen.unseen[0]!.parent?.command).toContain(sock);
|
||||
expect(sessionRegistry.getSessionsForRig(rig.id)[0]!.status).toBe("running");
|
||||
|
||||
// The blind daemon's own identity poll sees no sessions and honestly records no pid; a second blind restart still
|
||||
// finds the seat's last observed process running.
|
||||
await new SeatIdentityReconciler({ db, tmux: new TmuxAdapter(execCommand) }).reconcileAll();
|
||||
expect(new SeatIdentityStore(db).getForNode(node.id)!.evidence.observedPid).toBeNull();
|
||||
expect(await reconciler.reconcile(rig.id)).toMatchObject({ detached: 0, unseen: [{ pid: panePid }] });
|
||||
|
||||
// Restarted with TMUX naming the seats' socket, the daemon sees the seat again.
|
||||
vi.stubEnv("TMUX", `${sock},0,0`);
|
||||
expect(await reconciler.reconcile(rig.id)).toMatchObject({ checked: 1, detached: 0, unseen: [] });
|
||||
vi.stubEnv("TMUX", undefined);
|
||||
|
||||
await run("tmux", ["-S", sock, "kill-server"], { env: seatEnv });
|
||||
for (let i = 0; i < 50; i++) {
|
||||
try { process.kill(panePid, 0); } catch { break; }
|
||||
|
||||
@@ -5,6 +5,7 @@ import { SessionRegistry } from "../src/domain/session-registry.js";
|
||||
import { EventBus } from "../src/domain/event-bus.js";
|
||||
import { Reconciler } from "../src/domain/reconciler.js";
|
||||
import { SeatIdentityStore } from "../src/domain/seat-identity-store.js";
|
||||
import { SeatIdentityReconciler } from "../src/domain/seat-identity-reconciler.js";
|
||||
import type { ProcessTableRow } from "../src/domain/process-table.js";
|
||||
import type { TmuxAdapter } from "../src/adapters/tmux.js";
|
||||
import { createFullTestDb } from "./helpers/test-app.js";
|
||||
@@ -22,11 +23,14 @@ const lstart = (d: Date) => `${DAYS[d.getDay()]} ${MONTHS[d.getMonth()]} ${Strin
|
||||
const OBSERVED = new Date("2026-10-24T04:40:00.000Z");
|
||||
const BEFORE = new Date(OBSERVED.getTime() - 10 * 60_000);
|
||||
const AFTER = new Date(OBSERVED.getTime() + 60_000);
|
||||
const LATER = new Date(OBSERVED.getTime() + 5 * 60_000);
|
||||
const SERVER = "tmux -S /tmp/seats.sock new-session -d -s anchor";
|
||||
const row = (pid: number, ppid: number, startedAt: Date, command: string): ProcessTableRow =>
|
||||
({ pid, ppid, pgid: pid, tpgid: -1, executableName: command.split(" ")[0]!, startedAt: lstart(startedAt), command });
|
||||
|
||||
const noServer = { probeSession: async () => ({ state: "transport_unavailable" as const }) } as unknown as TmuxAdapter;
|
||||
/** The restarted daemon's identity poll, on a server where none of the seats' sessions exist. */
|
||||
const blindPoll = { listSessions: async () => [] } as unknown as ConstructorParameters<typeof SeatIdentityReconciler>[0]["tmux"];
|
||||
|
||||
describe("startup reconcile: a seat on a tmux server this daemon can't see", () => {
|
||||
let db: Database.Database;
|
||||
@@ -34,6 +38,7 @@ describe("startup reconcile: a seat on a tmux server this daemon can't see", ()
|
||||
let eventBus: EventBus;
|
||||
let rigId: string;
|
||||
let sessionId: string;
|
||||
let nodeId: string;
|
||||
beforeEach(() => {
|
||||
db = createFullTestDb();
|
||||
sessionRegistry = new SessionRegistry(db);
|
||||
@@ -45,6 +50,7 @@ describe("startup reconcile: a seat on a tmux server this daemon can't see", ()
|
||||
sessionRegistry.updateStatus(session.id, "running");
|
||||
rigId = rig.id;
|
||||
sessionId = session.id;
|
||||
nodeId = node.id;
|
||||
new SeatIdentityStore(db).upsert({
|
||||
nodeId: node.id, verdict: "verified", evidenceSource: null, reason: null,
|
||||
evidence: { registeredPane: "%1", observedPid: 4242, observedPanePid: 4241, observedCommand: "claude", matchedLayer: 1 },
|
||||
@@ -58,10 +64,10 @@ describe("startup reconcile: a seat on a tmux server this daemon can't see", ()
|
||||
const status = () => sessionRegistry.getSessionsForRig(rigId)[0]!.status;
|
||||
const detachedEvents = () => (db.prepare("SELECT count(*) AS n FROM events WHERE type = 'session.detached'").get() as { n: number }).n;
|
||||
|
||||
it("leaves it as it was while its recorded process still runs, and names that process's tmux server", async () => {
|
||||
it("leaves it as it was while its recorded process still runs, and names that process's parent", async () => {
|
||||
const result = await reconcile(async () => [row(4241, 900, BEFORE, "claude"), row(900, 1, BEFORE, SERVER)]);
|
||||
expect(result).toMatchObject({ checked: 1, detached: 0,
|
||||
unseen: [{ sessionId, sessionName: "advisor-lead@demo", pid: 4241, server: { pid: 900, command: SERVER } }] });
|
||||
unseen: [{ sessionId, sessionName: "advisor-lead@demo", pid: 4241, parent: { pid: 900, command: SERVER } }] });
|
||||
expect(status()).toBe("running");
|
||||
expect(detachedEvents()).toBe(0);
|
||||
});
|
||||
@@ -69,7 +75,7 @@ describe("startup reconcile: a seat on a tmux server this daemon can't see", ()
|
||||
it("does the same when the server it looked at runs but doesn't have the session", async () => {
|
||||
const absent = { probeSession: async () => ({ state: "absent" as const }) } as unknown as TmuxAdapter;
|
||||
const result = await reconcile(async () => [row(4241, 900, BEFORE, "claude")], absent);
|
||||
expect(result).toMatchObject({ detached: 0, unseen: [{ pid: 4241, server: null }] });
|
||||
expect(result).toMatchObject({ detached: 0, unseen: [{ pid: 4241, parent: null }] });
|
||||
expect(status()).toBe("running");
|
||||
});
|
||||
|
||||
@@ -89,4 +95,35 @@ describe("startup reconcile: a seat on a tmux server this daemon can't see", ()
|
||||
const result = await reconcile(async () => [row(4241, 900, BEFORE, "claude")]);
|
||||
expect(result).toMatchObject({ detached: 1, unseen: [] });
|
||||
});
|
||||
|
||||
describe("on a second restart, after the restarted daemon's own identity poll couldn't see the seat", () => {
|
||||
const running = async () => [row(4241, 900, BEFORE, "claude"), row(900, 1, BEFORE, SERVER)];
|
||||
const firstRestartThenPoll = async () => {
|
||||
expect(await reconcile(running)).toMatchObject({ detached: 0, unseen: [{ pid: 4241 }] });
|
||||
await new SeatIdentityReconciler({ db, tmux: blindPoll, now: () => LATER }).reconcileAll();
|
||||
// The current verdict stays honest: the poll saw nothing and records no pid.
|
||||
expect(new SeatIdentityStore(db).getForNode(nodeId)).toMatchObject({
|
||||
verdict: "tmux_unavailable", observedAt: LATER.toISOString(), evidence: { observedPid: null } });
|
||||
};
|
||||
|
||||
it("still leaves it as it was while its last observed process runs", async () => {
|
||||
await firstRestartThenPoll();
|
||||
expect(await reconcile(running)).toMatchObject({ detached: 0, unseen: [{ pid: 4241, parent: { pid: 900 } }] });
|
||||
expect(status()).toBe("running");
|
||||
expect(detachedEvents()).toBe(0);
|
||||
});
|
||||
|
||||
it("detaches once that process is gone", async () => {
|
||||
await firstRestartThenPoll();
|
||||
expect(await reconcile(async () => [row(900, 1, BEFORE, SERVER)])).toMatchObject({ detached: 1, unseen: [] });
|
||||
expect(status()).toBe("detached");
|
||||
});
|
||||
|
||||
it("finds the seat again once the daemon looks at the seats' own server", async () => {
|
||||
await firstRestartThenPoll();
|
||||
const present = { probeSession: async () => ({ state: "present" as const }) } as unknown as TmuxAdapter;
|
||||
expect(await reconcile(running, present)).toMatchObject({ checked: 1, detached: 0, unseen: [] });
|
||||
expect(status()).toBe("running");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { expect, it } from "vitest";
|
||||
import { migrate } from "../src/db/migrate.js";
|
||||
import { seatIdentityVerdictsSchema } from "../src/db/migrations/046_seat_identity_verdicts.js";
|
||||
import { seatIdentityPanePidSchema } from "../src/db/migrations/119_seat_identity_pane_pid.js";
|
||||
import { seatIdentityProcessEvidenceSchema } from "../src/db/migrations/135_seat_identity_process_evidence.js";
|
||||
import { SeatIdentityStore } from "../src/domain/seat-identity-store.js";
|
||||
|
||||
it("upgrades legacy verdicts without inventing a pane root from the native child PID", () => {
|
||||
@@ -11,7 +12,7 @@ it("upgrades legacy verdicts without inventing a pane root from the native child
|
||||
migrate(db, [seatIdentityVerdictsSchema]);
|
||||
db.prepare(`INSERT INTO seat_identity_verdicts
|
||||
(node_id, verdict, observed_pid, observed_at) VALUES ('legacy', 'verified', 20, '2026-10-01T00:00:00Z')`).run();
|
||||
migrate(db, [seatIdentityVerdictsSchema, seatIdentityPanePidSchema]);
|
||||
migrate(db, [seatIdentityVerdictsSchema, seatIdentityPanePidSchema, seatIdentityProcessEvidenceSchema]);
|
||||
const store = new SeatIdentityStore(db);
|
||||
expect(db.prepare("SELECT observed_pid, observed_pane_pid FROM seat_identity_verdicts").get())
|
||||
.toEqual({ observed_pid: 20, observed_pane_pid: null });
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import Database from "better-sqlite3";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { migrate } from "../src/db/migrate.js";
|
||||
import { seatIdentityVerdictsSchema } from "../src/db/migrations/046_seat_identity_verdicts.js";
|
||||
import { seatIdentityPanePidSchema } from "../src/db/migrations/119_seat_identity_pane_pid.js";
|
||||
import { seatIdentityProcessEvidenceSchema } from "../src/db/migrations/135_seat_identity_process_evidence.js";
|
||||
import { SeatIdentityStore } from "../src/domain/seat-identity-store.js";
|
||||
import type { SeatIdentityVerdict } from "../src/domain/types.js";
|
||||
|
||||
// The last process observed for a seat is kept with its own time, apart from the current verdict, so a verdict that
|
||||
// couldn't see the seat stays honest without erasing what startup reconcile needs on the next restart.
|
||||
|
||||
const T0 = "2026-10-24T05:00:00.000Z";
|
||||
const T1 = "2026-10-24T05:00:05.000Z";
|
||||
const verdict = (sessionName: string, observedAt: string, pid: number | null, panePid: number | null = pid): SeatIdentityVerdict => ({
|
||||
nodeId: "n1", verdict: pid === null ? "tmux_unavailable" : "verified", evidenceSource: null,
|
||||
reason: pid === null ? "tmux_unavailable" : null, sessionName, observedAt,
|
||||
evidence: { registeredPane: "%1", observedPid: pid, ...(panePid === null ? {} : { observedPanePid: panePid }), observedCommand: null, matchedLayer: null },
|
||||
});
|
||||
|
||||
describe("a seat's last observed process", () => {
|
||||
const withStore = (test: (store: SeatIdentityStore) => void) => {
|
||||
const db = new Database(":memory:");
|
||||
try {
|
||||
migrate(db, [seatIdentityVerdictsSchema, seatIdentityPanePidSchema, seatIdentityProcessEvidenceSchema]);
|
||||
test(new SeatIdentityStore(db));
|
||||
} finally { db.close(); }
|
||||
};
|
||||
|
||||
it("survives a verdict for the same session that saw no process, which itself stays honest", () => withStore((store) => {
|
||||
store.upsert(verdict("s@r", T0, 42, 41));
|
||||
store.upsert(verdict("s@r", T1, null));
|
||||
expect(store.getForNode("n1")).toMatchObject({ verdict: "tmux_unavailable", observedAt: T1, evidence: { observedPid: null } });
|
||||
expect(store.getForNode("n1")!.evidence).not.toHaveProperty("observedPanePid");
|
||||
expect(store.getProcessEvidence("n1")).toEqual({ sessionName: "s@r", pid: 42, panePid: 41, observedAt: T0 });
|
||||
}));
|
||||
|
||||
it("is replaced by the next verdict that observes a process", () => withStore((store) => {
|
||||
store.upsert(verdict("s@r", T0, 42, 41));
|
||||
store.upsert(verdict("s@r", T1, 52, 51));
|
||||
expect(store.getProcessEvidence("n1")).toEqual({ sessionName: "s@r", pid: 52, panePid: 51, observedAt: T1 });
|
||||
}));
|
||||
|
||||
it("is cleared by a verdict without a process for another session", () => withStore((store) => {
|
||||
store.upsert(verdict("s@r", T0, 42, 41));
|
||||
store.upsert(verdict("s2@r", T1, null));
|
||||
expect(store.getProcessEvidence("n1")).toBeNull();
|
||||
}));
|
||||
|
||||
it("is backfilled at upgrade from verdicts that carried a pid", () => {
|
||||
const db = new Database(":memory:");
|
||||
try {
|
||||
migrate(db, [seatIdentityVerdictsSchema, seatIdentityPanePidSchema]);
|
||||
db.prepare(`INSERT INTO seat_identity_verdicts (node_id, verdict, observed_pid, observed_pane_pid, session_name, observed_at)
|
||||
VALUES ('n1', 'verified', 42, 41, 's@r', ?), ('n2', 'tmux_unavailable', NULL, NULL, 't@r', ?)`).run(T0, T0);
|
||||
migrate(db, [seatIdentityVerdictsSchema, seatIdentityPanePidSchema, seatIdentityProcessEvidenceSchema]);
|
||||
const store = new SeatIdentityStore(db);
|
||||
expect(store.getProcessEvidence("n1")).toEqual({ sessionName: "s@r", pid: 42, panePid: 41, observedAt: T0 });
|
||||
expect(store.getProcessEvidence("n2")).toBeNull();
|
||||
} finally { db.close(); }
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user