fix(daemon): classify tmux socket-absence as missing session and log startup reconcile

After a host reboot the tmux socket file at /tmp/tmux-<uid>/<name> is gone, so
`tmux has-session` exits non-zero with a transport-absent message rather than
the server/session-absent messages the existing classifier knows. The startup
reconciler caught the unknown error, recorded it silently, and skipped marking
the row detached, leaving DB rows that said `running`/`ready` against a tmux
reality where the sessions were gone.

Add `isTmuxTransportAbsentError` to the tmux adapter; classify post-reboot
socket-absence (`No such file or directory`) and `Connection refused` against
a tmux socket path as absence. Wire it into `hasSession`, `listSessions`,
`listWindows`, and `listPanes` for symmetry. Permission errors (`Permission
denied`, `Operation not permitted`, `EACCES`, `EPERM`) remain rethrown so
callers stay fail-closed; the classifier checks the permission keywords
before any absence pattern.

Capture each `Reconciler.reconcile(rig.id)` aggregate at startup and log a
compact summary line:

  startup reconcile: rigs=<n> checked=<n> detached=<n> errors=<n>

Per-rig errors are written as warnings without crashing startup. Logging is
guarded so it can never throw out of `createDaemon`.

Tests:
- packages/daemon/test/tmux-adapter.test.ts — 4 new `hasSession` cases
  (2 socket-absent variants, 2 permission rethrow regressions).
- packages/daemon/test/reconciler.test.ts — explicit socket-absent detach
  case + explicit permission-rethrow case.
- packages/daemon/test/startup.test.ts — assertion that the compact reconcile
  summary line appears in startup output.

Refs:
- openrig-work/field-notes/2026-04-28-openrig-post-reboot-tmux-restore-postmortem.md
- rigs/openrig-build/state/orch/velocity-slice-L1-cold-start-tmux-truth-repair-plan-20260428.md

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
mvschwarzandClaude Opus 4.7 committed 2026-04-28 12:26:48 -07:00
1 parent 17812d5d3b
commit 3ad1df6339
5 files changed
+169 -10

No files matched your search

+33 -8
View File
@@ -43,6 +43,30 @@ function isSessionAbsenceError(err: unknown): boolean {
msg.includes("no session");
}
// Post-reboot the tmux socket file at /tmp/tmux-<uid>/<name> is gone, so
// `tmux has-session` exits non-zero with a transport-absent message rather than
// a server/session-absent message. Treat that case as "no session" so cold-start
// reconciliation can detach stale rows. Permission errors must remain rethrown.
function isTmuxTransportAbsentError(err: unknown): boolean {
if (!(err instanceof Error)) return false;
const msg = err.message;
// Fail-closed: never classify a permission/authorization failure as absence.
if (/permission denied|operation not permitted|EACCES|EPERM/i.test(msg)) {
return false;
}
// tmux's socket-transport failure prefix; the parenthetical names the cause.
// "error connecting to /private/tmp/tmux-501/default (No such file or directory)"
// "error connecting to /private/tmp/tmux-501/default (Connection refused)"
if (msg.startsWith("error connecting to")) {
return /No such file or directory|Connection refused/.test(msg);
}
// Conservative bare-message variants that still reference a tmux socket path.
if (/tmux-\d+/.test(msg) && /No such file or directory|Connection refused/.test(msg)) {
return true;
}
return false;
}
function classifyWriteError(err: unknown): TmuxResult {
if (!(err instanceof Error)) {
return { ok: false, code: "unknown", message: String(err) };
@@ -122,7 +146,7 @@ export class TmuxAdapter {
const output = await this.exec(`tmux list-sessions -F "${SESSION_FORMAT}"`);
return parseLines(output, parseSessionLine);
} catch (err) {
if (isNoServerError(err)) return [];
if (isNoServerError(err) || isTmuxTransportAbsentError(err)) return [];
throw err;
}
}
@@ -132,7 +156,7 @@ export class TmuxAdapter {
const output = await this.exec(`tmux list-windows -t ${shellQuote(sessionName)} -F "${WINDOW_FORMAT}"`);
return parseLines(output, parseWindowLine);
} catch (err) {
if (isNoServerError(err)) return [];
if (isNoServerError(err) || isTmuxTransportAbsentError(err)) return [];
throw err;
}
}
@@ -142,7 +166,7 @@ export class TmuxAdapter {
const output = await this.exec(`tmux list-panes -t ${shellQuote(target)} -F "${PANE_FORMAT}"`);
return parseLines(output, parsePaneLine);
} catch (err) {
if (isNoServerError(err)) return [];
if (isNoServerError(err) || isTmuxTransportAbsentError(err)) return [];
throw err;
}
}
@@ -155,13 +179,14 @@ export class TmuxAdapter {
await this.exec(`tmux has-session -t ${shellQuote(name)}`);
return true; // exit 0 = session exists
} catch (err) {
// Known-absence patterns: session genuinely doesn't exist or tmux not running.
// Return false so post-crash restore can proceed for dead sessions.
if (isNoServerError(err) || isSessionAbsenceError(err)) {
// Known-absence patterns: session genuinely doesn't exist, tmux not running,
// or the tmux socket file is gone post-reboot.
// Return false so cold-start reconciliation can detach stale rows.
if (isNoServerError(err) || isSessionAbsenceError(err) || isTmuxTransportAbsentError(err)) {
return false;
}
// Unexpected probe failure (permission denied, socket error, etc.) — rethrow
// so callers can fail closed rather than treating a probe failure as absence.
// Unexpected probe failure (permission denied, etc.) — rethrow so callers
// can fail closed rather than treating a probe failure as absence.
throw err;
}
}
+28 -2
View File
@@ -170,12 +170,38 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
// Connect to cmux at startup — degrades gracefully if absent
await cmuxAdapter.connect();
// Reconcile all managed rigs — marks stale sessions as detached
// Reconcile all managed rigs — marks stale sessions as detached.
// Capture aggregate counts and log a compact summary so cold-start truth
// repair is visible in daemon output instead of silently swallowed.
const reconciler = new Reconciler({ db, sessionRegistry, eventBus, tmuxAdapter });
const rigs = rigRepo.listRigs();
let reconcileChecked = 0;
let reconcileDetached = 0;
let reconcileErrors = 0;
for (const rig of rigs) {
await reconciler.reconcile(rig.id);
try {
const result = await reconciler.reconcile(rig.id);
reconcileChecked += result.checked;
reconcileDetached += result.detached;
reconcileErrors += result.errors.length;
for (const e of result.errors) {
try {
// eslint-disable-next-line no-console
console.warn(`startup reconcile warning: rig=${rig.id} session=${e.sessionId} error=${e.error}`);
} catch { /* logging must never throw */ }
}
} catch (err) {
reconcileErrors += 1;
try {
// eslint-disable-next-line no-console
console.warn(`startup reconcile warning: rig=${rig.id} error=${err instanceof Error ? err.message : String(err)}`);
} catch { /* logging must never throw */ }
}
}
try {
// eslint-disable-next-line no-console
console.log(`startup reconcile: rigs=${rigs.length} checked=${reconcileChecked} detached=${reconcileDetached} errors=${reconcileErrors}`);
} catch { /* logging must never throw */ }
const podRepo = new PodRepository(db);
const rigSpecExporter = new RigSpecExporter({ rigRepo, sessionRegistry, podRepo });
+43
View File
@@ -260,4 +260,47 @@ describe("Reconciler", () => {
const sessions = sessionRegistry.getSessionsForRig(rig.id);
expect(sessions[0]!.status).toBe("running");
});
// L1 cold-start tmux truth repair: when the tmux adapter classifies a post-reboot
// socket-absence as "no session" (hasSession returns false), reconcile must
// detach the row honestly so DB state matches tmux reality.
it("socket-absent at adapter layer (hasSession returns false): session detached + event emitted", async () => {
const { rig } = seedRigWithSessions([
{ logicalId: "dev1-impl", sessionName: "r01-dev1-impl", status: "running" },
]);
const reconciler = createReconciler(mockTmuxAdapter({ "r01-dev1-impl": false }));
const result = await reconciler.reconcile(rig.id);
expect(result.checked).toBe(1);
expect(result.detached).toBe(1);
const sessions = sessionRegistry.getSessionsForRig(rig.id);
expect(sessions[0]!.status).toBe("detached");
const events = db
.prepare("SELECT * FROM events WHERE type = 'session.detached'")
.all();
expect(events).toHaveLength(1);
});
it("permission error at adapter layer (hasSession rethrows): session NOT detached, error recorded", async () => {
const { rig } = seedRigWithSessions([
{ logicalId: "dev1-impl", sessionName: "r01-dev1-impl", status: "running" },
]);
const permErr = new Error("error connecting to /private/tmp/tmux-501/default (Operation not permitted)");
const reconciler = createReconciler(
mockTmuxAdapter({}, { "r01-dev1-impl": permErr })
);
const result = await reconciler.reconcile(rig.id);
expect(result.checked).toBe(0);
expect(result.detached).toBe(0);
expect(result.errors).toHaveLength(1);
expect(result.errors[0]!.error).toContain("Operation not permitted");
const sessions = sessionRegistry.getSessionsForRig(rig.id);
expect(sessions[0]!.status).toBe("running");
});
});
+35
View File
@@ -230,4 +230,39 @@ describe("createDaemon startup composition", () => {
db.close();
});
// L1 cold-start tmux truth repair: startup must surface a compact reconcile
// summary so silent reconciliation drift is visible in daemon output.
it("startup logs compact reconcile summary line", async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-test-"));
const dbPath = path.join(tmpDir, "test.sqlite");
seedDbWithStaleSessions(dbPath, [
{ rigName: "r01", logicalId: "dev1-impl", sessionName: "r01-dev1-impl" },
]);
const tmuxExec: ExecFn = async (cmd: string) => {
if (cmd.includes("has-session")) throw new Error("session not found");
return "";
};
const cmuxExec: ExecFn = async () => { throw Object.assign(new Error(""), { code: "ENOENT" }); };
const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
try {
const { db } = await createDaemon({ dbPath, tmuxExec, cmuxExec });
const calls = logSpy.mock.calls.map((c: unknown[]) => String(c[0]));
const summary = calls.find((line) => line.startsWith("startup reconcile:"));
expect(summary).toBeDefined();
expect(summary).toMatch(/rigs=1\b/);
expect(summary).toMatch(/checked=1\b/);
expect(summary).toMatch(/detached=1\b/);
expect(summary).toMatch(/errors=0\b/);
db.close();
} finally {
logSpy.mockRestore();
fs.rmSync(tmpDir, { recursive: true });
}
});
});
+30
View File
@@ -195,6 +195,36 @@ describe("TmuxAdapter", () => {
}));
await expect(adapter.hasSession("any-session")).rejects.toThrow("exit code 127");
});
// L1 cold-start tmux truth repair: post-reboot socket absence must classify
// as "no session" so the reconciler can detach stale rows without manual fix.
it("returns false when tmux socket is gone post-reboot (No such file or directory)", async () => {
const adapter = new TmuxAdapter(mockExec({
"has-session": { error: new Error("error connecting to /private/tmp/tmux-501/default (No such file or directory)") },
}));
expect(await adapter.hasSession("any-session")).toBe(false);
});
it("returns false on Connection refused against a tmux socket path", async () => {
const adapter = new TmuxAdapter(mockExec({
"has-session": { error: new Error("error connecting to /private/tmp/tmux-501/default (Connection refused)") },
}));
expect(await adapter.hasSession("any-session")).toBe(false);
});
it("rethrows on 'Operation not permitted' (permission must remain fail-closed)", async () => {
const adapter = new TmuxAdapter(mockExec({
"has-session": { error: new Error("error connecting to /private/tmp/tmux-501/default (Operation not permitted)") },
}));
await expect(adapter.hasSession("any-session")).rejects.toThrow("Operation not permitted");
});
it("rethrows on EACCES (permission must remain fail-closed)", async () => {
const adapter = new TmuxAdapter(mockExec({
"has-session": { error: new Error("EACCES: permission denied, /private/tmp/tmux-501/default") },
}));
await expect(adapter.hasSession("any-session")).rejects.toThrow("EACCES");
});
});
describe("createSession", () => {