mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-08 08:11:17 -10:00
fix(daemon): classify tmux socket-absence as missing session and log startup reconcile
After a host reboot the tmux socket file at /tmp/tmux-<uid>/<name> is gone, so `tmux has-session` exits non-zero with a transport-absent message rather than the server/session-absent messages the existing classifier knows. The startup reconciler caught the unknown error, recorded it silently, and skipped marking the row detached, leaving DB rows that said `running`/`ready` against a tmux reality where the sessions were gone. Add `isTmuxTransportAbsentError` to the tmux adapter; classify post-reboot socket-absence (`No such file or directory`) and `Connection refused` against a tmux socket path as absence. Wire it into `hasSession`, `listSessions`, `listWindows`, and `listPanes` for symmetry. Permission errors (`Permission denied`, `Operation not permitted`, `EACCES`, `EPERM`) remain rethrown so callers stay fail-closed; the classifier checks the permission keywords before any absence pattern. Capture each `Reconciler.reconcile(rig.id)` aggregate at startup and log a compact summary line: startup reconcile: rigs=<n> checked=<n> detached=<n> errors=<n> Per-rig errors are written as warnings without crashing startup. Logging is guarded so it can never throw out of `createDaemon`. Tests: - packages/daemon/test/tmux-adapter.test.ts — 4 new `hasSession` cases (2 socket-absent variants, 2 permission rethrow regressions). - packages/daemon/test/reconciler.test.ts — explicit socket-absent detach case + explicit permission-rethrow case. - packages/daemon/test/startup.test.ts — assertion that the compact reconcile summary line appears in startup output. Refs: - openrig-work/field-notes/2026-04-28-openrig-post-reboot-tmux-restore-postmortem.md - rigs/openrig-build/state/orch/velocity-slice-L1-cold-start-tmux-truth-repair-plan-20260428.md Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
1 parent
17812d5d3b
commit
3ad1df6339
5 files changed
+169
-10
No files matched your search
@@ -43,6 +43,30 @@ function isSessionAbsenceError(err: unknown): boolean {
|
||||
msg.includes("no session");
|
||||
}
|
||||
|
||||
// Post-reboot the tmux socket file at /tmp/tmux-<uid>/<name> is gone, so
|
||||
// `tmux has-session` exits non-zero with a transport-absent message rather than
|
||||
// a server/session-absent message. Treat that case as "no session" so cold-start
|
||||
// reconciliation can detach stale rows. Permission errors must remain rethrown.
|
||||
function isTmuxTransportAbsentError(err: unknown): boolean {
|
||||
if (!(err instanceof Error)) return false;
|
||||
const msg = err.message;
|
||||
// Fail-closed: never classify a permission/authorization failure as absence.
|
||||
if (/permission denied|operation not permitted|EACCES|EPERM/i.test(msg)) {
|
||||
return false;
|
||||
}
|
||||
// tmux's socket-transport failure prefix; the parenthetical names the cause.
|
||||
// "error connecting to /private/tmp/tmux-501/default (No such file or directory)"
|
||||
// "error connecting to /private/tmp/tmux-501/default (Connection refused)"
|
||||
if (msg.startsWith("error connecting to")) {
|
||||
return /No such file or directory|Connection refused/.test(msg);
|
||||
}
|
||||
// Conservative bare-message variants that still reference a tmux socket path.
|
||||
if (/tmux-\d+/.test(msg) && /No such file or directory|Connection refused/.test(msg)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function classifyWriteError(err: unknown): TmuxResult {
|
||||
if (!(err instanceof Error)) {
|
||||
return { ok: false, code: "unknown", message: String(err) };
|
||||
@@ -122,7 +146,7 @@ export class TmuxAdapter {
|
||||
const output = await this.exec(`tmux list-sessions -F "${SESSION_FORMAT}"`);
|
||||
return parseLines(output, parseSessionLine);
|
||||
} catch (err) {
|
||||
if (isNoServerError(err)) return [];
|
||||
if (isNoServerError(err) || isTmuxTransportAbsentError(err)) return [];
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -132,7 +156,7 @@ export class TmuxAdapter {
|
||||
const output = await this.exec(`tmux list-windows -t ${shellQuote(sessionName)} -F "${WINDOW_FORMAT}"`);
|
||||
return parseLines(output, parseWindowLine);
|
||||
} catch (err) {
|
||||
if (isNoServerError(err)) return [];
|
||||
if (isNoServerError(err) || isTmuxTransportAbsentError(err)) return [];
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -142,7 +166,7 @@ export class TmuxAdapter {
|
||||
const output = await this.exec(`tmux list-panes -t ${shellQuote(target)} -F "${PANE_FORMAT}"`);
|
||||
return parseLines(output, parsePaneLine);
|
||||
} catch (err) {
|
||||
if (isNoServerError(err)) return [];
|
||||
if (isNoServerError(err) || isTmuxTransportAbsentError(err)) return [];
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -155,13 +179,14 @@ export class TmuxAdapter {
|
||||
await this.exec(`tmux has-session -t ${shellQuote(name)}`);
|
||||
return true; // exit 0 = session exists
|
||||
} catch (err) {
|
||||
// Known-absence patterns: session genuinely doesn't exist or tmux not running.
|
||||
// Return false so post-crash restore can proceed for dead sessions.
|
||||
if (isNoServerError(err) || isSessionAbsenceError(err)) {
|
||||
// Known-absence patterns: session genuinely doesn't exist, tmux not running,
|
||||
// or the tmux socket file is gone post-reboot.
|
||||
// Return false so cold-start reconciliation can detach stale rows.
|
||||
if (isNoServerError(err) || isSessionAbsenceError(err) || isTmuxTransportAbsentError(err)) {
|
||||
return false;
|
||||
}
|
||||
// Unexpected probe failure (permission denied, socket error, etc.) — rethrow
|
||||
// so callers can fail closed rather than treating a probe failure as absence.
|
||||
// Unexpected probe failure (permission denied, etc.) — rethrow so callers
|
||||
// can fail closed rather than treating a probe failure as absence.
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,12 +170,38 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
|
||||
// Connect to cmux at startup — degrades gracefully if absent
|
||||
await cmuxAdapter.connect();
|
||||
|
||||
// Reconcile all managed rigs — marks stale sessions as detached
|
||||
// Reconcile all managed rigs — marks stale sessions as detached.
|
||||
// Capture aggregate counts and log a compact summary so cold-start truth
|
||||
// repair is visible in daemon output instead of silently swallowed.
|
||||
const reconciler = new Reconciler({ db, sessionRegistry, eventBus, tmuxAdapter });
|
||||
const rigs = rigRepo.listRigs();
|
||||
let reconcileChecked = 0;
|
||||
let reconcileDetached = 0;
|
||||
let reconcileErrors = 0;
|
||||
for (const rig of rigs) {
|
||||
await reconciler.reconcile(rig.id);
|
||||
try {
|
||||
const result = await reconciler.reconcile(rig.id);
|
||||
reconcileChecked += result.checked;
|
||||
reconcileDetached += result.detached;
|
||||
reconcileErrors += result.errors.length;
|
||||
for (const e of result.errors) {
|
||||
try {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`startup reconcile warning: rig=${rig.id} session=${e.sessionId} error=${e.error}`);
|
||||
} catch { /* logging must never throw */ }
|
||||
}
|
||||
} catch (err) {
|
||||
reconcileErrors += 1;
|
||||
try {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(`startup reconcile warning: rig=${rig.id} error=${err instanceof Error ? err.message : String(err)}`);
|
||||
} catch { /* logging must never throw */ }
|
||||
}
|
||||
}
|
||||
try {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`startup reconcile: rigs=${rigs.length} checked=${reconcileChecked} detached=${reconcileDetached} errors=${reconcileErrors}`);
|
||||
} catch { /* logging must never throw */ }
|
||||
|
||||
const podRepo = new PodRepository(db);
|
||||
const rigSpecExporter = new RigSpecExporter({ rigRepo, sessionRegistry, podRepo });
|
||||
|
||||
@@ -260,4 +260,47 @@ describe("Reconciler", () => {
|
||||
const sessions = sessionRegistry.getSessionsForRig(rig.id);
|
||||
expect(sessions[0]!.status).toBe("running");
|
||||
});
|
||||
|
||||
// L1 cold-start tmux truth repair: when the tmux adapter classifies a post-reboot
|
||||
// socket-absence as "no session" (hasSession returns false), reconcile must
|
||||
// detach the row honestly so DB state matches tmux reality.
|
||||
it("socket-absent at adapter layer (hasSession returns false): session detached + event emitted", async () => {
|
||||
const { rig } = seedRigWithSessions([
|
||||
{ logicalId: "dev1-impl", sessionName: "r01-dev1-impl", status: "running" },
|
||||
]);
|
||||
|
||||
const reconciler = createReconciler(mockTmuxAdapter({ "r01-dev1-impl": false }));
|
||||
const result = await reconciler.reconcile(rig.id);
|
||||
|
||||
expect(result.checked).toBe(1);
|
||||
expect(result.detached).toBe(1);
|
||||
|
||||
const sessions = sessionRegistry.getSessionsForRig(rig.id);
|
||||
expect(sessions[0]!.status).toBe("detached");
|
||||
|
||||
const events = db
|
||||
.prepare("SELECT * FROM events WHERE type = 'session.detached'")
|
||||
.all();
|
||||
expect(events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("permission error at adapter layer (hasSession rethrows): session NOT detached, error recorded", async () => {
|
||||
const { rig } = seedRigWithSessions([
|
||||
{ logicalId: "dev1-impl", sessionName: "r01-dev1-impl", status: "running" },
|
||||
]);
|
||||
|
||||
const permErr = new Error("error connecting to /private/tmp/tmux-501/default (Operation not permitted)");
|
||||
const reconciler = createReconciler(
|
||||
mockTmuxAdapter({}, { "r01-dev1-impl": permErr })
|
||||
);
|
||||
const result = await reconciler.reconcile(rig.id);
|
||||
|
||||
expect(result.checked).toBe(0);
|
||||
expect(result.detached).toBe(0);
|
||||
expect(result.errors).toHaveLength(1);
|
||||
expect(result.errors[0]!.error).toContain("Operation not permitted");
|
||||
|
||||
const sessions = sessionRegistry.getSessionsForRig(rig.id);
|
||||
expect(sessions[0]!.status).toBe("running");
|
||||
});
|
||||
});
|
||||
@@ -230,4 +230,39 @@ describe("createDaemon startup composition", () => {
|
||||
|
||||
db.close();
|
||||
});
|
||||
|
||||
// L1 cold-start tmux truth repair: startup must surface a compact reconcile
|
||||
// summary so silent reconciliation drift is visible in daemon output.
|
||||
it("startup logs compact reconcile summary line", async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "rigged-test-"));
|
||||
const dbPath = path.join(tmpDir, "test.sqlite");
|
||||
|
||||
seedDbWithStaleSessions(dbPath, [
|
||||
{ rigName: "r01", logicalId: "dev1-impl", sessionName: "r01-dev1-impl" },
|
||||
]);
|
||||
|
||||
const tmuxExec: ExecFn = async (cmd: string) => {
|
||||
if (cmd.includes("has-session")) throw new Error("session not found");
|
||||
return "";
|
||||
};
|
||||
const cmuxExec: ExecFn = async () => { throw Object.assign(new Error(""), { code: "ENOENT" }); };
|
||||
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
try {
|
||||
const { db } = await createDaemon({ dbPath, tmuxExec, cmuxExec });
|
||||
|
||||
const calls = logSpy.mock.calls.map((c: unknown[]) => String(c[0]));
|
||||
const summary = calls.find((line) => line.startsWith("startup reconcile:"));
|
||||
expect(summary).toBeDefined();
|
||||
expect(summary).toMatch(/rigs=1\b/);
|
||||
expect(summary).toMatch(/checked=1\b/);
|
||||
expect(summary).toMatch(/detached=1\b/);
|
||||
expect(summary).toMatch(/errors=0\b/);
|
||||
|
||||
db.close();
|
||||
} finally {
|
||||
logSpy.mockRestore();
|
||||
fs.rmSync(tmpDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -195,6 +195,36 @@ describe("TmuxAdapter", () => {
|
||||
}));
|
||||
await expect(adapter.hasSession("any-session")).rejects.toThrow("exit code 127");
|
||||
});
|
||||
|
||||
// L1 cold-start tmux truth repair: post-reboot socket absence must classify
|
||||
// as "no session" so the reconciler can detach stale rows without manual fix.
|
||||
it("returns false when tmux socket is gone post-reboot (No such file or directory)", async () => {
|
||||
const adapter = new TmuxAdapter(mockExec({
|
||||
"has-session": { error: new Error("error connecting to /private/tmp/tmux-501/default (No such file or directory)") },
|
||||
}));
|
||||
expect(await adapter.hasSession("any-session")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false on Connection refused against a tmux socket path", async () => {
|
||||
const adapter = new TmuxAdapter(mockExec({
|
||||
"has-session": { error: new Error("error connecting to /private/tmp/tmux-501/default (Connection refused)") },
|
||||
}));
|
||||
expect(await adapter.hasSession("any-session")).toBe(false);
|
||||
});
|
||||
|
||||
it("rethrows on 'Operation not permitted' (permission must remain fail-closed)", async () => {
|
||||
const adapter = new TmuxAdapter(mockExec({
|
||||
"has-session": { error: new Error("error connecting to /private/tmp/tmux-501/default (Operation not permitted)") },
|
||||
}));
|
||||
await expect(adapter.hasSession("any-session")).rejects.toThrow("Operation not permitted");
|
||||
});
|
||||
|
||||
it("rethrows on EACCES (permission must remain fail-closed)", async () => {
|
||||
const adapter = new TmuxAdapter(mockExec({
|
||||
"has-session": { error: new Error("EACCES: permission denied, /private/tmp/tmux-501/default") },
|
||||
}));
|
||||
await expect(adapter.hasSession("any-session")).rejects.toThrow("EACCES");
|
||||
});
|
||||
});
|
||||
|
||||
describe("createSession", () => {
|
||||
|
||||
Reference in new issue
Block a user