fix(runtime): recover long-idle sessions, survive terminal loss, journal crashes (#61)

* fix(runtime): recover long-idle sessions, survive terminal loss, journal crashes

A server left running while the user is away could leave the open page
unable to load anything (empty sidebar, endless Loading) until the
process was restarted, with no trace of why. Harden both sides:

- SessionSidebar: refresh the session list when the tab becomes visible
  again or the network returns — the SSE reconnect after laptop sleep /
  network change carries no list invalidation.
- useAgentSession: a CLOSED event stream now retries with capped backoff
  even while the agent is idle; previously only running sessions were
  retried, leaving idle streams dead silently until a page reload.
- process-lifecycle: translate SIGHUP into a graceful SIGTERM tree kill so
  a CLI launcher losing its terminal disposes the detached omp children
  instead of orphaning them.
- instrumentation: journal uncaughtException/unhandledRejection and
  event-loop stalls to ~/.omp/omp-web/diagnostics.log, so a crash-vs-stall
  wedge is diagnosable from disk when no terminal is watching.
- session-reader: an unsettled in-flight list load stops being coalesced
  after 60s instead of pinning the wedge until process restart.
- mcp-config: bound the synchronous git rev-parse with a 10s timeout so a
  hung spawn cannot block the entire event loop.

* fix(review): clear stale reconnect timer, gate late cache writes, fix MCP fallback caching, label sleep gaps

- useAgentSession: connectEvents drops any pending backoff timer first so a send-during-backoff cannot orphan a timer that later tears down the healthy stream; contract test pins it.
- session-reader: a watchdog-retired load can no longer overwrite fresher cache data when it resolves late (retired flag).
- mcp-config: transient git failures return the cwd fallback without poisoning the 30s project-root cache.
- instrumentation: stall watchdog compares CPU time so laptop sleep is journaled as sleep, not a misleading stall.

---------

Co-authored-by: kahme247 <[email protected]>

bin/process-lifecycle.js

@@ -6,6 +6,11 @@ const os = require("node:os");
 const { spawn } = require("node:child_process");
 
 const forwardedSignals = ["SIGINT", "SIGTERM"];
+// SIGHUP arrives when a CLI-run launcher loses its terminal (window closed,
+// session ended). The child must die gracefully, not verbatim: a verbatim
+// SIGHUP terminates Node without running exit handlers, orphaning the detached
+// omp children it spawned. Translate it to a graceful SIGTERM tree kill.
+const translatedSignals = { SIGHUP: "SIGTERM" };
 const shutdownTimeoutMs = 5_000;
 // Time to wait for a terminated process tree to actually exit (used as the
 // default for terminateChildProcess and by the update worker's stop gate).
@@ -75,20 +80,23 @@ function wireChildProcessLifecycle(child, parentProcess = process, timeoutMs = s
   const signalHandlers = new Map();
   let shutdownTimer;
   const platform = parentProcess.platform ?? process.platform;
-
   const forceKill = () => killChildTree(child, true, platform);
-
+  const killChildOnSignal = (childSignal) => {
+    if (shutdownTimer) {
+      forceKill();
+      return;
+    }
+    shutdownTimer = setTimeout(forceKill, timeoutMs);
+    shutdownTimer.unref?.();
+    killChildTree(child, false, platform, childSignal);
+  };
   for (const signal of forwardedSignals) {
-    const handler = () => {
-      if (shutdownTimer) {
-        forceKill();
-        return;
-      }
-
-      shutdownTimer = setTimeout(forceKill, timeoutMs);
-      shutdownTimer.unref?.();
-      killChildTree(child, false, platform, signal);
-    };
+    const handler = () => killChildOnSignal(signal);
+    signalHandlers.set(signal, handler);
+    parentProcess.on(signal, handler);
+  }
+  for (const [signal, childSignal] of Object.entries(translatedSignals)) {
+    const handler = () => killChildOnSignal(childSignal);
     signalHandlers.set(signal, handler);
     parentProcess.on(signal, handler);
   }

components/SessionSidebar.tsx

@@ -33,6 +33,10 @@ import {
 import { OmpWebTitle, SIDEBAR_BUTTON_TRANSITION, SidebarIconButton } from "./SessionSidebar-chrome";
 import { ProjectRow, ProjectWorktreeSwitcher } from "./SessionSidebar-rows";
 
+/** Deadline for one /api/sessions fetch. A wedged-but-listening server never
+ * answers; without this the initial spinner would pend forever. */
+const SESSIONS_FETCH_TIMEOUT_MS = 15_000;
+
 declare global {
   interface Window {
     piDesktop?: {
@@ -138,6 +142,12 @@ export const SessionSidebar = memo(function SessionSidebar({ selectedSessionId,
     sessionsAbortRef.current?.abort();
     const controller = new AbortController();
     sessionsAbortRef.current = controller;
+    // A wedged-but-listening server (the long-idle failure mode) accepts the
+    // TCP connection and never answers: without a deadline the fetch pends
+    // forever and the sidebar shows its initial spinner indefinitely. Abort
+    // with a TimeoutError so the catch surfaces an error and the
+    // visibility/online recovery below can retry later.
+    const timeout = setTimeout(() => controller.abort(new DOMException("Session list request timed out", "TimeoutError")), SESSIONS_FETCH_TIMEOUT_MS);
     try {
       if (showLoading) setLoading(true);
       const headers: Record<string, string> = {};
@@ -183,6 +193,7 @@ export const SessionSidebar = memo(function SessionSidebar({ selectedSessionId,
       if ((e as Error)?.name === "AbortError") return;
       setError(t("sessionSidebar.loadFailed", { detail: e instanceof Error ? e.message : String(e) }));
     } finally {
+      clearTimeout(timeout);
       initialLoadedRef.current = true;
       if (showLoading) setLoading(false);
     }
@@ -298,6 +309,22 @@ export const SessionSidebar = memo(function SessionSidebar({ selectedSessionId,
       source.close();
     };
   }, [loadSessions, scheduleRefresh]);
+  // Long-idle recovery: while the tab is hidden the SSE connection can die
+  // (laptop sleep, network change, tab freeze) and its EventSource reconnect
+  // carries no list invalidation. Refresh whenever the user actually comes
+  // back or the network returns, so the list is never left stale/empty.
+  useEffect(() => {
+    const onVisible = () => {
+      if (document.visibilityState === "visible") void loadSessions(false);
+    };
+    const onOnline = () => void loadSessions(false);
+    document.addEventListener("visibilitychange", onVisible);
+    window.addEventListener("online", onOnline);
+    return () => {
+      document.removeEventListener("visibilitychange", onVisible);
+      window.removeEventListener("online", onOnline);
+    };
+  }, [loadSessions]);
 
   useEffect(() => {
     const previous = previousRunningSessionIdsRef.current;

hooks/useAgentSession-stream.ts

@@ -237,6 +237,13 @@ export const BASH_STATE_RECONCILE_MS = 1_000;
 export const EVENT_STREAM_CONNECT_TIMEOUT_MS = 60_000;
 // Tell the user something is happening if the stream is still connecting.
 export const EVENT_STREAM_SLOW_CONNECT_MS = 4_000;
+// Manual reconnect backoff for a CLOSED event stream (fatal per EventSource,
+// so the browser will not retry on its own). Doubles per consecutive failure,
+// caps at the max; reset on any successful open. Without this, an idle
+// session whose stream died (server hiccup, wrapper respawn) stays dead
+// silently until a full page reload.
+export const EVENT_STREAM_RETRY_MIN_MS = 1_000;
+export const EVENT_STREAM_RETRY_MAX_MS = 30_000;
 
 export const SCROLL_KEYS = new Set(["ArrowUp", "ArrowDown", "PageUp", "PageDown", "Home", "End", " ", "Space", "Spacebar"]);
 export function isQuotaLikeError(text: string): boolean {

hooks/useAgentSession.test.mjs

@@ -23,3 +23,21 @@ test("fork carries the advisor choice to the new session id", async () => {
   assert.match(source, /setSessionAdvisorSpawn\(newSessionId, true\)/);
   assert.match(source, /omp-advisor-enabled:\$\{newSessionId\}/);
 });
+
+test("a CLOSED event stream retries with backoff even while the agent is idle", async () => {
+  const source = await readFile(new URL("./useAgentSession.ts", import.meta.url), "utf8");
+  // EventSource gives up on fatal errors; without a manual retry an idle
+  // session whose stream died (server hiccup, wrapper respawn) stays dead
+  // silently until a full page reload. The retry must not be gated on
+  // agentRunningRef — only on the session still being selected.
+  assert.match(source, /reconnectTimerRef\.current = setTimeout\(\(\) => \{\s*\n\s*reconnectTimerRef\.current = undefined;\s*\n\s*if \(sessionIdRef\.current === sid\) \{\s*\n\s*void connectEvents\(sid\);/);
+  // Backoff doubles per failure and caps, so a persistently failing endpoint
+  // (e.g. a 409 wrapper-less session) cannot tight-loop.
+  assert.match(source, /eventStreamRetryMsRef\.current = Math\.min\(delay \* 2, EVENT_STREAM_RETRY_MAX_MS\)/);
+  // A successful open resets the backoff to the floor.
+  assert.match(source, /es\.onopen = \(\) => \{\s*\n\s*eventStreamRetryMsRef\.current = EVENT_STREAM_RETRY_MIN_MS;/);
+  // Building a fresh stream drops any still-pending backoff timer first: a
+  // send during the wait must not let the orphaned timer fire later and tear
+  // down the healthy replacement mid-run.
+  assert.match(source, /const connectEvents = useCallback\(\(sid: string\)[\s\S]*?clearTimeout\(reconnectTimerRef\.current\);\s*\n\s*reconnectTimerRef\.current = undefined;\s*\n\s*if \(eventSourceRef\.current\)/);
+});

hooks/useAgentSession.ts

@@ -63,6 +63,8 @@ import {
   AGENT_STATE_RECONCILE_MS,
   BASH_STATE_RECONCILE_MS,
   EVENT_STREAM_CONNECT_TIMEOUT_MS,
+  EVENT_STREAM_RETRY_MAX_MS,
+  EVENT_STREAM_RETRY_MIN_MS,
   EVENT_STREAM_SLOW_CONNECT_MS,
   PROGRAMMATIC_SCROLL_IGNORE_MS,
   PROMPT_SETTLE_INITIAL_DELAY_MS,
@@ -318,7 +320,8 @@ export function useAgentSession(opts: UseAgentSessionOptions) {
   const activeSubagentCount = subagents.filter((subagent) => subagent.source !== "history" && subagent.status === "started").length;
 
   const eventSourceRef = useRef<EventSource | null>(null);
-  const reconnectTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const reconnectTimerRef = useRef<NodeJS.Timeout | undefined>(undefined);
+  const eventStreamRetryMsRef = useRef<number>(EVENT_STREAM_RETRY_MIN_MS);
   const sessionIdRef = useRef<string | null>(session?.id ?? null);
   // Guards stale branch/leaf context responses: two rapid navigate clicks must
   // not let the older response overwrite the newer branch's messages.
@@ -933,6 +936,11 @@ export function useAgentSession(opts: UseAgentSessionOptions) {
   const reconnectActionsRef = useRef<((sid: string) => void) | null>(null);
 
   const connectEvents = useCallback((sid: string): Promise<EventStreamConnectionResult> => {
+    // A backoff timer from an earlier CLOSED stream may still be pending (e.g.
+    // the user sent a message while it waited). It must not fire later and
+    // tear down the healthy stream built here — drop it before replacing.
+    clearTimeout(reconnectTimerRef.current);
+    reconnectTimerRef.current = undefined;
     if (eventSourceRef.current) {
       eventSourceRef.current.close();
       eventSourceRef.current = null;
@@ -951,10 +959,13 @@ export function useAgentSession(opts: UseAgentSessionOptions) {
         resolve({ status, source: es });
       };
       const timeout = setTimeout(() => settle("timeout"), EVENT_STREAM_CONNECT_TIMEOUT_MS);
-
       // The stream is live as soon as the response headers land, whether or not
-      // the server also sends an explicit `connected` frame.
-      es.onopen = () => settle("connected");
+      // the server also sends an explicit `connected` frame. A successful open
+      // also resets the manual-reconnect backoff.
+      es.onopen = () => {
+        eventStreamRetryMsRef.current = EVENT_STREAM_RETRY_MIN_MS;
+        settle("connected");
+      };
 
       es.onmessage = (e) => {
         try {
@@ -970,25 +981,29 @@ export function useAgentSession(opts: UseAgentSessionOptions) {
       };
       es.onerror = () => {
         if (es.readyState === EventSource.CLOSED) {
-          // Fatal error (404/500/content-type mismatch): browser won't
-          // auto-reconnect. Settle the Promise and manually reconnect for
-          // already-running sessions. Keep the timer in a ref so unmount or a
-          // session switch cancels it — otherwise an orphaned stream respawns
-          // (and can 404-loop) after the hook is torn down.
+          // Fatal error (404/409/500/content-type mismatch): the browser will
+          // not auto-reconnect. Settle the Promise and schedule a manual
+          // reconnect with capped backoff — regardless of whether the agent is
+          // running. Long-idle sessions (wrapper respawned server-side, server
+          // restart, proxy timeout) previously stayed dead silently until a
+          // page reload; the backoff bounds the retry loop against endpoints
+          // that keep failing (e.g. a 409 "not managed" wrapper-less session).
           settle("closed");
-          if (eventSourceRef.current === es && agentRunningRef.current) {
+          if (eventSourceRef.current === es) {
             eventSourceRef.current = null;
-            if (reconnectTimerRef.current) clearTimeout(reconnectTimerRef.current);
+            const delay = eventStreamRetryMsRef.current;
+            eventStreamRetryMsRef.current = Math.min(delay * 2, EVENT_STREAM_RETRY_MAX_MS);
+            clearTimeout(reconnectTimerRef.current);
             reconnectTimerRef.current = setTimeout(() => {
-              reconnectTimerRef.current = null;
-              if (agentRunningRef.current && sessionIdRef.current === sid) {
+              reconnectTimerRef.current = undefined;
+              if (sessionIdRef.current === sid) {
                 void connectEvents(sid);
-                // The reconnect restores the event stream, but host tools, URI
-                // schemes, and the subagent roster were registered on the old
-                // connection — re-register them so the agent keeps working.
-                reconnectActionsRef.current?.(sid);
+                // While the agent runs, the reconnect must also re-register
+                // host tools, URI schemes, and the subagent roster that were
+                // registered on the old connection.
+                if (agentRunningRef.current) reconnectActionsRef.current?.(sid);
               }
-            }, 1000);
+            }, delay);
           }
         }
         // Recoverable errors (CONNECTING): let EventSource auto-reconnect.
@@ -3121,10 +3136,8 @@ export function useAgentSession(opts: UseAgentSessionOptions) {
       eventCoalescerRef.current?.reset();
       eventSourceRef.current?.close();
       eventSourceRef.current = null;
-      if (reconnectTimerRef.current) {
-        clearTimeout(reconnectTimerRef.current);
-        reconnectTimerRef.current = null;
-      }
+      clearTimeout(reconnectTimerRef.current);
+      reconnectTimerRef.current = undefined;
       if (rosterRefreshTimerRef.current) {
         clearTimeout(rosterRefreshTimerRef.current);
         rosterRefreshTimerRef.current = null;

instrumentation.ts

@@ -1,3 +1,7 @@
+import { appendFileSync, mkdirSync, renameSync, statSync } from "fs";
+import { join } from "path";
+import { getConfigRoot } from "@/lib/omp/paths";
+
 export async function register(): Promise<void> {
   if (process.env.NEXT_RUNTIME !== "nodejs") return;
 
@@ -41,4 +45,57 @@ export async function register(): Promise<void> {
       console.warn(`[omp-web] omp utility warm-up failed (routes will retry on demand): ${detail} — ${hint}`);
     }
   })();
+
+  // Crash/stall journal: a long-running server that dies or wedges while the
+  // user is away leaves no trace in a terminal that no longer exists (CLI runs
+  // are killed with their terminal; pages then show endless loading until the
+  // process is restarted). Append fatal errors and event-loop stalls to a file
+  // so the next incident explains itself. Node's default crash semantics are
+  // preserved — this only adds the record before exiting.
+  const logDir = join(getConfigRoot(), "omp-web");
+  const logPath = join(logDir, "diagnostics.log");
+  const appendDiag = (kind: string, detail: string) => {
+    try {
+      mkdirSync(logDir, { recursive: true });
+      try {
+        if (statSync(logPath).size > 1_000_000) renameSync(logPath, `${logPath}.old`);
+      } catch { /* first write or unreadable — append anyway */ }
+      appendFileSync(logPath, `${new Date().toISOString()} [${kind}] ${detail}\n`, { encoding: "utf8" });
+    } catch { /* diagnostics must never crash the server */ }
+  };
+  const describe = (value: unknown) => (value instanceof Error ? `${value.name}: ${value.message}\n${value.stack ?? ""}` : String(value));
+  process.on("uncaughtException", (error) => {
+    appendDiag("crash", `uncaughtException ${describe(error)}`);
+    // An uncaughtException listener suppresses Node's default exit; keep the
+    // crash-visible semantics by exiting explicitly.
+    process.exit(2);
+  });
+  process.on("unhandledRejection", (reason) => {
+    appendDiag("crash", `unhandledRejection ${describe(reason)}`);
+    // Same as above: preserve Node's crash-on-unhandled-rejection default.
+    process.exit(2);
+  });
+  let lastTick = Date.now();
+  let lastCpu = process.cpuUsage();
+  const watchdog = setInterval(() => {
+    const now = Date.now();
+    const cpu = process.cpuUsage();
+    const drift = now - lastTick;
+    const cpuMs = (cpu.user - lastCpu.user + cpu.system - lastCpu.system) / 1000;
+    lastTick = now;
+    lastCpu = cpu;
+    if (drift > 45_000) {
+      // Wall-clock drift alone cannot tell a blocked loop from a sleeping
+      // machine: an hour with the lid closed looks like an hour-long stall
+      // but burns no CPU. Label accordingly so the journal does not mislead
+      // the next long-idle investigation.
+      const seconds = Math.round(drift / 1000);
+      if (cpuMs < Math.min(5_000, drift / 2)) {
+        appendDiag("sleep", `event loop gap of ~${seconds}s with negligible CPU time — machine was asleep/suspended or CPU-starved, not a synchronous block`);
+      } else {
+        appendDiag("stall", `event loop unresponsive for ~${seconds}s — a synchronous operation is blocking every request`);
+      }
+    }
+  }, 15_000);
+  watchdog.unref?.();
 }

lib/omp/mcp-config.ts

@@ -183,9 +183,12 @@ function projectRoot(cwd: string): string {
   if (cached && cached.expiresAt > Date.now()) return cached.root;
   let root: string;
   try {
-    root = resolve(execFileSync("git", ["-C", cwd, "rev-parse", "--show-toplevel"], { encoding: "utf8", windowsHide: true, stdio: ["ignore", "pipe", "ignore"] }).trim());
+    root = resolve(execFileSync("git", ["-C", cwd, "rev-parse", "--show-toplevel"], { encoding: "utf8", windowsHide: true, stdio: ["ignore", "pipe", "ignore"], timeout: 10_000 }).trim());
   } catch {
-    root = resolve(cwd);
+    // Transient git failure (hang, timeout, sick mount): fall back to the cwd
+    // but do NOT cache it — a cached fallback would aim MCP reads/writes at
+    // the wrong directory for the full TTL instead of retrying git next call.
+    return resolve(cwd);
   }
   if (projectRootCache.size > 500) projectRootCache.clear();
   projectRootCache.set(cwd, { root, expiresAt: Date.now() + PROJECT_ROOT_TTL_MS });

lib/session-reader.ts

@@ -111,10 +111,13 @@ export async function listAllSessions(): Promise<SessionInfo[]> {
     return globalThis.__piSessionListPromise;
   }
 
+  // Flipped once the watchdog below retires this scan: a hung load that
+  // resolves after the slot moved on must not overwrite fresher cache data.
+  let retired = false;
   const loadPromise = loadAllSessions().then((data) => {
     // An invalidation may happen while the scan is in flight. Do not let that
     // older result repopulate the cache after a session mutation.
-    if ((globalThis.__piSessionListGeneration ?? 0) === generation) {
+    if ((globalThis.__piSessionListGeneration ?? 0) === generation && !retired) {
       globalThis.__piSessionListCache = { data, ts: Date.now() };
     }
     return data;
@@ -125,6 +128,23 @@ export async function listAllSessions(): Promise<SessionInfo[]> {
       globalThis.__piSessionListPromiseGeneration = undefined;
     }
   });
+  // A load that never settles (a stuck sync syscall, a hung git spawn that
+  // slipped past its timeout) must not be handed to every future caller
+  // forever — the coalescing slot would pin the wedge until process restart.
+  // Drop the slot after a generous deadline; the cache stays unset, so the
+  // next request starts a fresh scan.
+  const watchdog = setTimeout(() => {
+    if (globalThis.__piSessionListPromise === trackedPromise) {
+      globalThis.__piSessionListPromise = undefined;
+      globalThis.__piSessionListPromiseGeneration = undefined;
+      retired = true;
+    }
+  }, SESSION_LIST_LOAD_DEADLINE_MS);
+  watchdog.unref?.();
+
+  globalThis.__piSessionListPromise = trackedPromise;
+  globalThis.__piSessionListPromiseGeneration = generation;
+  watchdog.unref?.();
 
   globalThis.__piSessionListPromise = trackedPromise;
   globalThis.__piSessionListPromiseGeneration = generation;
@@ -144,6 +164,8 @@ declare global {
 }
 
 const SESSION_LIST_CACHE_TTL_MS = 30_000;
+/** Beyond this, an unsettled in-flight list load stops being coalesced. */
+const SESSION_LIST_LOAD_DEADLINE_MS = 60_000;
 
 /** Invalidate the session LIST metadata (30s TTL result + generation gate)
  * and the directory-walk cache, but leave per-session parse caches intact.