feat(daemon,cli): decouple daemon health/listen from kernel-agent readiness

Forward-fix #3 (architectural) on Phase 05d. Daemon previously awaited
`bootKernelIfNeeded` inside `createDaemon`, which awaited the full
`BootstrapOrchestrator.bootstrap(...)` of the kernel rig. A broken
kernel agent (e.g., Codex unauthenticated mid-boot) stalled the entire
daemon HTTP surface; healthz never bound; CLI reported "daemon failed
to start" even though the daemon itself was fine — only the kernel
was sad.

Architectural decoupling:

- New KernelBootTracker class. Owns kernel-boot state (skipped /
  auth_blocked / spec_missing / booting / ready / partial_ready /
  bootstrap_failed / degraded) and aggregates agents[] live from the
  sessions table on read. Bootstrap promise is fire-and-forget.
- `bootKernelIfNeeded` now returns a tracker immediately. createDaemon
  finishes; server.ts binds healthz; the bootstrap runs in the
  background. Probe / variant / spec-missing / auth-block short-
  circuits are still synchronous-ish (no expensive work).
- New event type `kernel.agent.degraded` emitted exactly once via the
  tracker's degraded-timer (default 90s; override via
  OPENRIG_KERNEL_DEGRADED_MS env or the deps.degradedTimeoutMs
  injection point used by tests).

New observability surface:

- GET /api/kernel/status. Returns
  { kernel_state, agents[], first_unready_since, variant, detail }
  when the tracker is wired; 503 with `kernel_boot_tracker_unavailable`
  otherwise. Read-only, no auth needed (same posture as healthz).

New CLI affordance (current `rig daemon start` behavior preserved):

- `--wait-for-kernel` flag: after healthz binds, additionally polls
  /api/kernel/status until kernel_state is `ready` or `partial_ready`
  (the two "operator can use the kernel" states), with a default 60s
  timeout overridable via `--wait-for-kernel-ms`. Terminal non-ready
  states (auth_blocked / spec_missing / bootstrap_failed / degraded /
  skipped) short-circuit the poll so the CLI fails fast with the
  3-part error (Error / Reason / Fix lines) per banked discipline.

Tests:

- kernel-boot-tracker.test.ts (13): state transitions, agent
  aggregation (ready / partial_ready / booting), degraded-timer
  emission (exactly once + cancel on ready before deadline),
  sessionRegistry error-handling fallthrough.
- kernel-boot.test.ts (16): refactored for the tracker-returning API;
  short-circuit branches (skipped / auth_blocked / spec_missing) +
  fire-and-forget bootstrap (booting on launch + bootstrap_failed
  after flush).
- kernel-status-route.test.ts (4): 503 envelope on missing tracker;
  200 envelope on each state; snake_case agents[] projection.

Targeted: 59/59 across the 5 kernel-related test files.
Full daemon suite: 3177 passed / 1 skipped / 0 failed.

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
mvschwarzandClaude Opus 4.7 committed 2026-05-11 04:05:16 -07:00
1 parent 828cc99ad9
commit 270642d8bc
11 files changed
+1024 -175

No files matched your search

+36 -1
View File
@@ -83,7 +83,15 @@ export function daemonCommand(depsOverride?: LifecycleDeps): Command {
// and serves its HTTP API normally; just doesn't materialize the
// kernel rig.
.option("--no-kernel", "Skip kernel auto-boot (daemon serves without the kernel rig)")
.action(async (opts: { port?: string; host?: string; db?: string; kernel?: boolean }) => {
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
// After the daemon's healthz binds (current behavior preserved),
// additionally poll /api/kernel/status until kernel_state is
// ready / partial_ready, or the timeout elapses. Used by operators
// who want a "kernel-ready" signal at start-time rather than the
// weaker "daemon-ready". Default 60s; override with --wait-for-kernel-ms.
.option("--wait-for-kernel", "After daemon binds, also wait for kernel-agent readiness (default timeout 60s)")
.option("--wait-for-kernel-ms <ms>", "Override --wait-for-kernel timeout in milliseconds")
.action(async (opts: { port?: string; host?: string; db?: string; kernel?: boolean; waitForKernel?: boolean; waitForKernelMs?: string }) => {
try {
const { ConfigStore } = await import("../config-store.js");
const { SystemPreflight } = await import("../system-preflight.js");
@@ -136,6 +144,33 @@ export function daemonCommand(depsOverride?: LifecycleDeps): Command {
getDeps(),
);
console.log(`Daemon started on port ${state.port} (pid ${state.pid})`);
// V0.3.1 slice 05 forward-fix #3 architectural — --wait-for-kernel
// post-bind polling. Kernel boot is fire-and-forget after the
// daemon binds healthz, so without this flag the CLI doesn't
// know whether the kernel itself reached ready. Operators who
// need a kernel-ready signal opt in here.
if (opts.waitForKernel) {
const { waitForKernelReady } = await import("../daemon-lifecycle.js");
const timeoutMs = opts.waitForKernelMs && /^\d+$/.test(opts.waitForKernelMs)
? parseInt(opts.waitForKernelMs, 10)
: 60_000;
const baseUrl = `http://${state.host}:${state.port}`;
const result = await waitForKernelReady(baseUrl, timeoutMs);
if (result.ok) {
console.log(`Kernel ${result.kernelState}; variant=${result.variant ?? "(none)"}`);
} else {
// Honest 3-part error per banked discipline.
console.error(
`Error: kernel did not reach ready / partial_ready within ${timeoutMs}ms.\n` +
`Reason: kernel_state=${result.kernelState ?? "unknown"}` +
(result.detail ? `; ${result.detail}` : "") +
"\n" +
"Fix: inspect `rig ps --rig kernel` for stalled agents, or run `claude auth status` / `codex login status` to confirm runtime auth.",
);
process.exitCode = 1;
}
}
} catch (err) {
console.error(err instanceof Error ? err.message : String(err));
process.exitCode = 1;
+56
View File
@@ -513,3 +513,59 @@ async function fetchDaemonProbe(deps: LifecycleDeps, url: string): Promise<{ ok:
}),
]);
}
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
// Poll GET /api/kernel/status until kernel_state reaches ready or
// partial_ready (the two "operator can use the kernel" states) or the
// timeout elapses. Used by `rig daemon start --wait-for-kernel`.
//
// Returns { ok: true, ... } on success; { ok: false, ... } with the
// last observed kernelState + detail for the CLI's 3-part error.
export interface KernelReadyResult {
ok: boolean;
kernelState: string | null;
variant: string | null;
detail: string | null;
}
export async function waitForKernelReady(
baseUrl: string,
timeoutMs: number,
pollIntervalMs = 500,
): Promise<KernelReadyResult> {
const deadline = Date.now() + timeoutMs;
let last: KernelReadyResult = { ok: false, kernelState: null, variant: null, detail: null };
while (Date.now() < deadline) {
try {
const res = await fetch(`${baseUrl}/api/kernel/status`);
if (res.ok) {
const body = (await res.json()) as {
kernel_state?: string;
variant?: string | null;
detail?: string | null;
};
last = {
ok: body.kernel_state === "ready" || body.kernel_state === "partial_ready",
kernelState: body.kernel_state ?? null,
variant: body.variant ?? null,
detail: body.detail ?? null,
};
if (last.ok) return last;
// Terminal non-ready states: don't keep polling.
if (
body.kernel_state === "auth_blocked" ||
body.kernel_state === "spec_missing" ||
body.kernel_state === "bootstrap_failed" ||
body.kernel_state === "degraded" ||
body.kernel_state === "skipped"
) {
return last;
}
}
} catch {
// Transient fetch failure; keep polling until the deadline.
}
await new Promise((r) => setTimeout(r, pollIntervalMs));
}
return last;
}
@@ -0,0 +1,271 @@
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
// Decouples daemon health/listen from kernel-agent readiness.
//
// Prior behavior: `bootKernelIfNeeded` awaited `bootstrapOrchestrator.bootstrap(...)`
// inside `createDaemon`, so the daemon process couldn't reach `serve()` until
// the kernel rig's members had been launched + their startup deliveries
// completed. A broken kernel agent (e.g., Codex unauthenticated mid-boot)
// stalled healthz too. The CLI then reported "daemon failed to start" even
// though the daemon itself was perfectly happy — only the *kernel* was sad.
//
// New behavior: `bootKernelIfNeeded` builds a tracker, fires the bootstrap
// in the background, and returns immediately. The daemon binds healthz as
// soon as `createDaemon` completes (HTTP server bind happens in server.ts
// right after). Tracker state is published via GET /api/kernel/status so
// operators (and CLI flags like `--wait-for-kernel`) can observe progress.
//
// A configurable degraded-timer (default 90s) emits a single
// `kernel.agent.degraded` event if the kernel doesn't reach a ready / partial-
// ready state in time — telemetry that something is keeping the kernel
// stuck (auth / spec / tmux / etc.).
import type { EventBus } from "./event-bus.js";
import type { SessionRegistry } from "./session-registry.js";
import type { RigRepository } from "./rig-repository.js";
import type { BootstrapResult } from "./bootstrap-orchestrator.js";
export type KernelState =
| "skipped" // OPENRIG_NO_KERNEL=1 / VITEST auto-skip / kernel already managed
| "auth_blocked" // Both Claude Code + Codex unauthenticated; cannot pick a variant
| "spec_missing" // Selected variant's rig.yaml not found on disk
| "booting" // Bootstrap fired; awaiting member startup_status
| "partial_ready" // Bootstrap done; some members ready, others still pending/failed
| "ready" // Bootstrap done; all members reached ready
| "bootstrap_failed" // Bootstrap promise rejected or returned errors
| "degraded"; // Booting > degradedTimeoutMs without reaching ready/partial_ready
export interface KernelAgentStatus {
/** Session name (e.g. `advisor-lead@kernel`). */
sessionName: string;
/** Runtime declared in the agent profile (claude-code / codex / terminal). */
runtime: string;
/** Startup status from the sessions table. Same enum as session-registry. */
startupStatus: "pending" | "ready" | "attention_required" | "failed";
}
export interface KernelBootStatus {
kernelState: KernelState;
agents: KernelAgentStatus[];
/** ISO timestamp of when the kernel first entered booting; null in
* terminal/skipped states. */
firstUnreadySince: string | null;
/** Filename of the picked variant (rig.yaml / rig-claude-only.yaml /
* rig-codex-only.yaml). null when no variant was selected. */
variant: string | null;
/** Human-readable detail for auth_blocked / spec_missing /
* bootstrap_failed / degraded states. null otherwise. */
detail: string | null;
}
export interface KernelBootTrackerDeps {
eventBus: EventBus;
sessionRegistry: SessionRegistry;
rigRepo: RigRepository;
/** Milliseconds the tracker waits in `booting` before emitting
* `kernel.agent.degraded` and transitioning to the `degraded` state.
* Default 90_000 (90s) per IMPL-PRD §6.3 amendment. Tests pass
* shorter values; daemon startup honors the OPENRIG_KERNEL_DEGRADED_MS
* env override resolved by startup.ts. */
degradedTimeoutMs?: number;
}
export class KernelBootTracker {
private state: KernelState = "skipped";
private variant: string | null = null;
private detail: string | null = null;
private firstUnreadySince: string | null = null;
private degradedTimer: ReturnType<typeof setTimeout> | null = null;
private degradedEmitted = false;
private bootstrapInFlight = false;
constructor(private readonly deps: KernelBootTrackerDeps) {}
/** Mark the tracker as having intentionally not booted (--no-kernel,
* already-managed short-circuit, VITEST auto-skip). Terminal state;
* no degraded timer. */
setSkipped(detail: string): void {
this.cancelTimer();
this.state = "skipped";
this.detail = detail;
this.firstUnreadySince = null;
}
/** Auth-blocked terminal. Operator sees the 3-part error in detail. */
setAuthBlocked(message: string): void {
this.cancelTimer();
this.state = "auth_blocked";
this.detail = message;
this.firstUnreadySince = null;
}
/** Spec-missing terminal. Path is in detail for ops triage. */
setSpecMissing(specPath: string): void {
this.cancelTimer();
this.state = "spec_missing";
this.detail = specPath;
this.firstUnreadySince = null;
}
/** Begin tracking an in-flight bootstrap. The bootstrap promise
* is awaited internally; the caller does NOT block on it. */
startBooting(variant: string, bootstrapPromise: Promise<BootstrapResult>): void {
if (this.bootstrapInFlight) return;
this.bootstrapInFlight = true;
this.state = "booting";
this.variant = variant;
this.detail = null;
this.firstUnreadySince = new Date().toISOString();
this.degradedEmitted = false;
this.scheduleDegradedTimer();
bootstrapPromise
.then((result) => this.onBootstrapComplete(result))
.catch((err) => this.onBootstrapError(err));
}
/** Read current status. Computes agents[] live from the sessions
* table so the response always reflects the freshest startup_status. */
getStatus(): KernelBootStatus {
const agents = this.computeAgents();
let kernelState = this.state;
// Once bootstrap has completed (state == 'booting' before then),
// promote to ready / partial_ready based on agent startup_status.
if (this.state === "booting" && !this.bootstrapInFlight) {
kernelState = this.aggregateReadinessFromAgents(agents);
}
return {
kernelState,
agents,
firstUnreadySince:
kernelState === "ready" || kernelState === "skipped"
? null
: this.firstUnreadySince,
variant: this.variant,
detail: this.detail,
};
}
/** Stop the degraded timer. Safe to call from anywhere (idempotent).
* Production callers don't need this; tests + graceful daemon
* shutdown do. */
stop(): void {
this.cancelTimer();
}
private onBootstrapComplete(result: BootstrapResult): void {
this.bootstrapInFlight = false;
if (result.errors && result.errors.length > 0) {
this.cancelTimer();
this.state = "bootstrap_failed";
this.detail = result.errors.join("; ");
return;
}
// Bootstrap returned cleanly. State transitions to ready /
// partial_ready are computed on read from agents[]. Cancel the
// degraded timer ONLY when at least one agent is ready — until
// then, the kernel is still effectively booting and the operator
// wants the degraded telemetry if no agent ever reaches ready.
const agents = this.computeAgents();
const aggregated = this.aggregateReadinessFromAgents(agents);
if (aggregated === "ready" || aggregated === "partial_ready") {
this.cancelTimer();
}
}
private onBootstrapError(err: unknown): void {
this.bootstrapInFlight = false;
this.cancelTimer();
this.state = "bootstrap_failed";
this.detail = err instanceof Error ? err.message : String(err);
}
private aggregateReadinessFromAgents(
agents: KernelAgentStatus[],
): KernelState {
if (agents.length === 0) {
// Bootstrap finished but no agents are registered yet (race
// window between session insert + status update). Keep state
// as booting so the operator sees progress, not a false ready.
return "booting";
}
const readyCount = agents.filter((a) => a.startupStatus === "ready").length;
if (readyCount === agents.length) return "ready";
if (readyCount === 0) return "booting";
return "partial_ready";
}
private computeAgents(): KernelAgentStatus[] {
try {
const kernelRigs = this.deps.rigRepo.findRigsByName("kernel");
if (kernelRigs.length === 0) return [];
const out: KernelAgentStatus[] = [];
for (const rig of kernelRigs) {
const sessions = this.deps.sessionRegistry.getSessionsForRig(rig.id);
for (const s of sessions) {
// session-registry's Session shape carries startupStatus +
// sessionName; runtime comes from the node row. Read minimal
// fields here so the tracker doesn't pull in node-repository.
out.push({
sessionName: s.sessionName,
runtime: (s as { runtime?: string }).runtime ?? "unknown",
startupStatus: s.startupStatus,
});
}
}
return out;
} catch {
// Tracker must NEVER throw — /api/kernel/status returning a
// valid envelope with empty agents[] is more useful than a 500
// when the DB has a transient hiccup.
return [];
}
}
private scheduleDegradedTimer(): void {
const ms = this.deps.degradedTimeoutMs ?? 90_000;
if (ms <= 0) return;
this.cancelTimer();
this.degradedTimer = setTimeout(() => this.checkDegraded(), ms);
// Allow the daemon to exit cleanly without waiting on the timer.
if (typeof this.degradedTimer === "object" && "unref" in this.degradedTimer) {
(this.degradedTimer as unknown as { unref(): void }).unref();
}
}
private cancelTimer(): void {
if (this.degradedTimer !== null) {
clearTimeout(this.degradedTimer);
this.degradedTimer = null;
}
}
private checkDegraded(): void {
const agents = this.computeAgents();
const aggregated =
this.state === "booting"
? this.aggregateReadinessFromAgents(agents)
: this.state;
if (aggregated === "ready" || aggregated === "partial_ready") {
// Made it before the deadline; no degraded emission.
return;
}
// Promote to degraded + emit telemetry exactly once.
if (this.degradedEmitted) return;
this.degradedEmitted = true;
this.state = "degraded";
try {
this.deps.eventBus.emit({
type: "kernel.agent.degraded",
agents: agents.map((a) => ({
sessionName: a.sessionName,
runtime: a.runtime,
startupStatus: a.startupStatus,
})),
firstUnreadySince: this.firstUnreadySince,
detail: this.detail,
});
} catch {
// Best-effort telemetry; tracker must not throw.
}
}
}
+52 -46
View File
@@ -2,19 +2,27 @@
//
// Composes into rig daemon start per IMPL-PRD §6.1: probes runtime
// auth state, selects a variant (dual / claude-only / codex-only),
// and bootstraps the kernel rig via the in-process bootstrap pipeline.
// and fires the in-process bootstrap pipeline IN THE BACKGROUND.
// Short-circuits cleanly when:
// - OPENRIG_NO_KERNEL=1 is set (operator opt-out / test fixture)
// - A managed rig named `kernel` already exists in the DB
// Surfaces a 3-part-error (fact / reason / fix) when neither Claude
// Code nor Codex is authenticated. Failures during the bootstrap
// itself are logged but do NOT crash the daemon — the daemon serves
// without the kernel and the operator can retry manually.
// - Both runtimes are unauthenticated (auth_blocked terminal state)
// - The selected variant's spec file is missing
//
// Forward-fix #3 architectural amendment (IMPL-PRD §16 amended):
// The bootstrap promise is NOT awaited. KernelBootTracker is returned
// immediately so createDaemon can finish + server.ts can bind healthz.
// A broken kernel agent therefore can no longer keep the daemon's HTTP
// surface from starting — operators see "daemon ready; kernel <state>"
// rather than "daemon failed to start".
import nodePath from "node:path";
import { existsSync } from "node:fs";
import type { RigRepository } from "./rig-repository.js";
import type { BootstrapOrchestrator } from "./bootstrap-orchestrator.js";
import type { EventBus } from "./event-bus.js";
import type { SessionRegistry } from "./session-registry.js";
import { KernelBootTracker } from "./kernel-boot-tracker.js";
export type RuntimeAuthStatus = "ok" | "unavailable";
@@ -25,6 +33,8 @@ export interface RuntimeProbeResult {
export interface KernelBootDeps {
rigRepo: RigRepository;
sessionRegistry: SessionRegistry;
eventBus: EventBus;
bootstrapOrchestrator: BootstrapOrchestrator;
/** Absolute path to the daemon's specs dir (passed in so tests can
* inject a fixture root). Production callsite resolves to the
@@ -42,25 +52,25 @@ export interface KernelBootDeps {
/** Logger sink; defaults to console.log/warn so daemon stdout/stderr
* carry the boot trace. */
log?: (level: "info" | "warn" | "error", message: string) => void;
/** Degraded-timer override (ms). Forward-fix #3 default 90s; tests
* pass short values to exercise the timer path quickly. */
degradedTimeoutMs?: number;
}
export interface KernelBootResult {
outcome:
| "booted"
| "skipped_no_kernel_flag"
| "skipped_already_managed"
| "auth_blocked"
| "spec_missing"
| "bootstrap_failed";
/** Variant filename used when outcome=booted; null otherwise. */
variant?: string | null;
/** Human-readable detail for logs + tests. */
detail?: string;
}
/** Entry point. Idempotent: safe to call on every daemon startup. */
export async function bootKernelIfNeeded(deps: KernelBootDeps): Promise<KernelBootResult> {
/** Entry point. Idempotent: safe to call on every daemon startup.
* Returns the tracker immediately; bootstrap (if fired) runs in the
* background. Callers should NOT await the tracker's internal
* bootstrap promise — that's the whole point of the architectural
* decoupling. Tracker state is observable via getStatus() and the
* /api/kernel/status route. */
export async function bootKernelIfNeeded(deps: KernelBootDeps): Promise<KernelBootTracker> {
const log = deps.log ?? defaultLog;
const tracker = new KernelBootTracker({
eventBus: deps.eventBus,
sessionRegistry: deps.sessionRegistry,
rigRepo: deps.rigRepo,
degradedTimeoutMs: deps.degradedTimeoutMs,
});
// 1. --no-kernel opt-out (CLI flag projected via OPENRIG_NO_KERNEL).
// VITEST env additionally short-circuits the live runtime probe
@@ -75,7 +85,8 @@ export async function bootKernelIfNeeded(deps: KernelBootDeps): Promise<KernelBo
if (process.env["OPENRIG_NO_KERNEL"] === "1" || (process.env["VITEST"] === "true" && !probeInjected)) {
const reason = process.env["OPENRIG_NO_KERNEL"] === "1" ? "OPENRIG_NO_KERNEL=1" : "VITEST=true";
log("info", `kernel-boot: ${reason} — skipping kernel auto-boot`);
return { outcome: "skipped_no_kernel_flag" };
tracker.setSkipped(reason);
return tracker;
}
// 2. Already-managed short-circuit. If a rig named `kernel` exists
@@ -83,15 +94,18 @@ export async function bootKernelIfNeeded(deps: KernelBootDeps): Promise<KernelBo
// the builtin path stays out of the way.
if (kernelAlreadyManaged(deps.rigRepo)) {
log("info", "kernel-boot: kernel rig already managed; skipping builtin boot");
return { outcome: "skipped_already_managed" };
tracker.setSkipped("kernel rig already managed");
return tracker;
}
// 3. Probe runtime auth state to pick a variant.
const probe = await (deps.probeRuntimes ?? defaultProbeRuntimes)();
if (probe.claudeCode === "unavailable" && probe.codex === "unavailable") {
log("error", authBlockMessage());
return { outcome: "auth_blocked", detail: authBlockMessage() };
const msg = authBlockMessage();
log("error", msg);
tracker.setAuthBlocked(msg);
return tracker;
}
const variant = selectVariant(probe);
@@ -99,29 +113,21 @@ export async function bootKernelIfNeeded(deps: KernelBootDeps): Promise<KernelBo
if (!existsSync(specPath)) {
log("warn", `kernel-boot: spec missing at ${specPath} — skipping`);
return { outcome: "spec_missing", detail: specPath };
tracker.setSpecMissing(specPath);
return tracker;
}
// 4. Cold boot via the in-process bootstrap pipeline.
try {
log("info", `kernel-boot: booting kernel rig from ${variant}`);
const result = await deps.bootstrapOrchestrator.bootstrap({
mode: "apply",
sourceRef: specPath,
sourceKind: "rig_spec",
autoApprove: true,
cwdOverride: deps.cwdOverride,
});
if (result.errors && result.errors.length > 0) {
log("warn", `kernel-boot: bootstrap finished with errors: ${result.errors.join("; ")}`);
return { outcome: "bootstrap_failed", detail: result.errors.join("; ") };
}
return { outcome: "booted", variant };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
log("warn", `kernel-boot: bootstrap threw: ${msg}`);
return { outcome: "bootstrap_failed", detail: msg };
}
// 4. Cold boot via the in-process bootstrap pipeline — FIRED, not awaited.
log("info", `kernel-boot: booting kernel rig from ${variant}`);
const bootstrapPromise = deps.bootstrapOrchestrator.bootstrap({
mode: "apply",
sourceRef: specPath,
sourceKind: "rig_spec",
autoApprove: true,
cwdOverride: deps.cwdOverride,
});
tracker.startBooting(variant, bootstrapPromise);
return tracker;
}
/** True when a rig named `kernel` already exists in the DB. */
+6
View File
@@ -139,6 +139,12 @@ export type RigEvent =
| { type: "node.startup_failed"; rigId: string; nodeId: string; error: string }
| { type: "continuity.sync"; rigId: string; podId: string; nodeId: string }
| { type: "continuity.degraded"; rigId: string; podId: string; nodeId: string; reason: string }
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
// Emitted exactly once by KernelBootTracker when the kernel rig fails to
// reach ready / partial_ready within the configurable degraded-timer
// window (default 90s). Observability signal that healthz bound cleanly
// but the kernel itself is stuck — operator triage with `rig ps --rig kernel`.
| { type: "kernel.agent.degraded"; agents: Array<{ sessionName: string; runtime: string; startupStatus: string }>; firstUnreadySince: string | null; detail: string | null }
// Chat events
| { type: "chat.message"; rigId: string; messageId: string; sender: string; kind: string; body: string; topic?: string }
// Expansion events
@@ -0,0 +1,47 @@
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
//
// GET /api/kernel/status — read-only observability surface for the
// background kernel-boot. Pairs with the architectural decoupling in
// kernel-boot-tracker.ts: healthz binds early; this route lets
// operators (and CLI's `--wait-for-kernel`) observe kernel agent
// readiness independently of HTTP health.
//
// Returns one of two shapes:
//
// 200 { kernel_state, agents[], first_unready_since, variant, detail }
// — when the tracker is wired (the standard daemon composition).
//
// 503 { error: 'kernel_boot_tracker_unavailable', message: ... }
// — when AppDeps was constructed without a tracker (test fixtures
// or custom daemon compositions). Clear hint instead of a 500.
import { Hono } from "hono";
import type { KernelBootTracker } from "../domain/kernel-boot-tracker.js";
export const kernelStatusRoutes = new Hono();
kernelStatusRoutes.get("/status", (c) => {
const tracker = c.get("kernelBootTracker" as never) as KernelBootTracker | undefined;
if (!tracker) {
return c.json(
{
error: "kernel_boot_tracker_unavailable",
message:
"Kernel-boot tracker not wired into this daemon. Use --no-kernel or check startup logs for a kernel-boot failure that prevented tracker construction.",
},
503,
);
}
const status = tracker.getStatus();
return c.json({
kernel_state: status.kernelState,
agents: status.agents.map((a) => ({
session_name: a.sessionName,
runtime: a.runtime,
startup_status: a.startupStatus,
})),
first_unready_since: status.firstUnreadySince,
variant: status.variant,
detail: status.detail,
});
});
+9
View File
@@ -43,6 +43,7 @@ import type { UpCommandRouter } from "./domain/up-command-router.js";
import type { RigTeardownOrchestrator } from "./domain/rig-teardown.js";
import { upRoutes } from "./routes/up.js";
import { downRoutes } from "./routes/down.js";
import { kernelStatusRoutes } from "./routes/kernel-status.js";
import type { TranscriptStore } from "./domain/transcript-store.js";
import type { SessionTransport } from "./domain/session-transport.js";
import type { AgentActivityStore } from "./domain/agent-activity-store.js";
@@ -203,6 +204,12 @@ export interface AppDeps {
serviceOrchestrator?: import("./domain/service-orchestrator.js").ServiceOrchestrator;
composeAdapter?: import("./adapters/compose-services-adapter.js").ComposeServicesAdapter;
uiDistDir?: string | null;
/** V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
* Tracker exposed via GET /api/kernel/status. Optional because tests
* + custom daemon compositions may construct AppDeps without auto-
* booting the kernel; the route returns 503 with a clear message
* when the tracker isn't wired. */
kernelBootTracker?: import("./domain/kernel-boot-tracker.js").KernelBootTracker;
}
const MIME_TYPES: Record<string, string> = {
@@ -382,6 +389,7 @@ export function createApp(deps: AppDeps): Hono {
c.set("activityHookToken" as never, deps.activityHookToken);
c.set("serviceOrchestrator" as never, deps.serviceOrchestrator);
c.set("composeAdapter" as never, deps.composeAdapter);
c.set("kernelBootTracker" as never, deps.kernelBootTracker);
c.set("db" as never, deps.rigRepo.db);
await next();
});
@@ -409,6 +417,7 @@ export function createApp(deps: AppDeps): Hono {
app.route("/api/ps", psRoutes);
app.route("/api/up", upRoutes);
app.route("/api/down", downRoutes);
app.route("/api/kernel", kernelStatusRoutes);
app.route("/api/transcripts", transcriptRoutes());
app.route("/api/transport", transportRoutes());
app.route("/api/activity", activityRoutes);
+25 -10
View File
@@ -482,18 +482,31 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
});
// V0.3.1 slice 05 kernel-rig-as-default — auto-boot the kernel rig
// on daemon-start. Idempotent: --no-kernel (OPENRIG_NO_KERNEL=1)
// skips; an already-managed `kernel` rig short-circuits; the
// variant (dual / claude-only / codex-only) is selected by probing
// `claude auth status` + `codex login status`; both-unavailable
// surfaces a 3-part-error and the daemon continues without the
// kernel. The kernel rig is the only rig the daemon auto-boots —
// other rigs require explicit operator-initiated `rig up` /
// `rig restore` (per amended IMPL-PRD §16.2).
// on daemon-start. Forward-fix #3 architectural: the bootstrap is
// FIRED in the background, not awaited. createDaemon completes as
// soon as the tracker is created so server.ts can bind healthz
// independent of kernel-agent readiness. A broken kernel agent no
// longer keeps the daemon HTTP surface from starting.
//
// Tracker state is exposed via /api/kernel/status (route below) and
// the CLI's `rig daemon start --wait-for-kernel` flag polls it.
// After the configurable degraded-timer window (default 90s; env
// override OPENRIG_KERNEL_DEGRADED_MS for ops + test fixtures), the
// tracker emits a single `kernel.agent.degraded` event for
// observability. The kernel rig is still the only rig the daemon
// auto-boots; other rigs require explicit operator-initiated
// `rig up` / `rig restore` per amended IMPL-PRD §16.2.
let kernelBootTracker: import("./domain/kernel-boot-tracker.js").KernelBootTracker | undefined;
try {
const { bootKernelIfNeeded } = await import("./domain/kernel-boot.js");
const kernelBootResult = await bootKernelIfNeeded({
const degradedRaw = readOpenRigEnv("OPENRIG_KERNEL_DEGRADED_MS");
const degradedTimeoutMs = degradedRaw && /^\d+$/.test(degradedRaw)
? parseInt(degradedRaw, 10)
: undefined;
kernelBootTracker = await bootKernelIfNeeded({
rigRepo,
sessionRegistry,
eventBus,
bootstrapOrchestrator,
specsDir: nodePath.resolve(nodePath.dirname(new URL(import.meta.url).pathname), "..", "specs"),
// V0.3.1 slice 05 — kernel members run against the operator's
@@ -502,10 +515,11 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
// "cwd is inside the OpenRig installation". Use the resolved
// workspace.root setting as the per-operator default.
cwdOverride: runtimeSettings.workspaceRoot,
degradedTimeoutMs,
});
try {
// eslint-disable-next-line no-console
console.log(`kernel-boot: outcome=${kernelBootResult.outcome}${kernelBootResult.variant ? ` variant=${kernelBootResult.variant}` : ""}`);
console.log(`kernel-boot: tracker-state=${kernelBootTracker.getStatus().kernelState}`);
} catch { /* logging must never throw */ }
} catch (err) {
try {
@@ -665,6 +679,7 @@ export async function createDaemon(opts?: DaemonOptions): Promise<DaemonResult>
contextUsageStore,
serviceOrchestrator,
composeAdapter,
kernelBootTracker,
specReviewService,
specLibraryService: (() => {
const userSpecsRoot = getDefaultOpenRigPath("specs");
@@ -0,0 +1,297 @@
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
//
// KernelBootTracker unit tests. Tracker is the observable state surface
// for the background kernel-boot; getStatus() projects from the tracker's
// own state machine + the sessions table for agents[]. The bootstrap
// promise is fire-and-forget; tests synthesize promises directly so the
// orchestration aspects (probe / variant pick) are out of scope here.
import { describe, expect, it, vi } from "vitest";
import { KernelBootTracker } from "../src/domain/kernel-boot-tracker.js";
import type { EventBus } from "../src/domain/event-bus.js";
import type { SessionRegistry } from "../src/domain/session-registry.js";
import type { RigRepository } from "../src/domain/rig-repository.js";
function makeRigRepo(rigs: Array<{ id: string; name: string }>): RigRepository {
return {
listRigs: () => rigs,
findRigsByName: (name: string) => rigs.filter((r) => r.name === name),
} as unknown as RigRepository;
}
function makeSessionRegistry(
sessionsByRig: Record<string, Array<{ sessionName: string; runtime?: string; startupStatus: string }>>,
): SessionRegistry {
return {
getSessionsForRig: (rigId: string) => sessionsByRig[rigId] ?? [],
} as unknown as SessionRegistry;
}
function makeEventBus(): { bus: EventBus; emitted: Array<{ type: string }> } {
const emitted: Array<{ type: string }> = [];
const bus = {
emit: (event: { type: string }) => {
emitted.push(event);
return event;
},
} as unknown as EventBus;
return { bus, emitted };
}
async function flush(): Promise<void> {
await new Promise<void>((r) => setImmediate(r));
}
describe("KernelBootTracker — initial state", () => {
it("defaults to skipped with empty agents until set otherwise", () => {
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
});
const status = tracker.getStatus();
expect(status.kernelState).toBe("skipped");
expect(status.agents).toEqual([]);
expect(status.firstUnreadySince).toBeNull();
expect(status.variant).toBeNull();
tracker.stop();
});
it("setSkipped records the reason in detail", () => {
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
});
tracker.setSkipped("OPENRIG_NO_KERNEL=1");
const status = tracker.getStatus();
expect(status.kernelState).toBe("skipped");
expect(status.detail).toBe("OPENRIG_NO_KERNEL=1");
});
it("setAuthBlocked / setSpecMissing transition to the matching state with detail", () => {
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
});
tracker.setAuthBlocked("Error: ...");
expect(tracker.getStatus().kernelState).toBe("auth_blocked");
expect(tracker.getStatus().detail).toBe("Error: ...");
tracker.setSpecMissing("/fake/path");
expect(tracker.getStatus().kernelState).toBe("spec_missing");
expect(tracker.getStatus().detail).toBe("/fake/path");
});
});
describe("KernelBootTracker — agent readiness aggregation", () => {
it("getStatus aggregates ready when every kernel agent reaches startup_status=ready", async () => {
const rigId = "rig-kernel-1";
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({
[rigId]: [
{ sessionName: "advisor-lead@kernel", runtime: "claude-code", startupStatus: "ready" },
{ sessionName: "operator-agent@kernel", runtime: "codex", startupStatus: "ready" },
{ sessionName: "queue-worker@kernel", runtime: "codex", startupStatus: "ready" },
],
}),
rigRepo: makeRigRepo([{ id: rigId, name: "kernel" }]),
degradedTimeoutMs: 0,
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "ok", stages: [], errors: [], warnings: [],
} as never));
await flush();
const status = tracker.getStatus();
expect(status.kernelState).toBe("ready");
expect(status.agents).toHaveLength(3);
expect(status.firstUnreadySince).toBeNull();
tracker.stop();
});
it("getStatus aggregates partial_ready when some agents are ready and others are pending", async () => {
const rigId = "rig-kernel-2";
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({
[rigId]: [
{ sessionName: "advisor-lead@kernel", runtime: "claude-code", startupStatus: "ready" },
{ sessionName: "operator-agent@kernel", runtime: "codex", startupStatus: "pending" },
],
}),
rigRepo: makeRigRepo([{ id: rigId, name: "kernel" }]),
degradedTimeoutMs: 0,
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "ok", stages: [], errors: [], warnings: [],
} as never));
await flush();
expect(tracker.getStatus().kernelState).toBe("partial_ready");
tracker.stop();
});
it("getStatus stays booting while ALL agents are pending after bootstrap completes", async () => {
const rigId = "rig-kernel-3";
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({
[rigId]: [
{ sessionName: "advisor-lead@kernel", runtime: "claude-code", startupStatus: "pending" },
],
}),
rigRepo: makeRigRepo([{ id: rigId, name: "kernel" }]),
degradedTimeoutMs: 0,
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "ok", stages: [], errors: [], warnings: [],
} as never));
await flush();
expect(tracker.getStatus().kernelState).toBe("booting");
tracker.stop();
});
it("propagates startup_status from sessions table into agents[]", async () => {
const rigId = "rig-kernel-4";
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({
[rigId]: [
{ sessionName: "operator-agent@kernel", runtime: "codex", startupStatus: "failed" },
{ sessionName: "advisor-lead@kernel", runtime: "claude-code", startupStatus: "attention_required" },
],
}),
rigRepo: makeRigRepo([{ id: rigId, name: "kernel" }]),
degradedTimeoutMs: 0,
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "ok", stages: [], errors: [], warnings: [],
} as never));
await flush();
const agents = tracker.getStatus().agents;
expect(agents).toHaveLength(2);
expect(agents.find((a) => a.sessionName === "operator-agent@kernel")?.startupStatus).toBe("failed");
expect(agents.find((a) => a.sessionName === "advisor-lead@kernel")?.startupStatus).toBe("attention_required");
tracker.stop();
});
});
describe("KernelBootTracker — bootstrap result transitions", () => {
it("onBootstrapComplete with errors transitions to bootstrap_failed", async () => {
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
degradedTimeoutMs: 0,
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "failed", stages: [], errors: ["preflight: tmux missing"], warnings: [],
} as never));
await flush();
const status = tracker.getStatus();
expect(status.kernelState).toBe("bootstrap_failed");
expect(status.detail).toContain("tmux missing");
tracker.stop();
});
it("onBootstrapError (promise rejection) transitions to bootstrap_failed with thrown message", async () => {
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
degradedTimeoutMs: 0,
});
tracker.startBooting("rig.yaml", Promise.reject(new Error("network blip")));
await flush();
const status = tracker.getStatus();
expect(status.kernelState).toBe("bootstrap_failed");
expect(status.detail).toBe("network blip");
tracker.stop();
});
});
describe("KernelBootTracker — degraded timer telemetry", () => {
it("emits kernel.agent.degraded exactly once when boot stays unready past the timer", async () => {
const { bus, emitted } = makeEventBus();
const tracker = new KernelBootTracker({
eventBus: bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
degradedTimeoutMs: 10,
});
// Promise that never resolves keeps tracker in booting state past the timer
const blocked = new Promise<never>(() => {});
tracker.startBooting("rig.yaml", blocked as never);
expect(tracker.getStatus().kernelState).toBe("booting");
await new Promise((r) => setTimeout(r, 30));
const degraded = emitted.filter((e) => e.type === "kernel.agent.degraded");
expect(degraded).toHaveLength(1);
expect(tracker.getStatus().kernelState).toBe("degraded");
// Subsequent reads should not re-emit even if checkDegraded ran again
await new Promise((r) => setTimeout(r, 30));
expect(emitted.filter((e) => e.type === "kernel.agent.degraded")).toHaveLength(1);
tracker.stop();
});
it("does NOT emit degraded when at least one agent is ready before the timer fires", async () => {
const { bus, emitted } = makeEventBus();
const rigId = "rig-kernel-fast";
const tracker = new KernelBootTracker({
eventBus: bus,
sessionRegistry: makeSessionRegistry({
[rigId]: [{ sessionName: "advisor-lead@kernel", runtime: "claude-code", startupStatus: "ready" }],
}),
rigRepo: makeRigRepo([{ id: rigId, name: "kernel" }]),
degradedTimeoutMs: 50,
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "ok", stages: [], errors: [], warnings: [],
} as never));
await flush();
expect(tracker.getStatus().kernelState).toBe("ready");
await new Promise((r) => setTimeout(r, 80));
expect(emitted.filter((e) => e.type === "kernel.agent.degraded")).toHaveLength(0);
tracker.stop();
});
it("stop() cancels the degraded timer (idempotent)", async () => {
const { bus, emitted } = makeEventBus();
const tracker = new KernelBootTracker({
eventBus: bus,
sessionRegistry: makeSessionRegistry({}),
rigRepo: makeRigRepo([]),
degradedTimeoutMs: 10,
});
const blocked = new Promise<never>(() => {});
tracker.startBooting("rig.yaml", blocked as never);
tracker.stop();
tracker.stop(); // idempotent
await new Promise((r) => setTimeout(r, 30));
expect(emitted.filter((e) => e.type === "kernel.agent.degraded")).toHaveLength(0);
});
});
describe("KernelBootTracker — sessionRegistry error handling", () => {
it("getStatus returns empty agents[] when sessionRegistry throws (no 500)", () => {
const throwingRegistry: SessionRegistry = {
getSessionsForRig: vi.fn(() => {
throw new Error("DB connection lost");
}),
} as unknown as SessionRegistry;
const rigId = "rig-kernel-broken";
const tracker = new KernelBootTracker({
eventBus: makeEventBus().bus,
sessionRegistry: throwingRegistry,
rigRepo: makeRigRepo([{ id: rigId, name: "kernel" }]),
degradedTimeoutMs: 0,
});
const status = tracker.getStatus();
expect(status.agents).toEqual([]);
expect(status.kernelState).toBe("skipped");
tracker.stop();
});
});
+124 -118
View File
@@ -2,10 +2,10 @@
//
// Covers HG-2 (variant selection), HG-3 (auth-block 3-part error),
// HG-4 (already-managed short-circuit), HG-6 (--no-kernel flag via
// OPENRIG_NO_KERNEL env). Lifecycle in production is bootstrap-driven
// (BootstrapOrchestrator.bootstrap()); these tests inject a mocked
// orchestrator + probe so the logic is exercised without invoking the
// in-process bootstrap pipeline.
// OPENRIG_NO_KERNEL env). Forward-fix #3 architectural amendment:
// bootKernelIfNeeded now returns a KernelBootTracker; the bootstrap
// runs in the background. Tests await microtasks to observe the
// post-bootstrap tracker state.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
@@ -17,13 +17,34 @@ import {
kernelAlreadyManaged,
authBlockMessage,
type KernelBootDeps,
type RuntimeProbeResult,
} from "../src/domain/kernel-boot.js";
import type { RigRepository } from "../src/domain/rig-repository.js";
import type { BootstrapOrchestrator } from "../src/domain/bootstrap-orchestrator.js";
import type { SessionRegistry } from "../src/domain/session-registry.js";
import type { EventBus } from "../src/domain/event-bus.js";
function makeRigRepo(existingRigs: Array<{ name: string }>): RigRepository {
return { listRigs: () => existingRigs } as unknown as RigRepository;
function makeRigRepo(existingRigs: Array<{ id?: string; name: string }>): RigRepository {
return {
listRigs: () => existingRigs,
findRigsByName: (name: string) => existingRigs.filter((r) => r.name === name),
} as unknown as RigRepository;
}
function makeSessionRegistry(sessionsByRig: Record<string, Array<{ sessionName: string; runtime?: string; startupStatus: string }>> = {}): SessionRegistry {
return {
getSessionsForRig: (rigId: string) => sessionsByRig[rigId] ?? [],
} as unknown as SessionRegistry;
}
function makeEventBus(): { bus: EventBus; emitted: Array<{ type: string }> } {
const emitted: Array<{ type: string }> = [];
const bus = {
emit: (event: { type: string }) => {
emitted.push(event);
return event;
},
} as unknown as EventBus;
return { bus, emitted };
}
function makeBootstrapMock(result?: { errors?: string[]; throwError?: Error }) {
@@ -41,11 +62,35 @@ function makeBootstrapMock(result?: { errors?: string[]; throwError?: Error }) {
} as unknown as BootstrapOrchestrator;
}
/** Await pending microtasks so tracker's bootstrap-promise handlers can
* fire before status is read. The setImmediate hop is enough for
* vi.fn-wrapped async mocks that resolve in the same tick. */
async function flushPromises(): Promise<void> {
await new Promise<void>((resolve) => setImmediate(resolve));
}
function makeBaseDeps(
overrides: Partial<KernelBootDeps>,
specsDir: string,
): KernelBootDeps {
return {
rigRepo: makeRigRepo([]),
sessionRegistry: makeSessionRegistry(),
eventBus: makeEventBus().bus,
bootstrapOrchestrator: makeBootstrapMock(),
specsDir,
cwdOverride: specsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
degradedTimeoutMs: 0, // disable degraded timer for default tests
...overrides,
};
}
let tmpSpecsDir: string;
beforeEach(() => {
tmpSpecsDir = mkdtempSync(join(tmpdir(), "kernel-boot-"));
// Populate the 3 variant files so existsSync(specPath) succeeds.
const kernelDir = join(tmpSpecsDir, "rigs/launch/kernel");
mkdirSync(kernelDir, { recursive: true });
writeFileSync(join(kernelDir, "rig.yaml"), "name: kernel\n");
@@ -94,147 +139,108 @@ describe("authBlockMessage — 3-part error contract", () => {
});
describe("bootKernelIfNeeded — short-circuit branches", () => {
it("skips with outcome=skipped_no_kernel_flag when OPENRIG_NO_KERNEL=1", async () => {
it("returns skipped tracker when OPENRIG_NO_KERNEL=1", async () => {
process.env.OPENRIG_NO_KERNEL = "1";
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
bootstrapOrchestrator: makeBootstrapMock(),
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("skipped_no_kernel_flag");
expect((deps.bootstrapOrchestrator as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
const bootstrap = makeBootstrapMock();
const tracker = await bootKernelIfNeeded(makeBaseDeps({ bootstrapOrchestrator: bootstrap }, tmpSpecsDir));
expect(tracker.getStatus().kernelState).toBe("skipped");
expect(tracker.getStatus().detail).toBe("OPENRIG_NO_KERNEL=1");
expect((bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
});
it("skips with outcome=skipped_already_managed when a kernel rig already exists", async () => {
const deps: KernelBootDeps = {
it("returns skipped tracker when a kernel rig is already managed", async () => {
const bootstrap = makeBootstrapMock();
const tracker = await bootKernelIfNeeded(makeBaseDeps({
rigRepo: makeRigRepo([{ name: "kernel" }]),
bootstrapOrchestrator: makeBootstrapMock(),
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("skipped_already_managed");
expect((deps.bootstrapOrchestrator as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
bootstrapOrchestrator: bootstrap,
}, tmpSpecsDir));
expect(tracker.getStatus().kernelState).toBe("skipped");
expect(tracker.getStatus().detail).toContain("already managed");
expect((bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
});
it("returns auth_blocked when neither runtime is available", async () => {
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
bootstrapOrchestrator: makeBootstrapMock(),
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
it("returns auth_blocked tracker when neither runtime is available", async () => {
const bootstrap = makeBootstrapMock();
const tracker = await bootKernelIfNeeded(makeBaseDeps({
bootstrapOrchestrator: bootstrap,
probeRuntimes: async () => ({ claudeCode: "unavailable", codex: "unavailable" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("auth_blocked");
expect(result.detail).toMatch(/^Error: Kernel rig cannot boot/);
expect((deps.bootstrapOrchestrator as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
}, tmpSpecsDir));
const status = tracker.getStatus();
expect(status.kernelState).toBe("auth_blocked");
expect(status.detail).toMatch(/^Error: Kernel rig cannot boot/);
expect((bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
});
it("returns spec_missing when the chosen variant file doesn't exist", async () => {
it("returns spec_missing tracker when the chosen variant file doesn't exist", async () => {
rmSync(join(tmpSpecsDir, "rigs/launch/kernel/rig.yaml"));
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
bootstrapOrchestrator: makeBootstrapMock(),
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("spec_missing");
const bootstrap = makeBootstrapMock();
const tracker = await bootKernelIfNeeded(makeBaseDeps({ bootstrapOrchestrator: bootstrap }, tmpSpecsDir));
expect(tracker.getStatus().kernelState).toBe("spec_missing");
expect((bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).not.toHaveBeenCalled();
});
});
describe("bootKernelIfNeeded — happy path", () => {
it("invokes BootstrapOrchestrator.bootstrap with the dual variant when both runtimes available", async () => {
const bootstrap = makeBootstrapMock();
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
bootstrapOrchestrator: bootstrap,
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("booted");
expect(result.variant).toBe("rig.yaml");
const bootstrapMock = (bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap;
expect(bootstrapMock).toHaveBeenCalledOnce();
const opts = bootstrapMock.mock.calls[0]![0];
describe("bootKernelIfNeeded — fire-and-forget bootstrap", () => {
it("fires bootstrap in the background with the resolved variant + correct opts", async () => {
// Hold the bootstrap mock open so we can observe the in-flight
// booting state deterministically before the promise resolves.
let release: () => void = () => {};
const blocked = new Promise<void>((r) => { release = r; });
const bootstrap = {
bootstrap: vi.fn(async () => {
await blocked;
return { runId: "t", status: "ok", stages: [], errors: [], warnings: [] };
}),
} as unknown as BootstrapOrchestrator;
const tracker = await bootKernelIfNeeded(makeBaseDeps({ bootstrapOrchestrator: bootstrap }, tmpSpecsDir));
expect(tracker.getStatus().kernelState).toBe("booting");
expect(tracker.getStatus().variant).toBe("rig.yaml");
expect((bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap).toHaveBeenCalledOnce();
const opts = (bootstrap as unknown as { bootstrap: ReturnType<typeof vi.fn> }).bootstrap.mock.calls[0]![0];
expect(opts.mode).toBe("apply");
expect(opts.sourceRef).toBe(join(tmpSpecsDir, "rigs/launch/kernel", "rig.yaml"));
expect(opts.sourceKind).toBe("rig_spec");
expect(opts.autoApprove).toBe(true);
// V0.3.1 slice 05 forward-fix: cwdOverride threads through to
// bootstrap so kernel members run against the operator's
// workspace, not the daemon installation tree.
expect(opts.cwdOverride).toBe(tmpSpecsDir);
release();
tracker.stop();
});
it("returns bootstrap_failed and surfaces error detail when orchestrator reports errors", async () => {
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
it("transitions to bootstrap_failed when orchestrator returns errors (post-flush)", async () => {
const tracker = await bootKernelIfNeeded(makeBaseDeps({
bootstrapOrchestrator: makeBootstrapMock({ errors: ["preflight: tmux missing"] }),
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("bootstrap_failed");
expect(result.detail).toContain("tmux missing");
}, tmpSpecsDir));
await flushPromises();
const status = tracker.getStatus();
expect(status.kernelState).toBe("bootstrap_failed");
expect(status.detail).toContain("tmux missing");
tracker.stop();
});
it("returns bootstrap_failed and surfaces thrown error message", async () => {
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
it("transitions to bootstrap_failed and surfaces thrown error message (post-flush)", async () => {
const tracker = await bootKernelIfNeeded(makeBaseDeps({
bootstrapOrchestrator: makeBootstrapMock({ throwError: new Error("network blip") }),
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
probeRuntimes: async () => ({ claudeCode: "ok", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("bootstrap_failed");
expect(result.detail).toBe("network blip");
}, tmpSpecsDir));
await flushPromises();
const status = tracker.getStatus();
expect(status.kernelState).toBe("bootstrap_failed");
expect(status.detail).toBe("network blip");
tracker.stop();
});
it("uses the claude-only variant when only Claude is available", async () => {
const bootstrap = makeBootstrapMock();
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
bootstrapOrchestrator: bootstrap,
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
const tracker = await bootKernelIfNeeded(makeBaseDeps({
probeRuntimes: async () => ({ claudeCode: "ok", codex: "unavailable" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("booted");
expect(result.variant).toBe("rig-claude-only.yaml");
}, tmpSpecsDir));
expect(tracker.getStatus().variant).toBe("rig-claude-only.yaml");
tracker.stop();
});
it("uses the codex-only variant when only Codex is available", async () => {
const bootstrap = makeBootstrapMock();
const deps: KernelBootDeps = {
rigRepo: makeRigRepo([]),
bootstrapOrchestrator: bootstrap,
specsDir: tmpSpecsDir,
cwdOverride: tmpSpecsDir,
const tracker = await bootKernelIfNeeded(makeBaseDeps({
probeRuntimes: async () => ({ claudeCode: "unavailable", codex: "ok" }),
log: () => {},
};
const result = await bootKernelIfNeeded(deps);
expect(result.outcome).toBe("booted");
expect(result.variant).toBe("rig-codex-only.yaml");
}, tmpSpecsDir));
expect(tracker.getStatus().variant).toBe("rig-codex-only.yaml");
tracker.stop();
});
});
@@ -0,0 +1,101 @@
// V0.3.1 slice 05 kernel-rig-as-default — forward-fix #3 architectural.
//
// GET /api/kernel/status route tests. Verifies the route surfaces the
// tracker's state via a stable JSON envelope and 503s cleanly when no
// tracker is wired.
import { describe, expect, it } from "vitest";
import { Hono } from "hono";
import { kernelStatusRoutes } from "../src/routes/kernel-status.js";
import { KernelBootTracker } from "../src/domain/kernel-boot-tracker.js";
import type { EventBus } from "../src/domain/event-bus.js";
import type { SessionRegistry } from "../src/domain/session-registry.js";
import type { RigRepository } from "../src/domain/rig-repository.js";
function mountWithTracker(tracker: KernelBootTracker | undefined) {
const app = new Hono();
app.use("*", async (c, next) => {
c.set("kernelBootTracker" as never, tracker);
await next();
});
app.route("/api/kernel", kernelStatusRoutes);
return app;
}
function makeTracker(opts: {
rigs?: Array<{ id: string; name: string }>;
sessionsByRig?: Record<string, Array<{ sessionName: string; runtime: string; startupStatus: string }>>;
} = {}) {
const eventBus = { emit: () => undefined } as unknown as EventBus;
const sessionRegistry = {
getSessionsForRig: (rigId: string) => opts.sessionsByRig?.[rigId] ?? [],
} as unknown as SessionRegistry;
const rigRepo = {
listRigs: () => opts.rigs ?? [],
findRigsByName: (name: string) => (opts.rigs ?? []).filter((r) => r.name === name),
} as unknown as RigRepository;
return new KernelBootTracker({ eventBus, sessionRegistry, rigRepo, degradedTimeoutMs: 0 });
}
describe("GET /api/kernel/status", () => {
it("returns 503 with a clear error when no tracker is wired", async () => {
const app = mountWithTracker(undefined);
const res = await app.request("/api/kernel/status");
expect(res.status).toBe(503);
const body = await res.json();
expect(body.error).toBe("kernel_boot_tracker_unavailable");
expect(body.message).toContain("Kernel-boot tracker not wired");
});
it("returns 200 with skipped envelope for a fresh tracker", async () => {
const app = mountWithTracker(makeTracker());
const res = await app.request("/api/kernel/status");
expect(res.status).toBe(200);
const body = await res.json();
expect(body.kernel_state).toBe("skipped");
expect(body.agents).toEqual([]);
expect(body.first_unready_since).toBeNull();
expect(body.variant).toBeNull();
expect(body.detail).toBeNull();
});
it("returns 200 with auth_blocked envelope when tracker is auth-blocked", async () => {
const tracker = makeTracker();
tracker.setAuthBlocked("Error: ...\nReason: ...\nFix: ...");
const app = mountWithTracker(tracker);
const res = await app.request("/api/kernel/status");
const body = await res.json();
expect(body.kernel_state).toBe("auth_blocked");
expect(body.detail).toContain("Error:");
});
it("projects agents[] with snake_case keys from tracker getStatus()", async () => {
const rigId = "rig-kernel-route-1";
const tracker = makeTracker({
rigs: [{ id: rigId, name: "kernel" }],
sessionsByRig: {
[rigId]: [
{ sessionName: "advisor-lead@kernel", runtime: "claude-code", startupStatus: "ready" },
{ sessionName: "operator-agent@kernel", runtime: "codex", startupStatus: "pending" },
],
},
});
tracker.startBooting("rig.yaml", Promise.resolve({
runId: "t", status: "ok", stages: [], errors: [], warnings: [],
} as never));
await new Promise<void>((r) => setImmediate(r));
const app = mountWithTracker(tracker);
const res = await app.request("/api/kernel/status");
const body = await res.json();
expect(body.kernel_state).toBe("partial_ready");
expect(body.variant).toBe("rig.yaml");
expect(body.agents).toHaveLength(2);
expect(body.agents[0]).toMatchObject({
session_name: "advisor-lead@kernel",
runtime: "claude-code",
startup_status: "ready",
});
expect(body.first_unready_since).toMatch(/^\d{4}-\d{2}-\d{2}T/);
tracker.stop();
});
});