diff --git a/apps/fleet/app.json b/apps/fleet/app.json new file mode 100644 index 0000000..f080f90 --- /dev/null +++ b/apps/fleet/app.json @@ -0,0 +1,26 @@ +{ + "manifest_version": 1, + "id": "fleet", + "name": "FLEET", + "summary": "Your OpenRig machines at a glance: which accounts have room, which machines report, and what your rigs are doing.", + "category": "studio", + "maker": { + "name": "OpenRig", + "url": "https://github.com/mvschwarz" + }, + "surface": { + "entry": "app/fleet.html", + "path": "/surfaces/fleet.html", + "glyph": "▦", + "hint": "accounts, machines and rigs at a glance" + }, + "calls": { + "/api/fleet-view/fleet": { "required": true } + }, + "vendor": [], + "provider": { + "package": "@openrig/studio-fleet-view", + "required": true + }, + "_note": "Read only, and it needs no other app. It shows what OpenRig's own daemons report and says when something is not reported. It never asks for a sign-in, reads a token or spends an account's allowance." +} diff --git a/apps/fleet/app/fleet.html b/apps/fleet/app/fleet.html new file mode 100644 index 0000000..ecdb416 --- /dev/null +++ b/apps/fleet/app/fleet.html @@ -0,0 +1,307 @@ + + + + + + Fleet + + + +
+

Fleet

+

— means not reported here yet. Old readings show their time. An account shared by machines is counted once.

+

Reading…

+
+ +
+ + + + diff --git a/apps/observatory/app/observatory.html b/apps/observatory/app/observatory.html index de9da94..dd84021 100644 --- a/apps/observatory/app/observatory.html +++ b/apps/observatory/app/observatory.html @@ -119,6 +119,7 @@ const sv = fleet.served; if (!sv) return `asked at ${esc(fleet.polledAt ?? '')}`; const age = Math.max(sv.cacheAgeSeconds ?? 0, Math.round((now - Date.parse(sv.servedAt)) / 1000) + (sv.cacheAgeSeconds ?? 0)); + if (sv.stale) return `read by the provider ${ageText(sv.polledAt, now)} (the last good read, ${esc(age)} s old, past the ${esc(sv.keptForSeconds)} s keep${sv.refreshing ? '; a fresh read is on its way' : ''})`; return `read by the provider ${ageText(sv.polledAt, now)}${sv.fromCache ? ` (a kept read, ${esc(age)} s old; reads are kept up to ${esc(sv.keptForSeconds)} s)` : ''}`; } diff --git a/providers/studio-observatory/lib/fleet-reader.mjs b/providers/shared/fleet-reader.mjs similarity index 98% rename from providers/studio-observatory/lib/fleet-reader.mjs rename to providers/shared/fleet-reader.mjs index 15ccbb0..08e1632 100644 --- a/providers/studio-observatory/lib/fleet-reader.mjs +++ b/providers/shared/fleet-reader.mjs @@ -108,7 +108,11 @@ async function readSeats(ctx, host) { // Seats carry their own observedAt (the activity sample time); the read as a whole has none. observedAt: null, data: seats, - coverage: { rigsListed: rigs.length, rigsRead: rigs.length - failed.length, rigsFailed: failed, archivedRigs: "excluded (the route's default)", complete: failed.length === 0 }, + coverage: { + rigsListed: rigs.length, rigsRead: rigs.length - failed.length, rigsFailed: failed, archivedRigs: "excluded (the route's default)", complete: failed.length === 0, + // Each rig as the daemon's own summary describes it (whether it is running), for views that group by rig. + rigs: rigs.map((r) => ({ key: `${host.key}/${r.name ?? r.rigId}`, rigId: r.rigId, name: r.name ?? null, status: r.status ?? null, lifecycleState: r.lifecycleState ?? null, nodeCount: r.nodeCount ?? null, runningCount: r.runningCount ?? null })), + }, }; } diff --git a/providers/shared/healthcheck-core.mjs b/providers/shared/healthcheck-core.mjs new file mode 100644 index 0000000..1c7f6d2 --- /dev/null +++ b/providers/shared/healthcheck-core.mjs @@ -0,0 +1,39 @@ +// A container healthcheck for a read-only provider behind the Studio. It asks the provider's ready route +// THROUGH THE STUDIO'S PORT, so passing means the studio and that provider are both serving. It never +// reads a daemon: whether the fleet can be read is a reading the app shows, not a reason to call the app +// unhealthy. +// +// Healthy only when the ready route answers ready from the named provider. Otherwise unhealthy, with a +// line saying what is known. When only the studio answers, that line is "shell answers; provider not +// confirmed", which is a failure, never a pass. +export const LABELS = { + shellOnly: "shell answers; provider not confirmed", + down: "studio not answering", +}; + +async function get(fetchImpl, url, timeoutMs) { + try { + const r = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + const text = await r.text(); + let body = null; + try { body = JSON.parse(text); } catch { /* not JSON */ } + return { ok: r.ok, status: r.status, body }; + } catch (err) { + return { ok: false, status: null, error: String(err?.cause?.code ?? err?.message ?? err) }; + } +} + +export async function checkProvider({ base, readyPath, provider, readyLabel, fetch: fetchImpl = globalThis.fetch, timeoutMs = 4000 }) { + const ready = await get(fetchImpl, `${base}${readyPath}`, timeoutMs); + if (ready.ok && ready.body?.ready === true && ready.body?.provider === provider) return { healthy: true, label: readyLabel }; + const shell = await get(fetchImpl, `${base}/api/contract`, timeoutMs); + const why = ready.status ? `ready verb answered HTTP ${ready.status}${ready.body?.reason ? ` (${ready.body.reason})` : ""}` : `ready verb: ${ready.error ?? "no answer"}`; + if (shell.ok) return { healthy: false, label: LABELS.shellOnly, detail: why }; + return { healthy: false, label: LABELS.down, detail: why }; +} + +export async function runHealthcheck(opts) { + const r = await checkProvider(opts); + console.log(r.detail ? `${r.label} (${r.detail})` : r.label); + process.exit(r.healthy ? 0 : 1); +} diff --git a/providers/shared/reader-server.mjs b/providers/shared/reader-server.mjs new file mode 100644 index 0000000..3373f77 --- /dev/null +++ b/providers/shared/reader-server.mjs @@ -0,0 +1,144 @@ +// The HTTP side shared by the read-only providers built on the fleet reader (the Observatory's and +// Fleet's). Each provider names its own routes; the behaviour below is the same for both. +// +// Cached readings, no fan-out in the request path once a reading exists: +// - The first request waits for the first read of the fleet (there is nothing else to show). +// - After that, every request is answered at once from the last good read. Within the keep time it is +// current; after it, it is served labelled stale, with its age, and ONE refresh starts in the +// background. Nothing polls on a timer: with nobody asking, nothing is read. +// - A read that could not reach the configured daemon, or that failed, is never kept. +// - Every response says when its read was made, how old it is, and whether it is stale. +// +// One seat's drill-down reads that one seat on its own daemon, kept briefly and only while it belongs +// to the fleet read it was made from. The ready route answers from the provider process alone: no +// daemon call and no fleet read. +// +// Nothing here writes to a daemon, starts anything on one, or holds a token. +import http from "node:http"; +import { readFleet, readSeat, READS } from "./fleet-reader.mjs"; + +export const READY_MEANING = "this provider process is loaded and serving; it says nothing about whether any OpenRig daemon can be read"; +const SEAT_KEEP_MS = 10_000; +const SEATS_REMEMBERED = 64; +export const ALL_READS = [...READS, "subscriptions"]; + +function json(res, status, value) { + res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + res.end(JSON.stringify(value)); +} + +export function createFleetCache({ daemon, read = readFleet, reads = ALL_READS, ttlMs = 15_000, now = Date.now } = {}) { + let kept = null; // { fleet, at }: the last read that reached the configured daemon + let inFlight = null; // Promise<{ fleet, at }> + let previous = null; // the last kept read, for the restart signal + let last = null; // how the last fleet read came out, kept or not: what the ready route reports + async function fresh() { + let fleet; + try { fleet = await read({ daemonUrl: daemon, reads, previous }); } + catch (err) { last = { polledAt: new Date(now()).toISOString(), ok: false, status: "error", detail: String(err?.message ?? err) }; throw err; } + const at = now(); + const status = fleet?.hosts?.[0]?.status ?? "unknown"; + last = { polledAt: fleet?.polledAt ?? new Date(at).toISOString(), ok: status === "ok", status }; + if (status === "ok") { kept = { fleet, at }; previous = fleet; } + return { fleet, at }; + } + function refresh() { + inFlight ??= fresh().finally(() => { inFlight = null; }); + return inFlight; + } + async function get() { + const t = now(); + let entry, fromCache, stale = false; + if (kept) { + entry = kept; + fromCache = true; + if (t - kept.at >= ttlMs) { + stale = true; + // Refresh outside the request: this answer does not wait for it. + refresh().catch(() => {}); + } + } else { + fromCache = false; + entry = await refresh(); + } + const servedAt = now(); + return { + ...entry.fleet, + served: { + servedAt: new Date(servedAt).toISOString(), polledAt: entry.fleet.polledAt, + cacheAgeSeconds: Math.max(0, Math.round((servedAt - entry.at) / 1000)), keptForSeconds: Math.round(ttlMs / 1000), + fromCache, stale, refreshing: inFlight !== null, + }, + }; + } + get.lastRead = () => last; + // For tests and shutdown: the refresh currently running, if any. + get.pending = () => inFlight; + return get; +} + +export function createSeatCache({ fleet, readOne = readSeat, ttlMs = SEAT_KEEP_MS, max = SEATS_REMEMBERED, now = Date.now } = {}) { + const kept = new Map(); // key -> { value, at }, oldest first + return async function seat(key, tenureCursor = null) { + const id = `${key}\u0000${tenureCursor ?? ""}`; + // The fleet read comes first, and a kept seat is used only while it belongs to that same fleet + // read. Otherwise a seat could still read as fine after its machine has stopped answering. + const f = await fleet(); + const hit = kept.get(id); + if (hit && now() - hit.at < ttlMs && hit.value.fleetServed?.polledAt === f.polledAt) return { ...hit.value, fleetServed: f.served, served: { fromCache: true, cacheAgeSeconds: Math.round((now() - hit.at) / 1000), keptForSeconds: Math.round(ttlMs / 1000) } }; + const value = await readOne({ fleet: f, key, tenureCursor, now }); + value.fleetServed = f.served; + if (value.status === "ok") { + kept.delete(id); + kept.set(id, { value, at: now() }); + while (kept.size > max) kept.delete(kept.keys().next().value); + } + return { ...value, served: { fromCache: false, cacheAgeSeconds: 0, keptForSeconds: Math.round(ttlMs / 1000) } }; + }; +} + +// routes: { fleet: [paths answered with the whole fleet read], seat: path|null, ready: path } +export function createReaderServer({ name, pkg = {}, routes, daemon, read = readFleet, reads = ALL_READS, ttlMs = 15_000, now = Date.now, readOne = readSeat, startedAt = new Date().toISOString() }) { + const fleet = createFleetCache({ daemon, read, reads, ttlMs, now }); + const seat = routes.seat ? createSeatCache({ fleet, readOne, now }) : null; + const known = new Set([...routes.fleet, routes.ready, ...(routes.seat ? [routes.seat] : [])]); + const server = http.createServer(async (req, res) => { + const url = new URL(req.url, "http://127.0.0.1"); + if (!known.has(url.pathname)) return json(res, 404, { error: `no ${name} route ${url.pathname}` }); + if (req.method !== "GET" && req.method !== "HEAD") { + res.setHeader("allow", "GET, HEAD"); + return json(res, 405, { error: "read only: GET" }); + } + if (url.pathname === routes.ready) { + const body = { + schema: `openrig.${name}.ready/v1`, + ready: !server.draining, + meaning: READY_MEANING, + provider: pkg.name ?? null, version: pkg.version ?? null, + startedAt, servedAt: new Date(now()).toISOString(), + lastFleetRead: fleet.lastRead(), + ...(server.draining ? { reason: "the provider is shutting down" } : {}), + }; + return json(res, server.draining ? 503 : 200, body); + } + try { + if (seat && url.pathname === routes.seat) { + const key = url.searchParams.get("key"); + if (!key) return json(res, 400, { error: "one seat per call: give key=//" }); + const out = await seat(key, url.searchParams.get("cursor")); + return json(res, out.status === "ok" ? 200 : out.status === "seat_host_unknown" || out.status === "seat_not_found" ? 404 : out.status === "seat_key_required" ? 400 : 200, out); + } + return json(res, 200, await fleet()); + } catch (err) { + return json(res, 500, { error: String(err?.message ?? err) }); + } + }); + server.draining = false; + server.fleet = fleet; + return server; +} + +export function runReaderServer(server, { port, label }) { + for (const sig of ["SIGTERM", "SIGINT"]) process.on(sig, () => { server.draining = true; server.close(() => process.exit(0)); setTimeout(() => process.exit(0), 2000).unref(); }); + server.listen(port, "127.0.0.1", () => console.log(label)); +} diff --git a/providers/studio-fleet-view/fleet-view-server.mjs b/providers/studio-fleet-view/fleet-view-server.mjs new file mode 100644 index 0000000..b220291 --- /dev/null +++ b/providers/studio-fleet-view/fleet-view-server.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node +// Read-only fleet readings for the Fleet page, from OpenRig daemons' existing GET routes. The behaviour +// (cached readings served at once and refreshed outside the request; the ready route) is shared with the +// Observatory provider and described in providers/shared/reader-server.mjs. This file names Fleet's +// routes and the readings Fleet shows. It needs no Observatory. +// +// Nothing here writes to a daemon, starts anything on one, holds a token, or asks a model provider for +// anything: usage comes only from what OpenRig's daemons already report. +import fs from "node:fs"; +import { pathToFileURL } from "node:url"; +import { createReaderServer, runReaderServer } from "../shared/reader-server.mjs"; + +function arg(name, fallback = "") { + const i = process.argv.indexOf(name); + return i >= 0 ? process.argv[i + 1] : fallback; +} + +const port = Number(arg("--port", "9451")); +const daemonUrl = arg("--daemon", process.env.FLEET_DAEMON_URL || "http://127.0.0.1:7433"); +const cacheSeconds = Number(process.env.FLEET_CACHE_SECONDS ?? 15); +const PACKAGE = (() => { try { return JSON.parse(fs.readFileSync(new URL("./package.json", import.meta.url), "utf8")); } catch { return {}; } })(); + +export const FLEET_ROUTE = "/api/fleet-view/fleet"; +export const READY_ROUTE = "/api/fleet-view/ready"; +// What Fleet shows: machines (with their health summaries), rigs and seats, configured links and usage. +export const FLEET_READS = ["seats", "health", "bindings", "subscriptions"]; + +export function createFleetViewServer({ daemon = daemonUrl, read, ttlMs = cacheSeconds * 1000, now = Date.now } = {}) { + return createReaderServer({ + name: "fleet-view", pkg: PACKAGE, daemon, ttlMs, now, reads: FLEET_READS, + ...(read ? { read } : {}), + routes: { fleet: [FLEET_ROUTE], seat: null, ready: READY_ROUTE }, + }); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runReaderServer(createFleetViewServer(), { port, label: `studio-fleet-view on 127.0.0.1:${port} · daemon ${daemonUrl} · reads kept ${cacheSeconds} s` }); +} diff --git a/providers/studio-fleet-view/healthcheck.mjs b/providers/studio-fleet-view/healthcheck.mjs new file mode 100644 index 0000000..a15d2d3 --- /dev/null +++ b/providers/studio-fleet-view/healthcheck.mjs @@ -0,0 +1,15 @@ +#!/usr/bin/env node +// The Fleet container's healthcheck: the Fleet provider's ready route, asked through the studio's port +// (providers/shared/healthcheck-core.mjs has the rules). +import { pathToFileURL } from "node:url"; +import { checkProvider, runHealthcheck, LABELS as CORE } from "../shared/healthcheck-core.mjs"; + +export const PROVIDER = "@openrig/studio-fleet-view"; +export const LABELS = { ready: "provider ready: the studio and the fleet provider are serving", ...CORE }; +const opts = (o = {}) => ({ base: "http://127.0.0.1:9450", readyPath: "/api/fleet-view/ready", provider: PROVIDER, readyLabel: LABELS.ready, ...o }); + +export async function check(o = {}) { return checkProvider(opts(o)); } + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await runHealthcheck(opts({ base: process.env.FLEET_HEALTH_BASE || "http://127.0.0.1:9450" })); +} diff --git a/providers/studio-fleet-view/package.json b/providers/studio-fleet-view/package.json new file mode 100644 index 0000000..f7f9419 --- /dev/null +++ b/providers/studio-fleet-view/package.json @@ -0,0 +1,11 @@ +{ + "name": "@openrig/studio-fleet-view", + "version": "0.1.0", + "type": "module", + "license": "Apache-2.0", + "description": "Fleet readings for the Fleet page: machines, rigs, seats and subscription usage from each OpenRig daemon, with where and when each was measured.", + "scripts": { + "start": "node fleet-view-server.mjs", + "test": "node --test ../../test/fleet-app.test.mjs ../../test/fleet-studio.test.mjs" + } +} diff --git a/providers/studio-fleet-view/provider.json b/providers/studio-fleet-view/provider.json new file mode 100644 index 0000000..4f126bf --- /dev/null +++ b/providers/studio-fleet-view/provider.json @@ -0,0 +1,12 @@ +{ + "package": "@openrig/studio-fleet-view", + "run": { + "entry": "fleet-view-server.mjs", + "args": ["--port", "{{port}}"] + }, + "verbs": [ + "/api/fleet-view/fleet", + "/api/fleet-view/ready" + ], + "_note": "Read only. /api/fleet-view/fleet is the fleet as OpenRig's own daemons report it (the daemon on this machine, FLEET_DAEMON_URL, default http://127.0.0.1:7433, and each HTTP host in its host list): machines, rigs and seats, machine health summaries, configured subscription links and usage readings. A reading is served from the last good read at once and refreshed outside the request; nothing polls when nobody asks. No token is read, stored or sent, and no request is made to a model provider." +} diff --git a/providers/studio-observatory/healthcheck.mjs b/providers/studio-observatory/healthcheck.mjs index f75e2c4..65b60e5 100644 --- a/providers/studio-observatory/healthcheck.mjs +++ b/providers/studio-observatory/healthcheck.mjs @@ -1,43 +1,15 @@ #!/usr/bin/env node -// The Observatory container's healthcheck. It asks the provider's ready verb THROUGH THE STUDIO'S PORT, -// so passing means the studio and the observatory provider are both serving. It never reads a daemon: -// whether the fleet can be read is a reading the app shows, not a reason to call the app unhealthy. -// -// Exit 0 only when /api/observatory/ready answers ready from this provider. Otherwise exit 1 with a -// line saying what is known. When only the studio answers, that line is "shell answers; provider not -// confirmed", which is a failure, never a pass. +// The Observatory container's healthcheck: the Observatory provider's ready route, asked through the +// studio's port (providers/shared/healthcheck-core.mjs has the rules). import { pathToFileURL } from "node:url"; +import { checkProvider, runHealthcheck, LABELS as CORE } from "../shared/healthcheck-core.mjs"; export const PROVIDER = "@openrig/studio-observatory"; -export const LABELS = { - ready: "provider ready: the studio and the observatory provider are serving", - shellOnly: "shell answers; provider not confirmed", - down: "studio not answering", -}; +export const LABELS = { ready: "provider ready: the studio and the observatory provider are serving", ...CORE }; +const opts = (o = {}) => ({ base: "http://127.0.0.1:9440", readyPath: "/api/observatory/ready", provider: PROVIDER, readyLabel: LABELS.ready, ...o }); -async function get(fetchImpl, url, timeoutMs) { - try { - const r = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); - const text = await r.text(); - let body = null; - try { body = JSON.parse(text); } catch { /* not JSON */ } - return { ok: r.ok, status: r.status, body }; - } catch (err) { - return { ok: false, status: null, error: String(err?.cause?.code ?? err?.message ?? err) }; - } -} - -export async function check({ base = "http://127.0.0.1:9440", fetch: fetchImpl = globalThis.fetch, timeoutMs = 4000 } = {}) { - const ready = await get(fetchImpl, `${base}/api/observatory/ready`, timeoutMs); - if (ready.ok && ready.body?.ready === true && ready.body?.provider === PROVIDER) return { healthy: true, label: LABELS.ready }; - const shell = await get(fetchImpl, `${base}/api/contract`, timeoutMs); - const why = ready.status ? `ready verb answered HTTP ${ready.status}${ready.body?.reason ? ` (${ready.body.reason})` : ""}` : `ready verb: ${ready.error ?? "no answer"}`; - if (shell.ok) return { healthy: false, label: LABELS.shellOnly, detail: why }; - return { healthy: false, label: LABELS.down, detail: why }; -} +export async function check(o = {}) { return checkProvider(opts(o)); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const r = await check({ base: process.env.OBSERVATORY_HEALTH_BASE || undefined }); - console.log(r.detail ? `${r.label} (${r.detail})` : r.label); - process.exit(r.healthy ? 0 : 1); + await runHealthcheck(opts({ base: process.env.OBSERVATORY_HEALTH_BASE || "http://127.0.0.1:9440" })); } diff --git a/providers/studio-observatory/observatory-server.mjs b/providers/studio-observatory/observatory-server.mjs index 52f30ef..6388949 100644 --- a/providers/studio-observatory/observatory-server.mjs +++ b/providers/studio-observatory/observatory-server.mjs @@ -1,27 +1,15 @@ #!/usr/bin/env node // Read-only fleet views for the Observatory, built from OpenRig daemons' existing GET routes. -// -// One read of the whole fleet serves every verb: the configured daemon, plus every machine its -// host list names. A read is kept for a short time (OBSERVATORY_CACHE_SECONDS, default 15) so a page -// refresh, which asks for every verb at once, costs one read rather than one per verb. Concurrent -// requests share the read in flight. Every response says when that read was made and how old it is, -// so a kept read is never presented as a fresh one. A read that could not reach the configured daemon -// is not kept. -// -// One seat's drill-down (/api/observatory/seat?key=//) reads that one seat on -// its own daemon. Those reads are kept for a shorter time of their own, and only a bounded number of -// seats are remembered at once. -// -// GET /api/observatory/ready says only that this provider process is loaded and serving. It makes no -// daemon call and no fleet read; it reports the last fleet read the provider already made, as that read -// came out. A container healthcheck asks it through the studio's port, so "ready" there means the studio -// and this provider are both answering, which /api/contract alone (the studio's runtime) cannot show. +// The behaviour (cached readings served at once, refreshed outside the request; the seat drill-down; +// the ready route) is shared with the Fleet provider and described in providers/shared/reader-server.mjs. +// This file names the Observatory's routes. // // Nothing here writes to a daemon, starts anything on one, or holds a token. -import http from "node:http"; import fs from "node:fs"; import { pathToFileURL } from "node:url"; -import { readFleet, readSeat, READS } from "./lib/fleet-reader.mjs"; +import { createReaderServer, runReaderServer, createFleetCache, createSeatCache, READY_MEANING, ALL_READS } from "../shared/reader-server.mjs"; + +export { createFleetCache, createSeatCache, READY_MEANING }; function arg(name, fallback = "") { const i = process.argv.indexOf(name); @@ -45,114 +33,15 @@ export const VERBS = [ ]; export const FLEET_VERBS = VERBS.filter((v) => v !== "/api/observatory/seat" && v !== "/api/observatory/ready"); const PACKAGE = (() => { try { return JSON.parse(fs.readFileSync(new URL("./package.json", import.meta.url), "utf8")); } catch { return {}; } })(); -const STARTED_AT = new Date().toISOString(); -export const READY_MEANING = "this provider process is loaded and serving; it says nothing about whether any OpenRig daemon can be read"; -const SEAT_KEEP_MS = 10_000; -const SEATS_REMEMBERED = 64; -const ALL_READS = [...READS, "subscriptions"]; -function json(res, status, value) { - res.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); - res.end(JSON.stringify(value)); -} - -export function createFleetCache({ daemon, read = readFleet, ttlMs = cacheSeconds * 1000, now = Date.now } = {}) { - let kept = null; // { fleet, at } - let inFlight = null; // Promise<{ fleet, at }> - let previous = null; // the last read that reached the configured daemon, for the restart signal - let last = null; // how the last fleet read came out, kept or not: what /ready reports - async function fresh() { - let fleet; - try { fleet = await read({ daemonUrl: daemon, reads: ALL_READS, previous }); } - catch (err) { last = { polledAt: new Date(now()).toISOString(), ok: false, status: "error", detail: String(err?.message ?? err) }; throw err; } - const at = now(); - const status = fleet?.hosts?.[0]?.status ?? "unknown"; - last = { polledAt: fleet?.polledAt ?? new Date(at).toISOString(), ok: status === "ok", status }; - if (status === "ok") { kept = { fleet, at }; previous = fleet; } - return { fleet, at }; - } - async function get() { - const t = now(); - let entry, fromCache; - if (kept && t - kept.at < ttlMs) { entry = kept; fromCache = true; } - else { - fromCache = false; - inFlight ??= fresh().finally(() => { inFlight = null; }); - entry = await inFlight; - } - const servedAt = now(); - return { - ...entry.fleet, - served: { servedAt: new Date(servedAt).toISOString(), polledAt: entry.fleet.polledAt, cacheAgeSeconds: Math.max(0, Math.round((servedAt - entry.at) / 1000)), keptForSeconds: Math.round(ttlMs / 1000), fromCache }, - }; - } - // What the provider already knows, without reading anything. - get.lastRead = () => last; - return get; -} - -export function createSeatCache({ fleet, readOne = readSeat, ttlMs = SEAT_KEEP_MS, max = SEATS_REMEMBERED, now = Date.now } = {}) { - const kept = new Map(); // key -> { value, at }, oldest first - return async function seat(key, tenureCursor = null) { - const id = `${key}\u0000${tenureCursor ?? ""}`; - // The fleet read comes first, and a kept seat is used only while it belongs to that same fleet - // read. Otherwise a seat could still read as fine after its machine has stopped answering. - const f = await fleet(); - const hit = kept.get(id); - if (hit && now() - hit.at < ttlMs && hit.value.fleetServed?.polledAt === f.polledAt) return { ...hit.value, served: { fromCache: true, cacheAgeSeconds: Math.round((now() - hit.at) / 1000), keptForSeconds: Math.round(ttlMs / 1000) } }; - const value = await readOne({ fleet: f, key, tenureCursor, now }); - value.fleetServed = f.served; - if (value.status === "ok") { - kept.delete(id); - kept.set(id, { value, at: now() }); - while (kept.size > max) kept.delete(kept.keys().next().value); - } - return { ...value, served: { fromCache: false, cacheAgeSeconds: 0, keptForSeconds: Math.round(ttlMs / 1000) } }; - }; -} - -export function createObservatoryServer({ daemon = daemonUrl, read = readFleet, ttlMs = cacheSeconds * 1000, now = Date.now, readOne = readSeat } = {}) { - const fleet = createFleetCache({ daemon, read, ttlMs, now }); - const seat = createSeatCache({ fleet, readOne, now }); - const server = http.createServer(async (req, res) => { - const url = new URL(req.url, "http://127.0.0.1"); - if (!VERBS.includes(url.pathname)) return json(res, 404, { error: `no observatory route ${url.pathname}` }); - if (req.method !== "GET" && req.method !== "HEAD") { - res.setHeader("allow", "GET, HEAD"); - return json(res, 405, { error: "read only: GET" }); - } - if (url.pathname === "/api/observatory/ready") { - const body = { - schema: "openrig.observatory.ready/v1", - ready: !server.draining, - meaning: READY_MEANING, - provider: PACKAGE.name ?? null, version: PACKAGE.version ?? null, - startedAt: STARTED_AT, servedAt: new Date(now()).toISOString(), - lastFleetRead: fleet.lastRead(), - ...(server.draining ? { reason: "the provider is shutting down" } : {}), - }; - return json(res, server.draining ? 503 : 200, body); - } - try { - if (url.pathname === "/api/observatory/seat") { - const key = url.searchParams.get("key"); - if (!key) return json(res, 400, { error: "one seat per call: give key=//" }); - const out = await seat(key, url.searchParams.get("cursor")); - return json(res, out.status === "ok" ? 200 : out.status === "seat_host_unknown" || out.status === "seat_not_found" ? 404 : out.status === "seat_key_required" ? 400 : 200, out); - } - return json(res, 200, await fleet()); - } catch (err) { - return json(res, 500, { error: String(err?.message ?? err) }); - } +export function createObservatoryServer({ daemon = daemonUrl, read, ttlMs = cacheSeconds * 1000, now = Date.now, readOne } = {}) { + return createReaderServer({ + name: "observatory", pkg: PACKAGE, daemon, ttlMs, now, reads: ALL_READS, + ...(read ? { read } : {}), ...(readOne ? { readOne } : {}), + routes: { fleet: FLEET_VERBS, seat: "/api/observatory/seat", ready: "/api/observatory/ready" }, }); - server.draining = false; - return server; } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const server = createObservatoryServer(); - for (const sig of ["SIGTERM", "SIGINT"]) process.on(sig, () => { server.draining = true; server.close(() => process.exit(0)); setTimeout(() => process.exit(0), 2000).unref(); }); - server.listen(port, "127.0.0.1", () => { - console.log(`studio-observatory on 127.0.0.1:${port} · daemon ${daemonUrl} · reads kept ${cacheSeconds} s`); - }); + runReaderServer(createObservatoryServer(), { port, label: `studio-observatory on 127.0.0.1:${port} · daemon ${daemonUrl} · reads kept ${cacheSeconds} s` }); } diff --git a/rigs/observatory/agents/observatory-operator/agent.yaml b/rigs/agents/starter-ops/agent.yaml similarity index 74% rename from rigs/observatory/agents/observatory-operator/agent.yaml rename to rigs/agents/starter-ops/agent.yaml index 3cdc209..ed8f7fb 100644 --- a/rigs/observatory/agents/observatory-operator/agent.yaml +++ b/rigs/agents/starter-ops/agent.yaml @@ -1,6 +1,6 @@ -name: observatory-operator +name: starter-ops version: "0.1" -description: Starter operations agent; runs and shapes the Observatory, may grow its own rig, changes nothing outside it +description: Starter operations agent; runs its rig and its apps, knows the fleet routes, may grow its own rig, changes nothing outside it defaults: runtime: claude-code diff --git a/rigs/observatory/agents/observatory-operator/guidance/role.md b/rigs/agents/starter-ops/guidance/role.md similarity index 64% rename from rigs/observatory/agents/observatory-operator/guidance/role.md rename to rigs/agents/starter-ops/guidance/role.md index 9469295..dc95305 100644 --- a/rigs/observatory/agents/observatory-operator/guidance/role.md +++ b/rigs/agents/starter-ops/guidance/role.md @@ -2,7 +2,7 @@ You are the starter operations agent for this person's OpenRig setup. You help them run it, starting from what this rig gives you, and you may grow this rig as their needs grow. The Observatory is one -app you run and shape here; your other work does not depend on it. +app you may run and shape; your other work does not depend on it. This describes the role you are built for. It is not, by itself, permission on any particular machine: what you may do on a given machine is what its owner has actually enabled there. @@ -12,17 +12,45 @@ machine: what you may do on a given machine is what its owner has actually enabl This rig is yours. You may add to and change this rig (its members, its services and its app data) as normal, reviewed changes to its own files. -Its first app, the Observatory, is one Compose project named `openrig-observatory`: a single service, -`studio`, and one named volume for its data. That project, its service, its volume and this rig's own -reports are yours to set up, configure, start and recover. +Your rig's startup context says which apps it has. When it includes the Observatory, that is one +Compose project named `openrig-observatory`: a single service, `studio`, and one named volume for its +data. When it includes the Fleet page, that is one Compose project named `openrig-fleet-studio`, shaped +the same way. Your rig's app project, its service, its volume and this rig's own reports are yours to +set up, configure, start and recover. -You may run: +You may run, for the app your rig has: - `docker compose -p openrig-observatory ps`, `logs`, `up -d studio`, `restart studio` +- `docker compose -p openrig-fleet-studio ps`, `logs`, `up -d studio`, `restart studio` - `rig env status observatory` and `rig env logs observatory studio` -- `rig whoami --json`, `rig ps --json` and the Observatory's own pages, to look at the fleet +- `rig env status fleet-studio` and `rig env logs fleet-studio studio` +- `rig whoami --json`, `rig ps --json` and your app's own pages, to look at the fleet - `rig watchdog register --policy periodic-reminder` for a reminder to yourself, and no other - watchdog command + watchdog command (your startup context shows the exact form) + +## Routes you work with + +These are OpenRig's own commands; `rig --help` on this machine is the authority for their +options. Use them on this person's environment the way your role allows: read freely, and change only +what is yours. + +- `rig ps` (`--nodes`, `--json`, `--all-hosts`): rigs, seats, their activity and context use +- `rig queue`: work owed between seats +- `rig provider status`: accounts, configured seats and usage readings +- `rig auth status`: login state, without secrets +- `rig compact-plan`: a read-only plan for compacting seats in place +- `rig host pair
`: pair a second machine; the other machine's owner approves it +- `rig up` and `rig grow`: bring up or grow **your own** rig +- `rig watchdog`: reminders and early warnings for yourself +- `rig context list`, `rig context get `: shipped know-how, found by purpose +- the `openrig-upgrade` skill (`rig context get skills/core/openrig-upgrade`): upgrading in place + +## Know-how you carry, and anyone can use + +The practices you follow for recurring fleet work (recovering a stranded seat, reading usage honestly, +keeping context healthy) are in this machine's context library as `fleet-ops/...` entries. Read the +matching one with `rig context get` when the moment comes. They are not yours alone: any agent on the +machine can find and use them the same way. ## Customizing the Observatory @@ -60,8 +88,9 @@ Everything else is observed, never changed. Do not: - start, stop, restart, remove or bring down any other Compose project, container or volume - run `rig up`, `rig down`, `rig seat` or any other lifecycle command for another rig, seat or daemon -- change daemon settings, host lists, accounts, subscriptions or anyone's queue (the only exception is - the subscription-rotation job above, and only once it has been enabled for you) +- change daemon settings, host lists, accounts, subscriptions or anyone's queue (the only exceptions are + the subscription-rotation job above, once it has been enabled for you, and `rig host pair` for a machine + the person named, which its owner then approves) - read, copy or mount credentials, or anything under `~/.openrig`, `~/.claude` or `~/.codex` - publish or push the app's code upstream - change another rig, a host service, the OpenRig daemon, alert or retention policy, or any other @@ -88,7 +117,8 @@ the whole job there. ## Answering questions -- Answer "how is my fleet doing?" only from what the Observatory shows. For each machine, say +- Answer "how is my fleet doing?" only from what your rig's app (Fleet or the Observatory) and + OpenRig's own commands show. For each machine, say whether it was read, how old its readings are, and what could not be read and why. - Never turn a machine you could not read into "fine", "idle" or zero. Say "not read" and the reason. - Activity is activity. Do not call it productivity. diff --git a/rigs/agents/starter-ops/startup/context.md b/rigs/agents/starter-ops/startup/context.md new file mode 100644 index 0000000..efe139b --- /dev/null +++ b/rigs/agents/starter-ops/startup/context.md @@ -0,0 +1,21 @@ +# Startup context: getting your bearings + +1. `rig whoami --json`: confirm who and where you are, and which rig is yours. +2. Read your rig's own startup context (it follows this one): it says which apps this rig has and + where they are. +3. `rig ps --nodes --json`: what is running on this machine, and what needs attention. +4. `rig context list`: the know-how on this machine. Your rig's context says which entries to make + sure are there. +5. A reminder to yourself, for checking in regularly. A periodic reminder needs a small spec file + naming you and the message: + + ```yaml + target: + session: + message: "Check in: your rig, its apps and what needs attention." + ``` + + then `rig watchdog register --policy periodic-reminder --spec --target-session + --interval-seconds 1800 --registered-by `. Check `rig watchdog register --help` first. + +What you may do on this machine is what its owner has actually enabled here. diff --git a/rigs/fleet-studio/Dockerfile b/rigs/fleet-studio/Dockerfile new file mode 100644 index 0000000..c591151 --- /dev/null +++ b/rigs/fleet-studio/Dockerfile @@ -0,0 +1,41 @@ +# The Fleet image: the Studio SDK at a pinned public commit, plus the Fleet page and its provider +# from the checkout this rig is started from (the build context is the repository root). +FROM node:22.22.1-bookworm-slim@sha256:4f77a690f2f8946ab16fe1e791a3ac0667ae1c3575c3e4d0d4589e9ed5bfaf3d + +ARG SDK_REPO=https://github.com/mvschwarz/openrig-studio.git +ARG SDK_COMMIT=e7346bbdbd01539e6003ccccb38c44763236545b + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Fetch exactly the pinned commit, and refuse to build if something else arrived. +RUN git init -q /studio/sdk \ + && git -C /studio/sdk fetch -q --depth 1 "$SDK_REPO" "$SDK_COMMIT" \ + && git -C /studio/sdk checkout -q FETCH_HEAD \ + && test "$(git -C /studio/sdk rev-parse HEAD)" = "$SDK_COMMIT" + +COPY apps/fleet /studio/apps/apps/fleet +COPY providers/studio-fleet-view /studio/apps/providers/studio-fleet-view +COPY providers/shared /studio/apps/providers/shared + +# The SDK's install tool installs the APP, not its provider's dependencies; provisioning does +# those separately, so this step does too. The Fleet provider has no dependencies today, +# and the step says so rather than skipping silently. +RUN printf '{"apps":[],"appsRoot":"/studio/apps","port":9450}\n' > /studio/studio.json \ + && for p in $(node /studio/sdk/provision/needed-providers.mjs /studio/apps fleet); do \ + dir="/studio/apps/providers/$p"; \ + if [ -f "$dir/package-lock.json" ]; then npm ci --prefix "$dir" --omit=dev --no-audit --no-fund; \ + elif node -e "process.exit(Object.keys(require('$dir/package.json').dependencies || {}).length ? 0 : 1)"; then npm install --prefix "$dir" --omit=dev --no-audit --no-fund; \ + else echo "provider $p: no dependencies to install"; fi; \ + done \ + && cd /studio/sdk && OPENRIG_STUDIO_DIR=/studio node tools/install-app.mjs /studio/apps/apps/fleet --enable \ + && mkdir -p /data && chown -R node:node /studio /data + +LABEL org.opencontainers.image.title="openrig-fleet-studio" \ + openrig.fleet-studio.sdk-commit="$SDK_COMMIT" + +ENV OPENRIG_STUDIO_DIR=/studio +USER node +WORKDIR /studio/sdk +CMD ["node", "tools/studio.mjs", "--port", "9450"] diff --git a/rigs/fleet-studio/Dockerfile.dockerignore b/rigs/fleet-studio/Dockerfile.dockerignore new file mode 100644 index 0000000..6cf83dc --- /dev/null +++ b/rigs/fleet-studio/Dockerfile.dockerignore @@ -0,0 +1,5 @@ +# The build context is the repository root; the image needs only the Fleet page, its provider and the shared code. +* +!apps/fleet/** +!providers/studio-fleet-view/** +!providers/shared/** diff --git a/rigs/fleet-studio/README.md b/rigs/fleet-studio/README.md new file mode 100644 index 0000000..f9d5287 --- /dev/null +++ b/rigs/fleet-studio/README.md @@ -0,0 +1,38 @@ +# Fleet Studio launch rig + +A starter operations agent for an OpenRig machine, with the fleet know-how it carries: + +- recovering a stranded seat (a usage limit, a lost login, a restart) +- reading subscription usage honestly +- keeping agents' context healthy + +The know-how is three context packs in `context/fleet-ops/`. At startup the agent adds any that are +missing to the machine's context library, so **every** agent on the machine can find them with +`rig context list` and read them with `rig context get fleet-ops/`. You can also add them +yourself, without the agent: + +```sh +rig context add rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat +rig context add rigs/fleet-studio/context/fleet-ops/read-usage-honestly +rig context add rigs/fleet-studio/context/fleet-ops/keep-context-healthy +``` + +## Start it + +From a checkout of this repository: + +```sh +rig up rigs/fleet-studio/rig.yaml +``` + +The rig also runs the Fleet page in its own container (Compose project `openrig-fleet-studio`, port +9450): open http://127.0.0.1:9450/surfaces/fleet.html. It answers which accounts have room, which +machines report here and what the rigs are doing, from what OpenRig's own daemons report. It is read +only, needs no other app, and never asks for a sign-in or reads a token. The Observatory (`rigs/observatory/`) is an optional, +separate launch rig; Fleet Studio does not need it. Both rigs use the same starter operations agent +(`rigs/agents/starter-ops/`), each with its own startup context. + +## What it touches + +Its own rig and its own files. It reads the rest of the environment with OpenRig's commands and +changes nothing outside its rig; see the agent's role. diff --git a/rigs/fleet-studio/context/fleet-ops/keep-context-healthy/manifest.yaml b/rigs/fleet-studio/context/fleet-ops/keep-context-healthy/manifest.yaml new file mode 100644 index 0000000..2319507 --- /dev/null +++ b/rigs/fleet-studio/context/fleet-ops/keep-context-healthy/manifest.yaml @@ -0,0 +1,8 @@ +name: fleet-ops/keep-context-healthy +version: "1" +taxonomy: skills +purpose: "Keep agents' context healthy: check each seat's context use, plan compaction safely, set an early warning, and decide when a seat should hand over to a fresh successor instead." +files: + - path: practice.md + role: practice + summary: "Check context use, plan compact-in-place with rig compact-plan, warn early with a context watchdog, and hand over when that is better." diff --git a/rigs/fleet-studio/context/fleet-ops/keep-context-healthy/practice.md b/rigs/fleet-studio/context/fleet-ops/keep-context-healthy/practice.md new file mode 100644 index 0000000..8f97138 --- /dev/null +++ b/rigs/fleet-studio/context/fleet-ops/keep-context-healthy/practice.md @@ -0,0 +1,36 @@ +# Keep context healthy + +**When this applies:** a seat's context is filling up, its answers are getting worse, or you want warning before that +happens. + +## Check + +- `rig ps --nodes --json` shows each seat's `contextUsage`: how full it is, whether the sample is fresh, and when it was + sampled. +- `rig health` lists findings, including context pressure, with how fresh each finding is. +- `rig compact-plan` (`--rig `, `--refresh` to re-sample first, `--json`) is a **read-only** plan of which Claude + seats are candidates for compacting in place. It changes nothing. + +## Act + +1. **Compacting in place** is done inside the seat's own conversation, at a quiet moment between tasks. Agree it with + whoever owns that seat's work. +2. **Handing over** to a fresh successor is better when the seat's work is long-running and its context is no longer + trustworthy. `rig seat handover --dry-run` plans it without changing anything; give the real handover a + `--reason`, so the record says why. +3. **An early warning:** a `context-usage-threshold` watchdog sends a message before the limit: + `rig watchdog register --policy context-usage-threshold --target-session --interval-seconds + --registered-by --threshold-mb --message ""`. Check `rig watchdog register --help` for the + current options. +4. A rig can also choose a compaction strategy per seat in its rig spec (`compaction_strategy`). That's a change to that + rig, for its owner. + +## What needs a person + +Compacting or handing over a seat whose work belongs to someone else: ask that owner first. + +## Traps + +- **Compacting in the middle of a task.** Wait for a boundary. +- **Trusting a stale sample.** Check when it was taken; `--refresh` where it matters. +- **Starting a fresh seat when a handover would keep the work's continuity.** diff --git a/rigs/fleet-studio/context/fleet-ops/read-usage-honestly/manifest.yaml b/rigs/fleet-studio/context/fleet-ops/read-usage-honestly/manifest.yaml new file mode 100644 index 0000000..1b58f0e --- /dev/null +++ b/rigs/fleet-studio/context/fleet-ops/read-usage-honestly/manifest.yaml @@ -0,0 +1,8 @@ +name: fleet-ops/read-usage-honestly +version: "1" +taxonomy: skills +purpose: "Read subscription usage honestly: where each machine's usage windows and resets come from, how old they are, why readings from several machines are never added together, and what unknown means." +files: + - path: practice.md + role: practice + summary: "Read usage windows and resets with rig provider status, per machine, with their age; never a model request, never a total, never money." diff --git a/rigs/fleet-studio/context/fleet-ops/read-usage-honestly/practice.md b/rigs/fleet-studio/context/fleet-ops/read-usage-honestly/practice.md new file mode 100644 index 0000000..0a2a4e0 --- /dev/null +++ b/rigs/fleet-studio/context/fleet-ops/read-usage-honestly/practice.md @@ -0,0 +1,39 @@ +# Read subscription usage honestly + +**When this applies:** someone asks how much of their subscription is left, when it resets, or whether a seat can keep +working. + +## Where the readings come from + +- `rig provider status` (add `--json` for structure) shows accounts, which seat is configured for which account, and + usage signals. +- **Usage windows** (how much of each window is used, and when it resets) are only read when the machine's + `provider.usage_windows` setting is on. It is off by default. Turning it on is the machine owner's choice: + `rig config set provider.usage_windows true`. Ask before changing it. +- `rig provider status --refresh` reads the current windows natively first (it can take up to about 35 seconds, and needs + the setting on). +- **Several machines:** newer versions offer `rig provider status --all-hosts` (or `--hosts `) to see each + registered machine's readings side by side. Check `rig provider status --help` on your version. +- `rig usage` is per-seat **token** telemetry over time. It is not your subscription allowance; don't present it as one. + +## Read it honestly + +- **Every reading has an age.** Say when it was measured, not when you looked. An old reading is old. +- **Unknown is unknown.** A missing or unreadable window is "not known", never 0% and never "fine". +- **Each machine's reading stays separate.** Two machines can report the same shared allowance; adding them together + double-counts it. Show each, and say whether they agree. +- **Configured is not observed.** Which account a seat is set up for and which account it is actually using are two + different facts. Say which one you are quoting. +- **No money.** Subscriptions are read as usage and resets, not as a price. + +## What needs a person + +Turning on usage windows, and any decision to move a seat to another account (that belongs to whoever owns account +moves on the machine). + +## Traps + +- **Sending a model request to "check" usage.** It spends the allowance you are trying to read. +- **Adding readings from several machines into one total.** +- **Showing a missing window as zero.** +- **Quoting a configured account as the one in use.** diff --git a/rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat/manifest.yaml b/rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat/manifest.yaml new file mode 100644 index 0000000..87d4aa4 --- /dev/null +++ b/rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat/manifest.yaml @@ -0,0 +1,8 @@ +name: fleet-ops/recover-a-stranded-seat +version: "1" +taxonomy: skills +purpose: "Recover a stranded seat: an agent that stopped at a usage limit, lost its login, or was left stopped after a restart. How to recognise it, what to do with today's rig commands, and when to ask a person." +files: + - path: practice.md + role: practice + summary: "Recognise a seat stuck at a usage limit or a lost login, choose wait / sign in / resume, and keep its conversation." diff --git a/rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat/practice.md b/rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat/practice.md new file mode 100644 index 0000000..88a56b5 --- /dev/null +++ b/rigs/fleet-studio/context/fleet-ops/recover-a-stranded-seat/practice.md @@ -0,0 +1,49 @@ +# Recover a stranded seat + +**When this applies:** an agent seat has stopped doing work: it hit its subscription's usage limit, its login expired +or was lost, or it was left stopped after a machine restart. + +## Recognise it + +- `rig ps --nodes --json` shows each seat's `agentActivity` (`state`, `reason`, and when it was sampled) and its + attention state. A seat waiting on a person shows `needs_input` with a reason. +- `rig parked` diagnoses stopped seats that still owe work, for your rig (`--rig `) or one seat. +- `rig capture ` shows the seat's screen, where a usage-limit or sign-in message usually appears. +- `rig provider status --json` shows accounts, which seat is configured for which account, and usage signals with + their reset times when the machine has them (see the `fleet-ops/read-usage-honestly` practice). +- `rig auth status` shows login state without secrets. Check `rig auth status --help` for the runtimes it covers on your + version. + +## Choose a path + +**At a usage limit:** +1. Read the reset time from `rig provider status` (or the seat's own screen). Say how old that reading is. +2. If the reset is soon, wait. When it has passed, nudge the seat in its own conversation: + `rig send "Your usage limit has reset; carry on with your work."` +3. If work can't wait, a different account is a decision for whoever owns account moves on this machine. Ask them; + don't move accounts yourself unless that job has been enabled for you. + +**After a lost or expired login:** +1. Confirm it: `rig auth status`, and the seat's screen. +2. Signing in needs a person. Ask them **once**, plainly: which machine, which tool, and that the seat is waiting. +3. When they say it's done, check `rig auth status` again. For a Codex seat, `rig seat restart --keep-conversation + --reason "" --plan` shows what a restart would keep; run it again without `--plan` to pick up the new login in + the seat's own conversation. A Claude seat usually needs the person to finish signing in inside the seat itself. + +**After a machine restart:** `rig ps` shows which rigs are stopped. `rig up --existing ` brings a stopped rig +back from its saved state; seats resume their own conversations where they can. + +## What needs a person + +Signing in, choosing to use another account, and anything that spends money. Ask once, say exactly what is needed, then +carry on with what you can do. + +## Traps + +- **Starting a fresh seat to "fix" it** (`rig seat launch`) loses its conversation. Prefer waiting, resuming, or + restoring. +- **Checking usage by sending a model request.** That spends the very allowance you are reading. Use the readings the + machine already has. +- **Treating "I asked for a switch" as "the seat moved".** Check the seat's own state afterwards. +- **Asking the person again and again.** Once, then continue and report. +- **Copying credentials** between seats or machines. Never. diff --git a/rigs/fleet-studio/fleet-studio-startup.md b/rigs/fleet-studio/fleet-studio-startup.md new file mode 100644 index 0000000..ca018cd --- /dev/null +++ b/rigs/fleet-studio/fleet-studio-startup.md @@ -0,0 +1,36 @@ +# Startup context: Fleet Studio + +## What this rig has + +- You, the starter operations agent, and the fleet know-how in `context/fleet-ops/` under this rig's + directory (your working directory). +- The Fleet page, in its own container (Compose project `openrig-fleet-studio`), at + http://127.0.0.1:9450/surfaces/fleet.html: accounts, machines and rigs at a glance, read only. It + shows what OpenRig's daemons report and "—" for anything not reported; it never asks for a sign-in. + You also read the environment with OpenRig's own commands (your role lists them). +- If `rig env status fleet-studio` shows `studio: running (unhealthy)`, read + `docker compose -p openrig-fleet-studio logs studio`, restart `studio` once, and tell the person + what happened, what you did and anything you could not fix. Do not loop. +- The Observatory is optional. If the person wants it, it is its own launch rig; this rig does not + need it. + +## First: make sure the know-how is in this machine's library + +Check `rig context list --json` for these three entries: + +- `fleet-ops/recover-a-stranded-seat` +- `fleet-ops/read-usage-honestly` +- `fleet-ops/keep-context-healthy` + +Add only the ones that are missing, one at a time, from this rig's directory: + +`rig context add context/fleet-ops/` + +(`rig context add` refuses a name that is already there, so adding only what is missing keeps this +safe to repeat.) Then confirm with `rig context get fleet-ops/`. Once they are in the library, +any agent on this machine can find them by purpose with `rig context list`. + +## Then + +Point the person to the Fleet page, and tell them what you found: which rigs and seats are running, anything stranded or close to a +limit, and what is not known and why. Use the matching `fleet-ops` practice when one applies. diff --git a/rigs/fleet-studio/fleet-studio.compose.yaml b/rigs/fleet-studio/fleet-studio.compose.yaml new file mode 100644 index 0000000..3b4c9de --- /dev/null +++ b/rigs/fleet-studio/fleet-studio.compose.yaml @@ -0,0 +1,36 @@ +# The Fleet page as one container in its own Compose project. +# +# - Built from this checkout's Fleet page and provider plus the Studio SDK at a pinned commit (see Dockerfile). +# - Host networking: the studio listens on 127.0.0.1 only, and it reads the OpenRig daemon on +# 127.0.0.1 too, so the container shares the host's loopback. This works on Linux. On macOS, +# Docker Desktop's host networking is a separate opt-in feature and is not verified here. +# - Its only storage is its own named volume. Nothing from the host's home, OpenRig's home, +# ssh or any credential is mounted, and no token is passed in. +name: openrig-fleet-studio + +services: + studio: + build: + context: ../.. + dockerfile: rigs/fleet-studio/Dockerfile + image: openrig-fleet-studio:0.1.0 + network_mode: host + # Docker's init runs as PID 1, forwards signals and reaps the studio's child processes. + init: true + restart: unless-stopped + environment: + FLEET_DAEMON_URL: ${FLEET_DAEMON_URL:-http://127.0.0.1:7433} + volumes: + - fleet-studio-data:/data + healthcheck: + # The app's own health only: the studio AND the Fleet provider answering, asked through the + # studio's port (the provider's ready verb makes no daemon call). A daemon that is down is a + # reading the app shows, not a reason to call the app unhealthy. + test: ["CMD", "node", "/studio/apps/providers/studio-fleet-view/healthcheck.mjs"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s + +volumes: + fleet-studio-data: diff --git a/rigs/fleet-studio/rig.yaml b/rigs/fleet-studio/rig.yaml new file mode 100644 index 0000000..5df78f4 --- /dev/null +++ b/rigs/fleet-studio/rig.yaml @@ -0,0 +1,45 @@ +version: "0.2" +name: fleet-studio +summary: > + Fleet Studio: a starter operations agent for this OpenRig setup, carrying the fleet know-how + (recovering a stranded seat, reading usage honestly, keeping context healthy) and making it + available to every agent on the machine. Its Fleet page shows accounts, machines and rigs at a glance; + the Observatory is an optional addition, never required. + +services: + kind: compose + compose_file: fleet-studio.compose.yaml + project_name: openrig-fleet-studio + down_policy: down + # The rig comes up when the studio answers. Whether the Fleet provider behind it is serving is a + # runtime reading, which the container healthcheck reports and the agent acts on; waiting on it here + # would roll the whole rig back on the very failure the agent is there to handle. + wait_for: + - url: http://127.0.0.1:9450/api/contract + surfaces: + urls: + - name: Fleet + url: http://127.0.0.1:9450/surfaces/fleet.html + commands: + - name: Fleet status + command: "rig env status fleet-studio" + - name: Fleet logs + command: "rig env logs fleet-studio studio --tail 100" + +pods: + - id: ops + label: Operations + members: + - id: operator + agent_ref: "local:../agents/starter-ops" + runtime: claude-code + profile: default + cwd: "." + startup: + files: + - path: fleet-studio-startup.md + delivery_hint: send_text + required: true + edges: [] + +edges: [] diff --git a/rigs/observatory/Dockerfile b/rigs/observatory/Dockerfile index 7205be9..2b77d20 100644 --- a/rigs/observatory/Dockerfile +++ b/rigs/observatory/Dockerfile @@ -17,6 +17,7 @@ RUN git init -q /studio/sdk \ COPY apps/observatory /studio/apps/apps/observatory COPY providers/studio-observatory /studio/apps/providers/studio-observatory +COPY providers/shared /studio/apps/providers/shared # The SDK's install tool installs the APP, not its provider's dependencies; provisioning does # those separately, so this step does too. The observatory provider has no dependencies today, diff --git a/rigs/observatory/Dockerfile.dockerignore b/rigs/observatory/Dockerfile.dockerignore index 7afd1fb..269a329 100644 --- a/rigs/observatory/Dockerfile.dockerignore +++ b/rigs/observatory/Dockerfile.dockerignore @@ -2,3 +2,4 @@ * !apps/observatory/** !providers/studio-observatory/** +!providers/shared/** diff --git a/rigs/observatory/agents/observatory-operator/startup/context.md b/rigs/observatory/observatory-startup.md similarity index 75% rename from rigs/observatory/agents/observatory-operator/startup/context.md rename to rigs/observatory/observatory-startup.md index f86b995..efb671c 100644 --- a/rigs/observatory/agents/observatory-operator/startup/context.md +++ b/rigs/observatory/observatory-startup.md @@ -10,21 +10,19 @@ ## First steps -1. `rig whoami --json`: confirm who and where you are. -2. `rig env status observatory`: is the app's service up and healthy? The rig comes up as soon as the +1. `rig env status observatory`: is the app's service up and healthy? The rig comes up as soon as the studio answers, so the observatory provider may not be serving yet. Look at the `studio` line: healthy means the studio and the provider both answer (`/api/observatory/ready`); `unhealthy` with the studio still answering means "shell answers; provider not confirmed". Then: read the logs, restart `studio` once, and report. Docker does not restart an unhealthy container by itself, and you restart only once: no loop. -3. Open the hosts reading (`curl -s http://127.0.0.1:9440/api/observatory/hosts`) and tell the +2. Open the hosts reading (`curl -s http://127.0.0.1:9440/api/observatory/hosts`) and tell the person: - which machines were read - which were not, and why (unreachable, needs a token, ssh only, error) - where to open the app -4. Register one reminder to yourself to check the app every 30 minutes: - `rig watchdog register --policy periodic-reminder` with yourself as the target. On each reminder, - run step 2, check the readings' ages, and act only on the app (see your role). +3. On each periodic reminder (your general startup context shows how to register it), run step 1, + check the readings' ages, and act only on the app (see your role). ## Shaping the app @@ -36,5 +34,6 @@ enabled it for you. What you may do on this machine is what its owner has actual ## Limits to say out loud when they matter - Machines reachable only over ssh are listed but not read. -- Productivity, the oversight team's readings and subscription moves are not set up yet: only activity is shown, and nothing estimates them. Ceremony and subscriptions are shown, each with how it was read. +- Productivity, the oversight team's readings and subscription moves are not set up yet: only activity is + shown, and nothing estimates them. Ceremony and subscriptions are shown, each with how it was read. - This app reads open GET routes. It holds no token and asks for none. diff --git a/rigs/observatory/rig.yaml b/rigs/observatory/rig.yaml index b98a1ed..163d5c6 100644 --- a/rigs/observatory/rig.yaml +++ b/rigs/observatory/rig.yaml @@ -33,10 +33,15 @@ pods: label: Observatory members: - id: operator - agent_ref: "local:agents/observatory-operator" + agent_ref: "local:../agents/starter-ops" runtime: claude-code profile: default cwd: "." + startup: + files: + - path: observatory-startup.md + delivery_hint: send_text + required: true edges: [] edges: [] diff --git a/test/fleet-app.test.mjs b/test/fleet-app.test.mjs new file mode 100644 index 0000000..a92510d --- /dev/null +++ b/test/fleet-app.test.mjs @@ -0,0 +1,266 @@ +// The Fleet page: a small, read-only view of the person's machines, rigs and accounts, needing no other +// app. These pin the ways it could mislead (an unreachable machine shown as fine, a reading's age taken +// from the poll, usage that's off shown as room, a configured account shown as the one in use, a shared +// allowance added up), its independence from the Observatory, and the provider's cached-reading rule. +// Stand-in daemons on loopback with invented data. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { readFleet } from "../providers/shared/fleet-reader.mjs"; +import { createFleetCache } from "../providers/shared/reader-server.mjs"; +import { createFleetViewServer, FLEET_READS, FLEET_ROUTE, READY_ROUTE } from "../providers/studio-fleet-view/fleet-view-server.mjs"; +import { check, LABELS } from "../providers/studio-fleet-view/healthcheck.mjs"; +import { runtimeClosure, appClosure } from "./observatory-closure.test.mjs"; +import { startFakeDaemon, closedPortUrl, seat } from "./observatory-fake-daemon.mjs"; +import { loadPage } from "./observatory-page.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const root = path.resolve(here, ".."); +const pagePath = path.join(root, "apps/fleet/app/fleet.html"); +const NOW = Date.parse("2030-01-02T12:00:00.000Z"); +const now = () => NOW; +const started = []; +async function daemon(opts) { const d = await startFakeDaemon(opts); started.push(d); return d; } +test.afterEach(async () => { await Promise.all(started.splice(0).map((d) => d.close())); }); +const text = (html) => html.replace(/<[^>]+>/g, " ").replace(/&/g, "&").replace(/'/g, "'").replace(/\s+/g, " "); +const read = (url) => readFleet({ daemonUrl: url, reads: [...FLEET_READS], now }); +const fleetPage = () => loadPage(pagePath); +const render = async (fleet) => { const p = await fleetPage(); const o = p.renderPage(fleet, NOW); return { ...o, all: text(o.meta + " " + o.body) }; }; + +const usage = (observations, extra = {}) => ({ items: [{ provider: "codex", key: "acct-1", linkage: "same", notes: [], observations }], hosts: [], skipped: [], registry: { state: "read" }, seatAccounts: [], asOf: "2030-01-02T11:59:50.000Z", ...extra }); +const obs = (hostId, usedPercent, level = "ok", resetsAt = "2030-01-05T00:00:00.000Z") => ({ hostId, readAt: "2030-01-02T11:59:00.000Z", row: { identity: { planType: "team" }, observedAt: "2030-01-02T11:30:00.000Z", currentAvailability: "known", windows: [{ window: "weekly", usedPercent, resetsAt, observedAt: "2030-01-02T11:30:00.000Z", freshness: "fresh", level }] } }); + +// --- independence and boundaries ----------------------------------------------------------------- + +test("the page asks only for Fleet's own route, which the app declares and the Fleet provider serves", () => { + const html = fs.readFileSync(pagePath, "utf8"); + const app = JSON.parse(fs.readFileSync(path.join(root, "apps/fleet/app.json"), "utf8")); + const provider = JSON.parse(fs.readFileSync(path.join(root, "providers/studio-fleet-view/provider.json"), "utf8")); + const asked = [...new Set([...html.matchAll(/['"`](\/api\/[^'"`?$]+)['"`?]/g)].map((m) => m[1]))].sort(); + assert.deepEqual(asked, ["/api/fleet-view/fleet"]); + assert.deepEqual(Object.keys(app.calls), asked); + assert.equal(app.provider.package, "@openrig/studio-fleet-view"); + assert.equal(provider.package, app.provider.package); + assert.deepEqual(provider.verbs, [FLEET_ROUTE, READY_ROUTE]); + assert.equal(app.id, "fleet"); + assert.equal(app.surface.path, "/surfaces/fleet.html"); +}); + +test("Fleet depends on nothing of the Observatory's: no Observatory route, file or provider in what it runs or loads", () => { + const provider = JSON.parse(fs.readFileSync(path.join(root, "providers/studio-fleet-view/provider.json"), "utf8")); + assert.ok(provider.verbs.every((v) => !v.includes("observatory"))); + const { files, problems } = runtimeClosure(path.join(root, "providers/studio-fleet-view")); + assert.deepEqual(problems, []); + assert.ok(files.every((f) => !f.includes(`${path.sep}studio-observatory${path.sep}`)), "no Observatory provider file in Fleet's closure"); + const app = appClosure(path.join(root, "apps/fleet")); + assert.deepEqual(app.problems, []); + assert.deepEqual(app.files.map((f) => path.relative(path.join(root, "apps/fleet"), f)).sort(), ["app.json", path.join("app", "fleet.html")]); + // The page links to the Observatory for history; it does not call it. + assert.doesNotMatch(fs.readFileSync(pagePath, "utf8"), /fetch\([^)]*observatory/); +}); + +test("nothing Fleet runs or loads reads a credential, asks for a sign-in, or makes a model request", () => { + const files = [...runtimeClosure(path.join(root, "providers/studio-fleet-view")).files, ...appClosure(path.join(root, "apps/fleet")).files]; + const bad = /credentials\.json|\bauth\.json|\/v1\/messages|api\.anthropic\.com|api\.openai\.com|\boauth\b|authorization|\bbearer\b|access_?token/i; + const hits = []; + for (const f of files) fs.readFileSync(f, "utf8").split("\n").forEach((l, i) => { if (bad.test(l)) hits.push(`${path.relative(root, f)}:${i + 1}`); }); + assert.deepEqual(hits, []); + assert.ok(bad.test("fetch('https://api.anthropic.com/v1/messages')") && bad.test("read ~/.claude/.credentials.json"), "the pattern catches a planted request or credential read"); +}); + +// --- what the page says -------------------------------------------------------------------------- + +test("an unreachable machine is named as not reporting, never shown as fine or empty", async () => { + const gone = await closedPortUrl(); + const a = await daemon({ selfHostId: "box-a", hosts: [{ id: "gone", transport: "http", url: gone }] }); + const { all } = await render(await read(a.url)); + assert.match(all, /1 of 2 report here/); + assert.match(all, /Not reporting here: gone \(unreachable\)\./); + assert.doesNotMatch(all, /gone live/); +}); + +test("a seat's 'since' is when its machine sampled it, not when the page asked", async () => { + const a = await daemon({ selfHostId: "box-a", rigs: { "rig-1": { name: "alpha", nodes: [seat("dev.impl", { agentActivity: { state: "idle", reason: "no turn", evidenceSource: "pane_heuristic", sampledAt: new Date(NOW - 3 * 3600_000).toISOString() } })] } } }); + const { all } = await render(await read(a.url)); + assert.match(all, /dev\.impl idle since 3 h ago/); + assert.doesNotMatch(all, /since 0 s ago/); +}); + +test("usage that is off reads 'not connected', never zero or room", async () => { + const a = await daemon({ selfHostId: "box-a" }); + const { all } = await render(await read(a.url)); + assert.match(all, /Capacity Usage: not connected Usage readings are turned off on this machine\./); + assert.doesNotMatch(all, /\b0%|has room|No account is low or out/); +}); + +test("a shared allowance appears once, with its machines, and is never added up", async () => { + const b = await daemon({ selfHostId: "box-b" }); + const a = await daemon({ selfHostId: "box-a", hosts: [{ id: "second", transport: "http", url: b.url }], usageHosts: usage([obs("local", 40), obs("second", 40)]) }); + const { all } = await render(await read(a.url)); + assert.equal(all.match(/codex acct-1/g)?.length, 1, "one row for one account"); + assert.match(all, /box-a: weekly 40% used .* · box-b: weekly 40% used/); + assert.doesNotMatch(all, /80%/); +}); + +test("accounts are sorted out, then low, then the rest; ones without a reading fold into one named line", async () => { + const items = [ + { provider: "codex", key: "ok-acct", linkage: "single", notes: [], observations: [obs("local", 10, "ok")] }, + { provider: "claude", key: "out-acct", linkage: "single", notes: [], observations: [obs("local", 100, "exhausted", "2030-01-03T00:00:00.000Z")] }, + { provider: "codex", key: "low-acct", linkage: "single", notes: [], observations: [obs("local", 85, "nearing")] }, + { provider: "codex", key: "blind-acct", linkage: "unknown", notes: [], observations: [{ hostId: "local", readAt: "2030-01-02T11:59:00.000Z", row: { windows: [] } }] }, + ]; + const a = await daemon({ selfHostId: "box-a", usageHosts: { ...usage([]), items } }); + const { all } = await render(await read(a.url)); + const order = ["claude out-acct", "codex low-acct", "codex ok-acct"].map((k) => all.indexOf(k)); + assert.ok(order.every((i, n) => i > 0 && (n === 0 || i > order[n - 1])), `order ${order}`); + assert.match(all, /Capacity 1 out · 1 low claude out-acct: out, resets in 12 h 2 with no reading/); + // A configured account (the fixture's seat link) with no usage reading folds into the same line. + assert.match(all, /No reading yet: codex blind-acct, team plan A\./); +}); + +test("planned, default login and observed usage windows stay three separate columns; no account is ever shown as the one in use", async () => { + const a = await daemon({ + selfHostId: "box-a", + usageHosts: usage([obs("local", 10)], { seatAccounts: [{ hostId: "local", seats: [{ seatSession: "dev-impl@alpha", rigName: "alpha", configured: { source: "registry", profileRef: "team-a" }, observed: { windowsSampled: true, lastSampledAt: "2030-01-02T11:00:00.000Z", differsFrom: ["dev-review@alpha"], note: "possibly a different login: this seat's status line reports windows that differ from another seat's (a reset more than 60 s apart); not a verdict" } }, { seatSession: "dev-lead@alpha", rigName: "alpha", configured: { source: "registry", profileRef: "team-a" }, observed: { windowsSampled: true, lastSampledAt: "2030-01-02T11:30:00.000Z" } }, { seatSession: "dev-qa@alpha", rigName: "alpha", configured: { source: "default_login" } }] }] }), + }); + const p = await fleetPage(); + const fleet = await read(a.url); + const rows = p.seatAccountsRows(fleet, NOW); + const impl = rows.find((r) => r.seat === "dev-impl@alpha"); + assert.equal(impl.planned, "team plan A"); + assert.equal(impl.seen, "windows sampled 60 min ago; they differ from dev-review@alpha (possibly a different login: this seat's status line reports windows that differ from another seat's (a reset more than 60 s apart); not a verdict)"); + assert.equal(rows.find((r) => r.seat === "dev-lead@alpha").seen, "windows sampled 30 min ago", "a sample alone names no account"); + assert.equal(rows.find((r) => r.seat === "dev-qa@alpha").login, "default login"); + const { all } = await render(fleet); + assert.match(all, /Seat Planned Default login Usage windows observed/); + assert.doesNotMatch(all, /current account|currently using|a different account|the planned account/i); +}); + +test("a column with no values is cut", async () => { + const a = await daemon({ selfHostId: "box-a" }); + const { all } = await render(await read(a.url)); + assert.match(all, /Seat Planned dev-impl@alpha/); + assert.doesNotMatch(all, /Default login|Usage windows observed/); +}); + +test("the honesty rule is stated once, rows show '—' in grey, and a stale reading says so", async () => { + const html = fs.readFileSync(pagePath, "utf8"); + assert.equal(html.match(/— means not reported here yet\. Old readings show their time\. An account shared by machines is counted once\./g)?.length, 1); + const a = await daemon({ selfHostId: "box-a", rigs: { "rig-1": { name: "alpha", nodes: [seat("dev.impl", { agentActivity: undefined, assignedWorkCount: 0, inProgressWorkCount: 0, blockedWorkCount: 0 })] } } }); + const fleet = await read(a.url); + const { body, meta } = await render({ ...fleet, served: { polledAt: "2030-01-02T11:55:00.000Z", stale: true, refreshing: true } }); + assert.match(body, /—<\/span>/); + assert.doesNotMatch(text(body), /not declared|not reported(?! here:)/); + assert.match(meta, /Showing the last good reading, from 5 min ago; a fresh one is on its way\./); +}); + +test("rigs: running first by what needs the operator, a state bar with words, the rest folded", async () => { + const a = await daemon({ selfHostId: "box-a", rigs: { + "rig-1": { name: "calm", nodes: [seat("a.one")] }, + "rig-2": { name: "busy", nodes: [seat("b.one", { agentActivity: { state: "needs_input", reason: "x", sampledAt: "2030-01-02T11:59:00.000Z" } }), seat("b.two", { blockedWorkCount: 1 })] }, + "rig-3": { name: "parked", status: "stopped", nodes: [] }, + } }); + const { body, all } = await render(await read(a.url)); + assert.ok(all.indexOf("busy box-a") < all.indexOf("calm box-a")); + assert.match(all, /busy box-a · 2 seats 1 waiting · 1 blocked/); + assert.match(body, /
<\/span><\/span><\/div>/); + assert.match(all, /Not running: parked \(box-a\)\./); +}); + +test("the layout fits 390 px wide: no fixed width wider than the screen, cards that wrap", () => { + const css = fs.readFileSync(pagePath, "utf8").match(/