feat(evals): OPR.0.5.3.7 R6 — seed fixture context-library

A generator writes 13 minimal context packs at the canonical refs the eval cases select among
(the 10 case refs + 3 distractors so selection is a real choice, not a singleton), so a live run
has a real library to pull from. Output is derived (regenerate; never hand-edit under fixtures/);
production content ships via R2 packaging — this is the dev/eval mirror, and the live provider
points a spawned seat's config-resolved packs root at it (Track A).

A conformance guard verifies every fixture manifest through the daemon's OWN parseManifest (not a
second parser) and asserts the fixtures cover every ref the cases expect a seat to pull. 21 eval
tests green.
This commit is contained in:
v-openrig-build authored and mvschwarz committed 2026-08-25 18:42:21 -07:00
1 parent caa8619a77
commit 27b25a44b4
28 files changed
+242

No files matched your search

@@ -0,0 +1,55 @@
import { describe, it, expect } from "vitest";
import { readdirSync, readFileSync, existsSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseManifest } from "../src/domain/context-packs/manifest-parser.js";
import { loadEvalCasesFromDir } from "./helpers/eval-cases.js";
// slice-07 R6 — the seed fixture library must be valid packs the daemon can serve, AND must cover
// every ref the eval cases expect a seat to pull. Verifies through the daemon's OWN parseManifest
// (not a second parser) so a malformed fixture fails here, not at a live serve.
const HERE = dirname(fileURLToPath(import.meta.url));
const FIXTURES = resolve(HERE, "..", "..", "test-system", "evals", "fixtures");
const CASES_DIR = resolve(HERE, "..", "..", "test-system", "evals", "cases");
/** Every dir under fixtures/ that holds a manifest.yaml, as a ref (path relative to fixtures/). */
function fixtureRefs(): string[] {
const refs: string[] = [];
const walk = (dir: string, prefix: string) => {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const abs = join(dir, entry.name);
const ref = prefix ? `${prefix}/${entry.name}` : entry.name;
if (existsSync(join(abs, "manifest.yaml"))) refs.push(ref);
else walk(abs, ref);
}
};
walk(FIXTURES, "");
return refs;
}
describe("eval seed fixtures — valid packs covering the case refs", () => {
const refs = fixtureRefs();
it("every fixture manifest parses through the daemon parser", () => {
for (const ref of refs) {
const p = join(FIXTURES, ref, "manifest.yaml");
expect(() => parseManifest(readFileSync(p, "utf-8"), p)).not.toThrow();
}
});
it("covers every ref the cases expect a seat to pull", () => {
const { cases } = loadEvalCasesFromDir(CASES_DIR);
// Each expectedPattern is `rig context get\s+<ref>`; recover the ref tail.
const wanted = new Set<string>();
for (const c of cases) {
for (const p of c.expectedPatterns) {
const m = /rig context get\\s\+(\S+)/.exec(p);
if (m) wanted.add(m[1]!);
}
}
const have = new Set(refs);
const missing = [...wanted].filter((r) => !have.has(r));
expect(missing).toEqual([]);
});
});
@@ -0,0 +1,5 @@
# core/agent-startup-and-context-ingestion
orient a freshly-woken seat
Run `rig whoami --json` first; it is ground truth for who and where you are.
@@ -0,0 +1,6 @@
name: agent-startup-and-context-ingestion
version: "1"
purpose: "(eval fixture) orient a freshly-woken seat"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/attention-queue
see who actually holds the baton
Triage by baton, not by count; `rig queue transitions <id>` shows a park vs a strand.
@@ -0,0 +1,6 @@
name: attention-queue
version: "1"
purpose: "(eval fixture) see who actually holds the baton"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/cross-host-rig-commands
reach agents on other machines
Address a remote seat bare plus `--host <id>` (see `rig host ls`).
@@ -0,0 +1,6 @@
name: cross-host-rig-commands
version: "1"
purpose: "(eval fixture) reach agents on other machines"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/human-in-the-loop
when and how to reach the human
Reach the human by exception; orchestrators use discretion, others route through them.
@@ -0,0 +1,6 @@
name: human-in-the-loop
version: "1"
purpose: "(eval fixture) when and how to reach the human"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/openrig-upgrade
upgrade a running daemon without downtime
Use the sidecar-operator upgrade path so the rig stays up while the daemon upgrades.
@@ -0,0 +1,6 @@
name: openrig-upgrade
version: "1"
purpose: "(eval fixture) upgrade a running daemon without downtime"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/rig-bundles-and-shareable-artifacts
package a rig for a machine that never had the source
Build a portable rig with `rig bundle create`; install with `rig bundle install`.
@@ -0,0 +1,6 @@
name: rig-bundles-and-shareable-artifacts
version: "1"
purpose: "(eval fixture) package a rig for a machine that never had the source"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/rig-lifecycle
bring rigs and the whole fleet up/down/back
For a whole-box bring-back after a reboot, use `rig start` (not per-rig `rig up`).
@@ -0,0 +1,6 @@
name: rig-lifecycle
version: "1"
purpose: "(eval fixture) bring rigs and the whole fleet up/down/back"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/topology-mutation-and-seat-management
replace a seat's occupant without losing the address
Swap an occupant with `rig handover <seat>` — the seat, name, edges and queue stay put.
@@ -0,0 +1,6 @@
name: topology-mutation-and-seat-management
version: "1"
purpose: "(eval fixture) replace a seat's occupant without losing the address"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# core/watchdog
arm a wake you cannot perform yourself
You cannot wake yourself; arm `rig watchdog register --spec <yaml>` before you stop.
@@ -0,0 +1,6 @@
name: watchdog
version: "1"
purpose: "(eval fixture) arm a wake you cannot perform yourself"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# pm/requirements-writer
turn intent into requirements
Proportional structured requirements — three capture points, elastic middle.
@@ -0,0 +1,6 @@
name: requirements-writer
version: "1"
purpose: "(eval fixture) turn intent into requirements"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# process/systematic-debugging
chase a failure methodically
Instrument every boundary once and read where it actually breaks — one run beats five hypotheses.
@@ -0,0 +1,6 @@
name: systematic-debugging
version: "1"
purpose: "(eval fixture) chase a failure methodically"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# process/test-driven-development
red-first discipline
A failing test per chunk; if you cannot show it red, it is not a check.
@@ -0,0 +1,6 @@
name: test-driven-development
version: "1"
purpose: "(eval fixture) red-first discipline"
files:
- path: content.md
role: source
@@ -0,0 +1,5 @@
# process/writing-plans
write a plan before building
State the problem and intent, not the steps; separate DECIDED from OPEN.
@@ -0,0 +1,6 @@
name: writing-plans
version: "1"
purpose: "(eval fixture) write a plan before building"
files:
- path: content.md
role: source
@@ -0,0 +1,44 @@
#!/usr/bin/env node
/*
* slice-07 R6 — seed fixture context-library generator.
*
* Writes minimal context packs at the canonical refs the eval cases select among, so a LIVE eval
* run has a real library to pull from. Derived output (regenerate; never hand-edit under fixtures/).
* Real production content ships via R2 packaging; this is the dev/eval mirror. The live provider
* points a spawned seat's config-resolved packs root (OPENRIG_CONTEXT_PACKS_ROOT) at fixtures/.
*/
import { mkdirSync, writeFileSync, rmSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, "fixtures");
// ref = library path (ns/name); the pack dir sits at fixtures/<ref>/, served at that ref.
const ENTRIES = [
{ ref: "core/rig-lifecycle", purpose: "bring rigs and the whole fleet up/down/back", teaches: "For a whole-box bring-back after a reboot, use `rig start` (not per-rig `rig up`)." },
{ ref: "core/topology-mutation-and-seat-management", purpose: "replace a seat's occupant without losing the address", teaches: "Swap an occupant with `rig handover <seat>` — the seat, name, edges and queue stay put." },
{ ref: "core/agent-startup-and-context-ingestion", purpose: "orient a freshly-woken seat", teaches: "Run `rig whoami --json` first; it is ground truth for who and where you are." },
{ ref: "core/watchdog", purpose: "arm a wake you cannot perform yourself", teaches: "You cannot wake yourself; arm `rig watchdog register --spec <yaml>` before you stop." },
{ ref: "core/cross-host-rig-commands", purpose: "reach agents on other machines", teaches: "Address a remote seat bare plus `--host <id>` (see `rig host ls`)." },
{ ref: "core/rig-bundles-and-shareable-artifacts", purpose: "package a rig for a machine that never had the source", teaches: "Build a portable rig with `rig bundle create`; install with `rig bundle install`." },
{ ref: "core/openrig-upgrade", purpose: "upgrade a running daemon without downtime", teaches: "Use the sidecar-operator upgrade path so the rig stays up while the daemon upgrades." },
{ ref: "process/systematic-debugging", purpose: "chase a failure methodically", teaches: "Instrument every boundary once and read where it actually breaks — one run beats five hypotheses." },
{ ref: "core/attention-queue", purpose: "see who actually holds the baton", teaches: "Triage by baton, not by count; `rig queue transitions <id>` shows a park vs a strand." },
{ ref: "process/writing-plans", purpose: "write a plan before building", teaches: "State the problem and intent, not the steps; separate DECIDED from OPEN." },
// distractors — present so selection is a real choice, not a singleton
{ ref: "core/human-in-the-loop", purpose: "when and how to reach the human", teaches: "Reach the human by exception; orchestrators use discretion, others route through them." },
{ ref: "pm/requirements-writer", purpose: "turn intent into requirements", teaches: "Proportional structured requirements — three capture points, elastic middle." },
{ ref: "process/test-driven-development", purpose: "red-first discipline", teaches: "A failing test per chunk; if you cannot show it red, it is not a check." },
];
rmSync(ROOT, { recursive: true, force: true });
for (const e of ENTRIES) {
const dir = join(ROOT, e.ref);
mkdirSync(dir, { recursive: true });
const name = e.ref.split("/").pop();
const manifest = `name: ${name}\nversion: "1"\npurpose: "(eval fixture) ${e.purpose}"\nfiles:\n - path: content.md\n role: source\n`;
writeFileSync(join(dir, "manifest.yaml"), manifest);
writeFileSync(join(dir, "content.md"), `# ${e.ref}\n\n${e.purpose}\n\n${e.teaches}\n`);
}
console.log(`wrote ${ENTRIES.length} fixture packs under ${ROOT}`);