mirror of
https://github.com/mvschwarz/openrig.git
synced 2026-11-07 05:56:21 -10:00
feat(evals): OPR.0.5.3.7 R6 — seed fixture context-library
A generator writes 13 minimal context packs at the canonical refs the eval cases select among (the 10 case refs + 3 distractors so selection is a real choice, not a singleton), so a live run has a real library to pull from. Output is derived (regenerate; never hand-edit under fixtures/); production content ships via R2 packaging — this is the dev/eval mirror, and the live provider points a spawned seat's config-resolved packs root at it (Track A). A conformance guard verifies every fixture manifest through the daemon's OWN parseManifest (not a second parser) and asserts the fixtures cover every ref the cases expect a seat to pull. 21 eval tests green.
This commit is contained in:
1 parent
caa8619a77
commit
27b25a44b4
28 files changed
+242
No files matched your search
@@ -0,0 +1,55 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readdirSync, readFileSync, existsSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseManifest } from "../src/domain/context-packs/manifest-parser.js";
|
||||
import { loadEvalCasesFromDir } from "./helpers/eval-cases.js";
|
||||
|
||||
// slice-07 R6 — the seed fixture library must be valid packs the daemon can serve, AND must cover
|
||||
// every ref the eval cases expect a seat to pull. Verifies through the daemon's OWN parseManifest
|
||||
// (not a second parser) so a malformed fixture fails here, not at a live serve.
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURES = resolve(HERE, "..", "..", "test-system", "evals", "fixtures");
|
||||
const CASES_DIR = resolve(HERE, "..", "..", "test-system", "evals", "cases");
|
||||
|
||||
/** Every dir under fixtures/ that holds a manifest.yaml, as a ref (path relative to fixtures/). */
|
||||
function fixtureRefs(): string[] {
|
||||
const refs: string[] = [];
|
||||
const walk = (dir: string, prefix: string) => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const abs = join(dir, entry.name);
|
||||
const ref = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
if (existsSync(join(abs, "manifest.yaml"))) refs.push(ref);
|
||||
else walk(abs, ref);
|
||||
}
|
||||
};
|
||||
walk(FIXTURES, "");
|
||||
return refs;
|
||||
}
|
||||
|
||||
describe("eval seed fixtures — valid packs covering the case refs", () => {
|
||||
const refs = fixtureRefs();
|
||||
|
||||
it("every fixture manifest parses through the daemon parser", () => {
|
||||
for (const ref of refs) {
|
||||
const p = join(FIXTURES, ref, "manifest.yaml");
|
||||
expect(() => parseManifest(readFileSync(p, "utf-8"), p)).not.toThrow();
|
||||
}
|
||||
});
|
||||
|
||||
it("covers every ref the cases expect a seat to pull", () => {
|
||||
const { cases } = loadEvalCasesFromDir(CASES_DIR);
|
||||
// Each expectedPattern is `rig context get\s+<ref>`; recover the ref tail.
|
||||
const wanted = new Set<string>();
|
||||
for (const c of cases) {
|
||||
for (const p of c.expectedPatterns) {
|
||||
const m = /rig context get\\s\+(\S+)/.exec(p);
|
||||
if (m) wanted.add(m[1]!);
|
||||
}
|
||||
}
|
||||
const have = new Set(refs);
|
||||
const missing = [...wanted].filter((r) => !have.has(r));
|
||||
expect(missing).toEqual([]);
|
||||
});
|
||||
});
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
# core/agent-startup-and-context-ingestion
|
||||
|
||||
orient a freshly-woken seat
|
||||
|
||||
Run `rig whoami --json` first; it is ground truth for who and where you are.
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
name: agent-startup-and-context-ingestion
|
||||
version: "1"
|
||||
purpose: "(eval fixture) orient a freshly-woken seat"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# core/attention-queue
|
||||
|
||||
see who actually holds the baton
|
||||
|
||||
Triage by baton, not by count; `rig queue transitions <id>` shows a park vs a strand.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: attention-queue
|
||||
version: "1"
|
||||
purpose: "(eval fixture) see who actually holds the baton"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# core/cross-host-rig-commands
|
||||
|
||||
reach agents on other machines
|
||||
|
||||
Address a remote seat bare plus `--host <id>` (see `rig host ls`).
|
||||
@@ -0,0 +1,6 @@
|
||||
name: cross-host-rig-commands
|
||||
version: "1"
|
||||
purpose: "(eval fixture) reach agents on other machines"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# core/human-in-the-loop
|
||||
|
||||
when and how to reach the human
|
||||
|
||||
Reach the human by exception; orchestrators use discretion, others route through them.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: human-in-the-loop
|
||||
version: "1"
|
||||
purpose: "(eval fixture) when and how to reach the human"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# core/openrig-upgrade
|
||||
|
||||
upgrade a running daemon without downtime
|
||||
|
||||
Use the sidecar-operator upgrade path so the rig stays up while the daemon upgrades.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: openrig-upgrade
|
||||
version: "1"
|
||||
purpose: "(eval fixture) upgrade a running daemon without downtime"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
# core/rig-bundles-and-shareable-artifacts
|
||||
|
||||
package a rig for a machine that never had the source
|
||||
|
||||
Build a portable rig with `rig bundle create`; install with `rig bundle install`.
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
name: rig-bundles-and-shareable-artifacts
|
||||
version: "1"
|
||||
purpose: "(eval fixture) package a rig for a machine that never had the source"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# core/rig-lifecycle
|
||||
|
||||
bring rigs and the whole fleet up/down/back
|
||||
|
||||
For a whole-box bring-back after a reboot, use `rig start` (not per-rig `rig up`).
|
||||
@@ -0,0 +1,6 @@
|
||||
name: rig-lifecycle
|
||||
version: "1"
|
||||
purpose: "(eval fixture) bring rigs and the whole fleet up/down/back"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
# core/topology-mutation-and-seat-management
|
||||
|
||||
replace a seat's occupant without losing the address
|
||||
|
||||
Swap an occupant with `rig handover <seat>` — the seat, name, edges and queue stay put.
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
name: topology-mutation-and-seat-management
|
||||
version: "1"
|
||||
purpose: "(eval fixture) replace a seat's occupant without losing the address"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# core/watchdog
|
||||
|
||||
arm a wake you cannot perform yourself
|
||||
|
||||
You cannot wake yourself; arm `rig watchdog register --spec <yaml>` before you stop.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: watchdog
|
||||
version: "1"
|
||||
purpose: "(eval fixture) arm a wake you cannot perform yourself"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# pm/requirements-writer
|
||||
|
||||
turn intent into requirements
|
||||
|
||||
Proportional structured requirements — three capture points, elastic middle.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: requirements-writer
|
||||
version: "1"
|
||||
purpose: "(eval fixture) turn intent into requirements"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# process/systematic-debugging
|
||||
|
||||
chase a failure methodically
|
||||
|
||||
Instrument every boundary once and read where it actually breaks — one run beats five hypotheses.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: systematic-debugging
|
||||
version: "1"
|
||||
purpose: "(eval fixture) chase a failure methodically"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# process/test-driven-development
|
||||
|
||||
red-first discipline
|
||||
|
||||
A failing test per chunk; if you cannot show it red, it is not a check.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: test-driven-development
|
||||
version: "1"
|
||||
purpose: "(eval fixture) red-first discipline"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,5 @@
|
||||
# process/writing-plans
|
||||
|
||||
write a plan before building
|
||||
|
||||
State the problem and intent, not the steps; separate DECIDED from OPEN.
|
||||
@@ -0,0 +1,6 @@
|
||||
name: writing-plans
|
||||
version: "1"
|
||||
purpose: "(eval fixture) write a plan before building"
|
||||
files:
|
||||
- path: content.md
|
||||
role: source
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* slice-07 R6 — seed fixture context-library generator.
|
||||
*
|
||||
* Writes minimal context packs at the canonical refs the eval cases select among, so a LIVE eval
|
||||
* run has a real library to pull from. Derived output (regenerate; never hand-edit under fixtures/).
|
||||
* Real production content ships via R2 packaging; this is the dev/eval mirror. The live provider
|
||||
* points a spawned seat's config-resolved packs root (OPENRIG_CONTEXT_PACKS_ROOT) at fixtures/.
|
||||
*/
|
||||
import { mkdirSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(HERE, "fixtures");
|
||||
|
||||
// ref = library path (ns/name); the pack dir sits at fixtures/<ref>/, served at that ref.
|
||||
const ENTRIES = [
|
||||
{ ref: "core/rig-lifecycle", purpose: "bring rigs and the whole fleet up/down/back", teaches: "For a whole-box bring-back after a reboot, use `rig start` (not per-rig `rig up`)." },
|
||||
{ ref: "core/topology-mutation-and-seat-management", purpose: "replace a seat's occupant without losing the address", teaches: "Swap an occupant with `rig handover <seat>` — the seat, name, edges and queue stay put." },
|
||||
{ ref: "core/agent-startup-and-context-ingestion", purpose: "orient a freshly-woken seat", teaches: "Run `rig whoami --json` first; it is ground truth for who and where you are." },
|
||||
{ ref: "core/watchdog", purpose: "arm a wake you cannot perform yourself", teaches: "You cannot wake yourself; arm `rig watchdog register --spec <yaml>` before you stop." },
|
||||
{ ref: "core/cross-host-rig-commands", purpose: "reach agents on other machines", teaches: "Address a remote seat bare plus `--host <id>` (see `rig host ls`)." },
|
||||
{ ref: "core/rig-bundles-and-shareable-artifacts", purpose: "package a rig for a machine that never had the source", teaches: "Build a portable rig with `rig bundle create`; install with `rig bundle install`." },
|
||||
{ ref: "core/openrig-upgrade", purpose: "upgrade a running daemon without downtime", teaches: "Use the sidecar-operator upgrade path so the rig stays up while the daemon upgrades." },
|
||||
{ ref: "process/systematic-debugging", purpose: "chase a failure methodically", teaches: "Instrument every boundary once and read where it actually breaks — one run beats five hypotheses." },
|
||||
{ ref: "core/attention-queue", purpose: "see who actually holds the baton", teaches: "Triage by baton, not by count; `rig queue transitions <id>` shows a park vs a strand." },
|
||||
{ ref: "process/writing-plans", purpose: "write a plan before building", teaches: "State the problem and intent, not the steps; separate DECIDED from OPEN." },
|
||||
// distractors — present so selection is a real choice, not a singleton
|
||||
{ ref: "core/human-in-the-loop", purpose: "when and how to reach the human", teaches: "Reach the human by exception; orchestrators use discretion, others route through them." },
|
||||
{ ref: "pm/requirements-writer", purpose: "turn intent into requirements", teaches: "Proportional structured requirements — three capture points, elastic middle." },
|
||||
{ ref: "process/test-driven-development", purpose: "red-first discipline", teaches: "A failing test per chunk; if you cannot show it red, it is not a check." },
|
||||
];
|
||||
|
||||
rmSync(ROOT, { recursive: true, force: true });
|
||||
for (const e of ENTRIES) {
|
||||
const dir = join(ROOT, e.ref);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const name = e.ref.split("/").pop();
|
||||
const manifest = `name: ${name}\nversion: "1"\npurpose: "(eval fixture) ${e.purpose}"\nfiles:\n - path: content.md\n role: source\n`;
|
||||
writeFileSync(join(dir, "manifest.yaml"), manifest);
|
||||
writeFileSync(join(dir, "content.md"), `# ${e.ref}\n\n${e.purpose}\n\n${e.teaches}\n`);
|
||||
}
|
||||
console.log(`wrote ${ENTRIES.length} fixture packs under ${ROOT}`);
|
||||
Reference in new issue
Block a user