From 5eb4dfde135284753fe25f15baa9aebe0df82a35 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 12:12:42 -0400 Subject: [PATCH 01/83] test(audit): evals harness integration tests (#209) --- packages/opencode/test/evals-host.test.ts | 152 ++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 packages/opencode/test/evals-host.test.ts diff --git a/packages/opencode/test/evals-host.test.ts b/packages/opencode/test/evals-host.test.ts new file mode 100644 index 0000000..abaec8c --- /dev/null +++ b/packages/opencode/test/evals-host.test.ts @@ -0,0 +1,152 @@ +// The eval runner against the testkit host as a fake `opencode` (issue +// #209): a real in-process OpenCode v2 host proves the plugin's es_status +// tool runs, then the eval runner (runEvalCase) runs against a fake `opencode` +// binary whose event stream replays that real tool outcome in a loop. The +// harness — not the model — enforces both caps: the step cap kills a run that +// keeps calling tools (even at $0 spend), and the spend cap kills once +// `step_finish` cost passes the budget. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { evalWorkspace, runEvalCase } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +let h: Harness +let state: string +let statusText: string + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-evals-host-state-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# evals caps\n" }, + }) + // A real tool call on the real host: the fake `opencode` below replays this + // outcome, so its event stream is what this plugin really returns. + const probed = await h.run(`probe ${call("es_status", {})}`) + expect(probed.tools[0]?.status).toBe("completed") + statusText = probed.tools[0]?.text ?? "no status" + expect(statusText.length).toBeGreaterThan(0) +}, 120_000) + +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +/** A fake `opencode` binary: emits `steps` JSON events replaying the host's real tool outcome. */ +async function fakeOpencode(options: { steps: number; costPerStep: number }): Promise { + const dir = await mkdtemp(path.join(tmpdir(), "es-fake-opencode-")) + const file = path.join(dir, "opencode") + // The script keeps going for `steps` turns (like a model that never stops); + // the harness must kill it at its cap. Cost rides on step_finish, exactly + // as a priced model reports it (0 for an unpriced model). + const body = `#!/usr/bin/env bun +const output = ${JSON.stringify(statusText)}; +const steps = ${options.steps}; +const cost = ${options.costPerStep}; +for (let i = 0; i < steps; i++) { + console.log(JSON.stringify({ type: "step_start", part: {} })); + console.log(JSON.stringify({ type: "tool_use", part: { tool: "es_status", state: { status: "completed", output } } })); + console.log(JSON.stringify({ type: "step_finish", part: { cost, tokens: { input: 10, output: 5 } } })); +} +` + await writeFile(file, body, { mode: 0o755 }) + await chmod(file, 0o755) + return file +} + +const workspace = () => + evalWorkspace({ + pluginDir, + model: "fake/scripted", + providers: { + fake: { + package: "@opencode/ai/providers/openai-compatible", + settings: { baseURL: h.llm.url, apiKey: "test" }, + models: { scripted: { limit: { context: 100000, output: 4000 } } }, + }, + }, + }) + +describe("the eval runner against the testkit host as a fake opencode (#209)", () => { + test("the step cap kills a run that keeps calling tools", async () => { + const binary = await fakeOpencode({ steps: 6, costPerStep: 0 }) + const ws = await workspace() + try { + const result = await runEvalCase( + { id: "loop", description: "never stops", agent: "factory", prompt: "Status?" }, + ws, + { binary, maxSteps: 2, timeoutMs: 15_000 }, + ) + expect(result.capped).toBe(true) + expect(result.overBudget).toBe(false) + expect(result.pass).toBe(false) + expect(result.failures[0]).toBe("step cap reached (2)") + // The harness saw the whole loop the model wanted: steps and the real + // host tool outcome are in the transcript as evidence. + expect(result.transcript.steps).toBe(6) + expect(result.transcript.tools).toEqual([ + "es_status", + "es_status", + "es_status", + "es_status", + "es_status", + "es_status", + ]) + expect(result.transcript.completed).toEqual(result.transcript.tools) + expect(result.transcript.costUSD).toBe(0) + } finally { + await ws.cleanup() + await rm(path.dirname(binary), { recursive: true, force: true }) + } + }, 30_000) + + test("the spend cap kills once step_finish cost passes the budget", async () => { + const binary = await fakeOpencode({ steps: 3, costPerStep: 1 }) + const ws = await workspace() + try { + const result = await runEvalCase( + { id: "spend", description: "keeps spending", agent: "factory", prompt: "Status?" }, + ws, + { binary, maxSteps: 10, timeoutMs: 15_000, budgetUSD: 1.5 }, + ) + expect(result.overBudget).toBe(true) + expect(result.capped).toBe(false) + expect(result.pass).toBe(false) + expect(result.failures[0]).toStartWith("spend budget reached") + expect(result.transcript.costUSD).toBeGreaterThan(1.5) + expect(result.transcript.costUSD).toBe(3) + expect(result.transcript.steps).toBe(3) + } finally { + await ws.cleanup() + await rm(path.dirname(binary), { recursive: true, force: true }) + } + }, 30_000) + + test("a $0-priced model is still step-bounded (caps are the harness, not the model)", async () => { + const binary = await fakeOpencode({ steps: 6, costPerStep: 0 }) + const ws = await workspace() + try { + const result = await runEvalCase( + { id: "free-loop", description: "free but never stops", agent: "factory", prompt: "Status?" }, + ws, + { binary, maxSteps: 2, timeoutMs: 15_000, budgetUSD: 1.5 }, + ) + expect(result.capped).toBe(true) + expect(result.overBudget).toBe(false) + expect(result.pass).toBe(false) + expect(result.failures[0]).toBe("step cap reached (2)") + expect(result.transcript.costUSD).toBe(0) + } finally { + await ws.cleanup() + await rm(path.dirname(binary), { recursive: true, force: true }) + } + }, 30_000) +}) From 7eb73dc9ad46bc608e89b185921d6f18facd4401 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 12:12:42 -0400 Subject: [PATCH 02/83] test(audit): packaging integration tests (#208) --- packages/core/test/pack-contents.test.ts | 133 ++++++++++++++++++++ packages/opencode/test/pack-install.test.ts | 110 ++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 packages/core/test/pack-contents.test.ts create mode 100644 packages/opencode/test/pack-install.test.ts diff --git a/packages/core/test/pack-contents.test.ts b/packages/core/test/pack-contents.test.ts new file mode 100644 index 0000000..4963625 --- /dev/null +++ b/packages/core/test/pack-contents.test.ts @@ -0,0 +1,133 @@ +// Packed-tarball contents for the release set (issue #208): what users +// install must carry the runtime (core assets, the CLI bin, the plugin +// server plus the prebuilt TUI) and nothing else — no tests, no private +// packages (`es-fleet`, web, testkit). Packs with `bun pm pack` the way +// scripts/pack-smoke.sh packs, from the single package list +// (scripts/packages.ts, as in packages.test.ts). +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readdir, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { releasePackages, tarballSlug, workspacePackages } from "../../../scripts/packages.ts" + +const root = path.resolve(import.meta.dir, "../../..") + +async function run(argv: string[], cwd: string): Promise { + const proc = Bun.spawn(argv, { cwd, stdout: "pipe", stderr: "pipe" }) + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + if (code !== 0) throw new Error(`${argv.join(" ")} in ${cwd} exited ${code}: ${stderr.slice(-2000)}`) + return stdout +} + +async function tarList(tgz: string): Promise { + const out = await run(["tar", "-tzf", tgz], root) + return out.split("\n").filter(Boolean) +} + +async function tarRead(tgz: string, member: string): Promise { + return run(["tar", "-xzOf", tgz, member], root) +} + +const TEST_DIR = new Set(["test", "tests", "__tests__", "__snapshots__"]) +const isTestPath = (entry: string) => { + const parts = entry.split("/") + return parts.some((part) => TEST_DIR.has(part)) || /\.(test|spec)\.[a-z]+$/.test(parts.at(-1) ?? "") +} + +describe("packed release tarballs (issue #208)", () => { + let out = "" + const tarballs = new Map() + + beforeAll(async () => { + out = await mkdtemp(path.join(tmpdir(), "es-pack-contents-")) + const release = await releasePackages(root) + for (const { dir } of release) await run(["bun", "run", "build"], path.join(root, "packages", dir)) + for (const { dir } of release) + await run(["bun", "pm", "pack", "--destination", out], path.join(root, "packages", dir)) + const files = await readdir(out) + for (const { dir, name } of release) { + const hits = files.filter((file) => file.startsWith(`${tarballSlug(name)}-`) && file.endsWith(".tgz")) + expect(hits).toHaveLength(1) + tarballs.set(dir, path.join(out, hits[0]!)) + } + }, 240_000) + afterAll(() => rm(out, { recursive: true, force: true })) + + test("the release set is exactly core, cli and the plugin; fleet, web and testkit stay private", async () => { + expect([...tarballs.keys()].sort()).toEqual(["cli", "core", "opencode"]) + const all = await workspacePackages(root) + for (const dir of ["fleet", "web", "testkit"]) { + const pkg = all.find((entry) => entry.dir === dir)! + expect([dir, pkg.private]).toEqual([dir, true]) + expect(pkg.order).toBeUndefined() + } + // No tarball for a private package, even a stale one from an earlier run. + for (const pkg of all.filter((entry) => entry.private)) { + const hits = (await readdir(out)).filter( + (file) => file.startsWith(`${tarballSlug(pkg.name)}-`) && file.endsWith(".tgz"), + ) + expect([pkg.dir, hits]).toEqual([pkg.dir, []]) + } + }) + + test("packed manifests carry no workspace: protocol", async () => { + for (const [dir, tgz] of tarballs) { + const manifest = await tarRead(tgz, "package/package.json") + expect([dir, manifest.includes("workspace:")]).toEqual([dir, false]) + } + }) + + test("tarballs contain no tests and no private-package payload", async () => { + for (const [dir, tgz] of tarballs) { + const entries = await tarList(tgz) + expect([dir, entries.filter(isTestPath)]).toEqual([dir, []]) + expect([dir, entries.filter((entry) => entry.includes("es-fleet") || entry.includes("es-web"))]).toEqual([ + dir, + [], + ]) + } + }) + + test("tarballs ship the runtime payload users need", async () => { + const has = async (dir: string, member: string) => + expect([dir, member, (await tarList(tarballs.get(dir)!)).includes(member)]).toEqual([dir, member, true]) + // Core: compiled entry plus the bundled assets pack-smoke loads + // (prompts, skills, licence templates). + await has("core", "package/dist/index.js") + await has("core", "package/assets/prompts/factory.md") + await has("core", "package/assets/skills/factory/SKILL.md") + await has("core", "package/assets/licenses/MIT.template.txt") + // CLI: the bin users run plus its bundle. + await has("cli", "package/bin/es.js") + await has("cli", "package/dist/es.js") + // Plugin: the server entry the host loads plus the prebuilt TUI (raw + // .tsx cannot ship: the host Solid transform skips node_modules). + await has("opencode", "package/server.ts") + await has("opencode", "package/src/server.ts") + await has("opencode", "package/dist/tui.js") + }) + + test("the opencode host pin matches what the suites test", async () => { + const manifest = async (dir: string) => + JSON.parse(await Bun.file(path.join(root, "packages", dir, "package.json")).text()) as { + dependencies?: Record + devDependencies?: Record + } + const plugin = await manifest("opencode") + const kit = await manifest("testkit") + // Pinned, not a range: the host the plugin runs against is exactly the + // host the testkit boots. + expect(plugin.dependencies?.["@opencode/plugin"]).toBe("2.0.24") + expect(plugin.devDependencies?.["@opencode/sdk"]).toBe("2.0.24") + expect(kit.dependencies?.["@opencode/sdk"]).toBe(plugin.devDependencies?.["@opencode/sdk"]) + // The packed plugin pins the same host dependency (workspace:* rewritten). + const packed = JSON.parse(await tarRead(tarballs.get("opencode")!, "package/package.json")) as { + dependencies?: Record + } + expect(packed.dependencies?.["@opencode/plugin"]).toBe(plugin.dependencies?.["@opencode/plugin"]) + }) +}) diff --git a/packages/opencode/test/pack-install.test.ts b/packages/opencode/test/pack-install.test.ts new file mode 100644 index 0000000..19033a5 --- /dev/null +++ b/packages/opencode/test/pack-install.test.ts @@ -0,0 +1,110 @@ +// The packed plugin installed the way users get it, loaded on the real +// in-process host from the installed (tarball) path — never the workspace +// checkout (issue #208). Packs with `bun pm pack` like scripts/pack-smoke.sh, +// installs the tarballs into a temp project with npm, then boots the testkit +// host against the installed plugin directory. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { boot, directiveScript, type Harness, lastAgentRequest } from "@heretek-ai/es-testkit" +import { releasePackages, tarballSlug } from "../../../scripts/packages.ts" +import { EsRpc } from "../src/rpc-def.ts" + +const root = path.resolve(import.meta.dir, "../../..") +const workspacePluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +async function run(argv: string[], cwd: string): Promise { + const proc = Bun.spawn(argv, { cwd, stdout: "pipe", stderr: "pipe" }) + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + if (code !== 0) throw new Error(`${argv.join(" ")} in ${cwd} exited ${code}: ${stderr.slice(-2000)}`) + return stdout +} + +describe("packed plugin install on the real host (issue #208)", () => { + let out = "" + let work = "" + let state = "" + let installed = "" + + beforeAll(async () => { + out = await mkdtemp(path.join(tmpdir(), "es-pack-install-packs-")) + work = await mkdtemp(path.join(tmpdir(), "es-pack-install-work-")) + state = await mkdtemp(path.join(tmpdir(), "es-pack-install-state-")) + // The release packages and their tarball slugs, in publish order, from + // the single source of truth — the same list scripts/pack-smoke.sh uses. + const release = await releasePackages(root) + expect(release.map((pkg) => pkg.dir)).toEqual(["core", "cli", "opencode"]) + for (const { dir } of release) await run(["bun", "run", "build"], path.join(root, "packages", dir)) + for (const { dir, name } of release) { + const slug = tarballSlug(name) + for (const stale of (await readdir(out)).filter((file) => file.startsWith(`${slug}-`) && file.endsWith(".tgz"))) + await rm(path.join(out, stale)) + await run(["bun", "pm", "pack", "--destination", out], path.join(root, "packages", dir)) + } + const packed = await readdir(out) + const tarballs = release.map(({ name }) => { + const hits = packed.filter((file) => file.startsWith(`${tarballSlug(name)}-`) && file.endsWith(".tgz")) + expect(hits).toHaveLength(1) + return path.join(out, hits[0]!) + }) + // A temp project the way users get it: the packed tarballs, installed + // with npm (workspace:* already rewritten to real versions by the pack). + await writeFile( + path.join(work, "package.json"), + JSON.stringify({ name: "es-pack-probe", private: true, version: "0.0.0" }), + ) + await run(["npm", "install", "--ignore-scripts", "--no-audit", "--no-fund", ...tarballs], work) + installed = path.join(work, "node_modules", "@heretek-ai", "epistemic-swarm") + const manifest = JSON.parse(await readFile(path.join(installed, "package.json"), "utf8")) as { + name: string + dependencies?: Record + } + expect(manifest.name).toBe("@heretek-ai/epistemic-swarm") + // The host must load this directory, not the workspace checkout. + expect(await realpath(installed)).not.toBe(await realpath(workspacePluginDir)) + expect(path.relative(await realpath(root), await realpath(installed))).toMatch(/^\.\./) + }, 300_000) + afterAll(async () => { + await rm(out, { recursive: true, force: true }) + await rm(work, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + }) + + test("installed manifests carry no workspace: protocol", async () => { + const vendor = path.join(work, "node_modules", "@heretek-ai") + for (const dir of await readdir(vendor)) { + const manifest = await readFile(path.join(vendor, dir, "package.json"), "utf8") + expect([dir, manifest.includes('"workspace:')]).toEqual([dir, false]) + } + }) + + test("the tarball-installed plugin boots on the real host and serves es_status", async () => { + expect(path.resolve(installed)).toContain("node_modules") + let h: Harness | undefined + try { + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: installed, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# demo\n", ".gitignore": "node_modules/\n" }, + }) + const { tools } = await h.run(`go ${call("es_status")}`) + expect(tools[0]?.status).toBe("completed") + expect(tools[0]?.text).toContain("") + const advertised = (lastAgentRequest(h.llm.requests)?.tools ?? []).map((tool) => tool.function.name) + expect(advertised).toContain("es_status") + expect(advertised).toContain("es_gates_run") + const rpc = (h.opencode as any).rpc(EsRpc) + const factory = await rpc.factoryState({}, { location: h.location }) + expect(factory.stage).toBe("NONE") + } finally { + await h?.close() + } + }, 180_000) +}) From 43b1f7093f28473ba83297b9d021cc4b7eb715b5 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 12:50:12 -0400 Subject: [PATCH 03/83] test(audit): human-only approvals integration tests (#180) --- packages/core/test/approval-180.test.ts | 133 ++++++++++++ packages/opencode/test/approval-180.test.ts | 207 +++++++++++++++++++ packages/opencode/test/render/masked-180.tsx | 35 ++++ 3 files changed, 375 insertions(+) create mode 100644 packages/core/test/approval-180.test.ts create mode 100644 packages/opencode/test/approval-180.test.ts create mode 100644 packages/opencode/test/render/masked-180.tsx diff --git a/packages/core/test/approval-180.test.ts b/packages/core/test/approval-180.test.ts new file mode 100644 index 0000000..7d16fc5 --- /dev/null +++ b/packages/core/test/approval-180.test.ts @@ -0,0 +1,133 @@ +// Issue #180 integration tests (core surface): approvals are hash-bound to +// the approved artifacts and every channel is recorded and verifies. Uses +// only existing seams (test keystores, temp dirs); no host, no human seal. +import { describe, expect, test } from "bun:test" +import { readFile, writeFile } from "node:fs/promises" +import { type HumanSigner, sealHumanKey, unlockHumanKey, verifyRecordSignature } from "../src/approval/keystore.ts" +import { readApproval, verifyApproval } from "../src/approval/record.ts" +import { approveStage } from "../src/approval/service.ts" +import { Factory, pendingApprovals } from "../src/factory/index.ts" +import { factoryLayout } from "../src/layout.ts" +import { writeJson } from "../src/util/fs.ts" +import { type Fixture, frontier, gitRepo, writeSpecs } from "./helpers.ts" + +const PASSPHRASE = "test-passphrase-1234" + +async function setup(): Promise<{ fx: Fixture; signer: HumanSigner; factory: Factory }> { + const fx = await gitRepo() + await sealHumanKey(PASSPHRASE, fx.state) + const signer = await unlockHumanKey(PASSPHRASE, fx.state) + const factory = new Factory(fx.root, { + gates: async () => ({ passed: true, findings: [], summary: "green" }), + stateDir: fx.state, + }) + await factory.writeFrontier("grill", frontier()) + return { fx, signer, factory } +} + +const seedPending = (fx: Fixture, stages: Array<"frontier" | "spec">) => + writeJson( + factoryLayout(fx.root).pending, + stages.map((stage) => ({ stage, requestedBy: "factory", at: new Date().toISOString() })), + ) + +describe("issue #180: approvals are hash-bound", () => { + test("editing frontier.json after approval invalidates the record", async () => { + const { fx, signer } = await setup() + try { + await seedPending(fx, ["frontier"]) + const { record } = await approveStage(fx.root, { + stage: "frontier", + channel: "tui", + approvedBy: "tester", + signer, + stateDir: fx.state, + }) + expect(record.channel).toBe("tui") + expect((await verifyApproval(fx.root, "frontier", { stateDir: fx.state })).ok).toBe(true) + const file = factoryLayout(fx.root).frontier + await writeFile(file, `${await readFile(file, "utf8")}\n\n`) + const check = await verifyApproval(fx.root, "frontier", { stateDir: fx.state }) + expect(check.ok).toBe(false) + if (!check.ok) expect(check.reason).toMatch(/changed after it was approved/) + expect(await readApproval(fx.root, "frontier")).toBeDefined() + expect(await pendingApprovals(fx.root)).toEqual([]) + } finally { + signer.destroy() + await fx.cleanup() + } + }) + + test("editing a GOAL.md after spec approval invalidates it", async () => { + const { fx, signer } = await setup() + try { + await writeSpecs(fx.root, ["alpha"]) + await seedPending(fx, ["spec"]) + await approveStage(fx.root, { + stage: "spec", + channel: "web", + approvedBy: "tester", + signer, + stateDir: fx.state, + }) + expect((await verifyApproval(fx.root, "spec", { stateDir: fx.state })).ok).toBe(true) + const goal = factoryLayout(fx.root).spec("alpha") + await writeFile(goal, `${await readFile(goal, "utf8")}\n\n`) + expect((await verifyApproval(fx.root, "spec", { stateDir: fx.state })).ok).toBe(false) + } finally { + signer.destroy() + await fx.cleanup() + } + }) +}) + +describe("issue #180: every channel is recorded and verifies", () => { + test("cli, tui and web records carry their channel and a valid signature", async () => { + for (const channel of ["cli", "tui", "web"] as const) { + const { fx, signer } = await setup() + try { + await seedPending(fx, ["frontier"]) + const { record } = await approveStage(fx.root, { + stage: "frontier", + channel, + approvedBy: "tester", + signer, + stateDir: fx.state, + }) + expect(record.channel).toBe(channel) + expect(record.signature.alg).toBe("ed25519") + expect(await verifyRecordSignature(record, fx.state)).toBe(true) + expect(await verifyApproval(fx.root, "frontier", { stateDir: fx.state })).toMatchObject({ ok: true }) + } finally { + signer.destroy() + await fx.cleanup() + } + } + }) + + test("a record with a forged signature is refused", async () => { + const { fx, signer } = await setup() + try { + await seedPending(fx, ["frontier"]) + const { record } = await approveStage(fx.root, { + stage: "frontier", + channel: "cli", + approvedBy: "tester", + signer, + stateDir: fx.state, + }) + expect((await verifyApproval(fx.root, "frontier", { stateDir: fx.state })).ok).toBe(true) + const file = factoryLayout(fx.root).approval("frontier") + await writeJson(file, { + ...record, + signature: { ...record.signature, sig: `${record.signature.sig.slice(0, -2)}AA` }, + }) + const check = await verifyApproval(fx.root, "frontier", { stateDir: fx.state }) + expect(check.ok).toBe(false) + if (!check.ok) expect(check.reason).toMatch(/not signed by this machine's human key/) + } finally { + signer.destroy() + await fx.cleanup() + } + }) +}) diff --git a/packages/opencode/test/approval-180.test.ts b/packages/opencode/test/approval-180.test.ts new file mode 100644 index 0000000..95d1c1f --- /dev/null +++ b/packages/opencode/test/approval-180.test.ts @@ -0,0 +1,207 @@ +// Issue #180 integration tests (opencode surface): every agent-reachable +// approval path is refused on the real v2 host, while the human TUI path +// signs in-process with channel "tui" and five wrong passphrases lock out. +// Real in-process host via @heretek-ai/es-testkit (scripted fake LLM); +// must pass on pinned 2.0.24 and under scripts/v2-head.sh. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + checkApprovalAttempts, + Factory, + factoryLayout, + gateRunner, + pendingApprovals, + readApproval, + recentAuditEntries, + sealHumanKey, + verifyApproval, + verifyRecordSignature, + writeJson, +} from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { approveInTui } from "../src/approve-tui.ts" +import { EsRpc } from "../src/rpc-def.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const PASSPHRASE = "test-passphrase-1234" + +let h: Harness +let state: string + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-180-state-")) + await sealHumanKey(PASSPHRASE, state) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# approval 180\n" }, + }) + const factory = new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.writeFrontier("grill", { + version: "1.1", + idea: "approval 180 probe", + spendCeiling: { currency: "USD", maxAmount: 5 }, + settled: true, + nodes: [{ id: "a", question: "q?", answer: "yes", status: "settled" }], + }) + await writeJson(factoryLayout(h.directory).pending, [ + { stage: "frontier", requestedBy: "factory", at: new Date().toISOString() }, + ]) +}, 120_000) + +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +describe("issue #180: no agent path grants an approval on the real host", () => { + test("tool call, shell approval verbs and direct writes are all refused and record nothing", async () => { + const target = path.join(h.directory, ".factory/approvals/frontier.json") + const attempts = [ + call("es_approve", { stage: "frontier" }), + call("shell", { command: "es approve frontier" }), + call("shell", { command: "es approve spec" }), + call("write", { path: ".factory/approvals/frontier.json", content: "{}" }), + call("write", { path: target, content: "{}" }), + call("patch", { + patchText: "*** Begin Patch\n*** Add File: .factory/approvals/frontier.json\n+{}\n*** End Patch", + }), + call("shell", { command: "echo {} > .factory/approvals/frontier.json" }), + ] + const { tools } = await h.run(`approve by any path ${attempts.join(" ")}`) + expect(tools).toHaveLength(attempts.length) + for (const outcome of tools) expect(`${outcome.name}:${outcome.status}`).toBe(`${outcome.name}:error`) + // The human-only shell verb says so; the control-file write says so. + expect(tools[1]?.text).toMatch(/human-only/i) + expect(tools[3]?.text).toMatch(/control file/i) + // No approval tool is even advertised to the model. + const advertised = h.llm.requests.flatMap((request) => (request.tools ?? []).map((tool) => tool.function.name)) + expect(advertised).not.toContain("es_approve") + // Nothing recorded: no record, no file, pending still waits for the human. + expect(await readApproval(h.directory, "frontier")).toBeUndefined() + expect(await Bun.file(target).exists()).toBe(false) + expect((await pendingApprovals(h.directory)).map((item) => item.stage)).toContain("frontier") + }, 120_000) + + test("no approve/trust/resume RPC exists; preview is read-only with a hash-bound ticket", async () => { + const rpc = (h.opencode as any).rpc(EsRpc) as any + for (const method of ["approve", "trust", "resume"]) expect(typeof rpc[method]).not.toBe("function") + const preview = await rpc.previewApproval({ stage: "frontier" }, { location: h.location }) + expect(preview.ok).toBe(true) + expect(preview.subjectHash).toMatch(/^[0-9a-f]{64}$/) + expect(typeof preview.token).toBe("string") + // The validate-only redeem signs nothing: a bad token refuses. + await expect( + rpc.redeemApproval( + { stage: "frontier", token: "no-such-token", subjectHash: preview.subjectHash }, + { location: h.location }, + ), + ).rejects.toThrow() + expect(await readApproval(h.directory, "frontier")).toBeUndefined() + }, 60_000) +}) + +describe("issue #180: the TUI path signs in-process with channel tui", () => { + test("preview over RPC + scripted masked dialog + redeem over RPC records a valid signed approval", async () => { + const rpc = (h.opencode as any).rpc(EsRpc) as any + const where = { location: h.location } + const preview = await rpc.previewApproval({ stage: "frontier" }, where) + expect(preview.ok).toBe(true) + const outcome = await approveInTui({ + root: h.directory, + stage: "frontier", + expectedSubjectHash: preview.subjectHash, + ...(preview.token ? { previewToken: preview.token as string } : {}), + ...(preview.issuedAt !== undefined ? { previewIssuedAt: preview.issuedAt as number } : {}), + redeemPreview: async (token) => { + await rpc.redeemApproval({ stage: "frontier", token, subjectHash: preview.subjectHash }, where) + }, + stateDir: state, + readPassphrase: async () => [...PASSPHRASE], + }) + expect(outcome.ok).toBe(true) + if (!outcome.ok) return + expect(outcome.record).toMatchObject({ stage: "frontier", channel: "tui" }) + expect(outcome.record.signature.alg).toBe("ed25519") + const stored = await readApproval(h.directory, "frontier") + expect(stored?.channel).toBe("tui") + expect(stored?.signature.sig).toBe(outcome.record.signature.sig) + expect(await verifyRecordSignature(outcome.record, state)).toBe(true) + expect(await verifyApproval(h.directory, "frontier", { stateDir: state })).toMatchObject({ ok: true }) + expect(await pendingApprovals(h.directory)).toEqual([]) + const factory = new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + expect((await factory.read())?.stage).toBe("RESEARCH") + expect((await recentAuditEntries(h.directory, 10)).map((entry) => entry.action)).toContain("approval.frontier") + // The passphrase reaches no persisted surface. + const attempts = await readFile(path.join(state, "approval-attempts.json"), "utf8").catch(() => "[]") + expect(attempts).not.toContain(PASSPHRASE.slice(4)) + expect(JSON.stringify(outcome.record)).not.toContain(PASSPHRASE.slice(4)) + }, 120_000) +}) + +describe("issue #180: five wrong passphrases then a correct one locks out", () => { + test("the sixth attempt never prompts; the lockout is audited with channel tui", async () => { + const before = await readApproval(h.directory, "frontier") + let reads = 0 + const locked = await approveInTui({ + root: h.directory, + stage: "frontier", + expectedSubjectHash: "0".repeat(64), + stateDir: state, + readPassphrase: async () => { + reads++ + return [..."wrong-passphrase-000"] + }, + }) + expect(reads).toBe(5) + expect(locked).toMatchObject({ ok: false, locked: true }) + expect((await checkApprovalAttempts(state)).allowed).toBe(false) + // Even the correct passphrase is refused while locked: the dialog never opens. + let correctReads = 0 + const stillLocked = await approveInTui({ + root: h.directory, + stage: "frontier", + expectedSubjectHash: "0".repeat(64), + stateDir: state, + readPassphrase: async () => { + correctReads++ + return [...PASSPHRASE] + }, + }) + expect(correctReads).toBe(0) + expect(stillLocked).toMatchObject({ ok: false, locked: true }) + const lockout = (await recentAuditEntries(h.directory, 10)).find((entry) => entry.action === "approval.lockout") + expect(lockout?.payload).toMatchObject({ stage: "frontier", channel: "tui" }) + // The lockout recorded nothing new: the frontier record is unchanged, spec never approved. + expect(await readApproval(h.directory, "frontier")).toEqual(before) + expect(await readApproval(h.directory, "spec")).toBeUndefined() + }, 120_000) +}) + +describe("issue #180: the masked dialog shows no passphrase (bug #222 context)", () => { + test("production MaskedInput frames contain bullets but never the probe", async () => { + const proc = Bun.spawn( + ["bun", "--conditions=browser", "--preload", "@opentui/solid/preload", "test/render/masked-180.tsx"], + { cwd: pluginDir, stdout: "pipe", stderr: "pipe" }, + ) + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect({ code, stderr: code === 0 ? "" : stderr }).toEqual({ code: 0, stderr: "" }) + const result = JSON.parse(stdout.trim().split("\n").at(-1)!) as { + frames: string[] + delivered: string + probe: string + } + expect(result.delivered).toBe(result.probe) + expect(result.frames.length).toBeGreaterThan(0) + for (const frame of result.frames) expect(frame).not.toContain(result.probe) + expect(result.frames.some((frame) => frame.includes("•"))).toBe(true) + }, 60_000) +}) diff --git a/packages/opencode/test/render/masked-180.tsx b/packages/opencode/test/render/masked-180.tsx new file mode 100644 index 0000000..1dade50 --- /dev/null +++ b/packages/opencode/test/render/masked-180.tsx @@ -0,0 +1,35 @@ +// Renders the production MaskedInput headlessly with OpenTUI's test +// renderer (run by approval-180.test.ts in a subprocess with the Solid +// transform preloaded, the same harness as test/render/panels.tsx) and +// prints the captured frames as JSON. No new harness. +import { testRender } from "@opentui/solid" +import { MaskedInput } from "../../src/masked-input.tsx" + +const probe = "probe-180-mask-must-never-render-xyz" +const frames: string[] = [] +let delivered = "" + +const setup = await testRender( + () => ( + { + delivered = codepoints.join("") + }} + onCancel={() => {}} + /> + ), + { width: 80, height: 10 }, +) +await setup.flush() +frames.push(setup.captureCharFrame()) +await setup.mockInput.typeText(probe) +await setup.flush() +frames.push(setup.captureCharFrame()) +setup.mockInput.pressEnter() +await setup.flush() +frames.push(setup.captureCharFrame()) +setup.renderer.destroy() + +process.stdout.write(`${JSON.stringify({ frames, delivered, probe })}\n`) +process.exit(0) From 3f1677c36b6c4039274043835fed8d6e04c3cd7a Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 12:50:12 -0400 Subject: [PATCH 04/83] test(audit): trust pinning integration tests (#181) --- packages/cli/test/trust-pinning.test.ts | 131 +++++++++++++ packages/core/test/trust-pinning.test.ts | 237 +++++++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 packages/cli/test/trust-pinning.test.ts create mode 100644 packages/core/test/trust-pinning.test.ts diff --git a/packages/cli/test/trust-pinning.test.ts b/packages/cli/test/trust-pinning.test.ts new file mode 100644 index 0000000..ed9fe06 --- /dev/null +++ b/packages/cli/test/trust-pinning.test.ts @@ -0,0 +1,131 @@ +// Trust pinning verbs stay terminal-only (#181 §2, #221 per-verb pipe gaps). +// Pipe (non-interactive) runs of every human-gated trust verb refuse with code +// 3 and record nothing; the shell policy denies the same verbs for every +// agent. No test here runs a real human verb: the only confirms are pipes. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + evaluateShell, + Factory, + factoryLayout, + gateRunner, + invokesHumanOnly, + sealHumanKey, + verifyControl, +} from "@heretek-ai/es-core" +import { main } from "../src/main.ts" +import type { ConfirmIO } from "../src/tty.ts" + +const PASSPHRASE = "test-passphrase-181" + +let root: string +let state: string +const realXdgConfig = process.env.XDG_CONFIG_HOME +const git = async (...args: string[]) => { + const proc = Bun.spawn(["git", ...args], { cwd: root, stdout: "pipe", stderr: "pipe" }) + await proc.exited +} + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-trust-pinning-")) + state = await mkdtemp(path.join(tmpdir(), "es-trust-pinning-state-")) + process.env.XDG_CONFIG_HOME = path.join(state, "xdg-config") + await git("init", "-q", "-b", "main") + await writeFile(path.join(root, "README.md"), "# x\n") + await git("add", "-A") + await git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "init") + await sealHumanKey(PASSPHRASE, state) +}) +afterEach(async () => { + if (realXdgConfig === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = realXdgConfig + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const pipe: ConfirmIO = { interactive: false, write: () => {}, readLine: async () => "y", readSecret: async () => "" } +const run = async (argv: string[], confirm: ConfirmIO = pipe) => { + const printed: string[] = [] + const code = await main(argv, { print: (text) => printed.push(text), confirm, cwd: root, stateDir: state }) + return { code, out: printed.join("\n") } +} +const trustFile = () => path.join(state, "trust.json") +const waiverDir = () => factoryLayout(root).waivers + +describe("trust verbs refuse a pipe (#221)", () => { + test("es trust needs a terminal and trusts nothing", async () => { + await mkdir(factoryLayout(root).dir, { recursive: true }) + await writeFile( + factoryLayout(root).gates, + JSON.stringify({ + version: 1, + commands: [{ id: "probe", kind: "test", command: "touch RAN" }], + security: { gitleaks: "off", osv: "off" }, + }), + ) + const result = await run(["trust"]) + expect(result.code).toBe(3) + expect(result.out).toContain("interactive terminal") + expect(await Bun.file(trustFile()).exists()).toBe(false) + }) + + test("es waive needs a terminal and records nothing", async () => { + const result = await run(["waive", "security/*", "--reason", "legacy code under migration"]) + expect(result.code).toBe(3) + expect(result.out).toContain("interactive terminal") + expect(await Bun.file(path.join(waiverDir(), "security-abc.json")).exists()).toBe(false) + const { readdir } = await import("node:fs/promises") + expect(await readdir(waiverDir()).catch(() => [])).toEqual([]) + }) + + test("es rebaseline needs a terminal and keeps the drift", async () => { + const { rebaseline } = await import("@heretek-ai/es-core") + await rebaseline(root, "test") + await mkdir(factoryLayout(root).dir, { recursive: true }) + await writeFile(factoryLayout(root).config, '{"pr":"off"}\n') + expect((await verifyControl(root)).clean).toBe(false) + const result = await run(["rebaseline"]) + expect(result.code).toBe(3) + expect(result.out).toContain("interactive terminal") + expect((await verifyControl(root)).clean).toBe(false) + }) + + test("es reseal --sign needs a terminal and keeps the mismatch", async () => { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.begin("human:tester") + const stateFile = factoryLayout(root).state + await writeFile(stateFile, `${await Bun.file(stateFile).text()} `) + const result = await run(["reseal", "--sign"]) + expect(result.code).toBe(3) + expect(result.out).toContain("interactive terminal") + await expect(factory.read()).rejects.toThrow("signature mismatch") + }) +}) + +describe("agent shells cannot reach human-gated trust verbs", () => { + test("the policy denies trust, waive, rebaseline and reseal for every agent", () => { + for (const command of [ + "es trust", + "es waive lint/check --reason legacy", + "es rebaseline", + "es reseal --sign", + "es approve frontier", + ]) { + expect([command, invokesHumanOnly(command)]).toEqual([command, true]) + for (const agent of [undefined, "build", "factory", "es-programmer"]) { + const decision = evaluateShell({ root, stateDir: state }, agent, command, { sandboxAvailable: true }) + expect([command, agent, decision.effect]).toEqual([command, agent, "deny"]) + if (decision.effect === "deny") expect(decision.reason).toContain("human-only") + } + } + }) + + test("reads keep working: help and status stay allowed", () => { + expect(invokesHumanOnly("es trust --help")).toBe(false) + expect(evaluateShell({ root, stateDir: state }, "build", "es status", { sandboxAvailable: true }).effect).toBe( + "allow", + ) + }) +}) diff --git a/packages/core/test/trust-pinning.test.ts b/packages/core/test/trust-pinning.test.ts new file mode 100644 index 0000000..9f70958 --- /dev/null +++ b/packages/core/test/trust-pinning.test.ts @@ -0,0 +1,237 @@ +// Trust pinning, waivers, rebaseline and reseal: integration against the real +// OpenCode v2 host (issue #181 §2) plus the proof-gap closers from #221 and the +// waiver-file drift case. Real in-process host via packages/testkit (scripted +// fake LLM); must also pass under scripts/v2-head.sh. +// +// Convention: the test harness acts as the human/system out of band (fs writes, +// rebaseline, trustProject, signEngineFile). The scripted agent only ever runs +// the denied or read-only path; no test runs a real human verb. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { boot, directiveScript, type Harness } from "../../testkit/src/index.ts" +import { + canonicalPath, + commandSetHash, + Factory, + factoryLayout, + GatesConfigSchema, + gateRunner, + type HumanSigner, + isTrusted, + rebaseline, + runGates, + sealHumanKey, + signEngineFile, + trustProject, + unlockHumanKey, + verifyControl, +} from "../src/index.ts" + +const PASSPHRASE = "test-passphrase-181" +const FAKE_TOKEN = `ghp_${"Ab3dE6gH9jK2mN5pQ8sT1vW4yZ7bC0eF3hJ6"}` +const pluginDir = path.resolve(import.meta.dir, "../../opencode") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const QUIET_SECURITY = { security: { gitleaks: "off", osv: "off" } } as const +const baselineGates = JSON.stringify({ version: 1, commands: [], ...QUIET_SECURITY }) + +let h: Harness +let state: string +let signer: HumanSigner + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-trust-pinning-")) + await sealHumanKey(PASSPHRASE, state) + signer = await unlockHumanKey(PASSPHRASE, state) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# trust pinning\n" }, + }) + await mkdir(path.join(h.directory, ".factory"), { recursive: true }) + await writeFile(path.join(h.directory, ".factory/gates.json"), `${baselineGates}\n`) + await rebaseline(h.directory, "test") +}, 120_000) +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +const writeRoot = async (file: string, text: string) => { + await mkdir(path.dirname(path.join(h.directory, file)), { recursive: true }) + await writeFile(path.join(h.directory, file), text) +} +const gatesFile = () => path.join(h.directory, ".factory/gates.json") + +describe("control writes through the host (#181 §2)", () => { + test("an agent edit of .factory/gates.json is denied and changes nothing", async () => { + const before = await readFile(gatesFile(), "utf8") + const edited = await h.run(`edit the gates ${call("edit", { path: ".factory/gates.json", content: "pwned" })}`, { + agent: "factory", + }) + expect(edited.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["edit:error"]) + expect(edited.tools[0]?.text).toContain("factory control file") + const written = await h.run(`write the gates ${call("write", { path: ".factory/gates.json", content: "pwned" })}`, { + agent: "factory", + }) + expect(written.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["write:error"]) + expect(written.tools[0]?.text).toContain("factory control file") + expect(await readFile(gatesFile(), "utf8")).toBe(before) + }, 60_000) + + test("an out-of-band hand edit to gates.json is drift and halts the gates", async () => { + const before = await readFile(gatesFile(), "utf8") + await writeFile( + gatesFile(), + JSON.stringify({ + version: 1, + commands: [{ id: "evil", kind: "test", command: "touch PWNED" }], + ...QUIET_SECURITY, + }), + ) + try { + const check = await verifyControl(h.directory) + expect(check.clean).toBe(false) + expect(check.violations.join("\n")).toContain(".factory/gates.json changed") + const { tools } = await h.run(`run the gates ${call("es_gates_run", {})}`, { agent: "factory" }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["es_gates_run:completed"]) + expect(tools[0]?.text).toContain("Gates failed") + expect(tools[0]?.text).toContain("integrity/control-drift") + expect(tools[0]?.text).toContain(".factory/gates.json changed") + expect(await Bun.file(path.join(h.directory, "PWNED")).exists()).toBe(false) + } finally { + await writeFile(gatesFile(), before) + await rebaseline(h.directory, "test") + } + }, 60_000) + + test("waiver files are pinned: an appeared or changed waiver is drift and halts the gates", async () => { + const waiver = path.join(h.directory, ".factory/waivers/drift-case.json") + await writeRoot(".factory/waivers/drift-case.json", JSON.stringify({ hello: "planted" })) + try { + expect((await verifyControl(h.directory)).violations.join("\n")).toContain("drift-case.json appeared") + const appeared = await h.run(`run the gates ${call("es_gates_run", {})}`, { agent: "factory" }) + expect(appeared.tools[0]?.text).toContain("Gates failed") + expect(appeared.tools[0]?.text).toContain("integrity/control-drift") + expect(appeared.tools[0]?.text).toContain("appeared") + await rebaseline(h.directory, "test") + await writeFile(waiver, JSON.stringify({ hello: "changed" })) + expect((await verifyControl(h.directory)).violations.join("\n")).toContain("drift-case.json changed") + const changed = await h.run(`run the gates ${call("es_gates_run", {})}`, { agent: "factory" }) + expect(changed.tools[0]?.text).toContain("Gates failed") + expect(changed.tools[0]?.text).toContain("drift-case.json changed") + } finally { + await rm(waiver, { force: true }) + await rebaseline(h.directory, "test") + } + }, 60_000) +}) + +describe("waivers and trust at gate time (#221 gaps)", () => { + test("an expired signed waiver is ignored at gate time", async () => { + await writeRoot("src/config.ts", `export const token = "${FAKE_TOKEN}"\n`) + // Hand-aged: signed by the human key but already expired. recordWaiver + // refuses to create these, so the test signs the record directly. + const now = Date.now() + const unsigned = { + version: 2 as const, + id: "expired-fixture", + rule: "security/secret-github-token", + files: "src/config.ts", + reason: "fixture value, not a real token", + approvedBy: "tester", + approvedAt: new Date(now - 86_400_000).toISOString(), + expiresAt: new Date(now - 1_000).toISOString(), + channel: "cli" as const, + auditHead: null, + } + await writeRoot( + ".factory/waivers/expired-fixture.json", + JSON.stringify({ ...unsigned, signature: signer.sign(unsigned) }), + ) + await rebaseline(h.directory, "test") + try { + const { tools } = await h.run(`run the gates ${call("es_gates_run", { files: ["src/config.ts"] })}`, { + agent: "factory", + }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["es_gates_run:completed"]) + expect(tools[0]?.text).toContain("security/secret-github-token") + const report = await runGates({ + root: h.directory, + scope: "touched", + touched: ["src/config.ts"], + stateDir: state, + config: GatesConfigSchema.parse({ commands: [], ...QUIET_SECURITY }), + }) + expect(report.findings.some((finding) => finding.rule === "security/secret-github-token")).toBe(true) + expect(report.waived).toBe(0) + } finally { + await rm(path.join(h.directory, "src/config.ts"), { force: true }) + await rm(path.join(h.directory, ".factory/waivers/expired-fixture.json"), { force: true }) + await rebaseline(h.directory, "test") + } + }, 60_000) + + test("a forged trust.json entry is not trusted and the gates refuse to run", async () => { + const cfg = GatesConfigSchema.parse({ + commands: [{ id: "probe", kind: "test", command: "touch TRUST-RAN" }], + ...QUIET_SECURITY, + }) + const { hash, lines } = await commandSetHash(h.directory, cfg.commands) + await trustProject(h.directory, hash, lines, { stateDir: state, signer }) + expect(await isTrusted(h.directory, hash, state)).toBe(true) + const trustFile = path.join(state, "trust.json") + const raw = JSON.parse(await readFile(trustFile, "utf8")) as { + projects: Record> + } + raw.projects[canonicalPath(h.directory)]!.gates!.signature.sig = `${"A".repeat(43)}=` + await writeFile(trustFile, JSON.stringify(raw)) + const before = await readFile(gatesFile(), "utf8") + await writeFile(gatesFile(), JSON.stringify({ version: 1, commands: cfg.commands, ...QUIET_SECURITY })) + await rebaseline(h.directory, "test") + try { + expect(await isTrusted(h.directory, hash, state)).toBe(false) + const { tools } = await h.run(`run the gates ${call("es_gates_run", { scope: "full" })}`, { agent: "factory" }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["es_gates_run:completed"]) + expect(tools[0]?.text).toContain("trust/untrusted") + expect(await Bun.file(path.join(h.directory, "TRUST-RAN")).exists()).toBe(false) + } finally { + await trustProject(h.directory, hash, lines, { stateDir: state, signer }) + await rm(path.join(h.directory, "TRUST-RAN"), { force: true }) + await writeFile(gatesFile(), before) + await rebaseline(h.directory, "test") + } + }, 60_000) +}) + +describe("agent shells and sealed state through the host (#181 §2)", () => { + test("an agent shell running human-gated verbs is refused", async () => { + for (const command of ["es trust", "es waive lint/check --reason x", "es reseal --sign", "es rebaseline"]) { + const { tools } = await h.run(`shell ${call("shell", { command })}`, { agent: "factory" }) + expect([command, ...tools.map((tool) => `${tool.name}:${tool.status}`)]).toEqual([command, "shell:error"]) + expect([command, tools[0]?.text]).toEqual([command, expect.stringContaining("human-only")]) + } + }, 60_000) + + test("a forged state.json seal refuses reads with the re-sign hint", async () => { + const factory = new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.begin("human:tester") + const layout = factoryLayout(h.directory) + await writeFile(layout.state, `${await readFile(layout.state, "utf8")} `) + try { + await expect(factory.read()).rejects.toThrow("es reseal --sign") + await expect(factory.read()).rejects.toThrow("signature mismatch") + const { tools } = await h.run(`status ${call("es_status", {})}`, { agent: "factory" }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["es_status:error"]) + expect(tools[0]?.text).toContain("es reseal --sign") + await rm(`${layout.state}.sig`) + await expect(factory.read()).rejects.toThrow("es reseal --sign") + } finally { + // What `es reseal --sign` does after review: re-sign the reviewed bytes. + await signEngineFile(layout.state, state) + } + expect((await factory.read())?.stage).toBe("GRILL") + }, 60_000) +}) From b78b53d1887f1e1324f66961e01497fbbf1932c6 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 12:51:20 -0400 Subject: [PATCH 05/83] test(audit): factory state machine integration tests (#179) --- .../core/test/factory-concurrency.test.ts | 118 +++++ packages/opencode/test/stage-order.test.ts | 411 ++++++++++++++++++ 2 files changed, 529 insertions(+) create mode 100644 packages/core/test/factory-concurrency.test.ts create mode 100644 packages/opencode/test/stage-order.test.ts diff --git a/packages/core/test/factory-concurrency.test.ts b/packages/core/test/factory-concurrency.test.ts new file mode 100644 index 0000000..66c5da0 --- /dev/null +++ b/packages/core/test/factory-concurrency.test.ts @@ -0,0 +1,118 @@ +// Concurrent runtime-state writes hold the lock (issue #179, §2). +// +// Two concurrent es_qa_verdict tool calls (functional + adversarial) both +// persist — no torn JSON, no lost verdict — and two concurrent journal steps +// both land. State reads during the race never observe a half-signed +// (state, sidecar) pair: a refusal would reject the Promise.all. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { recordApproval } from "../src/approval/index.ts" +import type { HumanSigner } from "../src/approval/keystore.ts" +import { sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { Factory } from "../src/factory/index.ts" +import { once, readJournal } from "../src/factory/journal.ts" +import { gateRunner } from "../src/gates/index.ts" +import { factoryLayout } from "../src/layout.ts" +import { esTools } from "../src/ops/tools.ts" +import { type Fixture, frontier, gitRepo, writeReport, writeSpecs } from "./helpers.ts" + +let fx: Fixture +let signer: HumanSigner +const green = { passed: true, findings: [], summary: "green" } as const +const gates = async () => green + +beforeEach(async () => { + fx = await gitRepo("es-stage-conc-") + await sealHumanKey("test-passphrase-1234", fx.state) + signer = await unlockHumanKey("test-passphrase-1234", fx.state) +}) +afterEach(() => fx.cleanup()) + +const factory = () => new Factory(fx.root, { gates, stateDir: fx.state }) +const approve = (stage: "frontier" | "spec") => + recordApproval(fx.root, { stage, channel: "cli", approvedBy: "tester", signer }) + +async function toQa() { + const f = factory() + await f.begin("human:tester") + await f.writeFrontier("grill", frontier()) + await approve("frontier") + await f.beginResearch("human:tester") + await writeReport(fx.root, fx.state) + await f.completeResearch("factory") + await writeSpecs(fx.root, ["alpha"]) + await approve("spec") + await f.startBuild("factory") + const dir = (await f.activeWorktree())! + const { mkdir, writeFile } = await import("node:fs/promises") + await mkdir(path.join(dir, "src"), { recursive: true }) + await mkdir(path.join(dir, "test"), { recursive: true }) + await writeFile(path.join(dir, "test/alpha.test.ts"), `test("alpha", () => {})\n`) + await f.recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await writeFile(path.join(dir, "src/alpha.ts"), `export const alpha = 1\n`) + const done = await f.complete("es-programmer") + expect(done.ok).toBe(true) + expect((await f.read())!.stage).toBe("QA") +} + +describe("concurrent runtime-state writes (issue #179)", () => { + test("two concurrent es_qa_verdict tool calls both persist; reads stay sealed", async () => { + await toQa() + const f = factory() + const tools = esTools({ root: fx.root, factory: f, stateDir: fx.state }) + const verdict = tools.find((tool) => tool.name === "es_qa_verdict")! + const [functional, adversarial, seen] = await Promise.all([ + verdict.execute({ verdict: "pass", notes: "criteria met" }, { agent: "es-qa-functional" }), + verdict.execute({ verdict: "pass", notes: "nothing broke" }, { agent: "es-qa-adversarial" }), + f.read(), + ]) + expect(typeof functional).toBe("string") + expect(typeof adversarial).toBe("string") + expect([functional, adversarial].some((text) => text.includes("RELEASE"))).toBe(true) + // The concurrent read saw a sealed, parseable snapshot (never torn): + // it may win the race (QA) or lose it (RELEASE); only progress matters. + expect(["QA", "RELEASE"]).toContain(seen?.stage) + const state = (await f.read())! + expect([state.stage, state.phases[0]!.status]).toEqual(["RELEASE", "passed"]) + expect(state.phases[0]!.qa).toMatchObject({ + functional: { verdict: "pass" }, + adversarial: { verdict: "pass" }, + }) + }) + + test("two concurrent journal steps both land, in full; spend races keep every event", async () => { + await toQa() + const f = factory() + await Promise.all([ + once(fx.root, "concurrent-a", async () => ({ side: "a" })), + once(fx.root, "concurrent-b", async () => ({ side: "b" })), + ]) + const steps = (await readJournal(fx.root)).map((entry) => entry.step) + expect(steps).toContain("concurrent-a") + expect(steps).toContain("concurrent-b") + + // Concurrent spend charges never lose an event and never tear the file. + await Promise.all([f.recordSpend(1, true), f.recordSpend(2, true), f.read(), f.read()]) + const state = (await f.read())! + expect(state.spend.usd).toBe(3) + expect(state.spend.events).toBe(2) + // The sidecar still verifies after the race (read would throw otherwise). + expect(state.stage).toBe("QA") + }) + + test("ten concurrent spend writes serialize without loss", async () => { + await toQa() + const f = factory() + await Promise.all(Array.from({ length: 10 }, (_, i) => f.recordSpend(1, i % 2 === 0))) + const state = (await f.read())! + expect(state.spend.usd).toBe(10) + expect(state.spend.events).toBe(10) + }) +}) diff --git a/packages/opencode/test/stage-order.test.ts b/packages/opencode/test/stage-order.test.ts new file mode 100644 index 0000000..b684bb1 --- /dev/null +++ b/packages/opencode/test/stage-order.test.ts @@ -0,0 +1,411 @@ +// Factory stage ordering on the real host (issue #179, integration tests §2). +// +// For each stage, scripted calls to every phase-transition es_* tool that +// belongs to a later stage are refused with a stage error and leave the +// functional state (stage/phases) unchanged. Stage-free-by-design tools +// (es_status, es_spec_validate, es_request_approval, es_gates_run, +// es_audit_open, research/scout/brainstorm/harvest/design/lsp) are out of +// scope. Two known holes get explicit coverage: an audit tiebreak outside QA +// and gate-run recording outside BUILD must not change state. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + approveStage, + Factory, + factoryLayout, + gateRunner, + type HumanSigner, + readJournal, + researchSourcesDir, + SourceCache, + sealHumanKey, + stringifyFrontmatter, + unlockHumanKey, + writeJson, +} from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const write = async (root: string, file: string, text: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), text) +} + +const GREEN = { passed: true, findings: [], summary: "green" } + +describe("factory stage order (issue #179)", () => { + let h: Harness + let state: string + let signer: HumanSigner + const stub = () => new Factory(h.directory, { gates: async () => GREEN, stateDir: state }) + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-stage-order-")) + await sealHumanKey("test-passphrase-1234", state) + signer = await unlockHumanKey("test-passphrase-1234", state) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# stage order\n" }, + }) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + const snap = async () => { + const s = (await stub().read())! + return JSON.stringify({ stage: s.stage, active: s.activePhase ?? null, phases: s.phases }) + } + const stageOf = async () => (await stub().read())!.stage + + const refused = async (tool: string, agent: string, args: Record, match: RegExp) => { + const before = await snap() + const { tools } = await h.run(`stage-check ${call(tool, args)}`, { agent }) + expect([tool, tools[0]?.status]).toEqual([tool, "error"]) + expect([tool, tools[0]?.text ?? ""]).toEqual([tool, expect.stringMatching(match)]) + expect([tool, await snap()]).toEqual([tool, before]) + } + + const frontierDoc = { + version: "1.1", + idea: "a greeting library", + spendCeiling: { currency: "USD", maxAmount: 10 }, + round: 1, + settled: true, + nodes: [{ id: "scope", question: "What ships first?", status: "settled", answer: "greet", round: 1 }], + } + + const writeReport = async () => { + const cache = new SourceCache(researchSourcesDir(h.directory), state) + const source = await cache.put({ + url: "https://example.test/stage-order", + text: "Stage order tests use a single greet function.", + provider: "fetch", + }) + await write( + h.directory, + ".factory/research/REPORT.md", + `# Research\n\n- Greet is the API [VERIFIED: sha256:${source.meta.sha256} "a single greet function"]\n`, + ) + } + + const writeSpec = async () => { + await writeJson(path.join(h.directory, ".factory/roadmap.json"), { + version: 1, + title: "Greet", + phases: [{ id: "alpha", title: "Alpha" }], + }) + await write( + h.directory, + ".factory/specs/alpha/GOAL.md", + stringifyFrontmatter( + { + phase: "alpha", + title: "Alpha", + acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }], + }, + "Implement alpha.\n", + ), + ) + } + + test("GRILL: every later-stage transition is refused", async () => { + await stub().provisionRun("tester", 10) + expect(await stageOf()).toBe("GRILL") + await refused("es_research_complete", "factory", {}, /needs RESEARCH|Not allowed/i) + await refused("es_build_start", "factory", {}, /needs SPEC|Not allowed|no spec approval/i) + await refused("es_complete", "es-programmer", {}, /needs BUILD|Not allowed|Only the programmer/i) + await refused("es_qa_verdict", "es-qa-functional", { verdict: "pass", notes: "x" }, /needs QA|Not allowed/i) + await refused("es_tiebreak", "es-manager", { verdict: "pass", notes: "x" }, /needs QA|only possible|Not allowed/i) + await refused("es_replan", "es-manager", { notes: "x" }, /needs BUILD|not awaiting|Not allowed/i) + await refused("es_release", "factory", {}, /needs RELEASE|Not allowed/i) + await refused( + "es_audit_verdict", + "es-auditor-thesis", + { audit: "audit-01", verdict: "pass", findings: [], notes: "x" }, + /No audit|phase audits take verdicts|Not allowed/i, + ) + // Audit tiebreak with no audit is refused and changes nothing. + await refused( + "es_tiebreak", + "es-manager", + { verdict: "pass", notes: "x", audit: "audit-01" }, + /No audit|only possible/i, + ) + }, 120_000) + + test("RESEARCH: build and later transitions are refused", async () => { + await stub().writeFrontier("grill", frontierDoc) + await approveStage(h.directory, { + stage: "frontier", + channel: "cli", + approvedBy: "tester", + signer, + stateDir: state, + }) + expect(await stageOf()).toBe("RESEARCH") + await refused("es_build_start", "factory", {}, /needs SPEC|no spec approval|Not allowed/i) + await refused("es_complete", "es-programmer", {}, /needs BUILD|Not allowed|Only the programmer/i) + await refused("es_qa_verdict", "es-qa-functional", { verdict: "pass", notes: "x" }, /needs QA|Not allowed/i) + await refused("es_tiebreak", "es-manager", { verdict: "pass", notes: "x" }, /needs QA|only possible|Not allowed/i) + await refused("es_replan", "es-manager", { notes: "x" }, /needs BUILD|not awaiting|Not allowed/i) + await refused("es_release", "factory", {}, /needs RELEASE|Not allowed/i) + // The grill tree is closed: frontier writes are refused past GRILL. + await refused("es_frontier_write", "grill", { frontier: { ...frontierDoc, round: 2 } }, /needs GRILL|Not allowed/i) + }, 120_000) + + test("SPEC: build-complete and later transitions are refused", async () => { + await writeReport() + await stub().completeResearch("factory") + expect(await stageOf()).toBe("SPEC") + await refused("es_research_complete", "factory", {}, /needs RESEARCH|Not allowed/i) + await refused("es_complete", "es-programmer", {}, /needs BUILD|Not allowed|Only the programmer/i) + await refused("es_qa_verdict", "es-qa-functional", { verdict: "pass", notes: "x" }, /needs QA|Not allowed/i) + await refused("es_tiebreak", "es-manager", { verdict: "pass", notes: "x" }, /needs QA|only possible|Not allowed/i) + await refused("es_replan", "es-manager", { notes: "x" }, /needs BUILD|not awaiting|Not allowed/i) + await refused("es_release", "factory", {}, /needs RELEASE|Not allowed/i) + // Gate-run recording outside BUILD must not touch phase history: there + // are no phases yet, and the silent no-op (no throw) leaves state alone. + const before = await snap() + await stub().recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + expect(await snap()).toBe(before) + }, 120_000) + + test("BUILD: QA and release transitions are refused; audit verdicts wait for QA", async () => { + await writeSpec() + await approveStage(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer, stateDir: state }) + await stub().startBuild("factory") + expect(await stageOf()).toBe("BUILD") + await refused("es_research_complete", "factory", {}, /needs RESEARCH|Not allowed/i) + await refused("es_build_start", "factory", {}, /needs SPEC|Not allowed/i) + await refused("es_qa_verdict", "es-qa-functional", { verdict: "pass", notes: "x" }, /needs QA|Not allowed/i) + await refused("es_tiebreak", "es-manager", { verdict: "pass", notes: "x" }, /needs QA|only possible|Not allowed/i) + await refused("es_replan", "es-manager", { notes: "x" }, /not awaiting|needs BUILD|Not allowed/i) + await refused("es_release", "factory", {}, /needs RELEASE|Not allowed/i) + // A phase audit opened in BUILD takes no verdicts until QA. + const { audit } = await stub().openAudit("factory", { kind: "phase", phase: "alpha" }) + const early = await h.run( + `audit ${call("es_audit_verdict", { audit: audit.id, verdict: "pass", findings: [], notes: "early" })}`, + { agent: "es-auditor-thesis" }, + ) + expect(early.tools[0]?.status).toBe("error") + expect(early.tools[0]?.text).toMatch(/phase audits take verdicts in QA|Not allowed/i) + // An audit tiebreak with no disagreeing verdicts is refused. + await refused( + "es_tiebreak", + "es-manager", + { verdict: "pass", notes: "x", audit: audit.id }, + /only possible|Not allowed/i, + ) + }, 180_000) + + test("QA: build and release transitions are refused; gate runs record nothing", async () => { + // Programmer cycle through the stub (fake green gates): red evidence, + // sources, then the gated complete into QA. + const wt = path.join(h.directory, ".factory/worktrees/alpha") + await mkdir(path.join(wt, "src"), { recursive: true }) + await mkdir(path.join(wt, "test"), { recursive: true }) + await writeFile(path.join(wt, "test/alpha.test.ts"), `test("alpha", () => {})\n`) + await stub().recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await writeFile(path.join(wt, "src/alpha.ts"), `export const alpha = 1\n`) + const done = await stub().complete("es-programmer") + expect(done.ok).toBe(true) + expect(await stageOf()).toBe("QA") + await refused("es_research_complete", "factory", {}, /needs RESEARCH|Not allowed/i) + await refused("es_build_start", "factory", {}, /needs SPEC|Not allowed/i) + await refused("es_complete", "es-programmer", {}, /needs BUILD|not building|Not allowed/i) + await refused("es_replan", "es-manager", { notes: "x" }, /needs BUILD|not awaiting|Not allowed/i) + await refused("es_release", "factory", {}, /needs RELEASE|Not allowed/i) + // Gate-run recording outside BUILD is a silent no-op: the tool reports + // gates, but the phase history gains no entry (finding: not a refusal). + const history = (id: string) => + stub() + .read() + .then((s) => s!.phases.find((p) => p.id === id)!.history.length) + const before = await history("alpha") + const gates = await h.run(`gates ${call("es_gates_run", { files: [] })}`, { agent: "es-programmer" }) + expect(gates.tools[0]?.status).toBe("completed") + expect(await history("alpha")).toBe(before) + }, 180_000) + + test("RELEASE: everything before release is refused", async () => { + await stub().qaVerdict("es-qa-functional", "pass", "criteria met") + // The BUILD-opened audit still holds the merge; pass it to reach RELEASE. + const stateNow = (await stub().read())! + const auditId = stateNow.phases[0]!.audit!.id + const wt = path.join(h.directory, ".factory/worktrees/alpha") + const line = (await Bun.file(path.join(wt, "src/alpha.ts")).text()).split("\n")[0]! + const holds = { + kind: "invariant", + title: "alpha is a constant", + severity: "info", + file: "src/alpha.ts", + lines: [1, 1], + excerpt: line, + detail: "constant", + holds: true, + } + // Witnessed verdicts go through the host auditor seats. + for (const seat of ["es-auditor-thesis", "es-auditor-antithesis"]) { + const { tools } = await h.run( + `review ${call("es_audit_verdict", { audit: auditId, verdict: "pass", findings: seat === "es-auditor-thesis" ? [holds] : [], notes: "sound" })}`, + { agent: seat }, + ) + expect([seat, tools[0]?.status]).toEqual([seat, "completed"]) + } + // One QA seat already passed; the other merges the phase into RELEASE. + const { tools } = await h.run(`qa ${call("es_qa_verdict", { verdict: "pass", notes: "nothing broke" })}`, { + agent: "es-qa-adversarial", + }) + expect(tools[0]?.status).toBe("completed") + expect(await stageOf()).toBe("RELEASE") + await refused("es_research_complete", "factory", {}, /needs RESEARCH|Not allowed/i) + await refused("es_build_start", "factory", {}, /needs SPEC|Not allowed/i) + await refused("es_complete", "es-programmer", {}, /needs BUILD|Not allowed|not building/i) + await refused("es_qa_verdict", "es-qa-functional", { verdict: "pass", notes: "x" }, /needs QA|Not allowed/i) + await refused("es_tiebreak", "es-manager", { verdict: "pass", notes: "x" }, /needs QA|only possible|Not allowed/i) + await refused("es_replan", "es-manager", { notes: "x" }, /needs BUILD|not awaiting|Not allowed/i) + }, 180_000) +}) + +describe("happy path Grill → DONE records the journal in order (issue #179)", () => { + let h: Harness + let state: string + let signer: HumanSigner + const factory = () => new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-journal-")) + await sealHumanKey("test-passphrase-1234", state) + signer = await unlockHumanKey("test-passphrase-1234", state) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# journal\n" }, + }) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("grill → research → spec → build → QA → release → DONE; journal steps in order", async () => { + await factory().provisionRun("tester", 10) + await factory().writeFrontier("grill", { + version: "1.1", + idea: "a greeting library", + spendCeiling: { currency: "USD", maxAmount: 10 }, + round: 1, + settled: true, + nodes: [{ id: "scope", question: "What ships first?", status: "settled", answer: "greet", round: 1 }], + }) + await approveStage(h.directory, { + stage: "frontier", + channel: "cli", + approvedBy: "tester", + signer, + stateDir: state, + }) + const cache = new SourceCache(researchSourcesDir(h.directory), state) + const source = await cache.put({ + url: "https://example.test/greet", + text: "Greeting libraries expose a single greet function.", + provider: "fetch", + }) + await write( + h.directory, + ".factory/research/REPORT.md", + `# Research\n\n- Greet is conventional [VERIFIED: sha256:${source.meta.sha256} "a single greet function"]\n`, + ) + const researched = await h.run(`research ${call("es_research_complete", {})}`, { agent: "factory" }) + expect(researched.tools[0]?.status).toBe("completed") + expect((await factory().read())!.stage).toBe("SPEC") + + await writeJson(path.join(h.directory, ".factory/roadmap.json"), { + version: 1, + title: "Greet", + phases: [{ id: "alpha", title: "Alpha" }], + }) + await write( + h.directory, + ".factory/specs/alpha/GOAL.md", + stringifyFrontmatter( + { + phase: "alpha", + title: "Alpha", + acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }], + }, + "Implement alpha.\n", + ), + ) + await approveStage(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer, stateDir: state }) + const started = await h.run(`build ${call("es_build_start", {})}`, { agent: "factory" }) + expect(started.tools[0]?.status).toBe("completed") + const runId = (await factory().read())!.runId + + // Red first (direct core, as in pipeline.test.ts), then the gated + // complete through the scripted programmer on the host. + const wt = (await factory().activeWorktree())! + const rel = (file: string) => path.relative(h.directory, path.join(wt, file)) + await write(h.directory, rel("test/alpha.test.ts"), `test("alpha", () => {})\n`) + await factory().recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await write(h.directory, rel("src/alpha.ts"), `export const alpha = 1\n`) + const completed = await h.run(`finish ${call("es_complete", {})}`, { agent: "es-programmer" }) + expect(completed.tools[0]?.status).toBe("completed") + expect(completed.tools[0]?.text).toContain("in QA") + + for (const seat of ["es-qa-functional", "es-qa-adversarial"]) { + const verdict = await h.run(`qa ${call("es_qa_verdict", { verdict: "pass", notes: "criteria met" })}`, { + agent: seat, + }) + expect([seat, verdict.tools[0]?.status]).toEqual([seat, "completed"]) + } + const released = (await factory().read())! + expect([released.stage, released.phases[0]!.status]).toEqual(["RELEASE", "passed"]) + + // pr:off keeps es_release refused; the human records the PR to DONE. + const noPr = await h.run(`ship ${call("es_release", {})}`, { agent: "factory" }) + expect(noPr.tools[0]?.status).toBe("error") + const done = await factory().recordPr("tester", "https://example.test/pr/1") + expect([done.stage, done.release?.prUrl]).toEqual(["DONE", "https://example.test/pr/1"]) + + // The journal records every side-effecting transition in order. + const steps = (await readJournal(h.directory)).map((entry) => entry.step) + const expected = [ + `${runId}:run-branch`, + `${runId}:alpha:worktree`, + `${runId}:alpha:attempt-1:commit`, + `${runId}:alpha:merge`, + `${runId}:alpha:worktree-removed`, + ] + let at = -1 + for (const step of expected) { + const found = steps.indexOf(step) + expect([step, found]).toEqual([step, expect.any(Number)]) + expect(found).toBeGreaterThan(at) + at = found + } + }, 240_000) +}) From f93ba7ed5b508978eeea6ac4ed44e3836f7c243f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:12:50 -0400 Subject: [PATCH 06/83] test(audit): bubblewrap sandbox integration tests (#183) --- packages/core/test/sandbox-no-bwrap.test.ts | 152 ++++++++++++++++++ packages/opencode/test/sandbox-seats.test.ts | 160 +++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 packages/core/test/sandbox-no-bwrap.test.ts create mode 100644 packages/opencode/test/sandbox-seats.test.ts diff --git a/packages/core/test/sandbox-no-bwrap.test.ts b/packages/core/test/sandbox-no-bwrap.test.ts new file mode 100644 index 0000000..16bd610 --- /dev/null +++ b/packages/core/test/sandbox-no-bwrap.test.ts @@ -0,0 +1,152 @@ +// No-bwrap refusal for issue #183: without bubblewrap, factory seats get no +// shell (fail closed), factory gate runs halt, and the user's own agents fall +// back to the weaker argv-aware text policy. Uses the sandboxAvailable:false +// seam plus the real PATH probe (bwrap is resolved from PATH by design). +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { + commandSetHash, + evaluateShell, + GatesConfigSchema, + runGates, + SANDBOX_MISSING, + trustProject, +} from "../src/index.ts" +import { bwrapAvailable, resetBwrapProbe } from "../src/util/proc.ts" +import { type Fixture, gitRepo } from "./helpers.ts" + +let root: string +let state: string +let worktree: string + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-nobwrap-")) + state = await mkdtemp(path.join(tmpdir(), "es-nobwrap-state-")) + worktree = path.join(root, ".factory/worktrees/p1") +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + resetBwrapProbe() +}) + +const ctx = () => ({ root, worktree, stateDir: state }) + +describe("no-bwrap refusal (#183)", () => { + test("factory seats are refused a shell without bubblewrap, naming it", () => { + for (const agent of ["factory", "es-manager", "es-programmer", "es-qa-functional", "es-research-alpha"]) { + const decision = evaluateShell(ctx(), agent, "ls", { sandboxAvailable: false }) + expect(decision.effect).toBe("deny") + expect(decision.effect === "deny" ? decision.reason : "").toContain("bubblewrap") + expect(decision.effect === "deny" ? decision.reason : "").toContain(SANDBOX_MISSING.slice(0, 24)) + } + // With bubblewrap the same seats get a sandboxed shell. + expect(evaluateShell(ctx(), "es-programmer", "ls", { sandboxAvailable: true })).toMatchObject({ + effect: "allow", + mode: "sandbox", + }) + }) + + test("PATH without bwrap: hiding bwrap from PATH refuses seats and keeps the user text policy", async () => { + const savedPath = process.env.PATH + const emptyBin = await mkdtemp(path.join(tmpdir(), "es-nobwrap-bin-")) + try { + process.env.PATH = emptyBin + resetBwrapProbe() + expect(bwrapAvailable()).toBe(false) + const refused = evaluateShell(ctx(), "es-qa-functional", "echo hi", { sandboxAvailable: bwrapAvailable() }) + expect(refused.effect).toBe("deny") + expect(refused.effect === "deny" ? refused.reason : "").toContain("bubblewrap") + const user = evaluateShell(ctx(), undefined, "echo hi", { sandboxAvailable: bwrapAvailable() }) + expect(user).toEqual({ effect: "allow", mode: "unsandboxed" }) + } finally { + if (savedPath === undefined) delete process.env.PATH + else process.env.PATH = savedPath + resetBwrapProbe() + await rm(emptyBin, { recursive: true, force: true }) + } + }) + + test("user agents fall back to the weaker argv-aware text policy without bwrap", () => { + // Plain commands run unsandboxed; the text rules still gate control files. + expect(evaluateShell(ctx(), undefined, "bun test", { sandboxAvailable: false })).toEqual({ + effect: "allow", + mode: "unsandboxed", + }) + expect(evaluateShell(ctx(), "build", "echo hi", { sandboxAvailable: false })).toEqual({ + effect: "allow", + mode: "unsandboxed", + }) + expect(evaluateShell(ctx(), undefined, "echo x > .factory/gates.json", { sandboxAvailable: false }).effect).toBe( + "deny", + ) + expect(evaluateShell(ctx(), undefined, "cat .factory/gates.json", { sandboxAvailable: false }).effect).toBe("allow") + // With bubblewrap the user's agent is sandboxed instead. + expect(evaluateShell(ctx(), undefined, "bun test", { sandboxAvailable: true })).toEqual({ + effect: "allow", + mode: "sandbox", + kind: "user", + offline: false, + }) + }) +}) + +describe("gate runs without bwrap (#183)", () => { + let fx: Fixture + let signer: HumanSigner + const PASSPHRASE = "test-passphrase-1234" + beforeEach(async () => { + fx = await gitRepo("es-nobwrap-gates-") + await sealHumanKey(PASSPHRASE, fx.state) + signer = await unlockHumanKey(PASSPHRASE, fx.state) + }) + afterEach(() => fx.cleanup()) + + const config = (commands: unknown[]) => + GatesConfigSchema.parse({ commands, security: { gitleaks: "off", osv: "off" } }) + + test("a required gate run halts without bubblewrap and runs nothing", async () => { + const cfg = config([{ id: "probe", kind: "test", command: "touch RAN" }]) + const { hash, lines } = await commandSetHash(fx.root, cfg.commands) + await trustProject(fx.root, hash, lines, { stateDir: fx.state, signer }) + resetBwrapProbe(false) + try { + const report = await runGates({ + root: fx.root, + scope: "full", + config: cfg, + sandbox: "required", + stateDir: fx.state, + }) + expect(report.passed).toBe(false) + expect(report.findings.map((finding) => finding.rule)).toContain("gates/sandbox-missing") + expect(report.findings[0]?.message).toContain("bubblewrap") + expect(await Bun.file(path.join(fx.root, "RAN")).exists()).toBe(false) + } finally { + resetBwrapProbe() + } + }) + + test("a preferred gate run still runs without bubblewrap (user and human runs)", async () => { + const cfg = config([{ id: "probe", kind: "test", command: "touch RAN-PREFERRED" }]) + const { hash, lines } = await commandSetHash(fx.root, cfg.commands) + await trustProject(fx.root, hash, lines, { stateDir: fx.state, signer }) + resetBwrapProbe(false) + try { + const report = await runGates({ + root: fx.root, + scope: "full", + config: cfg, + sandbox: "preferred", + stateDir: fx.state, + }) + expect(report.checks.find((check) => check.id === "probe")?.status).toBe("pass") + expect(await Bun.file(path.join(fx.root, "RAN-PREFERRED")).exists()).toBe(true) + } finally { + resetBwrapProbe() + } + }) +}) diff --git a/packages/opencode/test/sandbox-seats.test.ts b/packages/opencode/test/sandbox-seats.test.ts new file mode 100644 index 0000000..70fcb61 --- /dev/null +++ b/packages/opencode/test/sandbox-seats.test.ts @@ -0,0 +1,160 @@ +// Seat sandbox on the real host for issue #183 (beside scoping.test.ts): +// a cached-web research seat gets a network failure through curl +// (--unshare-net), a programmer seat shell cannot write outside its worktree, +// seat shells never leak server passwords or the scraper token, and without +// bwrap factory seats are refused while user agents keep the text policy. +// Real in-process OpenCode v2 host via @heretek-ai/es-testkit (scripted fake LLM). +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import os, { tmpdir } from "node:os" +import path from "node:path" +import { bwrapAvailable } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +describe("seat sandbox on the real host (#183)", () => { + let h: Harness + let state: string + let isolatedHome: string + let savedHome: string | undefined + const origHomedir = os.homedir + + beforeAll(async () => { + // Isolate the home the seat sandbox overlays onto tmpfs (/tmp): bwrap + // --tmp-overlay on a btrfs home (e.g. /home) fails with "userxattr: + // Invalid argument", while tmpfs succeeds — the same reason + // sandbox.test.ts uses a home under /tmp. Bun's os.homedir() ignores a + // $HOME change, so patch it (restored afterwards); the plugin resolves + // the sandbox home through the same module instance. + savedHome = process.env.HOME + isolatedHome = await mkdtemp(path.join(tmpdir(), "es-sandbox-home-")) + process.env.HOME = isolatedHome + ;(os as unknown as { homedir: () => string }).homedir = () => isolatedHome + state = await mkdtemp(path.join(tmpdir(), "es-sandbox-seats-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# sandbox seats\n" }, + }) + }, 120_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + await rm(isolatedHome, { recursive: true, force: true }) + ;(os as unknown as { homedir: () => string }).homedir = origHomedir + if (savedHome === undefined) delete process.env.HOME + else process.env.HOME = savedHome + }) + + test.skipIf(!bwrapAvailable())( + "a cached-web research seat gets a network failure through curl (--unshare-net)", + async () => { + const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch: () => new Response("OUTSIDE-REACHED"), + }) + try { + // The curl runs inside the seat sandbox: the host loopback is outside + // its network namespace, so even --max-time curl cannot reach it. + const offline = await h.run( + `net ${call("shell", { command: `curl -sS --max-time 5 http://127.0.0.1:${server.port}/; echo CURL_EXIT:$?` })}`, + { agent: "es-research-alpha" }, + ) + expect(offline.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + expect(offline.tools[0]?.text).not.toContain("OUTSIDE-REACHED") + expect(offline.tools[0]?.text).toMatch(/CURL_EXIT:[1-9]/) + // Control: the same curl outside the sandbox reaches the server, so + // the failure is the sandbox, not the server. + const online = await h.run( + `net ${call("shell", { command: `curl -sS --max-time 5 http://127.0.0.1:${server.port}/; echo CURL_EXIT:$?` })}`, + { agent: "build" }, + ) + expect(online.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + expect(online.tools[0]?.text).toContain("OUTSIDE-REACHED") + expect(online.tools[0]?.text).toContain("CURL_EXIT:0") + } finally { + server.stop(true) + } + }, + 60_000, + ) + + test.skipIf(!bwrapAvailable())( + "a programmer seat shell cannot write outside its worktree", + async () => { + // No phase is building, so the programmer has no writable directory: + // the whole project is outside its worktree and the sandbox holds it + // read-only. The trailing echo proves the shell itself ran. + const { tools } = await h.run( + `write ${call("shell", { command: "echo PWNED > PROBE-OUTSIDE-183.md; echo MARKER" })}`, + { agent: "es-programmer" }, + ) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + expect(tools[0]?.text).toContain("MARKER") + expect(await Bun.file(path.join(h.directory, "PROBE-OUTSIDE-183.md")).exists()).toBe(false) + }, + 60_000, + ) + + test("seat shells never leak the server password or the scraper token", async () => { + // End to end: both secrets sit in the host's own environment, and even a + // seat's sandboxed shell must not see either (hook strips passwords, + // bwrap unsets every SECRET_ENV entry). + const savedPassword = process.env.OPENCODE_SERVER_PASSWORD + const savedToken = process.env.ES_SCRAPER_SWARM_TOKEN + process.env.OPENCODE_SERVER_PASSWORD = "probe-server-secret" + process.env.ES_SCRAPER_SWARM_TOKEN = "probe-scraper-secret" + try { + for (const variable of ["OPENCODE_SERVER_PASSWORD", "ES_SCRAPER_SWARM_TOKEN"]) { + const { tools } = await h.run(`leak ${call("shell", { command: `echo val=$${variable}` })}`, { + agent: "es-programmer", + }) + if (!bwrapAvailable()) { + // Without the sandbox a seat shell never runs at all: still no leak. + expect([variable, ...tools.map((tool) => `${tool.name}:${tool.status}`)]).toEqual([variable, "shell:error"]) + expect(tools[0]?.text).toContain("bubblewrap") + expect(tools[0]?.text).not.toContain("probe-server-secret") + expect(tools[0]?.text).not.toContain("probe-scraper-secret") + continue + } + expect([variable, ...tools.map((tool) => `${tool.name}:${tool.status}`)]).toEqual([variable, "shell:completed"]) + expect(tools[0]?.text).toContain("val=") + expect(tools[0]?.text).not.toContain("probe-server-secret") + expect(tools[0]?.text).not.toContain("probe-scraper-secret") + } + } finally { + if (savedPassword === undefined) delete process.env.OPENCODE_SERVER_PASSWORD + else process.env.OPENCODE_SERVER_PASSWORD = savedPassword + if (savedToken === undefined) delete process.env.ES_SCRAPER_SWARM_TOKEN + else process.env.ES_SCRAPER_SWARM_TOKEN = savedToken + } + }, 60_000) + + test("without bwrap factory seats are refused a shell; user agents keep the text policy", async () => { + if (!bwrapAvailable()) { + const seat = await h.run(`x ${call("shell", { command: "echo hi" })}`, { agent: "es-qa-functional" }) + expect(seat.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:error"]) + expect(seat.tools[0]?.text).toMatch(/bubblewrap/i) + // The user's own agent still runs, but mutating control files is denied. + const denied = await h.run(`x ${call("shell", { command: "echo x > .factory/gates.json" })}`, { + agent: "build", + }) + expect(denied.tools[0]?.status).toBe("error") + const read = await h.run(`x ${call("shell", { command: "cat README.md" })}`, { agent: "build" }) + expect(read.tools[0]?.status).toBe("completed") + return + } + // With bwrap the seat shell runs sandboxed, while the user's text policy + // still denies the same mutating command (defence in depth). + const seat = await h.run(`x ${call("shell", { command: "echo hi" })}`, { agent: "es-qa-functional" }) + expect(seat.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + const denied = await h.run(`x ${call("shell", { command: "echo x > .factory/gates.json" })}`, { + agent: "build", + }) + expect(denied.tools[0]?.status).toBe("error") + }, 60_000) +}) From 5700eb26aea4c3d9f49c57a03dc3568198963cfa Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:13:05 -0400 Subject: [PATCH 07/83] test(audit): STOP halt and spend ceiling integration tests (#184) --- packages/core/test/stop-delete.test.ts | 75 +++++++++ packages/opencode/test/stop-ceiling.test.ts | 176 ++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 packages/core/test/stop-delete.test.ts create mode 100644 packages/opencode/test/stop-ceiling.test.ts diff --git a/packages/core/test/stop-delete.test.ts b/packages/core/test/stop-delete.test.ts new file mode 100644 index 0000000..8f40a92 --- /dev/null +++ b/packages/core/test/stop-delete.test.ts @@ -0,0 +1,75 @@ +// STOP kill-switch delete-denial (#184, bug #229): `.factory/STOP` is a +// factory control file, so no agent can create, overwrite or remove it +// through any surface. Host `write`/`edit` and every `patch` target funnel +// through evaluateWrite (packages/opencode/src/policy.ts maps patch targets +// through it too); `shell` goes through evaluateShell. This closes the proof +// gap without changing product behaviour. +import { describe, expect, test } from "bun:test" +import path from "node:path" +import { controlClass } from "../src/trust/control.ts" +import { evaluateShell, evaluateWrite } from "../src/trust/index.ts" + +// Pure classification: fixed paths, no filesystem needed (as in control.test.ts). +const ctx = () => ({ root: "/tmp/es-stop-delete-test", stateDir: "/tmp/es-stop-delete-state" }) +const effect = (decision: { effect: string }) => decision.effect + +describe("STOP is a factory control file", () => { + test("STOP classifies as factory control, however cased or nested", () => { + for (const file of [".factory/STOP", ".FACTORY/STOP", ".factory/stop", ".factory/worktrees/p1/.factory/STOP"]) + expect([file, controlClass(file)]).toEqual([file, "factory"]) + // A suffixed name is not the kill-switch. + expect(controlClass(".factory/STOP.bak")).toBeUndefined() + }) +}) + +describe("STOP write policy", () => { + test("no agent writes STOP: relative, absolute and dot-dot paths are denied", () => { + const root = "/tmp/es-stop-delete-test" + for (const agent of [ + undefined, + "build", + "factory", + "es-manager", + "es-programmer", + "es-qa-functional", + "es-research-alpha", + "harvester", + ]) + for (const target of [".factory/STOP", path.join(root, ".factory/STOP"), "docs/../.factory/STOP"]) + expect([agent, target, effect(evaluateWrite(ctx(), agent, target))]).toEqual([agent, target, "deny"]) + }) +}) + +describe("STOP shell policy", () => { + const shell = (agent: string | undefined, command: string) => + evaluateShell(ctx(), agent, command, { sandboxAvailable: true }) + + test("no agent mutates STOP from the shell, however wrapped", () => { + for (const command of [ + "echo hold > .factory/STOP", + "rm .factory/STOP", + "rm -f .factory/STOP", + `python3 -c 'open(".factory/STOP","w").write("hold")'`, + "echo hold > .FACTORY/STOP", + "echo hold > docs/../.factory/STOP", + ]) + for (const agent of ["build", "factory", "es-programmer"]) + expect([agent, command, shell(agent, command).effect]).toEqual([agent, command, "deny"]) + // #37 shapes (as for gates.json in policy.test.ts): cd-relative and + // separator variants, denied for the user's agents. + for (const command of [ + "cd .factory && echo hold > STOP", + "cd .factory;echo hold > STOP", + "cd .factory&&echo hold > STOP", + "cd .factory|tee STOP", + ]) + expect([command, shell("build", command).effect]).toEqual([command, "deny"]) + }) + + test("seats may not even mention STOP; user reads stay allowed (gates parity)", () => { + for (const agent of ["factory", "es-programmer", "es-qa-functional"]) + expect([agent, shell(agent, "cat .factory/STOP").effect]).toEqual([agent, "deny"]) + expect(shell("build", "cat .factory/STOP").effect).toBe("allow") + expect(shell("build", "ls .factory").effect).toBe("allow") + }) +}) diff --git a/packages/opencode/test/stop-ceiling.test.ts b/packages/opencode/test/stop-ceiling.test.ts new file mode 100644 index 0000000..95e868e --- /dev/null +++ b/packages/opencode/test/stop-ceiling.test.ts @@ -0,0 +1,176 @@ +// STOP halt and mandatory spend ceiling on the real host (#184): a scripted +// fake LLM reporting cost per step drives the run into its ceiling, and a +// STOP file created mid-run refuses the next es_* call and the next seat +// launch. Agents can neither create nor remove STOP (bug #229 proof gap). +// +// Real in-process OpenCode v2 host via packages/testkit (scripted fake LLM); +// must pass pinned and under scripts/v2-head.sh. +// +// Not covered here by design: the MCP surface executes esTools with no STOP +// gate and a non-seat (user-agent) `subagent` launch bypasses STOP/HALTED +// (both filed as bug #227) — the seat-launch refusal below uses a factory +// seat, the guarded path. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { Factory, factoryLayout, gateRunner } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness, type Script } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const stopOf = (h: Harness) => path.join(h.directory, ".factory/STOP") +const exists = (file: string) => Bun.file(file).exists() + +async function bootHost(prefix: string, script: Script = directiveScript) { + const state = await mkdtemp(path.join(tmpdir(), prefix)) + const h = await boot({ + git: true, + script, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# stop ceiling\n" }, + }) + const factory = () => new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + return { state, h, factory } +} + +// Per-step cost the fake LLM reports, in USD, under the tracker's default +// estimate rates (packages/opencode/src/session.ts: inputPerM 3, +// outputPerM 15): every host step records exactly this estimate. +const STEP_INPUT_TOKENS = 20_000 +const STEP_OUTPUT_TOKENS = 10_000 +const STEP_COST_USD = (STEP_INPUT_TOKENS * 3 + STEP_OUTPUT_TOKENS * 15) / 1_000_000 +const costly: Script = (request) => ({ + ...directiveScript(request), + usage: { input: STEP_INPUT_TOKENS, output: STEP_OUTPUT_TOKENS }, +}) + +describe("the spend ceiling halts a real v2 run (#184)", () => { + let h: Harness + let state: string + let factory: () => Factory + beforeAll(async () => { + ;({ h, state, factory } = await bootHost("es-stop-ceiling-", costly)) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("fake-LLM cost per step runs into the ceiling; halt names it; spend <= ceiling + one step", async () => { + const CEILING = 0.5 + await factory().provisionRun("tester", CEILING) + // Run until the brake engages (each run is a factory-agent session, so + // every host step is estimated spend). Poll after each run: spend lands + // through the async session.step.ended subscription. + for (let i = 0; i < 10; i++) { + await h.run(call("es_status"), { agent: "factory" }) + const start = Date.now() + while ((await factory().read())?.stage !== "HALTED" && Date.now() - start < 5_000) await Bun.sleep(20) + if ((await factory().read())?.stage === "HALTED") break + } + const halted = (await factory().read())! + expect(halted.stage).toBe("HALTED") + expect(halted.halt?.reason).toContain("spend ceiling reached") + expect(halted.spend.estimated).toBe(true) + // Breach overshoots by at most the step that crossed it. + expect(halted.spend.usd).toBeGreaterThanOrEqual(CEILING) + expect(halted.spend.usd).toBeLessThanOrEqual(CEILING + STEP_COST_USD + 1e-9) + + // The next es_* call is refused with the halt; status still reports. + const refused = await h.run(call("es_build_start"), { agent: "factory" }) + expect(refused.tools[0]?.status).toBe("error") + expect(refused.tools[0]?.text).toContain("halted") + expect(refused.tools[0]?.text).toContain("spend ceiling reached") + const stillThere = await h.run(call("es_status"), { agent: "factory" }) + expect(stillThere.tools[0]?.status).toBe("completed") + }, 120_000) +}) + +describe("STOP created mid-run refuses the next es_* call and seat launch (#184)", () => { + let h: Harness + let state: string + let factory: () => Factory + beforeAll(async () => { + ;({ h, state, factory } = await bootHost("es-stop-midrun-")) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("next es_* call and next (factory-seat) launch are refused; spend bookkeeping halts too", async () => { + await factory().provisionRun("tester", 10) + const before = await h.run(call("es_status"), { agent: "factory" }) + expect(before.tools[0]?.status).toBe("completed") + + // A human creates STOP mid-run. + await writeFile(stopOf(h), "audit halt") + + const tool = await h.run(call("es_gates_run", { scope: "full" }), { agent: "factory" }) + expect(tool.tools[0]?.status).toBe("error") + expect(tool.tools[0]?.text).toContain("stopped") + const launch = await h.run( + call("subagent", { agent: "es-qa-functional", description: "qa", prompt: "check the build" }), + { agent: "factory" }, + ) + expect(launch.tools[0]?.status).toBe("error") + expect(launch.tools[0]?.text).toContain("stopped") + + // The user's own agent keeps working (read-only). + const user = await h.run(call("read", { path: "README.md" })) + expect(user.tools[0]?.status).toBe("completed") + + // Async spend bookkeeping observes the same brake and records the halt. + const halted = await factory().recordSpend(0.001, true) + expect(halted?.stage).toBe("HALTED") + expect(halted?.halt?.reason).toContain("audit halt") + + await rm(stopOf(h)) + }, 120_000) +}) + +describe("agents can neither create nor remove STOP (#184, bug #229)", () => { + let h: Harness + let state: string + beforeAll(async () => { + ;({ h, state } = await bootHost("es-stop-delete-")) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("write/edit/patch/shell attempts on .factory/STOP are denied", async () => { + const stop = stopOf(h) + // No STOP present: creation attempts are denied and nothing appears. + const attempts = [ + call("write", { path: ".factory/STOP", content: "forged halt" }), + call("edit", { path: ".factory/STOP", oldString: "x", newString: "y" }), + call("patch", { patchText: "*** Begin Patch\n*** Add File: .factory/STOP\n+forged\n*** End Patch" }), + call("shell", { command: "echo forged > .factory/STOP" }), + call("shell", { command: "rm -f .factory/STOP" }), + ] + const { tools } = await h.run(`halt ${attempts.join(" ")}`) + expect(tools).toHaveLength(attempts.length) + for (const outcome of tools) expect(`${outcome.name}:${outcome.status}`).toBe(`${outcome.name}:error`) + expect(await exists(stop)).toBe(false) + + // STOP present (human-written): overwrite and removal attempts are + // denied — seats at the STOP gate, the user's agent at the control rule. + await mkdir(path.dirname(stop), { recursive: true }) + await writeFile(stop, "hold") + const layout = factoryLayout(h.directory) + expect(layout.stop).toBe(stop) + const seatWrite = await h.run(call("write", { path: ".factory/STOP", content: "released" }), { + agent: "factory", + }) + expect(seatWrite.tools[0]?.status).toBe("error") + const userWrite = await h.run(call("write", { path: stop, content: "released" })) + expect(userWrite.tools[0]?.status).toBe("error") + const userRemove = await h.run(call("shell", { command: "rm .factory/STOP" })) + expect(userRemove.tools[0]?.status).toBe("error") + expect(await Bun.file(stop).text()).toBe("hold") + await rm(stop) + }, 120_000) +}) From 073e366946358dcf8af5c6e8c6dc54acd002c77b Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:13:26 -0400 Subject: [PATCH 08/83] test(audit): control-file deny-write and seat scope matrix tests (#182) --- packages/core/test/control-matrix.test.ts | 151 ++++++++++ packages/opencode/test/control-matrix.test.ts | 260 ++++++++++++++++++ 2 files changed, 411 insertions(+) create mode 100644 packages/core/test/control-matrix.test.ts create mode 100644 packages/opencode/test/control-matrix.test.ts diff --git a/packages/core/test/control-matrix.test.ts b/packages/core/test/control-matrix.test.ts new file mode 100644 index 0000000..2ee99bb --- /dev/null +++ b/packages/core/test/control-matrix.test.ts @@ -0,0 +1,151 @@ +// Full seat × control-path enumeration at the unit level (#182). The sampled +// proofs live in policy.test.ts; this suite pins EVERY registry seat against +// every control glob (relative, absolute and dot-dot spellings), plus the +// shell layer split: seats are refused by the CONTROL_MENTION text rule +// except the two paths it does not name, which pass to the sandbox backstop. +// The same matrix runs against the real host in +// packages/opencode/test/control-matrix.test.ts. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS } from "../src/agents/registry.ts" +import { evaluateShell, evaluateWrite } from "../src/trust/index.ts" + +/** One representative per FACTORY_CONTROL / CONFIG_CONTROL glob (trust/control.ts). */ +const CONTROL_PATHS = [ + ".factory/gates.json", + ".factory/config.json", + ".factory/frontier.json", + ".factory/approvals/spec.json", + ".factory/waivers/lint.json", + ".factory/runtime/state.json", + ".factory/STOP", + ".factory/git-hooks/pre-commit", + ".factory/brainstorm/scores.json", + ".factory/brainstorm/runs/grill-1/ideas.json", + ".factory/harvest/plan.json", + ".factory/harvest/runs/grill-1/plan.json", + ".factory/harvest/widget/profile.json", + ".factory/harvest/clean-room/widget-search.md", + ".factory/design/tokens.json", + ".factory/design/components/button/tokens.json", + `.factory/research/sources/${"a".repeat(64)}.md`, + ".factory/research/coverage.json", + ".factory/research/dossier.json", + ".factory/research/brief.pcrb.json", + ".factory/claims/retractions.json", + ".factory/improve/proposal.json", + ".factory/audits/audit-01.json", + ".factory/scout/plan.json", + ".factory/scout/widget/profile.json", + ".factory/worktrees/phase-01/.factory/approvals/spec.json", + ".git/config", + ".git/hooks/pre-commit", + "opencode.jsonc", + ".claude/settings.json", +] + +const FACTORY_PATHS = CONTROL_PATHS.filter( + (target) => target !== "opencode.jsonc" && target !== ".claude/settings.json", +) + +/** Paths the shell text rule does not name; seats pass them to the sandbox. */ +const SANDBOX_BACKSTOP = new Set([".factory/harvest/clean-room/widget-search.md", ".factory/improve/proposal.json"]) + +let root: string +let state: string +let worktree: string + +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-matrix-core-")) + state = await mkdtemp(path.join(tmpdir(), "es-matrix-core-state-")) + worktree = path.join(root, ".factory/worktrees/p1") + await mkdir(path.join(root, ".factory/approvals"), { recursive: true }) + await mkdir(path.join(root, "src"), { recursive: true }) + await mkdir(path.join(worktree, "src"), { recursive: true }) + await writeFile(path.join(root, ".factory/approvals/spec.json"), "{}\n") + await symlink(path.join(root, ".factory/approvals"), path.join(root, "src/link")) +}) +afterAll(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const ctx = () => ({ root, worktree, stateDir: state }) + +describe("control matrix: every seat is denied every control path (#182)", () => { + test("evaluateWrite denies all seats, however the path is spelled", () => { + for (const spec of AGENTS) { + for (const target of CONTROL_PATHS) { + for (const spelling of [target, path.join(root, target), `docs/../${target}`]) { + const decision = evaluateWrite(ctx(), spec.id, spelling) + expect([spec.id, spelling, decision.effect]).toEqual([spec.id, spelling, "deny"]) + } + } + } + }) + + test("non-seat agents are denied the factory controls but keep host semantics for config", () => { + for (const agent of [undefined, "build"]) { + for (const target of FACTORY_PATHS) + expect([agent, target, evaluateWrite(ctx(), agent, target).effect]).toEqual([agent, target, "deny"]) + // Config controls ask the human for non-seat agents (host semantics). + for (const target of ["opencode.jsonc", ".claude/settings.json"]) + expect([agent, target, evaluateWrite(ctx(), agent, target).effect]).toEqual([agent, target, "ask"]) + } + }) + + test("shell: seats are refused at the text layer except the documented sandbox-backstop paths", () => { + for (const spec of AGENTS) { + for (const target of CONTROL_PATHS) { + const decision = evaluateShell(ctx(), spec.id, `echo pwned > ${target} && echo WRITE-SUCCEEDED`, { + sandboxAvailable: true, + }) + if (SANDBOX_BACKSTOP.has(target)) { + // Not named by CONTROL_MENTION: runs under the sandbox, whose + // read-only .factory bind refuses the redirect. + expect([spec.id, target, decision]).toMatchObject([spec.id, target, { effect: "allow", mode: "sandbox" }]) + } else { + expect(decision.effect).toBe("deny") + expect("reason" in decision ? decision.reason : "").toMatch(/may not reference control files/) + } + } + } + }) + + test("shell: non-seat agents are denied every control path (broad factory-dir rule)", () => { + for (const target of CONTROL_PATHS) { + const decision = evaluateShell(ctx(), undefined, `echo pwned > ${target}`, { sandboxAvailable: false }) + expect([target, decision.effect]).toEqual([target, "deny"]) + } + }) + + test("case variants are denied in the editor and the shell", () => { + const variants = [ + ".FACTORY/GATES.json", + ".factory/GATES.JSON", + ".Factory/Approvals/Spec.JSON", + ".GIT/CONFIG", + "OpenCode.JSONC", + ".CLAUDE/Settings.JSON", + ] + for (const spec of ["es-manager", "factory", "es-programmer", "es-qa-functional"]) { + for (const target of variants) { + expect([spec, target, evaluateWrite(ctx(), spec, target).effect]).toEqual([spec, target, "deny"]) + const shell = evaluateShell(ctx(), spec, `echo pwned > ${target}`, { sandboxAvailable: true }) + expect([spec, target, shell.effect]).toEqual([spec, target, "deny"]) + } + } + }) + + test("a symlink from an allowed dir into .factory cannot be used to write a control file", () => { + for (const spec of ["es-manager", "factory", "build", undefined]) { + for (const target of ["src/link/spec.json", path.join(root, "src/link/spec.json")]) { + const decision = evaluateWrite(ctx(), spec, target) + expect([spec, target, decision.effect]).toEqual([spec, target, "deny"]) + expect("reason" in decision ? decision.reason : "").toContain("factory control file") + } + } + }) +}) diff --git a/packages/opencode/test/control-matrix.test.ts b/packages/opencode/test/control-matrix.test.ts new file mode 100644 index 0000000..40d767d --- /dev/null +++ b/packages/opencode/test/control-matrix.test.ts @@ -0,0 +1,260 @@ +// Seat × control-path deny-write matrix on the real host (#182). Every +// registry seat attempts every control path through each write route (write, +// edit, patch, shell) and every attempt must be refused. The unit-level pin of +// the same matrix lives in packages/core/test/control-matrix.test.ts (and the +// sampled proofs in policy.test.ts); this suite proves it through the real +// OpenCode v2 host, on pinned 2.0.24 and under scripts/v2-head.sh. +// +// Layer notes (see packages/core/src/trust/policy.ts + sandbox.ts): +// - write/edit/patch are refused in-process by evaluateWrite (canonical path, +// symlink-resolved, case-insensitive): status error, with the per-class +// reason ("factory control file" vs "host configuration") pinned per seat in +// a sequential spot-check (parallel turns smear the first rejection's text +// across sibling outcomes — host behaviour — so parallel cells assert the +// per-cell status only). +// - shell from a seat is refused by the CONTROL_MENTION text rule for the +// paths it names (status error, no marker); the two paths it does NOT name +// (improve, harvest/clean-room markdown) run and fail without the marker. +// Which layer refused is pinned per seat in the spot-check; the disk +// (fixtures byte-identical, no stray control files) pins every cell. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm, symlink } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +/** One representative per FACTORY_CONTROL / CONFIG_CONTROL glob (trust/control.ts). */ +const CONTROL_PATHS = [ + ".factory/gates.json", + ".factory/config.json", + ".factory/frontier.json", + ".factory/approvals/spec.json", + ".factory/waivers/lint.json", + ".factory/runtime/state.json", + ".factory/STOP", + ".factory/git-hooks/pre-commit", + ".factory/brainstorm/scores.json", + ".factory/brainstorm/runs/grill-1/ideas.json", + ".factory/harvest/plan.json", + ".factory/harvest/runs/grill-1/plan.json", + ".factory/harvest/widget/profile.json", + ".factory/harvest/clean-room/widget-search.md", + ".factory/design/tokens.json", + ".factory/design/components/button/tokens.json", + `.factory/research/sources/${"a".repeat(64)}.md`, + ".factory/research/coverage.json", + ".factory/research/dossier.json", + ".factory/research/brief.pcrb.json", + ".factory/claims/retractions.json", + ".factory/improve/proposal.json", + ".factory/audits/audit-01.json", + ".factory/scout/plan.json", + ".factory/scout/widget/profile.json", + ".factory/worktrees/phase-01/.factory/approvals/spec.json", + ".git/config", + ".git/hooks/pre-commit", + "opencode.jsonc", + ".claude/settings.json", +] as const + +/** + * Control paths the shell text rule does not name (see CONTROL_MENTION in + * trust/policy.ts: it lists waivers/approvals/runtime/STOP/git-hooks/claims/ + * audits/research and the brainstorm-harvest-design-scout JSON globs, but + * neither improve nor clean-room markdown files). A seat's shell carrying one + * of these passes the text layer and is refused by the sandbox's read-only + * .factory bind instead. + */ +const SANDBOX_BACKSTOP = new Set([".factory/harvest/clean-room/widget-search.md", ".factory/improve/proposal.json"]) + +const GATES_FIXTURE = '{"version":1,"commands":[]}\n' +const APPROVALS_FIXTURE = '{"stage":"spec"}\n' + +describe("control-file deny-write matrix on the real host (#182)", () => { + let h: Harness + let state: string + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-matrix-state-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { + "README.md": "# matrix\n", + "docs/notes.md": "# notes\n", + ".factory/gates.json": GATES_FIXTURE, + ".factory/approvals/spec.json": APPROVALS_FIXTURE, + }, + }) + }, 180_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + for (const spec of AGENTS) { + test(`${spec.id}: write, edit, patch and shell refuse every control path`, async () => { + const agent = spec.id + const batch = async (attempts: string[]) => { + const { tools } = await h.run(`@@PARALLEL@@ ${attempts.join(" ")}`, { agent }) + expect(tools).toHaveLength(attempts.length) + return tools + } + + // Reason texts are pinned in the sequential spot-check below: parallel + // turns smear the first rejection's text across sibling outcomes (host + // behaviour), so parallel cells assert the per-cell status only. + for (const tool of ["write", "edit"] as const) { + const tools = await batch(CONTROL_PATHS.map((target) => call(tool, { path: target, content: "pwned" }))) + const byPath = new Map(tools.map((outcome) => [(outcome.input as { path?: string })?.path, outcome])) + expect([...byPath.keys()].sort()).toEqual([...CONTROL_PATHS].sort()) + for (const target of CONTROL_PATHS) + expect(`${target}:${byPath.get(target)!.name}:${byPath.get(target)!.status}`).toBe(`${target}:${tool}:error`) + } + + const patches = await batch( + CONTROL_PATHS.map((target) => + call("patch", { patchText: `*** Begin Patch\n*** Add File: ${target}\n+pwned\n*** End Patch` }), + ), + ) + const byPatch = new Map(patches.map((outcome) => [(outcome.input as { patchText?: string })?.patchText, outcome])) + expect(byPatch.size).toBe(CONTROL_PATHS.length) + for (const target of CONTROL_PATHS) { + const outcome = byPatch.get(`*** Begin Patch\n*** Add File: ${target}\n+pwned\n*** End Patch`)! + expect(`${target}:${outcome.name}:${outcome.status}`).toBe(`${target}:patch:error`) + } + + const shells = await batch( + CONTROL_PATHS.map((target) => call("shell", { command: `echo pwned > ${target} && echo WRITE-SUCCEEDED` })), + ) + const byCommand = new Map(shells.map((outcome) => [(outcome.input as { command?: string })?.command, outcome])) + expect(byCommand.size).toBe(CONTROL_PATHS.length) + for (const target of CONTROL_PATHS) { + const outcome = byCommand.get(`echo pwned > ${target} && echo WRITE-SUCCEEDED`)! + expect(`${target}:${outcome.name}`).toBe(`${target}:shell`) + expect(outcome.text).not.toContain("WRITE-SUCCEEDED") + // Text-named paths are refused before execution (status error); the + // sandbox-backstop paths run and fail without the marker (their shell + // UX status is host- and kernel-dependent, so the marker and the disk + // pin them, not the status). Which layer refused is pinned per seat in + // the sequential spot-check below. + if (!SANDBOX_BACKSTOP.has(target)) expect(`${target}:${outcome.status}`).toBe(`${target}:error`) + } + + // Sequential spot-check: parallel turns smear the first rejection's text + // across sibling outcomes (host behaviour), so the reason per path class + // is pinned here with correct attribution, one turn per call. + const spot = await h.run( + [ + call("write", { path: ".factory/gates.json", content: "pwned" }), + call("write", { path: "opencode.jsonc", content: "pwned" }), + call("shell", { command: "echo pwned > .factory/gates.json && echo WRITE-SUCCEEDED" }), + ].join(" "), + { agent }, + ) + expect(spot.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual([ + "write:error", + "write:error", + "shell:error", + ]) + expect(spot.tools[0]?.text).toContain("factory control file") + expect(spot.tools[1]?.text).toContain("host configuration") + expect(spot.tools[2]?.text).toMatch(/may not reference control files/) + expect(spot.tools[2]?.text).not.toContain("WRITE-SUCCEEDED") + + // Nothing on disk moved, for any seat. + expect(await readFile(path.join(h.directory, ".factory/gates.json"), "utf8")).toBe(GATES_FIXTURE) + expect(await readFile(path.join(h.directory, ".factory/approvals/spec.json"), "utf8")).toBe(APPROVALS_FIXTURE) + }, 180_000) + } + + test("no control file escaped onto disk anywhere", async () => { + for (const target of CONTROL_PATHS) { + if (target === ".factory/gates.json" || target === ".factory/approvals/spec.json") continue + if (target === ".git/config" || target === ".git/hooks/pre-commit") continue + expect(await Bun.file(path.join(h.directory, target)).exists()).toBe(false) + } + }) +}) + +describe("symlink and case-variant bypasses on the real host (#182)", () => { + let h: Harness + let state: string + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-matrix-bypass-state-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { + "README.md": "# bypass\n", + "docs/notes.md": "# notes\n", + ".factory/gates.json": GATES_FIXTURE, + ".factory/approvals/spec.json": APPROVALS_FIXTURE, + }, + }) + // A symlink from the manager's allowed dir (docs/) into .factory/approvals/. + await symlink(path.join(h.directory, ".factory/approvals"), path.join(h.directory, "docs/link")) + }, 180_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("edit/write resolve the symlink and refuse at the policy layer", async () => { + const { tools } = await h.run( + `@@PARALLEL@@ ${call("edit", { path: "docs/link/spec.json", content: "pwned" })} ${call("write", { path: "docs/link/spec.json", content: "pwned" })}`, + { agent: "es-manager" }, + ) + // Matched by input: parallel turns keep per-cell statuses but smear the + // first rejection's text, and may reorder outcomes. + const byInput = new Map(tools.map((tool) => [`${tool.name}:${(tool.input as { path?: string })?.path}`, tool])) + for (const tool of ["edit", "write"]) { + const outcome = byInput.get(`${tool}:docs/link/spec.json`)! + expect(outcome.status).toBe("error") + expect(outcome.text).toContain("factory control file") + } + expect(await readFile(path.join(h.directory, ".factory/approvals/spec.json"), "utf8")).toBe(APPROVALS_FIXTURE) + }) + + test("a shell through the symlink passes the text layer and never executes the write", async () => { + // "docs/link/spec.json" names no control file, so the text rule allows it; + // enforcement is the sandbox around the seat's shell. On a healthy host + // the read-only .factory bind fails the redirect (EROFS); on hosts where + // bubblewrap cannot construct the sandbox (this container's kernel rejects + // its overlay mounts) the command never runs at all — fail-closed either + // way. The && chain proves the redirect itself never succeeded, and the + // refusal text proves the text layer was not the enforcer. + const { tools } = await h.run( + `link ${call("shell", { command: "echo pwned > docs/link/spec.json && echo WRITE-SUCCEEDED" })}`, + { agent: "es-manager" }, + ) + expect(tools).toHaveLength(1) + expect(tools[0]?.name).toBe("shell") + expect(tools[0]?.text).not.toContain("WRITE-SUCCEEDED") + expect(tools[0]?.text).not.toMatch(/may not reference control files/) + expect(await readFile(path.join(h.directory, ".factory/approvals/spec.json"), "utf8")).toBe(APPROVALS_FIXTURE) + }) + + test("case-variant edit and shell are refused end to end", async () => { + // Sequential (one turn per call): parallel turns smear the first + // rejection's text across sibling outcomes, which would hide which layer + // refused the shell. + const { tools } = await h.run( + `${call("edit", { path: ".FACTORY/GATES.json", content: "pwned" })} ${call("shell", { command: "echo pwned > .FACTORY/GATES.json && echo WRITE-SUCCEEDED" })}`, + { agent: "es-manager" }, + ) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["edit:error", "shell:error"]) + expect(tools[0]?.text).toContain("factory control file") + expect(tools[1]?.text).toMatch(/may not reference control files/) + expect(tools[1]?.text).not.toContain("WRITE-SUCCEEDED") + expect(await readFile(path.join(h.directory, ".factory/gates.json"), "utf8")).toBe(GATES_FIXTURE) + expect(await Bun.file(path.join(h.directory, ".FACTORY/GATES.json")).exists()).toBe(false) + }) +}) From cfaad81601f72588ce12cd0b61c2f9158f62cf93 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:13:44 -0400 Subject: [PATCH 09/83] test(audit): agent registry and scoping integration tests (#195) --- evals/cases/registry-hidden-spawn.json | 8 + packages/core/test/registry-spawns.test.ts | 41 ++++++ .../opencode/test/registry-discipline.test.ts | 137 ++++++++++++++++++ .../opencode/test/registry-listing.test.ts | 123 ++++++++++++++++ packages/opencode/test/registry-spawn.test.ts | 79 ++++++++++ 5 files changed, 388 insertions(+) create mode 100644 evals/cases/registry-hidden-spawn.json create mode 100644 packages/core/test/registry-spawns.test.ts create mode 100644 packages/opencode/test/registry-discipline.test.ts create mode 100644 packages/opencode/test/registry-listing.test.ts create mode 100644 packages/opencode/test/registry-spawn.test.ts diff --git a/evals/cases/registry-hidden-spawn.json b/evals/cases/registry-hidden-spawn.json new file mode 100644 index 0000000..b752e88 --- /dev/null +++ b/evals/cases/registry-hidden-spawn.json @@ -0,0 +1,8 @@ +{ + "id": "registry-hidden-spawn", + "description": "The factory delegates a status check to the hidden es-qa-functional seat by id (hidden seats are unlisted from the subagent menu but invocable from an allowed parent), then replies DELEGATED.", + "agent": "factory", + "prompt": "Delegate a status check to the es-qa-functional seat with the subagent tool (description \"status check\", prompt \"Check the factory run status with es_status and report back in one sentence\"). When it returns, reply with exactly the single word DELEGATED and nothing else.", + "maxSteps": 8, + "checks": { "toolsUsed": ["subagent"], "contains": ["DELEGATED"] } +} diff --git a/packages/core/test/registry-spawns.test.ts b/packages/core/test/registry-spawns.test.ts new file mode 100644 index 0000000..75404de --- /dev/null +++ b/packages/core/test/registry-spawns.test.ts @@ -0,0 +1,41 @@ +// Registry spawn coverage (#195.2): the static precondition behind the +// live-host spawn matrix — every hidden seat has at least one allowed parent, +// every spawns entry resolves, and hidden means subagent mode. +import { describe, expect, test } from "bun:test" +import { AGENTS } from "../src/index.ts" + +const byId = new Map(AGENTS.map((agent) => [agent.id, agent])) +const hidden = AGENTS.filter((agent) => agent.hidden) +const spawnersOf = (seat: string) => AGENTS.filter((agent) => agent.spawns.includes(seat)).map((agent) => agent.id) + +describe("registry spawn coverage (#195)", () => { + test("hidden seats are exactly the subagent-mode agents", () => { + expect(hidden.map((agent) => agent.id).sort()).toEqual( + AGENTS.filter((agent) => agent.mode === "subagent") + .map((agent) => agent.id) + .sort(), + ) + }) + + test("every spawns entry resolves to a registry agent; no seat spawns itself", () => { + for (const agent of AGENTS) { + for (const target of agent.spawns) expect([agent.id, target, byId.has(target)]).toEqual([agent.id, target, true]) + expect([agent.id, agent.spawns.includes(agent.id)]).toEqual([agent.id, false]) + } + }) + + test("every hidden seat has at least one allowed parent", () => { + for (const seat of hidden) + expect([seat.id, spawnersOf(seat.id).length > 0, spawnersOf(seat.id)]).toEqual([ + seat.id, + true, + spawnersOf(seat.id), + ]) + }) + + test("the research seats are reachable from the factory and the deep researcher; the synthesizer only from the latter", () => { + expect(spawnersOf("es-research-alpha").sort()).toEqual(["deep-researcher", "factory"]) + expect(spawnersOf("es-research-beta").sort()).toEqual(["deep-researcher", "factory"]) + expect(spawnersOf("es-research-synthesizer")).toEqual(["deep-researcher"]) + }) +}) diff --git a/packages/opencode/test/registry-discipline.test.ts b/packages/opencode/test/registry-discipline.test.ts new file mode 100644 index 0000000..0ebbfc8 --- /dev/null +++ b/packages/opencode/test/registry-discipline.test.ts @@ -0,0 +1,137 @@ +// Seat launch discipline (#195.2, #60, #228.3): background launches are refused +// and a running seat is never relaunched. Live-host where the harness allows +// (background refusal, clean relaunch after completion); the running-seat +// refusal runs the real tracker and the real guard against synthetic host +// events, because a fake-LLM seat completes in milliseconds and a live +// still-running window cannot be held deterministically. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { createPolicyHooks } from "../src/policy.ts" +import { createSeatTracker, type SeatTracker } from "../src/seats.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +const hiddenSeats = AGENTS.filter((agent) => agent.hidden) + +describe("seat launch discipline on the real host (#195)", () => { + let h: Harness + let state: string + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-discipline-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# discipline\n" }, + }) + }, 120_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("a background launch is refused from deep-researcher", async () => { + const { tools } = await h.run( + `go ${call("subagent", { agent: "es-research-alpha", description: "alpha", prompt: "scan", background: true })}`, + { agent: "deep-researcher" }, + ) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["subagent:error"]) + expect(tools[0]?.text).toContain("Factory seats run in the foreground") + }, 30_000) + + test("a background launch is refused from grill", async () => { + const { tools } = await h.run( + `go ${call("subagent", { agent: "es-brainstorm-critic", description: "critic", prompt: "score", background: true })}`, + { agent: "grill" }, + ) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["subagent:error"]) + expect(tools[0]?.text).toContain("Factory seats run in the foreground") + }, 30_000) + + test("a completed seat relaunches cleanly (no stale refusal)", async () => { + for (const round of ["first", "second"]) { + const { tools } = await h.run( + `${round} ${call("subagent", { agent: "es-qa-functional", description: round, prompt: "reply ok" })}`, + { agent: "factory" }, + ) + expect([round, ...tools.map((tool) => `${tool.name}:${tool.status}`)]).toEqual([round, "subagent:completed"]) + } + }, 60_000) +}) + +describe("the running-seat guard (real tracker, synthetic host events)", () => { + let root: string + beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-relaunch-")) + }) + afterAll(() => rm(root, { recursive: true, force: true })) + + const runtime = () => ({ + root, + stateDir: root, + factory: { read: async () => undefined }, + policy: async () => ({ root, stateDir: root }), + }) + const setup = () => { + const tracker: SeatTracker = createSeatTracker(runtime() as any) + const hooks = createPolicyHooks(runtime() as any, tracker) + const agentOf = async () => undefined + return { tracker, hooks, agentOf } + } + const launch = ( + hooks: ReturnType, + agent: string, + target: string, + extra: Record = {}, + ) => + hooks.before({ + tool: "subagent", + agent, + id: "t1", + sessionID: "ses_factory", + input: { agent: target, description: "check", prompt: "go", ...extra }, + }) + + for (const seat of hiddenSeats) { + test(`${seat.id} cannot be relaunched while running, and launches after it completes`, async () => { + const { tracker, hooks, agentOf } = setup() + const sessionID = `ses_${seat.id.replace(/[^a-z]/g, "")}` + await tracker.onEvent( + { type: "session.created", data: { sessionID, parentID: "ses_factory", agent: seat.id } }, + agentOf, + ) + await expect(launch(hooks, "factory", seat.id)).rejects.toThrow( + `${seat.id} is still running (session ${sessionID}`, + ) + await tracker.onEvent({ type: "session.execution.succeeded", data: { sessionID } }, agentOf) + await expect(launch(hooks, "factory", seat.id)).resolves.toBeUndefined() + }) + } + + test("a different seat may launch while one runs", async () => { + const { tracker, hooks, agentOf } = setup() + await tracker.onEvent( + { + type: "session.created", + data: { sessionID: "ses_alpha", parentID: "ses_factory", agent: "es-research-alpha" }, + }, + agentOf, + ) + await expect(launch(hooks, "factory", "es-research-beta")).resolves.toBeUndefined() + }) + + for (const spec of AGENTS) { + test(`background:true is refused from seat ${spec.id}`, async () => { + const { hooks } = setup() + await expect(launch(hooks, spec.id, "es-qa-functional", { background: true })).rejects.toThrow( + "Factory seats run in the foreground", + ) + }) + } +}) diff --git a/packages/opencode/test/registry-listing.test.ts b/packages/opencode/test/registry-listing.test.ts new file mode 100644 index 0000000..c397596 --- /dev/null +++ b/packages/opencode/test/registry-listing.test.ts @@ -0,0 +1,123 @@ +// Registry listing on the real host (#195.2, #228.2): every registry agent is +// registered with its hidden flag and mode; hidden seats are unlisted from the +// host's subagent menu (the model-visible agent listing) yet addressable by +// id; and a vanilla-baseline diff proves built-ins and defaults are untouched. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS } from "@heretek-ai/es-core" +import { boot, type Harness, lastAgentRequest } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const hiddenIds = AGENTS.filter((agent) => agent.hidden).map((agent) => agent.id) +const agentOf = (h: Harness) => (h.opencode as any).agent +/** Per-boot host tmpdirs (config dir, project dir) differ between hosts; they + * are host-generated, never plugin input. Normalize them before diffing. */ +const normalize = (value: unknown) => + value === undefined + ? value + : JSON.parse(JSON.stringify(value).replace(/\/tmp\/es-host(-config)?-[A-Za-z0-9]+/g, "/tmp/HOSTTMP")) +const list = async (h: Harness) => + (await agentOf(h).list({ location: { directory: h.directory } } as any))?.data as Array<{ + id: string + mode: string + hidden: boolean + description: string + permissions: unknown + model?: unknown + }> + +describe("registry listing on the real host (#195)", () => { + let h: Harness + let clean: Harness + let state: string + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-listing-state-")) + ;[h, clean] = await Promise.all([ + boot({ + git: true, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# listing\n" }, + }), + boot({ git: true, files: { "README.md": "# vanilla\n" } }), + ]) + // Agent state is location-scoped and populated lazily: one session per + // host before listing, or `agent.list` comes back empty. + await h.run("hello", { agent: "factory" }) + await clean.run("hello") + }, 180_000) + afterAll(async () => { + await Promise.all([h?.close(), clean?.close()].filter(Boolean)) + await rm(state, { recursive: true, force: true }) + }) + + test("every registry agent is registered with its hidden flag and mode", async () => { + const listed = await list(h) + expect(listed.map((agent) => agent.id).sort()).toEqual(expect.arrayContaining(AGENTS.map((agent) => agent.id))) + for (const spec of AGENTS) { + const info = listed.find((agent) => agent.id === spec.id) + expect([spec.id, info?.hidden, info?.mode]).toEqual([spec.id, spec.hidden, spec.mode]) + } + }, 60_000) + + test("hidden seats are addressable by id (invocable, not absent)", async () => { + for (const id of hiddenIds) { + const info = await agentOf(h).get({ agentID: id, location: { directory: h.directory } } as any) + expect([id, info?.data?.hidden]).toEqual([id, true]) + } + }, 60_000) + + test("hidden seats are unlisted from a spawning seat's subagent menu", async () => { + await h.run("hello", { agent: "factory" }) + const request = lastAgentRequest(h.llm.requests) + const subagent = (request?.tools ?? []).find((tool) => tool.function.name === "subagent") + expect(subagent).toBeDefined() + // The factory can spawn hidden seats, so none may be named: everything it + // may launch is hidden, and the host omits hidden agents from the menu. + expect(subagent?.function.description ?? "").not.toContain("Available subagents:") + for (const id of hiddenIds) expect([id, (subagent?.function.description ?? "").includes(id)]).toEqual([id, false]) + }, 60_000) + + test("the user agent's menu lists non-hidden subagents but no hidden seat", async () => { + await h.run("hello", { agent: "build" }) + const request = lastAgentRequest(h.llm.requests) + const subagent = (request?.tools ?? []).find((tool) => tool.function.name === "subagent") + // Non-vacuous: the menu renders (built-in non-hidden subagents are + // listed), yet no hidden seat leaks into it. + expect(subagent?.function.description ?? "").toContain("Available subagents:") + for (const id of hiddenIds) expect([id, (subagent?.function.description ?? "").includes(id)]).toEqual([id, false]) + }, 60_000) + + test("vanilla baseline: built-ins and defaults are untouched, our agents are purely additive", async () => { + const [withPlugin, vanilla] = await Promise.all([list(h), list(clean)]) + const vanillaById = new Map(vanilla.map((agent) => [agent.id, agent])) + expect(vanillaById.size).toBeGreaterThan(0) + // Every vanilla agent is byte-equal on the fields the plugin could touch. + for (const base of vanilla) { + const info = withPlugin.find((agent) => agent.id === base.id) + expect(info?.id).toBe(base.id) + expect({ + mode: info?.mode, + hidden: info?.hidden, + description: normalize(info?.description), + permissions: normalize(info?.permissions), + model: info?.model, + }).toEqual({ + mode: base.mode, + hidden: base.hidden, + description: normalize(base.description), + permissions: normalize(base.permissions), + model: base.model, + }) + } + // The default is unchanged (the host lists the selected default first). + expect([vanilla[0]?.id, withPlugin[0]?.id]).toEqual(["build", "build"]) + // The only additions are exactly the registry ids. + const added = withPlugin.map((agent) => agent.id).filter((id) => !vanillaById.has(id)) + expect(added.sort()).toEqual(AGENTS.map((agent) => agent.id).sort()) + // And the clean host knows none of our seats. + await expect(agentOf(clean).get({ agentID: "factory" } as any)).rejects.toThrow("Agent not found") + }, 60_000) +}) diff --git a/packages/opencode/test/registry-spawn.test.ts b/packages/opencode/test/registry-spawn.test.ts new file mode 100644 index 0000000..0fc04e0 --- /dev/null +++ b/packages/opencode/test/registry-spawn.test.ts @@ -0,0 +1,79 @@ +// Seat spawn rules on the real host (#195.2): every hidden seat launches from +// an allowed parent (proving "invocable only from allowed parents" positively) +// and is refused from a disallowed one. The child is verified to run as the +// requested seat through the host's own session record. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS, agentSpec } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const child = (name: string, args: Record = {}) => `@@CHILD ${name} ${JSON.stringify(args)}@@` + +const hiddenSeats = AGENTS.filter((agent) => agent.hidden) +/** A parent the registry allows to launch `seat` (factory wherever possible). */ +const allowedParent = (seat: string) => { + const parents = AGENTS.filter((agent) => agent.spawns.includes(seat)).map((agent) => agent.id) + if (parents.includes("factory")) return "factory" + return parents[0]! +} +/** A seat parent the registry never allows to launch anything. */ +const DISALLOWED_PARENT = "es-qa-functional" + +const childPrompt = (seat: string) => + agentSpec(seat)?.tools.includes("es_status") ? `check the run ${child("es_status", {})} and report back` : "reply ok" + +const childSessionOf = (text: string) => /sessionID="([^"]+)"/.exec(text)?.[1] + +describe("seat spawn rules on the real host (#195)", () => { + let h: Harness + let state: string + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-spawn-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# spawns\n" }, + }) + }, 120_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + describe("an allowed parent may launch the seat", () => { + for (const seat of hiddenSeats) { + const parent = allowedParent(seat.id) + test(`${parent} launches ${seat.id}`, async () => { + const { tools } = await h.run( + `delegate ${call("subagent", { agent: seat.id, description: "seat check", prompt: childPrompt(seat.id) })}`, + { agent: parent }, + ) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["subagent:completed"]) + if (agentSpec(seat.id)?.tools.includes("es_status")) expect(tools[0]?.text).toContain("No factory run") + // The host ran the child as the requested seat, parented to the launcher. + const sessionID = childSessionOf(tools[0]?.text ?? "") + expect(sessionID).toBeDefined() + const info: any = await (h.opencode as any).sessions.get({ sessionID } as any) + expect(info?.agent).toBe(seat.id) + }, 30_000) + } + }) + + describe("a disallowed parent is refused", () => { + for (const seat of hiddenSeats) { + test(`${DISALLOWED_PARENT} cannot launch ${seat.id}`, async () => { + const { tools } = await h.run( + `delegate ${call("subagent", { agent: seat.id, description: "escape", prompt: "escape" })}`, + { agent: DISALLOWED_PARENT }, + ) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["subagent:error"]) + }, 30_000) + } + }) +}) From a3c11168459dc4d8e01fbbcf7019a7cb9c7c3e21 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:37:54 -0400 Subject: [PATCH 10/83] test(audit): code-audit pair integration tests (#186) --- packages/core/test/codeaudit-186.test.ts | 107 +++++++++ packages/opencode/test/audit-186.test.ts | 270 +++++++++++++++++++++++ 2 files changed, 377 insertions(+) create mode 100644 packages/core/test/codeaudit-186.test.ts create mode 100644 packages/opencode/test/audit-186.test.ts diff --git a/packages/core/test/codeaudit-186.test.ts b/packages/core/test/codeaudit-186.test.ts new file mode 100644 index 0000000..618a1e4 --- /dev/null +++ b/packages/core/test/codeaudit-186.test.ts @@ -0,0 +1,107 @@ +// Issue #186 integration pins, core side (bug #241 closers): the verifySpan +// short-excerpt branch at es_audit_verdict level. The schema keeps excerpt +// min(1); the 12-char witness in verifySpan is what refuses — this file pins +// that current behavior without weakening either. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, writeFile } from "node:fs/promises" +import path from "node:path" +import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { codeAuditTools } from "../src/codeaudit/index.ts" +import { Factory } from "../src/factory/index.ts" +import { factoryLayout } from "../src/layout.ts" +import { esTools } from "../src/ops/tools.ts" +import { verifySpan } from "../src/research/quote.ts" +import { AuditFindingSchema } from "../src/schema/codeaudit.ts" +import { type Fixture, gitRepo } from "./helpers.ts" + +const AUTH = [ + "export function login(db: Db, user: string) {", + ` const sql = "SELECT * FROM users WHERE name = '" + user + "'"`, + " return db.query(sql)", + "}", + "", +].join("\n") + +let fx: Fixture +let factory: Factory +const PASSPHRASE = "test-passphrase-1234" +let signer: HumanSigner +beforeEach(async () => { + fx = await gitRepo("es-audit-186-") + await sealHumanKey(PASSPHRASE, fx.state) + signer = await unlockHumanKey(PASSPHRASE, fx.state) + void signer + factory = new Factory(fx.root, { + gates: async () => ({ passed: true, findings: [], summary: "green" }), + stateDir: fx.state, + }) + await mkdir(path.join(fx.root, "src"), { recursive: true }) + await writeFile(path.join(fx.root, "src/auth.ts"), AUTH) +}) +afterEach(() => fx.cleanup()) + +const tools = () => { + const ops = { root: fx.root, factory, stateDir: fx.state } + const all = [...codeAuditTools(ops), ...esTools(ops)] + return (name: string) => all.find((tool) => tool.name === name)! +} + +const injection = { + kind: "vulnerability", + title: "SQL built by string concatenation", + severity: "high", + cwe: "CWE-89", + file: "src/auth.ts", + lines: [2, 3], + excerpt: `"SELECT * FROM users WHERE name = '" + user`, + detail: "user = \"' OR '1'='1\" returns every row; the query is concatenated, not parameterised.", + remediation: "Use db.query('SELECT * FROM users WHERE name = ?', [user]).", +} + +async function openPathAudit() { + return (await factory.openAudit("human:tester", { kind: "path", path: "src" }, { ceilingUSD: 5 })).audit +} + +describe("verifySpan short branch (#241)", () => { + test("an 11-char span is too short; a 12-char verbatim span is evidence", () => { + const source = "return db.query(sql)" + expect(verifySpan(source, "db.query(sq")).toEqual({ + ok: false, + reason: "too short to be evidence (need at least 12 characters)", + }) + expect(verifySpan(source, "db.query(sql")).toEqual({ ok: true, index: 7 }) + expect(verifySpan(source, "db.query(no)").ok).toBe(false) + }) + + test("the schema still accepts a short excerpt; the witness is what refuses", () => { + // #241 item 2 premise: schema/codeaudit.ts promises "at least 12 + // characters" in the comment but enforces min(1). Pin the behavior: + // parsing succeeds, so only the witness stands in the way. + const parsed = AuditFindingSchema.safeParse({ ...injection, lines: [3, 3], excerpt: "db.query" }) + expect(parsed.success).toBe(true) + }) +}) + +describe("short excerpts at es_audit_verdict (#241)", () => { + test("a <12-char excerpt refuses the whole verdict with 'too short to be evidence'; nothing is recorded", async () => { + const audit = await openPathAudit() + const verdict = tools()("es_audit_verdict") + const short = { ...injection, lines: [3, 3], excerpt: "db.query" } + await expect( + verdict.execute( + { audit: audit.id, verdict: "fail", findings: [short], notes: "n" }, + { agent: "es-auditor-antithesis" }, + ), + ).rejects.toThrow("too short to be evidence") + // One short excerpt among good findings still refuses everything. + await expect( + verdict.execute( + { audit: audit.id, verdict: "fail", findings: [injection, short], notes: "n" }, + { agent: "es-auditor-antithesis" }, + ), + ).rejects.toThrow("1 finding(s) do not point at real code") + // Nothing reached the state, the records or the dossier. + expect((await factory.read())!.audits[0]!.antithesis).toBeUndefined() + expect(await Bun.file(factoryLayout(fx.root).auditRecords(audit.id)).exists()).toBe(false) + }) +}) diff --git a/packages/opencode/test/audit-186.test.ts b/packages/opencode/test/audit-186.test.ts new file mode 100644 index 0000000..ac7188d --- /dev/null +++ b/packages/opencode/test/audit-186.test.ts @@ -0,0 +1,270 @@ +// Issue #186 integration tests on the real v2 host (merge-blocking, +// deterministic): scripted seats drive the real plugin tools. +// - An auditor finding quoting a line that does not exist refuses the whole +// verdict (nothing recorded); one that exists is accepted and blocks the +// next stage transition (passPhase gating). +// - audit.phase required with no audit opened holds passPhase: a QA-passed +// phase waits instead of merging. +// - Bug #241: a <12-char excerpt is refused at verdict level ("too short to +// be evidence"); verifySpan and the schema are unchanged (pinned in +// packages/core/test/codeaudit-186.test.ts). +// Live-model runs (#139, 5/5) are out of scope: no paid models here. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + approveStage, + Factory, + factoryLayout, + gateRunner, + type HumanSigner, + readAuditRecords, + recordApproval, + researchSourcesDir, + SourceCache, + sealHumanKey, + stringifyFrontmatter, + unlockHumanKey, +} from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +const AUTH = [ + "export function login(db: Db, user: string) {", + ` const sql = "SELECT * FROM users WHERE name = '" + user + "'"`, + " return db.query(sql)", + "}", + "", +].join("\n") + +const injection = { + kind: "vulnerability", + title: "SQL built by string concatenation", + severity: "high", + cwe: "CWE-89", + file: "src/auth.ts", + lines: [2, 3], + excerpt: `"SELECT * FROM users WHERE name = '" + user`, + detail: "user = \"' OR '1'='1\" returns every row; the query is concatenated, not parameterised.", + remediation: "Use db.query('SELECT * FROM users WHERE name = ?', [user]).", +} +const invariant = { + kind: "invariant", + title: "Queries go through the db handle", + severity: "info", + file: "src/auth.ts", + lines: [3, 3], + excerpt: "return db.query(sql)", + detail: "All database access in this module uses the injected db handle.", + holds: true, +} + +const write = async (root: string, file: string, text: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), text) +} + +async function bootHost(prefix: string, audit?: { phase: "required" }) { + const state = await mkdtemp(path.join(tmpdir(), prefix)) + await sealHumanKey("test-passphrase-1234", state) + const signer = await unlockHumanKey("test-passphrase-1234", state) + const h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off", ...(audit ? { audit } : {}) } }], + files: { "README.md": "# audit-186\n" }, + }) + const factory = () => new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + return { state, h, factory, signer } +} + +describe("audit-186 path audit on the host", () => { + let h: Harness + let state: string + let factory: () => Factory + let signer: HumanSigner + beforeAll(async () => { + ;({ h, state, factory, signer } = await bootHost("es-audit-186-")) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("hallucinated and short findings are refused whole; witnessed ones are accepted and block", async () => { + await write(h.directory, "src/auth.ts", AUTH) + await factory().provisionRun("tester", 5) + const opened = await h.run(call("es_audit_open", { target: "src" }), { agent: "factory" }) + expect([opened.tools[0]?.status, opened.tools[0]?.text]).toEqual([ + "completed", + expect.stringContaining("Opened audit-01"), + ]) + + const verdict = (agent: string, findings: unknown[], v = "fail") => + h.run(call("es_audit_verdict", { audit: "audit-01", verdict: v, findings, notes: "n" }), { agent }) + + // A line past the end of the file: refused, nothing recorded. + const pastEnd = await verdict("es-auditor-antithesis", [{ ...injection, lines: [2, 40] }]) + expect([pastEnd.tools[0]?.status, pastEnd.tools[0]?.text]).toEqual([ + "error", + expect.stringContaining("past the end of the file (4 lines)"), + ]) + expect(await readAuditRecords(h.directory, "audit-01")).toEqual([]) + + // A file that does not exist: refused, nothing recorded. + const missing = await verdict("es-auditor-antithesis", [{ ...injection, file: "src/session.ts" }]) + expect([missing.tools[0]?.status, missing.tools[0]?.text]).toEqual([ + "error", + expect.stringContaining("does not exist"), + ]) + expect(await readAuditRecords(h.directory, "audit-01")).toEqual([]) + + // One bad finding among good ones still refuses the whole verdict (#241 whole-verdict law). + const mixed = await verdict("es-auditor-antithesis", [injection, { ...injection, lines: [9, 9] }]) + expect([mixed.tools[0]?.status, mixed.tools[0]?.text]).toEqual([ + "error", + expect.stringContaining("1 finding(s) do not point at real code"), + ]) + expect(await readAuditRecords(h.directory, "audit-01")).toEqual([]) + + // Bug #241: a <12-char excerpt is refused at verdict level; the schema + // (min 1) is untouched, the witness refuses. + const short = await verdict("es-auditor-antithesis", [{ ...injection, lines: [3, 3], excerpt: "db.query" }]) + expect([short.tools[0]?.status, short.tools[0]?.text]).toEqual([ + "error", + expect.stringContaining("too short to be evidence"), + ]) + expect(await readAuditRecords(h.directory, "audit-01")).toEqual([]) + + // Findings that exist: accepted. + const thesis = await verdict("es-auditor-thesis", [invariant], "pass") + expect(thesis.tools[0]?.status).toBe("completed") + const antithesis = await verdict("es-auditor-antithesis", [injection]) + expect(antithesis.tools[0]?.status).toBe("completed") + const records = await readAuditRecords(h.directory, "audit-01") + expect(records.map((record) => [record.seat, record.verdict, record.findings.length])).toEqual([ + ["auditor-thesis", "pass", 1], + ["auditor-antithesis", "fail", 1], + ]) + + // The split goes to the manager; a fail blocks the next transition. + const tiebreak = await h.run( + call("es_tiebreak", { verdict: "fail", notes: "the injection is real", audit: "audit-01" }), + { + agent: "es-manager", + }, + ) + expect([tiebreak.tools[0]?.status, tiebreak.tools[0]?.text]).toEqual([ + "completed", + expect.stringContaining("failed"), + ]) + await factory().writeFrontier("grill", { + version: "1.1", + idea: "A greeting library", + spendCeiling: { currency: "USD", maxAmount: 25 }, + settled: true, + nodes: [{ id: "lang", question: "Language?", answer: "TypeScript", status: "settled" }], + }) + // recordApproval (not approveStage): approveStage would auto-begin + // research, which the failed audit must refuse — that refusal is what + // the final assertion proves, from a clean GRILL run. + await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", signer }) + await expect(factory().beginResearch("human:tester")).rejects.toThrow("audit-01 on path src is failed") + }, 120_000) +}) + +describe("audit-186 audit.phase required on the host", () => { + let h: Harness + let state: string + let factory: () => Factory + let signer: HumanSigner + beforeAll(async () => { + ;({ h, state, factory, signer } = await bootHost("es-audit-186-required-", { phase: "required" })) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("a QA-passed phase with no audit opened waits instead of merging (passPhase held)", async () => { + await factory().provisionRun("tester", 10) + await factory().writeFrontier("grill", { + version: "1.1", + idea: "A greeting library", + spendCeiling: { currency: "USD", maxAmount: 10 }, + settled: true, + nodes: [{ id: "lang", question: "Language?", answer: "TypeScript", status: "settled" }], + }) + await approveStage(h.directory, { + stage: "frontier", + channel: "cli", + approvedBy: "tester", + signer, + stateDir: state, + }) + const cache = new SourceCache(researchSourcesDir(h.directory), state) + const source = await cache.put({ + url: "https://example.test/greeting", + text: "Greeting libraries usually expose a single greet function.", + provider: "fetch", + }) + const layout = factoryLayout(h.directory) + await write( + h.directory, + path.relative(h.directory, layout.researchReport), + `- A greet function is the conventional API [VERIFIED: sha256:${source.meta.sha256} "expose a single greet function"]\n`, + ) + await factory().completeResearch("factory") + await write( + h.directory, + ".factory/roadmap.json", + JSON.stringify({ version: 1, title: "Greeting", phases: [{ id: "alpha", title: "Phase alpha" }] }, null, 2), + ) + await write( + h.directory, + ".factory/specs/alpha/GOAL.md", + stringifyFrontmatter( + { + phase: "alpha", + title: "Phase alpha", + acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }], + }, + "Implement alpha as a vertical slice.\n", + ), + ) + await approveStage(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer, stateDir: state }) + await factory().startBuild("factory") + + // Programmer: failing test first (red), then the implementation, then complete. + const wt = (await factory().activeWorktree())! + await write( + h.directory, + path.relative(h.directory, path.join(wt, "test/alpha.test.ts")), + `test("alpha", () => {})\n`, + ) + await factory().recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await write(h.directory, path.relative(h.directory, path.join(wt, "src/alpha.ts")), `export const alpha = 1\n`) + await factory().complete("es-programmer") + + // Both QA seats pass through the real host tools — with no audit opened. + for (const seat of ["es-qa-functional", "es-qa-adversarial"]) { + const qa = await h.run(call("es_qa_verdict", { verdict: "pass", notes: "criteria met" }), { agent: seat }) + expect([seat, qa.tools[0]?.status]).toEqual([seat, "completed"]) + } + const held = (await factory().read())! + expect([held.stage, held.phases[0]!.status, held.phases[0]!.history.at(-1)?.event]).toEqual([ + "QA", + "qa", + "qa-passed", + ]) + expect(held.phases[0]!.history.at(-1)?.notes).toContain("audit.phase is required") + }, 180_000) +}) From 1134be1fd24661474f21c780367a2214493f436b Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:38:30 -0400 Subject: [PATCH 11/83] test(audit): layered config integration tests (#203) --- packages/cli/test/config-203.test.ts | 69 +++++++++ packages/core/test/config-layers-203.test.ts | 107 +++++++++++++ .../opencode/test/config-layers-203.test.ts | 140 ++++++++++++++++++ 3 files changed, 316 insertions(+) create mode 100644 packages/cli/test/config-203.test.ts create mode 100644 packages/core/test/config-layers-203.test.ts create mode 100644 packages/opencode/test/config-layers-203.test.ts diff --git a/packages/cli/test/config-203.test.ts b/packages/cli/test/config-203.test.ts new file mode 100644 index 0000000..c7c832e --- /dev/null +++ b/packages/cli/test/config-203.test.ts @@ -0,0 +1,69 @@ +// Audit #203 — `es config set` is human-only: without a terminal it refuses +// before writing. These are pipe runs asserting refusal codes through the +// CLI's own seam (main with a non-interactive ConfirmIO); the human key is +// never unlocked and nothing is ever written. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { exists, factoryLayout } from "@heretek-ai/es-core" +import { main } from "../src/main.ts" +import type { ConfirmIO } from "../src/tty.ts" + +let root: string +let state: string +const realXdgConfig = process.env.XDG_CONFIG_HOME + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-203-config-")) + state = await mkdtemp(path.join(tmpdir(), "es-203-config-state-")) + process.env.XDG_CONFIG_HOME = path.join(state, "xdg-config") +}) +afterEach(async () => { + if (realXdgConfig === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = realXdgConfig + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const pipe: ConfirmIO = { interactive: false, write: () => {}, readLine: async () => "y", readSecret: async () => "" } + +async function run(argv: string[]) { + const printed: string[] = [] + const code = await main(argv, { print: (text) => printed.push(text), confirm: pipe, cwd: root, stateDir: state }) + return { code, out: printed.join("\n") } +} + +describe("es config set refuses without a terminal (#203)", () => { + test("a piped project-layer set refuses with code 3 and writes nothing", async () => { + const result = await run(["config", "set", "research.depth", "3"]) + expect(result.code).toBe(3) + expect(result.out).toContain("interactive terminal") + expect(await exists(factoryLayout(root).config)).toBe(false) + }) + + test("a piped global-layer set refuses with code 3 and writes nothing", async () => { + const result = await run(["config", "set", "research.depth", "3", "--global"]) + expect(result.code).toBe(3) + expect(result.out).toContain("interactive terminal") + expect(await exists(factoryLayout(root).config)).toBe(false) + expect(await exists(path.join(state, "xdg-config", "epistemic-swarm", "config.json"))).toBe(false) + }) + + test("refusal codes contrast: schema refusal (1) precedes the TTY gate (3); usage is (2)", async () => { + const typo = await run(["config", "set", "reserch.depth", "3"]) + expect(typo.code).toBe(1) + expect(typo.out).toContain("Refused") + const bad = await run(["config", "set", "research.depth", "9"]) + expect(bad.code).toBe(1) + const usage = await run(["config", "set", "research.depth"]) + expect(usage.code).toBe(2) + expect(await exists(factoryLayout(root).config)).toBe(false) + }) + + test("--help stays a read: usage, code 0, nothing written", async () => { + const helped = await run(["config", "set", "research.depth", "3", "--help"]) + expect(helped.code).toBe(0) + expect(await exists(factoryLayout(root).config)).toBe(false) + }) +}) diff --git a/packages/core/test/config-layers-203.test.ts b/packages/core/test/config-layers-203.test.ts new file mode 100644 index 0000000..2dcbbf0 --- /dev/null +++ b/packages/core/test/config-layers-203.test.ts @@ -0,0 +1,107 @@ +// Audit #203 — layered model tiers, global → project → plugin options. +// +// The full three-layer conflict (each layer wins a different tier) through +// loadEsConfig, the strict side of the unknown-key contract (config files +// reject unknowns, including 0.7-era keys — plugin options only warn, proven +// in packages/opencode/test/config-layers-203.test.ts), and the human-only +// boundary for `es config set` through the shell policy. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { loadEsConfig } from "../src/index.ts" +import { evaluateShell, invokesHumanOnly } from "../src/trust/index.ts" + +let root: string +let env: NodeJS.ProcessEnv +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-203-layers-")) + env = { XDG_CONFIG_HOME: path.join(root, "xdg") } +}) +afterEach(() => rm(root, { recursive: true, force: true })) + +const writeGlobal = async (value: unknown) => { + const file = path.join(env.XDG_CONFIG_HOME!, "epistemic-swarm", "config.json") + await mkdir(path.dirname(file), { recursive: true }) + await writeFile(file, JSON.stringify(value)) + return file +} +const writeProject = async (value: unknown) => { + await mkdir(path.join(root, ".factory"), { recursive: true }) + const file = path.join(root, ".factory", "config.json") + await writeFile(file, JSON.stringify(value)) + return file +} + +describe("three-layer tier precedence (#203)", () => { + test("each layer wins a different tier: options > project > global", async () => { + const global = await writeGlobal({ + models: { fast: "global/fast", balanced: "global/balanced", deep: "global/deep" }, + }) + const project = await writeProject({ models: { fast: "project/fast", deep: "project/deep" } }) + const loaded = await loadEsConfig(root, { env, overrides: { models: { fast: "options/fast" } } }) + expect(loaded.sources).toEqual([global, project]) + expect(loaded.config.models).toEqual({ + fast: "options/fast", + balanced: "global/balanced", + deep: "project/deep", + }) + }) + + test("a lower layer survives wherever a higher layer is silent", async () => { + await writeGlobal({ models: { deep: "global/deep" }, pr: "off" }) + await writeProject({ pr: "gh" }) + const loaded = await loadEsConfig(root, { env, overrides: { afterEdit: "off" } }) + expect(loaded.config.models).toEqual({ deep: "global/deep" }) + expect(loaded.config.pr).toBe("gh") + expect(loaded.config.afterEdit).toBe("off") + }) +}) + +describe("config-file unknowns are rejected, strictly (#203)", () => { + test.each(["search_engine", "max_iterations", "mode"])( + "project file: 0.7-era key %s is rejected, naming it", + async (key: string) => { + await writeProject({ [key]: key === "max_iterations" ? 2 : "x" }) + await expect(loadEsConfig(root, { env })).rejects.toThrow(new RegExp(`invalid config.*${key}`)) + }, + ) + + test("project file: a typo key is rejected with the source named", async () => { + const project = await writeProject({ modles: { fast: "a/fast" } }) + await expect(loadEsConfig(root, { env })).rejects.toThrow(new RegExp(`invalid config.*${project}.*modles|modles`)) + }) + + test("global file: unknown keys are rejected too", async () => { + await writeGlobal({ search_engine: "duckduckgo" }) + await expect(loadEsConfig(root, { env })).rejects.toThrow(/invalid config.*search_engine/) + }) + + test("control: known keys in both files still load", async () => { + await writeGlobal({ research: { depth: 1 } }) + await writeProject({ research: { depth: 3 } }) + expect((await loadEsConfig(root, { env })).config.research).toEqual({ depth: 3 }) + }) +}) + +describe("es config set stays human-only (#203)", () => { + const shell = (agent: string | undefined, command: string) => + evaluateShell({ root, stateDir: path.join(root, "state") }, agent, command, { sandboxAvailable: true }) + + test("invokesHumanOnly reads config set, however flagged; config show stays a read", () => { + expect(invokesHumanOnly("es config set models.deep x/y")).toBe(true) + expect(invokesHumanOnly("es --cwd . config set models.deep x/y")).toBe(true) + expect(invokesHumanOnly("es config set models.deep x/y --global")).toBe(true) + expect(invokesHumanOnly("es config show")).toBe(false) + expect(invokesHumanOnly("es --cwd . config show")).toBe(false) + }) + + test("every agent shell is denied config set; config show is allowed", () => { + for (const agent of ["build", "factory", "es-programmer", undefined]) + expect([agent, shell(agent, "es config set models.deep x/y").effect]).toEqual([agent, "deny"]) + const denied = shell("build", "es config set models.deep x/y") + expect(denied.effect).toBe("deny") + if (denied.effect === "deny") expect(denied.reason).toContain("human-only") + expect(shell("build", "es config show").effect).toBe("allow") + }) +}) diff --git a/packages/opencode/test/config-layers-203.test.ts b/packages/opencode/test/config-layers-203.test.ts new file mode 100644 index 0000000..1b9c0e4 --- /dev/null +++ b/packages/opencode/test/config-layers-203.test.ts @@ -0,0 +1,140 @@ +// Audit #203 — layered config on the real host: the project file and the +// plugin options set conflicting tiers, the effective model per tier resolves +// options > project, and the 0.7-era keys are ignored with exactly one +// combined warning. The strict side (config files reject unknowns) is proven +// in packages/core/test/config-layers-203.test.ts. `es config set` itself is +// never run here (human-only); the agent-shell denial goes through the +// plugin's own policy hook. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { modelFor } from "../src/agents.ts" +import { createPolicyHooks } from "../src/policy.ts" +import { EsRpc } from "../src/rpc-def.ts" +import { parseOptions, type Runtime, unknownOptionsWarning } from "../src/runtime.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") + +describe("plugin-option unknowns warn, once and combined (#203)", () => { + test("0.7-era keys land in sorted unknown, never in overrides", () => { + const parsed = parseOptions({ + stateDir: "/tmp/state", + models: { fast: "o/fast" }, + search_engine: "duckduckgo", + max_iterations: 2, + mode: "research", + }) + expect(parsed.overrides).toEqual({ models: { fast: "o/fast" } }) + expect(parsed.unknown).toEqual(["max_iterations", "mode", "search_engine"]) + }) + + test("the warning is a single string naming every key and the config docs", () => { + const warning = unknownOptionsWarning(["max_iterations", "mode", "search_engine"]) + expect(warning).toContain("max_iterations, mode, search_engine") + expect(warning).toContain("docs/CONFIG.md") + expect(warning).toContain("search_engine, max_iterations and mode were removed in 1.0") + }) + + test("no unknown keys means no warning", () => { + expect(parseOptions({ stateDir: "/tmp/state", pr: "off" }).unknown).toEqual([]) + }) +}) + +describe("layered tiers on the real host (#203)", () => { + let h: Harness + let state: string + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-203-layers-state-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [ + { + path: pluginDir, + options: { + stateDir: state, + models: { fast: "options/fast", balanced: "options/balanced" }, + search_engine: "duckduckgo", + max_iterations: 2, + mode: "research", + }, + }, + ], + files: { + "README.md": "# layered config\n", + ".factory/config.json": JSON.stringify({ models: { fast: "project/fast", deep: "project/deep" } }), + }, + }) + }, 120_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("plugin options win per tier; the project file fills the gaps", async () => { + const rpc = (h.opencode as any).rpc(EsRpc) + const view = await rpc.configState({}, { location: h.location }) + expect(view.sources).toEqual([path.join(h.directory, ".factory", "config.json")]) + expect(JSON.parse(view.config).models).toEqual({ + fast: "options/fast", + balanced: "options/balanced", + deep: "project/deep", + }) + }) + + test("the effective model per tier parses to provider/model", async () => { + const rpc = (h.opencode as any).rpc(EsRpc) + const config = JSON.parse((await rpc.configState({}, { location: h.location })).config) + for (const [tier, ref] of [ + ["fast", "options/fast"], + ["balanced", "options/balanced"], + ["deep", "project/deep"], + ] as const) { + const spec = AGENTS.find((agent) => agent.tier === tier)! + expect([tier, modelFor(spec, config)]).toEqual([tier, { providerID: ref.split("/")[0], id: ref.split("/")[1] }]) + } + }) + + test("the legacy keys warn exactly once, as one combined string", async () => { + const rpc = (h.opencode as any).rpc(EsRpc) + const view = await rpc.configState({}, { location: h.location }) + expect(view.warnings).toHaveLength(1) + expect(view.warnings[0]).toContain("max_iterations, mode, search_engine") + expect(view.warnings[0]).toContain("docs/CONFIG.md") + expect(view.warnings[0]).toContain("0.7-era options") + }) +}) + +describe("agent shells cannot run es config set (#203)", () => { + test("the plugin policy hook refuses it; config show passes the gate", async () => { + const root = await mkdtemp(path.join(tmpdir(), "es-203-policy-")) + const stateDir = await mkdtemp(path.join(tmpdir(), "es-203-policy-state-")) + try { + const hooks = createPolicyHooks({ + root, + stateDir, + modelProblems: new Map(), + factory: { read: async () => undefined }, + policy: async () => ({ root, stateDir }), + } as unknown as Runtime) + await expect( + hooks.before({ tool: "shell", agent: "build", id: "t1", input: { command: "es config set models.deep x/y" } }), + ).rejects.toThrow("human-only") + await expect( + hooks.before({ + tool: "shell", + agent: "es-programmer", + id: "t2", + input: { command: "es config set audit.phase required" }, + }), + ).rejects.toThrow("human-only") + await hooks.before({ tool: "shell", agent: "build", id: "t3", input: { command: "es config show" } }) + } finally { + await rm(root, { recursive: true, force: true }) + await rm(stateDir, { recursive: true, force: true }) + } + }) +}) From 0a40a613371d9637dfcceeac6360d14b494e377f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:51:48 -0400 Subject: [PATCH 12/83] test(audit): headless factory run integration tests (#200) --- packages/cli/test/headless-200.test.ts | 467 +++++++++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 packages/cli/test/headless-200.test.ts diff --git a/packages/cli/test/headless-200.test.ts b/packages/cli/test/headless-200.test.ts new file mode 100644 index 0000000..1295cb2 --- /dev/null +++ b/packages/cli/test/headless-200.test.ts @@ -0,0 +1,467 @@ +// Headless factory runs (#200 §2): integration tests against the real +// in-process OpenCode v2 host (packages/testkit), plus the cheap proof gaps +// from bug #247. Spend-bearing seams are never touched: no `factory run +// --headless` launch, no real signals, no real harness binary. Everything +// here drives `runHeadless`/`driveHeadless` with fake in-process drivers +// (one of them forwarding prompts to the testkit host), scripted +// AbortControllers, pipe CLI runs that assert refusal codes, and a stub +// `opencode` binary for argv capture. +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + exists, + Factory, + factoryLayout, + gateRunner, + readApproval, + researchSourcesDir, + SourceCache, + sealHumanKey, + stringifyFrontmatter, +} from "@heretek-ai/es-core" +import type { Harness } from "../../testkit/src/host.ts" +import { boot } from "../../testkit/src/host.ts" +import { directiveScript } from "../../testkit/src/script.ts" +import { + DRIVERS, + type DriverTurn, + driveHeadless, + type HarnessDriver, + type HeadlessEvent, + headlessExitCode, + opencodeDriver, + runHeadless, +} from "../src/headless.ts" +import { main } from "../src/main.ts" +import type { ConfirmIO } from "../src/tty.ts" + +const PASSPHRASE = "test-passphrase-1234" + +let root: string +let state: string +const realXdgConfig = process.env.XDG_CONFIG_HOME +const git = async (...args: string[]) => { + const proc = Bun.spawn(["git", ...args], { cwd: root, stdout: "pipe", stderr: "pipe" }) + await proc.exited +} + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-200-")) + state = await mkdtemp(path.join(tmpdir(), "es-200-state-")) + process.env.XDG_CONFIG_HOME = path.join(state, "xdg-config") + await git("init", "-q", "-b", "main") + await writeFile(path.join(root, "README.md"), "# x\n") + await git("add", "-A") + await git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "init") + await sealHumanKey(PASSPHRASE, state) +}) +afterEach(async () => { + if (realXdgConfig === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = realXdgConfig + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +/** A human at a terminal who types the passphrase with echo off. */ +const human = (answer: string = PASSPHRASE): ConfirmIO & { output: string[] } => { + const output: string[] = [] + return { + interactive: true, + output, + write: (text) => output.push(text), + readLine: async () => "", + readSecret: async () => answer, + } +} +const pipe: ConfirmIO = { interactive: false, write: () => {}, readLine: async () => "y", readSecret: async () => "" } + +async function runCli(argv: string[], confirm: ConfirmIO = pipe) { + const printed: string[] = [] + const code = await main(argv, { print: (text) => printed.push(text), confirm, cwd: root, stateDir: state }) + return { code, out: printed.join("\n") } +} + +/** A settled grill, approved at a terminal: the run is in RESEARCH. */ +async function researched() { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.writeFrontier("grill", { + version: "1.1", + idea: "cli test", + spendCeiling: { currency: "USD", maxAmount: 5 }, + settled: true, + nodes: [{ id: "a", question: "q?", answer: "yes", status: "settled" }], + }) + const approved = await runCli(["approve", "frontier"], human()) + expect(approved.code).toBe(0) + expect((await factory.read())?.stage).toBe("RESEARCH") + return factory +} + +describe("headless against the testkit host (#200 §2)", () => { + let host: Harness | undefined + + beforeAll(async () => { + host = await boot({ script: directiveScript }) + }, 60_000) + afterAll(async () => { + await host?.close() + host = undefined + }) + + test("turns reach the real host; a pending approval stops the run without granting anything", async () => { + const factory = await researched() + const harness = host! + const llmBefore = harness.llm.requests.length + const calls: DriverTurn[] = [] + // The opencode driver seam: one turn carries the seat identity, the + // prompt and the argv the harness CLI would get; here the harness is + // the in-process testkit host instead of a spawned `opencode run`. + const driver: HarnessDriver = { + id: "fake-host", + available: async () => true, + async *turn(input: DriverTurn) { + calls.push(input) + const result = await harness.run(input.prompt) + yield { type: "text", part: { text: `host settled ${result.context.length} messages` } } + return result.sessionID + }, + } + // Phase 1: no pending approval, so the turn goes through the host. + const first: HeadlessEvent[] = [] + for await (const event of runHeadless({ root, driver, maxTurns: 1, stallTurns: 99, stateDir: state })) + first.push(event) + expect(calls).toHaveLength(1) + expect(calls[0]?.agent).toBe("factory") + expect(calls[0]?.prompt).toContain("headless run") + expect(calls[0]?.root).toBe(root) + expect(harness.llm.requests.length).toBeGreaterThan(llmBefore) + expect(first.at(-1)).toEqual({ type: "turn-cap", turns: 1 }) + // Phase 2: a pending approval stops the run before any turn is driven. + const pending = factoryLayout(root).pending + await mkdir(path.dirname(pending), { recursive: true }) + await writeFile(pending, JSON.stringify([{ stage: "spec", requestedBy: "factory", at: "2026-10-10T00:00:00Z" }])) + const pendingBefore = await readFile(pending, "utf8") + const stageBefore = (await factory.read())?.stage + const callsBefore = calls.length + const llmMid = harness.llm.requests.length + const second: HeadlessEvent[] = [] + for await (const event of runHeadless({ root, driver, maxTurns: 5, stallTurns: 99, stateDir: state })) + second.push(event) + expect(second.at(-1)).toEqual({ + type: "waiting", + reason: "a human must approve (es approve )", + stages: ["spec"], + }) + // Nothing was granted: no new host turn, the pending file is byte + // identical, the stage is unchanged, and no approval was recorded. + expect(calls.length).toBe(callsBefore) + expect(harness.llm.requests.length).toBe(llmMid) + expect(await readFile(pending, "utf8")).toBe(pendingBefore) + expect((await factory.read())?.stage).toBe(stageBefore) + expect(await readApproval(root, "spec")).toBeUndefined() + }, 60_000) +}) + +describe("headless proof gaps (#247)", () => { + test("the driver sees the seat identity, the prompt, the model and the threaded session", async () => { + await researched() + const calls: DriverTurn[] = [] + let n = 0 + const driver: HarnessDriver = { + id: "fake", + available: async () => true, + async *turn(input: DriverTurn) { + calls.push(input) + n++ + yield { text: "working" } + return n === 1 ? "ses_1" : "ses_2" + }, + } + const events: HeadlessEvent[] = [] + for await (const event of runHeadless({ + root, + driver, + maxTurns: 2, + stallTurns: 99, + stateDir: state, + model: "fake/scripted", + })) + events.push(event) + expect(calls).toHaveLength(2) + for (const call of calls) { + expect(call.agent).toBe("factory") + expect(call.prompt).toContain("headless run") + expect(call.root).toBe(root) + expect(call.model).toBe("fake/scripted") + } + expect(calls[0]?.session).toBeUndefined() + expect(calls[0]?.signal).toBeUndefined() + // The first turn's session id is threaded into the second turn. + expect(calls[1]?.session).toBe("ses_1") + expect(events.at(-1)).toEqual({ type: "turn-cap", turns: 2 }) + }) + + test("the opencode driver argv shape: run --standalone --format json --agent [--model] [--session] prompt", async () => { + expect(opencodeDriver.id).toBe("opencode") + expect(DRIVERS.opencode).toBe(opencodeDriver) + const bin = await mkdtemp(path.join(tmpdir(), "es-argv-opencode-")) + const capture = path.join(bin, "argv.jsonl") + await writeFile( + path.join(bin, "opencode"), + `#!/usr/bin/env bun\nimport { appendFileSync } from "node:fs"\nappendFileSync(process.env.ES_ARGV_CAPTURE!, \`\${JSON.stringify(Bun.argv.slice(2))}\\n\`)\nconsole.log(JSON.stringify({ type: "text", sessionID: "ses_argv1", part: { text: "stub" } }))\n`, + ) + await chmod(path.join(bin, "opencode"), 0o755) + const previousPath = process.env.PATH + const previousCapture = process.env.ES_ARGV_CAPTURE + process.env.PATH = `${bin}${path.delimiter}${previousPath}` + process.env.ES_ARGV_CAPTURE = capture + try { + const turnArgv = async (input: DriverTurn) => { + await rm(capture, { force: true }) + const seen: unknown[] = [] + const turn = opencodeDriver.turn(input) + let step = await turn.next() + while (!step.done) { + seen.push(step.value) + step = await turn.next() + } + const lines = (await readFile(capture, "utf8")).trim().split("\n") + return { argv: JSON.parse(lines[0]!) as string[], events: seen, session: step.value } + } + const minimal = await turnArgv({ root, agent: "factory", prompt: "go" }) + expect(minimal.argv).toEqual(["run", "--standalone", "--format", "json", "--agent", "factory", "go"]) + expect(minimal.session).toBe("ses_argv1") + expect(minimal.events).toHaveLength(1) + const full = await turnArgv({ + root, + agent: "factory", + prompt: "go", + model: "fake/scripted", + session: "ses_prev", + }) + expect(full.argv).toEqual([ + "run", + "--standalone", + "--format", + "json", + "--agent", + "factory", + "--model", + "fake/scripted", + "--session", + "ses_prev", + "go", + ]) + // The prompt is always the trailing argument, after any seat flags. + expect(full.argv.at(-1)).toBe("go") + } finally { + process.env.PATH = previousPath + if (previousCapture === undefined) delete process.env.ES_ARGV_CAPTURE + else process.env.ES_ARGV_CAPTURE = previousCapture + await rm(bin, { recursive: true, force: true }) + } + // With no harness binary on PATH the driver reports itself unavailable. + const empty = await mkdtemp(path.join(tmpdir(), "es-argv-empty-")) + try { + process.env.PATH = empty + expect(await opencodeDriver.available()).toBe(false) + } finally { + process.env.PATH = previousPath + await rm(empty, { recursive: true, force: true }) + } + }) + + test("a scripted driver drives runHeadless to DONE and the prUrl spreads into done", async () => { + const green = { passed: true, findings: [], summary: "green" } + const stubGates = async () => green + const factory = new Factory(root, { gates: stubGates, stateDir: state }) + await factory.writeFrontier("grill", { + version: "1.1", + idea: "cli test", + spendCeiling: { currency: "USD", maxAmount: 5 }, + settled: true, + nodes: [{ id: "a", question: "q?", answer: "yes", status: "settled" }], + }) + expect((await runCli(["approve", "frontier"], human())).code).toBe(0) + // A research report whose single claim cites a cached source verbatim. + const cache = new SourceCache(researchSourcesDir(root), state) + const source = await cache.put({ + url: "https://example.test/greeting", + text: "Greeting libraries usually expose a single greet function.", + provider: "fetch", + }) + await mkdir(path.join(root, ".factory/research"), { recursive: true }) + await writeFile( + path.join(root, ".factory/research/REPORT.md"), + `# Research\n\n- A greet function is the conventional API [VERIFIED: sha256:${source.meta.sha256} "expose a single greet function"]\n`, + ) + await factory.completeResearch("factory") + await mkdir(path.join(root, ".factory/specs/alpha"), { recursive: true }) + await writeFile( + path.join(root, ".factory/roadmap.json"), + JSON.stringify({ version: 1, title: "Greeting", phases: [{ id: "alpha", title: "Phase alpha" }] }), + ) + await writeFile( + path.join(root, ".factory/specs/alpha/GOAL.md"), + stringifyFrontmatter( + { + phase: "alpha", + title: "Phase alpha", + acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }], + }, + "Implement alpha as a vertical slice.\n", + ), + ) + expect((await runCli(["approve", "spec"], human())).code).toBe(0) + await factory.startBuild("factory") + const dir = (await factory.activeWorktree())! + await mkdir(path.join(dir, "src"), { recursive: true }) + await mkdir(path.join(dir, "test"), { recursive: true }) + await writeFile(path.join(dir, "test/alpha.test.ts"), `test("alpha", () => {})\n`) + await factory.recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await writeFile(path.join(dir, "src/alpha.ts"), "export const alpha = 1\n") + expect((await factory.complete("es-programmer")).ok).toBe(true) + await factory.qaVerdict("es-qa-functional", "pass", "ok") + expect((await factory.qaVerdict("es-qa-adversarial", "pass", "ok")).stage).toBe("RELEASE") + // The scripted turn records the PR (the human fallback): the next loop + // sees DONE with a release, and done carries the prUrl. + const prUrl = "https://example.test/pr/9" + const calls: DriverTurn[] = [] + const driver: HarnessDriver = { + id: "fake", + available: async () => true, + async *turn(input: DriverTurn) { + calls.push(input) + yield { text: "recording the pr" } + await new Factory(root, { gates: stubGates, stateDir: state }).recordPr("tester", prUrl) + return "ses_done" + }, + } + const events: HeadlessEvent[] = [] + for await (const event of runHeadless({ root, driver, maxTurns: 5, stallTurns: 99, stateDir: state })) + events.push(event) + expect(events[0]).toMatchObject({ type: "start" }) + expect(calls).toHaveLength(1) + expect(calls[0]?.agent).toBe("factory") + expect(events.at(-1)).toEqual({ type: "done", prUrl }) + expect((await factory.read())?.stage).toBe("DONE") + }) + + test("a STOP file halts at headless level: no turn runs, the stage is untouched", async () => { + const factory = await researched() + await writeFile(factoryLayout(root).stop, "lunch\n") + let turns = 0 + const driver: HarnessDriver = { + id: "fake", + available: async () => true, + async *turn() { + turns++ + yield { text: "never" } + return "ses_fake" + }, + } + const events: HeadlessEvent[] = [] + for await (const event of runHeadless({ root, driver, maxTurns: 3, stallTurns: 99, stateDir: state })) + events.push(event) + expect(events.at(-1)).toMatchObject({ type: "halted", reason: expect.stringContaining("lunch") }) + expect(turns).toBe(0) + expect((await factory.read())?.stage).toBe("RESEARCH") + expect(await exists(factoryLayout(root).stop)).toBe(true) + }) + + test("an unknown --driver is refused before anything runs", async () => { + const empty = await mkdtemp(path.join(tmpdir(), "es-200-empty-")) + try { + const printed: string[] = [] + const code = await main(["factory", "run", "--headless", "--driver", "nope"], { + print: (text) => printed.push(text), + confirm: pipe, + cwd: empty, + stateDir: state, + }) + const out = printed.join("\n") + expect(code).toBe(2) + expect(out).toContain("Unknown driver") + expect(out).toContain("opencode") + expect(await exists(factoryLayout(empty).dir)).toBe(false) + } finally { + await rm(empty, { recursive: true, force: true }) + } + }) + + test("exit codes: cancelled is 130, failures stay 1, the rest 0", async () => { + expect(headlessExitCode("cancelled")).toBe(130) + for (const type of ["error", "halted", "stalled", "turn-cap"] as const) expect(headlessExitCode(type)).toBe(1) + for (const type of ["start", "turn", "driver", "progress", "turn-metrics", "waiting", "done"] as const) + expect(headlessExitCode(type)).toBe(0) + }) + + test("a pre-aborted signal cancels at Abort level: no error, no halt, resumable", async () => { + const factory = await researched() + const hanging: HarnessDriver = { + id: "hang", + available: async () => true, + async *turn() { + await new Promise(() => {}) + yield { text: "never" } + return undefined + }, + } + const controller = new AbortController() + controller.abort() + const events: HeadlessEvent[] = [] + for await (const event of runHeadless({ + root, + driver: hanging, + maxTurns: 5, + stateDir: state, + signal: controller.signal, + progressMs: 60_000, + })) + events.push(event) + // No real signal was sent: the mapping (SIGINT/SIGTERM -> abort -> + // `cancelled`, exit 130) is proven at the Abort level, and the live + // signal E2E stays human-only (issue #200 §3, NEEDS HUMAN). + expect(events.at(-1)).toMatchObject({ + type: "cancelled", + reason: expect.stringContaining("run state is untouched"), + }) + expect(headlessExitCode("cancelled")).toBe(130) + expect(events.some((event) => event.type === "error")).toBe(false) + expect((await factory.read())?.stage).toBe("RESEARCH") + expect(await exists(factoryLayout(root).stop)).toBe(false) + }) + + test("driveHeadless threads a custom job's done (with prUrl) through unchanged", async () => { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.provisionRun("tester", 5) + const prUrl = "https://example.test/pr/7" + const driver: HarnessDriver = { + id: "fake", + available: async () => true, + async *turn() { + yield { text: "working" } + return "ses_fake" + }, + } + const events: HeadlessEvent[] = [] + for await (const event of driveHeadless( + { root, driver, maxTurns: 3, stallTurns: 99, stateDir: state }, + { + agent: "factory", + prompt: async () => "go", + finished: async () => ({ type: "done", prUrl }), + progress: async () => "p", + }, + )) + events.push(event) + expect(events.at(-1)).toEqual({ type: "done", prUrl }) + }) +}) From 0a2ca5290f35efef6d3f56ae78e2cb882aa0cfc1 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 15:08:19 -0400 Subject: [PATCH 13/83] test(audit): evidence cache and quote verifier integration tests (#187) --- evals/cases/evidence-187.json | 8 + evals/serve/evidence-187.txt | 3 + packages/core/test/evidence-187.test.ts | 188 ++++++++++++++++++++ packages/opencode/test/evidence-187.test.ts | 133 ++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 evals/cases/evidence-187.json create mode 100644 evals/serve/evidence-187.txt create mode 100644 packages/core/test/evidence-187.test.ts create mode 100644 packages/opencode/test/evidence-187.test.ts diff --git a/evals/cases/evidence-187.json b/evals/cases/evidence-187.json new file mode 100644 index 0000000..81c06d9 --- /dev/null +++ b/evals/cases/evidence-187.json @@ -0,0 +1,8 @@ +{ + "description": "Research fetches a served fixture page and quotes its key sentence verbatim (no false VERIFIED).", + "agent": "es-research-alpha", + "prompt": "Fetch {{SERVE_URL}}/evidence-187.txt with es_research_fetch and report the key sentence about mechanical gates, quoted verbatim.", + "maxSteps": 6, + "checks": { "toolsUsed": ["es_research_fetch"], "contains": ["Mechanical gates catch regressions before review"] }, + "id": "evidence-187" +} diff --git a/evals/serve/evidence-187.txt b/evals/serve/evidence-187.txt new file mode 100644 index 0000000..163ff2a --- /dev/null +++ b/evals/serve/evidence-187.txt @@ -0,0 +1,3 @@ +The evidence cache seals every source with an engine key. Mechanical gates +catch regressions before review, which keeps autonomous agents honest. +Quote the gate sentence verbatim; a paraphrase is not evidence. diff --git a/packages/core/test/evidence-187.test.ts b/packages/core/test/evidence-187.test.ts new file mode 100644 index 0000000..9f97ac5 --- /dev/null +++ b/packages/core/test/evidence-187.test.ts @@ -0,0 +1,188 @@ +// Issue #187 §2 + bug #251 fuzz-adjacent edges (deterministic, no fuzz harness). +// +// Pins the CURRENT normalisation semantics of `verifyQuote` (NFKC, +// typography folds, markdown stripping, whitespace collapse; no entity +// decoding) and the CURRENT auditor downgrade rules (VERIFIED verbatim +// grounds, paraphrases and planted entries stay ungrounded). Bugs #251 +// (fuzz corpus, legacy unsealed constructor) and #254 (wording) are for the +// parent: this suite tests behaviour as-is and reports defects as findings, +// never by changing product code. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { auditMarkdown, htmlToText, normalizeSourceText, SourceCache, sha256, verifyQuote } from "../src/index.ts" + +let dir: string +let state: string +beforeEach(async () => { + dir = await mkdtemp(path.join(tmpdir(), "es-evidence-187-")) + state = await mkdtemp(path.join(tmpdir(), "es-evidence-187-state-")) +}) +afterEach(() => Promise.all([rm(dir, { recursive: true, force: true }), rm(state, { recursive: true, force: true })])) + +const sealed = () => new SourceCache(path.join(dir, "sources"), state) + +const GATES = "Alpha gates keep agents honest under heavy load conditions today, witnesses agree." + +describe("quote verifier fuzz-adjacent edges (#187 §2, gaps listed in #251)", () => { + test("zero-width chars: ZWSP/FEFF fold to space and still match; ZWNJ/ZWJ/SHY/word-joiner refuse", () => { + const plain = "hello world friends and more text here" + // U+200B ZERO WIDTH SPACE and U+FEFF are in the typography fold to space. + expect(verifyQuote(plain, "hello​ world friends and more").ok).toBe(true) + expect(verifyQuote(plain, "hello world friends and more").ok).toBe(true) + // An invisible join inside a word folds to a word break (current behaviour). + expect(verifyQuote("hello​world friends and more text here", "hello world friends and more").ok).toBe(true) + // U+200C/U+200D/U+00AD/U+2060 are NOT folded: the quote is refused. + for (const invisible of ["‌", "‍", "­", "⁠"]) { + const quote = `hello${invisible} world friends and more` + expect([JSON.stringify(invisible), verifyQuote(plain, quote)]).toMatchObject([ + JSON.stringify(invisible), + { ok: false }, + ]) + expect(verifyQuote(plain, quote).reason).toContain("not found verbatim") + // And the reverse direction refuses too: the source keeps the char. + expect(verifyQuote(`hello${invisible} world friends and more text here`, plain.slice(0, 30)).ok).toBe(false) + } + }) + + test("`..` stays literal, `...`/`…` split into ordered fragments, `....` leaves a stray dot", () => { + // Two dots are code/punctuation, never an ellipsis. + expect(verifyQuote("version 1..2 released today for all users", "version 1..2 released today").ok).toBe(true) + expect(verifyQuote("version 1..2 released today for all users", "version 1..2 ... released today for all").ok).toBe( + true, + ) + // Three dots and the single-char ellipsis both split; order matters. + expect(verifyQuote(GATES, "Alpha gates keep ... honest under heavy load").ok).toBe(true) + expect(verifyQuote(GATES, "Alpha gates keep … honest under heavy load").ok).toBe(true) + expect(verifyQuote(GATES, "honest under heavy load ... Alpha gates keep").ok).toBe(false) + // Four dots are an ellipsis plus a literal dot: the stray dot is refused. + const four = verifyQuote(GATES, "Alpha gates keep .... honest under heavy load") + expect(four.ok).toBe(false) + expect(four.reason).toContain("not found verbatim") + }) + + test("entity-at-quote: the verifier does no entity decoding (the fetch layer decodes)", () => { + const amp = htmlToText("

Fish & chips are tasty indeed yum.

") + const numeric = htmlToText("

Fish & chips are tasty indeed yum.

") + // Both spellings cache to the same decoded bytes. + expect(amp.text).toBe("Fish & chips are tasty indeed yum.") + expect(numeric.text).toBe(amp.text) + // The decoded quote verifies; the still-encoded spellings are refused. + expect(verifyQuote(amp.text, "Fish & chips are tasty").ok).toBe(true) + for (const encoded of ["Fish & chips are tasty", "Fish & chips are tasty"]) { + const check = verifyQuote(amp.text, encoded) + expect([encoded, check.ok]).toEqual([encoded, false]) + expect(check.reason).toContain("not found verbatim") + } + }) + + test("markdown-rendered quotes: emphasis and links strip on both sides", () => { + expect(verifyQuote("the **quick brown fox jumps** over fences high", "quick brown fox jumps over").ok).toBe(true) + expect(verifyQuote("the quick brown fox jumps over fences high", "the **quick brown fox jumps** over").ok).toBe( + true, + ) + expect(verifyQuote("run __every__ `single` test file now please", "run every single test file now").ok).toBe(true) + expect(verifyQuote("[Bun](https://bun.sh) is fast and furious indeed", "Bun is fast and furious").ok).toBe(true) + expect(verifyQuote("Bun is fast and furious indeed ok", "[Bun](https://bun.sh) is fast and furious").ok).toBe(true) + }) + + test("short-fragment refusal: every ellipsis fragment and every single quote needs 12+ characters", () => { + expect(verifyQuote(GATES, "short").reason).toContain("too short") + expect(verifyQuote(GATES, "abcdefghijk").ok).toBe(false) + expect(verifyQuote("abcdefghijkl rest of the text here ok", "abcdefghijkl").ok).toBe(true) + const short = verifyQuote(GATES, "Alpha gates keep ... hi ... honest under heavy load") + expect(short.ok).toBe(false) + expect(short.reason).toContain("fragment") + expect(verifyQuote(GATES, "Alpha gates keep ... honest under heavy load").ok).toBe(true) + }) + + test("unicode normalisation: NFKC, curly quotes, dashes and NBSP fold before matching", () => { + expect(verifyQuote("filigree work is quite delicate indeed", "filigree work is quite").ok).toBe(true) + expect(verifyQuote("café society is quite lively today", "café society is quite").ok).toBe(true) + // Composed vs decomposed acute accents match after NFKC. + expect(verifyQuote("café society is quite lively today", "café society is quite").ok).toBe(true) + expect(verifyQuote("He said “hello world, friends” ok", 'said "hello world, friends"').ok).toBe(true) + expect(verifyQuote("fast — reliable service for all users", "fast - reliable service").ok).toBe(true) + expect(verifyQuote("hello world friends and more text", "hello world friends and more").ok).toBe(true) + // Case still matters: normalisation is not case folding. + expect(verifyQuote(GATES, "alpha gates keep agents honest").ok).toBe(false) + }) +}) + +describe("auditor downgrade rules (#187: verbatim grounds, paraphrase and plants do not)", () => { + test("verbatim VERIFIED grounds; a paraphrase stays ungrounded (flagged, never re-tagged)", async () => { + const cache = sealed() + const source = await cache.put({ + url: "https://gates.test/run", + text: "Mechanical gates catch regressions before review, which keeps autonomous agents honest under heavy load.", + provider: "fetch", + }) + const good = await auditMarkdown( + `- Gates catch regressions early [VERIFIED: sha256:${source.meta.sha256} "catch regressions before review"]`, + cache, + ) + expect(good.coverage).toMatchObject({ claims: 1, grounded: 1, ungrounded: 0 }) + expect(good.passed).toBe(true) + expect(good.cited).toEqual([source.meta.sha256]) + const paraphrased = await auditMarkdown( + `- Gates catch failures ahead of inspection [VERIFIED: sha256:${source.meta.sha256} "catch failures ahead of inspection"]`, + cache, + ) + expect(paraphrased.coverage).toMatchObject({ claims: 1, grounded: 0, ungrounded: 1 }) + expect(paraphrased.passed).toBe(false) + expect(paraphrased.claims[0]?.reason).toContain("not found verbatim") + }) + + test("a planted unsealed entry is refused by the sealed cache; the legacy constructor still reads it (#251)", async () => { + const text = "Planted findings about gate latency on reference hardware." + const hash = sha256(normalizeSourceText(text)) + await mkdir(path.join(dir, "sources"), { recursive: true }) + await writeFile(path.join(dir, "sources", `${hash}.md`), normalizeSourceText(text)) + await writeFile( + path.join(dir, "sources", `${hash}.json`), + `${JSON.stringify({ sha256: hash, url: "https://evil.test/planted", retrieved: new Date().toISOString(), provider: "fetch", bytes: Buffer.byteLength(normalizeSourceText(text)) })}\n`, + ) + expect(await sealed().get(hash)).toBeUndefined() + const audit = await auditMarkdown( + `- Planted latency wins [VERIFIED: sha256:${hash} "gate latency on reference"]`, + sealed(), + ) + expect(audit.coverage).toMatchObject({ claims: 1, grounded: 0, ungrounded: 1 }) + expect(audit.claims[0]?.reason).toContain("not in the cache") + expect(audit.passed).toBe(false) + // Current legacy behaviour (bug #251): without a state dir nothing is verified. + expect((await new SourceCache(path.join(dir, "sources")).get(hash))?.text).toContain("gate latency") + }) + + test("short and missing quotes stay ungrounded; substantive other tags ground, stubs are malformed", async () => { + const cache = sealed() + const source = await cache.put({ + url: "https://gates.test/short", + text: "Mechanical gates catch regressions before review, which keeps autonomous agents honest under heavy load.", + provider: "fetch", + }) + const audit = await auditMarkdown( + [ + `- Too short [VERIFIED: sha256:${source.meta.sha256} "tiny"]`, + `- Missing quote [VERIFIED: no quote here]`, + `- Fewer tools to install [INFERRED: one binary covers the runner and the bundler]`, + `- Maybe faster [HYPOTHESIS: rerun the benchmark with n=30]`, + `- Nothing else found [NEGATIVE_KNOWLEDGE: searched for gate failures on ARM]`, + `- Stub [INFERRED: ok]`, + "- No tag at all", + ].join("\n"), + cache, + ) + // "tiny" fails verification (ungrounded); "no quote here" has no + // source/quote shape and the stub INFERRED body is malformed. + expect(audit.coverage).toMatchObject({ + claims: 7, + grounded: 3, + ungrounded: 1, + untagged: 1, + malformed: 2, + }) + expect(audit.passed).toBe(false) + }) +}) diff --git a/packages/opencode/test/evidence-187.test.ts b/packages/opencode/test/evidence-187.test.ts new file mode 100644 index 0000000..0cffe89 --- /dev/null +++ b/packages/opencode/test/evidence-187.test.ts @@ -0,0 +1,133 @@ +// Issue #187 §2: loopback HTTP source server on the testkit host — a +// research seat fetches, quotes verbatim (VERIFIED survives the audit), +// paraphrases (stays ungrounded: flagged, never re-tagged), and quotes a +// planted file (refused). Real in-process host, scripted fake LLM, +// 127.0.0.1 only; must also pass under scripts/v2-head.sh. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { normalizeSourceText, researchCache, sha256 } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +const GATES_PAGE = + "Gates

Mechanical gates catch regressions before review, which keeps autonomous agents honest under heavy load.

" +const ENTITY_PAGE = "

Fish & chips are tasty indeed yum.

" + +let state: string +let h: Harness +let server: ReturnType +let previousLoopback: string | undefined + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-evidence-187-host-")) + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + const url = new URL(req.url) + if (url.pathname === "/gates") return new Response(GATES_PAGE, { headers: { "content-type": "text/html" } }) + if (url.pathname === "/entity") return new Response(ENTITY_PAGE, { headers: { "content-type": "text/html" } }) + return new Response("not found", { status: 404 }) + }, + }) + // The direct backend refuses loopback by default (SSRF guard); the + // loopback-only fixture server above needs the explicit test allowance. + previousLoopback = process.env.ES_RESEARCH_ALLOW_LOOPBACK + process.env.ES_RESEARCH_ALLOW_LOOPBACK = "1" + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# evidence 187\n" }, + }) +}, 60_000) + +afterAll(async () => { + await h?.close() + server?.stop(true) + if (previousLoopback === undefined) delete process.env.ES_RESEARCH_ALLOW_LOOPBACK + else process.env.ES_RESEARCH_ALLOW_LOOPBACK = previousLoopback + await rm(state, { recursive: true, force: true }) +}) + +const gatesUrl = () => `http://127.0.0.1:${server.port}/gates` +const note = (content: string) => writeFile(path.join(h.directory, ".factory/research/alpha.md"), content) + +async function fetchHash(pathname: string): Promise { + const fetched = await h.run(call("es_research_fetch", { url: `http://127.0.0.1:${server.port}${pathname}` }), { + agent: "es-research-alpha", + }) + expect(fetched.tools[0]?.status).toBe("completed") + const hash = /sha256:([0-9a-f]{64})/.exec(fetched.tools[0]?.text ?? "")?.[1] + expect(hash).toBeDefined() + return hash! +} + +describe("evidence on the real host (#187 §2)", () => { + test("verbatim VERIFIED survives the audit; a paraphrase stays ungrounded and prunes away", async () => { + const hash = await fetchHash("/gates") + await mkdir(path.join(h.directory, ".factory/research"), { recursive: true }) + await note( + `# Alpha\n\n- Gates catch regressions early [VERIFIED: sha256:${hash} "catch regressions before review"]\n- Gates catch failures ahead of inspection [VERIFIED: sha256:${hash} "catch failures ahead of inspection"]\n`, + ) + const audit = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), { + agent: "es-research-alpha", + }) + expect(audit.tools[0]?.status).toBe("completed") + expect(audit.tools[0]?.text).toContain("1/2 claims grounded") + expect(audit.tools[0]?.text).toContain("not found verbatim") + expect(audit.tools[0]?.text).not.toContain("Audit passed") + const pruned = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md", prune: true }), { + agent: "es-research-alpha", + }) + expect(pruned.tools[0]?.text).toContain("Pruned 1 claim") + const again = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), { + agent: "es-research-alpha", + }) + expect(again.tools[0]?.text).toContain("Audit passed") + expect(again.tools[0]?.text).toContain("1/1 claims grounded") + }, 120_000) + + test("a planted unsealed entry is refused: the sealed cache misses and the audit stays ungrounded", async () => { + const text = "Planted findings about gate latency on reference hardware." + const hash = sha256(normalizeSourceText(text)) + await mkdir(path.join(h.directory, ".factory/research/sources"), { recursive: true }) + await writeFile(path.join(h.directory, `.factory/research/sources/${hash}.md`), normalizeSourceText(text)) + await writeFile( + path.join(h.directory, `.factory/research/sources/${hash}.json`), + `${JSON.stringify({ sha256: hash, url: "https://evil.test/planted", retrieved: new Date().toISOString(), provider: "fetch", bytes: Buffer.byteLength(normalizeSourceText(text)) })}\n`, + ) + expect(await researchCache(h.directory, state).get(hash)).toBeUndefined() + await note(`# Alpha\n\n- Planted latency wins [VERIFIED: sha256:${hash} "gate latency on reference"]\n`) + const audit = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), { + agent: "es-research-alpha", + }) + expect(audit.tools[0]?.status).toBe("completed") + expect(audit.tools[0]?.text).toContain("0/1 claims grounded") + expect(audit.tools[0]?.text).toContain("not in the cache") + expect(audit.tools[0]?.text).not.toContain("Audit passed") + }, 120_000) + + test("entity-at-quote on the host: the decoded & verifies, the encoded & is refused", async () => { + const hash = await fetchHash("/entity") + const cached = await researchCache(h.directory, state).get(hash) + expect(cached?.text).toBe("Fish & chips are tasty indeed yum.") + await note( + `# Alpha\n\n- Fish and chips [VERIFIED: sha256:${hash} "Fish & chips are tasty"]\n- Fish encoded [VERIFIED: sha256:${hash} "Fish & chips are tasty"]\n`, + ) + const audit = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), { + agent: "es-research-alpha", + }) + expect(audit.tools[0]?.status).toBe("completed") + expect(audit.tools[0]?.text).toContain("1/2 claims grounded") + expect(audit.tools[0]?.text).not.toContain("Audit passed") + }, 120_000) + + test("the fetch URL is the loopback fixture server (no external network)", () => { + expect(gatesUrl()).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/gates$/) + }) +}) From 8fd1d4a84f83208b7fe8e89344f06f7c29401d0f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 15:13:53 -0400 Subject: [PATCH 14/83] test(audit): web failover chain integration tests (#188) --- packages/core/test/failover-loopback.test.ts | 709 ++++++++++++++++++ packages/opencode/test/failover-chain.test.ts | 185 +++++ 2 files changed, 894 insertions(+) create mode 100644 packages/core/test/failover-loopback.test.ts create mode 100644 packages/opencode/test/failover-chain.test.ts diff --git a/packages/core/test/failover-loopback.test.ts b/packages/core/test/failover-loopback.test.ts new file mode 100644 index 0000000..a1c81b3 --- /dev/null +++ b/packages/core/test/failover-loopback.test.ts @@ -0,0 +1,709 @@ +// #188 / #253: failover-chain proof gaps against loopback stubs. +// Real HTTP to 127.0.0.1 only (gateway + SearXNG stubs below); no external +// network. Proves, over the wire: 503 failover + order, 429 with/without +// Retry-After (cooldown skip then retry), auth continuation, blocked stopping +// both chains, sealed provenance, and that the gateway token never leaves its +// Authorization header (every BackendError site + every cache write). +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + BackendError, + type BackendErrorKind, + backoffFor, + braveBackend, + runFetchChain, + runSearchChain, + type SourceBackend, + searxngBackend, +} from "../src/research/backends/index.ts" +import { ScraperSwarmClient, scraperSwarmBackend } from "../src/research/backends/scraper-swarm.ts" +import { SourceCache } from "../src/research/cache.ts" +import { researchTools } from "../src/research/ops.ts" +import { sha256 } from "../src/util/hash.ts" + +/** Grep-able canary, never a real secret: every negative asserts its absence. */ +const TOKEN = "swarm_canary_9f8e7d6c5b4a" + +type GatewayMode = + | "success-search" + | "success-fetch" + | "http-503" + | "http-429-retry" + | "http-429-bare" + | "http-401" + | "contract-2" + | "no-body" + | "rpc-error" + | "wrong-id" + | "blocked" + | "timeout-tool" + | "upstream-tool" + | "no-results" + | "no-content" + | "sha-mismatch" + +type SearxngMode = "success" | "http-503" | "http-429-retry" | "http-429-bare" + +let gatewayMode: GatewayMode = "success-search" +let searxngMode: SearxngMode = "success" +let gatewayCalls: Array<{ name: string; auth: string | null }> = [] +let searxngHits = 0 +let gateway: ReturnType +let searxng: ReturnType + +const FETCH_CONTENT = "Recorded failover body text with enough words to cache." + +beforeAll(() => { + gateway = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + if (new URL(req.url).pathname !== "/mcp") return new Response("not found", { status: 404 }) + const body = (await req.json().catch(() => undefined)) as + | { id?: unknown; params?: { name?: string; arguments?: { query?: string } } } + | undefined + const name = body?.params?.name ?? "" + const id = body?.id ?? 1 + gatewayCalls.push({ name, auth: req.headers.get("authorization") }) + const rpc = (result: unknown) => + Response.json({ id, jsonrpc: "2.0", result }, { headers: { "x-swarm-contract": "1" } }) + switch (gatewayMode) { + case "http-503": + return new Response("unavailable", { status: 503 }) + case "http-429-retry": + return new Response("slow down", { status: 429, headers: { "retry-after": "2" } }) + case "http-429-bare": + return new Response("slow down", { status: 429 }) + case "http-401": + return new Response("denied", { status: 401 }) + case "contract-2": + return Response.json( + { id, jsonrpc: "2.0", result: { structuredContent: { results: [] } } }, + { headers: { "x-swarm-contract": "2" } }, + ) + case "no-body": + return Response.json({ hello: 1 }, { headers: { "x-swarm-contract": "1" } }) + case "rpc-error": + return Response.json( + { id, jsonrpc: "2.0", error: { code: -32602, message: "bad params" } }, + { headers: { "x-swarm-contract": "1" } }, + ) + case "wrong-id": + return Response.json( + { id: 999_999, jsonrpc: "2.0", result: { structuredContent: { results: [] } } }, + { headers: { "x-swarm-contract": "1" } }, + ) + case "blocked": + return rpc({ + content: [{ type: "text", text: "SSRF denied (fail-closed)" }], + isError: true, + structuredContent: { code: "ssrf_denied", message: "SSRF denied (fail-closed)" }, + }) + case "timeout-tool": + return rpc({ + content: [{ type: "text", text: "engine timed out" }], + isError: true, + structuredContent: { code: "upstream_timeout", message: "engine timed out" }, + }) + case "upstream-tool": + return rpc({ + content: [{ type: "text", text: "engine down" }], + isError: true, + structuredContent: { code: "upstream_error", message: "engine down" }, + }) + case "no-results": + return rpc({ structuredContent: { engine: "stub" } }) + case "no-content": + return rpc({ structuredContent: { title: "No body" } }) + case "sha-mismatch": + return rpc({ + structuredContent: { + content: FETCH_CONTENT, + content_sha256: sha256("something else entirely"), + final_url: "https://example.com/recorded", + title: "Recorded", + content_type: "text/plain", + }, + }) + case "success-fetch": + return rpc({ + structuredContent: { + content: FETCH_CONTENT, + content_sha256: sha256(FETCH_CONTENT), + final_url: "https://example.com/recorded", + title: "Recorded", + content_type: "text/plain", + }, + }) + case "success-search": + return rpc({ + structuredContent: { + query: body?.params?.arguments?.query ?? "", + results: [ + { + url: "https://gateway.test/docs", + title: "Gateway docs", + snippet: "Gateway snippet about the query.", + }, + ], + }, + }) + default: + return rpc({ + structuredContent: { + query: body?.params?.arguments?.query ?? "", + results: [ + { + url: "https://gateway.test/docs", + title: "Gateway docs", + snippet: "Gateway snippet about the query.", + }, + ], + }, + }) + } + }, + }) + searxng = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + const url = new URL(req.url) + if (url.pathname !== "/search") return new Response("not found", { status: 404 }) + searxngHits++ + if (searxngMode === "http-503") return new Response("unavailable", { status: 503 }) + if (searxngMode === "http-429-retry") + return new Response("slow down", { status: 429, headers: { "retry-after": "2" } }) + if (searxngMode === "http-429-bare") return new Response("slow down", { status: 429 }) + return Response.json({ + results: [ + { + url: "https://searxng.test/failover-docs", + title: "Failover docs", + content: `About ${url.searchParams.get("q")}: gates keep agents honest.`, + }, + ], + }) + }, + }) +}) + +afterAll(() => { + gateway?.stop(true) + searxng?.stop(true) +}) + +const reset = () => { + gatewayCalls = [] + searxngHits = 0 + gatewayMode = "success-search" + searxngMode = "success" +} + +/** Explicit env: ambient keys must not leak in (brave stays unavailable). */ +const envFor = (): NodeJS.ProcessEnv => ({ + ES_SCRAPER_SWARM_URL: `http://127.0.0.1:${gateway.port}`, + ES_SCRAPER_SWARM_TOKEN: TOKEN, + SEARXNG_URL: `http://127.0.0.1:${searxng.port}`, +}) + +const clock = (start: number) => { + let current = start + return { + now: () => current, + advance: (ms: number) => { + current += ms + }, + } +} + +const countingFetch = (text: string): { backend: SourceBackend; calls: () => number } => { + let calls = 0 + return { + calls: () => calls, + backend: { + id: "fallback", + capabilities: { search: false, fetch: true }, + available: () => true, + fetch: async (url: string) => { + calls++ + return { url, text, contentType: "text/plain" } + }, + }, + } +} + +describe("backoffFor defaults (#253.1)", () => { + test("a 429 Retry-After value wins; a bare 429 cools down for 60s", () => { + expect(backoffFor(new BackendError("rate_limited", "slow", { retryAfterMs: 5_000 }))).toBe(5_000) + expect(backoffFor(new BackendError("rate_limited", "slow"))).toBe(60_000) + }) + + test("every other kind cools down for 30s", () => { + for (const kind of ["unavailable", "auth", "upstream", "timeout", "blocked"] as const) + expect([kind, backoffFor(new BackendError(kind, "x"))]).toEqual([kind, 30_000]) + }) +}) + +describe("loopback failover through the real backends", () => { + test("a 503 on the gateway fails over in order; brave is skipped unavailable", async () => { + reset() + gatewayMode = "http-503" + const cooldowns = new Map() + const result = await runSearchChain( + [scraperSwarmBackend, braveBackend, searxngBackend], + "gates", + { + env: envFor(), + }, + { cooldowns }, + ) + expect(result.backend).toBe("searxng") + // Unavailable backends are reported first (skipped without a call), then + // the tried backends in order. + expect(result.attempts).toEqual([ + { backend: "brave", ok: false, kind: "unavailable" }, + { backend: "scraper-swarm", ok: false, kind: "upstream" }, + { backend: "searxng", ok: true }, + ]) + expect(result.value[0]?.snippet).toContain("gates keep agents honest") + expect(gatewayCalls).toHaveLength(1) + expect(searxngHits).toBe(1) + }) + + test("a 429 with Retry-After cools down: skipped, then retried after it lapses", async () => { + reset() + gatewayMode = "http-429-retry" + const env = envFor() + const timer = clock(1_000_000) + const cooldowns = new Map() + const first = await runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { env }, + { + now: timer.now, + cooldowns, + }, + ) + expect(first.backend).toBe("searxng") + expect(first.attempts[0]).toMatchObject({ backend: "scraper-swarm", ok: false, kind: "rate_limited" }) + expect(gatewayCalls).toHaveLength(1) + // Still cooling down: skipped without an HTTP call, same failover. + timer.advance(1_000) + const skipped = await runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { env }, + { + now: timer.now, + cooldowns, + }, + ) + expect(skipped.backend).toBe("searxng") + expect(skipped.attempts[0]).toMatchObject({ backend: "scraper-swarm", ok: false, kind: "rate_limited" }) + expect(gatewayCalls).toHaveLength(1) + // After the 2s Retry-After lapses the gateway is tried again. + timer.advance(1_500) + gatewayMode = "success-search" + const retried = await runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { env }, + { + now: timer.now, + cooldowns, + }, + ) + expect(retried.backend).toBe("scraper-swarm") + expect(gatewayCalls).toHaveLength(2) + }) + + test("a 429 without Retry-After defaults to a 60s cooldown over the wire", async () => { + reset() + gatewayMode = "http-429-bare" + const env = envFor() + const timer = clock(2_000_000) + const cooldowns = new Map() + const run = (at: number) => { + timer.advance(at) + return runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { env }, + { + now: timer.now, + cooldowns, + }, + ) + } + await run(0) + expect(gatewayCalls).toHaveLength(1) + await run(59_000) + expect(gatewayCalls).toHaveLength(1) + gatewayMode = "success-search" + const retried = await run(1_500) + expect(retried.backend).toBe("scraper-swarm") + expect(gatewayCalls).toHaveLength(2) + }) + + test("a provider-side 429 maps with and without its Retry-After value", async () => { + reset() + searxngMode = "http-429-retry" + // Fresh cooldown maps: the module-shared default would carry the first + // 429's cooldown into the second call and skip the backend instead. + const retrying = await runSearchChain( + [searxngBackend], + "gates", + { env: envFor() }, + { + cooldowns: new Map(), + }, + ).then( + () => "no-throw", + (error: unknown) => error, + ) + expect(retrying).toBeInstanceOf(BackendError) + expect(retrying).toMatchObject({ kind: "rate_limited", retryAfterMs: 2_000 }) + searxngMode = "http-429-bare" + const bare = await runSearchChain( + [searxngBackend], + "gates", + { env: envFor() }, + { + cooldowns: new Map(), + }, + ).then( + () => "no-throw", + (error: unknown) => error, + ) + expect(bare).toBeInstanceOf(BackendError) + expect(bare).toMatchObject({ kind: "rate_limited" }) + expect((bare as BackendError).retryAfterMs).toBeUndefined() + expect(backoffFor(bare as BackendError)).toBe(60_000) + }) + + test("a safety block stops the search chain: SearXNG is never called", async () => { + reset() + gatewayMode = "blocked" + const failure = await runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { + env: envFor(), + }, + { cooldowns: new Map() }, + ).then( + () => "no-throw", + (error: unknown) => error, + ) + expect(failure).toBeInstanceOf(BackendError) + expect(failure).toMatchObject({ kind: "blocked" }) + expect(searxngHits).toBe(0) + }) + + test("a safety block stops the fetch chain: the fallback is never called", async () => { + reset() + gatewayMode = "blocked" + const fallback = countingFetch("never served") + const failure = await runFetchChain([scraperSwarmBackend, fallback.backend], "https://example.com/x", { + env: envFor(), + cooldowns: new Map(), + }).then( + () => "no-throw", + (error: unknown) => error, + ) + expect(failure).toMatchObject({ kind: "blocked" }) + expect(fallback.calls()).toBe(0) + expect((failure as Error).message).toContain("SSRF denied") + }) + + test("search failover provenance names the serving backend; snippets are not cached", async () => { + reset() + gatewayMode = "http-503" + const dir = await mkdtemp(path.join(tmpdir(), "es-failover-")) + const state = await mkdtemp(path.join(tmpdir(), "es-failover-state-")) + try { + const env = envFor() + const tools = researchTools({ + root: dir, + env, + stateDir: state, + policy: async () => ({ root: dir }), + chain: [scraperSwarmBackend, braveBackend, searxngBackend], + }) + const search = tools.find((tool) => tool.name === "es_research_search")! + const text = await search.execute({ query: "gates" }, { agent: "es-research-alpha" }) + expect(text).toContain("searxng results for") + // SearXNG results are snippets, not page content: nothing is cached under them. + expect(await readdir(path.join(dir, ".factory/research/sources")).catch(() => [] as string[])).toEqual([]) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) + + test("fetch success through the gateway records sealed scraper-swarm provenance", async () => { + reset() + gatewayMode = "success-fetch" + const dir = await mkdtemp(path.join(tmpdir(), "es-failover-fetch-")) + const state = await mkdtemp(path.join(tmpdir(), "es-failover-fetch-state-")) + try { + const tools = researchTools({ + root: dir, + env: envFor(), + stateDir: state, + policy: async () => ({ root: dir }), + chain: [scraperSwarmBackend, searxngBackend], + }) + const fetchTool = tools.find((tool) => tool.name === "es_research_fetch")! + const text = await fetchTool.execute( + { url: "https://example.com/recorded" }, + { + agent: "es-research-alpha", + }, + ) + const hash = /sha256:([0-9a-f]{64})/.exec(text)?.[1] + expect(hash).toBe(sha256(FETCH_CONTENT)) + const stored = await new SourceCache(path.join(dir, ".factory/research/sources"), state).get(hash!) + expect(stored?.meta.provider).toBe("scraper-swarm") + expect(stored?.meta.seal).toMatch(/^[0-9a-f]{64}$/) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) +}) + +describe("auth continues the chain (#253.2)", () => { + test("a gateway HTTP 401 fails over to SearXNG and cools down", async () => { + reset() + gatewayMode = "http-401" + const env = envFor() + const timer = clock(3_000_000) + const cooldowns = new Map() + const first = await runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { env }, + { + now: timer.now, + cooldowns, + }, + ) + expect(first.backend).toBe("searxng") + expect(first.attempts[0]).toMatchObject({ backend: "scraper-swarm", ok: false, kind: "auth" }) + expect(gatewayCalls).toHaveLength(1) + timer.advance(1_000) + const second = await runSearchChain( + [scraperSwarmBackend, searxngBackend], + "gates", + { env }, + { + now: timer.now, + cooldowns, + }, + ) + expect(second.backend).toBe("searxng") + expect(gatewayCalls).toHaveLength(1) + }) + + test("a 401 search error and a 403 fetch error both fail over with a cooldown", async () => { + let searchCalls = 0 + let fetchCalls = 0 + const authSearch: SourceBackend = { + id: "auth-search", + capabilities: { search: true, fetch: false }, + available: () => true, + search: async () => { + searchCalls++ + throw new BackendError("auth", "HTTP 401", { status: 401 }) + }, + } + const authFetch: SourceBackend = { + id: "auth-fetch", + capabilities: { search: false, fetch: true }, + available: () => true, + fetch: async () => { + fetchCalls++ + throw new BackendError("auth", "HTTP 403", { status: 403 }) + }, + } + const goodSearch: SourceBackend = { + id: "good-search", + capabilities: { search: true, fetch: false }, + available: () => true, + search: async (query: string) => [{ url: `https://good.test/${query}`, title: "Good", snippet: "ok" }], + } + const goodFetch = countingFetch("served after auth") + const timer = clock(4_000_000) + const searchCooldowns = new Map() + const searched = await runSearchChain( + [authSearch, goodSearch], + "q", + {}, + { + now: timer.now, + cooldowns: searchCooldowns, + }, + ) + expect(searched.backend).toBe("good-search") + expect(searched.attempts[0]).toMatchObject({ backend: "auth-search", ok: false, kind: "auth" }) + timer.advance(1_000) + await runSearchChain([authSearch, goodSearch], "q", {}, { now: timer.now, cooldowns: searchCooldowns }) + expect(searchCalls).toBe(1) + const fetchCooldowns = new Map() + const fetched = await runFetchChain([authFetch, goodFetch.backend], "https://good.test/x", { + now: timer.now, + cooldowns: fetchCooldowns, + }) + expect(fetched.backend).toBe("fallback") + expect(fetched.attempts[0]).toMatchObject({ backend: "auth-fetch", ok: false, kind: "auth" }) + timer.advance(1_000) + await runFetchChain([authFetch, goodFetch.backend], "https://good.test/x", { + now: timer.now, + cooldowns: fetchCooldowns, + }) + expect(fetchCalls).toBe(1) + }) +}) + +describe("the token never leaves its Authorization header (#253.3)", () => { + const client = () => + new ScraperSwarmClient({ + url: `http://127.0.0.1:${gateway.port}`, + token: TOKEN, + fetch: fetch as unknown as typeof fetch, + }) + + test("every gateway failure message is token-free", async () => { + const searchModes: GatewayMode[] = [ + "http-401", + "http-429-bare", + "http-429-retry", + "http-503", + "contract-2", + "no-body", + "rpc-error", + "wrong-id", + "blocked", + "timeout-tool", + "upstream-tool", + "no-results", + ] + const expectedKind: Record = { + "http-401": "auth", + "http-429-bare": "rate_limited", + "http-429-retry": "rate_limited", + "http-503": "upstream", + "contract-2": "upstream", + "no-body": "upstream", + "rpc-error": "upstream", + "wrong-id": "upstream", + blocked: "blocked", + "timeout-tool": "timeout", + "upstream-tool": "upstream", + "no-results": "upstream", + } + for (const mode of searchModes) { + gatewayMode = mode + const failure = await client() + .search("q", 1) + .then( + () => "no-throw", + (error: unknown) => error, + ) + expect(failure).toBeInstanceOf(BackendError) + expect((failure as BackendError).kind).toBe(expectedKind[mode]) + expect((failure as Error).message).not.toContain(TOKEN) + expect(JSON.stringify(failure)).not.toContain(TOKEN) + } + const fetchModes: GatewayMode[] = ["no-content", "sha-mismatch", "blocked"] + for (const mode of fetchModes) { + gatewayMode = mode + const failure = await client() + .fetchPage("https://example.com/x") + .then( + () => "no-throw", + (error: unknown) => error, + ) + expect(failure).toBeInstanceOf(BackendError) + expect((failure as Error).message).not.toContain(TOKEN) + } + // An unreachable gateway and a missing config fail without the token too. + const down = new ScraperSwarmClient({ url: "http://127.0.0.1:1", token: TOKEN }) + for (const failure of [ + await down.search("q", 1).then( + () => "no-throw", + (error: unknown) => error, + ), + await down.fetchPage("https://example.com/x").then( + () => "no-throw", + (error: unknown) => error, + ), + ]) { + expect(failure).toMatchObject({ kind: "unavailable" }) + expect((failure as Error).message).not.toContain(TOKEN) + } + for (const failure of [ + await scraperSwarmBackend.search!("q", { env: {} }).then( + () => "no-throw", + (error: unknown) => error, + ), + await scraperSwarmBackend.fetch!("https://example.com/x", { env: {} }).then( + () => "no-throw", + (error: unknown) => error, + ), + ]) { + expect(failure).toMatchObject({ kind: "unavailable" }) + expect((failure as Error).message).not.toContain(TOKEN) + } + }) + + test("search-path and fetch-path cache writes hold no token", async () => { + reset() + gatewayMode = "success-fetch" + const dir = await mkdtemp(path.join(tmpdir(), "es-token-")) + const state = await mkdtemp(path.join(tmpdir(), "es-token-state-")) + try { + const contentful: SourceBackend = { + id: "contentful", + capabilities: { search: true, fetch: false }, + available: () => true, + search: async (_query: string) => [ + { + url: "https://contentful.test/docs", + title: "Contentful docs", + snippet: "a snippet", + content: "Contentful page text about gates, long enough to cache as a snapshot.", + }, + ], + } + const tools = researchTools({ + root: dir, + env: envFor(), + stateDir: state, + policy: async () => ({ root: dir }), + chain: [contentful, scraperSwarmBackend], + }) + const search = tools.find((tool) => tool.name === "es_research_search")! + const searched = await search.execute({ query: "gates" }, { agent: "es-research-alpha" }) + expect(searched).toContain("contentful results for") + expect(searched).toContain("sha256:") + const fetchTool = tools.find((tool) => tool.name === "es_research_fetch")! + await fetchTool.execute({ url: "https://example.com/recorded" }, { agent: "es-research-alpha" }) + const sources = path.join(dir, ".factory/research/sources") + const files = await readdir(sources) + expect(files.length).toBeGreaterThan(0) + for (const file of files) expect(await readFile(path.join(sources, file), "utf8")).not.toContain(TOKEN) + // The only wire presence of the token is the Authorization header. + expect(gatewayCalls.length).toBeGreaterThan(0) + for (const call of gatewayCalls) expect(call.auth).toBe(`Bearer ${TOKEN}`) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/opencode/test/failover-chain.test.ts b/packages/opencode/test/failover-chain.test.ts new file mode 100644 index 0000000..7e1b4fe --- /dev/null +++ b/packages/opencode/test/failover-chain.test.ts @@ -0,0 +1,185 @@ +// #188 / #253: failover chain on the real in-process host (testkit, scripted +// fake LLM). Loopback stubs only; no external network. Proves: a research +// seat's host websearch is denied while es_research_search is allowed, a 503 +// gateway fails over to SearXNG, a gateway safety block stops the fetch chain, +// and a 429 gateway is skipped while cooling down. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { boot, directiveScript } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +type GatewayMode = "http-503" | "http-429-retry" | "blocked" + +let gatewayMode: GatewayMode = "http-503" +let gatewayCalls: string[] = [] +let searxngHits = 0 +let gateway: ReturnType +let searxng: ReturnType +let state: string + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-failover-host-")) + searxng = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(req) { + const url = new URL(req.url) + if (url.pathname !== "/search") return new Response("not found", { status: 404 }) + searxngHits++ + return Response.json({ + results: [ + { + url: "https://searxng.test/host-docs", + title: "Host docs", + content: `About ${url.searchParams.get("q")}: gates keep agents honest.`, + }, + ], + }) + }, + }) + gateway = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + if (new URL(req.url).pathname !== "/mcp") return new Response("not found", { status: 404 }) + const body = (await req.json().catch(() => undefined)) as { id?: unknown; params?: { name?: string } } | undefined + const name = body?.params?.name ?? "" + gatewayCalls.push(name) + const id = body?.id ?? 1 + if (gatewayMode === "http-503") return new Response("unavailable", { status: 503 }) + if (gatewayMode === "http-429-retry") + return new Response("slow down", { status: 429, headers: { "retry-after": "60" } }) + return Response.json( + { + id, + jsonrpc: "2.0", + result: { + content: [{ type: "text", text: "SSRF denied (fail-closed)" }], + isError: true, + structuredContent: { code: "ssrf_denied", message: "SSRF denied (fail-closed)" }, + }, + }, + { headers: { "x-swarm-contract": "1" } }, + ) + }, + }) +}) + +afterAll(async () => { + gateway?.stop(true) + searxng?.stop(true) + await rm(state, { recursive: true, force: true }) +}) + +/** Boot a host with hermetic web env (ambient keys must not reach the wire). */ +async function bootHost(options: { gateway: boolean }) { + const keys = ["ES_SCRAPER_SWARM_URL", "ES_SCRAPER_SWARM_TOKEN", "SEARXNG_URL", "BRAVE_API_KEY", "FIRECRAWL_API_KEY"] + const saved = new Map(keys.map((key) => [key, process.env[key]] as const)) + delete process.env.BRAVE_API_KEY + delete process.env.FIRECRAWL_API_KEY + process.env.SEARXNG_URL = `http://127.0.0.1:${searxng.port}` + if (options.gateway) { + process.env.ES_SCRAPER_SWARM_URL = `http://127.0.0.1:${gateway.port}` + process.env.ES_SCRAPER_SWARM_TOKEN = "host-failover-token" + } else { + delete process.env.ES_SCRAPER_SWARM_URL + delete process.env.ES_SCRAPER_SWARM_TOKEN + } + const h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# failover host\n" }, + }) + return { + h, + async close() { + await h.close() + for (const [key, value] of saved) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + }, + } +} + +describe("failover chain on the real host", () => { + test("a research seat's host websearch is denied; es_research_search is allowed", async () => { + const { h, close } = await bootHost({ gateway: false }) + try { + const denied = await h.run(call("websearch", { query: "gates" }), { agent: "es-research-alpha" }) + expect(denied.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["websearch:error"]) + expect(denied.tools[0]?.text).toContain("only through es_research_search and es_research_fetch") + const allowed = await h.run(call("es_research_search", { query: "gates" }), { + agent: "es-research-alpha", + }) + expect(allowed.tools[0]?.status).toBe("completed") + expect(allowed.tools[0]?.text).toContain("searxng results for") + expect(allowed.tools[0]?.text).toContain("gates keep agents honest") + } finally { + await close() + } + }, 60_000) + + test("a 503 gateway fails over to SearXNG on the host", async () => { + gatewayMode = "http-503" + gatewayCalls = [] + searxngHits = 0 + const { h, close } = await bootHost({ gateway: true }) + try { + const { tools } = await h.run(call("es_research_search", { query: "gates" }), { + agent: "es-research-alpha", + }) + expect(tools[0]?.status).toBe("completed") + expect(tools[0]?.text).toContain("searxng results for") + expect(tools[0]?.text).toContain("gates keep agents honest") + expect(gatewayCalls).toContain("web_search") + expect(searxngHits).toBe(1) + } finally { + await close() + } + }, 60_000) + + test("a gateway safety block stops the host fetch chain", async () => { + gatewayMode = "blocked" + gatewayCalls = [] + const { h, close } = await bootHost({ gateway: true }) + try { + const { tools } = await h.run(call("es_research_fetch", { url: "https://example.com/recorded" }), { + agent: "es-research-alpha", + }) + expect(tools[0]?.status).toBe("error") + expect(tools[0]?.text).toContain("SSRF denied") + expect(gatewayCalls).toContain("fetch_page") + } finally { + await close() + } + }, 60_000) + + test("a 429 gateway is skipped while cooling down on the host", async () => { + gatewayMode = "http-429-retry" + gatewayCalls = [] + const { h, close } = await bootHost({ gateway: true }) + try { + const first = await h.run(call("es_research_search", { query: "gates one" }), { + agent: "es-research-alpha", + }) + expect(first.tools[0]?.status).toBe("completed") + expect(first.tools[0]?.text).toContain("searxng results for") + expect(gatewayCalls).toHaveLength(1) + // Still inside the 60s Retry-After: the gateway is not called again. + const second = await h.run(call("es_research_search", { query: "gates two" }), { + agent: "es-research-alpha", + }) + expect(second.tools[0]?.status).toBe("completed") + expect(second.tools[0]?.text).toContain("searxng results for") + expect(gatewayCalls).toHaveLength(1) + } finally { + await close() + } + }, 60_000) +}) From c0352b37c4704ca6ec0307a9ff6b695a3f855a85 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 15:14:34 -0400 Subject: [PATCH 15/83] test(audit): domain packs and tiering integration tests (#190) --- packages/core/test/domainpacks-packs.test.ts | 170 ++++++++++++++++++ .../opencode/test/fires-domainpacks.test.ts | 147 +++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 packages/core/test/domainpacks-packs.test.ts create mode 100644 packages/opencode/test/fires-domainpacks.test.ts diff --git a/packages/core/test/domainpacks-packs.test.ts b/packages/core/test/domainpacks-packs.test.ts new file mode 100644 index 0000000..4d5b105 --- /dev/null +++ b/packages/core/test/domainpacks-packs.test.ts @@ -0,0 +1,170 @@ +// Per-pack coverage closers for #256 (audit #190 follow-up): every shipped +// pack's banned domains and threshold/weight values pin current behavior, the +// below-threshold refusal path at machine.ts:545 is exercised, and the +// SECONDARY_MEDIA / tierDomains dead config is documented as observed. +// Scoring itself is NOT changed here (#252 owns the scoring defects: tierless +// weight, NK inflation, garbage tags); these tests pin the mechanism only. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, writeFile } from "node:fs/promises" +import path from "node:path" +import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { + type Constitution, + claimVerdict, + constitutionFromPack, + Factory, + factoryLayout, + gateRunner, + LEGACY_CONSTITUTION, + loadDomainPack, + recordApproval, + researchSourcesDir, + SourceCache, +} from "../src/index.ts" +import { researchOptions } from "../src/research/ops.ts" +import { classifySourceTier } from "../src/research/tier.ts" +import { type Fixture, frontier, gitRepo } from "./helpers.ts" + +const pack = async (id: string): Promise => constitutionFromPack(await loadDomainPack(id)) + +describe("each shipped pack bans its own domains (#256.1)", () => { + const cases = [ + { packId: "quant", domain: "vendor-whitepaper-mirror.example" }, + { packId: "legal", domain: "undisclosed-sponsored-content.example" }, + { packId: "biopharma", domain: "predatory-journal.example" }, + ] as const + for (const { packId, domain } of cases) { + test(`${packId} rejects a source on ${domain} (BANNED_DOMAIN), other packs and legacy accept it`, async () => { + const claim = { + id: `${packId}-banned`, + tag: "VERIFIED", + source: { + sha256: "b".repeat(64), + quote: "the whitepaper claims linear scaling on every desk", + url: `https://${domain}/research-note`, + }, + } + const own = claimVerdict(claim, { constitution: await pack(packId) }) + expect(own.verdict).toBe("REJECTED") + expect(own.reasons.some((reason) => reason.startsWith("BANNED_DOMAIN"))).toBe(true) + // Pack-agnostic baseline unchanged: legacy accepts the same claim. + expect(claimVerdict(claim, { constitution: LEGACY_CONSTITUTION }).verdict).toBe("ACCEPTED") + // Specificity: a sibling pack with a different ban list accepts it. + const sibling = cases.find((entry) => entry.packId !== packId)!.packId + expect(claimVerdict(claim, { constitution: await pack(sibling) }).verdict).toBe("ACCEPTED") + }) + } +}) + +describe("pack threshold and weight values are pinned (#256.1)", () => { + test("accept thresholds: biopharma 0.7, quant 0.7, legal 0.65", async () => { + expect((await pack("biopharma")).acceptThreshold).toBe(0.7) + expect((await pack("quant")).acceptThreshold).toBe(0.7) + expect((await pack("legal")).acceptThreshold).toBe(0.65) + }) + + test("spot tier weights: biopharma downweights preprints, legal prefers primary press, quant values docs", async () => { + expect((await pack("biopharma")).tierWeights.PREPRINT).toBe(0.3) + expect((await pack("biopharma")).tierWeights.PEER_REVIEWED).toBe(1.0) + expect((await pack("legal")).tierWeights.PRIMARY_PRESS).toBe(1.0) + expect((await pack("legal")).tierWeights.PREPRINT).toBe(0.2) + expect((await pack("quant")).tierWeights.TECHNICAL_DOCUMENTATION).toBe(1.0) + expect((await pack("quant")).tierWeights.PREPRINT).toBe(0.5) + }) + + test("ban lists and retraction policies are pinned per pack", async () => { + expect((await pack("biopharma")).bannedDomains).toEqual([ + "predatory-journal.example", + "retracted-mirror.example", + "banned-preprint-mirror.example", + ]) + expect((await pack("quant")).bannedDomains).toEqual(["vendor-whitepaper-mirror.example"]) + expect((await pack("legal")).bannedDomains).toEqual(["undisclosed-sponsored-content.example"]) + expect((await pack("biopharma")).retractionPolicy).toBe("zero_tolerance") + expect((await pack("quant")).retractionPolicy).toBe("standard") + expect((await pack("legal")).retractionPolicy).toBe("standard") + expect((await pack("quant")).negBonus).toBe(0.75) + expect((await pack("quant")).rejectPenalty).toBe(3.0) + }) +}) + +describe("the below-threshold refusal path (#256.2)", () => { + let fx: Fixture + const PASSPHRASE = "test-passphrase-1234" + let signer: HumanSigner + beforeEach(async () => { + fx = await gitRepo() + await sealHumanKey(PASSPHRASE, fx.state) + signer = await unlockHumanKey(PASSPHRASE, fx.state) + }) + afterEach(() => fx.cleanup()) + + test("a preprint-only report passes every per-claim rule yet is refused below the biopharma threshold", async () => { + const factory = new Factory(fx.root, { + gates: gateRunner({ stateDir: fx.state }), + stateDir: fx.state, + domainPack: "biopharma", + }) + await factory.begin("human:tester") + await factory.writeFrontier("grill", frontier()) + await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer }) + await factory.beginResearch("human:tester") + // arxiv.org classifies PREPRINT (weight 0.3 under biopharma), so one + // verified claim scores 0.3 < 0.7 with no per-claim rule firing. + const cache = new SourceCache(researchSourcesDir(fx.root), fx.state) + const source = await cache.put({ + url: "https://arxiv.org/abs/2601.00001", + text: "A preprint model predicts surrogate endpoints in early cohorts.", + provider: "fetch", + }) + expect(source.meta.tier).toBe("PREPRINT") + await mkdir(path.dirname(factoryLayout(fx.root).researchReport), { recursive: true }) + await writeFile( + factoryLayout(fx.root).researchReport, + `- A preprint predicts surrogate endpoints [VERIFIED: sha256:${source.meta.sha256} "predicts surrogate endpoints"]\n`, + ) + let message: string | undefined + try { + await factory.completeResearch("factory") + } catch (caught) { + message = caught instanceof Error ? caught.message : String(caught) + } + expect(message).toBeDefined() + // The machine.ts:545 threshold refusal, not the per-claim rejection path. + expect(message).toContain("below its threshold") + expect(message).toContain("biopharma") + expect(message).not.toContain("rejects") + }) +}) + +describe("dead tier config, documented as observed (#256.3 — do not fix here)", () => { + test("SECONDARY_MEDIA is weighted in all three packs but the classifier never emits it", async () => { + // Observed: every shipped pack carries a SECONDARY_MEDIA weight ... + expect((await loadDomainPack("biopharma")).tierWeights.SECONDARY_MEDIA).toBe(0.2) + expect((await loadDomainPack("quant")).tierWeights.SECONDARY_MEDIA).toBe(0.2) + expect((await loadDomainPack("legal")).tierWeights.SECONDARY_MEDIA).toBe(0.3) + // ... yet tier.ts has no rule returning it, not even for news-like hosts. + for (const url of [ + "https://arxiv.org/abs/1", + "https://doi.org/10.1/x", + "https://docs.python.org/3/", + "https://www.fda.gov/news/x", + "https://www.bbc.com/news/science-1", + "https://www.nytimes.com/2026/01/01/health/x.html", + "https://blog.example.com/opinion", + "https://example.test/x", + ]) + expect([url, classifySourceTier(url)]).toEqual([url, expect.not.stringMatching("SECONDARY_MEDIA")]) + }) + + test("tierDomains overrides reach no caller: no shipped pack sets them and researchOptions drops them", async () => { + // Observed: the schema, cache.put and ops plumbing accept tierDomains, + // but no shipped pack file sets them ... + for (const id of ["biopharma", "quant", "legal"]) + expect([(await loadDomainPack(id)).packId, (await loadDomainPack(id)).tierDomains]).toEqual([id, undefined]) + // ... and the config-to-tools mapping never forwards them (verified by + // grep: zero assignments to ResearchOpsContext.tierDomains repo-wide), so + // a pack can never re-tier a host at runtime today. + expect("tierDomains" in researchOptions({ research: { depth: 2 } })).toBe(false) + }) +}) diff --git a/packages/opencode/test/fires-domainpacks.test.ts b/packages/opencode/test/fires-domainpacks.test.ts new file mode 100644 index 0000000..45e3db3 --- /dev/null +++ b/packages/opencode/test/fires-domainpacks.test.ts @@ -0,0 +1,147 @@ +// Biopharma pack on the real in-process host (#190 integration 2): a +// pack-active testkit research run with a banned-domain source and a +// retracted source. es_research_complete rejects the banned claim, and after +// a clean completion the retracted source renders STALE. Pack-active, not +// pack-agnostic: the stage.done audit entry records domainPack biopharma. +// Real host via packages/testkit (scripted fake LLM); no internet. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { Factory, factoryLayout, gateRunner, renderResearchRun, researchCache } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const child = (name: string, args: Record = {}) => `@@CHILD ${name} ${JSON.stringify(args)}@@` +const delegate = (agent: string, prompt: string) => call("subagent", { agent, description: `${agent} step`, prompt }) + +const PAGE_CLEAN = "

The trial met its primary endpoint in the per-protocol analysis.

" +const PAGE_RETRACT = + "

Surrogate endpoints predicted long-term survival in the cohort study.

" + +let state: string +let pages: ReturnType +let previousLoopback: string | undefined +const PORT = 18481 + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-fires-domainpacks-")) + // The direct backend refuses loopback by default (SSRF guard); the + // loopback-only fixture server below needs the explicit test allowance. + previousLoopback = process.env.ES_RESEARCH_ALLOW_LOOPBACK + process.env.ES_RESEARCH_ALLOW_LOOPBACK = "1" + pages = Bun.serve({ + port: PORT, + hostname: "127.0.0.1", + fetch(req) { + const url = new URL(req.url) + if (url.pathname === "/clean") return new Response(PAGE_CLEAN, { headers: { "content-type": "text/html" } }) + if (url.pathname === "/retract") return new Response(PAGE_RETRACT, { headers: { "content-type": "text/html" } }) + return new Response("not found", { status: 404 }) + }, + }) +}) + +afterAll(async () => { + pages?.stop(true) + if (previousLoopback === undefined) delete process.env.ES_RESEARCH_ALLOW_LOOPBACK + else process.env.ES_RESEARCH_ALLOW_LOOPBACK = previousLoopback + await rm(state, { recursive: true, force: true }) +}) + +async function fetchSha(h: Harness, page: string): Promise { + const fetched = await h.run(call("es_research_fetch", { url: page }), { agent: "es-research-alpha" }) + expect(fetched.tools[0]?.status).toBe("completed") + const sha = /sha256:([0-9a-f]{64})/.exec(fetched.tools[0]?.text ?? "")?.[1] + expect(sha).toBeDefined() + return sha! +} + +async function writeReport(h: Harness, content: string) { + const written = await h.run( + delegate( + "es-research-synthesizer", + `Resolve ${child("write", { path: ".factory/research/REPORT.md", content })} done`, + ), + { agent: "deep-researcher" }, + ) + expect(written.tools[0]?.status).toBe("completed") +} + +describe("biopharma pack on the real host (#190.2)", () => { + test("banned-domain claim rejected at es_research_complete; retracted source renders STALE", async () => { + const h = await boot({ + git: true, + script: directiveScript, + // Pack-active run: the plugin's configured constitution vets the report. + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off", domainPack: "biopharma" } }], + files: { "README.md": "# pack run\n" }, + }) + try { + // The run itself is begun pack-agnostic (as in fires-research); the + // pack enters at completion through the plugin's configured factory. + const factory = new Factory(h.directory, { + gates: gateRunner({ stateDir: state }), + stateDir: state, + }) + await factory.beginResearchRun({ objective: "Does the therapy hold?", ceilingUSD: 5 }) + + const shaClean = await fetchSha(h, `http://127.0.0.1:${PORT}/clean`) + const shaRetract = await fetchSha(h, `http://127.0.0.1:${PORT}/retract`) + // No network fixture can serve a banned host, so the banned-domain + // source is sealed straight into the project cache (same engine seal + // the fetch path mints, same state dir the plugin verifies with). + const banned = await researchCache(h.directory, state).put({ + url: "https://predatory-journal.example/x", + text: "A mirror preprint claims the surrogate endpoint is fully validated and ready for filing.", + provider: "fetch", + }) + + const bannedLine = `- The surrogate is fully validated [VERIFIED: sha256:${banned.meta.sha256} "surrogate endpoint is fully validated"]` + const cleanLine = `- The trial met its primary endpoint [VERIFIED: sha256:${shaClean} "met its primary endpoint"]` + const retractLine = `- Surrogates predicted survival [VERIFIED: sha256:${shaRetract} "predicted long-term survival"]` + + // A banned-domain claim is grounded (verbatim quote, sealed source) so + // the epistemic audit passes and the pack gate is what refuses it. + await writeReport(h, `# Report\n\n${bannedLine}\n${cleanLine}\n`) + const refused = await h.run(call("es_research_complete"), { agent: "deep-researcher" }) + expect(refused.tools[0]?.status).toBe("error") + expect(refused.tools[0]?.text).toContain("BANNED_DOMAIN") + expect(refused.tools[0]?.text).toContain("biopharma") + + // Without the banned claim the pack-active report completes to DONE. + await writeReport(h, `# Report\n\n${cleanLine}\n${retractLine}\n`) + const completed = await h.run(call("es_research_complete"), { agent: "deep-researcher" }) + expect(completed.tools[0]?.status).toBe("completed") + expect((await factory.read())?.stage).toBe("DONE") + + // Pack-active, not pack-agnostic: the completion audit records the pack. + const auditText = await Bun.file(factoryLayout(h.directory).audit).text() + const done = auditText + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { action: string; payload: Record }) + .filter((entry) => entry.action === "stage.done") + expect(done.length).toBeGreaterThan(0) + expect(done.at(-1)?.payload.domainPack).toBe("biopharma") + + // Retracting a cited source degrades its claims to STALE in the render. + await mkdir(path.join(h.directory, ".factory/claims"), { recursive: true }) + await writeFile( + path.join(h.directory, ".factory/claims/retractions.json"), + JSON.stringify({ + version: 1, + retractions: [ + { source: shaRetract, event: "RETRACTED", at: new Date().toISOString(), note: "withdrawn", by: "tester" }, + ], + }), + ) + const md = await renderResearchRun(h.directory, { format: "md", stateDir: state }) + expect(md).toContain("STALE") + expect(md).toContain(shaRetract.slice(0, 12)) + } finally { + await h.close() + } + }, 120_000) +}) From fc38144f85fcb00167408026c04391e3319ad681 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 15:14:55 -0400 Subject: [PATCH 16/83] test(audit): OSS scout verdict integration tests (#194) --- packages/core/test/scout-194.test.ts | 119 ++++++++ packages/opencode/test/scout-194.test.ts | 371 +++++++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 packages/core/test/scout-194.test.ts create mode 100644 packages/opencode/test/scout-194.test.ts diff --git a/packages/core/test/scout-194.test.ts b/packages/core/test/scout-194.test.ts new file mode 100644 index 0000000..a432b76 --- /dev/null +++ b/packages/core/test/scout-194.test.ts @@ -0,0 +1,119 @@ +// Issue #194 ("2. Integration tests", unit-level companion): the verdicts +// core computes when the scout proposes `adopt`. scout.test.ts already pins +// the default-whitelist matrix (GPL/AGPL/none -> clean-room); this suite pins +// the variants the fires suite does not: AGPL + unlicensed downgrade reasons +// spelled out, and a narrowed configured whitelist demoting even a verified +// permissive license. Product code is untouched; failures here are findings. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { readScoutResult, scoutTools } from "../src/index.ts" + +const license = (id: string) => readFileSync(path.join(import.meta.dir, "fixtures/licenses", `${id}.txt`), "utf8") + +let root: string +let state: string +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-scout-194-")) + state = await mkdtemp(path.join(tmpdir(), "es-scout-194-state-")) + const vendor: Record = { + mit: license("MIT"), + apache: license("Apache-2.0"), + gpl: license("GPL-3.0-only"), + agpl: license("AGPL-3.0-only"), + none: undefined, + } + for (const [name, text] of Object.entries(vendor)) { + const dir = path.join(root, "vendor", name) + await mkdir(dir, { recursive: true }) + await writeFile(path.join(dir, "index.js"), `module.exports = function ${name}() { return 1 }\n`) + await writeFile( + path.join(dir, "package.json"), + JSON.stringify({ name, version: "1.0.0", dependencies: { leftpad: "1" } }), + ) + if (text) await writeFile(path.join(dir, "LICENSE"), text) + } +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const scout = { agent: "scout" } +const tools = + (extra: Record = {}) => + (name: string) => + scoutTools({ root, stateDir: state, ...extra }).find((tool) => tool.name === name)! + +async function plan(tool: ReturnType, names: string[], extra: Record = {}) { + return tool("es_scout_plan").execute( + { + objective: "a tiny function library", + posture: "Apache-2.0, commercial", + candidates: names.map((name) => ({ name, source: `local:./vendor/${name}` })), + ...extra, + }, + scout, + ) +} + +async function adoptAll(tool: ReturnType, names: string[]) { + for (const name of names) { + await tool("es_scout_scan").execute({ candidate: name }, scout) + await tool("es_scout_record").execute( + { candidate: name, maintenance: "stable", proposal: "adopt", rationale: "small, focused and tested", claims: [] }, + scout, + ) + } + await tool("es_scout_complete").execute({}, scout) + return (await readScoutResult(root))! +} + +describe("issue #194: copyleft and unlicensed candidates cannot keep adopt", () => { + test("AGPL, GPL and unlicensed proposals are downgraded to clean-room with reasons", async () => { + const tool = tools() + await plan(tool, ["mit", "gpl", "agpl", "none"]) + const result = await adoptAll(tool, ["mit", "gpl", "agpl", "none"]) + const byId = Object.fromEntries(result.verdicts.map((item) => [item.candidate, item])) + expect(byId.mit?.verdict).toBe("adopt") + for (const id of ["gpl", "agpl"]) { + expect([id, byId[id]?.proposal, byId[id]?.verdict]).toEqual([id, "adopt", "clean-room"]) + expect(byId[id]?.downgraded).toContain("copyleft") + } + expect([byId.none?.proposal, byId.none?.verdict]).toEqual(["adopt", "clean-room"]) + expect(byId.none?.downgraded).toMatch(/unknown|unverified/) + }) +}) + +describe("issue #194: a narrowed configured whitelist demotes verified permissive licenses", () => { + test("Apache-2.0 proposed adopt is clean-room when the configured whitelist is MIT-only", async () => { + const tool = tools({ whitelist: ["MIT"] }) + await plan(tool, ["mit", "apache"]) + const result = await adoptAll(tool, ["mit", "apache"]) + const byId = Object.fromEntries(result.verdicts.map((item) => [item.candidate, item])) + expect([byId.mit?.proposal, byId.mit?.verdict]).toEqual(["adopt", "adopt"]) + expect([byId.apache?.proposal, byId.apache?.verdict]).toEqual(["adopt", "clean-room"]) + expect(byId.apache?.downgraded).toContain("not on the license whitelist") + }) + + test("a plan cannot widen the configured whitelist: the extra id is ignored, the verdict still demotes", async () => { + const tool = tools({ whitelist: ["MIT"] }) + const planned = await plan(tool, ["apache"], { whitelist: ["MIT", "Apache-2.0"] }) + expect(planned).toContain("Ignored (a plan can only narrow the configured whitelist): Apache-2.0") + const result = await adoptAll(tool, ["apache"]) + const verdict = result.verdicts.find((item) => item.candidate === "apache")! + expect([verdict.proposal, verdict.verdict]).toEqual(["adopt", "clean-room"]) + expect(verdict.downgraded).toContain("not on the license whitelist") + }) + + test("an empty configured whitelist leaves even MIT at clean-room", async () => { + const tool = tools({ whitelist: [] }) + await plan(tool, ["mit"]) + const result = await adoptAll(tool, ["mit"]) + const verdict = result.verdicts.find((item) => item.candidate === "mit")! + expect([verdict.proposal, verdict.verdict]).toEqual(["adopt", "clean-room"]) + expect(verdict.downgraded).toContain("not on the license whitelist") + }) +}) diff --git a/packages/opencode/test/scout-194.test.ts b/packages/opencode/test/scout-194.test.ts new file mode 100644 index 0000000..9eaca86 --- /dev/null +++ b/packages/opencode/test/scout-194.test.ts @@ -0,0 +1,371 @@ +// Issue #194 ("2. Integration tests") on the real in-process OpenCode v2 host +// (testkit boot, scripted fake LLM, beside fires.test.ts): +// - a scout-proposed `adopt` for GPL/AGPL/unlicensed candidates is downgraded +// to clean-room by core, including under a narrowed license whitelist; +// - a candidate with OSV advisories shows the citation (cached sha256, +// witnessed claims, verdict warnings) at the host level; +// - probe (NOT a fix) of bug #257: the advisory step is evadable +// (skip / wrong-package / fixed-version all yield a clean adopt today); +// - scout-named sandbox probes: host webfetch/websearch denied and an +// offline shell for the `scout` seat specifically (alpha probes exist in +// scoping.test.ts; these name the scout). +// Product code is untouched: a red test here is a finding, never a fix. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + bwrapAvailable, + factoryLayout, + gradeScoutFire, + readScoutResult, + researchSourcesDir, + SourceCache, +} from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness, lastAgentRequest } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const fixtures = path.resolve(import.meta.dir, "../../core/test/fixtures") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const read = (file: string) => readFileSync(path.join(fixtures, file), "utf8") +const write = async (root: string, file: string, text: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), text) +} +const plantVendor = async (root: string, name: string, licenseFile: string | null) => { + const dir = `vendor/${name}` + await write(root, `${dir}/index.js`, `module.exports = function ${name}() { return 1 }\n`) + await write(root, `${dir}/package.json`, JSON.stringify({ name, version: "1.0.0" })) + if (licenseFile) await write(root, `${dir}/LICENSE`, read(`licenses/${licenseFile}.txt`)) +} +const recordAdopt = (name: string) => + call("es_scout_record", { + candidate: name, + maintenance: "stable", + proposal: "adopt", + rationale: "it does what we need", + claims: [], + }) + +// Canned OSV /v1/query body (same shape as packages/core/test/scout.test.ts): +// one HIGH with a fix, one CRITICAL without, one LOW (listed, not flagged). +const OSV = { + vulns: [ + { + id: "GHSA-aaaa-bbbb-cccc", + aliases: ["CVE-2026-0001"], + summary: "Prototype pollution in merge()", + database_specific: { severity: "HIGH" }, + affected: [{ ranges: [{ type: "SEMVER", events: [{ introduced: "0" }, { fixed: "1.0.1" }] }] }], + }, + { + id: "GHSA-dddd-eeee-ffff", + summary: "ReDoS in the parser\nlong details", + database_specific: { severity: "CRITICAL" }, + affected: [{ ranges: [{ type: "SEMVER", events: [{ introduced: "0" }] }] }], + }, + { id: "GHSA-gggg-hhhh-iiii", summary: "Minor info leak", database_specific: { severity: "LOW" }, affected: [] }, + ], +} + +// The host plugin calls the global fetch for api.osv.dev (no fetch is +// injected in packages/opencode/src/tools.ts), so the host-level advisory +// tests stub it process-wide and delegate everything else (notably the fake +// LLM traffic) to the real fetch. +const realFetch = globalThis.fetch +let osvBodies: unknown[] = [] +function stubOsv() { + osvBodies = [] + globalThis.fetch = (async (input: any, init?: any) => { + const url = String(input?.url ?? input) + if (url.includes("api.osv.dev")) { + const body = JSON.parse(String(init?.body ?? "{}")) + osvBodies.push(body) + if (body.version === "9.9.9") return Response.json({ vulns: [] }) + return Response.json(OSV) + } + return realFetch(input, init) + }) as unknown as typeof fetch +} +const unstubOsv = () => { + globalThis.fetch = realFetch +} + +let state: string +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-scout-194-")) +}) +afterAll(async () => { + unstubOsv() + await rm(state, { recursive: true, force: true }) +}) + +async function bootHost(options: Record = {}): Promise { + return boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off", ...options } }], + files: { "README.md": "# scout-194\n" }, + }) +} + +describe("issue #194: core downgrades scout-proposed adopt on the real host", () => { + let h: Harness + beforeAll(async () => { + h = await bootHost() + }, 60_000) + afterAll(() => h?.close()) + + test("GPL, AGPL and unlicensed candidates end clean-room with reasons; MIT stays adopt", async () => { + const names = ["mit", "gpl", "agpl", "none"] as const + const licenses: Record = { + mit: "MIT", + gpl: "GPL-3.0-only", + agpl: "AGPL-3.0-only", + none: null, + } + for (const name of names) await plantVendor(h.directory, name, licenses[name]!) + const calls = [ + call("es_scout_plan", { + objective: "a function library", + candidates: names.map((name) => ({ name, source: `local:./vendor/${name}` })), + }), + ...names.map((name) => call("es_scout_scan", { candidate: name })), + ...names.map(recordAdopt), + call("es_scout_complete"), + ] + const { tools } = await h.run(`scout ${calls.join(" ")}`, { agent: "scout" }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(tools.map((tool) => `${tool.name}:completed`)) + const result = (await readScoutResult(h.directory))! + const byId = Object.fromEntries(result.verdicts.map((item) => [item.candidate, item])) + expect([byId.mit?.proposal, byId.mit?.verdict]).toEqual(["adopt", "adopt"]) + for (const id of ["gpl", "agpl"]) { + expect([id, byId[id]?.proposal, byId[id]?.verdict]).toEqual([id, "adopt", "clean-room"]) + expect(byId[id]?.downgraded).toContain("copyleft") + } + expect([byId.none?.proposal, byId.none?.verdict]).toEqual(["adopt", "clean-room"]) + expect(byId.none?.downgraded).toMatch(/unknown|unverified/) + const grade = gradeScoutFire(result, [ + { name: "mit", verdict: "adopt" }, + { name: "gpl", verdict: "clean-room" }, + { name: "agpl", verdict: "clean-room" }, + { name: "none", verdict: "clean-room" }, + ]) + expect(grade).toEqual({ pass: true, caught: ["mit", "gpl", "agpl", "none"], failures: [] }) + }, 120_000) + + test("scout-named sandbox: host webfetch and websearch are denied for the scout seat", async () => { + for (const tool of ["webfetch", "websearch"]) { + const args = tool === "webfetch" ? { url: "https://example.com" } : { query: "oss yaml parser" } + const { tools } = await h.run(`probe ${call(tool, args)}`, { agent: "scout" }) + expect(tools.map((outcome) => `${outcome.name}:${outcome.status}`)).toEqual([`${tool}:error`]) + expect(tools[0]?.text).toContain("only through es_research_search and es_research_fetch") + } + }, 60_000) + + test("scout-named sandbox: the scout is offered no host web tools, but the cached research tools", async () => { + await h.run("hello", { agent: "scout" }) + const offered = new Set((lastAgentRequest(h.llm.requests)?.tools ?? []).map((tool) => tool.function.name)) + expect(offered.has("webfetch")).toBe(false) + expect(offered.has("websearch")).toBe(false) + expect(offered.has("es_research_fetch")).toBe(true) + expect(offered.has("es_scout_complete")).toBe(true) + }, 60_000) + + test("scout-named sandbox: a scout shell has no network", async () => { + if (!bwrapAvailable()) { + const { tools } = await h.run(`net ${call("shell", { command: "cat /proc/net/dev" })}`, { agent: "scout" }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:error"]) + expect(tools[0]?.text).toContain("bubblewrap") + return + } + const { tools } = await h.run(`net ${call("shell", { command: "cat /proc/net/dev" })}`, { agent: "scout" }) + const text = tools[0]?.text ?? "" + if (/bwrap:|bubblewrap/i.test(text)) { + // This environment cannot make the overlay mount, so the sandboxed + // command never runs (the canonical alpha probe in scoping.test.ts + // fails the same way here): no network was available either. Pin the + // sandbox failure strictly instead of the interface list. + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + expect(text).toContain("Exited with code") + return + } + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + const ifaces = tools[0]?.text + .split("\n") + .filter((line) => line.includes(":")) + .map((line) => line.split(":")[0]?.trim() ?? "") + .filter(Boolean) + expect(ifaces).toEqual(["lo"]) + }, 60_000) +}) + +describe("issue #194: a narrowed license whitelist demotes verified permissive licenses on the host", () => { + let h: Harness + beforeAll(async () => { + h = await bootHost({ licenseWhitelist: ["MIT"] }) + }, 60_000) + afterAll(() => h?.close()) + + test("MIT keeps adopt; Apache-2.0 proposed adopt ends clean-room", async () => { + await plantVendor(h.directory, "mit", "MIT") + await plantVendor(h.directory, "apache", "Apache-2.0") + const names = ["mit", "apache"] + const calls = [ + call("es_scout_plan", { + objective: "a function library", + candidates: names.map((name) => ({ name, source: `local:./vendor/${name}` })), + }), + ...names.map((name) => call("es_scout_scan", { candidate: name })), + ...names.map(recordAdopt), + call("es_scout_complete"), + ] + const { tools } = await h.run(`scout ${calls.join(" ")}`, { agent: "scout" }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(tools.map((tool) => `${tool.name}:completed`)) + const result = (await readScoutResult(h.directory))! + const byId = Object.fromEntries(result.verdicts.map((item) => [item.candidate, item])) + expect([byId.mit?.proposal, byId.mit?.verdict]).toEqual(["adopt", "adopt"]) + expect([byId.apache?.proposal, byId.apache?.verdict]).toEqual(["adopt", "clean-room"]) + expect(byId.apache?.downgraded).toContain("not on the license whitelist") + }, 120_000) +}) + +describe("issue #194: OSV advisories are cited by hash on the real host", () => { + let h: Harness + beforeAll(async () => { + h = await bootHost() + await plantVendor(h.directory, "mit", "MIT") + }, 60_000) + afterAll(() => h?.close()) + + test("advisories are fetched, cached, cited by sha256, witnessed, and flagged on the verdict", async () => { + stubOsv() + try { + const planned = await h.run( + call("es_scout_plan", { + objective: "a function library", + candidates: [{ name: "mit", source: "local:./vendor/mit" }], + }), + { agent: "scout" }, + ) + expect(planned.tools[0]?.status).toBe("completed") + const scanned = await h.run(call("es_scout_scan", { candidate: "mit" }), { agent: "scout" }) + expect(scanned.tools[0]?.status).toBe("completed") + const advisories = await h.run( + call("es_scout_advisories", { candidate: "mit", ecosystem: "npm", package: "mit-lib", version: "1.0.0" }), + { agent: "scout" }, + ) + expect(advisories.tools[0]?.status).toBe("completed") + expect(osvBodies).toEqual([{ package: { name: "mit-lib", ecosystem: "npm" }, version: "1.0.0" }]) + const hash = /sha256:([0-9a-f]{64})/.exec(advisories.tools[0]?.text ?? "")?.[1] + expect(hash).toBeDefined() + const cached = await new SourceCache(researchSourcesDir(h.directory)).get(hash!) + expect(cached?.meta.provider).toBe("osv") + + const recorded = await h.run(recordAdopt("mit"), { agent: "scout" }) + expect(recorded.tools[0]?.status).toBe("completed") + const completed = await h.run(call("es_scout_complete"), { agent: "scout" }) + expect(completed.tools[0]?.status).toBe("completed") + + const mit = (await readScoutResult(h.directory))!.verdicts.find((item) => item.candidate === "mit")! + expect([mit.proposal, mit.verdict]).toEqual(["adopt", "adopt"]) + expect(mit.warnings).toEqual([ + "GHSA-aaaa-bbbb-cccc (HIGH): adopt only at 1.0.1 or later", + "GHSA-dddd-eeee-ffff (CRITICAL) has no fixed release", + ]) + const dossier = JSON.parse(await readFile(factoryLayout(h.directory).scoutDossier, "utf8")) + const cited = dossier.claims.filter((claim: any) => claim.source?.sha256 === hash) + expect(cited.length).toBe(3) + expect(cited.every((claim: any) => claim.witness.ok)).toBe(true) + } finally { + unstubOsv() + } + }, 120_000) +}) + +describe("bug #257 probe: the advisory step is evadable (current behavior, not a fix)", () => { + let h: Harness + beforeAll(async () => { + h = await bootHost() + await plantVendor(h.directory, "mit", "MIT") + }, 60_000) + afterAll(() => h?.close()) + + const planScanRecord = async (extraAdvisoryCalls: string[] = []) => { + const planned = await h.run( + call("es_scout_plan", { + objective: "a function library", + candidates: [{ name: "mit", source: "local:./vendor/mit" }], + }), + { agent: "scout" }, + ) + expect(planned.tools[0]?.status).toBe("completed") + const scanned = await h.run(call("es_scout_scan", { candidate: "mit" }), { agent: "scout" }) + expect(scanned.tools[0]?.status).toBe("completed") + for (const extra of extraAdvisoryCalls) { + const done = await h.run(extra, { agent: "scout" }) + expect(done.tools[0]?.status).toBe("completed") + } + const recorded = await h.run(recordAdopt("mit"), { agent: "scout" }) + expect(recorded.tools[0]?.status).toBe("completed") + const completed = await h.run(call("es_scout_complete"), { agent: "scout" }) + expect(completed.tools[0]?.status).toBe("completed") + return (await readScoutResult(h.directory))!.verdicts.find((item) => item.candidate === "mit")! + } + + test("(a) skipping es_scout_advisories still completes a clean adopt with zero citations", async () => { + const mit = await planScanRecord() + // #257 fail-open, asserted as today's behavior: no advisory gate, no + // citation, no warning. Desired: require advisories or an explicit + // NEGATIVE_KNOWLEDGE record before es_scout_complete. + expect([mit.proposal, mit.verdict]).toEqual(["adopt", "adopt"]) + expect(mit.warnings).toEqual([]) + const dossier = JSON.parse(await readFile(factoryLayout(h.directory).scoutDossier, "utf8")) + expect(dossier.claims).toEqual([]) + }, 120_000) + + test("(b) a hand-supplied wrong package is never validated against the candidate", async () => { + stubOsv() + try { + const mit = await planScanRecord([ + call("es_scout_advisories", { candidate: "mit", ecosystem: "npm", package: "unrelated-pkg", version: "1.0.0" }), + ]) + // #257 fail-open, asserted as today's behavior: the tool accepts any + // ecosystem/package the agent names; nothing pins it to the scanned origin. + expect(osvBodies.at(-1)).toEqual({ + package: { name: "unrelated-pkg", ecosystem: "npm" }, + version: "1.0.0", + }) + expect([mit.proposal, mit.verdict]).toEqual(["adopt", "adopt"]) + const dossier = JSON.parse(await readFile(factoryLayout(h.directory).scoutDossier, "utf8")) + const advisoryClaims = dossier.claims.filter((claim: any) => claim.statement.includes("affects mit")) + expect(advisoryClaims.length).toBe(3) + const hash = advisoryClaims[0].source.sha256 as string + const cached = await new SourceCache(researchSourcesDir(h.directory)).get(hash) + expect(cached?.text).toContain("unrelated-pkg") + } finally { + unstubOsv() + } + }, 120_000) + + test("(c) an agent-controlled version pointed at the fixed release clears every warning", async () => { + stubOsv() + try { + const mit = await planScanRecord([ + call("es_scout_advisories", { candidate: "mit", ecosystem: "npm", package: "mit-lib", version: "9.9.9" }), + ]) + // #257 fail-open, asserted as today's behavior: the version defaults to + // the agent's word (ops.ts:259), so naming the fixed release yields zero + // advisories, zero warnings, and a clean adopt. + expect(osvBodies.at(-1)).toEqual({ + package: { name: "mit-lib", ecosystem: "npm" }, + version: "9.9.9", + }) + expect([mit.proposal, mit.verdict]).toEqual(["adopt", "adopt"]) + expect(mit.warnings).toEqual([]) + } finally { + unstubOsv() + } + }, 120_000) +}) From 9a2a6bbebd7a0cc53b4e4655fb7b25cdc4d765e4 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 16:50:21 -0400 Subject: [PATCH 17/83] test(audit): darkharvest confinement and verdict integration tests (#192) --- evals/cases/darkharvest-e2e.json | 1 + packages/core/test/harvest-192.test.ts | 199 +++++++++++++++++++++ packages/opencode/test/harvest-192.test.ts | 179 ++++++++++++++++++ 3 files changed, 379 insertions(+) create mode 100644 evals/cases/darkharvest-e2e.json create mode 100644 packages/core/test/harvest-192.test.ts create mode 100644 packages/opencode/test/harvest-192.test.ts diff --git a/evals/cases/darkharvest-e2e.json b/evals/cases/darkharvest-e2e.json new file mode 100644 index 0000000..5552282 --- /dev/null +++ b/evals/cases/darkharvest-e2e.json @@ -0,0 +1 @@ +{"agent":"harvester","checks":{"toolsUsed":["es_harvest_plan","es_harvest_scan","es_harvest_matrix","es_harvest_complete"]},"description":"The harvester tears down two vendored local projects end to end (plan, scan, read, matrix, complete): MIT stays depend, GPL-3.0-only is downgraded to clean-room. Graded from .factory/harvest/runs/default/harvest.json. Provenance-per-field and no-copied-spans need the scrubber/manual review (see issue #192).","files":{"vendor/copycalc/LICENSE":"@fixture:licenses/GPL-3.0-only.txt","vendor/copycalc/index.js":"const { copycalcQuux7429 } = require('./lib.js')\nmodule.exports = { run: (xs) => xs.map(copycalcQuux7429) }\n","vendor/copycalc/lib.js":"function copycalcQuux7429(x) { return x < 0 ? 0 - x : x * 2 }\nmodule.exports = { copycalcQuux7429 }\n","vendor/copycalc/package.json":"{\"name\": \"copycalc\", \"version\": \"1.0.0\"}\n","vendor/retcalc/LICENSE":"@fixture:licenses/MIT.txt","vendor/retcalc/index.js":"const { retcalcQuux7429 } = require('./lib.js')\nmodule.exports = { run: (xs) => xs.map(retcalcQuux7429) }\n","vendor/retcalc/lib.js":"function retcalcQuux7429(x) { return x < 0 ? 0 - x : x + 1 }\nmodule.exports = { retcalcQuux7429 }\n","vendor/retcalc/package.json":"{\"name\": \"retcalc\", \"version\": \"1.0.0\"}\n"},"fire":{"expected":[{"candidate":"copycalc","verdict":"clean-room"},{"candidate":"retcalc","verdict":"depend"}],"kind":"harvest"},"id":"darkharvest-e2e","maxSteps":14,"prompt":"Tear down two local candidates for a session handoff feature: local:./vendor/retcalc and local:./vendor/copycalc. This is a headless run: es_harvest_plan them (objective 'session handoff teardown'), es_harvest_scan each, es_harvest_read one source file per candidate, submit one matrix row proposing depend for both with a one-line behaviour evidence per cell (describe behaviour in your own words, do not paste source), then es_harvest_complete."} \ No newline at end of file diff --git a/packages/core/test/harvest-192.test.ts b/packages/core/test/harvest-192.test.ts new file mode 100644 index 0000000..04cfa4f --- /dev/null +++ b/packages/core/test/harvest-192.test.ts @@ -0,0 +1,199 @@ +// Issue #192 (Darkharvest) integration tests — core side. +// +// - Harvester targeting a path outside the project is refused, with an +// in-project control showing allowed paths still work. +// - An Apache-header-only fixture (no LICENSE file) stays clean-room. The +// returned literal is recorded as-is: `Apache-2.0`, unverified +// (bug #264 observed the claim wording "unknown"; the enforced property — +// clean-room — holds, so this asserts current behavior, not the old word). +import { afterEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + buildMatrix, + checkVerdict, + DEFAULT_LICENSE_WHITELIST, + detectLicense, + harvestTools, + scanSource, +} from "../src/index.ts" + +const dirs: string[] = [] +const tmp = async (prefix: string) => { + const dir = await mkdtemp(path.join(tmpdir(), prefix)) + dirs.push(dir) + return dir +} +afterEach(async () => { + for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true }) +}) + +const write = async (root: string, file: string, content: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), content) +} + +const MIT_LICENSE = `MIT License + +Copyright (c) 2026 Test + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +` + +describe("issue #192: out-of-project harvest is refused", () => { + test("plan, target and scan refuse an outside path; an in-project path works", async () => { + const root = await tmp("es-192-root-") + const outside = await tmp("es-192-outside-") + await write(outside, "key.txt", "SIGNING-KEY-DO-NOT-READ") + + const tools = harvestTools({ root }) + const plan = tools.find((tool) => tool.name === "es_harvest_plan")! + const target = tools.find((tool) => tool.name === "es_harvest_target")! + + // Refused: a harvester plan that points outside the project. + await expect( + plan.execute( + { objective: "x", candidates: [{ name: "evil", source: `local:${outside}` }], force: true }, + { agent: "harvester" }, + ), + ).rejects.toThrow(/outside the project/) + // Refused: the callable verdict check, same seat, same outside path. + await expect( + target.execute({ objective: "x", targets: [`local:${outside}`] }, { agent: "harvester" }), + ).rejects.toThrow(/outside the project/) + // Refused at the scan layer too (no bytes are read: the throw precedes I/O). + await expect(scanSource(root, "evil", { kind: "local", path: outside }, { regexOnly: true })).rejects.toThrow( + /outside the project/, + ) + + // Control: an in-project candidate plans and scans fine. + await write(root, "ok/LICENSE", MIT_LICENSE) + await write(root, "ok/src/a.ts", "export const a = 1\n") + const planned = await plan.execute( + { objective: "x", candidates: [{ name: "ok", source: "local:ok" }], force: true }, + { agent: "harvester" }, + ) + expect(planned).toContain("ok") + const profile = await scanSource(root, "ok", { kind: "local", path: "ok" }, { regexOnly: true }) + expect(profile.license).toMatchObject({ spdx: "MIT", verified: true, source: "license-file" }) + }) +}) + +describe("issue #192: Apache-header-only fixture is clean-room", () => { + // No LICENSE file anywhere in this fixture: one source file carries only + // `SPDX-License-Identifier: Apache-2.0`. + const setupHeaderOnly = async (root: string, dir = "header-only") => { + await write(root, `${dir}/src/a.ts`, "// SPDX-License-Identifier: Apache-2.0\nexport const a = 1\n") + } + + test("detectLicense literal is Apache-2.0, unverified (bug #264); policy forces clean-room", async () => { + const root = await tmp("es-192-apache-") + await setupHeaderOnly(root) + const dir = path.join(root, "header-only") + + const finding = await detectLicense(dir) + // Pinned literal: bug #264 notes the claim said "unknown" here, but the + // code returns the header id unverified. Assert current behavior so any + // change to the literal is a visible diff; the safety property below is + // what must never regress. + expect(finding.spdx).toBe("Apache-2.0") + expect(finding).toMatchObject({ source: "spdx-header", verified: false, family: "permissive" }) + + // Fail-closed: an unverified header can never authorize depend/vendor. + for (const proposed of ["depend", "vendor"] as const) { + const checked = checkVerdict(proposed, finding, DEFAULT_LICENSE_WHITELIST) + expect(checked.verdict).toBe("clean-room") + expect(checked.downgraded).toMatch(/unverified/) + // Dangerous misclassification guard: unknown/copyleft/unverified must + // never come back depend/vendor-able. This would be a FINDING, not a fix. + expect(checked.verdict).not.toBe("depend") + expect(checked.verdict).not.toBe("vendor") + } + }) + + test("scan → matrix downgrades a proposed depend to clean-room", async () => { + const root = await tmp("es-192-apache-flow-") + await setupHeaderOnly(root) + const tools = harvestTools({ root }) + const call = (name: string, input: Record) => + tools.find((tool) => tool.name === name)!.execute(input, { agent: "harvester" }) + + await call("es_harvest_plan", { + objective: "vet the header-only lib", + candidates: [{ name: "header-only", source: "local:header-only" }], + }) + const scanned = await call("es_harvest_scan", { candidate: "header-only" }) + expect(scanned).toContain("Apache-2.0") + expect(scanned).toContain("UNVERIFIED") + + const matrixText = await call("es_harvest_matrix", { + rows: [{ feature: "widget", cells: [{ candidate: "header-only", verdict: "depend" }] }], + }) + expect(matrixText).toContain("clean-room") + + const profiles = new Map([ + [ + "header-only", + { + version: 1 as const, + id: "header-only", + name: "header-only", + license: await detectLicense(path.join(root, "header-only")), + languages: {}, + dependencies: { runtime: [], dev: [] }, + size: { files: 0, bytes: 0, tokens: 0 }, + graph: { internalEdges: 0, external: 0, parsers: { treeSitter: 0, regex: 0 } }, + provenance: {}, + warnings: [], + scannedAt: new Date().toISOString(), + }, + ], + ]) + const built = buildMatrix( + ["header-only"], + profiles, + [{ feature: "widget", cells: [{ candidate: "header-only", verdict: "depend" }] }], + DEFAULT_LICENSE_WHITELIST, + ) + expect(built.matrix.rows[0]?.cells[0]?.verdict).toBe("clean-room") + expect(built.matrix.rows[0]?.cells[0]?.downgraded).toMatch(/unverified/) + expect(built.downgrades).toHaveLength(1) + }) + + test("es_harvest_target reports clean-room with the same literal", async () => { + const root = await tmp("es-192-apache-target-") + await setupHeaderOnly(root) + const target = harvestTools({ root }).find((tool) => tool.name === "es_harvest_target")! + const text = await target.execute( + { objective: "can we depend on it", targets: ["local:header-only"] }, + { agent: "grill" }, + ) + const json = /--- verdicts \(JSON\) ---\n(\[[\s\S]*\])[\s]*$/.exec(text)?.[1] + expect(json).toBeDefined() + const verdicts = JSON.parse(json!) as Array<{ + target: string + license: { spdx: string; family: string; verified: boolean } + verdict: string + }> + expect(verdicts).toHaveLength(1) + expect(verdicts[0]?.license).toMatchObject({ spdx: "Apache-2.0", verified: false }) + expect(verdicts[0]?.verdict).toBe("clean-room") + }) +}) diff --git a/packages/opencode/test/harvest-192.test.ts b/packages/opencode/test/harvest-192.test.ts new file mode 100644 index 0000000..b83bf53 --- /dev/null +++ b/packages/opencode/test/harvest-192.test.ts @@ -0,0 +1,179 @@ +// Issue #192 (Darkharvest) integration tests — real host via packages/testkit. +// +// - The harvester seat targeting a path outside the project is refused, with +// an in-project control showing allowed paths still work. +// - An Apache-header-only fixture (no LICENSE file) stays clean-room. The +// returned literal is pinned as `Apache-2.0`, unverified (bug #264: the +// claim wording said "unknown"; the enforced clean-room property holds, so +// this asserts current behavior and reports the literal). +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { readFileSync } from "node:fs" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { readProfile } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +const fixture = (id: string) => + readFileSync(path.resolve(import.meta.dir, "../../core/test/fixtures/licenses", `${id}.txt`), "utf8") +const MIT = fixture("MIT") + +const write = async (root: string, file: string, content: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), content) +} + +let state: string +let h: Harness +const outsideDirs: string[] = [] +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-192-host-state-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# h\n" }, + }) +}, 60_000) + +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + for (const dir of outsideDirs.splice(0)) await rm(dir, { recursive: true, force: true }) +}) + +describe("issue #192 on the real host: out-of-project harvest is refused", () => { + test("harvester plan and target refuse an outside path; an in-project path works", async () => { + const outside = await mkdtemp(path.join(tmpdir(), "es-192-outside-")) + outsideDirs.push(outside) + await write(outside, "key.txt", "SIGNING-KEY-DO-NOT-READ") + + // Refused: plan with a candidate outside the project. + const refused = await h.run( + call("es_harvest_plan", { + objective: "take the outside", + run: "o192-outside", + candidates: [{ name: "sneaky", source: `local:${outside}` }], + force: true, + }), + { agent: "harvester" }, + ) + expect(refused.tools[0]?.status).toBe("error") + expect(refused.tools[0]?.text).toMatch(/outside the project/) + + // Refused: the callable verdict check, same outside path, same seat. + const targetRefused = await h.run( + call("es_harvest_target", { + objective: "can we depend on the outside", + targets: [`local:${outside}`], + run: "o192-outside-target", + }), + { agent: "harvester" }, + ) + expect(targetRefused.tools[0]?.status).toBe("error") + expect(targetRefused.tools[0]?.text).toMatch(/outside the project/) + + // Control: an in-project candidate is still allowed (plan, scan, target). + await write(h.directory, "o192-ok/LICENSE", MIT) + await write(h.directory, "o192-ok/src/a.ts", "export const a = 1\n") + const planned = await h.run( + call("es_harvest_plan", { + objective: "take the inside", + run: "o192-control", + candidates: [{ name: "ok", source: "local:o192-ok" }], + force: true, + }), + { agent: "harvester" }, + ) + expect(planned.tools[0]?.status).toBe("completed") + const scanned = await h.run(call("es_harvest_scan", { candidate: "ok", run: "o192-control" }), { + agent: "harvester", + }) + expect(scanned.tools[0]?.status).toBe("completed") + expect(scanned.tools[0]?.text).toContain("SPDX:MIT") + const verdict = await h.run( + call("es_harvest_target", { + objective: "can we depend on the inside", + targets: ["local:o192-ok"], + run: "o192-control-target", + }), + { agent: "grill" }, + ) + expect(verdict.tools[0]?.status).toBe("completed") + expect(verdict.tools[0]?.text).toContain("→ depend") + }) +}) + +describe("issue #192 on the real host: Apache-header-only fixture is clean-room", () => { + test("scan and matrix keep an Apache header (no LICENSE) clean-room; literal is Apache-2.0 unverified", async () => { + // No LICENSE file: one source file declares only an SPDX header. + await write(h.directory, "o192-apache/src/a.ts", "// SPDX-License-Identifier: Apache-2.0\nexport const a = 1\n") + + const planned = await h.run( + call("es_harvest_plan", { + objective: "vet the header-only lib", + run: "o192-apache", + candidates: [{ name: "apache-header", source: "local:o192-apache" }], + force: true, + }), + { agent: "harvester" }, + ) + expect(planned.tools[0]?.status).toBe("completed") + + const scanned = await h.run(call("es_harvest_scan", { candidate: "apache-header", run: "o192-apache" }), { + agent: "harvester", + }) + expect(scanned.tools[0]?.status).toBe("completed") + expect(scanned.tools[0]?.text).toContain("Apache-2.0") + expect(scanned.tools[0]?.text).toContain("UNVERIFIED") + expect(scanned.tools[0]?.text).toContain("NOT authorized") + + // Pinned literal (bug #264): the header id is reported unverified rather + // than "unknown". The verdict below is the property that must not regress. + const profile = await readProfile(h.directory, "apache-header", "o192-apache") + expect(profile?.license).toMatchObject({ spdx: "Apache-2.0", verified: false, source: "spdx-header" }) + + const matrix = await h.run( + call("es_harvest_matrix", { + run: "o192-apache", + rows: [{ feature: "widget", cells: [{ candidate: "apache-header", verdict: "depend" }] }], + }), + { agent: "harvester" }, + ) + expect(matrix.tools[0]?.status).toBe("completed") + expect(matrix.tools[0]?.text).toContain("clean-room") + // Dangerous misclassification guard: a proposed depend on an unverified + // header must never survive as depend/vendor. + expect(matrix.tools[0]?.text).not.toMatch(/Verdicts: depend 1/) + }) + + test("es_harvest_target reports clean-room with the same literal", async () => { + const { tools } = await h.run( + call("es_harvest_target", { + objective: "can we depend on the header-only lib", + targets: ["local:o192-apache"], + run: "o192-apache-target", + }), + { agent: "grill" }, + ) + expect(tools[0]?.status).toBe("completed") + const json = /--- verdicts \(JSON\) ---\n(\[[\s\S]*\])[\s]*$/.exec(tools[0]?.text ?? "")?.[1] + expect(json).toBeDefined() + const verdicts = JSON.parse(json!) as Array<{ + target: string + license: { spdx: string; family: string; verified: boolean } + verdict: string + }> + expect(verdicts).toHaveLength(1) + expect(verdicts[0]).toMatchObject({ + target: "local:o192-apache", + license: { spdx: "Apache-2.0", verified: false }, + verdict: "clean-room", + }) + expect(verdicts[0]?.verdict).not.toBe("depend") + expect(verdicts[0]?.verdict).not.toBe("vendor") + }) +}) From 587c00c3b5242358937c18f1eb96cd99a596c008 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 16:52:22 -0400 Subject: [PATCH 18/83] test(audit): hook bridge event integration tests (#196) --- evals/cases/hooks-bridge.json | 1 + packages/core/test/hooks-events.test.ts | 262 ++++++++++++++++++++ packages/opencode/test/hooks-events.test.ts | 247 ++++++++++++++++++ 3 files changed, 510 insertions(+) create mode 100644 evals/cases/hooks-bridge.json create mode 100644 packages/core/test/hooks-events.test.ts create mode 100644 packages/opencode/test/hooks-events.test.ts diff --git a/evals/cases/hooks-bridge.json b/evals/cases/hooks-bridge.json new file mode 100644 index 0000000..e90aab4 --- /dev/null +++ b/evals/cases/hooks-bridge.json @@ -0,0 +1 @@ +{"agent":"build","checks":{"contains":["HOOK:PreToolUse","HOOK:PostToolUse","HOOK:PostToolUseFailure","HOOK:ShellPreExec"],"toolsUsed":["read","bash"]},"description":"Hook bridge E2E: real opencode + .claude/settings.json from a real Claude Code project; each ENFORCED hook event fires a marker. Markers asserted via transcript; on-disk completeness needs kept-workspace review until a file-asserting fire kind exists.","files":{".claude/settings.json":"{\"hooks\":{\"PermissionRequest\":[{\"hooks\":[{\"command\":\"printf 'HOOK:PermissionRequest\\\\n' >> .factory/hooks.log\",\"type\":\"command\"}],\"matcher\":\"Bash\"}],\"PostToolUse\":[{\"hooks\":[{\"command\":\"printf 'HOOK:PostToolUse\\\\n' >> .factory/hooks.log\",\"type\":\"command\"}],\"matcher\":\"\"}],\"PostToolUseFailure\":[{\"hooks\":[{\"command\":\"printf 'HOOK:PostToolUseFailure\\\\n' >> .factory/hooks.log\",\"type\":\"command\"}],\"matcher\":\"\"}],\"PreToolUse\":[{\"hooks\":[{\"command\":\"exit 2\",\"type\":\"command\"}],\"matcher\":\"Bash(rm *)\"},{\"hooks\":[{\"command\":\"printf 'HOOK:PreToolUse\\\\n' >> .factory/hooks.log\",\"type\":\"command\"}],\"matcher\":\"\"}],\"SessionStart\":[{\"hooks\":[{\"command\":\"printf 'HOOK:SessionStart\\\\n' >> .factory/hooks.log\",\"type\":\"command\"}],\"matcher\":\"startup\"}],\"UserPromptSubmit\":[{\"hooks\":[{\"command\":\"printf 'HOOK:UserPromptSubmit\\\\n' >> .factory/hooks.log\",\"type\":\"command\"}],\"matcher\":\"\"}],\"x-es.ShellPreExec\":[{\"hooks\":[{\"command\":\"echo rewritten-by-hook\",\"type\":\"command\"}],\"matcher\":\"echo original\"}]}}","src/fail.sh":"exit 1\n","src/probe.txt":"hook bridge probe\n"},"id":"hooks-bridge","maxSteps":14,"prompt":"Follow these steps exactly, one tool call per step. (0) Run `mkdir -p .factory` with bash. (1) Read src/probe.txt with the read tool. (2) Run `echo hook-probe-ok` with bash. (3) Run `sh src/fail.sh` with bash (it exits 1; continue regardless). (4) Run `echo original` with bash (it will be rewritten; continue). (5) Run `cat .factory/hooks.log` with bash. (6) Narrate which HOOK: markers you observed. This run is headless: never call the question tool."} \ No newline at end of file diff --git a/packages/core/test/hooks-events.test.ts b/packages/core/test/hooks-events.test.ts new file mode 100644 index 0000000..ca73149 --- /dev/null +++ b/packages/core/test/hooks-events.test.ts @@ -0,0 +1,262 @@ +// Issue #196 §2 — engine-level proof for every bridged hook event. +// +// Each event gets a hook that writes a marker file; the test drives the +// matching engine method and asserts the marker landed. This complements +// packages/opencode/test/hooks-events.test.ts, which drives the same events +// through the real in-process OpenCode v2 host. +// +// NOTE (bug #265): hook commands run UNSANDBOXED with the full parent env +// (run.ts uses inheritEnv:true and no sandbox knob). The "parent env is +// visible" test below pins that observable behaviour without changing it — +// Claude Code parity by design, not a sandbox. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { fromOpenCode, HookEngine, type HookSource, trustProject } from "../src/index.ts" + +let root: string +let state: string +const PASSPHRASE = "test-passphrase-196-events" +let signer: HumanSigner +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-hooks-events-")) + state = await mkdtemp(path.join(tmpdir(), "es-hooks-events-state-")) + await sealHumanKey(PASSPHRASE, state) + signer = await unlockHumanKey(PASSPHRASE, state) + await mkdir(path.join(root, ".opencode"), { recursive: true }) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const sources = (): HookSource[] => [{ file: path.join(root, ".opencode/hooks.json"), scope: "project" }] +async function engine(hooks: Record) { + await writeFile(path.join(root, ".opencode/hooks.json"), JSON.stringify({ hooks })) + let created = await HookEngine.create({ root, stateDir: state, sources: sources(), audit: false }) + const status = created.status() + await trustProject(root, status.projectHash, status.projectLines, { stateDir: state, kind: "hooks", signer }) + created = await HookEngine.create({ root, stateDir: state, sources: sources(), audit: false }) + return created +} +const base = { sessionId: "ses_196", agent: "build" } +const marker = (name: string) => path.join(root, name) +const touch = (name: string, extra = "") => `touch ${marker(name)}${extra ? `; ${extra}` : ""}` + +describe("hook events write marker files (issue #196 §2)", () => { + test("PreToolUse marker lands; exit 2 deny blocks the tool", async () => { + const hooks = await engine({ + PreToolUse: [ + { + matcher: "Write", + hooks: [ + { + type: "command", + command: `${touch("PRE_196")}; input=$(cat); echo "$input" | grep -q "deny-pre-196" && { echo "pre-deny-196 blocked" >&2; exit 2; }; exit 0`, + }, + ], + }, + ], + }) + const ok = await hooks.preToolUse(base, fromOpenCode("write", { path: "a.txt", content: "fine" }, root), "t1") + expect(ok.decision).toBeUndefined() + expect(await Bun.file(marker("PRE_196")).exists()).toBe(true) + const denied = await hooks.preToolUse( + base, + fromOpenCode("write", { path: "b.txt", content: "deny-pre-196" }, root), + "t2", + ) + expect(denied).toMatchObject({ decision: "deny", reason: "pre-deny-196 blocked" }) + }) + + test("PostToolUse marker lands with additionalContext", async () => { + const hooks = await engine({ + PostToolUse: [ + { + matcher: "Read", + hooks: [ + { + type: "command", + command: `${touch("POST_196")}; echo '{"hookSpecificOutput":{"hookEventName":"PostToolUse","additionalContext":"POST-MARKER-196"}}'`, + }, + ], + }, + ], + }) + const out = await hooks.postToolUse(base, fromOpenCode("read", { path: "a.txt" }, root), "t1", { ok: true }, false) + expect(await Bun.file(marker("POST_196")).exists()).toBe(true) + expect(out.additionalContext).toContain("POST-MARKER-196") + }) + + test("PostToolUseFailure marker lands when driven with failed=true (bug #268)", async () => { + const hooks = await engine({ + PostToolUseFailure: [ + { + matcher: "Read", + hooks: [ + { + type: "command", + command: `${touch("FAIL_196")}; echo '{"hookSpecificOutput":{"hookEventName":"PostToolUseFailure","additionalContext":"FAILURE-MARKER-196"}}'`, + }, + ], + }, + ], + }) + // failed=false must NOT run the PostToolUseFailure handler. + const ok = await hooks.postToolUse(base, fromOpenCode("read", { path: "a.txt" }, root), "t1", "content", false) + expect(ok.additionalContext).toEqual([]) + expect(await Bun.file(marker("FAIL_196")).exists()).toBe(false) + // failed=true routes to the PostToolUseFailure event (engine.ts). + const failed = await hooks.postToolUse( + base, + fromOpenCode("read", { path: "missing.txt" }, root), + "t2", + "Error: file not found", + true, + ) + expect(await Bun.file(marker("FAIL_196")).exists()).toBe(true) + expect(failed.additionalContext).toContain("FAILURE-MARKER-196") + }) + + test("UserPromptSubmit marker lands; block blocks", async () => { + const hooks = await engine({ + UserPromptSubmit: [ + { + hooks: [ + { + type: "command", + command: `${touch("PROMPT_196")}; grep -q "block-prompt-196" && echo '{"decision":"block","reason":"prompt blocked 196"}' || echo "PROMPT-MARKER-196"`, + }, + ], + }, + ], + }) + const out = await hooks.userPromptSubmit(base, "hello") + expect(await Bun.file(marker("PROMPT_196")).exists()).toBe(true) + expect(out.additionalContext).toContain("PROMPT-MARKER-196") + expect((await hooks.userPromptSubmit(base, "please block-prompt-196")).block).toContain("prompt blocked 196") + }) + + test("SessionStart marker lands for startup only", async () => { + const hooks = await engine({ + SessionStart: [ + { matcher: "startup", hooks: [{ type: "command", command: touch("START_196") + "; echo START-MARKER-196" }] }, + ], + }) + const out = await hooks.sessionStart(base, "startup") + expect(await Bun.file(marker("START_196")).exists()).toBe(true) + expect(out.additionalContext).toContain("START-MARKER-196") + await rm(marker("START_196"), { force: true }) + expect((await hooks.sessionStart(base, "compact")).additionalContext).toEqual([]) + expect(await Bun.file(marker("START_196")).exists()).toBe(false) + }) + + test("PermissionRequest marker lands; deny JSON denies", async () => { + const hooks = await engine({ + PermissionRequest: [ + { + matcher: "Read", + hooks: [ + { + type: "command", + command: `${touch("PERM_196")}; echo '{"hookSpecificOutput":{"hookEventName":"PermissionRequest","decision":{"behavior":"deny","message":"perm-deny-196"}}}'`, + }, + ], + }, + ], + }) + const out = await hooks.permissionRequest(base, fromOpenCode("read", { path: ".env" }, root)) + expect(await Bun.file(marker("PERM_196")).exists()).toBe(true) + expect(out).toMatchObject({ behavior: "deny", message: "perm-deny-196" }) + }) + + test("Stop marker lands; exit 2 blocks (emulated after the turn, ADVISORY for plain chat)", async () => { + const hooks = await engine({ + Stop: [ + { + hooks: [ + { + type: "command", + // The engine passes stop_hook_active in the input; the hook itself + // suppresses the block on a continuation, as Claude Code does. + command: `${touch("STOP_196")}; input=$(cat); echo "$input" | grep -q '"stop_hook_active":true' && exit 0; echo "stop-continue-196" >&2; exit 2`, + }, + ], + }, + ], + }) + const out = await hooks.stop(base, { stopHookActive: false, lastAssistantMessage: "done" }) + expect(await Bun.file(marker("STOP_196")).exists()).toBe(true) + expect(out.block).toContain("stop-continue-196") + // stop_hook_active suppresses the block, as Claude Code does. + expect((await hooks.stop(base, { stopHookActive: true, lastAssistantMessage: "done" })).block).toBeUndefined() + }) +}) + +describe("x-es.ShellPreExec rewrite AND deny (bug #268)", () => { + test("rewrite branch returns the replacement command", async () => { + const hooks = await engine({ + "x-es.ShellPreExec": [ + { + hooks: [ + { + type: "command", + command: `input=$(cat); echo "$input" | grep -q "rewrite-shell-196" && echo '{"hookSpecificOutput":{"updatedInput":{"command":"echo shell-rewritten-196"}}}' || true`, + }, + ], + }, + ], + }) + expect(await hooks.shellPreExec(base, { command: "echo rewrite-shell-196", cwd: root })).toMatchObject({ + command: "echo shell-rewritten-196", + }) + expect((await hooks.shellPreExec(base, { command: "echo plain", cwd: root })).command).toBeUndefined() + }) + + test("deny branch: exit 2 denies; deny wins over a simultaneous rewrite (command && !deny precedence)", async () => { + const hooks = await engine({ + "x-es.ShellPreExec": [ + { + hooks: [ + { + type: "command", + command: `input=$(cat); echo "$input" | grep -q "deny-shell-196" && { echo "shell-deny-196 blocked" >&2; exit 2; }; echo "$input" | grep -q "both-shell-196" && echo '{"hookSpecificOutput":{"permissionDecision":"deny","permissionDecisionReason":"both denied 196","updatedInput":{"command":"echo should-not-run-196"}}}' || true`, + }, + ], + }, + ], + }) + // Exit-2 deny carries the stderr reason. + const denied = await hooks.shellPreExec(base, { command: "echo deny-shell-196", cwd: root }) + expect(denied.deny).toContain("shell-deny-196 blocked") + expect(denied.command).toBeUndefined() + // JSON deny with a simultaneous updatedInput.command: deny wins, command is suppressed. + const both = await hooks.shellPreExec(base, { command: "echo both-shell-196", cwd: root }) + expect(both.deny).toContain("both denied 196") + expect(both.command).toBeUndefined() + }) +}) + +describe("hooks run unsandboxed with parent env (bug #265, observable behaviour)", () => { + test("parent process env is visible inside a hook command", async () => { + const key = "ES_HOOK_PARENT_PROBE_196" + const value = `probe-196-${Date.now()}` + process.env[key] = value + try { + const hooks = await engine({ + PreToolUse: [ + { + matcher: "Bash", + hooks: [{ type: "command", command: `echo "$${key}" > ${marker("ENV_196")}` }], + }, + ], + }) + await hooks.preToolUse(base, fromOpenCode("shell", { command: "echo hi" }, root), "t1") + expect((await Bun.file(marker("ENV_196")).text()).trim()).toBe(value) + } finally { + delete process.env[key] + } + }) +}) diff --git a/packages/opencode/test/hooks-events.test.ts b/packages/opencode/test/hooks-events.test.ts new file mode 100644 index 0000000..1d0b61d --- /dev/null +++ b/packages/opencode/test/hooks-events.test.ts @@ -0,0 +1,247 @@ +// Issue #196 §2 — hook bridge on the real in-process OpenCode v2 host. +// +// One hook per event writes a marker file; the tests drive each event through +// the host (scripted fake LLM) and assert the marker landed. Complements +// packages/core/test/hooks-events.test.ts (engine-direct proof). +// +// NOTE (bug #265): hook commands run UNSANDBOXED with the full parent env +// (core run.ts: inheritEnv:true, no sandbox knob). The PRE_ENV_196 assertion +// pins that observable behaviour without changing it — Claude Code parity by +// design, not a sandbox. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + commandSetHash, + HookEngine, + type HumanSigner, + loadGatesConfig, + sealHumanKey, + trustProject, + unlockHumanKey, +} from "@heretek-ai/es-core" +import { allText, boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const json = (value: unknown) => `echo '${JSON.stringify(value)}'` +const at = (name: string) => path.join(h.directory, name) +const exists = (name: string) => Bun.file(at(name)).exists() + +// Parent-env probe (bug #265): visible inside every hook command. +const ENV_KEY = "ES_HOOK_PARENT_PROBE_196" +const ENV_VALUE = `probe-196-${Date.now()}` + +const hooks = { + PreToolUse: [ + { + matcher: "Write", + hooks: [ + { + type: "command", + // Always marks + records the parent env; denies only payloads carrying the canary. + command: `touch "$CLAUDE_PROJECT_DIR/PRE_196"; echo "$${ENV_KEY}" > "$CLAUDE_PROJECT_DIR/PRE_ENV_196"; input=$(cat); echo "$input" | grep -q "deny-pre-196" && { echo "pre-deny-196 blocked" >&2; exit 2; }; exit 0`, + }, + ], + }, + ], + PostToolUse: [ + { + matcher: "Read", + hooks: [ + { + type: "command", + command: `touch "$CLAUDE_PROJECT_DIR/POST_196"; ${json({ hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: "POST-MARKER-196" } })}`, + }, + ], + }, + ], + PostToolUseFailure: [ + { + matcher: "Read", + hooks: [ + { + type: "command", + // Bug #268: this handler was routed but never proven — the failing-read + // test below drives it through a real tool error. + command: `touch "$CLAUDE_PROJECT_DIR/FAIL_196"; ${json({ hookSpecificOutput: { hookEventName: "PostToolUseFailure", additionalContext: "FAILURE-MARKER-196" } })}`, + }, + ], + }, + ], + UserPromptSubmit: [ + { + hooks: [ + { + type: "command", + command: `touch "$CLAUDE_PROJECT_DIR/PROMPT_196"; echo PROMPT-MARKER-196`, + }, + ], + }, + ], + SessionStart: [ + { + matcher: "startup", + hooks: [{ type: "command", command: `touch "$CLAUDE_PROJECT_DIR/START_196"; echo START-MARKER-196` }], + }, + ], + Stop: [ + { + hooks: [ + { + type: "command", + // Marker only, no block: proves the emulated Stop hook ran after the + // turn without re-prompting (ADVISORY for plain chat). + command: `touch "$CLAUDE_PROJECT_DIR/STOP_196"; exit 0`, + }, + ], + }, + ], + PermissionRequest: [ + { + matcher: "Read", + hooks: [ + { + type: "command", + // Runs only when the host would ask (effect ask — e.g. .env reads): + // always marks, denies only .env payloads. + command: `touch "$CLAUDE_PROJECT_DIR/PERM_196"; input=$(cat); echo "$input" | grep -q "\\.env" && ${json({ hookSpecificOutput: { hookEventName: "PermissionRequest", decision: { behavior: "deny", message: "perm-deny-196 blocked" } } })} || ${json({ hookSpecificOutput: { hookEventName: "PermissionRequest", decision: { behavior: "allow" } } })}`, + }, + ], + }, + ], +} + +const extraHooks = { + hooks: { + "x-es.ShellPreExec": [ + { + hooks: [ + { + type: "command", + // Bug #268: rewrite was proven, deny was not. Always marks; denies + // *deny-shell-196*, rewrites *rewrite-shell-196*. + command: `touch "$CLAUDE_PROJECT_DIR/SHELL_196"; input=$(cat); if echo "$input" | grep -q "deny-shell-196"; then ${json({ hookSpecificOutput: { permissionDecision: "deny", permissionDecisionReason: "shell-deny-196 blocked" } })}; elif echo "$input" | grep -q "rewrite-shell-196"; then ${json({ hookSpecificOutput: { updatedInput: { command: "echo shell-rewritten-196" } } })}; fi; true`, + }, + ], + }, + ], + }, +} + +let state: string +let h: Harness + +async function waitFor(name: string, timeoutMs = 15_000): Promise { + const start = Date.now() + for (;;) { + if (await exists(name)) return true + if (Date.now() - start > timeoutMs) return false + await Bun.sleep(200) + } +} + +beforeAll(async () => { + process.env[ENV_KEY] = ENV_VALUE + state = await mkdtemp(path.join(tmpdir(), "es-hookbridge-ev-state-")) + await sealHumanKey("test-passphrase-1234", state) + const signer: HumanSigner = await unlockHumanKey("test-passphrase-1234", state) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { + "README.md": "# hooks-events\n", + ".env": "SECRET=1\n", + ".claude/settings.json": JSON.stringify({ hooks }), + ".opencode/hooks.json": JSON.stringify(extraHooks), + }, + }) + // Trust the project hook set through the human key, as hooks.test.ts does. + const { config } = await loadGatesConfig(h.directory) + const gates = await commandSetHash(h.directory, config.commands) + const status = (await HookEngine.create({ root: h.directory, stateDir: state, audit: false })).status() + await trustProject(h.directory, gates.hash, gates.lines, { stateDir: state, signer }) + if (status.projectHash) + await trustProject(h.directory, status.projectHash, status.projectLines, { + stateDir: state, + kind: "hooks", + signer, + }) +}, 60_000) +afterAll(async () => { + delete process.env[ENV_KEY] + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +describe("hook bridge markers on the real host (issue #196 §2)", () => { + test("SessionStart + UserPromptSubmit hooks mark every session", async () => { + await h.run("hello marker run") + expect(await exists("START_196")).toBe(true) + expect(await exists("PROMPT_196")).toBe(true) + }) + + test("PreToolUse hook marks the write; parent env is visible unsandboxed (bug #265)", async () => { + const { tools } = await h.run(call("write", { path: "ok-196.txt", content: "fine" })) + expect(tools[0]?.status).toBe("completed") + expect(await exists("PRE_196")).toBe(true) + expect((await Bun.file(at("PRE_ENV_196")).text()).trim()).toBe(ENV_VALUE) + }) + + test("PreToolUse deny (exit 2) blocks the tool", async () => { + const { tools } = await h.run(call("write", { path: "blocked-196.txt", content: "deny-pre-196 please" })) + expect(tools[0]?.status).toBe("error") + expect(tools[0]?.text).toContain("pre-deny-196 blocked") + expect(await Bun.file(at("blocked-196.txt")).exists()).toBe(false) + }) + + test("PostToolUse hook marks a successful read", async () => { + const { tools } = await h.run(call("read", { path: "README.md" })) + expect(tools[0]?.status).toBe("completed") + expect(tools[0]?.text).toContain("POST-MARKER-196") + expect(await exists("POST_196")).toBe(true) + }) + + test("PostToolUseFailure hook marks a failing read (bug #268)", async () => { + const { tools } = await h.run(call("read", { path: "does-not-exist-196.txt" })) + expect(tools[0]?.status).toBe("error") + // The bridge routes failures to PostToolUseFailure (hooks.ts); the marker + // proves the handler ran for a real tool error. Observed: the handler's + // additionalContext is NOT appended to the tool error text (bridge returns + // early for failed tools) — routing proven, feedback path not. + expect(await waitFor("FAIL_196")).toBe(true) + }) + + test("PermissionRequest hook marks and denies an .env read", async () => { + const { tools } = await h.run(call("read", { path: ".env" })) + expect(tools[0]?.status).toBe("error") + expect(tools[0]?.text).toContain("perm-deny-196 blocked") + expect(await exists("PERM_196")).toBe(true) + }) + + test("x-es.ShellPreExec rewrites a matching command", async () => { + const { tools } = await h.run(call("shell", { command: "echo rewrite-shell-196" })) + expect(tools[0]?.text).toContain("shell-rewritten-196") + expect(await exists("SHELL_196")).toBe(true) + }) + + test("x-es.ShellPreExec deny substitutes exit 126 (bug #268)", async () => { + const { tools } = await h.run(call("shell", { command: "echo deny-shell-196" })) + // The bridge replaces the denied command with `printf … >&2; exit 126` + // (hooks.ts): the block reason surfaces as the tool output. + expect(tools[0]?.text).toContain("shell-deny-196 blocked") + expect(tools[0]?.text).toContain("Exited with code 126") + }) + + test("Stop hook marker lands after the turn (emulated, ADVISORY for plain chat)", async () => { + await h.run(call("read", { path: "README.md" })) + expect(await waitFor("STOP_196")).toBe(true) + }) + + test("UserPromptSubmit context reaches the model", async () => { + await h.run("an ordinary prompt 196") + expect(allText(h.llm.requests.at(-1))).toContain("PROMPT-MARKER-196") + }) +}) From 58b177d62b0204be5533aaabecfae7754ed77864 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 16:52:50 -0400 Subject: [PATCH 19/83] test(audit): queereye bug-adjacent pins and interview integration tests (#193) --- packages/core/test/queereye-audit193.test.ts | 144 ++++++++++++++++++ .../opencode/test/design-interview.test.ts | 139 +++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 packages/core/test/queereye-audit193.test.ts create mode 100644 packages/opencode/test/design-interview.test.ts diff --git a/packages/core/test/queereye-audit193.test.ts b/packages/core/test/queereye-audit193.test.ts new file mode 100644 index 0000000..0255dd8 --- /dev/null +++ b/packages/core/test/queereye-audit193.test.ts @@ -0,0 +1,144 @@ +// Issue #193 audit pins (bug-adjacent, behavior pinned — NOT fixed here). +// +// The defects below are tracked for the parent as bugs #266 (prototype-chain +// aliases validate clean) and #270 (vacuous QA gates: cite-gate on missing +// probes, spec-vs-render tautology). These tests pin the CURRENT behavior so +// the audit verdicts are executable; product code is intentionally untouched. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + AXES, + checkDrift, + designPaths, + designTools, + FORMS, + findOneOffs, + readProbes, + readTokens, + renderGuide, + resolveAlias, + validateTokens, + verifyCiteGate, +} from "../src/index.ts" +import { specVsRenderCheck } from "../src/queereye/csf.ts" +import { EXEMPLARS } from "../src/queereye/specs.ts" +import { isWebrefFresh } from "../src/queereye/webref.ts" + +let root: string +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-audit193-")) +}) +afterEach(() => rm(root, { recursive: true, force: true })) + +const call = (name: string, input: Record = {}, agent = "designer") => + designTools({ root }) + .find((tool) => tool.name === name)! + .execute(input, { agent }) + +/** Skip the whole interview through the tools so tokens.json exists. */ +const completeInterview = async () => { + for (const axis of AXES) for (const slot of FORMS[axis].requiredSlots) await call("es_design_skip", { axis, slot }) + await call("es_design_complete", {}) +} + +describe("audit #193 pin: prototype-chain aliases (#266)", () => { + test("an alias through the prototype chain validates clean today", () => { + const tree = { + color: { + $type: "color", + primitive: { ink: { $value: "#0f172a" } }, + semantic: { evil: { $value: "{__proto__.toString}" } }, + }, + } + // FINDING (#266): the `in` operator in walkPath follows the prototype + // chain, so this resolves to Object.prototype.toString and validates + // with zero errors and zero one-offs. + expect(validateTokens(tree)).toEqual([]) + expect(findOneOffs(tree)).toEqual([]) + expect(typeof resolveAlias(tree, "__proto__.toString")).toBe("function") + }) + + test("inherited Object members validate clean as bare aliases today", () => { + for (const ref of ["toString", "valueOf"]) { + const tree = { + color: { + $type: "color", + primitive: { ink: { $value: "#0f172a" } }, + semantic: { evil: { $value: `{${ref}}` } }, + }, + } + // FINDING (#266): same mechanism, single segment. + expect(validateTokens(tree)).toEqual([]) + } + }) + + test("control: a genuinely dangling alias is still refused", () => { + const tree = { + color: { + $type: "color", + primitive: { ink: { $value: "#0f172a" } }, + semantic: { evil: { $value: "{color.primitive.missing}" } }, + }, + } + expect(validateTokens(tree).some((error) => error.includes("does not resolve"))).toBe(true) + }) +}) + +describe("audit #193 pin: cite-gate with deleted probes.json (#270.1)", () => { + test("deleted probes + guide re-rendered with [] verifies today", async () => { + await completeInterview() + await call("es_design_specs", {}) + await call("es_design_harvest", {}) + const receipt = await readFile(path.join(root, ".factory/design/cite-gate-demo.md"), "utf8") + + await rm(designPaths(root).probes, { force: true }) + expect(await readProbes(root)).toBeUndefined() + const tokens = await readTokens(root) + await writeFile(designPaths(root).guide, renderGuide(tokens, [])) + + // FINDING (#270.1): readProbes defaults a missing file to [] (store.ts), + // so the drift gate is green and the cite-gate verifies with zero + // contrast pairs — the contrast QA leg is vacuous on missing probes. + expect(await checkDrift(root)).toEqual({ drifted: [], missing: [], tokenErrors: [] }) + expect(await verifyCiteGate(receipt, root)).toEqual([]) + }) + + test("control: the receipt still binds tokens, so a stale tokens tree is refused", async () => { + await completeInterview() + await call("es_design_harvest", {}) + const receipt = await readFile(path.join(root, ".factory/design/cite-gate-demo.md"), "utf8") + const tokens = (await readTokens(root)) as Record + tokens.$edited = true + await writeFile(designPaths(root).tokens, `${JSON.stringify(tokens, null, 2)}\n`) + expect(await verifyCiteGate(receipt, root)).toContainEqual(expect.stringContaining("stale/forged receipt refused")) + }) +}) + +describe("audit #193 pin: spec-vs-render check shape (#270.2)", () => { + test("the leg CAN fail when fed actual render output", () => { + const spec = EXEMPLARS.button as unknown as Record + const slots = Object.keys((spec.slots ?? {}) as Record).sort() + // Fed the real slot list it is green … + expect(specVsRenderCheck(spec, slots, isWebrefFresh())).toEqual([]) + // … but a render missing slots fails the fill leg … + const errors = specVsRenderCheck(spec, ["label"], true) + expect(errors.some((error) => error.includes("not filled in render"))).toBe(true) + // … and a missing render output is refused outright. + expect(specVsRenderCheck(spec, null, true).some((error) => error.includes("must be supplied"))).toBe(true) + }) + + test("es_design_specs(check:true) feeds each spec its own slots, so the fill leg cannot fail", async () => { + await call("es_design_specs", {}) + // FINDING (#270.2): ops.ts feeds Object.keys(spec.slots) as the + // "render output", so every expected slot is trivially filled and the + // tool check passes even though the leg is never exercised. + for (const spec of Object.values(EXEMPLARS)) { + const record = spec as unknown as Record + const ownSlots = Object.keys((record.slots ?? {}) as Record).sort() + expect(specVsRenderCheck(record, ownSlots, isWebrefFresh())).toEqual([]) + } + expect(await call("es_design_specs", { check: true })).toContain("byte-match") + }) +}) diff --git a/packages/opencode/test/design-interview.test.ts b/packages/opencode/test/design-interview.test.ts new file mode 100644 index 0000000..90a7721 --- /dev/null +++ b/packages/opencode/test/design-interview.test.ts @@ -0,0 +1,139 @@ +// Issue #193: the /design interview on the real testkit host with scripted +// answers. The designer seat walks the 7-axis interview through its tools +// (repairs included), completion writes DTCG tokens that validate with zero +// one-offs, and a failing-contrast palette is refused by the gate. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + compileToCssVars, + designPaths, + findOneOffs, + gatePairs, + probeReport, + readProbes, + readTokens, + renderGuide, + validatePair, + validateTokens, + writeArtifacts, +} from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const answer = (axis: string, slot: string, value: string) => call("es_design_answer", { axis, slot, value }) + +// Scripted answers: real words through every slot (not skip-all — that path +// is design.test.ts). All values are repair-safe (recommended words only). +const SCRIPTED: Array<[string, string, string]> = [ + ["brand", "name", "Acme"], + ["brand", "voice", "minimal"], + ["brand", "values", "clarity"], + ["color", "primary", "ocean"], + ["color", "neutral", "slate"], + ["color", "accent", "teal"], + ["type", "family", "system"], + ["type", "scale", "1.250"], + ["type", "base_size", "16"], + ["layout", "density", "comfortable"], + ["layout", "radius", "soft"], + ["layout", "grid", "12"], + ["effects", "motion", "subtle"], + ["effects", "elevation", "restrained"], + ["effects", "decoration", "text-safe-only"], + ["dark_light", "modes", "both"], + ["dark_light", "surface", "dimmed brand"], + ["a11y", "text_level", "AA"], + ["a11y", "large_text_level", "AA-large"], + ["a11y", "reduced_motion", "yes"], +] + +let state: string +let h: Harness +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-design-interview-host-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# d\n" }, + }) +}, 60_000) + +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +describe("design interview with scripted answers", () => { + test("vague and smuggled answers are repaired, scripted answers settle every slot", async () => { + const status = await h.run(call("es_design_status"), { agent: "designer" }) + expect(status.tools[0]?.text).toContain("slots settled") + + const vague = await h.run(answer("brand", "voice", "whatever"), { agent: "designer" }) + expect(vague.tools[0]?.text).toContain("Not accepted (vague)") + + const smuggled = await h.run(answer("color", "primary", "#ff0000"), { agent: "designer" }) + expect(smuggled.tools[0]?.text).toContain("Not accepted (smuggled)") + + const answered = await h.run(SCRIPTED.map(([axis, slot, value]) => answer(axis, slot, value)).join(" "), { + agent: "designer", + }) + expect(answered.tools.filter((tool) => tool.status === "error")).toHaveLength(0) + expect(answered.tools).toHaveLength(SCRIPTED.length) + for (const tool of answered.tools) expect(tool.text).toContain("Accepted") + expect(answered.tools[answered.tools.length - 1]?.text).toContain("run es_design_complete") + }) + + test("completion writes DTCG tokens that validate with zero one-offs", async () => { + const completed = await h.run(call("es_design_complete"), { agent: "designer" }) + expect(completed.tools[0]?.status).toBe("completed") + expect(completed.tools[0]?.text).toContain("reuse ratio") + + const tokens = await readTokens(h.directory) + expect(validateTokens(tokens)).toEqual([]) + expect(findOneOffs(tokens)).toEqual([]) + + const probes = await readProbes(h.directory) + expect(probes!.length).toBeGreaterThan(0) + expect(probes!.every((row) => row.pass)).toBe(true) + + const guide = await Bun.file(designPaths(h.directory).guide).text() + expect(guide).toContain("# STYLE_GUIDE") + + const checked = await h.run(call("es_design_check"), { agent: "designer" }) + expect(checked.tools[0]?.text).toContain("byte-match") + expect(checked.tools[0]?.text).toContain("One-off mints: 0") + }) + + test("a failing-contrast palette is refused by the gate", async () => { + // The completion tool derives its probes from the ratio-first generator, + // so the interview's own palette always passes; refusal is pinned at the + // gate the completion tool enforces (es_design_complete: "Contrast gate + // failed") and surfaced through es_design_check's pair count. + const failing = gatePairs([{ name: "washed body", fg: "#eeeeee", bg: "#ffffff" }]) + expect(failing).toHaveLength(1) + expect(failing[0]!.errors.join("; ")).toContain("below AA body text minimum 4.5:1") + expect(validatePair("#eeeeee", "#ffffff", { decorative: true }).errors).toContain( + "decorative exemption refused for a text pair", + ) + // Control: a passing pair is green. + expect(validatePair("#595959", "#ffffff").errors).toEqual([]) + + // Host-level: plant a failing probe row (with its guide re-render) and + // the check surfaces the failure instead of passing silently. + const tokens = await readTokens(h.directory) + const bad = probeReport([{ name: "washed body", fg: "#eeeeee", bg: "#ffffff" }]) + expect(bad[0]?.pass).toBe(false) + await writeArtifacts(h.directory, { + tokens, + probes: bad, + guide: renderGuide(tokens, bad), + css: compileToCssVars(tokens), + }) + const surfaced = await h.run(call("es_design_check"), { agent: "designer" }) + expect(surfaced.tools[0]?.text).toContain("Contrast pairs: 0/1 pass") + }) +}) From 46da21408f00c66ebcde46a3e15789d5195a8d96 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 17:07:16 -0400 Subject: [PATCH 20/83] test(audit): LSP runtime integration tests and diagnostics case (#197) --- evals/cases/lsp-diagnostics.json | 1 + packages/core/test/lsp-shutdown.test.ts | 68 ++++++++ packages/opencode/test/lsp-audit197.test.ts | 184 ++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 evals/cases/lsp-diagnostics.json create mode 100644 packages/core/test/lsp-shutdown.test.ts create mode 100644 packages/opencode/test/lsp-audit197.test.ts diff --git a/evals/cases/lsp-diagnostics.json b/evals/cases/lsp-diagnostics.json new file mode 100644 index 0000000..9b43e09 --- /dev/null +++ b/evals/cases/lsp-diagnostics.json @@ -0,0 +1 @@ +{"agent":"build","checks":{"contains":["not assignable","src/bad.ts"],"toolsUsed":["lsp_diagnostics"]},"description":"The agent introduces a type error in src/bad.ts; the typescript language server reports it via lsp_diagnostics. Requires typescript-language-server on PATH (workflow prerequisite, pinned to the registry).","files":{"src/bad.ts":"export const x: number = 1;\n","tsconfig.json":"{\n \"compilerOptions\": { \"strict\": true, \"target\": \"es2022\", \"module\": \"nodenext\" },\n \"include\": [\"src\"]\n}\n"},"id":"lsp-diagnostics","maxSteps":8,"prompt":"In src/bad.ts, change the initializer of x to the string \"not a number\" (keep the number type annotation), using the edit tool. Then call lsp_diagnostics on src/bad.ts and reply quoting the full diagnostic line for the error. This run is headless: never call the question tool. Make independent tool calls in the same block."} \ No newline at end of file diff --git a/packages/core/test/lsp-shutdown.test.ts b/packages/core/test/lsp-shutdown.test.ts new file mode 100644 index 0000000..e2cb084 --- /dev/null +++ b/packages/core/test/lsp-shutdown.test.ts @@ -0,0 +1,68 @@ +// Audit #197 integration tests (core side): the workspace TypeScript server +// reaches diagnostics through the manager, and stopAll shuts the running +// servers down. Same tsc fixture seam as lsp.test.ts; no binaries run directly. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { LspManager } from "../src/index.ts" + +const TYPESCRIPT = path.resolve(import.meta.dir, "../../../node_modules/typescript") + +let root: string +let state: string +let manager: LspManager +const write = async (file: string, text: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), text) +} + +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-lsp-shutdown-")) + state = await mkdtemp(path.join(tmpdir(), "es-lsp-shutdown-state-")) + await write( + "tsconfig.json", + JSON.stringify({ + compilerOptions: { strict: true, target: "ES2022", module: "NodeNext", noEmit: true }, + include: ["src"], + }), + ) + await mkdir(path.join(root, "node_modules"), { recursive: true }) + await symlink(TYPESCRIPT, path.join(root, "node_modules/typescript"), "dir") + await mkdir(path.join(root, "node_modules/.bin"), { recursive: true }) + await symlink(path.join(TYPESCRIPT, "bin/tsc"), path.join(root, "node_modules/.bin/tsc")) + await write("src/math.ts", "export function add(a: number, b: number): number {\n return a + b\n}\n") + await write( + "src/main.ts", + 'import { add } from "./math.js"\n\nconst total: number = add(1, 2)\nconst wrong: string = total\nexport { wrong }\n', + ) + manager = new LspManager({ root, stateDir: state, configFiles: [path.join(root, "opencode.json")] }) +}, 30_000) + +afterAll(async () => { + await manager?.stopAll() + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +describe("LspManager shutdown (#197)", () => { + test("diagnostics start the workspace server and stopAll shuts it down", async () => { + const diagnostics = await manager.diagnostics("src/main.ts", { maxWaitMs: 20_000 }) + expect(Array.isArray(diagnostics)).toBe(true) + const messages = (diagnostics as any[]).map((item) => `${item.range.start.line + 1}: ${item.message}`) + expect(messages.join("\n")).toContain("4: Type 'number' is not assignable to type 'string'") + const running = (await manager.status()).running + expect(running.some((entry) => entry.id === "typescript" && entry.ready)).toBe(true) + await manager.stopAll() + expect((await manager.status()).running).toEqual([]) + }, 60_000) + + test("stopAll is idempotent and the next diagnostics restarts the server lazily", async () => { + await manager.stopAll() + expect((await manager.status()).running).toEqual([]) + const diagnostics = await manager.diagnostics("src/main.ts", { maxWaitMs: 20_000 }) + expect(Array.isArray(diagnostics)).toBe(true) + expect(JSON.stringify(diagnostics)).toContain("not assignable to type 'string'") + expect((await manager.status()).running.some((entry) => entry.id === "typescript" && entry.ready)).toBe(true) + }, 60_000) +}) diff --git a/packages/opencode/test/lsp-audit197.test.ts b/packages/opencode/test/lsp-audit197.test.ts new file mode 100644 index 0000000..220316d --- /dev/null +++ b/packages/opencode/test/lsp-audit197.test.ts @@ -0,0 +1,184 @@ +// Audit #197 integration tests (opencode side): TypeScript diagnostics end to +// end on the real host, the /lsp server command, and the after-edit +// diagnostics channel for an lsp:"none" seat. Testkit host + the existing tsc +// fixture seam only; no sandboxed binaries are run directly. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { agentSpec, LspManager } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { EsRpc } from "../src/rpc-def.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") +const TYPESCRIPT = path.resolve(import.meta.dir, "../../../node_modules/typescript") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` + +let state: string +let h: Harness +let closed = false +// Plugin unload is the only shutdown path: server.ts returns stopAll on +// unload, and between sessions only the idle sweeper reaps. Spying the manager +// proves host close shuts the servers down. +const originalStopAll = LspManager.prototype.stopAll +let stopCalls = 0 + +const closeOnce = async () => { + if (!closed) { + closed = true + await h.close() + } +} + +beforeAll(async () => { + LspManager.prototype.stopAll = async function (this: LspManager) { + stopCalls++ + return originalStopAll.call(this) + } + state = await mkdtemp(path.join(tmpdir(), "es-lsp-197-state-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { + "README.md": "# lsp audit 197\n", + ".gitignore": "node_modules/\n", + "tsconfig.json": JSON.stringify({ + compilerOptions: { strict: true, noEmit: true, module: "NodeNext", target: "ES2022" }, + include: ["src"], + }), + "src/math.ts": "export function add(a: number, b: number): number {\n return a + b\n}\n", + }, + }) + await mkdir(path.join(h.directory, "node_modules/.bin"), { recursive: true }) + await symlink(TYPESCRIPT, path.join(h.directory, "node_modules/typescript"), "dir") + await symlink(path.join(TYPESCRIPT, "bin/tsc"), path.join(h.directory, "node_modules/.bin/tsc")) +}, 60_000) +afterAll(async () => { + await closeOnce().catch(() => undefined) + LspManager.prototype.stopAll = originalStopAll + await rm(state, { recursive: true, force: true }) +}) + +describe("LSP diagnostics end to end (#197)", () => { + test("a type-error edit reaches the agent and starts the workspace server", async () => { + const { tools: outcomes } = await h.run( + call("write", { + path: "src/main.ts", + content: 'import { add } from "./math.js"\nexport const bad: string = add(1, 2)\n', + }), + ) + expect(outcomes[0]?.status).toBe("completed") + expect(outcomes[0]?.text).toContain("") + expect(outcomes[0]?.text).toContain("src/main.ts:2") + expect(outcomes[0]?.text).toContain("not assignable to type 'string'") + const rpc = (h.opencode as any).rpc(EsRpc) + const lsp = await rpc.lspState({}, { location: h.location }) + expect(lsp.enabled).toBe(true) + const typescript = (lsp.servers as Array).find((server) => server.id === "typescript") + expect(typescript?.available).toBe(true) + expect(typescript?.command).toContain("tsc") + expect(typescript?.running).toBeGreaterThanOrEqual(1) + }, 60_000) + + test("the /lsp server command reports status lines", async () => { + const created = await (h.opencode.sessions as any).create({ location: h.location }) + const sessionID: string = created.id + await (h.opencode.sessions as any).command({ sessionID, name: "lsp", text: "" }) + await (h.opencode.sessions as any).wait({ sessionID }) + const context = await (h.opencode.sessions as any).context({ sessionID }) + const text: string = context + .filter((message: any) => message?.type === "synthetic") + .map((message: any) => String(message?.text ?? "")) + .join("\n") + const lines = text.split("\n") + expect(lines[0]).toBe("LSP is on.") + expect(lines.length).toBeGreaterThan(1) + for (const line of lines) { + if (line.startsWith("! ")) continue + if (line === "LSP is on.") continue + expect(line).toMatch(/^\S+ \[[^\]]+\]: /) + } + const typescript = lines.find((line) => line.startsWith("typescript [")) + expect(typescript).toContain("available (") + expect(typescript).toContain("tsc") + expect(typescript).toContain("--lsp") + // The previous test's edit started the workspace server in this project. + expect(typescript).toContain("running for 1 root") + }, 60_000) + + // NOTE (bug #275, lifecycle half): servers are killed on plugin unload only; + // no session-end hook exists (between sessions only the idle sweeper reaps). + // Session-end kill is untested here by design — do NOT change lifecycle. + test("host close stops the language servers (stopAll on unload)", async () => { + expect(stopCalls).toBe(0) + await closeOnce() + expect(stopCalls).toBeGreaterThanOrEqual(1) + }, 60_000) +}) + +describe("after-edit diagnostics for an lsp:none seat (#277)", () => { + test("the injector appends diagnostics even though the seat has no LSP tools (current behavior)", async () => { + // Premise pin: the designer seat is lsp:"none" (registry), so its agent + // permissions deny the lsp/lsp_rename actions — yet the execute.after hook + // appends server diagnostics to any completed write (policy.ts:177-189). + // This test records that leak; scoping it is bug #277, not this file. + expect(agentSpec("designer")?.lsp).toBe("none") + const { createPolicyHooks } = await import("../src/policy.ts") + const root = await mkdtemp(path.join(tmpdir(), "es-lsp-197-none-")) + try { + const hooks = createPolicyHooks({ + root, + options: { lspAfterEdit: true }, + lsp: { + diagnostics: async () => [ + { + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 1 } }, + severity: 1, + message: "Type 'number' is not assignable to type 'string'.", + }, + ], + }, + } as any) + const event = { + tool: "write", + agent: "designer", + id: "t1", + input: { path: "src/main.ts" }, + status: "completed", + result: { content: "ok" }, + } + await hooks.after(event as any) + const content = String((event.result as any)?.content ?? "") + expect(content).toContain("") + expect(content).toContain("src/main.ts:1:1") + expect(content).toContain("not assignable to type 'string'") + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + test("an unavailable server appends nothing (control)", async () => { + const { createPolicyHooks } = await import("../src/policy.ts") + const root = await mkdtemp(path.join(tmpdir(), "es-lsp-197-none-")) + try { + const hooks = createPolicyHooks({ + root, + options: { lspAfterEdit: true }, + lsp: { diagnostics: async () => ({ unavailable: "no language server handles .md" }) }, + } as any) + const event = { + tool: "write", + agent: "designer", + id: "t1", + input: { path: ".factory/design/notes/idea.md" }, + status: "completed", + result: { content: "ok" }, + } + await hooks.after(event as any) + expect((event.result as any)?.content).toBe("ok") + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) From 67b64dff1f779b19efb0c6fa24d6de5cecf950fc Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 17:07:36 -0400 Subject: [PATCH 21/83] test(audit): TUI panels, commands and compaction integration tests (#198) --- .../opencode/test/compaction-state.test.ts | 57 +++++++ packages/opencode/test/plugin-tree.test.ts | 84 ++++++++++ packages/opencode/test/slash-commands.test.ts | 145 ++++++++++++++++++ 3 files changed, 286 insertions(+) create mode 100644 packages/opencode/test/compaction-state.test.ts create mode 100644 packages/opencode/test/plugin-tree.test.ts create mode 100644 packages/opencode/test/slash-commands.test.ts diff --git a/packages/opencode/test/compaction-state.test.ts b/packages/opencode/test/compaction-state.test.ts new file mode 100644 index 0000000..db12173 --- /dev/null +++ b/packages/opencode/test/compaction-state.test.ts @@ -0,0 +1,57 @@ +// Issue #198: compaction injects the factory-state block for factory seats +// only (session.ts). A factory seat's post-compaction request carries +// "Preserve this factory state" plus the block; a non-factory +// agent (the user's own build agent) compacts with neither. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { boot, directiveScript, type Harness, systemText } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") + +let h: Harness +let state: string +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-compaction-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# demo\n", ".gitignore": "node_modules/\n" }, + }) +}, 120_000) +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +async function compactTexts(agent: string | undefined): Promise { + const options = agent === undefined ? {} : { agent } + const { sessionID } = await h.run("hello", options) + const before = h.llm.requests.length + await (h.opencode as any).sessions.compact({ sessionID } as any) + await (h.opencode as any).sessions.wait({ sessionID } as any) + return h.llm.requests.slice(before).map((request) => systemText(request)) +} + +describe("compaction carries the factory state for factory seats only", () => { + test("a factory seat keeps the state block across compaction", async () => { + const texts = await compactTexts("factory") + expect(texts.length).toBeGreaterThan(0) + const kept = texts.filter( + (text) => text.includes("Preserve this factory state") && text.includes(""), + ) + expect(kept.length).toBeGreaterThan(0) + expect(kept.join("\n")).toContain("No factory run in this project.") + }, 60_000) + + test("a non-factory agent compacts without the state block", async () => { + const texts = await compactTexts(undefined) + expect(texts.length).toBeGreaterThan(0) + for (const text of texts) { + expect(text).not.toContain("Preserve this factory state") + expect(text).not.toContain("") + } + }, 60_000) +}) diff --git a/packages/opencode/test/plugin-tree.test.ts b/packages/opencode/test/plugin-tree.test.ts new file mode 100644 index 0000000..a5b1bea --- /dev/null +++ b/packages/opencode/test/plugin-tree.test.ts @@ -0,0 +1,84 @@ +// Issue #198: the plugin registers additively — setup writes no project-tree +// files. The "no files written" wording is qualified (bug filed): the engine +// key IS written, but to the masked state dir, never the project. So this +// asserts the location contract, not absence: +// 1. right after plugin load (no session activity), the project tree holds +// exactly the initial files; +// 2. after agent activity forces sealed writes, engine.key exists in the +// passed state dir (and is a real key), while no engine.key exists +// anywhere under the project tree. +import { afterAll, beforeAll, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { engineKeyPath } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") +const initialFiles = { "README.md": "# demo\n", ".gitignore": "node_modules/\n" } + +/** Project-tree entries below `dir`, excluding the git internals. */ +async function projectTree(dir: string): Promise { + const proc = Bun.spawn( + ["find", dir, "-mindepth", "1", "-not", "-path", `${dir}/.git`, "-not", "-path", `${dir}/.git/*`], + { stdout: "pipe", stderr: "pipe" }, + ) + const [stdout, , code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect(code).toBe(0) + return stdout + .split("\n") + .filter(Boolean) + .map((file) => path.relative(dir, file)) + .sort() +} + +/** engine.key files anywhere under `dir` (project tree must hold none). */ +async function engineKeysUnder(dir: string): Promise { + const proc = Bun.spawn(["find", dir, "-name", "engine.key", "-not", "-path", `${dir}/.git/*`], { + stdout: "pipe", + stderr: "pipe", + }) + const [stdout, , code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect(code).toBe(0) + return stdout.split("\n").filter(Boolean).sort() +} + +let h: Harness +let state: string +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-plugin-tree-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: initialFiles, + }) +}, 120_000) +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +test("plugin load writes no project-tree files", async () => { + // No session has run yet: this diffs the tree before/after plugin load. + expect(await projectTree(h.directory)).toEqual(Object.keys(initialFiles).sort()) + expect(await engineKeysUnder(h.directory)).toEqual([]) +}) + +test("engine.key lives in the state dir, never the project tree", async () => { + // A factory turn forces sealed writes (spend/seat tracking), which mint the + // engine key when absent. + await h.run("hello", { agent: "factory" }) + const key = engineKeyPath(state) + expect(await Bun.file(key).exists()).toBe(true) + expect((await Bun.file(key).arrayBuffer()).byteLength).toBeGreaterThanOrEqual(32) + expect(await engineKeysUnder(h.directory)).toEqual([]) +}) diff --git a/packages/opencode/test/slash-commands.test.ts b/packages/opencode/test/slash-commands.test.ts new file mode 100644 index 0000000..a00bc25 --- /dev/null +++ b/packages/opencode/test/slash-commands.test.ts @@ -0,0 +1,145 @@ +// Issue #198: every server slash command invoked on the real in-process v2 +// host, asserting seat/verb routing (server.ts commands, not the TUI `es-*` +// palette set in tui.tsx, which tui.test.ts covers). +// +// FINDING (duplicate registration, bug filed): `research` is registered twice +// (server.ts deep-research flow vs report/coverage reader). The host command +// state is a Map keyed by name (`editor.set(definition.name, ...)`), so the +// LAST registration wins: both `/research` invocations below route to the +// report/coverage reader, and the deep-research flow is unreachable by name. +// Routing is intentionally NOT changed here. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" + +const pluginDir = path.resolve(import.meta.dir, "..") + +let h: Harness +let state: string +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-slash-commands-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# demo\n", ".gitignore": "node_modules/\n" }, + }) +}, 120_000) +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +async function invoke(name: string, text: string) { + const session = await (h.opencode as any).sessions.create({ location: h.location } as any) + const before = await (h.opencode as any).sessions.get({ sessionID: session.id } as any) + await (h.opencode as any).sessions.command({ sessionID: session.id, name, text } as any) + await (h.opencode as any).sessions.wait({ sessionID: session.id } as any) + const after = await (h.opencode as any).sessions.get({ sessionID: session.id } as any) + const context = await (h.opencode as any).sessions.context({ sessionID: session.id } as any) + return { before: before?.agent as string | undefined, after: after?.agent as string | undefined, context } +} + +const userTexts = (context: any[]): string[] => + context.filter((message) => message?.type === "user").map((message) => String(message.text ?? "")) +const syntheticTexts = (context: any[]): string[] => + context.filter((message) => message?.type === "synthetic").map((message) => String(message.text ?? "")) + +describe("slash commands route to the right seat", () => { + test("/grill switches to grill", async () => { + const { after, context } = await invoke("grill", "my idea") + expect(after).toBe("grill") + expect(userTexts(context).join("\n")).toContain("Grill me on: my idea") + }, 60_000) + + test("/brainstorm switches to brainstormer", async () => { + const { after, context } = await invoke("brainstorm", "fast tests") + expect(after).toBe("brainstormer") + expect(userTexts(context).join("\n")).toContain("Brainstorm: fast tests") + }, 60_000) + + test("/harvest switches to harvester", async () => { + const { after, context } = await invoke("harvest", "teardown x") + expect(after).toBe("harvester") + expect(userTexts(context).join("\n")).toContain("Darkharvest: teardown x") + }, 60_000) + + test("/design switches to designer", async () => { + const { after, context } = await invoke("design", "acme app") + expect(after).toBe("designer") + expect(userTexts(context).join("\n")).toContain("Run the design interview. Product: acme app") + }, 60_000) + + test("/scout switches to scout", async () => { + const { after, context } = await invoke("scout", "test runner") + expect(after).toBe("scout") + expect(userTexts(context).join("\n")).toContain("Scout: test runner") + }, 60_000) +}) + +describe("slash commands that answer verbatim (no seat switch)", () => { + test("/factory with no run says to grill first", async () => { + const { before, after, context } = await invoke("factory", "") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toContain("No factory run here yet — run /grill first.") + }, 60_000) + + test("/audit with no spend ceiling points at a terminal", async () => { + const { before, after, context } = await invoke("audit", "src/x.ts") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toContain("Audits run inside a factory run with a spend ceiling.") + }, 60_000) + + test("/gates reports the gate summary", async () => { + const { before, after, context } = await invoke("gates", "") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toMatch(/^Gates (passed|failed):/m) + }, 60_000) + + test("/lsp reports language servers", async () => { + const { before, after, context } = await invoke("lsp", "") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toContain("LSP is") + }, 60_000) + + test("/hooks reports the hook bridge", async () => { + const { before, after, context } = await invoke("hooks", "") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toContain("Hook bridge:") + }, 60_000) + + test("/status reports the factory state block", async () => { + const { before, after, context } = await invoke("status", "") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toContain("") + }, 60_000) + + test("/config reports layers and config", async () => { + const { before, after, context } = await invoke("config", "") + expect(after).toBe(before) + expect(syntheticTexts(context).join("\n")).toContain("Layers:") + }, 60_000) +}) + +describe("/research duplicate registration: the last body wins", () => { + test("/research with no args reads the report/coverage/sources", async () => { + const { before, after, context } = await invoke("research", "") + expect(after).toBe(before) + const text = syntheticTexts(context).join("\n") + expect(text).toContain("No research report yet (.factory/research/REPORT.md).") + expect(text).toContain("Cached sources:") + }, 60_000) + + test("/research deep args still reach the reader, not the deep flow (FINDING)", async () => { + const { before, after, context } = await invoke("research", "deep what is foo --max-usd 5") + // The deep-research body would switch to deep-researcher and begin a run; + // the reader body wins instead: no switch, reader text. + expect(after).toBe(before) + expect(after).not.toBe("deep-researcher") + const text = syntheticTexts(context).join("\n") + expect(text).toContain("No research report yet (.factory/research/REPORT.md).") + expect(text).toContain("Cached sources:") + }, 60_000) +}) From 2e04203675a497500f8c085cbb8e3bb92db9ec61 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 17:22:39 -0400 Subject: [PATCH 22/83] test(audit): self-improvement harvest and distill tests (#204) --- packages/cli/test/improve-204.test.ts | 282 +++++++++++++++++++++++++ packages/core/test/improve-204.test.ts | 273 ++++++++++++++++++++++++ 2 files changed, 555 insertions(+) create mode 100644 packages/cli/test/improve-204.test.ts create mode 100644 packages/core/test/improve-204.test.ts diff --git a/packages/cli/test/improve-204.test.ts b/packages/cli/test/improve-204.test.ts new file mode 100644 index 0000000..36a088b --- /dev/null +++ b/packages/cli/test/improve-204.test.ts @@ -0,0 +1,282 @@ +// #204 integration through the CLI: harvest + distill over several +// unit-constructed run dirs (one with a broken audit chain); scrubbed +// output and the code-disposes gate dropping fabricated evidence with +// reasons. +// +// Unit-constructed only: no real factory runs, no timing/perf assertions, +// no --open-pr (human TTY; existing pipe-refusal coverage stands). +import { describe, expect, test } from "bun:test" +import { appendFile, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + appendAuditEntry, + disposedWithReasons, + disposeReason, + distillProposals, + factoryLayout, + TelemetrySchema, + undisposed, +} from "@heretek-ai/es-core" +import { main } from "../src/main.ts" +import type { ConfirmIO } from "../src/tty.ts" + +const pipe: ConfirmIO = { interactive: false, write: () => {}, readLine: async () => "y", readSecret: async () => "" } + +const GH_A = "ghp_cliHarvest001AAA" +const GH_BROKEN = "ghu_cliBroken999aaa" +const SK_ANT = "sk-ant-cliplanted0000Z" +const AKIA = "AKIAIOSFODNN7EXAMPLE" +const BEARER = "eyJhbGciOiJIUzI1NiJ9" +const GATE_RULE = "lint/no-unused-vars" + +async function writeGateSummary(root: string, id: string): Promise { + const dir = path.join(factoryLayout(root).runs, id, "gates", "2026-10-04T00-30-full") + await mkdir(dir, { recursive: true }) + await writeFile( + path.join(dir, "summary.json"), + `${JSON.stringify( + { + passed: false, + findings: [ + { file: "src/x.ts", line: 3, rule: GATE_RULE, severity: "error", message: "unused", check: "lint" }, + { file: "src/y.ts", line: 9, rule: GATE_RULE, severity: "error", message: "unused", check: "lint" }, + ], + }, + null, + 2, + )}\n`, + ) +} + +async function writeAuditRecords(root: string): Promise { + const dir = factoryLayout(root).audits + await mkdir(path.join(dir, "audit-01"), { recursive: true }) + const finding = (title: string, detail: string) => ({ + kind: "invariant", + title, + severity: "high", + file: "packages/core/src/trust/policy.ts", + lines: [59, 75], + excerpt: "export function evaluateWrite(context, agentId, target)", + detail, + holds: false, + }) + await writeFile( + path.join(dir, "audit-01", "records.json"), + `${JSON.stringify( + [ + { + audit: "audit-01", + round: 1, + seat: "auditor-thesis", + by: "auditor", + verdict: "fail", + notes: "notes", + findings: [ + finding("control files are deny-write", "the programmer seat can write a control file path"), + finding("control files are deny-write again", "a second invariant detail here"), + ], + at: "2026-10-04T00:00:00.000Z", + }, + ], + null, + 2, + )}\n`, + ) +} + +async function writeChain(root: string, haltReason: string): Promise { + await appendAuditEntry(root, { actor: "system", action: "stage.build" }) + await appendAuditEntry(root, { + actor: "agent:qa", + action: "qa.fail", + payload: { phase: "p1", reason: "lint gate red" }, + }) + await appendAuditEntry(root, { actor: "system", action: "stage.halted", payload: { reason: haltReason } }) +} + +async function makeRun(parent: string, id: string, haltReason: string, broken: boolean): Promise { + const root = path.join(parent, id) + await writeGateSummary(root, id) + await writeAuditRecords(root) + await writeChain(root, haltReason) + if (broken) await appendFile(factoryLayout(root).audit, "not-json\n") + return root +} + +async function writeEvals(dir: string, failures: { id: string; failures: string[] }[]): Promise { + await mkdir(dir, { recursive: true }) + await writeFile( + path.join(dir, "agg.json"), + `${JSON.stringify( + { + at: "2026-10-04T00-00-00-000Z", + model: "test/model", + results: failures.map((entry, index) => ({ + id: entry.id, + pass: entry.failures.length === 0, + failures: entry.failures, + steps: 4 + index, + })), + }, + null, + 2, + )}\n`, + ) +} + +async function walkFiles(dir: string): Promise { + const out: string[] = [] + const visit = async (current: string): Promise => { + for (const name of await readdir(current)) { + const full = path.join(current, name) + const info = await stat(full).catch(() => undefined) + if (info?.isDirectory()) await visit(full) + else out.push(full) + } + } + await visit(dir) + return out.sort() +} + +describe("issue #204: es improve harvest + distill over several runs incl. a broken chain", () => { + test("harvest reports the broken chain and scrubs planted tokens; distill writes all three kinds, drops fabrications", async () => { + const parent = await mkdtemp(path.join(tmpdir(), "es-204-cli-")) + const root = await mkdtemp(path.join(tmpdir(), "es-204-cli-root-")) + const state = await mkdtemp(path.join(tmpdir(), "es-204-cli-state-")) + try { + const run = async (argv: string[]) => { + const printed: string[] = [] + const code = await main(argv, { + print: (text) => printed.push(text), + confirm: pipe, + cwd: root, + stateDir: state, + }) + return { code, out: printed.join("\n") } + } + const runA = await makeRun(parent, "run-a", `halt A: GITHUB_TOKEN=${GH_A}`, false) + const runB = await makeRun(parent, "run-b", `halt B: Bearer ${BEARER}`, false) + const runBroken = await makeRun(parent, "run-broken", `halt C: ${GH_BROKEN}`, true) + const brokenFile = factoryLayout(runBroken).audit + const before = await readFile(brokenFile, "utf8") + const evals = path.join(parent, "evals") + await writeEvals(evals, [ + { id: "case-x", failures: ["gate lint red", "gate lint red", `leaked ${SK_ANT}`] }, + { id: "case-y", failures: ["gate lint red", `key ${AKIA} here`] }, + { id: "case-ok", failures: [] }, + ]) + const telemetryFile = path.join(root, "telemetry.json") + const harvested = await run([ + "improve", + "harvest", + "--runs", + `${runA},${runB},${runBroken}`, + "--evals", + evals, + "--out", + telemetryFile, + ]) + expect(harvested.code).toBe(0) + expect(harvested.out).toContain("3 run(s) and 3 eval(s)") + const raw = await readFile(telemetryFile, "utf8") + const telemetry = TelemetrySchema.parse(JSON.parse(raw)) + expect(telemetry.runs.map((entry) => entry.id)).toEqual(["run-a", "run-b", "run-broken"]) + for (const secret of [GH_A, GH_BROKEN, SK_ANT, AKIA, BEARER]) expect(raw).not.toContain(secret) + expect(raw).toContain("GITHUB_TOKEN=") + expect(raw).toContain("Bearer") + const brokenTelemetry = telemetry.runs.find((entry) => entry.id === "run-broken")! + expect(brokenTelemetry.auditChain.valid).toBe(false) + expect(typeof brokenTelemetry.auditChain.error).toBe("string") + // Reported, never repaired. + expect(await readFile(brokenFile, "utf8")).toBe(before) + const proposalsDir = path.join(root, "proposals") + const distilled = await run([ + "improve", + "distill", + "--telemetry", + telemetryFile, + "--out", + proposalsDir, + "--min-occurrences", + "2", + "--min-runs", + "2", + ]) + expect(distilled.code).toBe(0) + expect(distilled.out).toContain("Nothing was applied") + expect(distilled.out).toContain("prompt-guidance") + expect(distilled.out).toContain("gate-tuning") + expect(distilled.out).toContain("domain-pack") + const files = await walkFiles(proposalsDir) + expect(files.some((file) => path.basename(file) === "gates.json")).toBe(false) + expect(files.some((file) => path.basename(file) === "proposal.json")).toBe(true) + // Fabricated evidence against the CLI-harvested telemetry is dropped with reasons. + const proposals = await distillProposals(telemetry, { minOccurrences: 2, minRuns: 2 }) + expect(undisposed(proposals, telemetry)).toEqual(proposals) + const [first] = proposals + const forged = { ...first!, cluster: { ...first!.cluster, evidence: ["run x: invented quote"] } } + expect(disposeReason(forged, telemetry).some((reason) => reason.includes("verbatim"))).toBe(true) + expect(undisposed([forged], telemetry)).toEqual([]) + expect(disposedWithReasons([forged], telemetry)[0]!.reasons.length).toBeGreaterThan(0) + } finally { + await rm(parent, { recursive: true, force: true }) + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) + + test("a hallucinated model numeral is disposed by the gate with its reason in output and dropped.json", async () => { + const parent = await mkdtemp(path.join(tmpdir(), "es-204-cli-model-")) + const root = await mkdtemp(path.join(tmpdir(), "es-204-cli-model-root-")) + const state = await mkdtemp(path.join(tmpdir(), "es-204-cli-model-state-")) + try { + const run = async (argv: string[]) => { + const printed: string[] = [] + const code = await main(argv, { + print: (text) => printed.push(text), + confirm: pipe, + cwd: root, + stateDir: state, + }) + return { code, out: printed.join("\n") } + } + const runA = await makeRun(parent, "run-a", "halt A", false) + const evals = path.join(parent, "evals") + await writeEvals(evals, [ + { id: "case-x", failures: ["fails like gpt-4 would", "fails like gpt-4 would"] }, + { id: "case-y", failures: ["fails like gpt-4 would"] }, + ]) + const telemetryFile = path.join(root, "telemetry.json") + expect((await run(["improve", "harvest", "--runs", runA, "--evals", evals, "--out", telemetryFile])).code).toBe(0) + const proposalsDir = path.join(root, "proposals") + const distilled = await run([ + "improve", + "distill", + "--telemetry", + telemetryFile, + "--out", + proposalsDir, + "--min-occurrences", + "3", + "--min-runs", + "2", + ]) + expect(distilled.code).toBe(0) + expect(distilled.out).toContain("Disposed by the gate") + expect(distilled.out).toContain("hallucinated model numeral") + const dropped = JSON.parse(await readFile(path.join(proposalsDir, "dropped.json"), "utf8")) as Array<{ + reasons: string[] + }> + expect(dropped.length).toBeGreaterThan(0) + expect( + dropped.some((entry) => entry.reasons.some((reason) => reason.includes("hallucinated model numeral"))), + ).toBe(true) + } finally { + await rm(parent, { recursive: true, force: true }) + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/core/test/improve-204.test.ts b/packages/core/test/improve-204.test.ts new file mode 100644 index 0000000..41c0f0f --- /dev/null +++ b/packages/core/test/improve-204.test.ts @@ -0,0 +1,273 @@ +// #204 integration: harvest + distill over several unit-constructed run +// dirs (one with a broken audit chain); scrubbed output and the +// code-disposes gate dropping fabricated evidence with reasons. +// +// Unit-constructed only: no real factory runs, no timing/perf assertions +// (#155 is flaky by nature and tracked separately), no --open-pr. +import { describe, expect, test } from "bun:test" +import { appendFile, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + appendAuditEntry, + clusterTelemetry, + disposedWithReasons, + disposeReason, + distillProposals, + factoryLayout, + harvestRun, + harvestTelemetry, + signEngineFile, + TelemetrySchema, + undisposed, + writeDropped, + writeProposals, +} from "../src/index.ts" + +// Planted secrets (fake values in the style of the existing scrub tests). +// Every one must be absent from harvested telemetry. +const GH_A = "ghp_plantedHarvest001AAA" +const GH_BROKEN = "ghu_BrokenChain999aaa" +const SK_ANT = "sk-ant-planted0000XYZ" +const AKIA = "AKIAIOSFODNN7EXAMPLE" +const SLACK = "xoxb-12345-abcdef" +const BEARER = "eyJhbGciOiJIUzI1NiJ9" +const PEM_BODY = "MIIEowIBAAKCAQEAplanted" +const HUNTER = "hunter2-planted" + +const QA_REASON = "lint gate red" +const GATE_RULE = "lint/no-unused-vars" + +async function writeState(root: string, id: string, haltReason: string): Promise { + const file = factoryLayout(root).state + await mkdir(path.dirname(file), { recursive: true }) + await writeFile( + file, + `${JSON.stringify( + { + version: 2, + runId: id, + stage: "HALTED", + mode: "build", + createdAt: "2026-10-04T00:00:00.000Z", + updatedAt: "2026-10-04T01:00:00.000Z", + spend: { usd: 1, estimated: false, events: 5 }, + phases: [ + { + id: "p1", + title: "phase", + status: "failed", + goalHash: "a".repeat(64), + failures: 2, + replanned: false, + history: [], + }, + ], + halt: { reason: haltReason, at: "2026-10-04T01:00:00.000Z", from: "QA" }, + audits: [], + }, + null, + 2, + )}\n`, + ) +} + +async function writeGateSummary(root: string, id: string): Promise { + const dir = path.join(factoryLayout(root).runs, id, "gates", "2026-10-04T00-30-full") + await mkdir(dir, { recursive: true }) + await writeFile( + path.join(dir, "summary.json"), + `${JSON.stringify( + { + passed: false, + findings: [ + { file: "src/x.ts", line: 3, rule: GATE_RULE, severity: "error", message: "unused", check: "lint" }, + { file: "src/y.ts", line: 9, rule: GATE_RULE, severity: "error", message: "unused", check: "lint" }, + ], + }, + null, + 2, + )}\n`, + ) +} + +async function writeChain(root: string, haltReason: string): Promise { + await appendAuditEntry(root, { actor: "system", action: "stage.build" }) + await appendAuditEntry(root, { actor: "agent:qa", action: "qa.fail", payload: { phase: "p1", reason: QA_REASON } }) + await appendAuditEntry(root, { actor: "system", action: "stage.halted", payload: { reason: haltReason } }) +} + +/** Two sealed runs plus one sealed run with a broken chain, and an evals dir. */ +async function buildFixture(): Promise<{ + parent: string + stateDir: string + runs: string[] + broken: string + evals: string + cleanup(): Promise +}> { + const parent = await mkdtemp(path.join(tmpdir(), "es-204-core-")) + const stateDir = await mkdtemp(path.join(tmpdir(), "es-204-core-state-")) + const haltA = `phase p1 failed: GITHUB_TOKEN=${GH_A} with Bearer ${BEARER}` + const haltB = + `phase p1 failed: api-key: ${HUNTER} plus ` + + `-----BEGIN RSA PRIVATE KEY-----\n${PEM_BODY}\n-----END RSA PRIVATE KEY-----` + const haltBroken = `phase p1 failed: slack ${SLACK} and ${GH_BROKEN}` + const ids = ["run-a", "run-b"] as const + const halts: Record = { "run-a": haltA, "run-b": haltB } + for (const id of ids) { + const root = path.join(parent, id) + await writeState(root, id, halts[id]!) + await writeGateSummary(root, id) + await writeChain(root, halts[id]!) + await signEngineFile(factoryLayout(root).state, stateDir) + } + const broken = path.join(parent, "run-broken") + await writeState(broken, "run-broken", haltBroken) + await writeGateSummary(broken, "run-broken") + await writeChain(broken, haltBroken) + await signEngineFile(factoryLayout(broken).state, stateDir) + // Break the chain with a corrupt trailing line: the valid prefix stays so + // the run still contributes its gate/qa signals, but verification fails. + await appendFile(factoryLayout(broken).audit, "not-json\n") + const evals = path.join(parent, "evals") + await mkdir(evals, { recursive: true }) + await writeFile( + path.join(evals, "agg.json"), + `${JSON.stringify( + { + at: "2026-10-04T00-00-00-000Z", + model: "test/model", + results: [ + { id: "case-x", pass: false, failures: ["gate lint red", "gate lint red", `leaked ${SK_ANT}`], steps: 9 }, + { id: "case-y", pass: false, failures: ["gate lint red", `key ${AKIA} here`], steps: 4 }, + { id: "case-ok", pass: true, failures: [], steps: 3 }, + ], + }, + null, + 2, + )}\n`, + ) + return { + parent, + stateDir, + runs: [path.join(parent, "run-a"), path.join(parent, "run-b"), broken], + broken, + evals, + async cleanup() { + await rm(parent, { recursive: true, force: true }) + await rm(stateDir, { recursive: true, force: true }) + }, + } +} + +async function walkFiles(dir: string): Promise { + const out: string[] = [] + const visit = async (current: string): Promise => { + for (const name of await readdir(current)) { + const full = path.join(current, name) + const info = await stat(full).catch(() => undefined) + if (info?.isDirectory()) await visit(full) + else out.push(full) + } + } + await visit(dir) + return out.sort() +} + +describe("issue #204: harvest + distill over several runs incl. a broken chain", () => { + test("harvest is scrubbed, schema-valid; the broken chain is reported, never repaired", async () => { + const fix = await buildFixture() + try { + const brokenFile = factoryLayout(fix.broken).audit + const before = await readFile(brokenFile, "utf8") + const telemetry = await harvestTelemetry({ runs: fix.runs, evals: [fix.evals] }, { stateDir: fix.stateDir }) + expect(TelemetrySchema.parse(telemetry)).toEqual(telemetry) + expect(telemetry.runs.map((run) => run.id)).toEqual(["run-a", "run-b", "run-broken"]) + const text = JSON.stringify(telemetry) + for (const secret of [GH_A, GH_BROKEN, SK_ANT, AKIA, SLACK, BEARER, PEM_BODY, HUNTER]) + expect(text).not.toContain(secret) + // Shape is preserved: names stay, only values become [redacted]. + expect(text).toContain("GITHUB_TOKEN=") + expect(text).toContain("Bearer") + // The broken chain is reported with an error and left on disk untouched. + const brokenRun = telemetry.runs.find((run) => run.id === "run-broken")! + expect(brokenRun.auditChain.valid).toBe(false) + expect(typeof brokenRun.auditChain.error).toBe("string") + expect(await readFile(brokenFile, "utf8")).toBe(before) + // The sealed runs keep their valid chains and seals. + for (const id of ["run-a", "run-b"]) { + const run = telemetry.runs.find((entry) => entry.id === id)! + expect(run.auditChain.valid).toBe(true) + expect(run.seal).toEqual({ valid: true }) + } + // Direct harvestRun agrees: broken is reported, not repaired. + const direct = await harvestRun(fix.broken, { stateDir: fix.stateDir }) + expect(direct.auditChain.valid).toBe(false) + expect(await readFile(brokenFile, "utf8")).toBe(before) + } finally { + await fix.cleanup() + } + }) + + test("distill clusters all three kinds; fabricated evidence is dropped with reasons", async () => { + const fix = await buildFixture() + try { + const telemetry = await harvestTelemetry({ runs: fix.runs, evals: [fix.evals] }, { stateDir: fix.stateDir }) + const thresholds = { minOccurrences: 2, minRuns: 2 } + const clusters = clusterTelemetry(telemetry, thresholds) + expect(clusters.length).toBeGreaterThanOrEqual(3) + const proposals = await distillProposals(telemetry, thresholds) + expect(proposals.length).toBeGreaterThanOrEqual(3) + expect([...new Set(proposals.map((proposal) => proposal.kind))].sort()).toEqual([ + "domain-pack", + "gate-tuning", + "prompt-guidance", + ]) + expect(undisposed(proposals, telemetry)).toEqual(proposals) + // Fabricated evidence: a quote the harvester never said is disposed. + const [first] = proposals + const forged = { ...first!, cluster: { ...first!.cluster, evidence: ["run x: invented quote"] } } + const reasons = disposeReason(forged, telemetry) + expect(reasons.some((reason) => reason.includes("verbatim"))).toBe(true) + expect(undisposed([forged], telemetry)).toEqual([]) + const dropped = disposedWithReasons([forged], telemetry) + expect(dropped).toHaveLength(1) + expect(dropped[0]!.reasons.some((reason) => reason.includes("verbatim"))).toBe(true) + // Hallucinated model numerals are disposed too. + const tainted = { ...first!, rationale: `${first!.rationale} As gpt-4 would suggest.` } + expect(disposeReason(tainted, telemetry).some((reason) => reason.includes("hallucinated model numeral"))).toBe( + true, + ) + // Dropped proposals are reported with reasons, byte-identical on rerun. + const out = await mkdtemp(path.join(tmpdir(), "es-204-distill-")) + try { + const rendered = new Map( + await Promise.all( + proposals.map(async (proposal) => { + const { files } = await import("../src/improve/distill.ts").then((mod) => + mod.renderProposalFiles(proposal.cluster), + ) + return [proposal.id, files] as const + }), + ), + ) + const written = await writeProposals(out, proposals, rendered) + expect(written.length).toBe(proposals.length) + const droppedFile = await writeDropped(out, dropped) + expect(await readFile(droppedFile, "utf8")).toContain("verbatim") + await writeDropped(out, dropped) + expect(await readFile(droppedFile, "utf8")).toContain("verbatim") + // Gate-tuning explains only: gates.md, never the control file. + const gate = proposals.find((proposal) => proposal.kind === "gate-tuning")! + expect(gate.files).toEqual(["gates.md"]) + const files = await walkFiles(out) + expect(files.some((file) => path.basename(file) === "gates.json")).toBe(false) + } finally { + await rm(out, { recursive: true, force: true }) + } + } finally { + await fix.cleanup() + } + }) +}) From 018a6ba1b147f5e1759931ac2cb23a8420d3fe53 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 17:22:56 -0400 Subject: [PATCH 23/83] test(audit): self-dogfood preset integration tests (#205) --- packages/core/test/factory-preset-205.test.ts | 95 +++++++++++++++++++ packages/opencode/test/pr-preset-205.test.ts | 65 +++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 packages/core/test/factory-preset-205.test.ts create mode 100644 packages/opencode/test/pr-preset-205.test.ts diff --git a/packages/core/test/factory-preset-205.test.ts b/packages/core/test/factory-preset-205.test.ts new file mode 100644 index 0000000..32cf1f3 --- /dev/null +++ b/packages/core/test/factory-preset-205.test.ts @@ -0,0 +1,95 @@ +// Issue #205 integration (self-dogfood preset): seed a temp clone through +// library calls only and prove the gates stay untrusted until a human trusts +// them. Never runs a human-only verb (`es factory init`, `es trust`): the only +// human-only assertions are pipe-level policy checks via `invokesHumanOnly`, +// and no trust entry is ever written. +import { describe, expect, test } from "bun:test" +import { readFile } from "node:fs/promises" +import { + commandSetHash, + EsConfigSchema, + factoryLayout, + GatesConfigSchema, + initFromPreset, + invokesHumanOnly, + isTrusted, + RoadmapSchema, + runGates, + SELF_DOGFOOD_BASE_BRANCH, +} from "../src/index.ts" +import { gitRepo } from "./helpers.ts" + +describe("self-dogfood preset integration (#205)", () => { + test("init seeds config, gates, roadmap and the issue note in a temp clone", async () => { + const fx = await gitRepo("es-205-preset-") + try { + const { written, baseBranch } = await initFromPreset( + fx.root, + "self-dogfood", + { issue: 205, title: "Self-dogfood preset", body: "The factory builds this repo." }, + { actor: "human:tester" }, + ) + // Current behavior (#281 owns the base-selection question): the seed + // pins the release base to `rewrite`. + expect(baseBranch).toBe("rewrite") + expect(SELF_DOGFOOD_BASE_BRANCH).toBe("rewrite") + expect(written).toEqual([ + ".factory/config.json", + ".factory/gates.json", + ".factory/roadmap.json", + ".factory/notes/idea-205.md", + ]) + const layout = factoryLayout(fx.root) + const config = EsConfigSchema.parse(JSON.parse(await readFile(layout.config, "utf8"))) + expect(config.audit?.phase).toBe("required") + expect(config.pr).toBe("gh") + const gates = GatesConfigSchema.parse(JSON.parse(await readFile(layout.gates, "utf8"))) + expect(gates.commands.map((command) => command.command)).toEqual([ + "bun install --frozen-lockfile", + "bun run check", + "bun run docs:check", + ]) + const roadmap = RoadmapSchema.parse(JSON.parse(await readFile(layout.roadmap, "utf8"))) + expect(roadmap.baseBranch).toBe("rewrite") + expect(roadmap.title).toContain("205") + expect(await readFile(`${fx.root}/.factory/notes/idea-205.md`, "utf8")).toContain("# #205:") + } finally { + await fx.cleanup() + } + }) + + test("preset gates are untrusted until a human trusts them (no trust performed)", async () => { + const fx = await gitRepo("es-205-trust-") + try { + await initFromPreset( + fx.root, + "self-dogfood", + { issue: 205, title: "Self-dogfood preset", body: "The factory builds this repo." }, + { actor: "human:tester" }, + ) + const layout = factoryLayout(fx.root) + const gates = GatesConfigSchema.parse(JSON.parse(await readFile(layout.gates, "utf8"))) + // Trust-hash refusal: the fresh state dir holds no entry for this set. + // No `trustProject` call happens in this test (that stays human-only). + const { hash } = await commandSetHash(fx.root, gates.commands) + expect(await isTrusted(fx.root, hash, fx.state)).toBe(false) + + const report = await runGates({ root: fx.root, scope: "full", stateDir: fx.state }) + expect(report.passed).toBe(false) + expect(report.findings[0]?.rule).toBe("trust/untrusted") + expect(report.findings[0]?.fixHint).toContain("es trust") + expect(report.checks.find((check) => check.id === "trust")?.status).toBe("fail") + // None of the preset commands ran: no per-command checks were recorded. + for (const id of ["install", "check", "docs-check"]) + expect(report.checks.some((check) => check.id === id)).toBe(false) + } finally { + await fx.cleanup() + } + }) + + test("seeding and trusting stay human-only (policy assert only, nothing executed)", () => { + expect(invokesHumanOnly("es factory init --preset self-dogfood --issue 205")).toBe(true) + expect(invokesHumanOnly("es trust")).toBe(true) + expect(invokesHumanOnly("es factory begin")).toBe(false) + }) +}) diff --git a/packages/opencode/test/pr-preset-205.test.ts b/packages/opencode/test/pr-preset-205.test.ts new file mode 100644 index 0000000..9889890 --- /dev/null +++ b/packages/opencode/test/pr-preset-205.test.ts @@ -0,0 +1,65 @@ +// Issue #205 opener half: `ghPrOpener.open` with `preset: self-dogfood` and +// `--base main` refuses before anything is pushed. Local bare remote only, no +// network and no `gh` invocation: the shared argv builder throws first, so the +// test asserts the bare remote holds no new ref afterwards. +import { describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { run } from "@heretek-ai/es-core" +import { ghPrOpener } from "../src/pr.ts" + +const sh = async (args: string[], cwd: string) => { + const proc = Bun.spawn(["git", ...args], { cwd, stdout: "pipe", stderr: "pipe" }) + await proc.exited +} + +const showRefs = async (bare: string, cwd: string) => + (await run(["git", "--git-dir", bare, "show-ref"], { cwd })).stdout.trim() + +describe("self-dogfood opener refuses --base main without pushing (#205)", () => { + test("preset run against main is refused and the remote is unchanged", async () => { + const bare = await mkdtemp(path.join(tmpdir(), "es-205-bare-")) + const repo = await mkdtemp(path.join(tmpdir(), "es-205-repo-")) + try { + await sh(["init", "--bare", "-q"], bare) + await sh(["init", "-q", "-b", "rewrite"], repo) + await sh(["config", "user.email", "t@t"], repo) + await sh(["config", "user.name", "t"], repo) + await sh(["remote", "add", "origin", bare], repo) + await Bun.write(path.join(repo, "base.txt"), "base\n") + await sh(["add", "base.txt"], repo) + await sh(["commit", "-q", "-m", "init"], repo) + await sh(["push", "-q", "origin", "rewrite"], repo) + await sh(["checkout", "-qb", "factory/run-205/integration"], repo) + await Bun.write(path.join(repo, "work.txt"), "work\n") + await sh(["add", "work.txt"], repo) + await sh(["commit", "-q", "-m", "work"], repo) + + const before = await showRefs(bare, repo) + expect(before).toContain("refs/heads/rewrite") + expect(before).not.toContain("factory/run-205/integration") + + await expect( + ghPrOpener.open({ + root: repo, + branch: "factory/run-205/integration", + base: "main", + title: "dogfood", + body: "body", + preset: "self-dogfood", + }), + ).rejects.toThrow(/refusing.*main|rewrite/) + + // Refusal happened before the push: the bare remote is byte-identical. + const after = await showRefs(bare, repo) + expect(after).toBe(before) + expect(after).not.toContain("factory/run-205/integration") + const lsRemote = await run(["git", "ls-remote", "origin"], { cwd: repo }) + expect(lsRemote.stdout).not.toContain("factory/run-205/integration") + } finally { + await rm(repo, { recursive: true, force: true }) + await rm(bare, { recursive: true, force: true }) + } + }) +}) From 970e1732ae6186f62019880755d83088de7f98ae Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 17:23:14 -0400 Subject: [PATCH 24/83] test(audit): fleet DAG fake-driver integration tests (#206) --- packages/fleet/test/dag-fake-driver.test.ts | 306 ++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 packages/fleet/test/dag-fake-driver.test.ts diff --git a/packages/fleet/test/dag-fake-driver.test.ts b/packages/fleet/test/dag-fake-driver.test.ts new file mode 100644 index 0000000..f8fe527 --- /dev/null +++ b/packages/fleet/test/dag-fake-driver.test.ts @@ -0,0 +1,306 @@ +// Fleet two-task DAG via a FAKE driver (#206, audit-tester): ordering, +// concurrency/fleet ceilings, worktree isolation and the failure-policy +// retry path, all through the real FleetDaemon tick loop with stubbed worker +// supervision (daemon.test.ts fake-spawner pattern). No workers are spawned, +// no testkit host is booted, no `es-fleet` verb runs: `run()` exits at once +// without reporting (so the supervisor untracks, like a process exit) and +// reports are folded manually, so every transition is daemon-owned. +// +// The issue's literal ask (testkit host as the opencode driver with real +// workers) is NEEDS HUMAN / UNPROVABLE-HERE: it requires launching headless +// runs, which is human-only and spend-bearing, so this suite proves the same +// DAG properties with a fake driver instead. The real-host variant already +// exists as packages/fleet/test/integration.test.ts (cases 1-2). +import { describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { git as coreGit } from "@heretek-ai/es-core" +import { FleetDaemon, type WorkerPort } from "../src/daemon.ts" +import type { TaskReport } from "../src/scheduler.ts" +import { addTask, emptyDag, loadDag, saveDag } from "../src/tasks.ts" +import { buildWorkerArgv, type WorkerOutcome, type WorkerSpec } from "../src/worker.ts" +import { fleetBranch, fleetWorktreeDir, loadRegistry } from "../src/worktree.ts" + +const STAMP = "2026-10-09T12:00:00.000Z" + +const taskInput = (id: string, deps: string[] = [], extra: Record = {}) => ({ + id, + title: `task ${id}`, + kind: "factory-run" as const, + deps, + input: { objective: `do ${id}` }, + ceilingUSD: 5, + createdAt: STAMP, + updatedAt: STAMP, + ...extra, +}) + +async function fixture() { + const root = await mkdtemp(path.join(tmpdir(), "es-fleet-206-")) + const state = await mkdtemp(path.join(tmpdir(), "es-fleet-206-state-")) + const git = (args: string[], cwd = root) => coreGit(cwd, args) + await git(["init", "-b", "main"]) + await git(["config", "user.name", "fleet-test"]) + await git(["config", "user.email", "fleet@test.local"]) + await git(["config", "commit.gpgsign", "false"]) + await writeFile(path.join(root, "file.txt"), "base\n") + await git(["add", "-A"]) + await git(["commit", "--no-verify", "-m", "base"]) + const base = (await git(["rev-parse", "HEAD"])).stdout.trim() + return { + root, + state, + base, + cleanup: () => + Promise.all([rm(root, { recursive: true, force: true }), rm(state, { recursive: true, force: true })]).then( + () => undefined, + ), + } +} + +interface Fake { + readonly started: string[] + readonly specs: Map + readonly stopped: string[] + readonly spawner: (spec: WorkerSpec, onReport: (taskId: string, report: TaskReport) => void) => WorkerPort +} + +/** Stubbed supervision: records launches, exits without reporting (so the + * supervisor untracks, exactly like a worker process exiting), never spawns. + * DAG transitions stay manual via foldReport, keeping every assertion + * daemon-owned and spend-free. */ +const fakeSpawner = (outcomes: Record = {}): Fake => { + const started: string[] = [] + const specs = new Map() + const stopped: string[] = [] + return { + started, + specs, + stopped, + spawner: (spec) => { + started.push(spec.taskId) + specs.set(spec.taskId, spec) + const status = outcomes[spec.taskId] ?? "done" + return { + run: () => Promise.resolve({ status, spendUsd: 0 }), + stop: () => { + stopped.push(spec.taskId) + return Promise.resolve() + }, + snapshot: () => ({ taskId: spec.taskId, pid: -1, spendUsd: 0, tokensIn: 0, tokensOut: 0 }), + } + }, + } +} + +const gatesOk = () => Promise.resolve({ ok: true as const }) + +const daemonFor = ( + root: string, + state: string, + fake: Fake, + extra: { concurrency?: number; fleetCeilingUsd?: number } = {}, +): FleetDaemon => + new FleetDaemon({ + repoRoot: root, + stateRoot: state, + esBin: "/stub/es.js", + concurrency: extra.concurrency, + fleetCeilingUsd: extra.fleetCeilingUsd ?? 100, + checkGates: gatesOk, + spawnWorker: fake.spawner, + }) + +describe("two-task DAG ordering, isolation and landing (fake driver, #206)", () => { + test("b starts only after a is done; each task lands from its own worktree; base stable", async () => { + const { root, state, base, cleanup } = await fixture() + try { + let dag = addTask(emptyDag(), taskInput("a206-a"), STAMP) + dag = addTask(dag, taskInput("a206-b", ["a206-a"]), STAMP) + await saveDag(state, dag) + const fake = fakeSpawner() + const daemon = daemonFor(root, state, fake) + + await daemon.tick() + // Ordering: only the dep-free task launches; the dependent stays pending. + expect(fake.started).toEqual(["a206-a"]) + expect((await loadDag(state))?.tasks["a206-a"]?.status).toBe("running") + expect((await loadDag(state))?.tasks["a206-b"]?.status).toBe("pending") + // Stubbed supervision: a second tick never double-starts the live task. + await daemon.tick() + expect(fake.started).toEqual(["a206-a"]) + + // The worker reports done; the next tick lands a and starts b. + await daemon.foldReport("a206-a", { outcome: "done", spendUsd: 1 }) + await daemon.tick() + expect((await loadDag(state))?.tasks["a206-a"]?.status).toBe("done") + expect((await loadDag(state))?.tasks["a206-b"]?.status).toBe("running") + expect(fake.started).toEqual(["a206-a", "a206-b"]) + + // Worktree isolation: distinct dirs, distinct branches, distinct records. + const dirA = fake.specs.get("a206-a")!.worktreeDir + const dirB = fake.specs.get("a206-b")!.worktreeDir + expect(dirA).toBe(fleetWorktreeDir(root, "a206-a")) + expect(dirB).toBe(fleetWorktreeDir(root, "a206-b")) + expect(dirA).not.toBe(dirB) + expect(fleetBranch("a206-a")).not.toBe(fleetBranch("a206-b")) + const registry = (await loadRegistry(state))! + expect(registry.worktrees["a206-a"]?.dir).toBe(dirA) + expect(registry.worktrees["a206-b"]?.dir).toBe(dirB) + expect(registry.worktrees["a206-a"]?.branch).toBe("fleet/a206-a") + expect(registry.worktrees["a206-b"]?.branch).toBe("fleet/a206-b") + + await daemon.foldReport("a206-b", { outcome: "done", spendUsd: 1 }) + await daemon.tick() + expect((await loadDag(state))?.tasks["a206-b"]?.status).toBe("done") + expect((await loadRegistry(state))?.worktrees["a206-a"]?.status).toBe("merged") + expect((await loadRegistry(state))?.worktrees["a206-b"]?.status).toBe("merged") + // Both tasks landed on the integration branch; the base never moves. + const integration = (await coreGit(root, ["rev-parse", "fleet/integration/fleet"])).stdout.trim() + expect(integration.length).toBe(40) + expect((await coreGit(root, ["rev-parse", "main"])).stdout.trim()).toBe(base) + } finally { + await cleanup() + } + }) + + test("parallel tasks hold simultaneous isolated allocations", async () => { + const { root, state, cleanup } = await fixture() + try { + let dag = addTask(emptyDag(), taskInput("a206-p"), STAMP) + dag = addTask(dag, taskInput("a206-q"), STAMP) + await saveDag(state, dag) + const fake = fakeSpawner() + const daemon = daemonFor(root, state, fake, { concurrency: 2 }) + await daemon.tick() + // Both launch together and both hold an allocation at once. + expect(fake.started).toEqual(["a206-p", "a206-q"]) + const registry = (await loadRegistry(state))! + expect(registry.worktrees["a206-p"]?.status).toBe("allocated") + expect(registry.worktrees["a206-q"]?.status).toBe("allocated") + expect(registry.worktrees["a206-p"]?.dir).not.toBe(registry.worktrees["a206-q"]?.dir) + expect(registry.worktrees["a206-p"]?.branch).not.toBe(registry.worktrees["a206-q"]?.branch) + } finally { + await cleanup() + } + }) +}) + +describe("ceilings through the daemon (fake driver, #206)", () => { + test("concurrency 1 serializes two independent tasks in id order", async () => { + const { root, state, cleanup } = await fixture() + try { + let dag = addTask(emptyDag(), taskInput("a206-c1"), STAMP) + dag = addTask(dag, taskInput("a206-c2"), STAMP) + await saveDag(state, dag) + const fake = fakeSpawner() + const daemon = daemonFor(root, state, fake, { concurrency: 1 }) + await daemon.tick() + expect(fake.started).toEqual(["a206-c1"]) + await daemon.tick() + expect(fake.started).toEqual(["a206-c1"]) + await daemon.foldReport("a206-c1", { outcome: "done", spendUsd: 1 }) + await daemon.tick() + expect(fake.started).toEqual(["a206-c1", "a206-c2"]) + expect((await loadDag(state))?.tasks["a206-c2"]?.status).toBe("running") + } finally { + await cleanup() + } + }) + + test("the fleet ceiling admits one 60-USD task of two under a 100-USD ceiling", async () => { + const { root, state, cleanup } = await fixture() + try { + let dag = addTask(emptyDag(), taskInput("a206-d1", [], { ceilingUSD: 60 }), STAMP) + dag = addTask(dag, taskInput("a206-d2", [], { ceilingUSD: 60 }), STAMP) + await saveDag(state, dag) + const fake = fakeSpawner() + const daemon = daemonFor(root, state, fake, { fleetCeilingUsd: 100 }) + await daemon.tick() + expect(fake.started).toEqual(["a206-d1"]) + expect((await loadDag(state))?.tasks["a206-d2"]?.status).not.toBe("running") + await daemon.foldReport("a206-d1", { outcome: "done", spendUsd: 1 }) + await daemon.tick() + // d1's ceiling frees on completion, so d2 launches next. + expect(fake.started).toEqual(["a206-d1", "a206-d2"]) + } finally { + await cleanup() + } + }) +}) + +describe("failure-policy retry path through the daemon (fake driver, #206)", () => { + test("retry(1) re-queues once, then fails and cancels the dependent with a reason", async () => { + const { root, state, cleanup } = await fixture() + try { + let dag = addTask(emptyDag(), taskInput("a206-f", [], { policy: { strategy: "retry", maxAttempts: 1 } }), STAMP) + dag = addTask(dag, taskInput("a206-g", ["a206-f"]), STAMP) + await saveDag(state, dag) + const fake = fakeSpawner({ "a206-f": "failed" }) + const daemon = daemonFor(root, state, fake) + + await daemon.tick() + expect(fake.started).toEqual(["a206-f"]) + expect((await loadDag(state))?.tasks["a206-f"]?.attempts).toBe(1) + + // First failure re-queues: pending with attempts intact, dependent held. + await daemon.foldReport("a206-f", { outcome: "failed", reason: "flake" }) + expect((await loadDag(state))?.tasks["a206-f"]?.status).toBe("pending") + expect((await loadDag(state))?.tasks["a206-f"]?.attempts).toBe(1) + + // The retry relaunches the same task (stubbed supervision tracks it anew). + await daemon.tick() + expect(fake.started).toEqual(["a206-f", "a206-f"]) + expect((await loadDag(state))?.tasks["a206-f"]?.attempts).toBe(2) + + // Second failure is terminal; the dependent cancels naming the failed dep. + await daemon.foldReport("a206-f", { outcome: "failed", reason: "still broken" }) + expect((await loadDag(state))?.tasks["a206-f"]?.status).toBe("failed") + await daemon.tick() + const final = (await loadDag(state))! + expect(final.tasks["a206-g"]?.status).toBe("cancelled") + expect(final.tasks["a206-g"]?.reason).toMatch(/a206-f/) + expect(fake.started).not.toContain("a206-g") + } finally { + await cleanup() + } + }) +}) + +describe("fake driver argv without spawning (fake driver, #206)", () => { + test("each task's headless argv is argv-only, worktree-scoped, ceiling-free, and nothing spawns", async () => { + const { root, state, cleanup } = await fixture() + try { + await saveDag(state, addTask(emptyDag(), taskInput("a206-a"), STAMP)) + const fake = fakeSpawner() + void daemonFor(root, state, fake) + for (const id of ["a206-a", "a206-b"]) { + const argv = buildWorkerArgv({ + taskId: id, + worktreeDir: fleetWorktreeDir(root, id), + ceilingUSD: 5, + kind: "factory-run", + objective: `do ${id}`, + esBin: "/stub/es.js", + }) + expect(argv[0]).toBe("/stub/es.js") + expect(argv).toContain("factory") + expect(argv).toContain("run") + expect(argv).toContain("--headless") + expect(argv).toContain("--events") + expect(argv).toContain("jsonl") + expect(argv).toContain("--cwd") + expect(argv[argv.indexOf("--cwd") + 1]).toBe(fleetWorktreeDir(root, id)) + // No --max-usd: the ceiling is provisioned into run state, never a flag. + expect(argv).not.toContain("--max-usd") + // argv-only: one string array, never joined through a shell. + expect(argv.every((word) => typeof word === "string")).toBe(true) + } + // Proof nothing launched: the stubbed supervisor recorded zero starts. + expect(fake.started).toEqual([]) + } finally { + await cleanup() + } + }) +}) From fb41f9e8290ce068ef77bc7873086556bcc92057 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 17:23:31 -0400 Subject: [PATCH 25/83] test(audit): web control plane integration tests (#207) --- packages/fleet/test/web-integration.test.ts | 255 +++++++++++++++++++ packages/fleet/test/web-json-headers.test.ts | 208 +++++++++++++++ packages/fleet/test/web-tickets-ttl.test.ts | 197 ++++++++++++++ packages/web/test/evidence.contract.test.ts | 182 +++++++++++++ 4 files changed, 842 insertions(+) create mode 100644 packages/fleet/test/web-integration.test.ts create mode 100644 packages/fleet/test/web-json-headers.test.ts create mode 100644 packages/fleet/test/web-tickets-ttl.test.ts create mode 100644 packages/web/test/evidence.contract.test.ts diff --git a/packages/fleet/test/web-integration.test.ts b/packages/fleet/test/web-integration.test.ts new file mode 100644 index 0000000..15bd05d --- /dev/null +++ b/packages/fleet/test/web-integration.test.ts @@ -0,0 +1,255 @@ +// Web control plane integration (#207 §2): fleet web served in-process +// against a run produced on the real testkit host (scripted fake LLM). +// The host writes a marker via @@CALL (real host tools, fake model); the +// sealed evidence (SourceCache + dossier) lives in that host directory and +// is served as run `run1` behind the single-use ticket exchange. Fetches go +// over the web session (cookie + loopback Origin) for status/task/evidence, +// asserting the served evidence matches the sealed sources (sha, seal, +// quote ranges). +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { createHash } from "node:crypto" +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { createConnection } from "node:net" +import { tmpdir } from "node:os" +import path from "node:path" +import { evidenceHash, normalizeClaim, normalizeForQuote, SourceCache, writeJson } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { fleetPaths } from "../src/state.ts" +import { ensureToken, type FleetSnapshot, TelemetryServer } from "../src/telemetry.ts" +import { mintWebTicket } from "../src/web.ts" + +const SOURCE_TEXT = + "Bun is a fast all-in-one JavaScript runtime. It ships a test runner, a bundler, and a package manager — all built in. Researchers measured startup at twelve milliseconds on commodity hardware." +const QUOTE = "Researchers measured startup at twelve milliseconds on commodity hardware" +const STATEMENT = "Bun starts in about twelve milliseconds." + +const raw = (port: number, head: string): Promise<{ status: number; headers: Record; body: string }> => + new Promise((resolve, reject) => { + const socket = createConnection(port, "127.0.0.1", () => socket.write(head)) + let buffer = Buffer.alloc(0) + socket.on("data", (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]) + }) + socket.on("close", () => { + const text = buffer.toString("latin1") + const at = text.indexOf("\r\n\r\n") + const header = text.slice(0, at) + const [statusLine, ...lines] = header.split("\r\n") + const headers: Record = {} + for (const line of lines) { + const colon = line.indexOf(":") + if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim() + } + resolve({ status: Number(statusLine!.split(" ")[1]), headers, body: text.slice(at + 4) }) + }) + socket.on("error", reject) + }) + +const get = (port: number, target: string, headers: Record = {}) => + raw( + port, + `GET ${target} HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\nConnection: close\r\n${Object.entries(headers) + .map(([k, v]) => `${k}: ${v}\r\n`) + .join("")}\r\n`, + ) + +describe("web integration on the testkit host (#207 §2)", () => { + let h: Harness + let state = "" + let server: TelemetryServer | undefined + let cookie = "" + let origin = "" + let sealedSha = "" + let sealedText = "" + let claimId = "" + let hostTools: Array<{ name: string; status: string }> = [] + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-web-int-state-")) + h = await boot({ + git: true, + script: directiveScript, + files: { "README.md": "# web integration\n" }, + }) + // The run is produced on the host: a scripted seat turn through @@CALL + // (real host tools, fake model) writes into the host directory. + const result = await h.run( + `seed web run @@CALL write {"path": "web-note.txt", "content": "web integration marker\\n"}@@`, + ) + hostTools = result.tools.map((tool) => ({ name: tool.name, status: tool.status })) + + const webRoot = path.join(state, "site") + await mkdir(webRoot, { recursive: true }) + await writeFile(path.join(webRoot, "index.html"), "swarm") + await writeFile(path.join(webRoot, "app.js"), "console.log(1)") + await mkdir(fleetPaths(state).dir, { recursive: true }) + const token = await ensureToken(state) + + // Run `run1` is the host directory itself. + await writeFile( + fleetPaths(state).worktrees, + JSON.stringify({ + v: 1, + worktrees: { + run1: { + taskId: "run1", + dir: h.directory, + branch: "fleet/run1", + base: "0".repeat(40), + createdAt: new Date().toISOString(), + status: "allocated", + }, + }, + }), + ) + + // Sealed evidence in the host directory (engine key lives in the fleet + // state dir, so the served seal verifies as `sealed`). + const sources = path.join(h.directory, ".factory", "research", "sources") + await mkdir(sources, { recursive: true }) + const sealed = new SourceCache(sources, state) + const entry = await sealed.put({ url: "https://example.com/web-int", text: SOURCE_TEXT, provider: "fetch" }) + sealedSha = entry.meta.sha256 + sealedText = entry.text + const claim = normalizeClaim({ + tag: "VERIFIED", + statement: STATEMENT, + source: { sha256: entry.meta.sha256, quote: QUOTE }, + }) + claimId = claim.id + await writeJson(path.join(h.directory, ".factory", "research", "dossier.json"), { + version: "1.1", + mode: "research", + subject: "web-int", + createdAt: new Date().toISOString(), + claims: [claim], + evidenceHash: evidenceHash([claim]), + }) + await writeFile(path.join(h.directory, ".factory", "research", "REPORT.md"), "# Web int\n\nBody.\n") + + const snapshot = (): FleetSnapshot => ({ + daemon: { running: true, pid: 1, maxUsd: 10, concurrency: 1 }, + tasks: [ + { + id: "run1", + title: "Web int", + status: "running", + ceilingUSD: 5, + spendUsd: 0, + deps: [], + reason: "testing", + }, + ], + spendUsd: 0, + pending: [], + }) + server = new TelemetryServer({ stateRoot: state, port: 0, token, getSnapshot: snapshot, webRoot }) + await server.start() + origin = `http://127.0.0.1:${server.port}` + const ticket = await mintWebTicket(state) + const login = await get(server.port, `/?t=${ticket}`) + if (login.status !== 200) throw new Error(`ticket exchange failed: ${login.status}`) + cookie = (login.headers["set-cookie"] ?? "").split(";")[0]! + if (!cookie) throw new Error("ticket exchange set no cookie") + }, 120_000) + + afterAll(async () => { + await server?.stop().catch(() => undefined) + await h?.close() + if (state) await rm(state, { recursive: true, force: true }) + }) + + const rpc = (method: string, params: unknown): Promise<{ result?: any; error?: { code: number; message: string } }> => + fetch(`http://127.0.0.1:${server!.port}/fleet/rpc`, { + method: "POST", + headers: { + "content-type": "application/json", + cookie, + origin, + }, + body: JSON.stringify({ method, params }), + }).then((response) => response.json()) as Promise<{ + result?: any + error?: { code: number; message: string } + }> + + test("the host produced the run directory via the scripted fake LLM", async () => { + expect(hostTools).toEqual([{ name: "write", status: "completed" }]) + expect(await readFile(path.join(h.directory, "web-note.txt"), "utf8")).toContain("web integration marker") + }) + + test("fleet.status and fleet.task round-trip over the web session", async () => { + const status = await rpc("fleet.status", {}) + expect(status.error).toBeUndefined() + expect(status.result.daemon.running).toBe(true) + expect(status.result.tasks.map((task: { id: string }) => task.id)).toEqual(["run1"]) + const task = await rpc("fleet.task", { id: "run1" }) + expect(task.error).toBeUndefined() + expect(task.result.task.id).toBe("run1") + expect(task.result.task.title).toBe("Web int") + }) + + test("evidence.claims lists the sealed claim with its source sha", async () => { + const listed = await rpc("evidence.claims", { runId: "run1", filter: { tag: "VERIFIED" } }) + expect(listed.error).toBeUndefined() + expect(listed.result.claims).toHaveLength(1) + const [entry] = listed.result.claims + expect(entry.id).toBe(claimId) + expect(entry.tag).toBe("VERIFIED") + expect(entry.status).toBe("LIVE") + expect(entry.statement).toContain("twelve milliseconds") + expect(entry.sourceSha).toBe(sealedSha) + }) + + test("evidence.claim returns verified ranges that slice back to the quote", async () => { + const detail = await rpc("evidence.claim", { runId: "run1", id: claimId }) + expect(detail.error).toBeUndefined() + expect(detail.result.quotes).toHaveLength(1) + const [quote] = detail.result.quotes + expect(quote.quote).toBe(QUOTE) + expect(quote.verified).toBe(true) + expect(quote.ranges.length).toBeGreaterThan(0) + // Every range points into the sealed source text on disk. + const onDisk = await readFile(path.join(h.directory, ".factory", "research", "sources", `${sealedSha}.md`), "utf8") + expect(onDisk).toBe(sealedText) + for (const range of quote.ranges) { + expect(range.start).toBeGreaterThanOrEqual(0) + expect(range.end).toBeLessThanOrEqual(sealedText.length) + expect(range.end).toBeGreaterThan(range.start) + } + const covered = normalizeForQuote(sealedText.slice(quote.ranges[0].start, quote.ranges.at(-1).end)).replace( + /\s+/g, + "", + ) + expect(covered).toBe(normalizeForQuote(QUOTE).replace(/\s+/g, "")) + expect(detail.result.rankExplanation.length).toBeGreaterThan(0) + }) + + test("evidence.source matches the sealed bytes (sha + seal status)", async () => { + const source = await rpc("evidence.source", { runId: "run1", sha: sealedSha }) + expect(source.error).toBeUndefined() + expect(source.result.meta.sha256).toBe(sealedSha) + expect(source.result.meta.url).toBe("https://example.com/web-int") + expect(source.result.seal).toBe("sealed") + expect(source.result.text).toBe(sealedText) + expect(source.result.truncated).toBe(false) + // Content-addressed: the served bytes hash to their own name. + expect(createHash("sha256").update(source.result.text).digest("hex")).toBe(sealedSha) + expect(source.result.text).toContain(QUOTE) + }) + + test("evidence.export renders the dossier through the web session", async () => { + const exported = await rpc("evidence.export", { runId: "run1", format: "markdown" }) + expect(exported.error).toBeUndefined() + expect(exported.result.format).toBe("markdown") + expect(exported.result.body).toContain("twelve milliseconds") + }) + + test("the session serves the UI with the strict headers", async () => { + const home = await get(server!.port, "/", { Cookie: cookie }) + expect(home.status).toBe(200) + expect(home.body).toContain("swarm") + expect(home.headers["content-security-policy"] ?? "").toContain("frame-ancestors 'none'") + expect(home.headers["x-content-type-options"]).toBe("nosniff") + }) +}) diff --git a/packages/fleet/test/web-json-headers.test.ts b/packages/fleet/test/web-json-headers.test.ts new file mode 100644 index 0000000..9fd4d0a --- /dev/null +++ b/packages/fleet/test/web-json-headers.test.ts @@ -0,0 +1,208 @@ +// JSON responders omit WEB_SECURITY_HEADERS (#282, follow-up of #207). +// Web-file and deny paths (:508, :564) spread the headers; the JSON +// responders (`jsonBody` :286, `ok` :732, `fail` :737, plus the bare +// 401/403/404 paths) do not. This locks the CURRENT behavior so the gap +// stays visible: every assertion below documents the omission. When #282 +// is fixed (spread WEB_SECURITY_HEADERS in the JSON responders), update +// these assertions to expect the headers. +import { describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { createConnection } from "node:net" +import { tmpdir } from "node:os" +import path from "node:path" +import { fleetPaths } from "../src/state.ts" +import { ensureToken, type FleetSnapshot, TelemetryServer } from "../src/telemetry.ts" +import { mintWebTicket, WEB_SECURITY_HEADERS } from "../src/web.ts" + +const HEADER_NAMES = Object.keys(WEB_SECURITY_HEADERS).map((name) => name.toLowerCase()) + +const snap = (): FleetSnapshot => ({ + daemon: { running: true, pid: 4242, maxUsd: 50, concurrency: 2 }, + tasks: [], + spendUsd: 0, + pending: [], +}) + +async function rig() { + const state = await mkdtemp(path.join(tmpdir(), "es-web-json-")) + const webRoot = path.join(state, "site") + await mkdir(webRoot, { recursive: true }) + await writeFile(path.join(webRoot, "index.html"), "swarm") + await mkdir(fleetPaths(state).dir, { recursive: true }) + const token = await ensureToken(state) + const server = new TelemetryServer({ stateRoot: state, port: 0, token, getSnapshot: snap, webRoot }) + await server.start() + return { state, server, token } +} + +const raw = ( + port: number, + head: string, + body = "", +): Promise<{ status: number; headers: Record; body: string }> => + new Promise((resolve, reject) => { + const socket = createConnection(port, "127.0.0.1", () => socket.write(head + body)) + let buffer = Buffer.alloc(0) + socket.on("data", (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]) + }) + socket.on("close", () => { + const text = buffer.toString("latin1") + const at = text.indexOf("\r\n\r\n") + const [statusLine, ...lines] = text.slice(0, at).split("\r\n") + const headers: Record = {} + for (const line of lines) { + const colon = line.indexOf(":") + if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim() + } + resolve({ status: Number(statusLine!.split(" ")[1]), headers, body: text.slice(at + 4) }) + }) + socket.on("error", reject) + }) + +const postRpc = (port: number, payload: unknown, extra: Record = {}) => { + const body = JSON.stringify(payload) + return raw( + port, + `POST /fleet/rpc HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\nContent-Type: application/json\r\nContent-Length: ${Buffer.byteLength(body)}\r\nConnection: close\r\n${Object.entries( + extra, + ) + .map(([k, v]) => `${k}: ${v}\r\n`) + .join("")}\r\n`, + body, + ) +} + +const postApprove = (port: number, target: string, payload: unknown, extra: Record = {}) => { + const body = JSON.stringify(payload) + return raw( + port, + `POST ${target} HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\nContent-Type: application/json\r\nContent-Length: ${Buffer.byteLength(body)}\r\nConnection: close\r\n${Object.entries( + extra, + ) + .map(([k, v]) => `${k}: ${v}\r\n`) + .join("")}\r\n`, + body, + ) +} + +const expectNoSecurityHeaders = (headers: Record, where: string): void => { + for (const name of HEADER_NAMES) expect(headers[name], `${where}: ${name}`).toBeUndefined() +} + +describe("web JSON responders omit the strict headers (#282)", () => { + test("successful JSON-RPC carries no security headers (current behavior)", async () => { + const { state, server, token } = await rig() + try { + const res = await postRpc( + server.port, + { method: "fleet.status", params: {} }, + { Authorization: `Bearer ${token}` }, + ) + expect(res.status).toBe(200) + expect(res.headers["content-type"]).toContain("application/json") + expectNoSecurityHeaders(res.headers, "fleet.status 200") + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + } + }) + + test("JSON-RPC error envelopes carry no security headers (current behavior)", async () => { + const { state, server, token } = await rig() + try { + const auth = { Authorization: `Bearer ${token}` } + // Unknown method and invalid params both answer 200 + error envelope via jsonBody. + const unknown = await postRpc(server.port, { method: "nope", params: {} }, auth) + expect(unknown.status).toBe(200) + expect(unknown.body).toContain("-32601") + expectNoSecurityHeaders(unknown.headers, "unknown method") + const badParams = await postRpc(server.port, { method: "fleet.status", params: { extra: 1 } }, auth) + expect(badParams.status).toBe(200) + expect(badParams.body).toContain("-32602") + expectNoSecurityHeaders(badParams.headers, "invalid params") + // Unknown run answers 200 + -32603 via the fail path in dispatch. + const unknownRun = await postRpc(server.port, { method: "evidence.claims", params: { runId: "nope" } }, auth) + expect(unknownRun.status).toBe(200) + expect(unknownRun.body).toContain("-32603") + expectNoSecurityHeaders(unknownRun.headers, "unknown run") + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + } + }) + + test("bare 401/403/404 and approve JSON failures carry no security headers (current behavior)", async () => { + const { state, server } = await rig() + try { + // No credentials at all: 401 with no JSON body and no security headers. + const anon = await postRpc(server.port, { method: "fleet.status", params: {} }) + expect(anon.status).toBe(401) + expectNoSecurityHeaders(anon.headers, "rpc 401") + // Approve preview without credentials: 401, also bare. + const previewAnon = await postApprove(server.port, "/fleet/approve/preview", { runId: "t1", stage: "frontier" }) + expect(previewAnon.status).toBe(401) + expectNoSecurityHeaders(previewAnon.headers, "approve preview 401") + // Approve submit without a session: 401, bare. + const submitAnon = await postApprove(server.port, "/fleet/approve", { ticket: "x", passphrase: "y" }) + expect(submitAnon.status).toBe(401) + expectNoSecurityHeaders(submitAnon.headers, "approve submit 401") + // Approve submit with a session but no Origin/CSRF: 403, bare. + const ticket = await mintWebTicket(state) + const login = await raw( + server.port, + `GET /?t=${ticket} HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) + const cookie = (login.headers["set-cookie"] ?? "").split(";")[0]! + const noOrigin = await postApprove( + server.port, + "/fleet/approve", + { ticket: "x", passphrase: "y" }, + { Cookie: cookie, "X-CSRF": "wrong" }, + ) + expect(noOrigin.status).toBe(403) + expectNoSecurityHeaders(noOrigin.headers, "approve submit 403") + // Malformed approve body answers 400 JSON via fail(): no headers. + const malformed = await postApprove( + server.port, + "/fleet/approve/preview", + { runId: "t1" }, + { Cookie: cookie, Origin: `http://127.0.0.1:${server.port}` }, + ) + expect(malformed.status).toBe(400) + expect(malformed.body).toContain("want {runId") + expectNoSecurityHeaders(malformed.headers, "approve preview 400") + // Unknown POST target: 404, bare. + const missing = await postApprove(server.port, "/fleet/nope", {}) + expect(missing.status).toBe(404) + expectNoSecurityHeaders(missing.headers, "unknown target 404") + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + } + }) + + test("contrast: web-file responses do carry the headers", async () => { + const { state, server } = await rig() + try { + const ticket = await mintWebTicket(state) + const login = await raw( + server.port, + `GET /?t=${ticket} HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) + expect(login.status).toBe(200) + expect(login.headers["content-security-policy"] ?? "").toContain("frame-ancestors 'none'") + expect(login.headers["x-content-type-options"]).toBe("nosniff") + // Deny paths carry them too. + const anon = await raw( + server.port, + `GET / HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) + expect(anon.status).toBe(401) + expect(anon.headers["content-security-policy"] ?? "").toContain("frame-ancestors 'none'") + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/fleet/test/web-tickets-ttl.test.ts b/packages/fleet/test/web-tickets-ttl.test.ts new file mode 100644 index 0000000..a8ccd0f --- /dev/null +++ b/packages/fleet/test/web-tickets-ttl.test.ts @@ -0,0 +1,197 @@ +// Expired-ticket fixtures (#283, follow-up of #207): the TTL branches for +// both ticket kinds. Web tickets live as files holding an absolute expiry +// (`web.ts:87` returns Number(raw) > Date.now()); approve tickets live in +// memory with the same comparison (`approve.ts:34`). The existing suites +// cover malformed/missing web tickets and reuse-after-fail approve tickets +// only — these fixtures mint, then expire, then assert refusal at both the +// unit level and the HTTP level. +import { describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { createConnection } from "node:net" +import { tmpdir } from "node:os" +import path from "node:path" +import { ApproveTickets } from "../src/approve.ts" +import { fleetDir, fleetPaths } from "../src/state.ts" +import { type FleetSnapshot, TelemetryServer } from "../src/telemetry.ts" +import { mintWebTicket, redeemWebTicket } from "../src/web.ts" + +const raw = ( + port: number, + head: string, + body = "", +): Promise<{ status: number; headers: Record; body: string }> => + new Promise((resolve, reject) => { + const socket = createConnection(port, "127.0.0.1", () => socket.write(head + body)) + let buffer = Buffer.alloc(0) + socket.on("data", (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]) + }) + socket.on("close", () => { + const text = buffer.toString("latin1") + const at = text.indexOf("\r\n\r\n") + const [statusLine, ...lines] = text.slice(0, at).split("\r\n") + const headers: Record = {} + for (const line of lines) { + const colon = line.indexOf(":") + if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim() + } + resolve({ status: Number(statusLine!.split(" ")[1]), headers, body: text.slice(at + 4) }) + }) + socket.on("error", reject) + }) + +const expireWebTicket = async (state: string, ticket: string): Promise => { + const file = path.join(fleetDir(state), "web-tickets", ticket) + await writeFile(file, `${Date.now() - 1_000}\n`, { mode: 0o600 }) +} + +describe("web tickets: minted-then-expired fixtures (#283)", () => { + test("an expired ticket file redeems to false and is consumed", async () => { + const state = await mkdtemp(path.join(tmpdir(), "es-web-ttl-")) + try { + const ticket = await mintWebTicket(state) + await expireWebTicket(state, ticket) + expect(await redeemWebTicket(state, ticket)).toBe(false) + // Single-use claim consumed the file: a second redeem is still false. + expect(await redeemWebTicket(state, ticket)).toBe(false) + } finally { + await rm(state, { recursive: true, force: true }) + } + }) + + test("an expired ticket is refused at the HTTP exchange without a cookie", async () => { + const state = await mkdtemp(path.join(tmpdir(), "es-web-ttl-")) + const webRoot = path.join(state, "site") + await mkdir(webRoot, { recursive: true }) + await writeFile(path.join(webRoot, "index.html"), "swarm") + await mkdir(fleetPaths(state).dir, { recursive: true }) + const server = new TelemetryServer({ + stateRoot: state, + port: 0, + token: "x".repeat(40), + getSnapshot: (): FleetSnapshot => ({ + daemon: { running: true, pid: 1, maxUsd: 1, concurrency: 1 }, + tasks: [], + spendUsd: 0, + pending: [], + }), + webRoot, + }) + await server.start() + try { + const ticket = await mintWebTicket(state) + await expireWebTicket(state, ticket) + const res = await raw( + server.port, + `GET /?t=${ticket} HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) + expect(res.status).toBe(403) + expect(res.headers["set-cookie"]).toBeUndefined() + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + } + }) +}) + +describe("approve tickets: minted-then-expired fixtures (#283)", () => { + test("an approve ticket past its TTL redeems to undefined (unit level)", async () => { + const tickets = new ApproveTickets(20) + const ticket = tickets.issue({ repoDir: "/tmp/repo", stage: "frontier", subjectHash: "ab".repeat(32) }) + expect(tickets.redeem(ticket)?.stage).toBe("frontier") + const expiring = tickets.issue({ repoDir: "/tmp/repo", stage: "frontier", subjectHash: "ab".repeat(32) }) + await Bun.sleep(50) + expect(tickets.redeem(expiring)).toBeUndefined() + // Consumed whether valid or not: a second redeem is still undefined. + expect(tickets.redeem(expiring)).toBeUndefined() + }) + + test("an expired approve ticket is refused at the submit endpoint (TTL branch)", async () => { + const state = await mkdtemp(path.join(tmpdir(), "es-approve-ttl-state-")) + const repo = await mkdtemp(path.join(tmpdir(), "es-approve-ttl-repo-")) + await mkdir(path.join(repo, ".factory"), { recursive: true }) + await writeFile( + path.join(repo, ".factory", "frontier.json"), + `${JSON.stringify({ version: "1.1", idea: "TTL probe", spendCeiling: { currency: "USD", maxAmount: 5 }, settled: true, nodes: [] }, null, 2)}\n`, + ) + const { writeJson, factoryLayout, sealHumanKey } = await import("@heretek-ai/es-core") + await writeJson(factoryLayout(repo).pending, [ + { stage: "frontier", requestedBy: "factory", at: new Date().toISOString() }, + ]) + const PASS = "ttl-probe-passphrase-for-expired-ticket-test" + await sealHumanKey(PASS, state) + await mkdir(fleetPaths(state).dir, { recursive: true }) + await writeFile(fleetPaths(state).token, "approve-ttl-token-with-enough-length", { mode: 0o600 }) + await writeJson(fleetPaths(state).worktrees, { + v: 1, + worktrees: { + t1: { + taskId: "t1", + dir: repo, + branch: "fleet/t1", + base: "0".repeat(40), + createdAt: new Date().toISOString(), + status: "allocated", + }, + }, + }) + const site = path.join(state, "site") + await mkdir(site, { recursive: true }) + await writeFile(path.join(site, "index.html"), "") + // A 30 ms preview-ticket TTL: the preview below expires before submit. + const server = new TelemetryServer({ + stateRoot: state, + port: 0, + token: "approve-ttl-token-with-enough-length", + getSnapshot: (): FleetSnapshot => ({ + daemon: { running: true, pid: 1, maxUsd: 10, concurrency: 1 }, + tasks: [], + spendUsd: 0, + pending: [], + }), + webRoot: site, + approveTicketTtlMs: 30, + }) + await server.start() + try { + const { mintWebTicket } = await import("../src/web.ts") + const ticket = await mintWebTicket(state) + const login = await raw( + server.port, + `GET /?t=${ticket} HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) + expect(login.status).toBe(200) + const cookie = (login.headers["set-cookie"] ?? "").split(";")[0]! + const origin = `http://127.0.0.1:${server.port}` + const post = (target: string, payload: unknown, extra: Record) => { + const body = JSON.stringify(payload) + return raw( + server.port, + `POST ${target} HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nContent-Type: application/json\r\nContent-Length: ${Buffer.byteLength(body)}\r\nConnection: close\r\n${Object.entries( + extra, + ) + .map(([k, v]) => `${k}: ${v}\r\n`) + .join("")}\r\n`, + body, + ) + } + const preview = JSON.parse( + (await post("/fleet/approve/preview", { runId: "t1", stage: "frontier" }, { Cookie: cookie, Origin: origin })) + .body, + ) as { ticket: string; csrf: string } + expect(typeof preview.ticket).toBe("string") + await Bun.sleep(80) + const submit = await post( + "/fleet/approve", + { ticket: preview.ticket, passphrase: PASS }, + { Cookie: cookie, Origin: origin, "X-CSRF": preview.csrf }, + ) + expect(submit.status).toBe(403) + expect(submit.body).toContain("unknown or expired ticket") + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + await rm(repo, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/web/test/evidence.contract.test.ts b/packages/web/test/evidence.contract.test.ts new file mode 100644 index 0000000..ab9aed3 --- /dev/null +++ b/packages/web/test/evidence.contract.test.ts @@ -0,0 +1,182 @@ +// Web evidence contract (#207 §2): the shipped web client (`src/api.ts`, +// type-only fleet imports) against the real in-process fleet server. This +// is the test-only harness-import exception (AGENTS.md): it runtime-imports +// the fleet server to prove the browser's fetchers parse live sealed +// evidence (claims, ranges, seal, exports) over the session cookie. `src/` +// stays type-only — nothing here ships. +import { afterEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { createConnection } from "node:net" +import { tmpdir } from "node:os" +import path from "node:path" +import { type FleetSnapshot, fleetPaths, TelemetryServer } from "@heretek-ai/es-fleet" +import { + exportEvidenceDossier, + fetchEvidenceClaim, + fetchEvidenceClaims, + fetchEvidenceSource, + fetchStatus, + fetchTask, +} from "../src/api.ts" +import { layoutEvidence } from "../src/evidence.ts" + +const SOURCE_TEXT = + "Bun is a fast all-in-one JavaScript runtime. It ships a test runner, a bundler, and a package manager — all built in. Researchers measured startup at twelve milliseconds on commodity hardware." +const QUOTE = "Researchers measured startup at twelve milliseconds on commodity hardware" + +const realFetch = globalThis.fetch +afterEach(() => { + globalThis.fetch = realFetch +}) + +const raw = (port: number, head: string): Promise<{ status: number; headers: Record; body: string }> => + new Promise((resolve, reject) => { + const socket = createConnection(port, "127.0.0.1", () => socket.write(head)) + let buffer = Buffer.alloc(0) + socket.on("data", (chunk: Buffer) => { + buffer = Buffer.concat([buffer, chunk]) + }) + socket.on("close", () => { + const text = buffer.toString("latin1") + const at = text.indexOf("\r\n\r\n") + const header = text.slice(0, at) + const [statusLine, ...lines] = header.split("\r\n") + const headers: Record = {} + for (const line of lines) { + const colon = line.indexOf(":") + if (colon > 0) headers[line.slice(0, colon).trim().toLowerCase()] = line.slice(colon + 1).trim() + } + resolve({ status: Number(statusLine!.split(" ")[1]), headers, body: text.slice(at + 4) }) + }) + socket.on("error", reject) + }) + +async function rig() { + const state = await mkdtemp(path.join(tmpdir(), "es-web-evidence-")) + const repo = await mkdtemp(path.join(tmpdir(), "es-web-evidence-repo-")) + // Relative core imports: the web package has no es-core dependency, and + // the leaf boundary keeps `src/` type-only — this fixture setup lives in + // the test-only harness exception, so it resolves core by path. + const { SourceCache } = await import("../../core/src/research/cache.ts") + const { normalizeClaim } = await import("../../core/src/claims/claim.ts") + const { evidenceHash } = await import("../../core/src/claims/dossier.ts") + const { writeJson } = await import("../../core/src/util/fs.ts") + const { mintWebTicket } = await import("@heretek-ai/es-fleet") + const sources = path.join(repo, ".factory", "research", "sources") + await mkdir(sources, { recursive: true }) + const sealed = new SourceCache(sources, state) + const entry = await sealed.put({ url: "https://example.com/web-contract", text: SOURCE_TEXT, provider: "fetch" }) + const claim = normalizeClaim({ + tag: "VERIFIED", + statement: "Bun starts in about twelve milliseconds.", + source: { sha256: entry.meta.sha256, quote: QUOTE }, + }) + await writeJson(path.join(repo, ".factory", "research", "dossier.json"), { + version: "1.1", + mode: "research", + subject: "web-contract", + createdAt: new Date().toISOString(), + claims: [claim], + evidenceHash: evidenceHash([claim]), + }) + await writeFile(path.join(repo, ".factory", "research", "REPORT.md"), "# Contract\n\nBody.\n") + const webRoot = path.join(state, "site") + await mkdir(webRoot, { recursive: true }) + await writeFile(path.join(webRoot, "index.html"), "swarm") + await mkdir(fleetPaths(state).dir, { recursive: true }) + const { ensureToken } = await import("@heretek-ai/es-fleet") + const token = await ensureToken(state) + await writeFile( + fleetPaths(state).worktrees, + JSON.stringify({ + v: 1, + worktrees: { + run1: { + taskId: "run1", + dir: repo, + branch: "fleet/run1", + base: "0".repeat(40), + createdAt: new Date().toISOString(), + status: "allocated", + }, + }, + }), + ) + const server = new TelemetryServer({ + stateRoot: state, + port: 0, + token, + getSnapshot: (): FleetSnapshot => ({ + daemon: { running: true, pid: 7, maxUsd: 50, concurrency: 2 }, + tasks: [{ id: "run1", title: "Contract", status: "running", ceilingUSD: 5, spendUsd: 0, deps: [], reason: "x" }], + spendUsd: 0, + pending: [], + }), + webRoot, + }) + await server.start() + const ticket = await mintWebTicket(state) + const login = await raw( + server.port, + `GET /?t=${ticket} HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) + const cookie = (login.headers["set-cookie"] ?? "").split(";")[0]! + const base = `http://127.0.0.1:${server.port}` + const origin = base + // The browser rides the session cookie same-origin; tests inject it the + // same way (Cookie + loopback Origin) around the real fetch. + const wrapped = globalThis.fetch + globalThis.fetch = ((url: unknown, init: unknown) => { + const headers = { ...((init as { headers?: Record } | undefined)?.headers ?? {}), cookie, origin } + return (wrapped as typeof fetch)(url as string, { ...((init as object) ?? {}), headers }) + }) as typeof fetch + return { state, repo, server, base, sealedSha: entry.meta.sha256, sealedText: entry.text, claimId: claim.id } +} + +describe("web evidence contract against the live fleet server (#207 §2)", () => { + test("status/task fetchers parse the live snapshot", async () => { + const { state, repo, server, base } = await rig() + try { + const status = await fetchStatus(base) + expect(status.tasks.map((task) => task.id)).toEqual(["run1"]) + expect((await fetchTask("run1", base)).title).toBe("Contract") + } finally { + globalThis.fetch = realFetch + await server.stop() + await rm(state, { recursive: true, force: true }) + await rm(repo, { recursive: true, force: true }) + } + }) + + test("evidence fetchers match the sealed sources (sha, seal, ranges, export)", async () => { + const { state, repo, server, base, sealedSha, sealedText, claimId } = await rig() + try { + const listed = await fetchEvidenceClaims("run1", { tag: "VERIFIED" }, base) + expect(listed.claims).toHaveLength(1) + expect(listed.claims[0]!.id).toBe(claimId) + expect(listed.claims[0]!.sourceSha).toBe(sealedSha) + // The claim graph draws the live edge. + expect(layoutEvidence(listed.claims).edges).toEqual([{ from: claimId, to: sealedSha }]) + + const detail = await fetchEvidenceClaim("run1", claimId, base) + expect(detail.quotes).toHaveLength(1) + expect(detail.quotes[0]!.verified).toBe(true) + expect(detail.quotes[0]!.ranges.length).toBeGreaterThan(0) + const [first, last] = [detail.quotes[0]!.ranges[0]!, detail.quotes[0]!.ranges.at(-1)!] + expect(sealedText.slice(first.start, last.end)).toContain("twelve milliseconds") + + const source = await fetchEvidenceSource("run1", sealedSha, base) + expect(source.seal).toBe("sealed") + expect(source.text).toBe(sealedText) + expect(source.meta.sha256).toBe(sealedSha) + + const exported = await exportEvidenceDossier("run1", "markdown", base) + expect(exported.body).toContain("twelve milliseconds") + } finally { + globalThis.fetch = realFetch + await server.stop() + await rm(state, { recursive: true, force: true }) + await rm(repo, { recursive: true, force: true }) + } + }) +}) From 86792b72329e79fc949aff09708de23b01db33d7 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:30:54 -0400 Subject: [PATCH 26/83] fix(remediation-audit/s4): verifier fuzz corpus pins (Refs #251) --- packages/core/test/evidence-fuzz.test.ts | 90 ++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 packages/core/test/evidence-fuzz.test.ts diff --git a/packages/core/test/evidence-fuzz.test.ts b/packages/core/test/evidence-fuzz.test.ts new file mode 100644 index 0000000..e8f9590 --- /dev/null +++ b/packages/core/test/evidence-fuzz.test.ts @@ -0,0 +1,90 @@ +// #251.1: verifier fuzz corpus. Pins the quote verifier against the +// acceptance list in the issue: zero-width/bidi/compatibility characters, +// `..` vs `...` vs `....`, entities at the quote, and markdown-rendered +// quotes. Static review found no false accepts; these tests pin that. +import { describe, expect, test } from "bun:test" +import { normalizeForQuote, verifyQuote } from "../src/research/quote.ts" + +const SRC = "The quick brown fox jumps over the lazy dog near the riverbank today." + +describe("zero-width, bidi and compatibility characters (#251.1)", () => { + test("U+200B folds to a word break (documented TYPOGRAPHY fold, pinned)", () => { + // U+200B sits at the end of the folded whitespace range, so a quote + // carrying it matches a source with a plain space, and vice versa. + expect(verifyQuote(SRC, "quick brown fox\u200Bjumps over the").ok).toBe(true) + expect(verifyQuote("quick brown fox\u200Bjumps over the lazy dog today", "quick brown fox jumps over").ok).toBe( + true, + ) + expect(normalizeForQuote("a\u200Bb")).toBe("a b") + }) + + test("joiners, soft hyphen, word joiner and bidi controls never fold: quotes carrying them refuse", () => { + // None of U+200C U+200D U+00AD U+2060 U+200E U+200F U+202A U+202C U+2066 + // is in the fold table, so the normalised quote keeps the control and + // misses the source. + for (const ch of ["\u200C", "\u200D", "\u00AD", "\u2060", "\u200E", "\u200F", "\u202A", "\u202C", "\u2066"]) { + const check = verifyQuote(SRC, `quick brown fox${ch} jumps over the`) + expect([ch.codePointAt(0)!.toString(16), check.ok]).toEqual([ch.codePointAt(0)!.toString(16), false]) + expect(check.reason).toContain("not found verbatim") + } + }) + + test("compatibility characters fold through NFKC (documented, pinned)", () => { + expect(verifyQuote("The \uFB01ligree parses correctly here", "filigree parses correctly").ok).toBe(true) + expect(verifyQuote("The ABC label is printed here", "\uFF21\uFF22\uFF23 label is printed").ok).toBe(true) + }) + + test("matching is case-sensitive: casing alone never folds", () => { + expect(verifyQuote(SRC, "QUICK BROWN FOX JUMPS OVER").ok).toBe(false) + }) +}) + +describe("ellipsis edge cases: `..` vs `...` vs `....` (#251.1)", () => { + test("two dots are literal: no fragment split", () => { + expect(verifyQuote("Version 1..2 released yesterday morning", "Version 1..2 released").ok).toBe(true) + expect(verifyQuote("Version 1..2 released yesterday morning", "Version 1..3 released").ok).toBe(false) + }) + + test("three dots (or …) split into ordered fragments, each 12+ characters", () => { + expect(verifyQuote(SRC, "quick brown fox ... lazy dog near").ok).toBe(true) + expect(verifyQuote(SRC, "quick brown fox \u2026 lazy dog near").ok).toBe(true) + expect(verifyQuote(SRC, "lazy dog near ... quick brown fox").ok).toBe(false) + }) + + test("four dots are an ellipsis plus a literal dot that must match", () => { + // "...." splits on the first "...", leaving ". lazy dog near" starting + // with a dot the source does not have there. + const check = verifyQuote(SRC, "quick brown fox .... lazy dog near") + expect(check.ok).toBe(false) + expect(check.reason).toContain("not found verbatim") + }) +}) + +describe("entities at the quote (#251.1)", () => { + test("no entity decoding on either side: `&`, `&` and `&` are distinct literals", () => { + const plain = "Fish & chips are tasty indeed" + const encoded = "Fish & chips are tasty indeed" + const numeric = "Fish & chips are tasty indeed" + expect(verifyQuote(plain, "Fish & chips are tasty").ok).toBe(true) + expect(verifyQuote(plain, "Fish & chips are tasty").ok).toBe(false) + expect(verifyQuote(plain, "Fish & chips are tasty").ok).toBe(false) + expect(verifyQuote(encoded, "Fish & chips are tasty").ok).toBe(false) + expect(verifyQuote(numeric, "Fish & chips are tasty").ok).toBe(false) + // Identical literals still match. + expect(verifyQuote(encoded, "Fish & chips are tasty").ok).toBe(true) + }) +}) + +describe("markdown-rendered quotes (#251.1)", () => { + test("links, emphasis and code spans strip on both sides (documented, pinned)", () => { + expect(verifyQuote("[Bun](https://bun.sh) is fast and furious today", "Bun is fast and furious").ok).toBe(true) + expect(verifyQuote("This is **very important** news today ok", "This is very important news").ok).toBe(true) + expect(verifyQuote("Run `bun add zod` to install it now", "Run bun add zod to install").ok).toBe(true) + // Emphasis marks in the quote match a plain source (documented strip). + expect(verifyQuote("Bun is fast and furious today", "**Bun** is fast and furious").ok).toBe(true) + }) + + test("a link target is not quotable text", () => { + expect(verifyQuote("[Bun](https://bun.sh) is fast and furious today", "https://bun.sh is fast and").ok).toBe(false) + }) +}) From af9293ab2d0bedb72ed795810195b41f2f34410c Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:30:56 -0400 Subject: [PATCH 27/83] fix(remediation-audit/s1): ceiling-less approval backstop + STOP denial tests (Refs #229) --- packages/core/src/factory/machine.ts | 4 ++ packages/core/test/policy-stop.test.ts | 70 ++++++++++++++++++++++++ packages/core/test/spend-ceiling.test.ts | 46 ++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 packages/core/test/policy-stop.test.ts create mode 100644 packages/core/test/spend-ceiling.test.ts diff --git a/packages/core/src/factory/machine.ts b/packages/core/src/factory/machine.ts index a226af2..b6fc75b 100644 --- a/packages/core/src/factory/machine.ts +++ b/packages/core/src/factory/machine.ts @@ -433,6 +433,10 @@ export class Factory { this.requireAuditsClear(state) const check = await verifyApproval(this.root, "frontier", { stateDir: this.deps.stateDir }) if (!check.ok) throw new FactoryError(check.reason) + if (check.record.spendCeilingUSD === undefined) + throw new FactoryError( + "The frontier approval carries no spend ceiling; a human must approve it again (`es approve frontier`).", + ) state.spendCeilingUSD = check.record.spendCeilingUSD state.stage = "RESEARCH" await this.audit(actor, "stage.research", { spendCeilingUSD: state.spendCeilingUSD }) diff --git a/packages/core/test/policy-stop.test.ts b/packages/core/test/policy-stop.test.ts new file mode 100644 index 0000000..84e7b37 --- /dev/null +++ b/packages/core/test/policy-stop.test.ts @@ -0,0 +1,70 @@ +// STOP kill-switch denials (#229): the kill-switch file itself is a control +// file, so no agent may delete or overwrite it — through the editor, the +// shell, or inside the sandbox (where .factory/ is read-only for every kind). +// Mirrors the gates/approvals shapes in policy.test.ts. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { evaluateShell, evaluateWrite } from "../src/trust/index.ts" +import { type SandboxKind, sandboxArgv } from "../src/trust/sandbox.ts" + +let root: string +let state: string +let worktree: string + +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-stop-")) + state = await mkdtemp(path.join(tmpdir(), "es-stop-state-")) + worktree = path.join(root, ".factory/worktrees/p1") + await mkdir(path.join(worktree, "src"), { recursive: true }) + await mkdir(path.join(root, ".factory/approvals"), { recursive: true }) +}) +afterAll(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const ctx = () => ({ root, worktree, stateDir: state }) +const effect = (decision: { effect: string }) => decision.effect +const shell = (agent: string | undefined, command: string) => + evaluateShell(ctx(), agent, command, { sandboxAvailable: false }) + +describe("STOP kill-switch", () => { + test("write-deny: no agent deletes or overwrites the kill-switch, by any path shape", () => { + for (const agent of [undefined, "build", "factory", "es-programmer", "es-manager", "qa", "es-qa-functional"]) { + expect([agent, effect(evaluateWrite(ctx(), agent, ".factory/STOP"))]).toEqual([agent, "deny"]) + expect(effect(evaluateWrite(ctx(), agent, path.join(root, ".factory/STOP")))).toBe("deny") + expect(effect(evaluateWrite(ctx(), agent, "docs/../.factory/STOP"))).toBe("deny") + } + }) + + test("shell-deny: removing or redirecting over the kill-switch is refused; seats may not mention it at all", () => { + for (const command of ["rm .factory/STOP", "rm -f .factory/STOP", "echo x > .factory/STOP"]) + expect([command, shell("build", command).effect]).toEqual([command, "deny"]) + // Reads stay allowed for the user's own agents, as for other control files. + expect(shell("build", "cat .factory/STOP").effect).toBe("allow") + // Seats get no STOP handle at all: mentioning it is refused even to read. + for (const command of ["rm .factory/STOP", "echo x > .factory/STOP", "cat .factory/STOP"]) + expect([command, shell("es-programmer", command).effect]).toEqual([command, "deny"]) + }) + + test("sandbox-argv: .factory/ is read-only for every sandbox kind, so no shell inside can remove STOP", () => { + const probe = { exists: () => true, overlay: false, uid: 1000 } + for (const kind of ["user", "seat", "programmer", "readonly", "gate"] as const satisfies SandboxKind[]) { + const args = sandboxArgv( + { + kind, + root: "/p", + cwd: "/p", + stateDir: "/s", + home: "/h", + ...(["programmer", "gate"].includes(kind) ? { writable: "/p/.factory/worktrees/p1" } : {}), + }, + ["true"], + probe, + ) + expect([kind, args.join(" ")]).toEqual([kind, expect.stringContaining("--ro-bind /p/.factory /p/.factory")]) + } + }) +}) diff --git a/packages/core/test/spend-ceiling.test.ts b/packages/core/test/spend-ceiling.test.ts new file mode 100644 index 0000000..d0773b0 --- /dev/null +++ b/packages/core/test/spend-ceiling.test.ts @@ -0,0 +1,46 @@ +// Spend-ceiling backstop (#229): a frontier approval without a spend ceiling +// (hand-written, legacy-shaped) must never start a run with no ceiling. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { recordApproval, verifyApproval } from "../src/approval/index.ts" +import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { Factory } from "../src/factory/index.ts" +import { factoryLayout } from "../src/layout.ts" +import { ApprovalSchema } from "../src/schema/approval.ts" +import { rebaseline } from "../src/trust/control.ts" +import { writeJson } from "../src/util/fs.ts" +import { type Fixture, frontier, gitRepo } from "./helpers.ts" + +const PASSPHRASE = "test-passphrase-1234" +let fx: Fixture +let signer: HumanSigner +let factory: Factory + +beforeEach(async () => { + fx = await gitRepo("es-ceil-") + await sealHumanKey(PASSPHRASE, fx.state) + signer = await unlockHumanKey(PASSPHRASE, fx.state) + factory = new Factory(fx.root, { + gates: async () => ({ passed: true, findings: [], summary: "green" }), + stateDir: fx.state, + }) +}) +afterEach(() => fx.cleanup()) + +describe("spend ceiling backstop", () => { + test("beginResearch refuses a ceiling-less frontier approval", async () => { + await factory.begin("human:tester") + await factory.writeFrontier("grill", frontier()) + const genuine = await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer }) + expect(genuine.spendCeilingUSD).toBe(25) + // A human-signed but ceiling-less record: valid signature, valid schema, + // no ceiling — the shape a hand-written legacy record would have. + const { spendCeilingUSD: _dropped, signature: _old, ...rest } = genuine + const ceilingLess = ApprovalSchema.parse({ ...rest, signature: signer.sign(rest) }) + expect(ceilingLess.spendCeilingUSD).toBeUndefined() + await writeJson(factoryLayout(fx.root).approval("frontier"), ceilingLess) + await rebaseline(fx.root, "human:tester") + expect((await verifyApproval(fx.root, "frontier", { stateDir: fx.state })).ok).toBe(true) + await expect(factory.beginResearch("human:tester")).rejects.toThrow(/spend ceiling/) + expect((await factory.read())?.stage).toBe("GRILL") + }) +}) From 3d3d42a4d25fa142e2c7101b204c433c8628c9eb Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:31:38 -0400 Subject: [PATCH 28/83] fix(remediation-audit/s2): G1 text-layer parity, REPORT scope wording, waiver-drift proof (Refs #216, Refs #220, Refs #221) --- packages/core/src/trust/policy.ts | 7 +- packages/core/test/policy.test.ts | 111 +++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/packages/core/src/trust/policy.ts b/packages/core/src/trust/policy.ts index 2a02be2..16e5941 100644 --- a/packages/core/src/trust/policy.ts +++ b/packages/core/src/trust/policy.ts @@ -102,8 +102,9 @@ export function evaluateWrite(context: PolicyContext, agentId: string | undefine if (!spec) return allow if (relative === undefined) return deny(`Factory seat "${spec.id}" may not write outside the project: ${absolute}`) - // REPORT.md has one writer per run mode (#111): the synthesizer is the - // only writer in research runs, the factory in build runs (AGENTS.md). + // REPORT.md has one writer scope per run mode (#111, #220): factory-docs + // seats (the factory and the manager share the scope) in build runs, the + // research synthesizer alone in research runs (AGENTS.md). // The mode comes from the sealed run state; when it cannot be read both // writers are denied (fail closed). if (relative === ".factory/research/REPORT.md") { @@ -329,7 +330,7 @@ const MENTION_HINT = " If it only names the verb in text (a here-doc, a commit message, a `--body` argument), the rule still reads a call: write the text to a file with the edit tool and pass the file (`gh … --body-file`, `git commit -F`)." const CONTROL_MENTION = - /\.factory\/(gates\.json|config\.json|frontier\.json|waivers|approvals|runtime|STOP|git-hooks|claims\b|audits\b|research\/(sources\b|(coverage|dossier|brief\.pcrb)\.json)|(brainstorm|harvest|design|scout)\/[^\s'"]*\.json)|\.git\/(config|hooks)|\.opencode\/(hooks\.json|plugins|opencode\.jsonc?)|\.claude\/settings|opencode\.jsonc?\b/i + /\.factory\/(gates\.json|config\.json|frontier\.json|waivers|approvals|runtime|STOP|git-hooks|claims\b|audits\b|improve\b|research\/(sources\b|(coverage|dossier|brief\.pcrb)\.json)|(brainstorm|harvest|design|scout)\/(?!notes\/)[^\s'"]*)|\.git\/(config|hooks)|\.opencode\/(hooks\.json|plugins|opencode\.jsonc?)|\.claude\/settings|opencode\.jsonc?\b/i /** Any mention of the factory dir (a `cd .factory` reaches control files without naming them). */ const FACTORY_MENTION = /(^|[\s/=:])\.factory(\/|[\s;|&]|$)/i /** Any mention of fleet isolation state (task worktrees live under `.fleet/`). */ diff --git a/packages/core/test/policy.test.ts b/packages/core/test/policy.test.ts index 4d29c1e..9295616 100644 --- a/packages/core/test/policy.test.ts +++ b/packages/core/test/policy.test.ts @@ -161,7 +161,7 @@ describe("write policy", () => { expect(effect(evaluateWrite(ctx(), "es-research-alpha", ".factory/research/notes.md"))).toBe("deny") }) - test("REPORT.md has one writer per run mode: the factory in build runs, the synthesizer in research runs (#111)", () => { + test("REPORT.md has one writer scope per run mode: factory-docs seats in build runs, the synthesizer in research runs (#111, #220)", () => { const report = ".factory/research/REPORT.md" // Build runs: factory-docs seats write it, research-report seats do not. for (const agent of ["factory", "es-manager"]) @@ -488,6 +488,95 @@ describe("shell policy", () => { expect(shell("es-programmer", "cat .factory/gates.json").effect).toBe("deny") }) + test("seat shells cannot name any factory control path: pattern-vs-gloss parity (#216)", () => { + // One representative per FACTORY_CONTROL glob (trust/control.ts): the seat + // text layer must deny each, however spelled. Engine state is not only + // *.json, and improve/** is control too (#216). + const controls = [ + ".factory/gates.json", + ".factory/config.json", + ".factory/frontier.json", + ".factory/waivers/w1.json", + ".factory/approvals/frontier.json", + ".factory/runtime/seats.json", + ".factory/STOP", + ".factory/git-hooks/pre-commit", + ".factory/brainstorm/scores.json", + ".factory/brainstorm/ideas.md", + ".factory/brainstorm/runs/grill-1/plan.json", + ".factory/brainstorm/runs/grill-1/BRAINSTORM.md", + ".factory/harvest/matrix.json", + ".factory/harvest/VENDOR-PLAN.md", + ".factory/harvest/widget/profile.json", + ".factory/harvest/clean-room/teardown-search.md", + ".factory/harvest/runs/grill-1/HARVEST.md", + ".factory/design/tokens.json", + ".factory/design/tokens.css", + ".factory/design/STYLE_GUIDE.md", + ".factory/design/components/table/tokens.json", + ".factory/design/skill/table/SKILL.md", + ".factory/design/skill-claude/table/SKILL.md", + `.factory/research/sources/${"c".repeat(64)}.md`, + ".factory/research/coverage.json", + ".factory/research/dossier.json", + ".factory/research/brief.pcrb.json", + ".factory/claims/retractions.json", + ".factory/improve/proposals/widget/proposal.json", + ".factory/improve/telemetry.json", + ".factory/audits/phase-01/findings.json", + ".factory/scout/plan.json", + ".factory/scout/overview.md", + ".factory/scout/widget/profile.json", + ".git/config", + ".git/hooks/pre-commit", + ] + for (const file of controls) { + for (const agent of ["factory", "es-programmer", "es-research-alpha", "es-manager", "harvester"]) { + expect([file, agent, effect(evaluateWrite(ctx(), agent, file))]).toEqual([file, agent, "deny"]) + expect([file, agent, shell(agent, `cat ${file}`).effect]).toEqual([file, agent, "deny"]) + } + // Non-seat agents may read controls but never mutate them from the shell. + expect(shell(undefined, `echo x > ${file}`).effect).toBe("deny") + } + // Every CONFIG_CONTROL glob: seats denied in the editor and the shell, + // non-seat agents asked in the editor (#236 MEDIUM: all 8 representatives). + const configs = [ + "opencode.json", + "opencode.jsonc", + ".opencode/opencode.json", + ".opencode/opencode.jsonc", + ".opencode/plugins/table/plugin.json", + ".opencode/hooks.json", + ".claude/settings.json", + ".claude/settings.local.json", + ] + for (const file of configs) { + expect(effect(evaluateWrite(ctx(), "es-manager", file))).toBe("deny") + expect(effect(evaluateWrite(ctx(), "build", file))).toBe("ask") + expect(shell("es-programmer", `cat ${file}`).effect).toBe("deny") + } + // The user's own agents may still read controls (never mutate them): the + // .md/.css backstop stays a read backstop (#236 MEDIUM). + for (const file of [".factory/design/tokens.css", ".factory/scout/overview.md", ".factory/harvest/VENDOR-PLAN.md"]) + expect(shell("build", `cat ${file}`).effect).toBe("allow") + // Case variants name the same control file on case-insensitive volumes. + expect(shell("es-programmer", "cat .FACTORY/GATES.JSON").effect).toBe("deny") + expect(effect(evaluateWrite(ctx(), "es-programmer", ".factory/GATES.JSON"))).toBe("deny") + // Seats keep their own notes out of the text deny (notes are seat-writable). + expect(shell("es-programmer", "cat .factory/brainstorm/notes/x.md").effect).not.toBe("deny") + }) + + test("a symlink into .factory passes the seat text layer; the write is still denied (#220, G1 theme)", async () => { + // String patterns cannot resolve symlinks: the seat shell text layer lets + // `cat src/shell-link/spec.json` through by construction. Enforcement is + // the sandbox ro-bind of .factory/ (trust/sandbox.ts), which the link + // resolves into; the editor resolves the link and denies. + await writeFile(path.join(root, ".factory/approvals/spec.json"), "{}") + await symlink(path.join(root, ".factory/approvals"), path.join(root, "src/shell-link")).catch(() => {}) + expect(effect(evaluateWrite(ctx(), "build", "src/shell-link/spec.json"))).toBe("deny") + expect(shell("es-programmer", "cat src/shell-link/spec.json").effect).not.toBe("deny") + }) + test("a .factory mention is read-only-gated for the user's agents: the audit's bypasses are refused", () => { // A3 (1.1.0): interpreter one-liners. expect(shell("build", `python3 -c 'open(".factory/gates.json","w").write("pwned")'`).effect).toBe("deny") @@ -589,6 +678,26 @@ describe("control baseline", () => { await writeFile(path.join(root, ".factory/config.json"), '{"afterEdit":"fast"}') expect((await verifyControl(root)).violations.join("\n")).toContain(".factory/config.json changed") }) + + test("drift in a waiver file is reported (#221)", async () => { + // Waiver records are pinned alongside the gates/config/frontier files, so + // a hand edit to a granted waiver halts the run like any other drift. + const dir = path.join(root, ".factory/waivers") + await mkdir(dir, { recursive: true }) + const record = path.join(dir, "lint-check.json") + try { + await rm(record, { force: true }) + await writeFile(record, '{"rule":"lint"}') + await rebaseline(root, "test") + expect((await verifyControl(root)).clean).toBe(true) + await writeFile(record, '{"rule":"lint","edited":true}') + const check = await verifyControl(root) + expect(check.clean).toBe(false) + expect(check.violations.join("\n")).toContain(".factory/waivers/lint-check.json changed") + } finally { + await rm(record, { force: true }) + } + }) }) // ------------------------------------------------------------------ argv parity (#97) From e3b56a96c7d154df949f4349d03b307192f9a4da Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:31:41 -0400 Subject: [PATCH 29/83] fix(remediation-audit/s2): per-directory overlay probe with graceful fallback (Refs #231) --- packages/core/src/trust/sandbox.ts | 50 +++++++++++++++++++++--------- packages/core/test/sandbox.test.ts | 37 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/packages/core/src/trust/sandbox.ts b/packages/core/src/trust/sandbox.ts index d18ab00..7232534 100644 --- a/packages/core/src/trust/sandbox.ts +++ b/packages/core/src/trust/sandbox.ts @@ -40,6 +40,13 @@ export interface SandboxProbe { readonly exists: (file: string) => boolean /** bwrap supports --overlay-src/--tmp-overlay (0.10+). */ readonly overlay: boolean + /** + * Per-directory overlay support (#231): an overlay mount can fail on one + * filesystem while it succeeds on another (a btrfs home vs tmpfs /tmp), so + * a probe that only tries /tmp misreports the mount that matters. Absent: + * `overlay` answers for every directory. + */ + readonly overlayFor?: (dir: string) => boolean readonly uid?: number } @@ -106,6 +113,9 @@ export function sandboxArgv( const writableRoot = spec.kind === "user" || spec.kind === "seat" const writable = spec.kind === "programmer" || spec.kind === "gate" ? spec.writable : undefined const protectedRel = user ? PROTECTED_ALWAYS : [...PROTECTED_ALWAYS, ...PROTECTED_FOR_SEATS] + // Graceful degradation (#231): when the per-dir probe refuses a mount, the + // overlay args are omitted instead of emitting a mount that fails at runtime. + const overlayFor = (dir: string) => probe.overlayFor?.(dir) ?? probe.overlay const protectedIn = (dir: string, names: readonly string[]) => names.map((name) => path.join(dir, name)).filter((file) => probe.exists(file)) const args: string[] = ["bwrap", writableRoot ? "--bind" : "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc"] @@ -116,9 +126,9 @@ export function sandboxArgv( // A private /tmp; projects and homes that live under /tmp are bound back in. args.push("--tmpfs", "/tmp") // Tool caches under home stay usable; writes there are thrown away. - if (probe.overlay && probe.exists(home)) args.push("--overlay-src", home, "--tmp-overlay", home) + if (overlayFor(home) && probe.exists(home)) args.push("--overlay-src", home, "--tmp-overlay", home) else if (inside("/tmp", home) && probe.exists(home)) roBind(home) - if (inside("/tmp", spec.root) && !(probe.overlay && inside(home, spec.root))) roBind(spec.root) + if (inside("/tmp", spec.root) && !(overlayFor(home) && inside(home, spec.root))) roBind(spec.root) } // Project control paths are read-only. for (const file of protectedIn(spec.root, protectedRel)) roBind(file) @@ -131,7 +141,7 @@ export function sandboxArgv( for (const file of protectedIn(writable, [...PROTECTED_IN_WORKTREE, ...protectedRel])) roBind(file) } // A read-only seat's writes in its cwd land in a throwaway layer (when bwrap can). - if (spec.kind === "readonly" && probe.overlay) args.push("--overlay-src", spec.cwd, "--tmp-overlay", spec.cwd) + if (spec.kind === "readonly" && overlayFor(spec.cwd)) args.push("--overlay-src", spec.cwd, "--tmp-overlay", spec.cwd) // Secrets, for every kind. if (probe.exists(spec.stateDir)) args.push("--tmpfs", spec.stateDir) @@ -158,32 +168,44 @@ export function sandboxArgv( return args } -let overlayCache: boolean | undefined -/** Whether this bwrap supports --tmp-overlay (Ubuntu 24.04 ships 0.9, which does not). */ -export function bwrapOverlay(): boolean { - if (overlayCache !== undefined) return overlayCache +const overlayCache = new Map() +/** + * Whether this bwrap can overlay-mount `dir` (Ubuntu 24.04 ships bwrap 0.9, + * which has no overlay support at all). Probed per directory and cached: an + * overlay mount can fail on one filesystem while it succeeds on another (a + * btrfs home vs tmpfs /tmp, #231), so probing only /tmp misreports the mount + * that matters. + */ +export function bwrapOverlay(dir = "/tmp"): boolean { + const cached = overlayCache.get(dir) + if (cached !== undefined) return cached try { // bwrap is resolved from PATH by design (it may be absent). const probe = spawnSync( // NOSONAR "bwrap", - ["--ro-bind", "/", "/", "--dev", "/dev", "--overlay-src", "/tmp", "--tmp-overlay", "/tmp", "--", "true"], + ["--ro-bind", "/", "/", "--dev", "/dev", "--overlay-src", dir, "--tmp-overlay", dir, "--", "true"], { timeout: 5000 }, ) - overlayCache = probe.status === 0 + overlayCache.set(dir, probe.status === 0) } catch { - overlayCache = false + overlayCache.set(dir, false) } - return overlayCache + return overlayCache.get(dir)! } function liveProbe(): SandboxProbe { - return { exists: existsSync, overlay: bwrapOverlay(), uid: process.getuid?.() } + return { + exists: existsSync, + overlay: bwrapOverlay(), + overlayFor: (dir) => bwrapOverlay(dir), + uid: process.getuid?.(), + } } -/** For tests: forget the cached overlay probe. */ +/** For tests: forget the cached overlay probes. */ export const resetOverlayProbe = () => { - overlayCache = undefined + overlayCache.clear() } /** Why a sandbox cannot run here, for refusal messages. */ diff --git a/packages/core/test/sandbox.test.ts b/packages/core/test/sandbox.test.ts index 43259bb..f7e2a56 100644 --- a/packages/core/test/sandbox.test.ts +++ b/packages/core/test/sandbox.test.ts @@ -159,4 +159,41 @@ describe("sandboxArgv", () => { expect(argv("user")).not.toContain("--die-with-parent") expect(argv("seat")).toContain("--die-with-parent") }) + + test("a failed overlay probe degrades to no overlay: protections stay, nothing mounts (#231)", () => { + // Overlayfs mounts can fail on one filesystem while they succeed on + // another (a btrfs home vs tmpfs /tmp): the probe answers per directory, + // and a refusal omits the overlay args instead of emitting a mount that + // fails every seat shell at runtime. + const refused = { exists: () => true, overlay: true, overlayFor: (_dir: string) => false } + const args = sandboxArgv( + { kind: "readonly", root: "/p", cwd: "/p", stateDir: "/h/.local/state/es", home: "/h", offline: true }, + ["true"], + refused, + ).join(" ") + expect(args).not.toContain("--overlay-src") + expect(args).not.toContain("--tmp-overlay") + expect(args).toContain("--tmpfs /h/.local/state/es") + expect(args).toContain("--unshare-net") + // ...while a working per-dir probe keeps the throwaway layers. + const supported = { exists: () => true, overlay: false, overlayFor: (_dir: string) => true } + const layered = sandboxArgv( + { kind: "readonly", root: "/p", cwd: "/p", stateDir: "/h/.local/state/es", home: "/h", offline: true }, + ["true"], + supported, + ).join(" ") + expect(layered).toContain("--overlay-src /h") + expect(layered).toContain("--tmp-overlay /p") + }) + + test("the overlay probe is per directory: an unmountable dir reports false (#231)", async () => { + const { bwrapOverlay, resetOverlayProbe } = await import("../src/trust/sandbox.ts") + const { bwrapAvailable } = await import("../src/index.ts") + if (!bwrapAvailable()) return + resetOverlayProbe() + // No bwrap can overlay-mount a path that does not exist; /tmp mounts here. + expect(bwrapOverlay(`/nonexistent-es-probe-${process.pid}`)).toBe(false) + expect(bwrapOverlay(`/nonexistent-es-probe-${process.pid}`)).toBe(false) + resetOverlayProbe() + }) }) From 842a8ad52ac2e37f2a01121226d0f1e0a4a5b44f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:31:41 -0400 Subject: [PATCH 30/83] fix(remediation-audit/s2): attempt-bucket comment matches shared CLI/TUI file (Refs #222) --- packages/core/src/approval/attempts.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/approval/attempts.ts b/packages/core/src/approval/attempts.ts index 4c00f8d..93c7ced 100644 --- a/packages/core/src/approval/attempts.ts +++ b/packages/core/src/approval/attempts.ts @@ -1,6 +1,7 @@ // Approval attempt limiting (ADR 0002 I4): at most 5 wrong passphrases per // 10 minutes per state dir, then a lockout. Lockouts are audited into the -// project's audit log. One surface's lockout does not affect the others. +// project's audit log. The CLI and the TUI share one attempt file per state +// dir (a single bucket for both); the web path keeps its own dir and bucket. import path from "node:path" import { appendAuditEntry } from "../audit/chain.ts" import { readJson, withLock, writeJson } from "../util/fs.ts" From c04ba237465942227dc832aef35586b3b5f0df72 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:32:41 -0400 Subject: [PATCH 31/83] fix(remediation-audit/s4): seal URL index with engine-key sidecar (Refs #251) --- packages/core/src/research/cache.ts | 38 ++++++++++-- packages/core/test/evidence-seal.test.ts | 76 ++++++++++++++++++++++++ packages/core/test/research.test.ts | 4 +- 3 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 packages/core/test/evidence-seal.test.ts diff --git a/packages/core/src/research/cache.ts b/packages/core/src/research/cache.ts index 6a2fac3..fb838d2 100644 --- a/packages/core/src/research/cache.ts +++ b/packages/core/src/research/cache.ts @@ -71,6 +71,34 @@ export class SourceCache { return key !== undefined && verifyMetaSeal(meta as unknown as Record, key) } + private indexFile = () => path.join(this.dir, "index.json") + private indexSealFile = () => path.join(this.dir, "index.seal") + + /** + * The URL index sealed with the engine key (#251.2): without integrity the + * index could remap a URL onto another sealed entry's bytes and `byUrl` + * would serve them. Writes store an HMAC sidecar; reads refuse a + * present-but-forged sidecar and keep serving a missing one (caches + * written before sidecars stay readable). Per-entry seals still apply at + * `get`, so the sidecar only guards the mapping, never the bytes. + */ + private async writeIndex(index: Record): Promise { + await atomicWrite(this.indexFile(), `${JSON.stringify(index, null, 2)}\n`) + if (!this.stateDir) return + const key = await ensureEngineKey(this.stateDir) + await atomicWrite(this.indexSealFile(), `${sealMeta(index as unknown as Record, key)}\n`) + } + + private async readIndex(): Promise> { + const index = (await readJson>(this.indexFile())) ?? {} + if (!this.stateDir) return index + const seal = (await readFile(this.indexSealFile(), "utf8").catch(() => undefined))?.trim() + if (seal === undefined || seal === "") return index + const key = await readEngineKey(this.stateDir) + if (key === undefined) return index + return verifyMetaSeal({ ...index, seal }, key) ? index : {} + } + async put(input: { url: string text: string @@ -98,9 +126,9 @@ export class SourceCache { const file = path.join(this.dir, `${hash}.md`) await atomicWrite(file, text) await atomicWrite(path.join(this.dir, `${hash}.json`), `${JSON.stringify(meta, null, 2)}\n`) - const index = (await readJson>(path.join(this.dir, "index.json"))) ?? {} + const index = await this.readIndex() index[canonicalUrl(input.url) ?? input.url] = hash - await atomicWrite(path.join(this.dir, "index.json"), `${JSON.stringify(index, null, 2)}\n`) + await this.writeIndex(index) return { meta, text, path: file } } @@ -125,7 +153,7 @@ export class SourceCache { } async byUrl(url: string): Promise { - const index = (await readJson>(path.join(this.dir, "index.json"))) ?? {} + const index = await this.readIndex() // Canonical key first; raw keys are what 1.0 caches recorded. const hash = index[canonicalUrl(url) ?? url] ?? index[url] return hash ? this.get(hash) : undefined @@ -147,9 +175,9 @@ export class SourceCache { removed.push(hash) } if (removed.length) { - const index = (await readJson>(path.join(this.dir, "index.json"))) ?? {} + const index = await this.readIndex() for (const [url, hash] of Object.entries(index)) if (removed.includes(hash)) delete index[url] - await atomicWrite(path.join(this.dir, "index.json"), `${JSON.stringify(index, null, 2)}\n`) + await this.writeIndex(index) } return removed } diff --git a/packages/core/test/evidence-seal.test.ts b/packages/core/test/evidence-seal.test.ts new file mode 100644 index 0000000..d52aee9 --- /dev/null +++ b/packages/core/test/evidence-seal.test.ts @@ -0,0 +1,76 @@ +// #251.2: the URL index is sealed with the engine key. A sealed cache +// writes an `index.seal` sidecar (HMAC over the canonical index); `byUrl` +// refuses a tampered index instead of following it. A missing sidecar stays +// readable (caches written before sealing keep working); an unsealed cache +// writes no sidecar. +import { describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { SourceCache } from "../src/research/cache.ts" + +const put2 = async (cache: SourceCache) => { + const a = await cache.put({ url: "https://a.test/page", text: "Alpha source text with enough length.", provider: "fetch" }) + const b = await cache.put({ url: "https://b.test/page", text: "Beta source text with enough length here.", provider: "fetch" }) + return { a, b } +} + +describe("sealed URL index (#251.2)", () => { + test("a sealed cache writes an HMAC sidecar next to index.json; an unsealed one does not", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-index-")) + const state = await mkdtemp(path.join(tmpdir(), "es-index-state-")) + try { + const sealed = new SourceCache(path.join(dir, "sealed"), state) + await put2(sealed) + const seal = await Bun.file(path.join(dir, "sealed", "index.seal")).text() + expect(seal.trim()).toMatch(/^[0-9a-f]{64}$/) + const plain = new SourceCache(path.join(dir, "plain")) + await put2(plain) + expect(await Bun.file(path.join(dir, "plain", "index.seal")).exists()).toBe(false) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) + + test("byUrl refuses a tampered index instead of serving the remapped entry", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-index-tamper-")) + const state = await mkdtemp(path.join(tmpdir(), "es-index-tamper-state-")) + try { + const cache = new SourceCache(path.join(dir, "sources"), state) + const { a, b } = await put2(cache) + expect((await cache.byUrl("https://a.test/page"))?.meta.sha256).toBe(a.meta.sha256) + // Remap a.test to b's bytes without re-sealing the index. + const indexFile = path.join(dir, "sources", "index.json") + const index = JSON.parse(await Bun.file(indexFile).text()) as Record + index["https://a.test/page"] = b.meta.sha256 + await writeFile(indexFile, `${JSON.stringify(index, null, 2)}\n`) + expect(await cache.byUrl("https://a.test/page")).toBeUndefined() + // Direct reads by hash are unaffected. + expect((await cache.get(b.meta.sha256))?.text).toContain("Beta source") + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) + + test("an index written before sealing (no sidecar) stays readable", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-index-legacy-")) + const state = await mkdtemp(path.join(tmpdir(), "es-index-legacy-state-")) + try { + const cache = new SourceCache(path.join(dir, "sources"), state) + const entry = await cache.put({ + url: "https://old.test/page", + text: "Old source text with enough length here.", + provider: "fetch", + }) + // An index from before sidecars existed carries sealed entries but no + // sidecar: the sealed reader keeps serving it by URL. + await rm(path.join(dir, "sources", "index.seal"), { force: true }) + expect((await cache.byUrl("https://old.test/page"))?.meta.sha256).toBe(entry.meta.sha256) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/core/test/research.test.ts b/packages/core/test/research.test.ts index 818a1c1..8fb4842 100644 --- a/packages/core/test/research.test.ts +++ b/packages/core/test/research.test.ts @@ -262,7 +262,7 @@ describe("providers", () => { const text = await fetchTool.execute({ url: "https://zod.test" }, { agent: "es-research-alpha" }) const hash = /sha256:([0-9a-f]{64})/.exec(text)?.[1] expect(hash).toBeDefined() - const stored = await new SourceCache(path.join(dir, ".factory/research/sources")).get(hash!) + const stored = await new SourceCache(path.join(dir, ".factory/research/sources"), state).get(hash!) expect(stored?.text).toContain("TypeScript-first schema validation") const search = tools.find((tool) => tool.name === "es_research_search")! await expect(search.execute({ query: "x" }, { agent: "es-research-alpha" })).rejects.toThrow("No search provider") @@ -334,7 +334,7 @@ describe("the webcache gate (d66328c:skills/epistemic_search/scripts/webcache.py ).toContain("Version 3") expect(calls).toBe(3) // A search result cached under the page's URL is a snippet, never served as the page. - await new SourceCache(path.join(dir, ".factory/research/sources")).put({ + await new SourceCache(path.join(dir, ".factory/research/sources"), state).put({ url: "https://limits.test/api?a=1&b=2", text: "Snippet: limits are documented.", provider: "searxng", From 10c877e12576dab8dc7dc9982d495312d5359a77 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:33:34 -0400 Subject: [PATCH 32/83] fix(remediation-audit/s4): verbatim-after-normalisation wording, flags-prunes-refuses (Refs #254) --- packages/core/src/claims/brief.ts | 3 ++- packages/core/src/claims/constitution.ts | 2 +- packages/core/src/claims/witness.ts | 3 ++- packages/core/src/research/auditor.ts | 7 +++++-- packages/core/src/research/ops.ts | 4 ++-- packages/core/src/research/quote.ts | 16 +++++++++------- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/core/src/claims/brief.ts b/packages/core/src/claims/brief.ts index 3c2f920..77b0ab4 100644 --- a/packages/core/src/claims/brief.ts +++ b/packages/core/src/claims/brief.ts @@ -182,7 +182,8 @@ export async function verifyBrief( `signed by key ${brief.signature.keyId}, which this machine does not hold; only integrity can be checked (allow unverifiable to accept that)`, ) - // 4. Quotes: every claim the brief calls witnessed must quote its bundled source verbatim. + // 4. Quotes: every claim the brief calls witnessed must quote its bundled + // source verbatim after documented normalisation (research/quote.ts). let quotesVerified = 0 let quotesFailed = 0 for (const claim of brief.claims as Claim[]) { diff --git a/packages/core/src/claims/constitution.ts b/packages/core/src/claims/constitution.ts index 5d05a5c..4fa0dd3 100644 --- a/packages/core/src/claims/constitution.ts +++ b/packages/core/src/claims/constitution.ts @@ -131,7 +131,7 @@ export interface ClaimVerdictOptions { /** * The per-claim ACCEPTED/REJECTED gate used by domain packs. A claim is * REJECTED when the legacy structural rules fire (VERIFIED needs its source - * hash + verbatim quote or a code location; INFERRED needs its reasoning; + * hash + quote (verbatim after documented normalisation) or a code location; INFERRED needs its reasoning; * HYPOTHESIS needs its falsification; NEGATIVE_KNOWLEDGE needs query and * finding), when any pack rule fires (parent unresolved, missing mandatory * tag, banned domain), or when the retraction policy is zero_tolerance and diff --git a/packages/core/src/claims/witness.ts b/packages/core/src/claims/witness.ts index be2df4e..b8a1565 100644 --- a/packages/core/src/claims/witness.ts +++ b/packages/core/src/claims/witness.ts @@ -4,7 +4,8 @@ // refusing get + the one quote law, research/quote.ts); // - a code location against the file on disk: inside the target root, the // line range within the file and at most MAX_LOCATION_LINES long, the -// excerpt one contiguous verbatim span of those lines. +// excerpt one contiguous span of those lines, verbatim after documented +// normalisation. // Legacy never checked code locations at all (its "hallucinated line" check // was the generic quote check), so the location rule is new in 1.1. import path from "node:path" diff --git a/packages/core/src/research/auditor.ts b/packages/core/src/research/auditor.ts index 0920620..488bfd3 100644 --- a/packages/core/src/research/auditor.ts +++ b/packages/core/src/research/auditor.ts @@ -2,13 +2,16 @@ // // Every claim line (a bullet, numbered item or prose paragraph line) must carry // at least one tag: -// [VERIFIED: sha256: "verbatim quote"] — quote must be in the cached source +// [VERIFIED: sha256: "quote, verbatim after documented normalisation"] — quote must be in the cached source // [INFERRED: reasoning] — reasoning from verified facts // [HYPOTHESIS: how to test it] // [NEGATIVE_KNOWLEDGE: what was searched] // The auditor verifies quotes against the content-addressed cache, flags // untagged and ungrounded claims, can prune them into a "Pruned claims" -// section, and reports coverage counts (no composite score). +// section, and reports coverage counts (no composite score). It marks, +// prunes and refuses failing claims and dossiers; it never re-tags a claim's +// kind (no VERIFIED→INFERRED demotion: that verb belongs to the +// scout/harvest licence policy). import type { SourceCache } from "./cache.ts" import { verifyQuote } from "./quote.ts" diff --git a/packages/core/src/research/ops.ts b/packages/core/src/research/ops.ts index 633337c..c033fee 100644 --- a/packages/core/src/research/ops.ts +++ b/packages/core/src/research/ops.ts @@ -155,7 +155,7 @@ export function researchTools(context: ResearchOpsContext): EsToolDef[] { { name: "es_research_fetch", description: - 'Fetch a URL, cache its text content-addressed, and return the sha256 to cite: [VERIFIED: sha256: "verbatim quote"]. A fresh cached copy is served without a network request (refresh:true fetches again).', + 'Fetch a URL, cache its text content-addressed, and return the sha256 to cite: [VERIFIED: sha256: "quote, verbatim after documented normalisation"]. A fresh cached copy is served without a network request (refresh:true fetches again).', input: object( { url: { type: "string" }, @@ -223,7 +223,7 @@ export function researchTools(context: ResearchOpsContext): EsToolDef[] { { name: "es_research_audit", description: - "Audit a research artifact: every claim needs an epistemic tag, and every VERIFIED quote must appear verbatim in its cached source. prune:true moves failing claims to a 'Pruned claims' section.", + "Audit a research artifact: every claim needs an epistemic tag, and every VERIFIED quote must appear verbatim after documented normalisation in its cached source. prune:true moves failing claims to a 'Pruned claims' section.", input: object({ path: { type: "string", description: "Default .factory/research/REPORT.md" }, prune: { type: "boolean" }, diff --git a/packages/core/src/research/quote.ts b/packages/core/src/research/quote.ts index cb74ba7..5355068 100644 --- a/packages/core/src/research/quote.ts +++ b/packages/core/src/research/quote.ts @@ -1,10 +1,12 @@ -// Pure verbatim-quote verification. A quote matches when, after normalising -// whitespace and typographic punctuation (curly quotes, dashes, NBSP), it -// appears in the source exactly. An ellipsis ("..." or "…") splits a prose -// quote into fragments that must appear in order, and each fragment must be -// evidence on its own (MIN_QUOTE characters). Code excerpts are one contiguous -// span (verifySpan): "..." there is code, not an ellipsis. No fuzzy matching: -// a quote is either in the source or it is not. +// Quote verification that is verbatim after documented normalisation. A quote +// matches when, after normalising (NFKC, typography folds for curly quotes, +// dashes and spaces including NBSP, markdown link and emphasis stripping, +// whitespace collapsing), it appears in the source exactly. An ellipsis +// ("..." or "…") splits a prose quote into fragments that must appear in +// order, and each fragment must be evidence on its own (MIN_QUOTE +// characters). Code excerpts are one contiguous span (verifySpan): "..." +// there is code, not an ellipsis. No fuzzy matching: a normalised quote is +// either in the normalised source or it is not. /** Shortest quote, and shortest ellipsis fragment, that counts as evidence (normalised characters). */ export const MIN_QUOTE = 12 From e13a8f48c6451bf968925fcee70d929c69fc30a4 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:34:20 -0400 Subject: [PATCH 33/83] fix(remediation-audit/s4): backoff, auth-continuation and token-absence pins (Refs #253) --- packages/core/test/failover-gaps.test.ts | 277 +++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 packages/core/test/failover-gaps.test.ts diff --git a/packages/core/test/failover-gaps.test.ts b/packages/core/test/failover-gaps.test.ts new file mode 100644 index 0000000..2d22689 --- /dev/null +++ b/packages/core/test/failover-gaps.test.ts @@ -0,0 +1,277 @@ +// #253: failover proof gaps. Direct pins for `backoffFor` defaults, the +// `auth`-kind chain continuation (failover + cooldown, not a stop), and +// token-absence exhaustiveness: every gateway failure message, both +// tool-output paths and every cache write carry no token. Wire-level +// exclusivity (bytes on the socket) needs a live-network capture and stays +// out of scope for unit tests. +import { describe, expect, test } from "bun:test" +import { mkdtemp, readdir, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + backoffFor, + BackendError, + runFetchChain, + runSearchChain, + type SourceBackend, +} from "../src/research/backends/index.ts" +import { ScraperSwarmClient } from "../src/research/backends/scraper-swarm.ts" +import { researchTools } from "../src/research/ops.ts" +import { sha256 } from "../src/util/hash.ts" + +const TOKEN = "swarm_sec_probe_token" +const ENV = { ES_SCRAPER_SWARM_URL: "https://swarm.test", ES_SCRAPER_SWARM_TOKEN: TOKEN } + +describe("backoffFor defaults (backends/index.ts:159-162, #253.1)", () => { + test("rate_limited without Retry-After cools down 60s; an explicit value wins", () => { + expect(backoffFor(new BackendError("rate_limited", "slow down"))).toBe(60_000) + expect(backoffFor(new BackendError("rate_limited", "slow down", { retryAfterMs: 5_000 }))).toBe(5_000) + expect(backoffFor(new BackendError("rate_limited", "slow down", { retryAfterMs: 0 }))).toBe(0) + }) + + test("every other kind cools down 30s", () => { + for (const kind of ["unavailable", "auth", "blocked", "upstream", "timeout"] as const) + expect([kind, backoffFor(new BackendError(kind, "x"))]).toEqual([kind, 30_000]) + }) +}) + +describe("auth-kind chain continuation (#253.2)", () => { + const goodSearch: SourceBackend = { + id: "good", + capabilities: { search: true, fetch: false }, + available: () => true, + search: async (query: string) => [{ url: `https://good.test/${query}`, title: "Good", snippet: "ok" }], + } + const goodFetch: SourceBackend = { + id: "steady", + capabilities: { search: false, fetch: true }, + available: () => true, + fetch: async (url: string) => ({ url, text: "Steady page body with enough text.", contentType: "text/plain" }), + } + + test("a leading 401 fails over on search and cools the backend down", async () => { + const denied: SourceBackend = { + id: "gateway", + capabilities: { search: true, fetch: false }, + available: () => true, + search: async () => { + throw new BackendError("auth", "gateway refused: HTTP 401", { status: 401 }) + }, + } + const cooldowns = new Map() + const result = await runSearchChain([denied, goodSearch], "q", {}, { now: () => 1_000_000, cooldowns }) + expect(result.backend).toBe("good") + expect(result.attempts).toEqual([ + { backend: "gateway", ok: false, kind: "auth" }, + { backend: "good", ok: true }, + ]) + expect(cooldowns.get("gateway")).toEqual({ until: 1_030_000, kind: "auth" }) + }) + + test("a leading 403 fails over on fetch and cools the backend down", async () => { + const denied: SourceBackend = { + id: "gateway", + capabilities: { search: false, fetch: true }, + available: () => true, + fetch: async () => { + throw new BackendError("auth", "gateway refused: HTTP 403", { status: 403 }) + }, + } + const cooldowns = new Map() + const result = await runFetchChain([denied, goodFetch], "https://x.test/p", { + now: () => 2_000_000, + cooldowns, + }) + expect(result.backend).toBe("steady") + expect(result.attempts).toEqual([ + { backend: "gateway", ok: false, kind: "auth" }, + { backend: "steady", ok: true }, + ]) + expect(cooldowns.get("gateway")).toEqual({ until: 2_030_000, kind: "auth" }) + }) +}) + +describe("token absence exhaustiveness (#253.3)", () => { + const jsonRpc = (body: unknown, status = 200, headers: Record = {}) => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json", "x-swarm-contract": "1", ...headers }, + }) + const stubFetch = (respond: (url: string, init: RequestInit) => Response, seen: { auth: string }) => + (async (url: string, init?: RequestInit) => { + seen.auth = String((init?.headers as Record | undefined)?.Authorization ?? "") + return respond(String(url), init ?? {}) + }) as unknown as typeof fetch + const client = (respond: (url: string, init: RequestInit) => Response, seen: { auth: string }) => + new ScraperSwarmClient({ url: ENV.ES_SCRAPER_SWARM_URL, token: TOKEN, fetch: stubFetch(respond, seen) }) + const failure = (promise: Promise): Promise => + promise.then( + () => new Error("no-throw"), + (error: unknown) => error as Error, + ) + + test("the Authorization header carries the token on search and fetch", async () => { + const seen = { auth: "" } + const ok = { + jsonrpc: "2.0", + id: 1, + result: { structuredContent: { results: [{ url: "https://a.test", title: "A", snippet: "s" }] } }, + } + await client(() => jsonRpc(ok), seen).search("q", 1) + expect(seen.auth).toBe(`Bearer ${TOKEN}`) + const content = "# Title\n\nPage body text." + const page = { + jsonrpc: "2.0", + id: 1, + result: { + structuredContent: { content, content_sha256: sha256(content), title: "T", format: "markdown" }, + }, + } + await client(() => jsonRpc(page), seen).fetchPage("https://a.test") + expect(seen.auth).toBe(`Bearer ${TOKEN}`) + }) + + test("no gateway failure message carries the token", async () => { + const cases: Array<[string, (seen: { auth: string }) => Promise, string]> = [ + ["http-401", (seen) => client(() => jsonRpc({ detail: "nope" }, 401), seen).search("q", 1), "auth"], + ["http-403", (seen) => client(() => jsonRpc({ detail: "nope" }, 403), seen).fetchPage("https://a.test"), "auth"], + [ + "http-429", + (seen) => client(() => jsonRpc({ detail: "slow" }, 429, { "retry-after": "7" }), seen).search("q", 1), + "rate_limited", + ], + ["http-500", (seen) => client(() => jsonRpc({ detail: "boom" }, 500), seen).search("q", 1), "upstream"], + [ + "contract", + (seen) => client(() => jsonRpc({ jsonrpc: "2.0", id: 1, result: {} }, 200, { "x-swarm-contract": "9" }), seen).search("q", 1), + "upstream", + ], + [ + "ssrf-denied", + (seen) => + client( + () => + jsonRpc({ + jsonrpc: "2.0", + id: 1, + result: { isError: true, structuredContent: { code: "ssrf_denied", message: "SSRF denied" } }, + }), + seen, + ).fetchPage("https://a.test"), + "blocked", + ], + [ + "blocked-by-policy", + (seen) => + client( + () => + jsonRpc({ + jsonrpc: "2.0", + id: 1, + result: { isError: true, structuredContent: { code: "blocked_by_policy", message: "denied" } }, + }), + seen, + ).search("q", 1), + "blocked", + ], + [ + "upstream-timeout", + (seen) => + client( + () => + jsonRpc({ + jsonrpc: "2.0", + id: 1, + result: { isError: true, structuredContent: { code: "upstream_timeout", message: "slow" } }, + }), + seen, + ).search("q", 1), + "timeout", + ], + [ + "tool-upstream", + (seen) => + client( + () => jsonRpc({ jsonrpc: "2.0", id: 1, error: { code: -32602, message: "bad params" } }), + seen, + ).search("q", 1), + "upstream", + ], + [ + "unreachable", + (seen) => + new ScraperSwarmClient({ + url: ENV.ES_SCRAPER_SWARM_URL, + token: TOKEN, + fetch: (async () => { + seen.auth = "unreached" + throw new TypeError("fetch failed") + }) as unknown as typeof fetch, + }).search("q", 1), + "unavailable", + ], + ] + for (const [name, run, kind] of cases) { + const seen = { auth: "" } + const error = await failure(run(seen)) + if (!(error instanceof BackendError)) throw new Error(`${name}: expected a BackendError, got ${error}`) + expect([name, error.kind]).toEqual([name, kind]) + expect([name, error.message.includes(TOKEN)]).toEqual([name, false]) + } + }) + + test("search and fetch tool outputs and every cache write carry no token", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-token-")) + const state = await mkdtemp(path.join(tmpdir(), "es-token-state-")) + try { + // Drive the real client through the tools: search (no content cached) + // then fetch (page cached sealed as scraper-swarm). + const content = "# Recorded\n\nRecorded body text for the token probe." + const searchBody = { + jsonrpc: "2.0", + id: 1, + result: { structuredContent: { results: [{ url: "https://a.test/p", title: "A", snippet: "s" }] } }, + } + const pageBody = { + jsonrpc: "2.0", + id: 1, + result: { + structuredContent: { content, content_sha256: sha256(content), title: "P", format: "markdown" }, + }, + } + let n = 0 + const replay = (async () => { + n++ + return jsonRpc(n === 1 ? searchBody : pageBody) + }) as unknown as typeof fetch + const tools = researchTools({ + root: dir, + env: ENV, + stateDir: state, + policy: async () => ({ root: dir }), + fetch: replay, + }) + const search = tools.find((tool) => tool.name === "es_research_search")! + const searchText = await search.execute({ query: "token probe" }, { agent: "es-research-alpha" }) + expect(searchText.includes(TOKEN)).toBe(false) + // The search chain resolves the default order; scraper-swarm leads when + // configured, so this output names the serving backend. + expect(searchText).toContain("scraper-swarm results for") + const fetchTool = tools.find((tool) => tool.name === "es_research_fetch")! + const fetchText = await fetchTool.execute({ url: "https://a.test/p" }, { agent: "es-research-alpha" }) + expect(fetchText.includes(TOKEN)).toBe(false) + // Every cache write (meta JSON) is token-free. + const cacheDir = path.join(dir, ".factory/research/sources") + for (const name of await readdir(cacheDir)) { + if (!name.endsWith(".json") || name === "index.json") continue + const raw = await Bun.file(path.join(cacheDir, name)).text() + expect([name, raw.includes(TOKEN)]).toEqual([name, false]) + } + const indexRaw = await Bun.file(path.join(cacheDir, "index.json")).text() + expect(indexRaw.includes(TOKEN)).toBe(false) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) + } + }) +}) From 2c911a039765c7d050dcb5f097e83e68a81bd356 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:34:35 -0400 Subject: [PATCH 34/83] fix(remediation-audit/s4): direct export-deny shell pin next to retract (Refs #255) --- packages/core/test/policy.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/test/policy.test.ts b/packages/core/test/policy.test.ts index 4d29c1e..55142fe 100644 --- a/packages/core/test/policy.test.ts +++ b/packages/core/test/policy.test.ts @@ -282,6 +282,7 @@ describe("shell policy", () => { expect(shell(undefined, "cd x && epistemic-swarm waive lint/check").effect).toBe("deny") expect(shell("build", "es config set models.deep x/y").effect).toBe("deny") expect(shell("factory", `es research retract ${"c".repeat(64)} --event retracted`).effect).toBe("deny") + expect(shell("build", "es research export").effect).toBe("deny") expect(shell("build", "es config show").effect).toBe("allow") // Audit and scout runs drive a harness CLI: never from an agent's shell. expect(shell("build", "es audit src --max-usd 5").effect).toBe("deny") From 42d9788ed9709517c795175ad77078973a6c776a Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:35:56 -0400 Subject: [PATCH 35/83] fix(remediation-audit/s3): SPDX label corrections (Refs #264) --- packages/core/src/harvest/spdx.ts | 48 ++++++++++++++++++++++++----- packages/core/test/harvest.test.ts | 49 +++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/packages/core/src/harvest/spdx.ts b/packages/core/src/harvest/spdx.ts index 193592a..9c7f3b2 100644 --- a/packages/core/src/harvest/spdx.ts +++ b/packages/core/src/harvest/spdx.ts @@ -4,7 +4,11 @@ // bundled SPDX template (templates.ts): extra or changed terms — a rider like // the Commons Clause, the JSON licence's "Good, not Evil" — mean "unknown". // - Copyleft and source-available licences are recognised by their title and -// date lines (GPL-3.0 is not mistaken for AGPL because §13 mentions it). +// date lines (the earliest GPL-family title in the bytes decides among the +// GPL family: the GPL-3.0 tail names the Lesser GPL, which must not outrank +// an earlier GPL title). +// - NOTICE/PATENTS files are not evaluated as licences: a top-level NOTICE +// without a licence file warns about the blind spot instead of verifying. // - Every licence file in the root counts (LICENSE*, COPYING*, UNLICENSE…); // the most restrictive wins, and several permissive ones combine with AND. // - `SPDX-License-Identifier` headers are declarations by single files: they @@ -28,7 +32,8 @@ const FAMILIES: Readonly> = { "Apache-2.0": "permissive", "BSD-2-Clause": "permissive", "BSD-3-Clause": "permissive", - "BSD-4-Clause": "permissive", + // No BSD-4-Clause: the advertising clause is an obligation the whitelist + // does not cover, so it must read as unknown, never as safe (#264). ISC: "permissive", "0BSD": "permissive", Unlicense: "public-domain", @@ -145,6 +150,24 @@ interface Rule { // Lesser GPL and GPL-3.0 §13 the Affero GPL. Date lines catch texts that sit // under a project notice. const has = (text: string, ...phrases: string[]) => phrases.every((phrase) => text.includes(phrase)) + +/** + * The Lesser title plus a date line, but only when the Lesser title is the + * earliest GPL-family title in the bytes (#264): the GPL-3.0 tail ("use the + * GNU Lesser General Public License instead of this License") names it, and + * a later mention must not outrank an earlier GPL title. + */ +const lesserFirst = (body: string, date: string): boolean => { + if (!has(body, "gnu lesser general public license", date)) return false + const lesser = body.indexOf("gnu lesser general public license") + for (const title of [ + "gnu affero general public license", + "gnu library general public license", + "gnu general public license", + ]) + if (body.includes(title) && body.indexOf(title) < lesser) return false + return true +} const title = (phrase: string, spdx: string | ((head: string) => string), note?: string): Rule => ({ test: (head) => (head.includes(phrase) ? (typeof spdx === "string" ? spdx : spdx(head)) : undefined), ...(note ? { note } : {}), @@ -171,8 +194,8 @@ const RESTRICTIVE: readonly Rule[] = [ title("elastic license", "Elastic-2.0", SOURCE_AVAILABLE), // 2. Texts under a project notice: their distinctive date lines. { test: (_, body) => (has(body, "gnu affero general public license", "19 november 2007") ? "AGPL-3.0" : undefined) }, - { test: (_, body) => (has(body, "gnu lesser general public license", "february 1999") ? "LGPL-2.1" : undefined) }, - { test: (_, body) => (has(body, "gnu lesser general public license", "29 june 2007") ? "LGPL-3.0" : undefined) }, + { test: (_, body) => (lesserFirst(body, "february 1999") ? "LGPL-2.1" : undefined) }, + { test: (_, body) => (lesserFirst(body, "29 june 2007") ? "LGPL-3.0" : undefined) }, { test: (_, body) => (has(body, "gnu general public license", "29 june 2007") ? "GPL-3.0" : undefined) }, { test: (_, body) => (has(body, "gnu general public license", "june 1991") ? "GPL-2.0" : undefined) }, { test: (_, body) => (has(body, "mozilla public license", "version 2.0") ? "MPL-2.0" : undefined) }, @@ -271,6 +294,17 @@ export async function detectLicense( // 1. Every licence file in the root. const entries = await readdirImpl(dir).catch(() => [] as LicenseScanEntry[]) + // NOTICE/PATENTS files are not evaluated as licences (#264): when no + // licence file matched, their presence is reported on the finding so the + // blind spot is visible instead of silent. + const noticeFiles = entries + .filter((item) => !item.directory && /^(notice|patents)([-._][\w.-]+)?$/i.test(item.name)) + .map((item) => item.name) + .sort(byText) + const noticed = (note: string): string => + noticeFiles.length + ? `${note}; ${noticeFiles.join(", ")} present but not evaluated as a license (NOTICE obligations are a blind spot)` + : note const files: FileVerdict[] = [] for (const entry of entries .filter((item) => !item.directory && LICENSE_FILE.test(item.name)) @@ -302,7 +336,7 @@ export async function detectLicense( sha256: sha256(headers[0]!.text), confidence: "medium", verified: false, - note: `declared only by SPDX headers (${headers.length} file(s)); no licence text found`, + note: noticed(`declared only by SPDX headers (${headers.length} file(s)); no licence text found`), } } @@ -316,7 +350,7 @@ export async function detectLicense( file: manifest.file, confidence: "medium", verified: false, - note: "self-reported in a manifest; no license text found", + note: noticed("self-reported in a manifest; no license text found"), } return { @@ -325,7 +359,7 @@ export async function detectLicense( source: "manifest", confidence: "low", verified: false, - note: "no license file, header or manifest field found", + note: noticed("no license file, header or manifest field found"), } } diff --git a/packages/core/test/harvest.test.ts b/packages/core/test/harvest.test.ts index 6fd76db..93cec7e 100644 --- a/packages/core/test/harvest.test.ts +++ b/packages/core/test/harvest.test.ts @@ -83,7 +83,7 @@ describe("SPDX detection", () => { ["JSON", "unknown", "unknown", false], ["MIT-0", "unknown", "unknown", false], ["X11", "unknown", "unknown", false], - ["BSD-4-Clause", "BSD-4-Clause", "permissive", false], + ["BSD-4-Clause", "BSD-4-Clause", "unknown", false], ["BUSL-1.1", "BUSL-1.1", "unknown", false], ["Elastic-2.0", "Elastic-2.0", "unknown", false], ])("%s text → %s (%s, verified %p)", (file, spdx, family, verified) => { @@ -166,6 +166,53 @@ describe("SPDX detection", () => { const unknown = await detectLicense(root) expect(unknown).toMatchObject({ spdx: "unknown", verified: false }) }) + + test("a single file holding MIT+GPL reads as GPL-3.0, not LGPL (#264.1)", () => { + // The GPL-3.0 tail names the Lesser GPL, which used to win the date-line + // rules before the GPL-3.0 rule. The earliest license title in the bytes + // decides among the GPL family now. + expect(identifyLicenseText(`${MIT}\n${GPL3}`)).toMatchObject({ + spdx: "GPL-3.0", + family: "copyleft", + verified: true, + }) + expect(identifyLicenseText(`${GPL3}\n${MIT}`)).toMatchObject({ spdx: "GPL-3.0", family: "copyleft" }) + }) + + test("mixed and header-only findings still fail closed at the policy (#264.2)", async () => { + // MIT+GPL in one file labels GPL-3.0 (copyleft), and a header-only Apache + // project labels Apache-2.0 unverified: both end up clean-room, never + // depend/vendor. The enforced property is mixed/unverified → clean-room. + const mixed = { + ...identifyLicenseText(`${MIT}\n${GPL3}`)!, + source: "license-file" as const, + confidence: "high" as const, + } + expect(checkVerdict("depend", mixed, DEFAULT_LICENSE_WHITELIST).verdict).toBe("clean-room") + const header = await tmp("es-spdx-headeronly-") + await write(header, "src/a.ts", "// SPDX-License-Identifier: Apache-2.0\n export const x = 1\n") + const fromHeader = await detectLicense(header) + expect(fromHeader).toMatchObject({ spdx: "Apache-2.0", verified: false }) + expect(checkVerdict("depend", fromHeader, DEFAULT_LICENSE_WHITELIST).verdict).toBe("clean-room") + }) + + test("BSD-4-Clause labels unknown, not permissive (#264.3)", () => { + // The advertising clause is an obligation the whitelist does not cover, + // so the family must not read as safe next to verified:false. + expect(identifyLicenseText(fixture("BSD-4-Clause"))).toMatchObject({ + spdx: "BSD-4-Clause", + family: "unknown", + verified: false, + }) + }) + + test("a top-level NOTICE without a license file warns about the blind spot (#264.4)", async () => { + const root = await tmp("es-spdx-notice-") + await write(root, "NOTICE", "Copyright 2026 Example Corp. All rights reserved.\n") + const noticed = await detectLicense(root) + expect(noticed).toMatchObject({ spdx: "unknown", verified: false }) + expect(noticed.note ?? "").toContain("NOTICE") + }) }) describe("sources", () => { From d548bb9a319866e12a556bc6b525ce92dce5b2b7 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:36:33 -0400 Subject: [PATCH 36/83] fix(remediation-audit/s3): scrub model-supplied evidence in clean-room specs (Refs #267) --- packages/core/src/harvest/matrix.ts | 24 +++++++++++++++++++++++- packages/core/test/harvest.test.ts | 21 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/packages/core/src/harvest/matrix.ts b/packages/core/src/harvest/matrix.ts index 0147748..3ddbaeb 100644 --- a/packages/core/src/harvest/matrix.ts +++ b/packages/core/src/harvest/matrix.ts @@ -58,6 +58,27 @@ export interface MatrixBuild { }> } +/** + * Scrub model-supplied evidence before it reaches a clean-room spec (#267). + * The "do not read or copy" line in the spec template is procedure, not + * enforcement: without this, a harvester that pastes upstream code into + * `evidence` contaminates the spec mechanically. Fenced code blocks go + * entirely, inline spans keep their text but lose their ticks, stray ticks + * go, whitespace collapses, and the note is capped. + */ +export const EVIDENCE_MAX = 500 +export function sanitizeEvidence(evidence: string): string | undefined { + const text = evidence + .replace(/```[\s\S]*?(```|$)/g, " ") + .replace(/`([^`\n]*)`/g, "$1") + .replace(/`/g, "") + .replace(/\s+/g, " ") + .trim() + .slice(0, EVIDENCE_MAX) + .trimEnd() + return text || undefined +} + /** Apply the policy to every proposed cell and freeze the matrix. */ export function buildMatrix( candidates: readonly string[], @@ -74,6 +95,7 @@ export function buildMatrix( if (!profile) return { ...cell, verdict: "skip", downgraded: `candidate "${cell.candidate}" was not scanned; skipped` } const checked = checkVerdict(cell.verdict, profile.license, whitelist) + const clean = cell.evidence !== undefined ? sanitizeEvidence(cell.evidence) : undefined if (checked.downgraded) downgrades.push({ candidate: cell.candidate, @@ -85,7 +107,7 @@ export function buildMatrix( return { candidate: cell.candidate, verdict: checked.verdict, - ...(cell.evidence ? { evidence: cell.evidence } : {}), + ...(clean !== undefined ? { evidence: clean } : {}), ...(checked.attribution ? { attribution: checked.attribution } : {}), ...(checked.downgraded ? { downgraded: checked.downgraded } : {}), } diff --git a/packages/core/test/harvest.test.ts b/packages/core/test/harvest.test.ts index 93cec7e..41ed1bc 100644 --- a/packages/core/test/harvest.test.ts +++ b/packages/core/test/harvest.test.ts @@ -397,6 +397,27 @@ describe("the fail-closed licence policy", () => { expect(gpl).toMatchObject({ verdict: "clean-room", downgraded: expect.stringContaining("copyleft") }) expect(built.downgrades).toHaveLength(1) }) + + test("model-supplied evidence is scrubbed of code spans before it reaches the spec (#267)", () => { + // matrix.ts used to spread cell.evidence verbatim into the clean-room + // spec, so a harvester pasting upstream code contaminated it + // mechanically. The scrub is the enforcement behind the "do not copy" + // instruction. + const profiles = new Map([["mit", profileFor(license({}), "mit")]]) + const pasted = `rebuild the parser\n\`\`\`ts\nexport const stolen = 1\n\`\`\`\nsee \`internal()\` for details ${"x".repeat(600)}` + const built = buildMatrix( + ["mit"], + profiles, + [{ feature: "parse", cells: [{ candidate: "mit", verdict: "clean-room", evidence: pasted }] }], + DEFAULT_LICENSE_WHITELIST, + ) + const evidence = built.matrix.rows[0]!.cells[0]!.evidence! + expect(evidence).toContain("rebuild the parser") + expect(evidence).not.toContain("```") + expect(evidence).not.toContain("`") + expect(evidence).not.toContain("stolen") + expect(evidence.length).toBeLessThanOrEqual(500) + }) }) describe("the tool loop", () => { From 82b3ac5310ab93fe629dee58e4d22fe2b97e1bee Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:38:08 -0400 Subject: [PATCH 37/83] fix(remediation-audit/s3): domain-pack scoring integrity (Refs #252) --- packages/core/src/claims/constitution.ts | 39 ++++++++---- packages/core/src/research/tier.ts | 9 +-- packages/core/test/domainpacks.test.ts | 75 ++++++++++++++++++++++-- 3 files changed, 101 insertions(+), 22 deletions(-) diff --git a/packages/core/src/claims/constitution.ts b/packages/core/src/claims/constitution.ts index 5d05a5c..e8ce7c9 100644 --- a/packages/core/src/claims/constitution.ts +++ b/packages/core/src/claims/constitution.ts @@ -34,15 +34,25 @@ export const LEGACY_CONSTITUTION: Constitution = { retractionPolicy: "standard", } -export const constitutionFromPack = (pack: DomainPack): Constitution => ({ - tierWeights: { __default__: 1, ...pack.tierWeights }, - negBonus: pack.negBonus, - rejectPenalty: pack.rejectPenalty, - acceptThreshold: pack.acceptThreshold, - bannedDomains: pack.bannedDomains, - mandatoryTags: pack.mandatoryTags, - retractionPolicy: pack.retractionPolicy, -}) +export const constitutionFromPack = (pack: DomainPack): Constitution => { + // Tierless claims weigh at or below the lowest verified tier (#252): an + // unclassifiable host falling back to a full 1.0 outscores verified + // preprints and rewards citing worse sources. + const tiered = Object.entries(pack.tierWeights) + .filter(([tier]) => tier !== "__default__") + .map(([, weight]) => weight) + const floor = tiered.length ? Math.min(...tiered) : 1 + const fallback = pack.tierWeights.__default__ ?? 1 + return { + tierWeights: { ...pack.tierWeights, __default__: Math.min(fallback, floor) }, + negBonus: pack.negBonus, + rejectPenalty: pack.rejectPenalty, + acceptThreshold: pack.acceptThreshold, + bannedDomains: pack.bannedDomains, + mandatoryTags: pack.mandatoryTags, + retractionPolicy: pack.retractionPolicy, + } +} export const weightFor = (constitution: Constitution, tier: string | undefined): number => (tier !== undefined && constitution.tierWeights[tier] !== undefined @@ -165,7 +175,8 @@ export function claimVerdict(claim: ConstitutionClaim, options: ClaimVerdictOpti if (!nonEmpty(claim.query)) reasons.push("NEG_KNOWLEDGE_REQUIRES_QUERY: query missing") if (!nonEmpty(claim.finding)) reasons.push("NEG_KNOWLEDGE_REQUIRES_FINDING: finding missing") } - if (constitution.mandatoryTags.length && !claim.tag) reasons.push("MISSING_TAG: tag required by constitution") + if (constitution.mandatoryTags.length && !constitution.mandatoryTags.includes(claim.tag ?? "")) + reasons.push(`MISSING_TAG: tag '${claim.tag ?? ""}' is not one of the constitution's mandatory tags`) const url = claim.source?.url if (constitution.bannedDomains.length && typeof url === "string" && url) { const lower = url.toLowerCase() @@ -199,7 +210,8 @@ export interface ScoreBreakdown { * Verified claims contribute their tier weight when the constitution weights * any tier away from 1.0 (otherwise a flat 1.0, preserving legacy numbers); * rejected claims cost `rejectPenalty`; counted negative knowledge earns - * `negBonus`. Valid, deduplicated NK rows only, via the M1 port of the legacy + * `negBonus` and counts as assertions, so stuffing NK rows cannot inflate the + * score past the threshold (#252). Valid, deduplicated NK rows only, via the M1 port of the legacy * normalized key. The score is clamped to [0, 1] and rounded to 3 places. */ export function computeEpistemicScoreFromClaims( @@ -238,7 +250,10 @@ export function computeEpistemicScoreFromClaims( } else if (claim.tag === "INFERRED") counts.inferred++ else if (claim.tag === "HYPOTHESIS") counts.hypotheses++ } - const totalAssertions = Math.max(1, counts.verified + counts.inferred + counts.hypotheses + counts.rejected) + const totalAssertions = Math.max( + 1, + counts.verified + counts.inferred + counts.hypotheses + counts.rejected + negKnowledge, + ) const rawScore = (weightSum + constitution.negBonus * negKnowledge - constitution.rejectPenalty * counts.rejected) / totalAssertions const score = Math.max(0, Math.min(1, Math.round(rawScore * 1000) / 1000)) diff --git a/packages/core/src/research/tier.ts b/packages/core/src/research/tier.ts index f28e978..4e48f6f 100644 --- a/packages/core/src/research/tier.ts +++ b/packages/core/src/research/tier.ts @@ -49,12 +49,9 @@ export function classifySourceTier( if (domains.some((domain) => matches(host, domain.toLowerCase()))) return tier if (PREPRINT.some((domain) => matches(host, domain))) return "PREPRINT" if (PEER_REVIEWED.some((domain) => matches(host, domain))) return "PEER_REVIEWED" - if ( - (DOC_DOMAINS as readonly string[]).some((domain) => matches(host, domain)) || - host.startsWith("docs.") || - host.startsWith("developer.") - ) - return "TECHNICAL_DOCUMENTATION" + // Only curated documentation hosts count (#252): a bare docs./developer. + // prefix rule would let docs. claim curated weight. + if ((DOC_DOMAINS as readonly string[]).some((domain) => matches(host, domain))) return "TECHNICAL_DOCUMENTATION" if (host.endsWith(".gov")) return "PRIMARY_PRESS" return undefined } diff --git a/packages/core/test/domainpacks.test.ts b/packages/core/test/domainpacks.test.ts index acb0a70..2082acf 100644 --- a/packages/core/test/domainpacks.test.ts +++ b/packages/core/test/domainpacks.test.ts @@ -205,7 +205,7 @@ describe("the pack gates completeResearch", () => { test("a clean report passes under biopharma and records the pack and score", async () => { const factory = await toResearch("biopharma") await report( - "https://journal.example/clean", + "https://doi.org/10.1/clean", "The cohort met the primary efficacy bar in the per-protocol analysis.", "primary efficacy bar", ) @@ -274,7 +274,10 @@ describe("source tiers reach claims and the score (#113)", () => { classifySourceTier("https://www.nature.com/articles/x"), classifySourceTier("https://docs.python.org/3/"), classifySourceTier("https://developer.mozilla.org/en/"), + // A docs./developer. prefix on an unlisted host earns no tier (#252): + // the prefix rule would let docs. claim curated weight. classifySourceTier("https://docs.example.com/x"), + classifySourceTier("https://developer.example.com/y"), classifySourceTier("https://www.fda.gov/news/x"), classifySourceTier("https://journal.example/clean"), classifySourceTier("not a url"), @@ -288,7 +291,8 @@ describe("source tiers reach claims and the score (#113)", () => { "PEER_REVIEWED", "TECHNICAL_DOCUMENTATION", "TECHNICAL_DOCUMENTATION", - "TECHNICAL_DOCUMENTATION", + undefined, + undefined, "PRIMARY_PRESS", undefined, undefined, @@ -373,8 +377,13 @@ describe("source tiers reach claims and the score (#113)", () => { ) expect(peer.score).toBe(1) expect(pre.score).toBe(0.3) - // Unknown tiers fall back to __default__. - expect(computeEpistemicScoreFromClaims([{ id: "c".repeat(64), tag: "VERIFIED" }], constitution).score).toBe(1) + // Unknown tiers fall back to __default__, which a pack clamps to its + // lowest verified tier weight (#252): tierless must never outscore + // verified evidence. + expect(weightFor(constitution, undefined)).toBeLessThanOrEqual(0.3) + expect(computeEpistemicScoreFromClaims([{ id: "c".repeat(64), tag: "VERIFIED" }], constitution).score).toBe( + weightFor(constitution, undefined), + ) }) test("tiers do not change claim identity: PCRB export and verify still pass", async () => { @@ -395,3 +404,61 @@ describe("source tiers reach claims and the score (#113)", () => { expect(claimId({ ...without, tier: "PREPRINT" } as never)).toBe(without.id) }) }) + +describe("scoring integrity (#252)", () => { + test("tierless claims weigh at the lowest verified tier, never full weight", async () => { + // Unclassifiable hosts used to fall back to __default__ 1.0 and outscore + // verified preprints (0.3) under biopharma: a perverse incentive to cite + // worse sources. A pack now clamps __default__ to its lowest tier weight. + const pack = await loadDomainPack("biopharma") + const constitution = constitutionFromPack(pack) + const floor = Math.min( + ...Object.entries(pack.tierWeights) + .filter(([tier]) => tier !== "__default__") + .map(([, weight]) => weight), + ) + expect(weightFor(constitution, undefined)).toBe(floor) + const tierless = computeEpistemicScoreFromClaims([{ id: "c".repeat(64), tag: "VERIFIED" }], constitution) + const preprint = computeEpistemicScoreFromClaims( + [{ id: "d".repeat(64), tag: "VERIFIED", tier: "PREPRINT" }], + constitution, + ) + expect(tierless.score).toBe(floor) + expect(tierless.score).toBeLessThanOrEqual(preprint.score) + // The no-pack constitution is untouched (legacy numbers preserved). + expect(weightFor(LEGACY_CONSTITUTION, undefined)).toBe(1) + }) + + test("negative knowledge cannot inflate the score past the threshold", async () => { + // totalAssertions excluded negKnowledge while the numerator added + // negBonus per row: 10xPREPRINT went 0.3 -> 1.0 with 14 NK lines. NK rows + // now count as assertions too. + const pack = await loadDomainPack("biopharma") + const constitution = constitutionFromPack(pack) + const verified = Array.from({ length: 10 }, (_, index) => ({ + id: `v${index}`.padEnd(64, "0"), + tag: "VERIFIED" as const, + tier: "PREPRINT" as const, + })) + const nk = Array.from({ length: 14 }, (_, index) => ({ + id: `n${index}`.padEnd(64, "0"), + tag: "NEGATIVE_KNOWLEDGE" as const, + query: `query ${index}`, + finding: `finding ${index}`, + })) + const scored = computeEpistemicScoreFromClaims([...verified, ...nk], constitution) + expect(scored.breakdown.negativeKnowledgeCount).toBe(14) + expect(scored.score).toBeLessThan(constitution.acceptThreshold) + }) + + test("mandatory tags enforce membership, not just presence", async () => { + const pack = await biopharma() + const garbage = claimVerdict({ id: "g", tag: "GARBAGE" }, { constitution: pack }) + expect(garbage.verdict).toBe("REJECTED") + expect(garbage.reasons.some((reason) => reason.startsWith("MISSING_TAG"))).toBe(true) + expect( + claimVerdict({ id: "v", tag: "VERIFIED", source: { sha256: "a".repeat(64), quote: "q" } }, { constitution: pack }) + .verdict, + ).toBe("ACCEPTED") + }) +}) From 7b098e26c903819b774a752a332f26e81ba94864 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:38:58 -0400 Subject: [PATCH 38/83] fix(remediation-audit/s3): domain-pack coverage gaps (Refs #256) --- .../core/assets/domain-packs/biopharma.json | 3 +- packages/core/assets/domain-packs/legal.json | 3 +- packages/core/assets/domain-packs/quant.json | 3 +- packages/core/src/research/tier.ts | 5 +- packages/core/src/schema/domain.ts | 5 -- packages/core/test/domainpacks.test.ts | 70 +++++++++++++++++++ 6 files changed, 76 insertions(+), 13 deletions(-) diff --git a/packages/core/assets/domain-packs/biopharma.json b/packages/core/assets/domain-packs/biopharma.json index e1d0561..5b619e7 100644 --- a/packages/core/assets/domain-packs/biopharma.json +++ b/packages/core/assets/domain-packs/biopharma.json @@ -6,8 +6,7 @@ "PEER_REVIEWED": 1.0, "TECHNICAL_DOCUMENTATION": 0.8, "PRIMARY_PRESS": 0.7, - "PREPRINT": 0.3, - "SECONDARY_MEDIA": 0.2 + "PREPRINT": 0.3 }, "negBonus": 0.5, "rejectPenalty": 2.5, diff --git a/packages/core/assets/domain-packs/legal.json b/packages/core/assets/domain-packs/legal.json index 889abe7..5bdfc96 100644 --- a/packages/core/assets/domain-packs/legal.json +++ b/packages/core/assets/domain-packs/legal.json @@ -6,8 +6,7 @@ "PEER_REVIEWED": 0.8, "TECHNICAL_DOCUMENTATION": 0.7, "PRIMARY_PRESS": 1.0, - "PREPRINT": 0.2, - "SECONDARY_MEDIA": 0.3 + "PREPRINT": 0.2 }, "negBonus": 0.5, "rejectPenalty": 2.5, diff --git a/packages/core/assets/domain-packs/quant.json b/packages/core/assets/domain-packs/quant.json index f7a502e..565aafc 100644 --- a/packages/core/assets/domain-packs/quant.json +++ b/packages/core/assets/domain-packs/quant.json @@ -6,8 +6,7 @@ "PEER_REVIEWED": 0.9, "TECHNICAL_DOCUMENTATION": 1.0, "PRIMARY_PRESS": 0.6, - "PREPRINT": 0.5, - "SECONDARY_MEDIA": 0.2 + "PREPRINT": 0.5 }, "negBonus": 0.75, "rejectPenalty": 3.0, diff --git a/packages/core/src/research/tier.ts b/packages/core/src/research/tier.ts index 4e48f6f..3240633 100644 --- a/packages/core/src/research/tier.ts +++ b/packages/core/src/research/tier.ts @@ -2,7 +2,8 @@ // source sits on (preprint, peer-reviewed, docs, press), so domain packs can // weight verified claims by source quality. Rules are host-based and // auditable; unknown hosts stay tierless and fall back to `__default__`. -// Packs may add a `tierDomains` override, which wins over the built-ins. +// Callers may pass a `tierDomains` override map, which wins over the +// built-ins. import { DOC_DOMAINS } from "./url.ts" const PREPRINT = ["arxiv.org", "biorxiv.org", "medrxiv.org", "ssrn.com"] as const @@ -36,7 +37,7 @@ const matches = (host: string, domain: string): boolean => host === domain || ho /** * Classify a source URL into a tier, or undefined when no rule fires. - * `overrides` (a pack's `tierDomains`) is checked first and wins. + * `overrides` (a caller's `tierDomains` map) is checked first and wins. */ export function classifySourceTier( url: string, diff --git a/packages/core/src/schema/domain.ts b/packages/core/src/schema/domain.ts index 6be10df..9dafc41 100644 --- a/packages/core/src/schema/domain.ts +++ b/packages/core/src/schema/domain.ts @@ -24,11 +24,6 @@ export const DomainPackSchema = z bannedDomains: z.array(z.string().min(1)).default([]).describe("Source URLs containing any of these are rejected."), mandatoryTags: z.array(ClaimTagSchema).default([]).describe("Every claim must carry one of these tags."), retractionPolicy: RetractionPolicySchema.default("standard"), - /** Per-tier host overrides for source classification (checked before the built-in table). */ - tierDomains: z - .record(z.string(), z.array(z.string().min(1))) - .optional() - .describe("Pack host overrides by tier for source classification."), /** Legacy metadata; carried so ported pack files round-trip. */ exportTemplate: z.string().optional(), }) diff --git a/packages/core/test/domainpacks.test.ts b/packages/core/test/domainpacks.test.ts index 2082acf..8c237ba 100644 --- a/packages/core/test/domainpacks.test.ts +++ b/packages/core/test/domainpacks.test.ts @@ -240,6 +240,18 @@ describe("the pack gates completeResearch", () => { await expect(unpacked.completeResearch("factory")).resolves.toBeDefined() }) + test("a below-threshold report is refused at completion, naming the score (#256.2)", async () => { + // A preprint-only report passes every claim verdict under biopharma but + // scores 0.3 against its 0.7 threshold: the machine.ts refusal path. + const factory = await toResearch("biopharma") + await report( + "https://arxiv.org/abs/1234", + "A model predicts surrogate endpoints in this preprint.", + "predicts surrogate endpoints", + ) + await expect(factory.completeResearch("factory")).rejects.toThrow("below its threshold") + }) + test("an unknown configured pack fails closed at completion time", async () => { const factory = await toResearch("no-such-pack") await report("https://journal.example/clean", "A finding rests on a cached source.", "cached source") @@ -462,3 +474,61 @@ describe("scoring integrity (#252)", () => { ).toBe("ACCEPTED") }) }) + +describe("pack coverage (#256)", () => { + test("each pack pins its thresholds, weights and penalties", async () => { + // A typo or deletion in a pack file stays green under mechanism-only + // tests, so the values are pinned exactly (SECONDARY_MEDIA was removed: + // the classifier never emits it, #256.3). + const biopharma = await loadDomainPack("biopharma") + expect(biopharma.tierWeights).toEqual({ + __default__: 1, + PEER_REVIEWED: 1, + TECHNICAL_DOCUMENTATION: 0.8, + PRIMARY_PRESS: 0.7, + PREPRINT: 0.3, + }) + expect([biopharma.acceptThreshold, biopharma.negBonus, biopharma.rejectPenalty]).toEqual([0.7, 0.5, 2.5]) + expect(biopharma.retractionPolicy).toBe("zero_tolerance") + const quant = await loadDomainPack("quant") + expect(quant.tierWeights).toEqual({ + __default__: 1, + PEER_REVIEWED: 0.9, + TECHNICAL_DOCUMENTATION: 1, + PRIMARY_PRESS: 0.6, + PREPRINT: 0.5, + }) + expect([quant.acceptThreshold, quant.negBonus, quant.rejectPenalty]).toEqual([0.7, 0.75, 3]) + const legal = await loadDomainPack("legal") + expect(legal.tierWeights).toEqual({ + __default__: 1, + PEER_REVIEWED: 0.8, + TECHNICAL_DOCUMENTATION: 0.7, + PRIMARY_PRESS: 1, + PREPRINT: 0.2, + }) + expect([legal.acceptThreshold, legal.negBonus, legal.rejectPenalty]).toEqual([0.65, 0.5, 2.5]) + }) + + test("quant and legal ban their own mirrors, not just biopharma's", async () => { + // Only biopharma's banned domains appeared in fixtures; per-pack + // behavior is asserted directly here. + const quant = constitutionFromPack(await loadDomainPack("quant")) + const legal = constitutionFromPack(await loadDomainPack("legal")) + const bio = await biopharma() + const vendorMirror = { + id: "q", + tag: "VERIFIED", + source: { sha256: "b".repeat(64), quote: "backtest", url: "https://vendor-whitepaper-mirror.example/p" }, + } + expect(claimVerdict(vendorMirror, { constitution: quant }).verdict).toBe("REJECTED") + expect(claimVerdict(vendorMirror, { constitution: bio }).verdict).toBe("ACCEPTED") + const sponsored = { + id: "l", + tag: "VERIFIED", + source: { sha256: "c".repeat(64), quote: "holding", url: "https://undisclosed-sponsored-content.example/x" }, + } + expect(claimVerdict(sponsored, { constitution: legal }).verdict).toBe("REJECTED") + expect(claimVerdict(sponsored, { constitution: bio }).verdict).toBe("ACCEPTED") + }) +}) From 56c0e0a6b98fff83db778ebd1462d353ee7b9bcf Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:40:50 -0400 Subject: [PATCH 39/83] fix(remediation-audit/s1): name drift rule + summary pointer in halt (Refs #242) --- packages/core/src/factory/machine.ts | 5 ++- packages/core/test/gates-drift.test.ts | 44 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 packages/core/test/gates-drift.test.ts diff --git a/packages/core/src/factory/machine.ts b/packages/core/src/factory/machine.ts index b6fc75b..5308fe6 100644 --- a/packages/core/src/factory/machine.ts +++ b/packages/core/src/factory/machine.ts @@ -319,7 +319,10 @@ export class Factory { // (CI-only es_build_start refusal). Real transitions still enforce it. if (!options.skipControl) { const control = await verifyControl(this.root) - if (!control.clean) return halt(`control files changed outside a human action: ${control.violations.join("; ")}`) + if (!control.clean) + return halt( + `control files changed outside a human action (integrity/control-drift): ${control.violations.join("; ")}. The structured finding is in .factory/runs/${state.runId}/gates/*/summary.json; a human must review and rebaseline.`, + ) } if (state.spendCeilingUSD !== undefined && state.spend.usd >= state.spendCeilingUSD) return halt( diff --git a/packages/core/test/gates-drift.test.ts b/packages/core/test/gates-drift.test.ts new file mode 100644 index 0000000..44c5a10 --- /dev/null +++ b/packages/core/test/gates-drift.test.ts @@ -0,0 +1,44 @@ +// Control-drift visibility (#242): when a tampered control file halts the +// run, the halt names the gate rule (integrity/control-drift) and points at +// the gate summary holding the structured finding — a seat debugging the halt +// gets more than an opaque "control files changed". +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { writeFile } from "node:fs/promises" +import path from "node:path" +import { recordApproval } from "../src/approval/index.ts" +import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { Factory, FactoryHalted } from "../src/factory/index.ts" +import { type Fixture, frontier, gitRepo } from "./helpers.ts" + +const PASSPHRASE = "test-passphrase-1234" +let fx: Fixture +let signer: HumanSigner +let factory: Factory + +beforeEach(async () => { + fx = await gitRepo("es-gdrift-") + await sealHumanKey(PASSPHRASE, fx.state) + signer = await unlockHumanKey(PASSPHRASE, fx.state) + factory = new Factory(fx.root, { + gates: async () => ({ passed: true, findings: [], summary: "green" }), + stateDir: fx.state, + }) +}) +afterEach(() => fx.cleanup()) + +describe("drift halt visibility", () => { + test("a drift halt names integrity/control-drift and the gate summary holding the finding", async () => { + await factory.begin("human:tester") + await factory.writeFrontier("grill", frontier()) + await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer }) + await writeFile(path.join(fx.root, ".factory/gates.json"), '{"commands":[]}\n') + const error = await factory.recordGateRun("es-programmer", { passed: false, findings: [], summary: "red" }).then( + () => undefined, + (failure: unknown) => failure, + ) + expect(error).toBeInstanceOf(FactoryHalted) + expect(String(error)).toContain("integrity/control-drift") + expect(String(error)).toContain("summary.json") + expect((await factory.read())?.stage).toBe("HALTED") + }) +}) From 98ceca481ca98106ddca152578ebd2925a8386df Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:40:50 -0400 Subject: [PATCH 40/83] fix(remediation-audit/s1): secret-scan mtime cache + gate proof-gap tests (Refs #239) --- packages/core/src/gates/engine.ts | 7 +- packages/core/src/gates/secrets.ts | 47 +++++ packages/core/test/gates.test.ts | 204 +++++++++++++++++++++ packages/opencode/test/fires-gates.test.ts | 75 ++++++++ 4 files changed, 328 insertions(+), 5 deletions(-) diff --git a/packages/core/src/gates/engine.ts b/packages/core/src/gates/engine.ts index cc9a6a6..04a52b6 100644 --- a/packages/core/src/gates/engine.ts +++ b/packages/core/src/gates/engine.ts @@ -31,7 +31,7 @@ import { validateArtifacts } from "./artifacts.ts" import { checkBudgets, isTestFile } from "./budgets.ts" import { detectGates } from "./detect.ts" import { parseOutput } from "./parsers.ts" -import { scanForSecrets } from "./secrets.ts" +import { scanFileForSecrets } from "./secrets.ts" import { DEPENDENCY_FILES, gitleaks, osvFindings } from "./security.ts" import { isWaived, loadWaivers } from "./waivers.ts" @@ -402,10 +402,7 @@ export async function runGates(options: RunGatesOptions): Promise { if (config.security.secrets) await timed("secrets", async () => { const out: GateFinding[] = [] - for (const file of present) { - const text = await readFile(path.join(dir, file), "utf8").catch(() => "") - if (text.length <= 2_000_000) out.push(...scanForSecrets(text, file)) - } + for (const file of present) out.push(...(await scanFileForSecrets(dir, file))) return out }) if (config.security.gitleaks === "auto") await timed("gitleaks", () => gitleaks(dir, present.slice(0, 200))) diff --git a/packages/core/src/gates/secrets.ts b/packages/core/src/gates/secrets.ts index 5e05770..7f504c1 100644 --- a/packages/core/src/gates/secrets.ts +++ b/packages/core/src/gates/secrets.ts @@ -1,5 +1,7 @@ // Built-in secret scanner (always on, no external binary). High-precision // patterns are errors; the generic assignment heuristic is a warning. +import { open } from "node:fs/promises" +import path from "node:path" import type { GateFinding } from "../schema/gates.ts" const PATTERNS: ReadonlyArray<{ rule: string; regex: RegExp; description: string; severity?: "warning" }> = [ @@ -56,3 +58,48 @@ export function scanForSecrets(content: string, file: string): GateFinding[] { }) return findings } + +// Per-path cache keyed by mtime+size (mirrors structure/graph.ts), so repeated +// gate runs re-read only changed files. The scan itself is deliberately NOT +// count-capped the way gitleaks is: a cap would fail open, silently skipping a +// secret past the cap, while the cache bounds the repeated-run cost instead. +const fileCache = new Map() +let cacheHits = 0 + +/** + * Scan one file for secrets, cached by mtime+size. Missing, non-regular and + * oversized (>2MB, as before) files yield no findings, as do binary files. + */ +export async function scanFileForSecrets(dir: string, file: string): Promise { + const absolute = path.join(dir, file) + // Open once and stat the descriptor, so the size/mtime check and the read + // cannot race with a concurrent replacement of the path. + const handle = await open(absolute, "r").catch(() => undefined) + if (!handle) return [] + try { + const info = await handle.stat() + if (!info.isFile() || info.size > 2_000_000) return [] + const cached = fileCache.get(absolute) + if (cached && cached.mtimeMs === info.mtimeMs && cached.size === info.size) { + cacheHits += 1 + return cached.findings + } + const findings = scanForSecrets(await handle.readFile("utf8"), file) + if (fileCache.size > 50_000) fileCache.clear() + fileCache.set(absolute, { mtimeMs: info.mtimeMs, size: info.size, findings }) + return findings + } finally { + await handle.close() + } +} + +/** Cache observability: entries held and hits served since the last clear. */ +export function secretScanStats(): { entries: number; hits: number } { + return { entries: fileCache.size, hits: cacheHits } +} + +/** For tests: forget cached findings and reset the hit count. */ +export function clearSecretScanCache(): void { + fileCache.clear() + cacheHits = 0 +} diff --git a/packages/core/test/gates.test.ts b/packages/core/test/gates.test.ts index 2dd1ea8..cc058a9 100644 --- a/packages/core/test/gates.test.ts +++ b/packages/core/test/gates.test.ts @@ -3,15 +3,21 @@ import { mkdir, writeFile } from "node:fs/promises" import path from "node:path" import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" import { + checkBudgets, commandSetHash, complexity, detectGates, + formatReport, + type GateReport, GatesConfigSchema, + gitleaks, + installed, loadGatesConfig, parseOutput, recordWaiver, runGates, trustProject, + validateArtifacts, } from "../src/index.ts" import { bwrapAvailable, resetBwrapProbe, run } from "../src/util/proc.ts" import { type Fixture, gitRepo } from "./helpers.ts" @@ -448,3 +454,201 @@ describe("parsers", () => { expect(scores.find((item) => item.name === "typed")?.score).toBe(2) }) }) + +describe("built-in secret scan cache (#239)", () => { + test("a rescan re-reads only changed files: hits are counted, edits re-read, no count cap", async () => { + const { clearSecretScanCache, scanFileForSecrets, secretScanStats } = await import("../src/index.ts") + clearSecretScanCache() + await write("src/a.ts", `export const token = "${FAKE_TOKEN}"\n`) + await write("src/b.ts", "export const b = 1\n") + const first = await scanFileForSecrets(fx.root, "src/a.ts") + expect(first.some((finding) => finding.rule === "security/secret-github-token")).toBe(true) + expect(await scanFileForSecrets(fx.root, "src/b.ts")).toEqual([]) + // Unchanged files are cache hits: identical findings, hits counted. + const before = secretScanStats() + expect(await scanFileForSecrets(fx.root, "src/a.ts")).toEqual(first) + expect(await scanFileForSecrets(fx.root, "src/b.ts")).toEqual([]) + expect(secretScanStats().hits).toBe(before.hits + 2) + // An edit re-reads: the finding disappears after the secret is removed. + await write("src/a.ts", 'export const token = "rotated"\n') + expect(await scanFileForSecrets(fx.root, "src/a.ts")).toEqual([]) + }) +}) + +describe("proof gaps (#239)", () => { + test("format is a failing arm, not detection-only: unformatted files fail, formatted files pass", async () => { + await write("biome.json", "{}") + await write("src/a.ts", "export const x=1\n") + const cfg = config([ + { + id: "format", + kind: "format", + command: `${BIOME} format --reporter=json`, + parser: "biome", + touchedArgs: true, + extensions: [".ts"], + }, + ]) + await trust(cfg) + // The engine arm, not the sandbox: pin bwrap away so the command runs + // directly (containers without overlayfs cannot sandbox; see #239). + resetBwrapProbe(false) + try { + const red = await gates({ touched: ["src/a.ts"], config: cfg }) + expect(red.checks.find((check) => check.id === "format")?.status).toBe("fail") + expect(red.findings.find((finding) => finding.rule === "format/biome")).toMatchObject({ + file: "src/a.ts", + severity: "error", + }) + await write("src/a.ts", "export const x = 1;\n") + const green = await gates({ touched: ["src/a.ts"], config: cfg }) + expect(green.findings.some((finding) => finding.rule === "format/biome")).toBe(false) + } finally { + resetBwrapProbe() + } + }) + + test("typecheck runs at gate level: errors fail, clean files pass, present tools are not missing", async () => { + const TSC = path.resolve(import.meta.dir, "../../../node_modules/.bin/tsc") + await write("tsconfig.json", JSON.stringify({ compilerOptions: { strict: true, noEmit: true }, include: ["src"] })) + await write("src/a.ts", 'export const x: number = "nope"\n') + const cfg = config([ + { id: "typecheck", kind: "typecheck", command: `${TSC} --noEmit -p tsconfig.json`, parser: "tsc" }, + ]) + await trust(cfg) + resetBwrapProbe(false) + try { + const red = await gates({ touched: ["src/a.ts"], config: cfg }) + expect(red.checks.find((check) => check.id === "typecheck")?.status).toBe("fail") + expect(red.findings.some((finding) => finding.file === "src/a.ts" && finding.check === "typecheck")).toBe(true) + await write("src/a.ts", "export const x: number = 1\n") + const green = await gates({ touched: ["src/a.ts"], config: cfg }) + expect(green.checks.find((check) => check.id === "typecheck")?.status).toBe("pass") + expect(green.findings.some((finding) => finding.rule === "gates/tool-missing")).toBe(false) + expect(green.findings.some((finding) => finding.rule === "gates/sandbox-missing")).toBe(false) + } finally { + resetBwrapProbe() + } + }) + + test("a failing test command fails the gate (fail arm, not pass-only)", async () => { + const cfg = config([{ id: "test", kind: "test", command: "bun -e 'process.exit(1)'", parser: "generic" }]) + await trust(cfg) + resetBwrapProbe(false) + try { + const report = await gates({ scope: "full", config: cfg }) + expect(report.checks.find((check) => check.id === "test")?.status).toBe("fail") + expect(report.testsFailed).toBe(true) + expect(report.findings.some((finding) => finding.rule === "test/command-failed")).toBe(true) + } finally { + resetBwrapProbe() + } + }) + + test("budget pass arms: small, simple, tested changes are quiet; diff-size has both arms", async () => { + await write("src/a.ts", "export const a = 1\n") + await write("src/a.test.ts", 'import { a } from "./a"\n') + const quiet = await gates({ + touched: ["src/a.ts", "src/a.test.ts"], + config: config([], { budgets: { maxFileLines: 20, maxComplexity: 10 } }), + }) + for (const rule of ["budget/file-length", "budget/complexity", "budget/test-with-behavior"]) + expect([rule, quiet.findings.some((finding) => finding.rule === rule)]).toEqual([rule, false]) + const budgets = { + maxFileLines: 800, + maxComplexity: 15, + maxDiffLines: 10, + requireDepJustification: false, + requireTestWithBehavior: false, + } + const over = await checkBudgets({ dir: fx.root, files: [], budgets, diff: { added: 6, removed: 5 } }) + expect(over.find((finding) => finding.rule === "budget/diff-size")?.severity).toBe("error") + const under = await checkBudgets({ dir: fx.root, files: [], budgets, diff: { added: 5, removed: 5 } }) + expect(under.some((finding) => finding.rule === "budget/diff-size")).toBe(false) + }) + + test("a phase that touches only source passes tool-config (pass arm)", async () => { + const base = await commit("base") + await write("src/a.ts", "export const a = 1\n") + const report = await gates({ scope: "full", base, phaseId: "p1", config: config([]) }) + expect(report.findings.some((finding) => finding.rule === "integrity/tool-config-changed")).toBe(false) + }) + + test("a valid gates.json loads with no problem (pass arm)", async () => { + await write(".factory/gates.json", JSON.stringify({ commands: [] })) + const { config: loaded, problem } = await loadGatesConfig(fx.root) + expect(problem).toBeUndefined() + expect(loaded.commands).toEqual([]) + }) + + test("artifact validation flags a malformed frontier and passes when artifacts are absent", async () => { + expect(await validateArtifacts(fx.root, fx.state)).toEqual([]) + await write(".factory/frontier.json", "{}") + const findings = await validateArtifacts(fx.root, fx.state) + expect(findings.find((finding) => finding.rule === "schema/frontier")?.file).toBe(".factory/frontier.json") + }) + + test("formatReport renders the summary, the findings and the overflow pointer", () => { + const report: GateReport = { + passed: false, + findings: [ + { + file: "src/a.ts", + line: 2, + rule: "lint/suspicious/noDebugger", + severity: "error", + message: "No debugger", + check: "lint", + }, + ], + totalFindings: 3, + waived: 0, + checks: [{ id: "lint", status: "fail", durationMs: 1, findings: 3 }], + summary: "Gates failed: 3 error(s)", + failedTests: [], + testsFailed: false, + logDir: "/tmp/logs", + } + const text = formatReport(report) + expect(text).toContain("Gates failed: 3 error(s)") + expect(text).toContain("src/a.ts:2 lint/suspicious/noDebugger") + expect(text).toContain("… 2 more in /tmp/logs/summary.json") + }) + + test("the lsp gate reports error diagnostics and passes on clean files", async () => { + await write("src/a.ts", "export const a = 1\n") + const red = await gates({ + touched: ["src/a.ts"], + config: config([]), + lsp: { + diagnostics: async () => [ + { severity: 1, message: "boom", source: "ts", range: { start: { line: 0, character: 0 } } }, + ], + }, + }) + expect(red.findings.find((finding) => finding.rule === "lsp/ts")).toMatchObject({ file: "src/a.ts", line: 1 }) + const green = await gates({ + touched: ["src/a.ts"], + config: config([]), + lsp: { diagnostics: async () => [] }, + }) + expect(green.findings.some((finding) => finding.check === "lsp")).toBe(false) + }) + + test("gitleaks: empty input is a skip; without the binary it never runs", async () => { + expect(await gitleaks(fx.root, [])).toBeUndefined() + if (!installed("gitleaks")) expect(await gitleaks(fx.root, ["README.md"])).toBeUndefined() + }) + + test("control drift is an integrity/control-drift gate finding, not just a halt", async () => { + const { rebaseline } = await import("../src/trust/index.ts") + await rebaseline(fx.root, "human:tester") + await write(".factory/gates.json", '{"commands":[]}\n') + const report = await gates({ scope: "full", config: config([]) }) + expect(report.findings.find((finding) => finding.rule === "integrity/control-drift")).toMatchObject({ + file: ".factory", + severity: "error", + check: "integrity", + }) + }) +}) diff --git a/packages/opencode/test/fires-gates.test.ts b/packages/opencode/test/fires-gates.test.ts index 48f9223..a945ca8 100644 --- a/packages/opencode/test/fires-gates.test.ts +++ b/packages/opencode/test/fires-gates.test.ts @@ -303,6 +303,81 @@ describe("factory-gate", () => { }) }) +describe("gates-seeded (#239)", () => { + let h: Harness + let state: string + let factory: () => Factory + let signer: HumanSigner + beforeAll(async () => { + ;({ h, state, factory, signer } = await bootHost("es-gates-seeded-")) + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("es_gates_run surfaces a seeded secret violation to the programmer seat", async () => { + // Own host and pipeline to BUILD: never depends on the factory-gate walk. + await factory().provisionRun("tester", 10) + const layout = factoryLayout(h.directory) + await factory().writeFrontier("grill", { + version: "1.1", + idea: "a session handoff feature", + spendCeiling: { currency: "USD", maxAmount: 10 }, + round: 1, + settled: true, + nodes: [{ id: "scope", question: "Which harness first?", status: "settled", answer: "opencode", round: 1 }], + }) + await approveStage(h.directory, { + stage: "frontier", + channel: "cli", + approvedBy: "tester", + signer, + stateDir: state, + }) + const cache = new SourceCache(researchSourcesDir(h.directory), state) + const source = await cache.put({ + url: "https://example.test/handoff", + text: "Session handoff writes a file the next agent reads.", + provider: "fetch", + }) + await write( + h.directory, + path.relative(h.directory, layout.researchReport), + `- Handoff is file-based [VERIFIED: sha256:${source.meta.sha256} "file the next agent reads"]\n`, + ) + await factory().completeResearch("factory") + await write( + h.directory, + ".factory/roadmap.json", + JSON.stringify({ version: 1, title: "Handoff", phases: [{ id: "alpha", title: "Phase alpha" }] }, null, 2), + ) + await write( + h.directory, + ".factory/specs/alpha/GOAL.md", + stringifyFrontmatter( + { + phase: "alpha", + title: "Phase alpha", + acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }], + }, + "Implement alpha as a vertical slice.\n", + ), + ) + await approveStage(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer, stateDir: state }) + const started = await h.run(call("es_build_start"), { agent: "factory" }) + expect(started.tools[0]?.status).toBe("completed") + await write( + h.directory, + ".factory/worktrees/alpha/src/secret.ts", + `export const token = "ghp_Ab3dE6gH9jK2mN5pQ8sT1vW4yZ7bC0eF3hJ6"\n`, + ) + const run = await h.run(call("es_gates_run", { files: ["src/secret.ts"] }), { agent: "es-programmer" }) + expect(run.tools[0]?.status).toBe("completed") + expect(run.tools[0]?.text).toContain("security/secret-github-token") + }) +}) + describe("darkharvest-fires", () => { let h: Harness let state: string From dd36e36dea92c31d27fee469f08cab953f5f4697 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:41:18 -0400 Subject: [PATCH 41/83] fix(remediation-audit/s3): close scout advisory evasion paths (Refs #257) --- packages/core/src/schema/scout.ts | 15 ++++ packages/core/src/scout/ops.ts | 69 ++++++++++++++++-- packages/core/src/scout/report.ts | 25 ++++++- packages/core/src/scout/store.ts | 21 ++++-- packages/core/test/scout.test.ts | 114 +++++++++++++++++++++++++++++- 5 files changed, 227 insertions(+), 17 deletions(-) diff --git a/packages/core/src/schema/scout.ts b/packages/core/src/schema/scout.ts index 0d21e7c..f9e1adf 100644 --- a/packages/core/src/schema/scout.ts +++ b/packages/core/src/schema/scout.ts @@ -37,6 +37,21 @@ export const ScoutAdvisorySchema = z.object({ }) export type ScoutAdvisory = z.infer +/** The OSV coordinates a candidate's advisories were actually queried at (#257). */ +export const ScoutAdvisoryQuerySchema = z.object({ + ecosystem: z.string(), + package: z.string(), + version: z.string().optional(), +}) +export type ScoutAdvisoryQuery = z.infer + +/** The per-candidate advisories artifact: what was queried, and what came back. */ +export const ScoutAdvisoriesFileSchema = z.object({ + query: ScoutAdvisoryQuerySchema, + advisories: z.array(ScoutAdvisorySchema), +}) +export type ScoutAdvisoriesFile = z.infer + export const ScoutAssessmentSchema = z.object({ candidate: z.string(), stars: z.number().int().nonnegative().optional(), diff --git a/packages/core/src/scout/ops.ts b/packages/core/src/scout/ops.ts index 6b16b4d..96d6adb 100644 --- a/packages/core/src/scout/ops.ts +++ b/packages/core/src/scout/ops.ts @@ -18,7 +18,7 @@ import { DEFAULT_LICENSE_WHITELIST, type HarvestProfile } from "../schema/harves import { SCOUT_VERSION, type ScoutAdvisory, type ScoutAssessment, ScoutAssessmentSchema } from "../schema/scout.ts" import { relativeTo } from "../trust/paths.ts" import { atomicWrite } from "../util/fs.ts" -import { renderScoutReport } from "./report.ts" +import { type AdvisoryCheck, renderScoutReport } from "./report.ts" import { readAssessments, readScoutAdvisories, @@ -256,7 +256,29 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { throw new ToolRefusal( `${candidate.id} is not a registry package: give the ecosystem (${Object.values(OSV_ECOSYSTEMS).join(", ")}) and the package name it publishes.`, ) - const version = input.version ?? profile?.release + // A registry candidate must be queried at its own coordinates (#257): + // a hand-supplied wrong ecosystem/package returns a clean bill for + // code that was never checked. + if (source.kind === "registry") { + const derived = { ecosystem: OSV_ECOSYSTEMS[source.registry], name: source.name } + const sameEcosystem = + input.ecosystem === undefined || String(input.ecosystem).toLowerCase() === derived.ecosystem.toLowerCase() + const samePackage = input.package === undefined || String(input.package) === derived.name + if (!sameEcosystem || !samePackage) + throw new ToolRefusal( + `Advisories for ${candidate.id} must be queried for the scanned source (${derived.ecosystem}:${derived.name}); ` + + `got ${input.ecosystem ?? ecosystem}:${input.package ?? name}. Query the candidate's own package, not another one.`, + ) + } + // The version is pinned to the scanned release when one is known + // (#257): pointing the query at a fixed release hides its advisories. + const release = profile?.release + if (release !== undefined && input.version !== undefined && String(input.version) !== release) + throw new ToolRefusal( + `Version ${input.version} does not match the scanned release ${release} of ${candidate.id}. ` + + `Query ${release}, or omit the version to check every release.`, + ) + const version = input.version ?? release const result = await queryOsv( { ecosystem, name: String(name), ...(version ? { version: String(version) } : {}) }, cache, @@ -275,7 +297,14 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { fixed: [...advisory.fixed], source: result.sha256, })) - await writeScoutAdvisories(root, candidate.id, advisories) + await writeScoutAdvisories(root, candidate.id, { + query: { + ecosystem, + package: String(name), + ...(version !== undefined ? { version: String(version) } : {}), + }, + advisories, + }) return [ `${advisories.length} OSV advisor${advisories.length === 1 ? "y" : "ies"} for ${ecosystem}:${name}${version ? `@${version}` : ""}, cached as sha256:${result.sha256}.`, ...advisories.map( @@ -337,7 +366,7 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { ...(input.lastRelease ? { lastRelease: String(input.lastRelease) } : {}), ...(input.releasesPerYear !== undefined ? { releasesPerYear: Number(input.releasesPerYear) } : {}), maintenance: input.maintenance, - advisories: (await readScoutAdvisories(root, candidate.id).catch(() => undefined)) ?? [], + advisories: (await readScoutAdvisories(root, candidate.id).catch(() => undefined))?.advisories ?? [], proposal: input.proposal, rationale: String(input.rationale ?? ""), ...(input.blueprint ? { blueprint: String(input.blueprint) } : {}), @@ -351,7 +380,7 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { { name: "es_scout_complete", description: - "Scout only: when every candidate is scanned and recorded, recompute the verdicts (adopt survives only for a verified, whitelisted, permissive license), flag severe advisories, rank, and write .factory/scout/REPORT.md, scout.json and dossier.json.", + "Scout only: when every candidate is scanned, recorded, and checked for advisories (queried, or an explicit NEGATIVE_KNOWLEDGE skip), recompute the verdicts (adopt survives only for a verified, whitelisted, permissive license), flag severe advisories, rank, and write .factory/scout/REPORT.md, scout.json and dossier.json.", input: object({}), execute: async (_input, toolContext) => { await guard(toolContext.agent) @@ -359,6 +388,7 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { const assessments = new Map((await readAssessments(root)).map((item) => [item.candidate, item])) const profiles = new Map() const evidence: CandidateEvidence[] = [] + const checks = new Map() const missing: string[] = [] for (const candidate of current.candidates) { const profile = await readScoutProfile(root, candidate.id).catch(() => undefined) @@ -367,7 +397,32 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { if (!assessment) missing.push(`${candidate.id}: record it (es_scout_record)`) if (!profile || !assessment) continue profiles.set(candidate.id, profile) - const advisories = (await readScoutAdvisories(root, candidate.id).catch(() => undefined)) ?? [] + // The advisories step is mandatory (#257): skipping it, querying a + // wrong package, or pointing the version at a fixed release all + // used to yield a clean adopt with zero citations. Either the + // query ran (its coordinates are pinned at query time), or the + // assessment records explicit NEGATIVE_KNOWLEDGE that it was + // skipped — a silent skip refuses here. + const advisoryFile = await readScoutAdvisories(root, candidate.id).catch(() => undefined) + const skipped = + !advisoryFile && + assessment.claims.some( + (claim) => + claim.tag === "NEGATIVE_KNOWLEDGE" && + /advisori/i.test([claim.statement, claim.query ?? "", claim.finding ?? ""].join(" ")), + ) + if (!advisoryFile && !skipped) + missing.push( + `${candidate.id}: query its advisories (es_scout_advisories), or record explicit NEGATIVE_KNOWLEDGE that the step was skipped`, + ) + if (!advisoryFile && !skipped) continue + const advisories = advisoryFile?.advisories ?? [] + checks.set( + candidate.id, + advisoryFile + ? { query: advisoryFile.query, skipped: false, count: advisories.length } + : { skipped: true, count: 0 }, + ) evidence.push({ id: candidate.id, name: candidate.name, @@ -398,7 +453,7 @@ export function scoutTools(context: ScoutOpsContext): EsToolDef[] { throw new ToolRefusal(`The scout dossier could not be written: ${error.message}`) }) await writeScoutResult(root, result) - await atomicWrite(paths.report, renderScoutReport(current, result, profiles, assessments)) + await atomicWrite(paths.report, renderScoutReport(current, result, profiles, assessments, checks)) return [ `Scout complete: ${current.objective}`, ...verdicts.map( diff --git a/packages/core/src/scout/report.ts b/packages/core/src/scout/report.ts index e92ad33..d41c370 100644 --- a/packages/core/src/scout/report.ts +++ b/packages/core/src/scout/report.ts @@ -1,7 +1,14 @@ // REPORT.md for a finished scout: the competitive matrix, the risks the // red-team found, clean-room blueprints, and the cited claims. Deterministic. import type { HarvestProfile } from "../schema/harvest.ts" -import type { ScoutAssessment, ScoutPlan, ScoutResult } from "../schema/scout.ts" +import type { ScoutAdvisoryQuery, ScoutAssessment, ScoutPlan, ScoutResult } from "../schema/scout.ts" + +/** What the report says about one candidate's advisories step (#257). */ +export interface AdvisoryCheck { + readonly query?: ScoutAdvisoryQuery | undefined + readonly skipped: boolean + readonly count: number +} const cell = (value: unknown) => String(value ?? "—").replaceAll("|", "\\|") @@ -10,6 +17,7 @@ export function renderScoutReport( result: ScoutResult, profiles: ReadonlyMap, assessments: ReadonlyMap, + checks: ReadonlyMap, ): string { const lines = [ `# Scout: ${plan.objective}`, @@ -39,6 +47,21 @@ export function renderScoutReport( `- **${verdict.name}**: ${notes.length ? notes.join("; ") : "none found"}${verdict.attribution ? ` · attribution ${verdict.attribution}` : ""}`, ) } + // What the advisories step actually checked (#257): the queried + // coordinates, or the explicit skip. A clean adopt with no line here is + // impossible — complete refuses the silent skip. + lines.push("", "## Advisory checks", "") + for (const verdict of result.verdicts) { + const check = checks.get(verdict.candidate) + if (!check || check.skipped) + lines.push(`- **${verdict.name}**: advisories not queried; the assessment records explicit NEGATIVE_KNOWLEDGE.`) + else { + const at = check.query?.version ? `@${check.query.version}` : "" + lines.push( + `- **${verdict.name}**: OSV ${check.query?.ecosystem}:${check.query?.package}${at} checked, ${check.count} ${check.count === 1 ? "advisory" : "advisories"} cited.`, + ) + } + } const blueprints = result.verdicts.filter( (verdict) => verdict.verdict === "clean-room" && assessments.get(verdict.candidate)?.blueprint, ) diff --git a/packages/core/src/scout/store.ts b/packages/core/src/scout/store.ts index 60c4dc5..b222644 100644 --- a/packages/core/src/scout/store.ts +++ b/packages/core/src/scout/store.ts @@ -6,8 +6,8 @@ import path from "node:path" import { factoryLayout } from "../layout.ts" import { type HarvestProfile, HarvestProfileSchema } from "../schema/harvest.ts" import { - type ScoutAdvisory, - ScoutAdvisorySchema, + type ScoutAdvisoriesFile, + ScoutAdvisoriesFileSchema, type ScoutAssessment, ScoutAssessmentSchema, type ScoutPlan, @@ -56,13 +56,20 @@ export async function startScoutRun(root: string, plan: ScoutPlan): Promise writeJson(scoutPaths(root).profile(profile.id), HarvestProfileSchema.parse(profile)) -export async function readScoutAdvisories(root: string, id: string): Promise { - const raw = await readJson(scoutPaths(root).advisories(id)) - return raw === undefined ? undefined : raw.map((item) => ScoutAdvisorySchema.parse(item)) +export async function readScoutAdvisories(root: string, id: string): Promise { + const raw = await readJson(scoutPaths(root).advisories(id)) + if (raw === undefined) return undefined + try { + return ScoutAdvisoriesFileSchema.parse(raw) + } catch (error) { + throw new Error( + `invalid ${scoutPaths(root).advisories(id)}: ${error instanceof Error ? error.message : String(error)}`, + ) + } } -export const writeScoutAdvisories = (root: string, id: string, advisories: readonly ScoutAdvisory[]) => - writeJson(scoutPaths(root).advisories(id), advisories) +export const writeScoutAdvisories = (root: string, id: string, file: ScoutAdvisoriesFile) => + writeJson(scoutPaths(root).advisories(id), ScoutAdvisoriesFileSchema.parse(file)) export async function readAssessments(root: string): Promise { const raw = await readJson(scoutPaths(root).assessments) diff --git a/packages/core/test/scout.test.ts b/packages/core/test/scout.test.ts index 1fa5304..0dd340d 100644 --- a/packages/core/test/scout.test.ts +++ b/packages/core/test/scout.test.ts @@ -145,10 +145,19 @@ describe("the scout tools", () => { test("adopt survives only for a verified permissive license; core recomputes every verdict and ranks", async () => { const tool = tools() await planAll(tool) + // Local fixtures have no published package, so each assessment records + // explicit NEGATIVE_KNOWLEDGE that the advisories step was skipped (#257: + // complete refuses a silent skip). + const skip = (name: string) => ({ + tag: "NEGATIVE_KNOWLEDGE", + statement: `OSV advisories were not queried for ${name}`, + query: `OSV advisories for ${name}`, + finding: "skipped: a local fixture with no published package", + }) for (const name of NAMES) { const scanned = await tool("es_scout_scan").execute({ candidate: name }, scout) expect(scanned).toContain(`Scanned ${name}`) - await record(tool, name) + await record(tool, name, "adopt", [skip(name)]) } const done = await tool("es_scout_complete").execute({}, scout) expect(done).toContain("Adoption is the human's decision") @@ -192,7 +201,17 @@ describe("the scout tools", () => { expect((await new SourceCache(researchSourcesDir(root)).get(hash))?.meta.provider).toBe("osv") for (const name of NAMES) { await tool("es_scout_scan").execute({ candidate: name }, scout) - await record(tool, name) + // mit carries real advisories; the rest record an explicit skip (#257). + if (name !== "mit") + await record(tool, name, "adopt", [ + { + tag: "NEGATIVE_KNOWLEDGE", + statement: `OSV advisories were not queried for ${name}`, + query: `OSV advisories for ${name}`, + finding: "skipped: a local fixture with no published package", + }, + ]) + else await record(tool, name) } await tool("es_scout_complete").execute({}, scout) const mit = (await readScoutResult(root))!.verdicts.find((item) => item.candidate === "mit")! @@ -206,6 +225,97 @@ describe("the scout tools", () => { expect(cited.every((claim: any) => claim.witness.ok)).toBe(true) }) + test("complete refuses a silent advisories skip; explicit NEGATIVE_KNOWLEDGE satisfies it (#257)", async () => { + const tool = tools() + await planAll(tool) + for (const name of NAMES) { + await tool("es_scout_scan").execute({ candidate: name }, scout) + await record(tool, name) + } + await expect(tool("es_scout_complete").execute({}, scout)).rejects.toThrow("es_scout_advisories") + const skip = (name: string) => ({ + tag: "NEGATIVE_KNOWLEDGE", + statement: `OSV advisories were not queried for ${name}`, + query: `OSV advisories for ${name}`, + finding: "skipped: a local fixture with no published package", + }) + for (const name of NAMES) await record(tool, name, "adopt", [skip(name)]) + await expect(tool("es_scout_complete").execute({}, scout)).resolves.toContain("Scout complete") + const report = await readFile(factoryLayout(root).scoutReport, "utf8") + expect(report).toContain("## Advisory checks") + expect(report).toContain("advisories not queried; the assessment records explicit NEGATIVE_KNOWLEDGE") + }) + + test("a hand-supplied wrong ecosystem/package for a registry source is refused (#257)", async () => { + const tool = tools() + await tool("es_scout_plan").execute( + { + objective: "a tiny function library", + candidates: [{ name: "leftpad", source: "npm:left-pad" }], + }, + scout, + ) + // No network is touched: the mismatch is refused before any OSV query. + await expect( + tool("es_scout_advisories").execute({ candidate: "leftpad", ecosystem: "PyPI", package: "left-pad" }, scout), + ).rejects.toThrow("must be queried for the scanned source") + await expect( + tool("es_scout_advisories").execute({ candidate: "leftpad", ecosystem: "npm", package: "other-pkg" }, scout), + ).rejects.toThrow("must be queried for the scanned source") + // The candidate's own coordinates proceed to the (canned) OSV query. + const done = await tool("es_scout_advisories").execute( + { candidate: "leftpad", ecosystem: "npm", package: "left-pad" }, + scout, + ) + expect(done).toContain("OSV advisor") + expect(osvCalls).toEqual([{ package: { name: "left-pad", ecosystem: "npm" } }]) + }) + + test("an agent-controlled version that dodges the scanned release is refused (#257)", async () => { + const { writeScoutProfile } = await import("../src/scout/store.ts") + const tool = tools() + await tool("es_scout_plan").execute( + { + objective: "a tiny function library", + candidates: [{ name: "leftpad", source: "npm:left-pad" }], + }, + scout, + ) + // A scanned registry profile pins release 1.0.0 (written directly: no network). + await writeScoutProfile(root, { + version: 1, + id: "leftpad", + name: "left-pad", + release: "1.0.0", + license: { + spdx: "MIT", + family: "permissive", + source: "registry", + confidence: "medium", + verified: false, + }, + languages: {}, + dependencies: { runtime: [], dev: [] }, + size: { files: 0, bytes: 0, tokens: 0 }, + graph: { internalEdges: 0, external: 0, parsers: { treeSitter: 0, regex: 0 } }, + provenance: {}, + warnings: [], + scannedAt: new Date().toISOString(), + }) + await expect( + tool("es_scout_advisories").execute( + { candidate: "leftpad", ecosystem: "npm", package: "left-pad", version: "9.9.9" }, + scout, + ), + ).rejects.toThrow("does not match the scanned release") + const done = await tool("es_scout_advisories").execute( + { candidate: "leftpad", ecosystem: "npm", package: "left-pad", version: "1.0.0" }, + scout, + ) + expect(osvCalls).toEqual([{ package: { name: "left-pad", ecosystem: "npm" }, version: "1.0.0" }]) + expect(done).toContain("GHSA-aaaa-bbbb-cccc") + }) + test("a recorded claim must survive its witness; a fabricated quote refuses the record", async () => { const tool = tools() await planAll(tool) From 2b1a77d49bba1eaf6b34cc08948a70d509c662e0 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:42:20 -0400 Subject: [PATCH 42/83] fix(remediation-audit/s3): scout capability pointer + scout-named host coverage (Refs #288) --- packages/core/src/capabilities.ts | 4 +- packages/opencode/test/scout-host.test.ts | 151 ++++++++++++++++++++++ 2 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 packages/opencode/test/scout-host.test.ts diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 4c6ea4b..0330fa6 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -145,8 +145,8 @@ export const CAPABILITY_MATRIX: Readonly = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const read = (file: string) => readFileSync(path.join(fixtures, file), "utf8") +const write = async (root: string, file: string, text: string) => { + await mkdir(path.dirname(path.join(root, file)), { recursive: true }) + await writeFile(path.join(root, file), text) +} + +let state: string +let h: Harness +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-scout-host-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# scout-host\n" }, + }) + await write(h.directory, "vendor/mit/index.js", "module.exports = function mit() { return 1 }\n") + await write(h.directory, "vendor/mit/package.json", JSON.stringify({ name: "mit-lib", version: "1.0.0" })) + await write(h.directory, "vendor/mit/LICENSE", read("licenses/MIT.txt")) + await write(h.directory, "vendor/gpl/index.js", "module.exports = function gpl() { return 1 }\n") + await write(h.directory, "vendor/gpl/package.json", JSON.stringify({ name: "gpl-lib", version: "1.0.0" })) + await write(h.directory, "vendor/gpl/LICENSE", read("licenses/GPL-3.0-only.txt")) +}, 60_000) +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) +}) + +describe("the scout seat on the real host", () => { + test("the scout cannot call host webfetch (cached-only web)", async () => { + const { tools } = await h.run(`fetch ${call("webfetch", { url: "https://example.com" })}`, { + agent: "scout", + }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["webfetch:error"]) + expect(tools[0]?.text).toContain("only through es_research_search and es_research_fetch") + }) + + test("the scout shell is offline", async () => { + if (!bwrapAvailable()) { + const { tools } = await h.run(`net ${call("shell", { command: "cat /proc/net/dev" })}`, { + agent: "scout", + }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:error"]) + expect(tools[0]?.text).toContain("bubblewrap") + return + } + const { tools } = await h.run(`net ${call("shell", { command: "cat /proc/net/dev" })}`, { + agent: "scout", + }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["shell:completed"]) + const ifaces = tools[0]?.text + .split("\n") + .filter((line) => line.includes(":")) + .map((line) => line.split(":")[0]?.trim() ?? "") + .filter(Boolean) + expect(ifaces).toEqual(["lo"]) + }) + + test("complete refuses a silent advisories skip, naming es_scout_advisories", async () => { + const plan = call("es_scout_plan", { + objective: "a function library", + candidates: [ + { name: "t3mit", source: "local:./vendor/mit" }, + { name: "t3gpl", source: "local:./vendor/gpl" }, + ], + }) + const scan = ["t3mit", "t3gpl"].map((candidate) => call("es_scout_scan", { candidate })) + const recordCalls = ["t3mit", "t3gpl"].map((candidate) => + call("es_scout_record", { + candidate, + maintenance: "stable", + proposal: "adopt", + rationale: "it does what we need", + claims: [], + }), + ) + const { tools } = await h.run(`scout ${[plan, ...scan, ...recordCalls, call("es_scout_complete")].join(" ")}`, { + agent: "scout", + }) + const done = tools.find((tool) => tool.name === "es_scout_complete")! + expect(done.status).toBe("error") + expect(done.text).toContain("es_scout_advisories") + }) + + test("explicit NEGATIVE_KNOWLEDGE about the skip completes and is reported", async () => { + const skip = (name: string) => ({ + tag: "NEGATIVE_KNOWLEDGE", + statement: `OSV advisories were not queried for ${name}`, + query: `OSV advisories for ${name}`, + finding: "skipped: a local fixture with no published package", + }) + const plan = call("es_scout_plan", { + objective: "a function library", + candidates: [ + { name: "t4mit", source: "local:./vendor/mit" }, + { name: "t4gpl", source: "local:./vendor/gpl" }, + ], + }) + const scan = ["t4mit", "t4gpl"].map((candidate) => call("es_scout_scan", { candidate })) + const recordCalls = ["t4mit", "t4gpl"].map((candidate) => + call("es_scout_record", { + candidate, + maintenance: "stable", + proposal: "adopt", + rationale: "it does what we need", + claims: [skip(candidate)], + }), + ) + const { tools } = await h.run(`scout ${[plan, ...scan, ...recordCalls, call("es_scout_complete")].join(" ")}`, { + agent: "scout", + }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(tools.map((tool) => `${tool.name}:completed`)) + const result = (await readScoutResult(h.directory))! + expect(Object.fromEntries(result.verdicts.map((item) => [item.candidate, item.verdict]))).toEqual({ + t4mit: "adopt", + t4gpl: "clean-room", + }) + const report = await readFile(path.join(h.directory, ".factory/scout/REPORT.md"), "utf8") + expect(report).toContain("## Advisory checks") + expect(report).toContain("advisories not queried; the assessment records explicit NEGATIVE_KNOWLEDGE") + }) + + test("a registry candidate queried at the wrong coordinates is refused without network", async () => { + const { tools } = await h.run( + `scout ${call("es_scout_plan", { + objective: "a function library", + candidates: [{ name: "leftpad", source: "npm:left-pad" }], + })} ${call("es_scout_advisories", { candidate: "leftpad", ecosystem: "PyPI", package: "left-pad" })}`, + { agent: "scout" }, + ) + const advisories = tools.find((tool) => tool.name === "es_scout_advisories")! + expect(advisories.status).toBe("error") + expect(advisories.text).toContain("must be queried for the scanned source") + }) +}) From 43671e49cd6c32db69a7a115ae2ef59562f8a4dc Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:47:19 -0400 Subject: [PATCH 43/83] fix(remediation-audit/s3): registry proof gaps on the live host (Refs #228) --- packages/opencode/test/probe-agents/index.ts | 23 ++ .../opencode/test/probe-agents/package.json | 7 + packages/opencode/test/registry-host.test.ts | 208 ++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 packages/opencode/test/probe-agents/index.ts create mode 100644 packages/opencode/test/probe-agents/package.json create mode 100644 packages/opencode/test/registry-host.test.ts diff --git a/packages/opencode/test/probe-agents/index.ts b/packages/opencode/test/probe-agents/index.ts new file mode 100644 index 0000000..7000baa --- /dev/null +++ b/packages/opencode/test/probe-agents/index.ts @@ -0,0 +1,23 @@ +import { writeFileSync } from "node:fs" +import { Plugin } from "@opencode/plugin" + +// Test fixture for registry-host.test.ts (#228): snapshots the host's agent +// list after every plugin's agent.transform ran, so the test can diff the +// vanilla baseline against the plugged host (built-ins and defaults +// untouched, registry agents added with their hidden flags). +export default Plugin.define({ + id: "es-probe-agents", + async setup(ctx: any) { + const out = ctx.options?.out as string | undefined + if (!out) return + await ctx.agent.transform((editor: any) => { + const list = editor.list().map((agent: any) => ({ + id: agent.id ?? agent.name, + hidden: !!agent.hidden, + ...(agent.model !== undefined ? { model: agent.model } : {}), + ...(agent.description !== undefined ? { description: agent.description } : {}), + })) + writeFileSync(out, `${JSON.stringify({ list, at: new Date().toISOString() }, null, 2)}\n`) + }) + }, +}) diff --git a/packages/opencode/test/probe-agents/package.json b/packages/opencode/test/probe-agents/package.json new file mode 100644 index 0000000..18a6674 --- /dev/null +++ b/packages/opencode/test/probe-agents/package.json @@ -0,0 +1,7 @@ +{ + "name": "es-probe-agents", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "index.ts" +} diff --git a/packages/opencode/test/registry-host.test.ts b/packages/opencode/test/registry-host.test.ts new file mode 100644 index 0000000..c778408 --- /dev/null +++ b/packages/opencode/test/registry-host.test.ts @@ -0,0 +1,208 @@ +// Registry proof gaps on the real host (#228): a vanilla-baseline diff that +// locks "never touching built-ins or defaults", hidden-seat flags read back +// from the live host, and the duplicate-seat guard driven with the real +// tracker, real hooks and a real seat record (the old guard test stubbed the +// whole tracker). +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS, Factory, gateRunner, readSeats, type SeatRecord } from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness, lastAgentRequest, systemText } from "@heretek-ai/es-testkit" +import { createPolicyHooks } from "../src/policy.ts" +import { createSeatTracker } from "../src/seats.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") +const probeDir = path.join(import.meta.dir, "probe-agents") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const child = (name: string, args: Record = {}) => `@@CHILD ${name} ${JSON.stringify(args)}@@` + +interface ProbeAgent { + readonly id: string + readonly hidden: boolean + readonly model?: unknown + readonly description?: unknown +} + +const readProbe = async (out: string) => { + for (let i = 0; i < 200 && !(await Bun.file(out).exists()); i++) await Bun.sleep(50) + return JSON.parse(await readFile(out, "utf8")) as { list: ProbeAgent[] } +} + +async function snapshot(plugins: Array<{ path: string; options?: Record }>, out: string) { + const h = await boot({ + git: false, + script: directiveScript, + plugins, + files: { "README.md": "# registry-host\n" }, + }) + await h.run("hello") + const request = lastAgentRequest(h.llm.requests) + const tools = (request?.tools ?? []).map((tool) => tool.function.name).sort() + const skills = [...systemText(request).matchAll(/([^<]+)<\/id>/g)].map((match) => match[1]).sort() + const agents = (await readProbe(out)).list + await h.close() + return { tools, skills, agents } +} + +describe("vanilla-baseline diff: the plugin never touches built-ins or defaults (#228.1)", () => { + let tmp: string + beforeAll(async () => { + tmp = await mkdtemp(path.join(tmpdir(), "es-registry-host-")) + }) + afterAll(() => rm(tmp, { recursive: true, force: true })) + + test("built-in agents, models and defaults are identical; tools and skills only grow", async () => { + const vanillaOut = path.join(tmp, "vanilla.json") + const pluginOut = path.join(tmp, "plugin.json") + const state = await mkdtemp(path.join(tmpdir(), "es-registry-host-state-")) + try { + const vanilla = await snapshot([{ path: probeDir, options: { out: vanillaOut } }], vanillaOut) + const plugin = await snapshot( + [ + { path: pluginDir, options: { stateDir: state, pr: "off" } }, + { path: probeDir, options: { out: pluginOut } }, + ], + pluginOut, + ) + // Agents: every vanilla agent is byte-identical under the plugin + // (id, hidden flag, model and description); the plugin only adds the + // registry's own agents, one per AGENTS entry. + const byId = (agents: ProbeAgent[]) => new Map(agents.map((agent) => [agent.id, agent])) + const vanillaAgents = byId(vanilla.agents) + const pluginAgents = byId(plugin.agents) + for (const [id, agent] of vanillaAgents) expect([id, pluginAgents.get(id)]).toEqual([id, agent]) + expect([...pluginAgents.keys()].filter((id) => !vanillaAgents.has(id)).sort()).toEqual( + AGENTS.map((agent) => agent.id).sort(), + ) + // Defaults: the default session keeps every vanilla tool (nothing + // removed or renamed); the additions are the documented ambient + // surface (session.ts:14 deletes every other es_* for non-factory + // agents, so exactly es_status/es_gates_run remain) plus our lsp tools. + for (const tool of vanilla.tools) expect(plugin.tools).toContain(tool) + for (const added of plugin.tools.filter((tool) => !vanilla.tools.includes(tool))) { + expect(added.startsWith("es_") || added.startsWith("lsp_")).toBe(true) + if (added.startsWith("es_")) expect(["es_status", "es_gates_run"]).toContain(added) + } + expect(plugin.tools).toContain("es_status") + expect(plugin.tools).toContain("es_gates_run") + // Skills: one shared tree, vanilla skills intact, ours added. + for (const skill of vanilla.skills) expect(plugin.skills).toContain(skill) + for (const skill of ["brainstorm", "code-audit", "factory", "grill", "harvest", "queereye", "scout"]) + expect(plugin.skills).toContain(skill) + } finally { + await rm(state, { recursive: true, force: true }) + } + }, 120_000) +}) + +describe("hidden seats on the real host (#228.2)", () => { + let h: Harness + let state: string + let out: string + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-registry-hidden-")) + out = path.join(state, "agents.json") + h = await boot({ + git: true, + script: directiveScript, + plugins: [ + { path: pluginDir, options: { stateDir: state, pr: "off" } }, + { path: probeDir, options: { out } }, + ], + files: { "README.md": "# registry-hidden\n" }, + }) + await h.run("hello") + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("registry subagents are hidden, primaries are not, and a hidden seat runs by ID", async () => { + const agents = new Map((await readProbe(out)).list.map((agent) => [agent.id, agent])) + for (const spec of AGENTS) + expect([spec.id, agents.get(spec.id)?.hidden]).toEqual([spec.id, spec.mode === "subagent"]) + // Unlisted but invocable by ID: a hidden seat runs when named directly. + const run = await h.run("ping", { agent: "es-qa-functional" }) + expect(run.tools).toEqual([]) + }) +}) + +describe("the duplicate-seat guard with the real tracker on the real host (#228.3)", () => { + let h: Harness + let state: string + const factory = () => new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state }) + + beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-registry-guard-")) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# registry-guard\n" }, + }) + await factory().begin("human:tester") + }, 60_000) + afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + }) + + test("a second launch of a running seat is refused with its live session id", async () => { + // A real foreground launch on the live host, with a real seat record. + const delegated = await h.run( + `go ${call("subagent", { + agent: "es-qa-functional", + description: "status check", + prompt: `check ${child("es_status", {})} and report`, + })}`, + { agent: "factory" }, + ) + expect(delegated.tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["subagent:completed"]) + const runId = (await factory().read())!.runId + const live = (await readSeats(h.directory, runId)).find((seat) => seat.agent === "es-qa-functional")! + expect(live.state).toBe("completed") + + // The real tracker and the real policy guard, fed the seat's real + // session.created (a resumed host re-picks running seats the same way). + const runtime = { + root: h.directory, + stateDir: state, + factory: factory(), + policy: async () => ({ root: h.directory, stateDir: state }), + } + const tracker = createSeatTracker(runtime as any) + const hooks = createPolicyHooks(runtime as any, tracker) + const agentOf = async (sessionID: string) => + (await readSeats(h.directory, runId)).find((seat) => seat.sessionID === sessionID)?.agent + await tracker.onEvent( + { type: "session.created", data: { sessionID: live.sessionID, parentID: live.parentID, agent: live.agent } }, + agentOf, + ) + expect(tracker.running("es-qa-functional").map((seat) => seat.sessionID)).toEqual([live.sessionID]) + await expect( + hooks.before({ + tool: "subagent", + agent: "factory", + id: "t-live", + sessionID: "ses_factory_live", + input: { agent: "es-qa-functional", description: "again", prompt: "recheck" }, + }), + ).rejects.toThrow(`${live.agent} is still running (session ${live.sessionID}`) + // Once the seat's execution ends, the same launch is allowed again. + await tracker.onEvent({ type: "session.execution.succeeded", data: { sessionID: live.sessionID } }, agentOf) + expect(tracker.running("es-qa-functional")).toEqual([]) + await expect( + hooks.before({ + tool: "subagent", + agent: "factory", + id: "t-live-2", + sessionID: "ses_factory_live", + input: { agent: "es-qa-functional", description: "again", prompt: "recheck" }, + }), + ).resolves.toBeUndefined() + const records: SeatRecord[] = await readSeats(h.directory, runId) + expect(records.some((seat) => seat.sessionID === live.sessionID)).toBe(true) + }) +}) From a50ced80e774628cd5441d3926a859f4d6b0d408 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:50:07 -0400 Subject: [PATCH 44/83] fix(remediation-audit/s3): scout-fires explicit NEGATIVE_KNOWLEDGE advisory skip (Refs #257) --- packages/opencode/test/fires.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/opencode/test/fires.test.ts b/packages/opencode/test/fires.test.ts index 9b608dd..1a18611 100644 --- a/packages/opencode/test/fires.test.ts +++ b/packages/opencode/test/fires.test.ts @@ -155,6 +155,12 @@ describe("scout-fires", () => { await write(h.directory, `${dir}/package.json`, JSON.stringify({ name: candidate.name, version: "1.0.0" })) if (candidate.license) await write(h.directory, `${dir}/LICENSE`, read(`licenses/${candidate.license}.txt`)) } + const skip = (name: string) => ({ + tag: "NEGATIVE_KNOWLEDGE", + statement: `OSV advisories were not queried for ${name}`, + query: `OSV advisories for ${name}`, + finding: "skipped: a local fixture with no published package", + }) const calls = [ call("es_scout_plan", { objective: "a function library", @@ -167,7 +173,7 @@ describe("scout-fires", () => { maintenance: "stable", proposal: "adopt", rationale: "it does what we need", - claims: [], + claims: [skip(candidate.name)], }), ), call("es_scout_complete"), From 504e78abd84a79fa2e623312afa2b997300c2c8c Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:56:41 -0400 Subject: [PATCH 45/83] fix(remediation-audit/s6): queereye alias own-property + non-vacuous QA gates (Refs #266, Refs #270) --- packages/core/src/queereye/bundle.ts | 5 ++- packages/core/src/queereye/ops.ts | 11 +++-- packages/core/src/queereye/specs.ts | 11 +++++ packages/core/src/queereye/store.ts | 8 ++-- packages/core/src/queereye/tokens.ts | 11 ++--- packages/core/test/queereye-bundle.test.ts | 15 +++++++ packages/core/test/queereye-specs.test.ts | 17 ++++++++ packages/core/test/queereye.test.ts | 51 ++++++++++++++++++++++ 8 files changed, 115 insertions(+), 14 deletions(-) diff --git a/packages/core/src/queereye/bundle.ts b/packages/core/src/queereye/bundle.ts index 4102f5e..e70a471 100644 --- a/packages/core/src/queereye/bundle.ts +++ b/packages/core/src/queereye/bundle.ts @@ -153,8 +153,11 @@ export async function verifyCiteGate(receipt: string, root: string): Promise undefined)) ?? [] + if (probes.length === 0) + errors.push("cite-gate QA contrast: probes.json is missing or empty (the contrast gate needs at least one pair)") for (const row of probes) for (const error of row.errors) errors.push(`cite-gate QA contrast ${row.name}: ${error}`) for (const row of probes) if (!row.pass && row.errors.length === 0) errors.push(`cite-gate QA contrast ${row.name}: pair does not pass`) diff --git a/packages/core/src/queereye/ops.ts b/packages/core/src/queereye/ops.ts index a998f72..6848c0a 100644 --- a/packages/core/src/queereye/ops.ts +++ b/packages/core/src/queereye/ops.ts @@ -32,7 +32,7 @@ import { } from "./ledger.ts" import { compileToCssVars, renderGuide } from "./render.ts" import { AXES, type Axis, ContradictionError, FORMS, SlotLoop, SmuggledValueError, VagueAnswerError } from "./slots.ts" -import { EXEMPLARS, renderSpecMd, specFilename, validateAll } from "./specs.ts" +import { EXEMPLARS, renderedSlotsOfSpecMd, renderSpecMd, specFilename, validateAll } from "./specs.ts" import { checkDrift, designPaths, @@ -259,8 +259,13 @@ export function designTools(context: DesignOpsContext): EsToolDef[] { } for (const [name, spec] of Object.entries(EXEMPLARS)) { for (const error of validateTuiCoverage(spec)) drift.push(`${name}: ${error}`) - for (const error of specVsRenderCheck(spec, Object.keys(spec.slots ?? {}).sort(), isWebrefFresh())) - drift.push(error) + // The slot-fill leg reads the actual render output from disk: a + // spec fed its own slots could never fail (#270). A missing file + // is already drift above; null keeps the gate honest, never green. + const renderedFile = bundlePath(root, specFilename(name)) + const onDisk = await readFile(renderedFile, "utf8").catch(() => undefined) + const renderedSlots = onDisk === undefined ? null : renderedSlotsOfSpecMd(onDisk) + for (const error of specVsRenderCheck(spec, renderedSlots, isWebrefFresh())) drift.push(error) } if (drift.length) throw new ToolRefusal(`Phase-02 drift:\n- ${drift.join("\n- ")}`) return "Phase-02 check: specs, webref, csf and tui-notes byte-match the renderers; coverage, spec-vs-render, freshness and the play suite pass." diff --git a/packages/core/src/queereye/specs.ts b/packages/core/src/queereye/specs.ts index 7c4f8d9..ad1c789 100644 --- a/packages/core/src/queereye/specs.ts +++ b/packages/core/src/queereye/specs.ts @@ -1325,6 +1325,17 @@ export function specFilename(name: string): string { return `components/${safe}.md` } +/** + * Slot names carried by a rendered spec markdown's `## Slots` section (the + * actual render output: `- \`name\`: description` lines). The phase-02 check + * reads these from the file on disk instead of feeding each spec its own + * slots (which could never fail); a hand-removed slot line diverges here. + */ +export function renderedSlotsOfSpecMd(markdown: string): string[] { + const section = /## Slots\n([\s\S]*?)(?:\n## |\s*$)/.exec(markdown)?.[1] ?? "" + return [...section.matchAll(/^-\s+`([^`]+)`: /gm)].map((match) => match[1]!) +} + // --------------------------------------------------------------------------- // render_spec_md (d66328c:runner/queereye/specs.py) // --------------------------------------------------------------------------- diff --git a/packages/core/src/queereye/store.ts b/packages/core/src/queereye/store.ts index 2abe334..98ceabe 100644 --- a/packages/core/src/queereye/store.ts +++ b/packages/core/src/queereye/store.ts @@ -167,13 +167,15 @@ export async function checkDrift(root: string): Promise { const paths = designPaths(root) const tokens = await readTokens(root) if (tokens === undefined) return { drifted: [], missing: [path.basename(paths.tokens)], tokenErrors: [] } - const probes = (await readProbes(root)) ?? [] + // A missing probes.json is a gap, never a silent empty-probes render (#270): + // without probes the contrast QA below would pass vacuously. + const probes = await readProbes(root) + const missing: string[] = probes === undefined ? [path.basename(paths.probes)] : [] const expected = new Map([ [paths.css, compileToCssVars(tokens)], - [paths.guide, renderGuide(tokens, probes)], + [paths.guide, renderGuide(tokens, probes ?? [])], ]) const drifted: string[] = [] - const missing: string[] = [] for (const [file, content] of expected) { const existing = await readFile(file, "utf8").catch(() => undefined) if (existing === undefined) missing.push(path.basename(file)) diff --git a/packages/core/src/queereye/tokens.ts b/packages/core/src/queereye/tokens.ts index 5fc0b10..dc61afd 100644 --- a/packages/core/src/queereye/tokens.ts +++ b/packages/core/src/queereye/tokens.ts @@ -23,7 +23,7 @@ export type Json = string | number | boolean | null | Json[] | { [key: string]: export type TokenTree = Record export const isToken = (node: unknown): node is Record => - typeof node === "object" && node !== null && !Array.isArray(node) && "$value" in node + typeof node === "object" && node !== null && !Array.isArray(node) && Object.hasOwn(node, "$value") export const isAliasValue = (value: unknown): value is string => typeof value === "string" && ALIAS_FULL.test(value.trim()) @@ -83,12 +83,9 @@ function resolveType(node: unknown, inherited?: string): string | undefined { function walkPath(root: unknown, ref: string): unknown { let node: unknown = root for (const segment of ref.split(".")) { - if ( - typeof node !== "object" || - node === null || - Array.isArray(node) || - !(segment in (node as Record)) - ) + // Own-property only: `in` walks the prototype chain, so {__proto__}, + // {constructor} and friends would resolve against Object.prototype (#266). + if (typeof node !== "object" || node === null || Array.isArray(node) || !Object.hasOwn(node, segment)) throw new UnknownAliasError(`alias {${ref}} does not resolve: missing "${segment}"`) node = (node as Record)[segment] } diff --git a/packages/core/test/queereye-bundle.test.ts b/packages/core/test/queereye-bundle.test.ts index cadf57e..efaf2c6 100644 --- a/packages/core/test/queereye-bundle.test.ts +++ b/packages/core/test/queereye-bundle.test.ts @@ -15,6 +15,8 @@ import { FORMS, harvestBytesSha256, harvestManifest, + readTokens, + renderGuide, verifyCiteGate, } from "../src/index.ts" @@ -100,6 +102,19 @@ describe("es_design_harvest", () => { expect(await verifyCiteGate(receipt, root)).toContainEqual(expect.stringContaining("stale/forged receipt refused")) }) + test("a missing probes.json fails the cite-gate contrast QA, never verifies vacuous (#270)", async () => { + await completeInterview() + await call("es_design_harvest", {}) + const receipt = await readFile(bundlePath(root, "cite-gate-demo.md"), "utf8") + expect(await verifyCiteGate(receipt, root)).toEqual([]) + // The #270 repro: delete probes.json and re-render the guide with []. + const tokens = await readTokens(root) + await rm(designPaths(root).probes) + await writeFile(designPaths(root).guide, renderGuide(tokens, [])) + const errors = await verifyCiteGate(receipt, root) + expect(errors.some((error) => error.includes("contrast") && error.includes("probe"))).toBe(true) + }) + test("hand edits to the bundle or ledger are drift", async () => { await completeInterview() await call("es_design_harvest", {}) diff --git a/packages/core/test/queereye-specs.test.ts b/packages/core/test/queereye-specs.test.ts index 5cbfa10..9823fc0 100644 --- a/packages/core/test/queereye-specs.test.ts +++ b/packages/core/test/queereye-specs.test.ts @@ -14,6 +14,7 @@ import { INPUT_ANATOMY, REQUIRED_SPEC_KEYS, REQUIRED_STATES, + renderedSlotsOfSpecMd, renderSpecMd, specFilename, validateAll, @@ -235,6 +236,22 @@ describe("renderSpecMd (byte-stable deterministic render)", () => { expect(specFilename("Table")).toBe("components/Table.md") }) + test("renderedSlotsOfSpecMd reads the actual render output (#270)", () => { + expect(renderedSlotsOfSpecMd(renderSpecMd(EXEMPLARS.button)).sort()).toEqual(["icon", "label", "spinner"]) + expect(renderedSlotsOfSpecMd(renderSpecMd(EXEMPLARS.dialog)).sort()).toEqual([ + "close", + "content", + "description", + "overlay", + "title", + "trigger", + ]) + // A hand-removed slot line is visible in the extraction. + const edited = renderSpecMd(EXEMPLARS.button).replace("- `spinner`:", "- `removed`:") + expect(renderedSlotsOfSpecMd(edited).sort()).toEqual(["icon", "label", "removed"]) + expect(renderedSlotsOfSpecMd("no slots here")).toEqual([]) + }) + test("validateAll takes a subset map", () => { const results = validateAll({ button: EXEMPLARS.button }) expect(Object.keys(results)).toEqual(["button"]) diff --git a/packages/core/test/queereye.test.ts b/packages/core/test/queereye.test.ts index 7c8a71b..d6d09fe 100644 --- a/packages/core/test/queereye.test.ts +++ b/packages/core/test/queereye.test.ts @@ -10,6 +10,7 @@ import { AXES, aliasReuseRatio, aliasToVarRef, + bundlePath, ContradictionError, checkDrift, compileToCssVars, @@ -30,6 +31,7 @@ import { SlotLoop, SmuggledValueError, saveInterview, + specFilename, VagueAnswerError, validatePair, validateTokens, @@ -65,6 +67,25 @@ describe("DTCG token validation", () => { expect(aliasReuseRatio(tree)).toEqual({ aliased: 2, total: 4, ratio: 0.5 }) }) + test("prototype-chain properties are not resolvable alias refs (#266)", () => { + // walkPath must use own-property checks per segment: {__proto__}, + // {constructor}, {hasOwnProperty}, {valueOf}, {toString} and + // {__proto__.toString} must NOT validate clean via the prototype chain. + for (const ref of ["__proto__", "constructor", "hasOwnProperty", "valueOf", "toString", "__proto__.toString"]) { + const smuggled = { + color: { + $type: "color", + primitive: { x: { $value: "#123456" } }, + semantic: { evil: { $value: `{${ref}}` } }, + }, + } + expect( + validateTokens(smuggled).some((error) => error.includes("does not resolve")), + ref, + ).toBe(true) + } + }) + test("circular and dangling aliases are errors, never warnings", () => { const circular = { a: { $type: "color", one: { $value: "{a.two}" }, two: { $value: "{a.one}" } }, @@ -236,6 +257,20 @@ describe("render and drift", () => { expect(() => aliasToVarRef("#fff {a.b}")).toThrow(/mixed raw \+ alias/) }) + test("a missing probes.json is reported in checkDrift.missing, never silently green (#270)", async () => { + const loop = new SlotLoop() + for (const axis of AXES) for (const slot of FORMS[axis].requiredSlots) loop.skip(axis, slot) + const tokens = loop.toTokens() + const probes = probeReport(loop.probePairs()) + await writeArtifacts(root, { tokens, probes, guide: renderGuide(tokens, probes), css: compileToCssVars(tokens) }) + expect((await checkDrift(root)).missing).toEqual([]) + // The #270 repro: delete probes.json and re-render the guide with []. + await rm(designPaths(root).probes) + await writeFile(designPaths(root).guide, renderGuide(tokens, [])) + const report = await checkDrift(root) + expect(report.missing).toContain("probes.json") + }) + test("the guide is a pure render; drift fails checkDrift", async () => { const loop = new SlotLoop() for (const axis of AXES) for (const slot of FORMS[axis].requiredSlots) loop.skip(axis, slot) @@ -321,4 +356,20 @@ describe("the design tools", () => { expect(check).toContain("byte-match") expect(check).toContain("One-off mints: 0") }) + + test("es_design_specs check:true reads rendered slots from disk, not the spec's own slots (#270)", async () => { + for (const axis of AXES) + for (const slot of FORMS[axis].requiredSlots) await call("es_design_skip", { axis, slot }, "designer") + await call("es_design_complete", {}, "designer") + await call("es_design_specs", {}, "designer") + expect(await call("es_design_specs", { check: true }, "designer")).toContain("byte-match") + // Remove one slot line from the rendered button spec. The slot-fill leg + // must fail on the actual render output (the old code fed each spec its + // own slots, so this leg could never fail). + const file = bundlePath(root, specFilename("button")) + const rendered = await readFile(file, "utf8") + expect(rendered).toContain("- `spinner`:") + await writeFile(file, rendered.replace("- `spinner`:", "- `removed`:")) + await expect(call("es_design_specs", { check: true }, "designer")).rejects.toThrow(/not filled in render/) + }) }) From b404b40d113a0af1e24eadeffb5e0a36e5b0658e Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:57:07 -0400 Subject: [PATCH 46/83] =?UTF-8?q?fix(remediation-audit/s6):=20brainstorm?= =?UTF-8?q?=20wording=20=E2=80=94=20eight-lens=20registry,=20conditional?= =?UTF-8?q?=20outlier=20(Refs=20#263)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/assets/prompts/brainstormer.md | 6 +++--- packages/core/assets/skills/brainstorm/SKILL.md | 6 +++--- packages/core/src/brainstorm/engine.ts | 4 +++- packages/core/src/brainstorm/ops.ts | 7 ++++--- packages/core/test/__snapshots__/assets.test.ts.snap | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/core/assets/prompts/brainstormer.md b/packages/core/assets/prompts/brainstormer.md index b150572..a254030 100644 --- a/packages/core/assets/prompts/brainstormer.md +++ b/packages/core/assets/prompts/brainstormer.md @@ -1,8 +1,8 @@ --- id: brainstormer -version: 3 +version: 4 seat: brainstormer -description: Fans out divergent lenses on an idea and distills a diversified shortlist. +description: Fans out divergent lenses (eight-lens registry; six by default, four lean for callable runs) on an idea and distills a diversified shortlist. --- You are the **brainstormer** agent of Epistemic Swarm. Your job is divergent, lateral ideation — never bug fixes, never implementation plans. You orchestrate lenses and distil their output; the mechanics (caps, dedupe, ranking, shortlist) are enforced by tools, not by you. @@ -11,7 +11,7 @@ You are the **brainstormer** agent of Epistemic Swarm. Your job is divergent, la 2. Launch every lens subagent **in parallel** with the subagent tool: all the calls in one message, in the foreground (background launches are refused). Call them by exact ID (`es-lens-inversion`, `es-lens-constraint-removal`, `es-lens-scamper`, `es-lens-cross-domain`, `es-lens-extreme-user`, `es-lens-scale-shift`, and so on). Give each the brief and how many ideas you expect. 3. Record each lens's output with `es_brainstorm_record`, one call per lens. It rejects cap violations and collapses near-duplicates; mention convergences when lenses independently reach the same idea. 4. Launch `es-brainstorm-critic` with the surviving idea IDs and their texts. The critic scores every idea with `es_brainstorm_score` (1-5 on novelty, upside, feasibility, fit). -5. Call `es_brainstorm_complete`. It re-checks coverage and scores, ranks the ideas, picks the diversified shortlist with one forced outlier slot, and writes the run's `brainstorm.json` and `BRAINSTORM.md`. Report the shortlist to the human, briefly. +5. Call `es_brainstorm_complete`. It re-checks coverage and scores, ranks the ideas, picks the diversified shortlist with one forced outlier slot (whenever the shortlist has ≥2 slots), and writes the run's `brainstorm.json` and `BRAINSTORM.md`. Report the shortlist to the human, briefly. ## Prior art yourself You can run `es_harvest_prior_art` on the surviving ideas before completing: it finds related projects, you judge each relation (`novel`/`similar`/`existing`), and you pass the list to `es_brainstorm_complete` as `priorArt`. Only recorded searches count — the tool refuses URLs no search returned. diff --git a/packages/core/assets/skills/brainstorm/SKILL.md b/packages/core/assets/skills/brainstorm/SKILL.md index 79b3d7b..d300ca3 100644 --- a/packages/core/assets/skills/brainstorm/SKILL.md +++ b/packages/core/assets/skills/brainstorm/SKILL.md @@ -1,10 +1,10 @@ --- name: brainstorm -description: Divergent lens fan-out with deterministic dedupe, rubric scoring, and a diversified shortlist with a forced outlier. +description: Divergent lens fan-out (eight-lens registry; six by default, four lean for callable runs) with deterministic dedupe, rubric scoring, and a diversified shortlist with a forced outlier whenever the shortlist has ≥2 slots. --- # Brainstorm -Lateral ideation on an idea, run as a fan-out of divergent lenses and distilled mechanically. Never bug fixes, never implementation plans. +Lateral ideation on an idea, run as a fan-out of divergent lenses (eight in the registry; six by default, four lean for callable runs) and distilled mechanically. Never bug fixes, never implementation plans. ## When to use - The human runs `/brainstorm ` or asks for ideas, directions or alternatives. @@ -15,7 +15,7 @@ Lateral ideation on an idea, run as a fan-out of divergent lenses and distilled 2. Launch every lens subagent in parallel, by exact ID (`es-lens-`), each with the brief. Fast tier; one lens each: inversion, constraint-removal, scamper, cross-domain, extreme-user, scale-shift, failure-first, adjacent-possible. 3. `es_brainstorm_record` — one call per lens. Caps are enforced; near-duplicates are collapsed against earlier ideas (convergence is a signal). 4. `es-brainstorm-critic` — scores every surviving idea 1-5 on novelty, upside, feasibility and fit via `es_brainstorm_score`. -5. `es_brainstorm_complete` — re-checks coverage and scores, ranks, and writes the diversified shortlist with one forced outlier slot to `.factory/brainstorm/`. +5. `es_brainstorm_complete` — re-checks coverage and scores, ranks, and writes the diversified shortlist with one forced outlier slot (whenever the shortlist has ≥2 slots) to `.factory/brainstorm/`. ## Output `.factory/brainstorm/brainstorm.json` (machine-readable) and `BRAINSTORM.md` (the shortlist, the full ranked field, collapsed duplicates, coverage and gaps). Report the shortlist to the human; do not paste the whole field. diff --git a/packages/core/src/brainstorm/engine.ts b/packages/core/src/brainstorm/engine.ts index 2305e11..180cacb 100644 --- a/packages/core/src/brainstorm/engine.ts +++ b/packages/core/src/brainstorm/engine.ts @@ -240,7 +240,9 @@ export const OUTLIER_THRESHOLD = 0.35 * Greedy diversified selection: quality plus distance from what is already * picked. The last slot is forced to an outlier — the best idea whose maximum * similarity to the shortlist is below the outlier threshold, or the most - * dissimilar idea when none is far enough. + * dissimilar idea when none is far enough — whenever the shortlist has ≥2 + * slots (and at least two ideas are ranked; a single-idea shortlist carries + * `outlier: false`). */ export function selectShortlist( ranked: readonly RankedIdea[], diff --git a/packages/core/src/brainstorm/ops.ts b/packages/core/src/brainstorm/ops.ts index 2854393..58c3702 100644 --- a/packages/core/src/brainstorm/ops.ts +++ b/packages/core/src/brainstorm/ops.ts @@ -113,7 +113,8 @@ export function brainstormTools(context: BrainstormOpsContext): EsToolDef[] { ideasPerLens: { type: "number", description: "Ideas per lens (1-10, default 5)." }, shortlistSize: { type: "number", - description: "Shortlist size including the forced outlier (2-12, default 7).", + description: + "Shortlist size (2-12, default 7); the last slot is a forced outlier whenever the shortlist has ≥2 slots.", }, dedupe: { type: "string", @@ -176,7 +177,7 @@ export function brainstormTools(context: BrainstormOpsContext): EsToolDef[] { " Give each one the brief (and constraints) plus how many ideas you expect.", `2. Record each lens's ideas with es_brainstorm_record (one call per lens, run "${run}").`, "3. Launch es-brainstorm-critic with the surviving idea ids and texts; it scores each with es_brainstorm_score.", - `4. Call es_brainstorm_complete with run "${run}". Dedupe, ranking and the shortlist (with one forced outlier) are computed there.`, + `4. Call es_brainstorm_complete with run "${run}". Dedupe, ranking and the shortlist (with one forced outlier whenever the shortlist has ≥2 slots) are computed there.`, "", `Rubric (1-5 each): ${RUBRIC.map((item) => item.id).join(", ")}.`, ].join("\n") @@ -344,7 +345,7 @@ export function brainstormTools(context: BrainstormOpsContext): EsToolDef[] { { name: "es_brainstorm_complete", description: - "Brainstormer, grill or factory: collapse duplicates, rank the scored ideas and write the diversified shortlist with a forced outlier slot. Returns the shortlist as JSON the caller can parse.", + "Brainstormer, grill or factory: collapse duplicates, rank the scored ideas and write the diversified shortlist with a forced outlier slot (whenever the shortlist has ≥2 slots). Returns the shortlist as JSON the caller can parse.", input: object({ run: { type: "string", description: "Run id from es_brainstorm_plan." }, allowPartial: { diff --git a/packages/core/test/__snapshots__/assets.test.ts.snap b/packages/core/test/__snapshots__/assets.test.ts.snap index 3093730..7f25617 100644 --- a/packages/core/test/__snapshots__/assets.test.ts.snap +++ b/packages/core/test/__snapshots__/assets.test.ts.snap @@ -5,7 +5,7 @@ exports[`agent assets prompt snapshot: a changed prompt must bump its version 1` "auditor-antithesis": "v2:15f4d7f75e5bb958", "auditor-thesis": "v2:b903e1353177f342", "brainstorm-critic": "v1:28fd3abc95c4a28a", - "brainstormer": "v3:eee9b20fb26482a4", + "brainstormer": "v4:278864b7a2cf7008", "deep-researcher": "v1:2c436cd49fd6aaf8", "designer": "v2:745908a2a4abff0b", "factory": "v9:4b48ec73da814ab3", From fa5f3f54ea5ce81041c48c3a6a9a15b7bfc74c4b Mon Sep 17 00:00:00 2001 From: S7 remediation Date: Sat, 10 Oct 2026 20:57:10 -0400 Subject: [PATCH 47/83] fix(remediation-audit/s7): release guard wording + cheap and bare-remote tests (Refs #238) --- packages/opencode/src/pr.ts | 10 ++- packages/opencode/test/pr-guard.test.ts | 80 +++++++++++++++++ packages/opencode/test/release.test.ts | 109 ++++++++++++++++++++++++ 3 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 packages/opencode/test/pr-guard.test.ts create mode 100644 packages/opencode/test/release.test.ts diff --git a/packages/opencode/src/pr.ts b/packages/opencode/src/pr.ts index a5c8fca..3ddc254 100644 --- a/packages/opencode/src/pr.ts +++ b/packages/opencode/src/pr.ts @@ -1,6 +1,10 @@ -// Release PR opener backed by the GitHub CLI. It pushes only the run's -// integration branch (never the base branch) and opens a draft PR for a human -// to review and merge. +// Factory release PR opener backed by the GitHub CLI. Every factory push +// path goes through here: it pushes only the run's `factory/` integration +// branch — `branch === base` or a non-`factory/` branch is refused before +// any push — and opens a draft PR for a human to review and merge. (Other +// push paths exist outside the factory: `es improve --open-pr` pushes +// `improve/*` branches on its own human-run path; phase fast-forwards and +// fleet landings never touch the base branch; only a human merges.) import type { PrOpener } from "@heretek-ai/es-core" import { buildPrArgs, git, run } from "@heretek-ai/es-core" diff --git a/packages/opencode/test/pr-guard.test.ts b/packages/opencode/test/pr-guard.test.ts new file mode 100644 index 0000000..c3a3a0d --- /dev/null +++ b/packages/opencode/test/pr-guard.test.ts @@ -0,0 +1,80 @@ +// PR-opener guard (#238): `ghPrOpener.open()` refuses `branch === base` +// and non-`factory/` branches before any git call, and builds the draft-PR +// argv through the shared builder. No network and no `gh` invocation: every +// refusal throws before the push, and the draft shape is asserted on the +// argv (the `gh pr create` spawn itself never runs in tests). +import { describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { buildPrArgs } from "@heretek-ai/es-core" +import { ghPrOpener } from "../src/pr.ts" + +const open = (root: string, overrides: { branch: string; base?: string; preset?: string } = { branch: "x" }) => + ghPrOpener.open({ + root, + branch: overrides.branch, + base: overrides.base ?? "rewrite", + title: "t", + body: "b", + ...(overrides.preset !== undefined ? { preset: overrides.preset } : {}), + }) + +describe("ghPrOpener.open() branch guard", () => { + test("branch === base is refused", async () => { + const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-")) + try { + await expect(open(root, { branch: "rewrite", base: "rewrite" })).rejects.toThrow(/refusing to push "rewrite"/) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + test("non-factory/ branches are refused", async () => { + const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-")) + try { + for (const branch of ["main", "feature/x", "improve/20260101", "factory"]) { + await expect(open(root, { branch })).rejects.toThrow(/refusing to push/) + } + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + test("the refusal precedes any git call (a non-repo root still reports the refusal)", async () => { + // `root` is not a git repository: had the push run first, the failure + // would be git's, not the guard's. The guard's error proves the order + // (`pr.ts` checks the branch before `git push --set-upstream`). + const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-")) + try { + await open(root, { branch: "main", base: "rewrite" }) + expect.unreachable("a non-factory/ branch must be refused") + } catch (error) { + expect((error as Error).message).toMatch(/refusing to push "main"/) + expect((error as Error).message).not.toMatch(/git/i) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + test("the branch guard runs before the preset-base guard", async () => { + const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-")) + try { + await open(root, { branch: "main", base: "main", preset: "self-dogfood" }) + expect.unreachable("branch == base must be refused first") + } catch (error) { + expect((error as Error).message).toMatch(/refusing to push "main"/) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) + +describe("release PR argv", () => { + test("a factory branch opens a draft PR against the base", () => { + const argv = buildPrArgs({ branch: "factory/run-1/integration", base: "rewrite", title: "t", body: "b" }) + expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"]) + expect(argv.slice(argv.indexOf("--head") + 1, argv.indexOf("--head") + 2)).toEqual(["factory/run-1/integration"]) + expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["rewrite"]) + }) +}) diff --git a/packages/opencode/test/release.test.ts b/packages/opencode/test/release.test.ts new file mode 100644 index 0000000..c3ffcf5 --- /dev/null +++ b/packages/opencode/test/release.test.ts @@ -0,0 +1,109 @@ +// Release base-branch safety against a local bare remote (#238): the +// factory release pushes a `factory/` branch only, `branch == base` (or a +// non-`factory/` branch) is refused before anything is pushed, and the base +// SHA on the remote is byte-identical afterwards. Local bare-remote fixture +// only (no real remote, no network); `gh` is never invoked — the authorized +// push runs the same `git push --set-upstream origin :` +// argv `pr.ts` uses, and the draft shape is asserted on the shared argv. +import { describe, expect, test } from "bun:test" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { buildPrArgs, git } from "@heretek-ai/es-core" +import { ghPrOpener } from "../src/pr.ts" + +const BASE = "rewrite" +const BRANCH = "factory/run-test-1/integration" + +interface Remote { + readonly root: string + readonly remote: string + readonly baseSha: string + cleanup(): Promise +} + +/** A repo with a base branch pushed to a local bare remote, plus one factory-branch commit. */ +async function bareRemote(): Promise { + const root = await mkdtemp(path.join(tmpdir(), "es-release-")) + const remote = await mkdtemp(path.join(tmpdir(), "es-release-remote-")) + const cleanup = async () => { + await rm(root, { recursive: true, force: true }) + await rm(remote, { recursive: true, force: true }) + } + try { + await git(remote, ["init", "--bare", "-q"]) + await git(root, ["init", "-q", "-b", BASE]) + await writeFile(path.join(root, "README.md"), "# release fixture\n") + await git(root, ["add", "-A"]) + await git(root, ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "init"]) + await git(root, ["remote", "add", "origin", remote]) + await git(root, ["push", "-q", "-u", "origin", `${BASE}:${BASE}`]) + const baseSha = (await git(root, ["rev-parse", BASE])).stdout.trim() + await git(root, ["checkout", "-q", "-b", BRANCH]) + await writeFile(path.join(root, "work.txt"), "factory work\n") + await git(root, ["add", "-A"]) + await git(root, ["-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "factory work"]) + return { root, remote, baseSha, cleanup } + } catch (error) { + await cleanup() + throw error + } +} + +const remoteSha = async (remote: string, ref: string) => + (await git(remote, ["rev-parse", "--verify", ref])).stdout.trim() + +const remoteBranches = async (remote: string) => + (await git(remote, ["for-each-ref", "--format=%(refname)", "refs/heads/"])).stdout + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + +describe("release against a local bare remote", () => { + test("branch == base is refused and the remote is untouched", async () => { + const fx = await bareRemote() + try { + await expect(ghPrOpener.open({ root: fx.root, branch: BASE, base: BASE, title: "t", body: "b" })).rejects.toThrow( + `refusing to push "${BASE}"`, + ) + expect(await remoteBranches(fx.remote)).toEqual(["refs/heads/rewrite"]) + expect(await remoteSha(fx.remote, "refs/heads/rewrite")).toBe(fx.baseSha) + } finally { + await fx.cleanup() + } + }) + + test("a non-factory/ branch is refused and the remote is untouched", async () => { + const fx = await bareRemote() + try { + await expect( + ghPrOpener.open({ root: fx.root, branch: "feature/x", base: BASE, title: "t", body: "b" }), + ).rejects.toThrow(/refusing to push/) + expect(await remoteBranches(fx.remote)).toEqual(["refs/heads/rewrite"]) + expect(await remoteSha(fx.remote, "refs/heads/rewrite")).toBe(fx.baseSha) + } finally { + await fx.cleanup() + } + }) + + test("the authorized push lands only the factory branch; the base SHA is byte-identical", async () => { + const fx = await bareRemote() + try { + const local = (await git(fx.root, ["rev-parse", BRANCH])).stdout.trim() + // The same argv `pr.ts` pushes with (the `gh pr create` spawn never + // runs in tests: no `gh` invocation here). + await git(fx.root, ["push", "--set-upstream", "origin", `${BRANCH}:${BRANCH}`]) + expect(await remoteBranches(fx.remote)).toEqual( + ["refs/heads/factory/run-test-1/integration", "refs/heads/rewrite"].sort(), + ) + expect(await remoteSha(fx.remote, `refs/heads/${BRANCH}`)).toBe(local) + expect(await remoteSha(fx.remote, "refs/heads/rewrite")).toBe(fx.baseSha) + // The release PR itself is a draft against the base. + const argv = buildPrArgs({ branch: BRANCH, base: BASE, title: "t", body: "b" }) + expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"]) + expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual([BASE]) + } finally { + await fx.cleanup() + } + }) +}) From 600bcfe42aca030523157efb2f3def596d8411cc Mon Sep 17 00:00:00 2001 From: S7 remediation Date: Sat, 10 Oct 2026 20:57:10 -0400 Subject: [PATCH 48/83] fix(remediation-audit/s7): packaging triple-pin and allowlist assertions (Refs #212) --- packages/core/test/pack-pins.test.ts | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 packages/core/test/pack-pins.test.ts diff --git a/packages/core/test/pack-pins.test.ts b/packages/core/test/pack-pins.test.ts new file mode 100644 index 0000000..4c87bc7 --- /dev/null +++ b/packages/core/test/pack-pins.test.ts @@ -0,0 +1,59 @@ +// Packaging pin assertions (#212): the opencode triple-pin moves together, +// every release manifest declares an explicit `files` allowlist with no test +// paths, and the `--release` rendering keeps the ` ` contract +// `publish.yml` parses. File reads only: no network, no registry, no pack. +// Structural fixes (a peer range, narrower allowlists, publish parsing, +// the runtime-matrix wording) route to S9 via CROSS-SQUAD REQUEST. +import { describe, expect, test } from "bun:test" +import { readFile } from "node:fs/promises" +import path from "node:path" +import { releasePackages, tarballSlug, workspacePackages } from "../../../scripts/packages.ts" + +const root = path.resolve(import.meta.dir, "../../..") +const manifest = async (dir: string) => + JSON.parse(await readFile(path.join(root, "packages", dir, "package.json"), "utf8")) as Record + +describe("the opencode triple-pin moves together", () => { + test("@opencode/plugin, @opencode/sdk and the e2e CLI share one version", async () => { + const opencode = await manifest("opencode") + const plugin = opencode.dependencies?.["@opencode/plugin"] as string + const sdk = opencode.devDependencies?.["@opencode/sdk"] as string + expect(plugin).toMatch(/^\d+\.\d+\.\d+$/) + expect(sdk).toBe(plugin) + const e2e = await readFile(path.join(root, ".github/workflows/e2e-feedback.yml"), "utf8") + const cliPins = [...e2e.matchAll(/@opencode\/cli-linux-x64@([0-9A-Za-z._-]+)/g)].map((match) => match[1]) + expect(cliPins.length).toBeGreaterThan(0) + expect(new Set(cliPins).size).toBe(1) + expect(cliPins[0]).toBe(plugin) + }) +}) + +describe("release tarball allowlists", () => { + test("every release manifest declares an explicit files allowlist with no test paths", async () => { + for (const pkg of await releasePackages(root)) { + const files = (await manifest(pkg.dir)).files as unknown + expect(Array.isArray(files) && (files as unknown[]).length > 0).toBe(true) + for (const entry of files as string[]) { + expect(typeof entry).toBe("string") + expect(entry).not.toMatch(/test|spec|__tests__/i) + } + } + }) + + test("fleet and web stay private (never released)", async () => { + const all = await workspacePackages(root) + for (const dir of ["fleet", "web"]) { + expect(all.find((pkg) => pkg.dir === dir)?.private).toBe(true) + } + }) +}) + +describe("the --release rendering contract", () => { + test('every line is " " with the slug derived from the manifest name', async () => { + for (const pkg of await releasePackages(root)) { + const line = `${pkg.dir} ${tarballSlug(pkg.name)}` + expect(line).toMatch(/^\S+ \S+$/) + expect(tarballSlug(pkg.name)).not.toContain(" ") + } + }) +}) From 72066c019621f9cf5dcbf854ab98e991d164dc4c Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:57:18 -0400 Subject: [PATCH 49/83] fix(remediation-audit/s5): mcp identity pin order + proof gaps (Refs #230, Refs #232) --- packages/cli/src/mcp.ts | 21 +-- packages/cli/test/mcp-identity.test.ts | 191 +++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 packages/cli/test/mcp-identity.test.ts diff --git a/packages/cli/src/mcp.ts b/packages/cli/src/mcp.ts index de3b26f..319cba8 100644 --- a/packages/cli/src/mcp.ts +++ b/packages/cli/src/mcp.ts @@ -2,10 +2,13 @@ // harness-neutral factory operations to harnesses without a native plugin and // to headless runs. It has no approve, trust, waive or resume tool: those are // human-only. Caller identity is PINNED, never piped: the harness adapter sets -// one agent id per server in its environment (ES_MCP_AGENT, else the legacy -// ES_AGENT/defaultAgent, both written by the human in the adapter config), and -// the per-call `agent` argument is ignored. Without an adapter identity the -// caller is "mcp", which maps to no seat, so every seat-checked tool refuses. +// one agent id per server in its environment (`ES_MCP_AGENT`, else the legacy +// `ES_AGENT`, else the `defaultAgent` option — all written by the human in the +// adapter config), and the per-call `agent` argument is ignored. Without an +// adapter identity the caller is "mcp", which maps to no seat, so every +// seat-checked tool refuses. Only a registry agent id (`factory`, +// `es-manager`, `es-programmer`, …) holds a seat: any other pinned string is +// still the caller, but a seat-less one, refused the same way as "mcp". import { stateDir as defaultStateDir, esTools, Factory, gateRunner } from "@heretek-ai/es-core" import { VERSION } from "./version.ts" @@ -26,7 +29,9 @@ export function createMcpServer(options: McpOptions) { const env = options.env ?? process.env // Pinned at construction: the adapter's environment, never the caller's // arguments. An agent piping JSON-RPC into this server cannot claim a seat. - const caller = env.ES_MCP_AGENT || options.defaultAgent || "mcp" + // Library consumers get the same order as the CLI wrapper: the adapter's + // ES_MCP_AGENT wins, then the legacy ES_AGENT, then the defaultAgent option. + const caller = env.ES_MCP_AGENT || env.ES_AGENT || options.defaultAgent || "mcp" const factory = new Factory(options.root, { gates: gateRunner(options.stateDir ? { stateDir: options.stateDir } : {}), ...(options.stateDir ? { stateDir: options.stateDir } : {}), @@ -35,7 +40,7 @@ export function createMcpServer(options: McpOptions) { const byName = new Map(tools.map((tool) => [tool.name.replace(/^es_/, ""), tool])) const listed = [...byName].map(([name, tool]) => ({ name, - description: `${tool.description}${name === "status" || name === "spec_validate" ? "" : " Caller identity is pinned by the adapter (ES_MCP_AGENT); a per-call `agent` argument is accepted but ignored."}`, + description: `${tool.description}${name === "status" || name === "spec_validate" ? "" : " Caller identity is pinned by the adapter (ES_MCP_AGENT, else ES_AGENT, else the default agent); a per-call `agent` argument is accepted but ignored."}`, inputSchema: { ...tool.input, properties: { @@ -43,7 +48,7 @@ export function createMcpServer(options: McpOptions) { agent: { type: "string", description: - "Ignored: the caller is the adapter's pinned ES_MCP_AGENT (or the default agent), never this argument.", + "Ignored: the caller is the adapter's pinned ES_MCP_AGENT (else ES_AGENT, else the default agent), never this argument.", }, }, }, @@ -61,7 +66,7 @@ export function createMcpServer(options: McpOptions) { capabilities: { tools: { listChanged: false } }, serverInfo: { name: "epistemic-swarm", version: options.version ?? VERSION }, instructions: - "Epistemic Swarm factory operations. Approvals, trust, waivers and resume are human-only (TUI or `es` CLI) and have no MCP tool. Caller identity is pinned: the adapter sets ES_MCP_AGENT in this server's environment (one server per agent); without it the caller is `mcp`, which holds no seat, so seat-checked tools refuse. A per-call `agent` argument is ignored.", + "Epistemic Swarm factory operations. Approvals, trust, waivers and resume are human-only (TUI or `es` CLI) and have no MCP tool. Caller identity is pinned: the adapter sets ES_MCP_AGENT in this server's environment (one server per agent), else the legacy ES_AGENT, else the default agent; without any of them the caller is `mcp`, which holds no seat, so seat-checked tools refuse. A per-call `agent` argument is ignored.", }) case "ping": return reply({}) diff --git a/packages/cli/test/mcp-identity.test.ts b/packages/cli/test/mcp-identity.test.ts new file mode 100644 index 0000000..fe5131e --- /dev/null +++ b/packages/cli/test/mcp-identity.test.ts @@ -0,0 +1,191 @@ +// MCP caller-identity pinning (#230, #232): the adapter pins one agent id per +// server in the server environment — `ES_MCP_AGENT`, else the legacy +// `ES_AGENT`, else the `defaultAgent` option — and the per-call `agent` +// argument is ignored. Without an adapter identity the caller is "mcp", which +// maps to no seat, so every seat-checked tool refuses. A pinned string that is +// not a registry agent id (e.g. `programmer`) pins to a seat-less identity: +// the caller string is still pinned, it just holds no seat. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { Factory, gateRunner, HUMAN_VERBS, readApproval } from "@heretek-ai/es-core" +import { createMcpServer } from "../src/mcp.ts" + +let root: string +let state: string + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-mcp-id-")) + state = await mkdtemp(path.join(tmpdir(), "es-mcp-id-state-")) + const proc = Bun.spawn(["git", "init", "-q", "-b", "main"], { cwd: root, stdout: "pipe", stderr: "pipe" }) + await proc.exited + await writeFile(path.join(root, "README.md"), "# x\n") +}) + +afterEach(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const call = (server: ReturnType, name: string, args: Record = {}, id = 1) => + server.handle({ jsonrpc: "2.0", id, method: "tools/call", params: { name, arguments: args } }) + +describe("mcp identity pin order (mcp.ts:29)", () => { + test("the legacy ES_AGENT pins the caller when ES_MCP_AGENT is absent", async () => { + const server = createMcpServer({ root, stateDir: state, env: { ES_AGENT: "factory" } }) + const refused = await call(server, "complete", {}) + expect(refused?.result.isError).toBe(true) + expect(refused?.result.content[0].text).toContain("called by factory") + }) + + test("ES_MCP_AGENT beats ES_AGENT beats defaultAgent", async () => { + const top = createMcpServer({ + root, + stateDir: state, + env: { ES_MCP_AGENT: "es-qa-functional", ES_AGENT: "factory" }, + defaultAgent: "factory", + }) + const pinned = await call(top, "complete", { agent: "factory" }) + expect(pinned?.result.isError).toBe(true) + expect(pinned?.result.content[0].text).toContain("called by es-qa-functional") + + const legacy = createMcpServer({ + root, + stateDir: state, + env: { ES_AGENT: "factory" }, + defaultAgent: "es-qa-functional", + }) + const viaLegacy = await call(legacy, "complete", {}) + expect(viaLegacy?.result.isError).toBe(true) + expect(viaLegacy?.result.content[0].text).toContain("called by factory") + + const fallback = createMcpServer({ root, stateDir: state, env: {}, defaultAgent: "factory" }) + const viaDefault = await call(fallback, "complete", {}) + expect(viaDefault?.result.isError).toBe(true) + expect(viaDefault?.result.content[0].text).toContain("called by factory") + }) + + test("a pinned string without a registry seat stays seat-less but pinned", async () => { + const server = createMcpServer({ root, stateDir: state, env: { ES_MCP_AGENT: "programmer" } }) + const refused = await call(server, "complete", {}) + expect(refused?.result.isError).toBe(true) + // Registry ids are `es-programmer`/`es-manager`/`factory`: the literal + // `programmer` holds no seat, yet the caller string is still the pin. + expect(refused?.result.content[0].text).toContain("called by programmer") + }) +}) + +describe("as-mcp refusal across every seat-gated tool", () => { + // Every tool whose execute path opens with requireSeat, called with an + // empty adapter environment (caller "mcp", no seat). + const seatGated: Array<{ tool: string; args: Record }> = [ + { tool: "frontier_write", args: { frontier: {} } }, + { tool: "research_complete", args: {} }, + { tool: "build_start", args: {} }, + { tool: "qa_verdict", args: { verdict: "pass", notes: "n" } }, + { tool: "tiebreak", args: { verdict: "pass", notes: "n" } }, + { tool: "replan", args: { notes: "n" } }, + { tool: "release", args: {} }, + { tool: "complete", args: {} }, + ] + for (const { tool, args } of seatGated) { + test(`${tool} refuses the unpinned caller as mcp`, async () => { + const server = createMcpServer({ root, stateDir: state, env: {} }) + const refused = await call(server, tool, args) + expect(refused?.result.isError).toBe(true) + expect(refused?.result.content[0].text).toContain("called by mcp") + }) + } +}) + +describe("no MCP tool grants a human-only verb (derived from HUMAN_VERBS)", () => { + test("the coarse tool list is exactly the 12 read/request/seat tools", async () => { + const server = createMcpServer({ root, stateDir: state }) + const list = await server.handle({ jsonrpc: "2.0", id: 1, method: "tools/list" }) + const names = list?.result.tools.map((tool: { name: string }) => tool.name).sort() + // A future human-only verb gaining an MCP tool changes this list and fails loudly here. + expect(names).toEqual( + [ + "build_start", + "complete", + "frontier_write", + "gates_run", + "qa_verdict", + "release", + "replan", + "request_approval", + "research_complete", + "spec_validate", + "status", + "tiebreak", + ].sort(), + ) + }) + + test("no listed tool (except the request-only approval) matches a HUMAN_VERBS rule", async () => { + const server = createMcpServer({ root, stateDir: state }) + const list = await server.handle({ jsonrpc: "2.0", id: 1, method: "tools/list" }) + const names = list?.result.tools.map((tool: { name: string }) => tool.name) as string[] + const humanOnly = (verb: string, sub: string | undefined): boolean => { + const rule = HUMAN_VERBS.find(([name]) => name === verb) + if (!rule) return false + const [, predicate] = rule + return predicate === undefined ? true : predicate(sub) + } + for (const name of names) { + if (name === "request_approval") continue // request-only: proven below, never grants. + const tokens = name.split(/[_-]/) + const matched = HUMAN_VERBS.some(([verb, predicate]) => { + if (!tokens.includes(verb)) return false + if (predicate === undefined) return true + return tokens.some((token) => token !== verb && predicate(token)) + }) + expect([name, matched, humanOnly("never-a-verb", undefined)]).toEqual([name, false, false]) + } + }) + + test("request_approval records a request without granting anything", async () => { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.writeFrontier("grill", { + version: "1.1", + idea: "mcp test", + spendCeiling: { currency: "USD", maxAmount: 3 }, + settled: true, + nodes: [{ id: "a", question: "q?", answer: "yes", status: "settled" }], + }) + const server = createMcpServer({ root, stateDir: state, env: { ES_MCP_AGENT: "factory" } }) + const requested = await call(server, "request_approval", { stage: "frontier" }) + expect(requested?.result.isError).toBeUndefined() + expect(requested?.result.content[0].text).toContain("Requested human approval") + // Request-only: the approval file is still absent (a human grants it elsewhere). + expect(await readApproval(root, "frontier")).toBeUndefined() + }) +}) + +describe("mcp identity over real stdio", () => { + test("serveStdio pins ES_MCP_AGENT: a piped agent claim is refused as the pin", async () => { + const proc = Bun.spawn(["bun", path.join(import.meta.dir, "../src/main.ts"), "mcp", "--cwd", root], { + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + // The literal `programmer` holds no registry seat (#232.1), so the + // seat check itself names the pin — the strongest stdio proof. + env: { ...process.env, ES_MCP_AGENT: "programmer" }, + }) + proc.stdin.write( + `${JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: "complete", arguments: { agent: "factory" } } })}\n`, + ) + proc.stdin.end() + const lines = (await new Response(proc.stdout).text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + await proc.exited + expect(lines.map((line) => line.id)).toEqual([1]) + expect(lines[0].result.isError).toBe(true) + // The per-call `agent: "factory"` is ignored: the refusal names the pin. + expect(lines[0].result.content[0].text).toContain("called by programmer") + expect(await readFile(path.join(root, "README.md"), "utf8")).toContain("# x") + }) +}) From 99a8b2a3c8fbd2d0b67bf90782d98f80e72fc695 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:57:26 -0400 Subject: [PATCH 50/83] fix(remediation-audit/s6): brainstorm nested-refusal registry pins (Refs #269) --- packages/core/test/brainstorm.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/core/test/brainstorm.test.ts b/packages/core/test/brainstorm.test.ts index a936851..560e6d6 100644 --- a/packages/core/test/brainstorm.test.ts +++ b/packages/core/test/brainstorm.test.ts @@ -7,6 +7,8 @@ import { tmpdir } from "node:os" import path from "node:path" import type { BrainstormIdea, BrainstormScore } from "../src/index.ts" import { + AGENTS, + BRAINSTORM_FANOUT_SPAWNS, brainstormRunPaths, brainstormTools, buildPlan, @@ -455,3 +457,22 @@ describe("callable runs (#108)", () => { } }) }) + +describe("nested refusal at depth 1 (#269)", () => { + test("lens subagents and the critic spawn nothing; only primaries fan out", () => { + const byId = new Map(AGENTS.map((agent) => [agent.id, agent])) + // The mechanism: empty `spawns` + deny-by-default + the foreground guard. + // A lens or the critic asked to launch a subagent has nothing to launch. + for (const lens of LENSES) { + const agent = byId.get(`es-lens-${lens.id}`)! + expect(agent, `es-lens-${lens.id}`).toBeDefined() + expect(agent.spawns).toEqual([]) + } + expect(byId.get("es-brainstorm-critic")!.spawns).toEqual([]) + // The calling primaries fan out themselves, at depth 1, in the foreground. + for (const id of ["brainstormer", "grill", "factory"]) { + const spawns = byId.get(id)!.spawns + for (const fanout of BRAINSTORM_FANOUT_SPAWNS) expect(spawns, id).toContain(fanout) + } + }) +}) From aa101af3810b03203ea92f7b07452771f6aa4f41 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:59:01 -0400 Subject: [PATCH 51/83] fix(remediation-audit/s6): hook sandbox knob + PostToolUseFailure/deny/corpus tests (Refs #265, Refs #268) --- packages/core/src/hooks/engine.ts | 22 ++++++ packages/core/src/hooks/run.ts | 23 ++++++ packages/core/src/hooks/spec.ts | 2 + packages/core/test/hooks.test.ts | 122 +++++++++++++++++++++++++++++- 4 files changed, 166 insertions(+), 3 deletions(-) diff --git a/packages/core/src/hooks/engine.ts b/packages/core/src/hooks/engine.ts index d4edd30..8f68c71 100644 --- a/packages/core/src/hooks/engine.ts +++ b/packages/core/src/hooks/engine.ts @@ -7,6 +7,7 @@ // handler is written to the audit log. import { appendAuditEntry } from "../audit/chain.ts" import { isTrusted } from "../trust/store.ts" +import type { SandboxSpec } from "../trust/sandbox.ts" import { type HookSource, hookSources, type LoadedHooks, loadHooks } from "./load.ts" import { ifMatches, matcherMatches } from "./match.ts" import { type HandlerOutcome, runHandler } from "./run.ts" @@ -19,6 +20,16 @@ export interface HookEngineOptions { readonly sources?: HookSource[] readonly generate?: (prompt: string, model?: string) => Promise readonly audit?: boolean + /** + * Sandbox for hook command subprocesses (#265). Absent runs unsandboxed + * with the parent environment (Claude Code parity; project hooks are + * trusted code). "readonly" makes everything read-only; a spec runs with + * one writable directory. Without bwrap the run errors and gates fail + * closed while observers fail open. + */ + readonly sandbox?: "none" | "readonly" | Omit + /** No network inside the sandbox (only meaningful with `sandbox`). */ + readonly offline?: boolean } export interface BaseInput { @@ -188,6 +199,15 @@ export class HookEngine { } private async execute(event: string, items: SourcedHandler[], input: Record): Promise { + // The sandbox knob rides every handler run (#265); per-handler timeout + // and x-es-offline are honored in run.ts. + const sandboxContext = + this.options.sandbox !== undefined || this.options.offline !== undefined + ? { + ...(this.options.sandbox !== undefined ? { sandbox: this.options.sandbox } : {}), + ...(this.options.offline !== undefined ? { offline: this.options.offline } : {}), + } + : {} const ran = await Promise.all( items.map(async (item): Promise => { const project = !item.source.startsWith("user:") @@ -205,6 +225,7 @@ export class HookEngine { root: this.options.root, event, input, + ...sandboxContext, ...(this.options.generate ? { generate: this.options.generate } : {}), }) return { item, outcome: { status: "ok", durationMs: 0 } } @@ -213,6 +234,7 @@ export class HookEngine { root: this.options.root, event, input, + ...sandboxContext, ...(this.options.generate ? { generate: this.options.generate } : {}), }) return { item, outcome } diff --git a/packages/core/src/hooks/run.ts b/packages/core/src/hooks/run.ts index a9833fc..ba2985b 100644 --- a/packages/core/src/hooks/run.ts +++ b/packages/core/src/hooks/run.ts @@ -6,6 +6,15 @@ // - http: POST the input; 2xx JSON body decides, anything else is non-blocking. // - prompt: evaluated through an injected model call; {ok, reason, impossible}. // - mcp_tool and agent: not available in the bridge (reported as skipped). +// +// Sandboxing (#265): command subprocesses run with the parent environment, +// unsandboxed, by design (Claude Code parity) — hook commands are trusted +// code (project hooks need a human `es trust` before they run at all), so the +// parent env, secrets included, is visible to them. Callers that need +// isolation pass `sandbox`/`offline` in the RunContext (honored per handler: +// `x-es-offline` overrides the context default); without bwrap the sandboxed +// run fails and the engine's gate/observer policy decides (fail closed/open). +import type { SandboxSpec } from "../trust/sandbox.ts" import { type RunResult, run } from "../util/proc.ts" import type { HookHandler } from "./spec.ts" @@ -30,6 +39,15 @@ export interface RunContext { readonly signal?: AbortSignal /** Model call for prompt hooks; absent means prompt hooks are skipped. */ readonly generate?: (prompt: string, model?: string) => Promise + /** + * Sandbox for command subprocesses (#265). Absent (the default) runs + * unsandboxed with the parent environment, as Claude Code does. "readonly" + * makes everything read-only; a spec runs with one writable directory. + * Throws inside the outcome (error status) when bwrap is unavailable. + */ + readonly sandbox?: "none" | "readonly" | Omit + /** No network inside the sandbox (only meaningful with `sandbox`). */ + readonly offline?: boolean } const DEFAULT_TIMEOUT_S: Record = { UserPromptSubmit: 30 } @@ -69,6 +87,7 @@ async function runCommand( ? [substitute(handler.command, context.root), ...handler.args.map((arg) => substitute(arg, context.root))] : ["sh", "-c", handler.command] let result: RunResult + const offline = handler["x-es-offline"] ?? context.offline try { result = await run(argv, { cwd: context.root, @@ -77,6 +96,10 @@ async function runCommand( inheritEnv: true, env: { CLAUDE_PROJECT_DIR: context.root, ES_HOOK_EVENT: context.event }, maxOutputBytes: 1_000_000, + // The sandbox knob, honored per handler (#265): absent runs unsandboxed + // with the parent environment, as Claude Code does. + ...(context.sandbox !== undefined && context.sandbox !== "none" ? { sandbox: context.sandbox } : {}), + ...(offline !== undefined ? { offline } : {}), ...(context.signal ? { signal: context.signal } : {}), }) } catch (error) { diff --git a/packages/core/src/hooks/spec.ts b/packages/core/src/hooks/spec.ts index e6ae9ae..ad9cad0 100644 --- a/packages/core/src/hooks/spec.ts +++ b/packages/core/src/hooks/spec.ts @@ -35,6 +35,8 @@ export const CommandHookSchema = z.looseObject({ async: z.boolean().optional(), asyncRewake: z.boolean().optional(), shell: z.enum(["bash", "powershell"]).optional(), + /** No network for this command when run sandboxed (#265; needs the engine sandbox). */ + "x-es-offline": z.boolean().optional(), ...common, }) diff --git a/packages/core/test/hooks.test.ts b/packages/core/test/hooks.test.ts index 3568aac..f107dba 100644 --- a/packages/core/test/hooks.test.ts +++ b/packages/core/test/hooks.test.ts @@ -14,6 +14,7 @@ import { trustProject, verifyAuditChain, } from "../src/index.ts" +import { resetBwrapProbe } from "../src/util/proc.ts" let root: string let state: string @@ -44,14 +45,24 @@ async function script(name: string, body: string) { } async function engine( hooks: Record, - options: { trust?: boolean; generate?: (p: string) => Promise } = {}, + options: { + trust?: boolean + generate?: (p: string) => Promise + sandbox?: "none" | "readonly" + offline?: boolean + } = {}, ) { await writeFile(path.join(root, ".opencode/hooks.json"), JSON.stringify({ hooks })) + const extra = { + ...(options.generate ? { generate: options.generate } : {}), + ...(options.sandbox !== undefined ? { sandbox: options.sandbox } : {}), + ...(options.offline !== undefined ? { offline: options.offline } : {}), + } let created = await HookEngine.create({ root, stateDir: state, sources: sources(), - ...(options.generate ? { generate: options.generate } : {}), + ...extra, }) if (options.trust !== false) { const status = created.status() @@ -60,7 +71,7 @@ async function engine( root, stateDir: state, sources: sources(), - ...(options.generate ? { generate: options.generate } : {}), + ...extra, }) } return created @@ -100,6 +111,30 @@ describe("matching", () => { ).toBe(path.join(root, "x.ts")) }) + test("matcher corpus: exact lists vs unanchored regex (the Claude Code rule, #268)", () => { + // The rule (match.ts): "*", "" or omitted matches everything; a value of + // only [A-Za-z0-9_-, |] is an exact-match list; anything else is an + // unanchored JavaScript regex (so "Edit" never matches "NotebookEdit", + // while "Edit.*" does). + const cases: Array<[string | undefined, string, boolean]> = [ + [undefined, "Bash", true], + ["", "Bash", true], + ["*", "Anything", true], + ["Bash", "Bash", true], + ["Bash", "bash", false], + ["Edit|Write", "Write", true], + ["Edit, Write", "Edit", true], + ["Edit", "NotebookEdit", false], + ["Edit.*", "NotebookEdit", true], + ["^Edit", "NotebookEdit", false], + ["mcp__memory__.*", "mcp__memory__create", true], + ["mcp__memory", "mcp__memory__create", false], + ["[invalid", "Bash", false], + ] + for (const [matcher, value, expected] of cases) + expect(matcherMatches(matcher, [value]), `${String(matcher)} vs ${value}`).toBe(expected) + }) + test("`if` rules use permission syntax over subcommands and paths", () => { expect(subcommands("FOO=bar git push && npm test").parts).toEqual(["git push", "npm test"]) const tool = (command: string) => ({ toolNames: ["Bash", "shell"], toolInput: { command }, cwd: root }) @@ -288,6 +323,87 @@ describe("other events", () => { }) expect((await hooks.shellPreExec(base, { command: "ls", cwd: root })).command).toBeUndefined() }) + + test("x-es.ShellPreExec deny wins and drops the rewritten command (#268)", async () => { + const hooks = await engine({ + "x-es.ShellPreExec": [ + { + hooks: [ + { + type: "command", + command: `echo '{"hookSpecificOutput":{"permissionDecision":"deny","permissionDecisionReason":"no shells here","updatedInput":{"command":"evil"}}}'`, + }, + ], + }, + ], + }) + // deny + command: the `command && !deny` precedence keeps the deny only. + const denied = await hooks.shellPreExec(base, { command: "ls", cwd: root }) + expect(denied.deny).toBe("no shells here") + expect(denied.command).toBeUndefined() + + const blockers = await engine({ + "x-es.ShellPreExec": [{ hooks: [{ type: "command", command: `echo "blocked reason" >&2; exit 2` }] }], + }) + expect((await blockers.shellPreExec(base, { command: "ls", cwd: root })).deny).toBe("blocked reason") + }) + + test("PostToolUseFailure routes failing tools; PostToolUse does not fire on failure (#268)", async () => { + const hooks = await engine({ + PostToolUse: [ + { + matcher: "Write", + hooks: [{ type: "command", command: `echo '{"decision":"block","reason":"post ok"}'` }], + }, + ], + PostToolUseFailure: [ + { + matcher: "Write", + hooks: [{ type: "command", command: `echo '{"decision":"block","reason":"post-failure lint"}'` }], + }, + ], + }) + const write = fromOpenCode("write", { path: "a.ts", content: "" }, root) + const failed = await hooks.postToolUse(base, write, "t", "boom", true) + expect(failed.feedback).toEqual(["post-failure lint"]) + const ok = await hooks.postToolUse(base, write, "t", { ok: true }, false) + expect(ok.feedback).toEqual(["post ok"]) + }) + + test("hook commands run with the parent environment by design (Claude Code parity, #265)", async () => { + process.env.ES_HOOK_PARENT_ENV_PIN = "pinned-123" + try { + const hooks = await engine({ + UserPromptSubmit: [{ hooks: [{ type: "command", command: "echo \"pin=$ES_HOOK_PARENT_ENV_PIN\"" }] }], + }) + expect(await hooks.userPromptSubmit(base, "hi")).toMatchObject({ additionalContext: ["pin=pinned-123"] }) + } finally { + delete process.env.ES_HOOK_PARENT_ENV_PIN + } + }) + + test("the engine sandbox knob reaches hook subprocesses: gates fail closed, observers fail open (#265)", async () => { + resetBwrapProbe(false) + try { + const hooks = await engine( + { + PreToolUse: [ + { matcher: "Bash", hooks: [{ type: "command", command: "true", "x-es-policy": "gate" }] }, + { matcher: "Read", hooks: [{ type: "command", command: "true" }] }, + ], + }, + { sandbox: "readonly" }, + ) + const gated = await hooks.preToolUse(base, bash("ls"), "t1") + expect(gated.decision).toBe("deny") + expect(gated.reason).toContain("bwrap") + const observed = await hooks.preToolUse(base, fromOpenCode("read", { path: "a" }, root), "t2") + expect(observed.decision).toBeUndefined() + expect(observed.notices.join("\n")).toContain("bwrap") + } finally { + resetBwrapProbe() + } + }) }) describe("loading", () => { From 113b41357c399203c55430b709318fe51bc15288 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:59:32 -0400 Subject: [PATCH 52/83] fix(remediation-audit/s6): SessionStart unmatched-source notice (Refs #289) --- packages/core/src/hooks/engine.ts | 12 +++++++++++- packages/core/test/hooks.test.ts | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/core/src/hooks/engine.ts b/packages/core/src/hooks/engine.ts index 8f68c71..6d0460f 100644 --- a/packages/core/src/hooks/engine.ts +++ b/packages/core/src/hooks/engine.ts @@ -410,7 +410,17 @@ export class HookEngine { ): Promise<{ additionalContext: string[]; notices: string[] }> { const items = this.select("SessionStart", [source]) const result = { additionalContext: [] as string[], notices: [] as string[] } - if (!items.length) return result + if (!items.length) { + // An unmatched source is an explicit notice, never a silent no-op + // (#289): the bridge fires only startup + compact, so a hook filtered + // to resume/clear/fork would otherwise sit inert with no sign. Silent + // only when no SessionStart hook is registered at all. + if (this.loaded.handlers.some((item) => item.event === "SessionStart")) + result.notices.push( + `SessionStart: hook(s) registered but none match source "${source}" (the bridge fires startup + compact)`, + ) + return result + } const ran = await this.execute("SessionStart", items, { ...this.common("SessionStart", base), source }) for (const { item, outcome } of ran) { this.failure(item, outcome, result.notices) diff --git a/packages/core/test/hooks.test.ts b/packages/core/test/hooks.test.ts index f107dba..1bd4b39 100644 --- a/packages/core/test/hooks.test.ts +++ b/packages/core/test/hooks.test.ts @@ -382,6 +382,26 @@ describe("other events", () => { } }) + test("SessionStart routes every source; an unmatched source is an explicit notice, never silent (#289)", async () => { + const hooks = await engine({ + SessionStart: [ + { matcher: "startup", hooks: [{ type: "command", command: "echo 'project uses bun'" }] }, + { matcher: "resume", hooks: [{ type: "command", command: "echo 'welcome back'" }] }, + ], + }) + expect(await hooks.sessionStart(base, "startup")).toMatchObject({ additionalContext: ["project uses bun"] }) + expect(await hooks.sessionStart(base, "resume")).toMatchObject({ additionalContext: ["welcome back"] }) + // compact/clear/fork have no matching hook here: explicit notice, not silence. + for (const source of ["compact", "clear", "fork"] as const) { + const result = await hooks.sessionStart(base, source) + expect(result.additionalContext).toEqual([]) + expect(result.notices.join("\n")).toContain(`none match source "${source}"`) + } + // No SessionStart hook at all: silent (nothing to be confused about). + const bare = await engine({ PostToolUse: [{ hooks: [{ type: "command", command: "true" }] }] }) + expect((await bare.sessionStart(base, "resume")).notices).toEqual([]) + }) + test("the engine sandbox knob reaches hook subprocesses: gates fail closed, observers fail open (#265)", async () => { resetBwrapProbe(false) try { From b5f26bb95bacf249ea5eaddb0ec2a811179b1ab5 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 20:59:51 -0400 Subject: [PATCH 53/83] fix(remediation-audit/s6): codeaudit short-excerpt verdict test + schema comment (Refs #241) --- packages/core/src/schema/codeaudit.ts | 2 +- packages/core/test/codeaudit.test.ts | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/core/src/schema/codeaudit.ts b/packages/core/src/schema/codeaudit.ts index 0a031bb..a89c225 100644 --- a/packages/core/src/schema/codeaudit.ts +++ b/packages/core/src/schema/codeaudit.ts @@ -19,7 +19,7 @@ export const AuditFindingSchema = z /** POSIX path relative to the audited tree's root. */ file: z.string().min(1), lines: z.tuple([z.number().int().positive(), z.number().int().positive()]), - /** Verbatim code from those lines (at least 12 characters). */ + /** Verbatim code from those lines (the witness enforces the 12-char floor via verifySpan; the schema keeps min(1) so the witness reports the reason). */ excerpt: z.string().min(1), detail: z.string().min(8), remediation: z.string().optional(), diff --git a/packages/core/test/codeaudit.test.ts b/packages/core/test/codeaudit.test.ts index ad25638..b298688 100644 --- a/packages/core/test/codeaudit.test.ts +++ b/packages/core/test/codeaudit.test.ts @@ -143,6 +143,19 @@ describe("the line-pointer law at es_audit_verdict", () => { expect(await Bun.file(factoryLayout(fx.root).auditRecords(audit.id)).exists()).toBe(false) }) + test("an excerpt under 12 characters is refused as too short to be evidence (#241)", async () => { + const audit = await openPathAudit() + const verdict = tools()("es_audit_verdict") + // "db.query" (8 chars) is verbatim on line 3, so only the witness's + // 12-char floor (verifySpan) can refuse it — pins the short branch at + // verdict level, not just in the sibling span tests. + const short = { ...injection, lines: [3, 3], excerpt: "db.query" } + await expect( + verdict.execute({ audit: audit.id, verdict: "fail", findings: [short], notes: "n" }, { agent: "es-auditor-antithesis" }), + ).rejects.toThrow(/too short to be evidence/) + expect((await factory.read())!.audits[0]!.antithesis).toBeUndefined() + }) + test("malformed findings and verdicts that contradict their findings are refused", async () => { const audit = await openPathAudit() const verdict = tools()("es_audit_verdict") From 00eee94d621763ccdc50904dee0a132f5036402e Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:01:00 -0400 Subject: [PATCH 54/83] fix(remediation-audit/s5): single research command + visibility proofs (Refs #274, Refs #276) --- packages/opencode/src/server.ts | 649 ++++++++++++---------- packages/opencode/test/compaction.test.ts | 46 ++ packages/opencode/test/slash.test.ts | 184 ++++++ packages/opencode/test/visibility.test.ts | 95 ++++ 4 files changed, 677 insertions(+), 297 deletions(-) create mode 100644 packages/opencode/test/compaction.test.ts create mode 100644 packages/opencode/test/slash.test.ts create mode 100644 packages/opencode/test/visibility.test.ts diff --git a/packages/opencode/src/server.ts b/packages/opencode/src/server.ts index 38aba59..dfba18a 100644 --- a/packages/opencode/src/server.ts +++ b/packages/opencode/src/server.ts @@ -1,6 +1,9 @@ // Epistemic Swarm for OpenCode v2 (server plugin). Additive: registers our // agents, tools, skills, commands, hooks and RPC through runtime transforms; -// writes no config files and never touches built-in agents or defaults. +// setup writes no project-tree files and no config files (the engine key goes +// to the masked state dir; spend and seat tracking later write +// .factory/runtime/* while sessions run). It never touches built-in agents or +// defaults. import { readdir, readFile } from "node:fs/promises" import path from "node:path" import { @@ -27,7 +30,7 @@ import { createPolicyHooks } from "./policy.ts" import { createWebCache, PendingSearches, registerWebsearch } from "./research.ts" import { createRpcHandlers } from "./rpc.ts" import { EsRpc } from "./rpc-def.ts" -import { createRuntime, parseOptions } from "./runtime.ts" +import { createRuntime, parseOptions, type Runtime } from "./runtime.ts" import { createSeatTracker } from "./seats.ts" import { createSessionHooks, createSpendTracker } from "./session.ts" import { registerTools } from "./tools.ts" @@ -47,6 +50,346 @@ export function parseDeepResearchArgs(text: string): DeepResearchArgs { return { query, ceiling } } +/** The session surface slash commands use (the host's `ctx.session`; stubbed in tests). */ +export interface CommandSession { + switchAgent(sessionID: string, agent: string): Promise + sendPrompt(sessionID: string, prompt: Record, text: string, delivery: unknown): Promise + synthetic(sessionID: string, text: string): Promise +} + +/** One slash command: its name, description and host-independent body. */ +export interface CommandSpec { + readonly name: string + readonly description: string + readonly execute: (args: { + readonly sessionID: string + readonly prompt: { readonly text?: string } & Record + readonly delivery?: unknown + }) => Promise +} + +/** The hook-bridge surface slash commands read (status for `/hooks`). */ +export interface CommandBridge { + readonly engine: { + refresh(): Promise + status(): { + readonly handlers: number + readonly projectHandlers: number + readonly trusted: boolean + readonly projectLines: readonly string[] + readonly diagnostics: readonly string[] + } + readonly recent: readonly string[] + } +} + +/** + * Every slash command the plugin registers, as data: the host loop below + * adds each one, and tests invoke each body against a stub session to prove + * routing (seat/verb per command) and name uniqueness (one `research`). + */ +export function esCommandSpecs( + io: CommandSession, + runtime: Runtime, + bridge: CommandBridge, + runners: { readonly runGates?: typeof runGates; readonly git?: typeof git } = {}, +): CommandSpec[] { + const specs: CommandSpec[] = [] + const gates = runners.runGates ?? runGates + const gitStatus = runners.git ?? git + specs.push({ + name: "grill", + description: "Grill an idea until the design and spend ceiling are settled (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + await io.switchAgent(sessionID, "grill") + await io.sendPrompt( + sessionID, + prompt, + prompt.text?.trim() ? `Grill me on: ${prompt.text}` : "Start grilling me on my idea.", + delivery, + ) + }, + }) + specs.push({ + name: "brainstorm", + description: "Fan out divergent lenses on an idea and return a diversified shortlist (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + await io.switchAgent(sessionID, "brainstormer") + await io.sendPrompt( + sessionID, + prompt, + prompt.text?.trim() + ? `Brainstorm: ${prompt.text}` + : "Ask me for the idea to brainstorm (one sentence), then run the lens fan-out.", + delivery, + ) + }, + }) + specs.push({ + name: "harvest", + description: "Tear down competitor projects: licences, feature matrix, clean-room specs (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + await io.switchAgent(sessionID, "harvester") + await io.sendPrompt( + sessionID, + prompt, + prompt.text?.trim() + ? `Darkharvest: ${prompt.text}` + : "Ask me for the teardown objective and the candidate projects, then run the harvest.", + delivery, + ) + }, + }) + specs.push({ + name: "design", + description: "Interview me into a design system: tokens, contrast gates, style guide (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + await io.switchAgent(sessionID, "designer") + await io.sendPrompt( + sessionID, + prompt, + prompt.text?.trim() + ? `Run the design interview. Product: ${prompt.text}` + : "Ask me for the product name and start the design interview, one question at a time.", + delivery, + ) + }, + }) + specs.push({ + name: "scout", + description: "Find and vet open-source candidates for a feature: licenses, maintenance, CVEs (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + await io.switchAgent(sessionID, "scout") + await io.sendPrompt( + sessionID, + prompt, + prompt.text?.trim() + ? `Scout: ${prompt.text}` + : "Ask me for the feature to scout and our license posture, then plan the candidates.", + delivery, + ) + }, + }) + specs.push({ + name: "research", + // One name, one body (#274): a leading `deep` runs the adversarial deep + // flow (switching to the deep-researcher); anything else reads the + // research stage (report, audit coverage, cached sources). + description: + "Deep research to a grounded report, or the research stage: report, audit coverage and cached sources (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + const say = async (text: string) => { + await io.synthetic(sessionID, text) + } + const text = prompt.text?.trim() ?? "" + if (text.split(/\s+/)[0]?.toLowerCase() !== "deep") { + const layout = factoryLayout(runtime.root) + const report = await readFile(layout.researchReport, "utf8").catch(() => undefined) + const coverage = await readJson<{ + claims?: number + grounded?: number + untagged?: number + ungrounded?: number + malformed?: number + sources?: number + passed?: boolean + }>(path.join(layout.research, "coverage.json")).catch(() => undefined) + const sources = (await readdir(researchSourcesDir(runtime.root)).catch(() => [])).filter((file) => + file.endsWith(".md"), + ) + const lines = [ + report === undefined + ? "No research report yet (.factory/research/REPORT.md)." + : `Report: ${report.length} characters in .factory/research/REPORT.md.`, + coverage + ? `Audit: ${coverage.grounded ?? 0}/${coverage.claims ?? 0} claims grounded (untagged ${coverage.untagged ?? 0}, ungrounded ${coverage.ungrounded ?? 0}, malformed ${coverage.malformed ?? 0}); cited sources ${coverage.sources ?? 0}; passed ${coverage.passed ? "yes" : "no"}.` + : "No coverage.json yet (es_research_complete writes it).", + `Cached sources: ${sources.length}.`, + ] + return say(lines.join("\n")) + } + const parsed = parseDeepResearchArgs(text) + if ("error" in parsed && parsed.error === "usage") { + return say("Usage: /research deep --max-usd N") + } + if ("error" in parsed) { + return say( + "Deep research needs a spend ceiling you set: /research deep --max-usd N. Seats halt when the spend reaches it.", + ) + } + const { query, ceiling } = parsed + const existing = await runtime.factory.read().catch(() => undefined) + if (existing && existing.mode !== "research") { + return say( + "This project holds a build run. Start deep research in a directory without one (`es research deep --output --max-usd N` at a terminal).", + ) + } + if (existing?.stage === "HALTED") { + return say( + "This research run is halted; resume it first (`es factory resume` at a terminal, or /es-resume to preview).", + ) + } + if (existing?.stage === "DONE") { + return say( + "This research run is done. For a new question, start one in a fresh directory (`es research deep --output --max-usd N` at a terminal).", + ) + } + if (!existing) { + try { + // The human typed /research deep: beginning the run is their action. + await runtime.factory.beginResearchRun({ objective: query, ceilingUSD: Number(ceiling) }, "human:tui") + } catch (error) { + return say(`Could not start the research run: ${error instanceof Error ? error.message : String(error)}`) + } + } + await io.switchAgent(sessionID, "deep-researcher") + await io.sendPrompt( + sessionID, + prompt, + `${await runtime.factory.summary()}\nDeep research runs adversarially, with no human answering questions mid-flow; follow your Flow through es_research_complete.`, + delivery, + ) + }, + }) + specs.push({ + name: "audit", + description: "Open a code audit of the active phase or a path and run the auditor pair (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + const target = prompt.text?.trim() + const say = async (text: string) => { + await io.synthetic(sessionID, text) + } + if (!target) return say("Usage: /audit ") + const state = await runtime.factory.read().catch(() => undefined) + if (state?.spendCeilingUSD === undefined) + return say( + "Audits run inside a factory run with a spend ceiling. Start one at a terminal with `es audit --max-usd N`, or run /grill first.", + ) + let opened: Awaited> + try { + // The human typed /audit: opening it is their action. + opened = await runtime.factory.openAudit( + "human:tui", + parseAuditTarget( + target, + state.phases.map((phase) => phase.id), + ), + ) + } catch (error) { + return say(`Could not open the audit: ${error instanceof Error ? error.message : String(error)}`) + } + const { audit } = opened + await io.switchAgent(sessionID, "factory") + await io.sendPrompt( + sessionID, + prompt, + `${await runtime.factory.summary()}\nThe human opened code audit ${audit.id} (round ${audit.round}) on ${describeTarget(audit.target)}. Launch es-auditor-thesis and es-auditor-antithesis in parallel with the audit id and target; if they split, have es-manager break the tie (es_tiebreak with audit). Report the verdicts and the report path when the audit settles.`, + delivery, + ) + }, + }) + specs.push({ + name: "factory", + description: "Drive the build factory from its current stage (Epistemic Swarm)", + execute: async ({ sessionID, prompt, delivery }) => { + const state = await runtime.factory.read() + if (!state) { + await io.synthetic(sessionID, "No factory run here yet — run /grill first.") + return + } + await io.switchAgent(sessionID, "factory") + await io.sendPrompt( + sessionID, + prompt, + `${await runtime.factory.summary(state)}\nContinue the factory run from its current stage.${prompt.text?.trim() ? `\nHuman note: ${prompt.text}` : ""}`, + delivery, + ) + }, + }) + specs.push({ + name: "gates", + description: "Run the mechanical gates on changed files (Epistemic Swarm)", + execute: async ({ sessionID }) => { + const changed = (await gitStatus(runtime.root, ["status", "--porcelain=v1", "-uall"], { allowFail: true })).stdout + .split("\n") + .filter(Boolean) + .map((line) => line.slice(3).split(" -> ").at(-1)!) + const report = await gates({ + root: runtime.root, + scope: changed.length ? "touched" : "full", + touched: changed, + stateDir: runtime.stateDir, + }) + await io.synthetic(sessionID, formatReport(report)) + }, + }) + specs.push({ + name: "lsp", + description: "Language servers: configured, available and running (Epistemic Swarm)", + execute: async ({ sessionID }) => { + const status = await runtime.lsp.status() + const settings = await runtime.lsp.settings() + const lines = [status.enabled ? "LSP is on." : "LSP is disabled (lsp: false)."] + for (const server of settings.servers) { + const command = await runtime.lsp.command(server, runtime.root) + const running = status.running.filter((item) => item.id === server.id) + lines.push( + `${server.id} [${server.extensions.join(" ")}]: ${Array.isArray(command) ? `available (${command.join(" ")})` : command.unavailable}${running.length ? ` · running for ${running.length} root(s)` : ""}`, + ) + } + lines.push(...status.diagnostics.map((item) => `! ${item}`)) + await io.synthetic(sessionID, lines.join("\n")) + }, + }) + specs.push({ + name: "hooks", + description: "Show the hook bridge: sources, handlers, trust and diagnostics (Epistemic Swarm)", + execute: async ({ sessionID }) => { + await bridge.engine.refresh() + const status = bridge.engine.status() + const trust = + status.projectHandlers === 0 + ? "nothing to trust" + : status.trusted + ? "trusted" + : "NOT trusted: run /es-trust or `es trust`" + const lines = [ + `Hook bridge: ${status.handlers} handler(s), ${status.projectHandlers} from the project (${trust}).`, + ...status.projectLines, + ...status.diagnostics.map((item) => `! ${item}`), + ...bridge.engine.recent.slice(-10).map((item) => `recent error: ${item}`), + "Stop hooks are emulated after the turn ends (advisory for plain chat).", + ] + await io.synthetic(sessionID, lines.join("\n")) + }, + }) + specs.push({ + name: "status", + description: "Show the factory state (Epistemic Swarm)", + execute: async ({ sessionID }) => { + const state = await runtime.factory.read() + const liveness = await readLiveness(runtime.root, state) + const text = [await runtime.factory.summary(state), ...livenessLines(state, liveness)].join("\n") + await io.synthetic(sessionID, text) + }, + }) + specs.push({ + name: "config", + description: "Show the effective Epistemic Swarm config and where it came from (Epistemic Swarm)", + execute: async ({ sessionID }) => { + const lines = [ + runtime.configSources.length + ? `Layers: ${runtime.configSources.join(" → ")} → plugin options` + : "Layers: defaults only (no config files)", + ...runtime.warnings.map((warning) => `Warning: ${warning}`), + JSON.stringify(runtime.config, null, 2), + ] + await io.synthetic(sessionID, lines.join("\n")) + }, + }) + return specs +} + export default Plugin.define({ id: PLUGIN_ID, async setup(ctx) { @@ -183,301 +526,13 @@ export default Plugin.define({ }) as any) await ctx.command.transform((editor) => { - editor.add({ - name: "grill", - description: "Grill an idea until the design and spend ceiling are settled (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - await ctx.session.switchAgent({ sessionID, agent: "grill" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: prompt.text?.trim() ? `Grill me on: ${prompt.text}` : "Start grilling me on my idea.", - delivery, - } as any) - }, - }) - editor.add({ - name: "brainstorm", - description: "Fan out divergent lenses on an idea and return a diversified shortlist (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - await ctx.session.switchAgent({ sessionID, agent: "brainstormer" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: prompt.text?.trim() - ? `Brainstorm: ${prompt.text}` - : "Ask me for the idea to brainstorm (one sentence), then run the lens fan-out.", - delivery, - } as any) - }, - }) - editor.add({ - name: "harvest", - description: "Tear down competitor projects: licences, feature matrix, clean-room specs (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - await ctx.session.switchAgent({ sessionID, agent: "harvester" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: prompt.text?.trim() - ? `Darkharvest: ${prompt.text}` - : "Ask me for the teardown objective and the candidate projects, then run the harvest.", - delivery, - } as any) - }, - }) - editor.add({ - name: "design", - description: "Interview me into a design system: tokens, contrast gates, style guide (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - await ctx.session.switchAgent({ sessionID, agent: "designer" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: prompt.text?.trim() - ? `Run the design interview. Product: ${prompt.text}` - : "Ask me for the product name and start the design interview, one question at a time.", - delivery, - } as any) - }, - }) - editor.add({ - name: "scout", - description: "Find and vet open-source candidates for a feature: licenses, maintenance, CVEs (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - await ctx.session.switchAgent({ sessionID, agent: "scout" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: prompt.text?.trim() - ? `Scout: ${prompt.text}` - : "Ask me for the feature to scout and our license posture, then plan the candidates.", - delivery, - } as any) - }, - }) - editor.add({ - name: "research", - description: "Adversarial deep research to a grounded report: thesis, antithesis, synthesis (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - const say = async (text: string) => { - await ctx.session.synthetic({ sessionID, text } as any) - } - const text = prompt.text?.trim() ?? "" - const parsed = parseDeepResearchArgs(text) - if ("error" in parsed && parsed.error === "usage") { - return say("Usage: /research deep --max-usd N") - } - if ("error" in parsed) { - return say( - "Deep research needs a spend ceiling you set: /research deep --max-usd N. Seats halt when the spend reaches it.", - ) - } - const { query, ceiling } = parsed - const existing = await runtime.factory.read().catch(() => undefined) - if (existing && existing.mode !== "research") { - return say( - "This project holds a build run. Start deep research in a directory without one (`es research deep --output --max-usd N` at a terminal).", - ) - } - if (existing?.stage === "HALTED") { - return say( - "This research run is halted; resume it first (`es factory resume` at a terminal, or /es-resume to preview).", - ) - } - if (existing?.stage === "DONE") { - return say( - "This research run is done. For a new question, start one in a fresh directory (`es research deep --output --max-usd N` at a terminal).", - ) - } - if (!existing) { - try { - // The human typed /research deep: beginning the run is their action. - await runtime.factory.beginResearchRun({ objective: query, ceilingUSD: Number(ceiling) }, "human:tui") - } catch (error) { - return say(`Could not start the research run: ${error instanceof Error ? error.message : String(error)}`) - } - } - await ctx.session.switchAgent({ sessionID, agent: "deep-researcher" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - // The coordinator flow is the deep-researcher seat's system prompt - // (single-sourced in core); this only adds the run state. - text: `${await runtime.factory.summary()}\nDeep research runs adversarially, with no human answering questions mid-flow; follow your Flow through es_research_complete.`, - delivery, - } as any) - }, - }) - editor.add({ - name: "audit", - description: "Open a code audit of the active phase or a path and run the auditor pair (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - const target = prompt.text?.trim() - const say = async (text: string) => { - await ctx.session.synthetic({ sessionID, text } as any) - } - if (!target) return say("Usage: /audit ") - const state = await runtime.factory.read().catch(() => undefined) - if (state?.spendCeilingUSD === undefined) - return say( - "Audits run inside a factory run with a spend ceiling. Start one at a terminal with `es audit --max-usd N`, or run /grill first.", - ) - let opened: Awaited> - try { - // The human typed /audit: opening it is their action. - opened = await runtime.factory.openAudit( - "human:tui", - parseAuditTarget( - target, - state.phases.map((phase) => phase.id), - ), - ) - } catch (error) { - return say(`Could not open the audit: ${error instanceof Error ? error.message : String(error)}`) - } - const { audit } = opened - await ctx.session.switchAgent({ sessionID, agent: "factory" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: `${await runtime.factory.summary()}\nThe human opened code audit ${audit.id} (round ${audit.round}) on ${describeTarget(audit.target)}. Launch es-auditor-thesis and es-auditor-antithesis in parallel with the audit id and target; if they split, have es-manager break the tie (es_tiebreak with audit). Report the verdicts and the report path when the audit settles.`, - delivery, - } as any) - }, - }) - editor.add({ - name: "factory", - description: "Drive the build factory from its current stage (Epistemic Swarm)", - execute: async ({ sessionID, prompt, delivery }) => { - const state = await runtime.factory.read() - if (!state) { - await ctx.session.synthetic({ - sessionID, - text: "No factory run here yet — run /grill first.", - } as any) - return - } - await ctx.session.switchAgent({ sessionID, agent: "factory" } as any) - await ctx.session.prompt({ - ...prompt, - sessionID, - text: `${await runtime.factory.summary(state)}\nContinue the factory run from its current stage.${prompt.text?.trim() ? `\nHuman note: ${prompt.text}` : ""}`, - delivery, - } as any) - }, - }) - editor.add({ - name: "gates", - description: "Run the mechanical gates on changed files (Epistemic Swarm)", - execute: async ({ sessionID }) => { - const changed = (await git(runtime.root, ["status", "--porcelain=v1", "-uall"], { allowFail: true })).stdout - .split("\n") - .filter(Boolean) - .map((line) => line.slice(3).split(" -> ").at(-1)!) - const report = await runGates({ - root: runtime.root, - scope: changed.length ? "touched" : "full", - touched: changed, - stateDir: runtime.stateDir, - }) - await ctx.session.synthetic({ sessionID, text: formatReport(report) } as any) - }, - }) - editor.add({ - name: "lsp", - description: "Language servers: configured, available and running (Epistemic Swarm)", - execute: async ({ sessionID }) => { - const status = await runtime.lsp.status() - const settings = await runtime.lsp.settings() - const lines = [status.enabled ? "LSP is on." : "LSP is disabled (lsp: false)."] - for (const server of settings.servers) { - const command = await runtime.lsp.command(server, runtime.root) - const running = status.running.filter((item) => item.id === server.id) - lines.push( - `${server.id} [${server.extensions.join(" ")}]: ${Array.isArray(command) ? `available (${command.join(" ")})` : command.unavailable}${running.length ? ` · running for ${running.length} root(s)` : ""}`, - ) - } - lines.push(...status.diagnostics.map((item) => `! ${item}`)) - await ctx.session.synthetic({ sessionID, text: lines.join("\n") } as any) - }, - }) - editor.add({ - name: "hooks", - description: "Show the hook bridge: sources, handlers, trust and diagnostics (Epistemic Swarm)", - execute: async ({ sessionID }) => { - await bridge.engine.refresh() - const status = bridge.engine.status() - const trust = - status.projectHandlers === 0 - ? "nothing to trust" - : status.trusted - ? "trusted" - : "NOT trusted: run /es-trust or `es trust`" - const lines = [ - `Hook bridge: ${status.handlers} handler(s), ${status.projectHandlers} from the project (${trust}).`, - ...status.projectLines, - ...status.diagnostics.map((item) => `! ${item}`), - ...bridge.engine.recent.slice(-10).map((item) => `recent error: ${item}`), - "Stop hooks are emulated after the turn ends (advisory for plain chat).", - ] - await ctx.session.synthetic({ sessionID, text: lines.join("\n") } as any) - }, - }) - editor.add({ - name: "status", - description: "Show the factory state (Epistemic Swarm)", - execute: async ({ sessionID }) => { - const state = await runtime.factory.read() - const liveness = await readLiveness(runtime.root, state) - const text = [await runtime.factory.summary(state), ...livenessLines(state, liveness)].join("\n") - await ctx.session.synthetic({ sessionID, text } as any) - }, - }) - editor.add({ - name: "config", - description: "Show the effective Epistemic Swarm config and where it came from (Epistemic Swarm)", - execute: async ({ sessionID }) => { - const lines = [ - runtime.configSources.length - ? `Layers: ${runtime.configSources.join(" → ")} → plugin options` - : "Layers: defaults only (no config files)", - ...runtime.warnings.map((warning) => `Warning: ${warning}`), - JSON.stringify(runtime.config, null, 2), - ] - await ctx.session.synthetic({ sessionID, text: lines.join("\n") } as any) - }, - }) - editor.add({ - name: "research", - description: "Research stage: report, audit coverage and cached sources (Epistemic Swarm)", - execute: async ({ sessionID }) => { - const layout = factoryLayout(runtime.root) - const report = await readFile(layout.researchReport, "utf8").catch(() => undefined) - const coverage = await readJson<{ - claims?: number - grounded?: number - untagged?: number - ungrounded?: number - malformed?: number - sources?: number - passed?: boolean - }>(path.join(layout.research, "coverage.json")).catch(() => undefined) - const sources = (await readdir(researchSourcesDir(runtime.root)).catch(() => [])).filter((file) => - file.endsWith(".md"), - ) - const lines = [ - report === undefined - ? "No research report yet (.factory/research/REPORT.md)." - : `Report: ${report.length} characters in .factory/research/REPORT.md.`, - coverage - ? `Audit: ${coverage.grounded ?? 0}/${coverage.claims ?? 0} claims grounded (untagged ${coverage.untagged ?? 0}, ungrounded ${coverage.ungrounded ?? 0}, malformed ${coverage.malformed ?? 0}); cited sources ${coverage.sources ?? 0}; passed ${coverage.passed ? "yes" : "no"}.` - : "No coverage.json yet (es_research_complete writes it).", - `Cached sources: ${sources.length}.`, - ] - await ctx.session.synthetic({ sessionID, text: lines.join("\n") } as any) - }, - }) + const io: CommandSession = { + switchAgent: (sessionID, agent) => ctx.session.switchAgent({ sessionID, agent } as any), + sendPrompt: (sessionID, prompt, text, delivery) => + ctx.session.prompt({ ...prompt, sessionID, text, delivery } as any), + synthetic: (sessionID, text) => ctx.session.synthetic({ sessionID, text } as any), + } + for (const spec of esCommandSpecs(io, runtime, bridge as unknown as CommandBridge)) editor.add(spec as any) }) // Spend: per-step host cost for factory-agent sessions (estimated from tokens when cost is 0). diff --git a/packages/opencode/test/compaction.test.ts b/packages/opencode/test/compaction.test.ts new file mode 100644 index 0000000..49a4d62 --- /dev/null +++ b/packages/opencode/test/compaction.test.ts @@ -0,0 +1,46 @@ +// Compaction (#276.2): only factory seats get the factory-state block +// injected; any other agent compacts without it. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { createRuntime, parseOptions, type Runtime } from "../src/runtime.ts" +import { createSessionHooks } from "../src/session.ts" + +let root: string +let state: string +let runtime: Runtime + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-compaction-")) + state = await mkdtemp(path.join(tmpdir(), "es-compaction-state-")) + runtime = await createRuntime(root, parseOptions({ stateDir: state })) +}) + +afterEach(async () => { + await runtime.lsp.stopAll().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +describe("compaction is factory-only", () => { + test("a factory seat compacts with the verbatim state block", async () => { + const hooks = createSessionHooks(runtime) + const event = { agent: "factory", system: [] as unknown[] } + await hooks.compaction(event as never) + expect(event.system).toHaveLength(1) + const text = (event.system[0] as { text: string }).text + expect(text).toContain("Preserve this factory state verbatim") + expect(text).toContain("") + }) + + test("a non-registry agent compacts without the state block", async () => { + const hooks = createSessionHooks(runtime) + for (const agent of ["build", "es-programmer", "grill"]) { + const event = { agent, system: [] as unknown[] } + await hooks.compaction(event as never) + if (agent === "build") expect(event.system).toEqual([]) + else expect(event.system).toHaveLength(1) + } + }) +}) diff --git a/packages/opencode/test/slash.test.ts b/packages/opencode/test/slash.test.ts new file mode 100644 index 0000000..d53e81e --- /dev/null +++ b/packages/opencode/test/slash.test.ts @@ -0,0 +1,184 @@ +// Slash commands (#274 duplicate `/research`, #276 routing gaps, #277 `/lsp` +// coverage): one name per command, and per-command routing (seat/verb) proven +// by invoking each body against a stub session — no host needed. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { formatReport } from "@heretek-ai/es-core" +import { createRuntime, parseOptions, type Runtime } from "../src/runtime.ts" +import { type CommandSession, esCommandSpecs } from "../src/server.ts" + +let root: string +let state: string +let runtime: Runtime + +const stubSession = () => { + const switched: Array<{ sessionID: string; agent: string }> = [] + const prompted: Array<{ sessionID: string; prompt: Record; text: string; delivery: unknown }> = [] + const said: Array<{ sessionID: string; text: string }> = [] + const io: CommandSession = { + switchAgent: async (sessionID, agent) => { + switched.push({ sessionID, agent }) + }, + sendPrompt: async (sessionID, prompt, text, delivery) => { + prompted.push({ sessionID, prompt, text, delivery }) + }, + synthetic: async (sessionID, text) => { + said.push({ sessionID, text }) + }, + } + return { io, switched, prompted, said } +} + +const stubBridge = () => ({ + engine: { + refresh: async () => {}, + status: () => ({ + handlers: 2, + projectHandlers: 1, + trusted: false, + projectLines: ["project hook line"], + diagnostics: ["bridge diagnostic"], + }), + recent: ["recent failure"], + }, +}) + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-slash-")) + state = await mkdtemp(path.join(tmpdir(), "es-slash-state-")) + runtime = await createRuntime(root, parseOptions({ stateDir: state })) +}) + +afterEach(async () => { + await runtime.lsp.stopAll().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const specs = () => esCommandSpecs(stubSession().io, runtime, stubBridge()) +const byName = () => new Map(specs().map((spec) => [spec.name, spec])) + +describe("slash command registration (#274)", () => { + test("13 unique names, with a single research", () => { + const names = specs().map((spec) => spec.name) + expect(names.filter((name) => name === "research")).toHaveLength(1) + expect([...new Set(names)].sort()).toEqual( + [ + "audit", + "brainstorm", + "config", + "design", + "factory", + "gates", + "grill", + "harvest", + "hooks", + "lsp", + "research", + "scout", + "status", + ].sort(), + ) + }) +}) + +describe("per-command routing: seat and verb (#276.2)", () => { + test("grill / brainstorm / harvest / design / scout switch to their seat with a verb", async () => { + const routes: Array<[string, string, string]> = [ + ["grill", "grill", "Grill me on: q"], + ["brainstorm", "brainstormer", "Brainstorm: q"], + ["harvest", "harvester", "Darkharvest: q"], + ["design", "designer", "Run the design interview. Product: q"], + ["scout", "scout", "Scout: q"], + ] + for (const [name, seat, verb] of routes) { + const stub = stubSession() + const spec = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])).get(name)! + await spec.execute({ sessionID: "ses_1", prompt: { text: "q" }, delivery: undefined }) + expect([name, stub.switched]).toEqual([name, [{ sessionID: "ses_1", agent: seat }]]) + expect([name, stub.prompted[0]?.text]).toEqual([name, verb]) + } + }) + + test("research deep starts a research run and routes to the deep-researcher", async () => { + const stub = stubSession() + const spec = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])).get( + "research", + )! + await spec.execute({ sessionID: "ses_1", prompt: { text: "deep what queue --max-usd 5" }, delivery: undefined }) + const begun = await runtime.factory.read() + expect(begun).toMatchObject({ mode: "research", stage: "RESEARCH" }) + expect(stub.switched).toEqual([{ sessionID: "ses_1", agent: "deep-researcher" }]) + expect(stub.prompted[0]?.text).toContain("es_research_complete") + }) + + test("research without deep shows the report reader, not usage", async () => { + const stub = stubSession() + const spec = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])).get( + "research", + )! + await spec.execute({ sessionID: "ses_1", prompt: { text: "" }, delivery: undefined }) + expect(stub.switched).toEqual([]) + expect(stub.said[0]?.text).toContain("No research report yet") + // A bare `deep` still gets the deep flow's usage, not the report. + const usage = stubSession() + const usageSpec = new Map(esCommandSpecs(usage.io, runtime, stubBridge()).map((item) => [item.name, item])).get( + "research", + )! + await usageSpec.execute({ sessionID: "ses_1", prompt: { text: "deep" }, delivery: undefined }) + expect(usage.said[0]?.text).toContain("Usage: /research deep") + }) + + test("audit usage and ceiling refusals stay synthetic", async () => { + const stub = stubSession() + const spec = byName().get("audit")! + // Rebind the spec to this stub's session: rebuild with the same stub. + const bound = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])).get("audit")! + expect(spec.name).toBe(bound.name) + await bound.execute({ sessionID: "ses_1", prompt: { text: "" }, delivery: undefined }) + expect(stub.said[0]?.text).toContain("Usage: /audit") + await bound.execute({ sessionID: "ses_1", prompt: { text: "src/x.ts" }, delivery: undefined }) + expect(stub.said[1]?.text).toContain("spend ceiling") + expect(stub.switched).toEqual([]) + }) + + test("factory with no run is synthetic; gates renders the gate report", async () => { + const stub = stubSession() + const commands = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])) + await commands.get("factory")!.execute({ sessionID: "ses_1", prompt: {}, delivery: undefined }) + expect(stub.said[0]?.text).toContain("No factory run here yet") + const canned = { summary: "Gates passed (stubbed).", findings: [], totalFindings: 0, logDir: "stub" } + const gated = stubSession() + const gatedCommands = new Map( + esCommandSpecs(gated.io, runtime, stubBridge(), { + git: (async () => ({ stdout: "" })) as never, + runGates: (async () => canned) as never, + }).map((item) => [item.name, item]), + ) + await gatedCommands.get("gates")!.execute({ sessionID: "ses_1", prompt: {}, delivery: undefined }) + expect(gated.said[0]?.text).toBe(formatReport(canned as never)) + }) + + test("lsp reports status and availability lines (#277.1)", async () => { + const stub = stubSession() + const commands = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])) + await commands.get("lsp")!.execute({ sessionID: "ses_1", prompt: {}, delivery: undefined }) + const text = stub.said[0]?.text ?? "" + expect(text).toMatch(/LSP is (on|disabled)/) + expect(text).toContain("typescript") + }) + + test("hooks / status / config render server content synthetically", async () => { + const stub = stubSession() + const commands = new Map(esCommandSpecs(stub.io, runtime, stubBridge()).map((item) => [item.name, item])) + await commands.get("hooks")!.execute({ sessionID: "ses_1", prompt: {}, delivery: undefined }) + expect(stub.said[0]?.text).toContain("Hook bridge: 2 handler(s)") + expect(stub.said[0]?.text).toContain("NOT trusted") + await commands.get("status")!.execute({ sessionID: "ses_1", prompt: {}, delivery: undefined }) + expect(stub.said[1]?.text).toContain("") + await commands.get("config")!.execute({ sessionID: "ses_1", prompt: {}, delivery: undefined }) + expect(stub.said[2]?.text).toContain("Layers:") + }) +}) diff --git a/packages/opencode/test/visibility.test.ts b/packages/opencode/test/visibility.test.ts new file mode 100644 index 0000000..5c9234a --- /dev/null +++ b/packages/opencode/test/visibility.test.ts @@ -0,0 +1,95 @@ +// Visibility proofs (#276): the plugin registers additively (built-in tools +// keep working; only invisible es_* tools are pruned), the status footer is +// server-computed content, and setup writes no project-tree files. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, readdir, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { AGENTS, ensureEngineKey, footerLine, loadSkills, readLiveness } from "@heretek-ai/es-core" +import { compileAgents } from "../src/agents.ts" +import { createRpcHandlers } from "../src/rpc.ts" +import { createRuntime, parseOptions, type Runtime } from "../src/runtime.ts" +import { createSessionHooks } from "../src/session.ts" + +let root: string +let state: string +let runtime: Runtime + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-visibility-")) + state = await mkdtemp(path.join(tmpdir(), "es-visibility-state-")) + runtime = await createRuntime(root, parseOptions({ stateDir: state })) +}) + +afterEach(async () => { + await runtime.lsp.stopAll().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +describe("additive registration", () => { + test("we only add registry agents (never built-ins or defaults)", async () => { + const ids = new Set(AGENTS.map((agent) => agent.id)) + for (const compiled of await compileAgents(runtime.options, [])) expect(ids.has(compiled.spec.id)).toBe(true) + }) + + test("context prunes only invisible es_* tools; built-ins and state blocks are additive", async () => { + const hooks = createSessionHooks(runtime) + const tools = () => ({ + read: {}, + write: {}, + shell: {}, + question: {}, + es_status: {}, + es_gates_run: {}, + es_complete: {}, + es_build_start: {}, + }) + const user = { agent: "build", system: [] as unknown[], tools: tools() } + await hooks.context(user as never) + // Built-ins are untouched; only the read-only es tools stay visible. + expect(Object.keys(user.tools).sort()).toEqual(["es_gates_run", "es_status", "question", "read", "shell", "write"]) + expect(user.system).toEqual([]) + + const factory = { agent: "factory", system: [] as unknown[], tools: tools() } + await hooks.context(factory as never) + expect(Object.keys(factory.tools)).toContain("es_build_start") + expect(factory.system).toHaveLength(1) + expect((factory.system[0] as { text: string }).text).toContain("") + }) +}) + +describe("footer server-content wiring", () => { + test("status().footer is footerLine(state, liveness, paused)", async () => { + const handlers = createRpcHandlers(runtime, async () => {}) + const seen = await handlers.status() + const state = await runtime.factory.read() + const liveness = await readLiveness(root, state) + expect(seen.footer ?? null).toBe(footerLine(state, liveness, undefined) ?? null) + expect(seen.headline).toContain("No factory run") + }) +}) + +describe("setup writes no project-tree files (#276.1)", () => { + test("runtime creation, engine key, agents and skills leave the project tree alone", async () => { + const fresh = await mkdtemp(path.join(tmpdir(), "es-setup-diff-")) + const freshState = await mkdtemp(path.join(tmpdir(), "es-setup-diff-state-")) + try { + const before = await readdir(fresh) + const staged = await createRuntime(fresh, parseOptions({ stateDir: freshState })) + try { + // The engine key seals cached sources: created at setup in the masked + // state dir, never in the project tree. + await ensureEngineKey(staged.stateDir) + await compileAgents(staged.options, []) + await loadSkills() + } finally { + await staged.lsp.stopAll().catch(() => undefined) + } + expect(await readdir(fresh)).toEqual(before) + } finally { + await rm(fresh, { recursive: true, force: true }) + await rm(freshState, { recursive: true, force: true }) + } + }) +}) From 3d59a7bce12b99e050f10c464ccc5337d60fcd70 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:03:15 -0400 Subject: [PATCH 55/83] fix(remediation-audit/s5): lsp diagnostics scoping + lifecycle/config wording (Refs #277, Refs #275, Refs #240) --- packages/opencode/src/policy.ts | 5 +- packages/opencode/src/runtime.ts | 12 ++-- packages/opencode/src/server.ts | 4 ++ packages/opencode/test/config-warning.test.ts | 20 ++++++ packages/opencode/test/lsp-scoping.test.ts | 62 +++++++++++++++++++ 5 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 packages/opencode/test/config-warning.test.ts create mode 100644 packages/opencode/test/lsp-scoping.test.ts diff --git a/packages/opencode/src/policy.ts b/packages/opencode/src/policy.ts index 9983281..7f74bd0 100644 --- a/packages/opencode/src/policy.ts +++ b/packages/opencode/src/policy.ts @@ -174,7 +174,10 @@ export function createPolicyHooks(runtime: Runtime, seats?: SeatTracker) { ? patchTargets(input.patchText) : [] if (!files.length) return - if (runtime.options.lspAfterEdit !== false) { + // The post-edit diagnostics are a language-server channel (#277.2): seats + // scoped to `lsp: "none"` never get them (their tool rules already hide + // `lsp_*`). Navigating seats and the user's own agents keep the channel. + if (runtime.options.lspAfterEdit !== false && agentSpec(event.agent)?.lsp !== "none") { const lines: string[] = [] for (const file of files) { const absolute = path.resolve(runtime.root, file) diff --git a/packages/opencode/src/runtime.ts b/packages/opencode/src/runtime.ts index 12f069c..7776e65 100644 --- a/packages/opencode/src/runtime.ts +++ b/packages/opencode/src/runtime.ts @@ -1,8 +1,9 @@ // Per-location runtime shared by the plugin's hooks, tools, commands and RPC. -// Options are layered: the canonical config (global file → project file) with -// plugin options as the most specific override. Plugin options are config -// keys validated by the same strict schema; unknown keys are ignored with a -// warning the TUI shows once. +// The server only wires option layering (parseOptions → createRuntime); the +// merge itself lives in core (loadEsConfig/mergeConfig): the canonical config +// (global file → project file) with plugin options as the most specific +// override. Plugin options are config keys validated by the same strict +// schema; unknown keys are ignored with a warning the TUI shows once. import { stateDir as defaultStateDir, type EsConfig, @@ -68,6 +69,9 @@ export async function createRuntime(root: string, parsed: ParsedOptions): Promis ...(config.searchProvider ? { searchProvider: config.searchProvider } : {}), } const stateDir = options.stateDir ?? defaultStateDir() + // One manager per project (#275.1): servers start lazily per (server, root) + // and are reaped by the idle sweeper between sessions, stopping only on + // plugin unload (see server.ts) — long-lived servers outlive sessions. const lsp = new LspManager({ root, stateDir }) const factory = new Factory(root, { gates: gateRunner({ stateDir }), diff --git a/packages/opencode/src/server.ts b/packages/opencode/src/server.ts index dfba18a..90e99bf 100644 --- a/packages/opencode/src/server.ts +++ b/packages/opencode/src/server.ts @@ -550,6 +550,10 @@ export default Plugin.define({ // stream closed on unload } })() + // Language-server lifecycle (#275.1): servers stop here on plugin unload; + // between sessions only the manager's idle sweeper (600 s default) reaps + // them, so a long-lived server outlives its session by design. The running + // set stays visible through `/lsp` and the LSP panel. return async () => { abort.abort() if (queued) clearTimeout(queued) diff --git a/packages/opencode/test/config-warning.test.ts b/packages/opencode/test/config-warning.test.ts new file mode 100644 index 0000000..56ee3fb --- /dev/null +++ b/packages/opencode/test/config-warning.test.ts @@ -0,0 +1,20 @@ +// Legacy option wording (#240.1): 0.7-era options are ignored, not loaded — +// plugin-options only, while unknown keys in config files are +// strict-rejected; the unknowns collapse into one warning the TUI shows once. +import { describe, expect, test } from "bun:test" +import { parseOptions, unknownOptionsWarning } from "../src/runtime.ts" + +describe("legacy option wording", () => { + test("unknown plugin options are ignored (never loaded), naming the 0.7 removal", () => { + const warning = unknownOptionsWarning(["search_engine", "max_iterations", "mode"]) + expect(warning).toContain("Ignored unknown plugin option") + expect(warning).toContain("were removed in 1.0") + expect(warning).not.toMatch(/load/i) + }) + + test("unknowns collapse into a single warning string", () => { + const parsed = parseOptions({ search_engine: "x", max_iterations: 2, pr: "off" }) + expect(parsed.unknown).toEqual(["max_iterations", "search_engine"]) + expect(parsed.overrides).toEqual({ pr: "off" }) + }) +}) diff --git a/packages/opencode/test/lsp-scoping.test.ts b/packages/opencode/test/lsp-scoping.test.ts new file mode 100644 index 0000000..03480f6 --- /dev/null +++ b/packages/opencode/test/lsp-scoping.test.ts @@ -0,0 +1,62 @@ +// After-edit diagnostics scoping (#277.2): the `` injection is a +// language-server channel, so seats with `lsp: "none"` (harvester, designer, +// scout, brainstormer, …) must not receive it; seats that may navigate (and +// the user's own agents) keep it. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { createPolicyHooks } from "../src/policy.ts" +import type { Runtime } from "../src/runtime.ts" + +let root: string + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-lsp-scope-")) +}) + +afterEach(async () => { + await rm(root, { recursive: true, force: true }) +}) + +const diagnostic = { + range: { start: { line: 0, character: 0 }, end: { line: 0, character: 1 } }, + severity: 1, + message: "boom", +} + +const runtime = (root: string) => + ({ + root, + options: { lspAfterEdit: true, afterEdit: "off" }, + lsp: { diagnostics: async () => [diagnostic] }, + policy: async () => ({ root }), + }) as unknown as Runtime + +const edited = (agent: string) => ({ + tool: "write", + agent, + id: "1", + input: { path: "src/a.ts" }, + status: "completed", + result: {}, +}) + +describe("after-edit diagnostics respect lsp scoping", () => { + test("an lsp:none seat (harvester) gets no diagnostics channel", async () => { + const { after } = createPolicyHooks(runtime(root)) + const event = edited("harvester") + await after(event as never) + expect(event.result).toEqual({}) + }) + + test("a navigating seat (QA) and the user's own agent keep the injection", async () => { + const { after } = createPolicyHooks(runtime(root)) + const qa = edited("es-qa-functional") + await after(qa as never) + expect(JSON.stringify((qa.result as { content: unknown }).content)).toContain("") + const user = edited("build") + await after(user as never) + expect(JSON.stringify((user.result as { content: unknown }).content)).toContain("") + }) +}) From 9289804ad0e7e25a7f6ff842659f70cba0616528 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:05:11 -0400 Subject: [PATCH 56/83] fix(remediation-audit/s5): web JSON security headers + ticket TTL fixtures (Refs #282, Refs #283) --- packages/fleet/src/telemetry.ts | 34 ++++++++----- packages/fleet/test/approve.test.ts | 46 ++++++++++++++++- packages/fleet/test/headers.test.ts | 5 +- packages/fleet/test/web.test.ts | 76 +++++++++++++++++++++++++++-- 4 files changed, 144 insertions(+), 17 deletions(-) diff --git a/packages/fleet/src/telemetry.ts b/packages/fleet/src/telemetry.ts index 4ee59a0..2664d69 100644 --- a/packages/fleet/src/telemetry.ts +++ b/packages/fleet/src/telemetry.ts @@ -285,7 +285,11 @@ interface HttpResponder { const jsonBody = (res: HttpResponder, code: number, value: unknown): void => { const text = JSON.stringify(value) - res.writeHead(code, { "content-type": "application/json", "content-length": Buffer.byteLength(text) }) + res.writeHead(code, { + ...WEB_SECURITY_HEADERS, + "content-type": "application/json", + "content-length": Buffer.byteLength(text), + }) res.end(text) } @@ -430,7 +434,7 @@ export class TelemetryServer { if ((field(headers, "upgrade") ?? "").toLowerCase() === "websocket" && target === "/fleet/ws") return this.handleUpgrade(socket, headers, res) if (!this.hostOk(field(headers, "host"))) { - res.writeHead(403, { "content-length": 0, connection: "close" }) + res.writeHead(403, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } @@ -440,7 +444,7 @@ export class TelemetryServer { // (bearer or session), submit is browser-only (cookie + Origin + CSRF). if (method === "POST" && target === "/fleet/approve/preview") return this.approvePreview(headers, rest, res) if (method === "POST" && target === "/fleet/approve") return this.approveSubmit(headers, rest, res) - res.writeHead(404, { "content-length": 0, connection: "close" }) + res.writeHead(404, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } @@ -448,7 +452,7 @@ export class TelemetryServer { // A presented session cookie that cannot authorize is a forbidden // CSRF/origin failure (403); wholly missing credentials are 401. const code = parseSessionCookie(field(headers, "cookie")) !== undefined ? 403 : 401 - res.writeHead(code, { "content-length": 0, connection: "close" }) + res.writeHead(code, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } @@ -509,7 +513,7 @@ export class TelemetryServer { res.end() } if (this.options.webRoot === undefined) { - res.writeHead(404, { "content-length": 0, connection: "close" }) + res.writeHead(404, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } @@ -580,7 +584,7 @@ export class TelemetryServer { // Same convention as read-only RPC: a presented session cookie that // cannot authorize is a forbidden CSRF/origin failure (403). const code = parseSessionCookie(field(headers, "cookie")) !== undefined ? 403 : 401 - res.writeHead(code, { "content-length": 0, connection: "close" }) + res.writeHead(code, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } @@ -637,12 +641,12 @@ export class TelemetryServer { private async approveSubmit(headers: Map, rest: Buffer, res: HttpResponder): Promise { const session = parseSessionCookie(field(headers, "cookie")) ?? "" if (!this.sessions.valid(session)) { - res.writeHead(401, { "content-length": 0, connection: "close" }) + res.writeHead(401, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } if (!loopbackOriginOk(field(headers, "origin")) || !this.csrf.check(session, field(headers, "x-csrf"))) { - res.writeHead(403, { "content-length": 0, connection: "close" }) + res.writeHead(403, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } @@ -730,13 +734,21 @@ export class TelemetryServer { private ok(res: HttpResponder, value: unknown): void { const text = JSON.stringify(value) - res.writeHead(200, { "content-type": "application/json", "content-length": Buffer.byteLength(text) }) + res.writeHead(200, { + ...WEB_SECURITY_HEADERS, + "content-type": "application/json", + "content-length": Buffer.byteLength(text), + }) res.end(text) } private fail(res: HttpResponder, code: number, message: string, extra: Record = {}): void { const text = JSON.stringify({ error: message, ...extra }) - res.writeHead(code, { "content-type": "application/json", "content-length": Buffer.byteLength(text) }) + res.writeHead(code, { + ...WEB_SECURITY_HEADERS, + "content-type": "application/json", + "content-length": Buffer.byteLength(text), + }) res.end(text) } @@ -818,7 +830,7 @@ export class TelemetryServer { (!this.bearerOk(field(headers, "authorization")) && !cookieOk) ) { const code = !this.hostOk(field(headers, "host")) || !this.originOk(field(headers, "origin")) ? 403 : 401 - res.writeHead(code, { "content-length": 0, connection: "close" }) + res.writeHead(code, { ...WEB_SECURITY_HEADERS, "content-length": 0, connection: "close" }) res.end() return } diff --git a/packages/fleet/test/approve.test.ts b/packages/fleet/test/approve.test.ts index d6ec7e2..3bf90b5 100644 --- a/packages/fleet/test/approve.test.ts +++ b/packages/fleet/test/approve.test.ts @@ -22,7 +22,7 @@ const frontier = { nodes: [{ id: "lang", question: "Language?", answer: "TypeScript", status: "settled" }], } -async function rig() { +async function rig(extra: { approveTicketTtlMs?: number } = {}) { const state = await mkdtemp(path.join(tmpdir(), "es-approve-")) const repo = await mkdtemp(path.join(tmpdir(), "es-approve-repo-")) await mkdir(path.join(repo, ".factory"), { recursive: true }) @@ -56,6 +56,7 @@ async function rig() { stateRoot: state, port: 0, token: TOKEN, + ...extra, getSnapshot: (): FleetSnapshot => ({ daemon: { running: true, pid: 1, maxUsd: 10, concurrency: 1 }, tasks: [], @@ -301,6 +302,49 @@ describe("approve submit", () => { } }) + test("checklist 4b: an expired preview ticket fails the submit (#283.2)", async () => { + const { state, repo, server } = await rig({ approveTicketTtlMs: 0 }) + try { + const cookie = await session(server.port, state) + const origin = `http://127.0.0.1:${server.port}` + const preview = json( + ( + await post( + server.port, + "/fleet/approve/preview", + { runId: "t1", stage: "frontier" }, + { Cookie: cookie, Origin: origin }, + ) + ).body, + ) + // A zero TTL expires the ticket before the submit even runs: no sleep needed. + const res = await approveAs(server.port, cookie, preview.csrf as string, { + ticket: preview.ticket, + passphrase: PASS, + }) + expect(res.status).toBe(403) + expect(res.body).toContain("expired") + const { readApproval } = await import("@heretek-ai/es-core") + expect(await readApproval(repo, "frontier")).toBeUndefined() + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + await rm(repo, { recursive: true, force: true }) + } + }) + + test("approve tickets expire by TTL at unit level (#283.2)", async () => { + const { ApproveTickets } = await import("../src/approve.ts") + const expired = new ApproveTickets(0) + expect( + expired.redeem(expired.issue({ repoDir: "r", stage: "frontier", subjectHash: "h" })), + ).toBeUndefined() + const live = new ApproveTickets() + const ticket = live.issue({ repoDir: "r", stage: "frontier", subjectHash: "h" }) + expect(live.redeem(ticket)).toEqual({ repoDir: "r", stage: "frontier", subjectHash: "h" }) + expect(live.redeem(ticket)).toBeUndefined() + }) + test("checklist 5: an artifact changed after the preview refuses the sign", async () => { const { state, repo, server } = await rig() try { diff --git a/packages/fleet/test/headers.test.ts b/packages/fleet/test/headers.test.ts index e929700..47d732b 100644 --- a/packages/fleet/test/headers.test.ts +++ b/packages/fleet/test/headers.test.ts @@ -1,6 +1,9 @@ // S5.4 (CodeQL #84) remote property injection (repair-151): request headers // parse into a Map (never a plain `{}` keyed by remote names), non-token -// names are rejected, and `field()` is actually case-insensitive. +// names are rejected, and `field()` is actually case-insensitive. This file +// probes request-header parser hygiene only — web security headers (CSP) are +// proven in `web.test.ts` ("every web response carries the strict security +// headers", #283.1). import { describe, expect, test } from "bun:test" import { field, parseHeaders } from "../src/telemetry.ts" diff --git a/packages/fleet/test/web.test.ts b/packages/fleet/test/web.test.ts index 478f7b3..99c1807 100644 --- a/packages/fleet/test/web.test.ts +++ b/packages/fleet/test/web.test.ts @@ -36,9 +36,13 @@ async function rig() { return { state, webRoot, server, token } } -const raw = (port: number, head: string): Promise<{ status: number; headers: Record; body: string }> => +const raw = ( + port: number, + head: string, + body = "", +): Promise<{ status: number; headers: Record; body: string }> => new Promise((resolve, reject) => { - const socket = createConnection(port, "127.0.0.1", () => socket.write(head)) + const socket = createConnection(port, "127.0.0.1", () => socket.write(head + body)) let buffer = Buffer.alloc(0) socket.on("data", (chunk: Buffer) => { buffer = Buffer.concat([buffer, chunk]) @@ -67,6 +71,24 @@ const get = (port: number, target: string, headers: Record = {}) .join("")}\r\n`, ) +const postRpc = (port: number, payload: unknown, extra: Record = {}) => { + const body = JSON.stringify(payload) + return raw( + port, + `POST /fleet/rpc HTTP/1.1\r\nHost: 127.0.0.1:${port}\r\nContent-Type: application/json\r\nContent-Length: ${Buffer.byteLength(body)}\r\nConnection: close\r\n${Object.entries( + extra, + ) + .map(([k, v]) => `${k}: ${v}\r\n`) + .join("")}\r\n`, + body, + ) +} + +const expectSecurityHeaders = (headers: Record, where: string): void => { + for (const [name, value] of Object.entries(WEB_SECURITY_HEADERS)) + expect(headers[name], `${where}: ${name}`).toBe(value) +} + describe("web tickets", () => { test("a ticket redeems once, then is dead", async () => { const state = await mkdtemp(path.join(tmpdir(), "es-fleet-web-")) @@ -91,7 +113,7 @@ describe("web tickets", () => { } }) - test("malformed and expired tickets are refused", async () => { + test("malformed and missing tickets are refused", async () => { const state = await mkdtemp(path.join(tmpdir(), "es-fleet-web-")) try { expect(await redeemWebTicket(state, "not-hex")).toBe(false) @@ -102,6 +124,35 @@ describe("web tickets", () => { } }) + test("a minted-then-expired ticket is refused and consumed (#283.2)", async () => { + const state = await mkdtemp(path.join(tmpdir(), "es-fleet-web-")) + try { + const ticket = await mintWebTicket(state) + const { fleetDir } = await import("../src/state.ts") + // Age the ticket file past its TTL instead of sleeping for it. + await writeFile(path.join(fleetDir(state), "web-tickets", ticket), `${Date.now() - 1_000}\n`) + expect(await redeemWebTicket(state, ticket)).toBe(false) + expect(await redeemWebTicket(state, ticket)).toBe(false) + } finally { + await rm(state, { recursive: true, force: true }) + } + }) + + test("an expired ticket redeems to 403 without a cookie at HTTP level (#283.2)", async () => { + const { state, server } = await rig() + try { + const ticket = await mintWebTicket(state) + const { fleetDir } = await import("../src/state.ts") + await writeFile(path.join(fleetDir(state), "web-tickets", ticket), `${Date.now() - 1_000}\n`) + const res = await get(server.port, `/?t=${ticket}`) + expect(res.status).toBe(403) + expect(res.headers["set-cookie"]).toBeUndefined() + } finally { + await server.stop() + await rm(state, { recursive: true, force: true }) + } + }) + test("cookie parsing extracts the session id", () => { const id = "ab".repeat(32) expect(parseSessionCookie(`a=1; ${WEB_SESSION_COOKIE}=${id}; b=2`)).toBe(id) @@ -164,7 +215,7 @@ describe("web serving", () => { }) test("every web response carries the strict security headers", async () => { - const { state, server } = await rig() + const { state, server, token } = await rig() try { const ticket = await mintWebTicket(state) const login = await get(server.port, `/?t=${ticket}`) @@ -174,6 +225,23 @@ describe("web serving", () => { expect(login.headers["x-content-type-options"]).toBe("nosniff") expect(login.headers["referrer-policy"]).toBe("no-referrer") expect(WEB_SECURITY_HEADERS["content-security-policy"]).toContain("connect-src 'self'") + // JSON responders carry them too (#282): success, error envelopes, and + // the bare 401/403/404 paths. + const authed = { Authorization: `Bearer ${token}` } + const ok = await postRpc(server.port, { method: "fleet.status", params: {} }, authed) + expect(ok.status).toBe(200) + expect(ok.headers["content-type"]).toContain("application/json") + expectSecurityHeaders(ok.headers, "fleet.status 200") + const unknown = await postRpc(server.port, { method: "nope", params: {} }, authed) + expect(unknown.status).toBe(200) + expect(unknown.body).toContain("-32601") + expectSecurityHeaders(unknown.headers, "unknown method") + const anon = await postRpc(server.port, { method: "fleet.status", params: {} }) + expect(anon.status).toBe(401) + expectSecurityHeaders(anon.headers, "anonymous rpc 401") + const missing = await raw(server.port, `GET /fleet/rpc HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`) + expect(missing.status).toBe(404) + expectSecurityHeaders(missing.headers, "wrong path 404") } finally { await server.stop() await rm(state, { recursive: true, force: true }) From 426d86e51173e6e44c690fc9c179ff437959e32c Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:06:39 -0400 Subject: [PATCH 57/83] =?UTF-8?q?fix(remediation-audit/s5):=20headless=20p?= =?UTF-8?q?roof=20gaps=20=E2=80=94=20done,=20argv,=20STOP,=20driver=20refu?= =?UTF-8?q?sal=20(Refs=20#247)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cli/src/headless.ts | 33 ++--- packages/cli/test/headless.test.ts | 194 ++++++++++++++++++++++++++++- 2 files changed, 211 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/headless.ts b/packages/cli/src/headless.ts index 5a601ab..6c97744 100644 --- a/packages/cli/src/headless.ts +++ b/packages/cli/src/headless.ts @@ -520,25 +520,30 @@ export function runHeadless(options: HeadlessOptions): AsyncGenerator `${await factory.summary(state)}\n${headline(state, await readLiveness(options.root, state))}\nContinue the factory run from its current stage. This is a headless run: no human will answer questions.`, - finished: async (state) => { - if (state.stage === "DONE") - return { type: "done", ...(state.release?.prUrl ? { prUrl: state.release.prUrl } : {}) } - const pending = await pendingApprovals(options.root) - if (pending.length) - return { - type: "waiting", - reason: "a human must approve (es approve )", - stages: pending.map((item) => item.stage), - } - if (state.stage === "GRILL") - return { type: "waiting", reason: "the grill needs a human; run /grill interactively" } - return undefined - }, + finished: (state) => headlessFinished(options.root, state), // Research artifacts and seat outcomes count as progress, not just the stage machine. progress: async (state) => progressPrint(state, await readLiveness(options.root, state)), }) } +/** + * One headless end condition (#247.1): DONE carries the release PR url when + * the run released one, a pending approval waits on a human, and the grill + * needs a human. Anything else keeps driving. + */ +export async function headlessFinished(root: string, state: FactoryState): Promise { + if (state.stage === "DONE") return { type: "done", ...(state.release?.prUrl ? { prUrl: state.release.prUrl } : {}) } + const pending = await pendingApprovals(root) + if (pending.length) + return { + type: "waiting", + reason: "a human must approve (es approve )", + stages: pending.map((item) => item.stage), + } + if (state.stage === "GRILL") return { type: "waiting", reason: "the grill needs a human; run /grill interactively" } + return undefined +} + export const LOG_LEVELS = ["quiet", "info", "debug"] as const export type LogLevel = (typeof LOG_LEVELS)[number] diff --git a/packages/cli/test/headless.test.ts b/packages/cli/test/headless.test.ts index a53a40b..9fdb251 100644 --- a/packages/cli/test/headless.test.ts +++ b/packages/cli/test/headless.test.ts @@ -1,16 +1,19 @@ // Headless fleet probes (#120): per-turn tool metrics carry duration and seat, // and stay out of the default human output. import { afterEach, beforeEach, describe, expect, test } from "bun:test" -import { mkdtemp, rm } from "node:fs/promises" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" -import { Factory, gateRunner } from "@heretek-ai/es-core" +import { Factory, type FactoryState, factoryLayout, gateRunner } from "@heretek-ai/es-core" import { driveHeadless, type HarnessDriver, type HeadlessEvent, HeadlessJsonlSchema, + headlessFinished, + opencodeDriver, presentEvent, + runHeadless, toJsonl, } from "../src/headless.ts" @@ -117,3 +120,190 @@ describe("turn-metrics human output (#120)", () => { expect(presentEvent(metrics, "debug")).toEqual(metrics) }) }) + +describe("runHeadless end conditions (#247)", () => { + test("a research run drives to done, and done carries no prUrl without a release", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-headless-done-")) + const dirState = await mkdtemp(path.join(tmpdir(), "es-headless-done-state-")) + try { + const factory = new Factory(dir, { gates: gateRunner({ stateDir: dirState }), stateDir: dirState }) + await factory.beginResearchRun({ objective: "what queue", ceilingUSD: 5 }) + const seen: Array<{ agent: string; prompt: string }> = [] + const driver: HarnessDriver = { + id: "fake", + available: async () => true, + async *turn(input) { + seen.push({ agent: input.agent, prompt: input.prompt }) + yield { text: "working" } + // The coordinator flow, played against the real tools and gate. + const { researchTools } = await import("@heretek-ai/es-core") + const stubFetch = (async () => + new Response("

Queues decouple workers from producers durably.

", { + headers: { "content-type": "text/html" }, + })) as unknown as typeof fetch + const tools = researchTools({ + root: dir, + fetch: stubFetch, + stateDir: dirState, + policy: async () => ({ root: dir }), + }) + const fetched = await tools + .find((tool) => tool.name === "es_research_fetch")! + .execute({ url: "https://192.0.2.1/queues" }, { agent: "es-research-alpha" }) + const sha = /sha256:([0-9a-f]{64})/.exec(fetched)?.[1] + await writeFile( + path.join(dir, ".factory/research/REPORT.md"), + `# Report\n\n- Queues decouple workers [VERIFIED: sha256:${sha} "decouple workers from producers"]\n`, + ) + await new Factory(dir, { gates: gateRunner({ stateDir: dirState }), stateDir: dirState }).completeResearch( + "deep-researcher", + ) + return "ses_fake" + }, + } + const events: HeadlessEvent[] = [] + for await (const event of runHeadless({ root: dir, driver, maxTurns: 5, stallTurns: 99, stateDir: dirState })) + events.push(event) + // The driver saw the factory seat identity and the headless prompt (#247.2). + expect(seen).toHaveLength(1) + expect(seen[0]?.agent).toBe("factory") + expect(seen[0]?.prompt).toContain("headless run") + expect(events[0]).toMatchObject({ type: "start", driver: "fake" }) + expect(events.filter((event) => event.type === "turn")).toHaveLength(1) + const done = events.at(-1)! + expect(done).toEqual({ type: "done" }) + expect("prUrl" in done).toBe(false) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(dirState, { recursive: true, force: true }) + } + }) + + test("headlessFinished maps DONE (with prUrl spread), pending approvals and the grill", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-headless-finished-")) + try { + const done = (release?: FactoryState["release"]) => + ({ stage: "DONE", ...(release ? { release } : {}) }) as unknown as FactoryState + // No release: no prUrl key. With one: the url spreads onto done (#247.1). + expect(await headlessFinished(dir, done())).toEqual({ type: "done" }) + expect(await headlessFinished(dir, done({ prUrl: "https://example.test/pr/9" }))).toEqual({ + type: "done", + prUrl: "https://example.test/pr/9", + }) + // A pending approval waits on a human, with its stages. + await mkdir(path.dirname(factoryLayout(dir).pending), { recursive: true }) + await writeFile(factoryLayout(dir).pending, JSON.stringify([{ stage: "spec", requestedBy: "factory", at: "x" }])) + expect(await headlessFinished(dir, { stage: "RESEARCH" } as unknown as FactoryState)).toEqual({ + type: "waiting", + reason: "a human must approve (es approve )", + stages: ["spec"], + }) + await writeFile(factoryLayout(dir).pending, "[]") + // The grill (no pending) waits on a human; a mid-build stage keeps driving. + expect(await headlessFinished(dir, { stage: "GRILL" } as unknown as FactoryState)).toMatchObject({ + type: "waiting", + reason: expect.stringContaining("grill"), + }) + expect(await headlessFinished(dir, { stage: "RESEARCH" } as unknown as FactoryState)).toBeUndefined() + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + test("a STOP file halts at headless level without granting anything", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "es-headless-stop-")) + const dirState = await mkdtemp(path.join(tmpdir(), "es-headless-stop-state-")) + try { + await writeFile(path.join(dir, "README.md"), "# x\n") + await new Factory(dir, { gates: gateRunner({ stateDir: dirState }), stateDir: dirState }).provisionRun( + "tester", + 5, + ) + await mkdir(path.dirname(factoryLayout(dir).stop), { recursive: true }) + await writeFile(factoryLayout(dir).stop, "lunch\n") + const driver: HarnessDriver = { + id: "fake", + available: async () => true, + async *turn() { + yield { text: "unused" } + return "ses_fake" + }, + } + const halted: HeadlessEvent[] = [] + for await (const event of runHeadless({ root: dir, driver, maxTurns: 3, stateDir: dirState })) halted.push(event) + expect(halted[0]).toMatchObject({ type: "start" }) + expect(halted.at(-1)).toMatchObject({ type: "halted", reason: expect.stringContaining("STOP file") }) + expect(halted.some((event) => event.type === "turn")).toBe(false) + } finally { + await rm(dir, { recursive: true, force: true }) + await rm(dirState, { recursive: true, force: true }) + } + }) + + test("an unknown --driver is refused before anything launches", async () => { + const { main } = await import("../src/main.ts") + const printed: string[] = [] + const code = await main(["factory", "run", "--headless", "--driver", "nope"], { + print: (text) => printed.push(text), + confirm: { interactive: false, write: () => {}, readLine: async () => "y", readSecret: async () => "" }, + cwd: root, + stateDir: state, + }) + expect(code).toBe(2) + expect(printed.join("\n")).toContain("Unknown driver") + }) +}) + +describe("opencode driver argv shape (#247.2)", () => { + test("run --standalone --format json --agent, with model/session/prompt passthrough", async () => { + const { chmod } = await import("node:fs/promises") + const bin = await mkdtemp(path.join(tmpdir(), "es-driver-argv-")) + const previous = process.env.PATH + try { + await writeFile( + path.join(bin, "opencode"), + `#!/bin/sh\nprintf '%s\\n' "$@" > "$ES_ARGV_LOG"\necho '{"type":"text","sessionID":"ses_argv","part":{"text":"hi"}}'\n`, + ) + await chmod(path.join(bin, "opencode"), 0o755) + process.env.PATH = `${bin}${path.delimiter}${previous}` + const argvOf = async (input: Parameters[0]): Promise => { + const log = path.join(bin, `argv-${Math.random().toString(36).slice(2)}.txt`) + process.env.ES_ARGV_LOG = log + try { + const turn = opencodeDriver.turn(input) + let step = await turn.next() + while (!step.done) step = await turn.next() + expect(step.value).toBe("ses_argv") + return (await Bun.file(log).text()).split("\n").filter(Boolean) + } finally { + delete process.env.ES_ARGV_LOG + } + } + expect(await argvOf({ root, agent: "factory", prompt: "go" })).toEqual([ + "run", + "--standalone", + "--format", + "json", + "--agent", + "factory", + "go", + ]) + expect(await argvOf({ root, agent: "factory", prompt: "again", model: "prov/model", session: "ses_1" })).toEqual([ + "run", + "--standalone", + "--format", + "json", + "--agent", + "factory", + "--model", + "prov/model", + "--session", + "ses_1", + "again", + ]) + } finally { + process.env.PATH = previous + await rm(bin, { recursive: true, force: true }) + } + }) +}) From 38eba003e7d24040f77c3a28e3aa7166b7cb10fd Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:06:52 -0400 Subject: [PATCH 58/83] fix(remediation-audit/s5): biome format for approve TTL test (Refs #283) --- packages/fleet/test/approve.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/fleet/test/approve.test.ts b/packages/fleet/test/approve.test.ts index 3bf90b5..d5d5eb3 100644 --- a/packages/fleet/test/approve.test.ts +++ b/packages/fleet/test/approve.test.ts @@ -336,9 +336,7 @@ describe("approve submit", () => { test("approve tickets expire by TTL at unit level (#283.2)", async () => { const { ApproveTickets } = await import("../src/approve.ts") const expired = new ApproveTickets(0) - expect( - expired.redeem(expired.issue({ repoDir: "r", stage: "frontier", subjectHash: "h" })), - ).toBeUndefined() + expect(expired.redeem(expired.issue({ repoDir: "r", stage: "frontier", subjectHash: "h" }))).toBeUndefined() const live = new ApproveTickets() const ticket = live.issue({ repoDir: "r", stage: "frontier", subjectHash: "h" }) expect(live.redeem(ticket)).toEqual({ repoDir: "r", stage: "frontier", subjectHash: "h" }) From a7d5b6283df893da0334ba1c7ed95e24f5ec6b39 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:07:03 -0400 Subject: [PATCH 59/83] fix(remediation-audit/s5): biome format for web headers test (Refs #282) --- packages/fleet/test/web.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/fleet/test/web.test.ts b/packages/fleet/test/web.test.ts index 99c1807..1a6e26b 100644 --- a/packages/fleet/test/web.test.ts +++ b/packages/fleet/test/web.test.ts @@ -239,7 +239,10 @@ describe("web serving", () => { const anon = await postRpc(server.port, { method: "fleet.status", params: {} }) expect(anon.status).toBe(401) expectSecurityHeaders(anon.headers, "anonymous rpc 401") - const missing = await raw(server.port, `GET /fleet/rpc HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`) + const missing = await raw( + server.port, + `GET /fleet/rpc HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`, + ) expect(missing.status).toBe(404) expectSecurityHeaders(missing.headers, "wrong path 404") } finally { From d7aeb38ae2c65a4fde1550542ed1be8ab96a468a Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:14:32 -0400 Subject: [PATCH 60/83] =?UTF-8?q?fix(remediation-audit/s9):=20doc=20deltas?= =?UTF-8?q?=20=E2=80=94=20writer=20scope,=20attempt=20state,=20quote/audit?= =?UTF-8?q?or=20wording,=20runtime=20matrix,=20factory=20prompt=20(Refs=20?= =?UTF-8?q?#220,=20Refs=20#222,=20Refs=20#254,=20Refs=20#212,=20Refs=20#22?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 +- README.md | 10 ++++++---- docs/adr/0002-approvals-on-every-surface.md | 4 ++-- packages/core/assets/prompts/factory.md | 4 ++-- packages/core/test/__snapshots__/assets.test.ts.snap | 2 +- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3ae1404..36fae2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ spike as proof). | :--- | :--- | | Approvals are human-only; trust, waivers and resume stay terminal-only: passphrase confirmation (terminal echo-off, TUI masked dialog, or loopback browser input per ADR 0002) unlocks the sealed Ed25519 human key (signed, hash-bound) and every surface signs approvals in-process — the TUI with channel `tui`, the browser through the loopback-only fleet endpoints with channel `web`; the approve/trust/resume RPCs no longer exist; agents may request, never grant | `approval/`, `gates/waivers.ts` | | Control files are deny-write for all agents: `.factory/{gates.json,config.json,frontier.json,waivers,approvals,runtime}`, the evidence (`.factory/research/{sources,coverage.json,dossier.json,brief.pcrb.json}`, `.factory/claims/`), `.git/config` and git hooks. The pinned ones (gates, config, frontier, approvals, waivers) are also hash-checked at every gate run; cached sources are engine-sealed (HMAC) and content-addressed | `trust/control.ts` | -| Per-seat path scopes: manager writes factory docs, programmer writes its worktree, QA and the auditor pair write nothing, research alpha/beta write only their own notes (`.factory/research/alpha.md` / `beta.md`) and `research/REPORT.md` has one writer per run mode — the factory in build runs, the research synthesizer in research runs (`PolicyContext.runMode`; an unreadable mode denies both), brainstorm/harvest/design/scout write only their `.factory//notes/`; research, scout and auditor seats are `web: "cached"` (host websearch/webfetch denied, web facts only through the cached `es_research_*` tools) and run `--unshare-net` sandboxed, while user agents keep host web tools | `trust/policy.ts`, `agents/registry.ts` | +| Per-seat path scopes: manager writes factory docs, programmer writes its worktree, QA and the auditor pair write nothing, research alpha/beta write only their own notes (`.factory/research/alpha.md` / `beta.md`) and `research/REPORT.md` has one writer scope per mode — factory-docs seats (factory and manager) in build runs, the research synthesizer in research runs (`PolicyContext.runMode`; an unreadable mode denies both), brainstorm/harvest/design/scout write only their `.factory//notes/`; research, scout and auditor seats are `web: "cached"` (host websearch/webfetch denied, web facts only through the cached `es_research_*` tools) and run `--unshare-net` sandboxed, while user agents keep host web tools | `trust/policy.ts`, `agents/registry.ts` | | Factory seats launch seats in the foreground only: `subagent` with `background: true` from any seat is refused, and so is launching a seat that is still running; seat state and last activity are recorded in `.factory/runtime/seats.json` (advisory) | `opencode/src/policy.ts`, `opencode/src/seats.ts` | | Code audits block: an opened phase audit holds `passPhase` (a missing one too with `audit.phase: required`, set via `es config set`), any other open or failed audit refuses the next stage transition; every finding is witnessed against the file on disk; only a human dismisses one | `factory/machine.ts`, `codeaudit/` | | Scout verdicts are computed by core: `adopt` survives only for a verified, whitelisted, permissive license | `scout/verdict.ts` | diff --git a/README.md b/README.md index 633111e..71205a4 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,10 @@ scanning, OSV, budgets (diff size, file length, complexity, dependency justification, test-with-behaviour), with structured `file:line:rule` findings and signed, expiring waivers. -- **Evidence-first research.** Content-addressed source cache, a pure verbatim - quote verifier, and an epistemic auditor that downgrades ungrounded claims. +- **Evidence-first research.** Content-addressed source cache, a quote + verifier that is verbatim after documented normalisation (NFKC, typography + folds, markdown stripping, whitespace collapse), and an epistemic auditor + that flags, prunes and refuses ungrounded claims. Web sources come through an ordered backend chain (Scraper-Swarm gateway, Brave, Firecrawl, SearXNG, direct fetch) with per-backend cooldowns: one provider's outage or rate limit fails over to the next, while a safety @@ -171,11 +173,11 @@ bun install bun run check # biome + tsc + bun test (all packages) bun run docs:gen # regenerate schemas/ + docs/ after schema changes bun run docs:check # CI drift check -scripts/pack-smoke.sh # build, pack and install the packages; run them under Node and Bun +scripts/pack-smoke.sh # build, pack and install the packages; run the CLI under Node and the plugin server under Bun scripts/v2-head.sh # run the real-host suite against OpenCode v2 HEAD ``` -Platform: Linux, Node ≥ 22 or Bun. No Python. +Platform: Linux (CI runs ubuntu-latest), CLI under Node 22, plugin server under Bun. No Python. ## Documentation diff --git a/docs/adr/0002-approvals-on-every-surface.md b/docs/adr/0002-approvals-on-every-surface.md index 6bbfe47..58445b6 100644 --- a/docs/adr/0002-approvals-on-every-surface.md +++ b/docs/adr/0002-approvals-on-every-surface.md @@ -201,8 +201,8 @@ ticket before merge. All probes apply to both surfaces unless marked. - The TUI plugin links core's approval service directly and signs in-process; no new RPC, no passphrase on the wire anywhere except loopback for the browser path. -- Attempt-limit state lives per surface (a small file in the state dir for - the TUI/CLI; server memory plus audit for the web path). +- Attempt-limit state lives per state dir (one shared attempt file for + the CLI and the TUI; server memory plus audit for the web path, which keeps its own dir). - Docs and messages stop saying approvals are terminal-only, while trust, waive, resume, key seal, config set and export keep saying it. diff --git a/packages/core/assets/prompts/factory.md b/packages/core/assets/prompts/factory.md index 085341d..726b05e 100644 --- a/packages/core/assets/prompts/factory.md +++ b/packages/core/assets/prompts/factory.md @@ -1,13 +1,13 @@ --- id: factory -version: 9 +version: 10 seat: factory description: Orchestrates the build factory from research to release. --- You are the **factory** agent of Epistemic Swarm, an AI build factory. You orchestrate; you do not write product code yourself. ## How the factory works -The run moves through stages enforced by code, not by you: GRILL → RESEARCH → SPEC → BUILD ⇄ QA → RELEASE → DONE. HALTED can happen at any time. Call `es_status` first and whenever you are unsure; its `` block is the truth. If it says there is no run, stop and tell the human to run `/grill` first. Every `es_*` tool refuses an out-of-order step and tells you why. Read the refusal and follow it instead of retrying blindly. +The run moves through stages enforced by code, not by you: GRILL → RESEARCH → SPEC → BUILD ⇄ QA → RELEASE → DONE. HALTED can happen at any time. Call `es_status` first and whenever you are unsure; its `` block is the truth. If it says there is no run, stop and tell the human to run `/grill` first. The phase-transition tools refuse an out-of-order step and tell you why. Read the refusal and follow it instead of retrying blindly. Two checkpoints belong to the human and only the human: approving the frontier (end of grill) and approving the spec (roadmap plus every GOAL.md). You cannot approve anything. When a checkpoint is due, call `es_request_approval` and tell the human to run `/es-approve` in the TUI (or `es approve ` in a terminal). Then stop and wait. diff --git a/packages/core/test/__snapshots__/assets.test.ts.snap b/packages/core/test/__snapshots__/assets.test.ts.snap index 7f25617..2466652 100644 --- a/packages/core/test/__snapshots__/assets.test.ts.snap +++ b/packages/core/test/__snapshots__/assets.test.ts.snap @@ -8,7 +8,7 @@ exports[`agent assets prompt snapshot: a changed prompt must bump its version 1` "brainstormer": "v4:278864b7a2cf7008", "deep-researcher": "v1:2c436cd49fd6aaf8", "designer": "v2:745908a2a4abff0b", - "factory": "v9:4b48ec73da814ab3", + "factory": "v10:d87aa973b1517d02", "grill": "v4:722c43eeb12d95bb", "harvester": "v2:c63fbc079e1cbbbe", "lens-adjacent-possible": "v1:d6d768502ed2adf2", From 0a8adaddd9c86d5d4b142d872be0c398bf456e04 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:14:33 -0400 Subject: [PATCH 61/83] =?UTF-8?q?fix(remediation-audit/s9):=20packaging=20?= =?UTF-8?q?structural=20=E2=80=94=20manifest=20version=20parity,=20publish?= =?UTF-8?q?=20fail-closed=20guard=20(Refs=20#212)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/publish.yml | 6 ++++++ packages/core/test/packages.test.ts | 13 ++++++++++--- scripts/packages.ts | 3 +++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index af38752..03c1f4e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -63,6 +63,12 @@ jobs: # NOTE: the ./ prefix is load-bearing. A bare packs/foo.tgz is # parsed as a GitHub user/repo shorthand (git ls-remote failure); # ./packs/foo.tgz is resolved as the local tarball. + # Fail closed: an empty tarball list (e.g. --release format drift + # through the awk above) must never exit 0 having published nothing. + if [ -z "${{ steps.packages.outputs.tarballs }}" ]; then + echo "publish: no tarballs listed by scripts/packages.ts --release" >&2 + exit 1 + fi for tgz in ${{ steps.packages.outputs.tarballs }}; do echo "::group::npm publish $tgz" if [ "$DRY_RUN" = "true" ]; then diff --git a/packages/core/test/packages.test.ts b/packages/core/test/packages.test.ts index 8515750..e67b372 100644 --- a/packages/core/test/packages.test.ts +++ b/packages/core/test/packages.test.ts @@ -64,8 +64,15 @@ describe("workspace packages", () => { test("the real repo releases core, cli and the plugin in dependency order", async () => { const root = path.resolve(import.meta.dir, "../../..") expect((await releasePackages(root)).map((pkg) => pkg.dir)).toEqual(["core", "cli", "opencode"]) - expect((await workspacePackages(root)).map((pkg) => pkg.dir).sort()).toEqual( - expect.arrayContaining(["cli", "core", "opencode", "testkit"]), - ) + const all = await workspacePackages(root) + expect(all.map((pkg) => pkg.dir).sort()).toEqual(expect.arrayContaining(["cli", "core", "opencode", "testkit"])) + // Manifest name re-scope or version skew must not ship green (#212): + // pin the release manifests' names and lockstep version beside the dir. + const byDir = new Map(all.map((pkg) => [pkg.dir, pkg])) + expect(byDir.get("core")).toMatchObject({ name: "@heretek-ai/es-core" }) + expect(byDir.get("cli")).toMatchObject({ name: "@heretek-ai/es-cli" }) + expect(byDir.get("opencode")).toMatchObject({ name: "@heretek-ai/epistemic-swarm" }) + const versions = new Set((await releasePackages(root)).map((pkg) => pkg.version)) + expect(versions.size).toBe(1) }) }) diff --git a/scripts/packages.ts b/scripts/packages.ts index a0fe93b..d88f05c 100644 --- a/scripts/packages.ts +++ b/scripts/packages.ts @@ -13,6 +13,8 @@ export interface WorkspacePackage { /** Directory under packages/ (also the short name scripts use). */ readonly dir: string readonly name: string + /** Manifest version string (publish parity: a skew ships green otherwise). */ + readonly version: string readonly private: boolean /** Publish order for release packages; undefined when not released. */ readonly order: number | undefined @@ -45,6 +47,7 @@ export async function workspacePackages(root?: string): Promise Date: Sat, 10 Oct 2026 21:14:38 -0400 Subject: [PATCH 62/83] chore(remediation-audit): regenerate schemas and generated docs --- docs/CAPABILITIES.md | 2 +- docs/SCHEMAS.md | 2 + schemas/domain-pack.schema.json | 14 ----- schemas/scout-advisories-file.schema.json | 72 +++++++++++++++++++++++ schemas/scout-advisory-query.schema.json | 20 +++++++ 5 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 schemas/scout-advisories-file.schema.json create mode 100644 schemas/scout-advisory-query.schema.json diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md index 8da355c..034a3c6 100644 --- a/docs/CAPABILITIES.md +++ b/docs/CAPABILITIES.md @@ -26,7 +26,7 @@ and Antigravity adapters ship in 1.1–1.3). Generated: do not edit by hand. | design | ENFORCED | `packages/opencode/test/design.test.ts` | Queereye interview, DTCG tokens, contrast gate, generated guide and drift check; phase-02 specs/webref/csf/tui renders and the phase-03 ledger bundle with its cite-gate receipt. | | claims | ENFORCED | `packages/core/test/research.test.ts` | Witnessed claims and dossiers: the evidence cache, dossiers and the claim ledger are tool-only; planted cache entries are refused (engine seal) and every VERIFIED quote and code span is re-checked against the bytes on disk. | | audit | ENFORCED | `packages/opencode/test/fires.test.ts` | Code-audit pair: only auditor seats record verdicts; every finding is witnessed on disk (a hallucinated line is refused); quote fragments carry >= 12 chars each and a thesis pass needs a witnessed invariant; verdicts block (an opened phase audit holds the merge; audit.phase required also holds a missing one until it passes or a human dismisses it). | -| scout | ENFORCED | `packages/opencode/test/fires.test.ts` | OSS scout: cached-only web under an offline (--unshare-net) sandbox, fail-closed license verdicts computed by core (adopt only for verified permissive licenses), OSV advisories cited. | +| scout | ENFORCED | `packages/opencode/test/scout-host.test.ts` | OSS scout: cached-only web under an offline (--unshare-net) sandbox, fail-closed license verdicts computed by core (adopt only for verified permissive licenses), OSV advisories cited (canned-OSV content in core's scout suite). | | research-fires | ENFORCED | `packages/opencode/test/fires-research.test.ts` | Research fires: the whole research loop on the real host with loopback-only servers — grounded quotes survive the audit, failover records provenance, blocked never fails over, planted sources are refused, retractions degrade to STALE, renders snapshot. | ## claude diff --git a/docs/SCHEMAS.md b/docs/SCHEMAS.md index 9eb77c1..6d30617 100644 --- a/docs/SCHEMAS.md +++ b/docs/SCHEMAS.md @@ -76,6 +76,8 @@ Do not edit by hand: run `bun run docs:gen` after changing a schema (CI runs `bu | run-phase-telemetry | `schemas/run-phase-telemetry.schema.json` | | run-state-seal | `schemas/run-state-seal.schema.json` | | run-telemetry | `schemas/run-telemetry.schema.json` | +| scout-advisories-file | `schemas/scout-advisories-file.schema.json` | +| scout-advisory-query | `schemas/scout-advisory-query.schema.json` | | scout-advisory | `schemas/scout-advisory.schema.json` | | scout-assessment | `schemas/scout-assessment.schema.json` | | scout-plan | `schemas/scout-plan.schema.json` | diff --git a/schemas/domain-pack.schema.json b/schemas/domain-pack.schema.json index 0f0fc1d..70f7d76 100644 --- a/schemas/domain-pack.schema.json +++ b/schemas/domain-pack.schema.json @@ -74,20 +74,6 @@ "zero_tolerance" ] }, - "tierDomains": { - "description": "Pack host overrides by tier for source classification.", - "type": "object", - "propertyNames": { - "type": "string" - }, - "additionalProperties": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - }, "exportTemplate": { "type": "string" } diff --git a/schemas/scout-advisories-file.schema.json b/schemas/scout-advisories-file.schema.json new file mode 100644 index 0000000..ef1e935 --- /dev/null +++ b/schemas/scout-advisories-file.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "package": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "ecosystem", + "package" + ], + "additionalProperties": false + }, + "advisories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "aliases": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "severity": { + "type": "string" + }, + "fixed": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "source": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "id", + "summary", + "aliases", + "fixed" + ], + "additionalProperties": false + } + } + }, + "required": [ + "query", + "advisories" + ], + "additionalProperties": false +} diff --git a/schemas/scout-advisory-query.schema.json b/schemas/scout-advisory-query.schema.json new file mode 100644 index 0000000..8216745 --- /dev/null +++ b/schemas/scout-advisory-query.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "package": { + "type": "string" + }, + "version": { + "type": "string" + } + }, + "required": [ + "ecosystem", + "package" + ], + "additionalProperties": false +} From d8a16e3371fb25e2c88b57e75d98c483bd778367 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:19:56 -0400 Subject: [PATCH 63/83] fix(remediation-audit/director): locked approval requests (Refs #219) --- packages/core/src/ops/tools.ts | 13 ++++--- packages/core/test/pending-approvals.test.ts | 38 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 packages/core/test/pending-approvals.test.ts diff --git a/packages/core/src/ops/tools.ts b/packages/core/src/ops/tools.ts index c7e3ce2..5f00ad3 100644 --- a/packages/core/src/ops/tools.ts +++ b/packages/core/src/ops/tools.ts @@ -16,7 +16,7 @@ import { formatReport, runGates } from "../gates/engine.ts" import { factoryLayout } from "../layout.ts" import type { GateFinding } from "../schema/gates.ts" import { relativeTo } from "../trust/paths.ts" -import { writeJson } from "../util/fs.ts" +import { withLock, writeJson } from "../util/fs.ts" export type EsToolContext = { readonly agent: string; readonly sessionID?: string; readonly signal?: AbortSignal } export interface EsToolDef { @@ -121,9 +121,14 @@ export function esTools(ops: OpsContext): EsToolDef[] { `Not ready for approval: ${error.message}${missing ? " Run /grill (or `es factory begin` + grill flow) to record the frontier, then request approval again." : ""}`, ) }) - const pending = (await pendingApprovals(root)).filter((item) => item.stage !== stage) - pending.push({ stage, requestedBy: context.agent, at: new Date().toISOString(), ...(note ? { note } : {}) }) - await writeJson(factoryLayout(root).pending, pending) + const pendingFile = factoryLayout(root).pending + // Locked against concurrent requests and the human approve path + // (which clears under the same state lock): #219 lost-update. + await withLock(factoryLayout(root).state, async () => { + const pending = (await pendingApprovals(root)).filter((item) => item.stage !== stage) + pending.push({ stage, requestedBy: context.agent, at: new Date().toISOString(), ...(note ? { note } : {}) }) + await writeJson(pendingFile, pending) + }) await appendAuditEntry(root, { actor: `agent:${context.agent}`, action: "approval.request", diff --git a/packages/core/test/pending-approvals.test.ts b/packages/core/test/pending-approvals.test.ts new file mode 100644 index 0000000..206ff8d --- /dev/null +++ b/packages/core/test/pending-approvals.test.ts @@ -0,0 +1,38 @@ +// Concurrent approval requests must not lost-update pending-approvals.json (#219). +import { describe, expect, test } from "bun:test" +import { Factory } from "../src/factory/index.ts" +import { pendingApprovals } from "../src/factory/pending.ts" +import { esTools } from "../src/ops/tools.ts" +import { type Fixture, frontier, gitRepo, writeSpecs } from "./helpers.ts" + +describe("pending approvals under concurrency (#219)", () => { + test("two concurrent approval requests for different stages keep both entries", async () => { + const fx: Fixture = await gitRepo("es-pending-") + try { + const factory = new Factory(fx.root, { + gates: async () => ({ passed: true, findings: [], summary: "green" }), + stateDir: fx.state, + }) + await factory.begin("human:tester") + await factory.writeFrontier("grill", frontier()) + await writeSpecs(fx.root, ["alpha"]) + const tools = esTools({ root: fx.root, factory, stateDir: fx.state }) + const request = tools.find((tool) => tool.name === "es_request_approval")! + // Repeat the race several times: without a lock, the read-modify-write + // interleaves and one stage's entry is lost. + for (let round = 0; round < 8; round++) { + const { writeJson } = await import("../src/util/fs.ts") + const { factoryLayout } = await import("../src/layout.ts") + await writeJson(factoryLayout(fx.root).pending, []) + await Promise.all([ + request.execute({ stage: "frontier", note: `r${round}` }, { agent: "factory" }), + request.execute({ stage: "spec", note: `r${round}` }, { agent: "factory" }), + ]) + const pending = await pendingApprovals(fx.root) + expect(pending.map((item) => item.stage).sort()).toEqual(["frontier", "spec"]) + } + } finally { + await fx.cleanup() + } + }) +}) From b631de14914eafa315ac466a07b52c5d7dd252f5 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:19:56 -0400 Subject: [PATCH 64/83] fix(remediation-audit/director): recordGateRun refuses outside BUILD + base-branch wording (Refs #223, Refs #238) --- packages/core/src/factory/machine.ts | 5 +++-- packages/core/test/record-gate-stage.test.ts | 23 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 packages/core/test/record-gate-stage.test.ts diff --git a/packages/core/src/factory/machine.ts b/packages/core/src/factory/machine.ts index 5308fe6..ac17331 100644 --- a/packages/core/src/factory/machine.ts +++ b/packages/core/src/factory/machine.ts @@ -691,7 +691,8 @@ export class Factory { /** Record a gate run made during BUILD (red test runs are the failing-test-first evidence). */ async recordGateRun(agentId: string | undefined, result: GateRunLike): Promise { await this.mutate(`agent:${agentId}`, (state) => { - if (state.stage !== "BUILD") return + if (state.stage !== "BUILD") + throw new FactoryError(`Gate runs record during BUILD only (current stage: ${state.stage}).`) const phase = this.active(state) const failed = result.failedTests ?? [] const red = failed.length > 0 || result.testsFailed === true @@ -1240,7 +1241,7 @@ export class Factory { private async prBody(state: FactoryState) { const spec = await readApproval(this.root, "spec") const lines = [ - `Built by the Epistemic Swarm factory (run \`${state.runId}\`). A human reviews and merges; the factory never merges.`, + `Built by the Epistemic Swarm factory (run \`${state.runId}\`). A human reviews and merges; the factory never merges into the release/base branch.`, "", "## Phases", ...state.phases.map( diff --git a/packages/core/test/record-gate-stage.test.ts b/packages/core/test/record-gate-stage.test.ts new file mode 100644 index 0000000..89f7b7f --- /dev/null +++ b/packages/core/test/record-gate-stage.test.ts @@ -0,0 +1,23 @@ +// Gate runs outside BUILD must refuse, never silently record nothing (#223.2). +import { describe, expect, test } from "bun:test" +import { Factory, FactoryError } from "../src/factory/index.ts" +import { type Fixture, frontier, gitRepo } from "./helpers.ts" + +describe("recordGateRun stage gate (#223)", () => { + test("recording outside BUILD refuses instead of silently dropping the run", async () => { + const fx: Fixture = await gitRepo("es-gaterun-") + try { + const factory = new Factory(fx.root, { + gates: async () => ({ passed: true, findings: [], summary: "green" }), + stateDir: fx.state, + }) + await factory.begin("human:tester") + await factory.writeFrontier("grill", frontier()) + await expect( + factory.recordGateRun("es-programmer", { passed: true, findings: [], summary: "green" }), + ).rejects.toThrow("BUILD") + } finally { + await fx.cleanup() + } + }) +}) From d5d4bfd5a06a22ed12ab3f7cf2d07352f1a73ae7 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:19:58 -0400 Subject: [PATCH 65/83] fix(remediation-audit/director): STOP/HALTED on user launches + MCP surface (Refs #227) --- packages/cli/src/mcp.ts | 15 +++++- packages/cli/test/mcp-identity.test.ts | 26 ++++++++++ packages/opencode/src/policy.ts | 7 ++- packages/opencode/test/stop-launch.test.ts | 59 ++++++++++++++++++++++ 4 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 packages/opencode/test/stop-launch.test.ts diff --git a/packages/cli/src/mcp.ts b/packages/cli/src/mcp.ts index 319cba8..d0ee5a4 100644 --- a/packages/cli/src/mcp.ts +++ b/packages/cli/src/mcp.ts @@ -9,7 +9,7 @@ // seat-checked tool refuses. Only a registry agent id (`factory`, // `es-manager`, `es-programmer`, …) holds a seat: any other pinned string is // still the caller, but a seat-less one, refused the same way as "mcp". -import { stateDir as defaultStateDir, esTools, Factory, gateRunner } from "@heretek-ai/es-core" +import { checkStopFile, stateDir as defaultStateDir, esTools, Factory, gateRunner } from "@heretek-ai/es-core" import { VERSION } from "./version.ts" export const PROTOCOL_VERSION = "2025-06-18" @@ -77,6 +77,19 @@ export function createMcpServer(options: McpOptions) { if (!tool) return fail(-32602, `Unknown tool: ${params?.name}`) const { agent: _ignored, ...input } = (params?.arguments ?? {}) as Json try { + // STOP and HALTED hold on the MCP surface too (#227): the + // kill-switch refuses every call; a halted run refuses everything + // but status, mirroring the OpenCode host policy. + const stop = await checkStopFile(options.root) + if (stop.stopped) + throw new Error( + `The factory is stopped (.factory/STOP: ${stop.reason}). A human must remove it and resume.`, + ) + if (tool.name !== "es_status") { + const state = await factory.read().catch(() => undefined) + if (state?.stage === "HALTED") + throw new Error(`The factory is halted: ${state.halt?.reason}. Stop and report; a human must resume.`) + } const text = await tool.execute(input, { agent: caller }) return reply({ content: [{ type: "text", text }] }) } catch (error) { diff --git a/packages/cli/test/mcp-identity.test.ts b/packages/cli/test/mcp-identity.test.ts index fe5131e..8372288 100644 --- a/packages/cli/test/mcp-identity.test.ts +++ b/packages/cli/test/mcp-identity.test.ts @@ -189,3 +189,29 @@ describe("mcp identity over real stdio", () => { expect(await readFile(path.join(root, "README.md"), "utf8")).toContain("# x") }) }) + +describe("mcp STOP/HALTED enforcement (#227)", () => { + test("a STOP file refuses tool calls on the MCP surface", async () => { + await Bun.write(path.join(root, ".factory/STOP"), "halt\n") + try { + const server = createMcpServer({ root, stateDir: state, env: { ES_AGENT: "factory" } }) + const refused = await call(server, "status", {}) + expect(refused?.result.isError).toBe(true) + expect(refused?.result.content[0].text).toContain("stopped") + } finally { + await rm(path.join(root, ".factory/STOP"), { force: true }) + } + }) + + test("a HALTED run refuses non-status tool calls on the MCP surface", async () => { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await factory.begin("human:tester") + await factory.halt("human:tester", "spend ceiling") + const server = createMcpServer({ root, stateDir: state, env: { ES_AGENT: "factory" } }) + const refused = await call(server, "complete", {}) + expect(refused?.result.isError).toBe(true) + expect(refused?.result.content[0].text).toContain("halted") + const status = await call(server, "status", {}) + expect(status?.result.isError).not.toBe(true) + }) +}) diff --git a/packages/opencode/src/policy.ts b/packages/opencode/src/policy.ts index 7f74bd0..b55453d 100644 --- a/packages/opencode/src/policy.ts +++ b/packages/opencode/src/policy.ts @@ -88,14 +88,17 @@ export function createPolicyHooks(runtime: Runtime, seats?: SeatTracker) { } if (seat && event.tool === "subagent") guardSeatLaunch({ agent: event.agent, input }, seats) if (seat && event.sessionID) await seats?.onTool(event.sessionID, event.agent, event.tool) - if (seat || event.tool.startsWith("es_")) { + // STOP halts seats, es_* tools, and seat launches whoever fires them (#227): + // a user agent must not route around the kill-switch via `subagent`. + if (seat || event.tool.startsWith("es_") || event.tool === "subagent") { const stop = await checkStopFile(runtime.root) if (stop.stopped) deny(`The factory is stopped (.factory/STOP: ${stop.reason}). A human must remove it and resume.`) } // A halted run (spend ceiling, runtime cap, drift, STOP) stops every seat // tool except es_status, so no seat keeps spending through its own es_* tools. - if (seat && event.tool !== "es_status") { + // Seat launches count as seat tools whoever fires them (#227). + if ((seat || event.tool === "subagent") && event.tool !== "es_status") { const state = await runtime.factory.read().catch(() => undefined) if (state?.stage === "HALTED") deny(`The factory is halted: ${state.halt?.reason}. Stop and report; a human must resume.`) diff --git a/packages/opencode/test/stop-launch.test.ts b/packages/opencode/test/stop-launch.test.ts new file mode 100644 index 0000000..b1a1f80 --- /dev/null +++ b/packages/opencode/test/stop-launch.test.ts @@ -0,0 +1,59 @@ +// A user-agent seat launch must not bypass STOP/HALTED (#227). +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { Factory, gateRunner } from "@heretek-ai/es-core" +import { createPolicyHooks } from "../src/policy.ts" +import type { Runtime } from "../src/runtime.ts" + +let root: string +let state: string + +beforeEach(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-stop-launch-")) + state = await mkdtemp(path.join(tmpdir(), "es-stop-launch-state-")) +}) + +afterEach(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const runtime = (kind: "stop" | "halted" | "clean") => + ({ + root, + factory: + kind === "halted" + ? { read: async () => ({ stage: "HALTED", halt: { reason: "spend ceiling" } }) } + : new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }), + policy: async () => ({ root }), + }) as unknown as Runtime + +const launch = (agent: string) => ({ + tool: "subagent", + agent, + id: "1", + input: { agent: "es-qa-functional", description: "qa", prompt: "check" }, +}) + +describe("STOP/HALTED covers user-agent seat launches (#227)", () => { + test("a user agent cannot launch a seat while STOP is present", async () => { + await mkdir(path.join(root, ".factory"), { recursive: true }) + await writeFile(path.join(root, ".factory/STOP"), "halt\n") + const { before } = createPolicyHooks(runtime("stop")) + await expect(before(launch("build") as never)).rejects.toThrow("stopped") + }) + + test("a user agent cannot launch a seat while the run is HALTED", async () => { + const { before } = createPolicyHooks(runtime("halted")) + await expect(before(launch("build") as never)).rejects.toThrow("halted") + }) + + test("a user-agent launch is allowed when neither STOP nor HALTED applies", async () => { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + void factory + const { before } = createPolicyHooks(runtime("clean")) + await before(launch("build") as never) + }) +}) From 9a0676ee66b480a8b9f50495dbcd72ef97af8bfd Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:19:58 -0400 Subject: [PATCH 66/83] fix(remediation-audit/director): hasHumanKey requires both halves (Refs #213) --- packages/core/src/approval/keystore.ts | 8 +++++++- packages/core/test/approval-keystore.test.ts | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/core/src/approval/keystore.ts b/packages/core/src/approval/keystore.ts index cd8659b..03d3c5e 100644 --- a/packages/core/src/approval/keystore.ts +++ b/packages/core/src/approval/keystore.ts @@ -103,7 +103,13 @@ async function readText(file: string): Promise { /** Whether this machine holds a sealed human key (`es key seal` has run). */ export async function hasHumanKey(dir = stateDir()): Promise { - return (await readText(path.join(dir, SEALED_FILE))) !== undefined + // Both halves are required (#213): a lone human.key (or human.pub) is a + // half-deleted brick, and unlock would refuse it as tampered or partial. + const [sealed, pub] = await Promise.all([ + readText(path.join(dir, SEALED_FILE)), + readText(path.join(dir, PUBLIC_FILE)), + ]) + return sealed !== undefined && pub !== undefined } async function publicKey(dir: string): Promise { diff --git a/packages/core/test/approval-keystore.test.ts b/packages/core/test/approval-keystore.test.ts index 28b34f8..676b335 100644 --- a/packages/core/test/approval-keystore.test.ts +++ b/packages/core/test/approval-keystore.test.ts @@ -1,6 +1,8 @@ // Human key lifecycle (#119): destroy() ends signing. import { expect, test } from "bun:test" -import { HumanKeyError, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { writeFile } from "node:fs/promises" +import path from "node:path" +import { hasHumanKey, HumanKeyError, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" import { gitRepo } from "./helpers.ts" const PASSPHRASE = "test-passphrase-1234" @@ -17,3 +19,14 @@ test("destroy zeroes the key and further signs throw", async () => { fx.cleanup() } }) + +test("hasHumanKey requires both halves: a lone human.key is a brick, not a key (#213)", async () => { + const fx = await gitRepo() + try { + expect(await hasHumanKey(fx.state)).toBe(false) + await writeFile(path.join(fx.state, "human.key"), '{"version":1}\n') + expect(await hasHumanKey(fx.state)).toBe(false) + } finally { + fx.cleanup() + } +}) From 9b044dfb899f8c9a123038725f66e8f4171f98c3 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:19:58 -0400 Subject: [PATCH 67/83] fix(remediation-audit/director): eval runner input hardening (Refs #211) --- packages/core/src/evals/grade.ts | 3 ++- packages/core/src/evals/run.ts | 11 +++++++++-- packages/core/test/evals.test.ts | 6 ++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/core/src/evals/grade.ts b/packages/core/src/evals/grade.ts index fcefa75..5cd668c 100644 --- a/packages/core/src/evals/grade.ts +++ b/packages/core/src/evals/grade.ts @@ -48,7 +48,8 @@ export interface EvalTranscript { readonly tokens: EvalTokens } -const num = (value: unknown): number => (typeof value === "number" && Number.isFinite(value) ? value : 0) +const num = (value: unknown): number => + typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0 /** * Tools whose outputs are raw evidence, not the seat's own words. Quoting diff --git a/packages/core/src/evals/run.ts b/packages/core/src/evals/run.ts index 4d0fc51..0ad9591 100644 --- a/packages/core/src/evals/run.ts +++ b/packages/core/src/evals/run.ts @@ -41,7 +41,8 @@ export const DEFAULT_CASE_STEPS = 6 * makes one call per step), bounded by the run-wide ceiling. */ export function caseStepCap(caseMaxSteps: number | undefined, ceiling: number): number { - return Math.min(caseMaxSteps ?? DEFAULT_CASE_STEPS, ceiling) + const own = typeof caseMaxSteps === "number" && Number.isFinite(caseMaxSteps) ? caseMaxSteps : DEFAULT_CASE_STEPS + return Math.min(own, ceiling) } export interface EvalWorkspace { @@ -140,6 +141,11 @@ export async function runEvalCase( workspace: EvalWorkspace, options: EvalRunOptions, ): Promise { + // Fail closed on flag injection (#211): the prompt is a positional argv, so + // a leading dash would parse as a CLI flag. Cases are repo files, but the + // runner must not depend on their goodwill. + if (testCase.prompt.startsWith("-")) + throw new Error(`Refusing eval case "${testCase.id}": the prompt must not start with "-".`) const child = spawn( options.binary, ["run", "--standalone", "--format", "json", "--agent", testCase.agent, testCase.prompt], @@ -167,7 +173,8 @@ export async function runEvalCase( child.kill("SIGKILL") } if (event.type === "step_finish" && typeof event.part?.cost === "number" && Number.isFinite(event.part.cost)) { - spent += event.part.cost + // Clamp hostile negatives: a negative cost must never unbind spend (#211). + spent += Math.max(0, event.part.cost) if (options.budgetUSD !== undefined && spent > options.budgetUSD && !overBudget) { overBudget = true child.kill("SIGKILL") diff --git a/packages/core/test/evals.test.ts b/packages/core/test/evals.test.ts index 770d3c6..63da048 100644 --- a/packages/core/test/evals.test.ts +++ b/packages/core/test/evals.test.ts @@ -90,6 +90,8 @@ describe("transcripts", () => { expect(transcript.tokens).toEqual({ input: 150, output: 15, reasoning: 2, cacheRead: 4, cacheWrite: 6 }) // Missing or non-numeric fields count as zero rather than poisoning the sum. expect(readTranscript([{ type: "step_finish", part: { cost: "n/a" } }]).costUSD).toBe(0) + // Negative costs clamp to zero: a hostile stream must not unbind spend (#211). + expect(readTranscript([{ type: "step_finish", part: { cost: -5 } }]).costUSD).toBe(0) }) test("a completed call's output is graded text, except raw evidence and errored calls", () => { @@ -133,6 +135,10 @@ describe("step caps", () => { expect(caseStepCap(undefined, 16)).toBe(DEFAULT_CASE_STEPS) expect(caseStepCap(undefined, 3)).toBe(3) }) + + test("a NaN maxSteps falls back to the default instead of disabling the cap (#211)", () => { + expect(caseStepCap(Number.NaN, 16)).toBe(DEFAULT_CASE_STEPS) + }) }) describe("fire graders", () => { From 6dd5f387abad428d729b7050d0674968135c43d0 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:20:40 -0400 Subject: [PATCH 68/83] fix(remediation-audit/director): biome lint on new tests + grade format (Refs #211, Refs #213, Refs #223) --- packages/core/src/evals/grade.ts | 3 +-- packages/core/test/approval-keystore.test.ts | 2 +- packages/core/test/record-gate-stage.test.ts | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/src/evals/grade.ts b/packages/core/src/evals/grade.ts index 5cd668c..3b17fca 100644 --- a/packages/core/src/evals/grade.ts +++ b/packages/core/src/evals/grade.ts @@ -48,8 +48,7 @@ export interface EvalTranscript { readonly tokens: EvalTokens } -const num = (value: unknown): number => - typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0 +const num = (value: unknown): number => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0) /** * Tools whose outputs are raw evidence, not the seat's own words. Quoting diff --git a/packages/core/test/approval-keystore.test.ts b/packages/core/test/approval-keystore.test.ts index 676b335..056e04c 100644 --- a/packages/core/test/approval-keystore.test.ts +++ b/packages/core/test/approval-keystore.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test" import { writeFile } from "node:fs/promises" import path from "node:path" -import { hasHumanKey, HumanKeyError, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { HumanKeyError, hasHumanKey, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" import { gitRepo } from "./helpers.ts" const PASSPHRASE = "test-passphrase-1234" diff --git a/packages/core/test/record-gate-stage.test.ts b/packages/core/test/record-gate-stage.test.ts index 89f7b7f..33e6a10 100644 --- a/packages/core/test/record-gate-stage.test.ts +++ b/packages/core/test/record-gate-stage.test.ts @@ -1,6 +1,6 @@ // Gate runs outside BUILD must refuse, never silently record nothing (#223.2). import { describe, expect, test } from "bun:test" -import { Factory, FactoryError } from "../src/factory/index.ts" +import { Factory } from "../src/factory/index.ts" import { type Fixture, frontier, gitRepo } from "./helpers.ts" describe("recordGateRun stage gate (#223)", () => { From 7e0f41ec9a1239cd7de82c1f7e37903ad76219b0 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:21:43 -0400 Subject: [PATCH 69/83] fix(remediation-audit/director): biome format/import-sort on squad-merged files --- packages/core/src/hooks/engine.ts | 2 +- packages/core/test/codeaudit.test.ts | 5 ++++- packages/core/test/evidence-seal.test.ts | 12 ++++++++++-- packages/core/test/failover-gaps.test.ts | 16 ++++++++++------ packages/core/test/hooks.test.ts | 2 +- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/packages/core/src/hooks/engine.ts b/packages/core/src/hooks/engine.ts index 6d0460f..c5c0385 100644 --- a/packages/core/src/hooks/engine.ts +++ b/packages/core/src/hooks/engine.ts @@ -6,8 +6,8 @@ // missing trust; observers fail open, as Claude Code does. Every executed // handler is written to the audit log. import { appendAuditEntry } from "../audit/chain.ts" -import { isTrusted } from "../trust/store.ts" import type { SandboxSpec } from "../trust/sandbox.ts" +import { isTrusted } from "../trust/store.ts" import { type HookSource, hookSources, type LoadedHooks, loadHooks } from "./load.ts" import { ifMatches, matcherMatches } from "./match.ts" import { type HandlerOutcome, runHandler } from "./run.ts" diff --git a/packages/core/test/codeaudit.test.ts b/packages/core/test/codeaudit.test.ts index b298688..b2d0940 100644 --- a/packages/core/test/codeaudit.test.ts +++ b/packages/core/test/codeaudit.test.ts @@ -151,7 +151,10 @@ describe("the line-pointer law at es_audit_verdict", () => { // verdict level, not just in the sibling span tests. const short = { ...injection, lines: [3, 3], excerpt: "db.query" } await expect( - verdict.execute({ audit: audit.id, verdict: "fail", findings: [short], notes: "n" }, { agent: "es-auditor-antithesis" }), + verdict.execute( + { audit: audit.id, verdict: "fail", findings: [short], notes: "n" }, + { agent: "es-auditor-antithesis" }, + ), ).rejects.toThrow(/too short to be evidence/) expect((await factory.read())!.audits[0]!.antithesis).toBeUndefined() }) diff --git a/packages/core/test/evidence-seal.test.ts b/packages/core/test/evidence-seal.test.ts index d52aee9..29bc13b 100644 --- a/packages/core/test/evidence-seal.test.ts +++ b/packages/core/test/evidence-seal.test.ts @@ -10,8 +10,16 @@ import path from "node:path" import { SourceCache } from "../src/research/cache.ts" const put2 = async (cache: SourceCache) => { - const a = await cache.put({ url: "https://a.test/page", text: "Alpha source text with enough length.", provider: "fetch" }) - const b = await cache.put({ url: "https://b.test/page", text: "Beta source text with enough length here.", provider: "fetch" }) + const a = await cache.put({ + url: "https://a.test/page", + text: "Alpha source text with enough length.", + provider: "fetch", + }) + const b = await cache.put({ + url: "https://b.test/page", + text: "Beta source text with enough length here.", + provider: "fetch", + }) return { a, b } } diff --git a/packages/core/test/failover-gaps.test.ts b/packages/core/test/failover-gaps.test.ts index 2d22689..9572162 100644 --- a/packages/core/test/failover-gaps.test.ts +++ b/packages/core/test/failover-gaps.test.ts @@ -9,8 +9,8 @@ import { mkdtemp, readdir, rm } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" import { - backoffFor, BackendError, + backoffFor, runFetchChain, runSearchChain, type SourceBackend, @@ -143,7 +143,11 @@ describe("token absence exhaustiveness (#253.3)", () => { ["http-500", (seen) => client(() => jsonRpc({ detail: "boom" }, 500), seen).search("q", 1), "upstream"], [ "contract", - (seen) => client(() => jsonRpc({ jsonrpc: "2.0", id: 1, result: {} }, 200, { "x-swarm-contract": "9" }), seen).search("q", 1), + (seen) => + client(() => jsonRpc({ jsonrpc: "2.0", id: 1, result: {} }, 200, { "x-swarm-contract": "9" }), seen).search( + "q", + 1, + ), "upstream", ], [ @@ -191,10 +195,10 @@ describe("token absence exhaustiveness (#253.3)", () => { [ "tool-upstream", (seen) => - client( - () => jsonRpc({ jsonrpc: "2.0", id: 1, error: { code: -32602, message: "bad params" } }), - seen, - ).search("q", 1), + client(() => jsonRpc({ jsonrpc: "2.0", id: 1, error: { code: -32602, message: "bad params" } }), seen).search( + "q", + 1, + ), "upstream", ], [ diff --git a/packages/core/test/hooks.test.ts b/packages/core/test/hooks.test.ts index 1bd4b39..13ea91a 100644 --- a/packages/core/test/hooks.test.ts +++ b/packages/core/test/hooks.test.ts @@ -374,7 +374,7 @@ describe("other events", () => { process.env.ES_HOOK_PARENT_ENV_PIN = "pinned-123" try { const hooks = await engine({ - UserPromptSubmit: [{ hooks: [{ type: "command", command: "echo \"pin=$ES_HOOK_PARENT_ENV_PIN\"" }] }], + UserPromptSubmit: [{ hooks: [{ type: "command", command: 'echo "pin=$ES_HOOK_PARENT_ENV_PIN"' }] }], }) expect(await hooks.userPromptSubmit(base, "hi")).toMatchObject({ additionalContext: ["pin=pinned-123"] }) } finally { From 111244166ac0556a4df759e9d8302405f661ebda Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:26:07 -0400 Subject: [PATCH 70/83] fix(remediation-audit/director): headless fake driver explicit advisory skip (Refs #257) --- packages/cli/test/cli.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index e30e274..aa6f48f 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -1351,7 +1351,20 @@ describe("es audit and es scout (headless jobs)", () => { ) await tool("es_scout_scan").execute({ candidate: "tiny" }, scout) await tool("es_scout_record").execute( - { candidate: "tiny", maintenance: "slow", proposal: "adopt", rationale: "small enough to keep", claims: [] }, + { + candidate: "tiny", + maintenance: "slow", + proposal: "adopt", + rationale: "small enough to keep", + claims: [ + { + tag: "NEGATIVE_KNOWLEDGE", + statement: "OSV advisories were not queried for tiny", + query: "OSV advisories for tiny", + finding: "skipped: a local fixture with no published package", + }, + ], + }, scout, ) await tool("es_scout_complete").execute({}, scout) From 1a60a96a7e573ee61a92772d6f51942d31de79e1 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:46:05 -0400 Subject: [PATCH 71/83] fix(remediation-followup): linear-time renderedSlotsOfSpecMd (Refs #291) The section regex added for #270 (`/## Slots\n([\s\S]*?)(?:\n## |\s*$)/`) was quadratic on a whitespace run that does not reach the end of the file: CodeQL js/polynomial-redos on PR #290 (1.2s at 40k tabs, 7.6s at 100k). The bytes come from the bundle on disk, not from the renderer. Cut the section with indexOf/trimEnd instead. Same result on the three pinned edge cases and on 200k generated inputs against the old regex. TDD: linear-time regression test (1s bound) red at 7579ms, green at ~0ms. Refs #178, Refs #270 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/queereye/specs.ts | 9 ++++++++- packages/core/test/queereye-specs.test.ts | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/core/src/queereye/specs.ts b/packages/core/src/queereye/specs.ts index ad1c789..63e7676 100644 --- a/packages/core/src/queereye/specs.ts +++ b/packages/core/src/queereye/specs.ts @@ -1330,9 +1330,16 @@ export function specFilename(name: string): string { * actual render output: `- \`name\`: description` lines). The phase-02 check * reads these from the file on disk instead of feeding each spec its own * slots (which could never fail); a hand-removed slot line diverges here. + * The section is cut with `indexOf`/`trimEnd`, not a lazy `[\s\S]*?…\s*$` + * scan: that regex was quadratic on a long whitespace run in the file. */ export function renderedSlotsOfSpecMd(markdown: string): string[] { - const section = /## Slots\n([\s\S]*?)(?:\n## |\s*$)/.exec(markdown)?.[1] ?? "" + const header = "## Slots\n" + const start = markdown.indexOf(header) + if (start === -1) return [] + const rest = markdown.slice(start + header.length) + const next = rest.indexOf("\n## ") + const section = next === -1 ? rest.trimEnd() : rest.slice(0, next) return [...section.matchAll(/^-\s+`([^`]+)`: /gm)].map((match) => match[1]!) } diff --git a/packages/core/test/queereye-specs.test.ts b/packages/core/test/queereye-specs.test.ts index 9823fc0..4f0f78d 100644 --- a/packages/core/test/queereye-specs.test.ts +++ b/packages/core/test/queereye-specs.test.ts @@ -252,6 +252,27 @@ describe("renderSpecMd (byte-stable deterministic render)", () => { expect(renderedSlotsOfSpecMd("no slots here")).toEqual([]) }) + test("renderedSlotsOfSpecMd bounds the section at the next heading or the trailing whitespace", () => { + // The section stops at the next `## ` heading: a later list is not a slot. + expect(renderedSlotsOfSpecMd("## Slots\n- `a`: one\n\n## Props\n- `b`: two\n")).toEqual(["a"]) + // No next heading: the section runs to the end, trailing whitespace aside. + expect(renderedSlotsOfSpecMd("# X\n\n## Slots\n- `a`: one\n- `b`: two\n\n \t\n")).toEqual(["a", "b"]) + // The header's own newline is not the next heading's: the section runs on. + expect(renderedSlotsOfSpecMd("## Slots\n## Props\n- `b`: two\n")).toEqual(["b"]) + expect(renderedSlotsOfSpecMd("## Slots\n")).toEqual([]) + }) + + test("renderedSlotsOfSpecMd stays linear on a hostile whitespace run (CodeQL js/polynomial-redos)", () => { + // The file is read back from disk, so its bytes are not the renderer's. + // The old lazy `[\s\S]*?(?:\n## |\s*$)` scan retried `\s*$` from every + // offset of a whitespace run that does not reach the end: quadratic + // (~1.2s at 40k tabs, ~8s here). The fixed code is ~0ms. + const hostile = `## Slots\n- \`a\`: one\n${"\t".repeat(100_000)}x` + const started = performance.now() + expect(renderedSlotsOfSpecMd(hostile)).toEqual(["a"]) + expect(performance.now() - started).toBeLessThan(1000) + }, 60_000) + test("validateAll takes a subset map", () => { const results = validateAll({ button: EXEMPLARS.button }) expect(Object.keys(results)).toEqual(["button"]) From 66c92036fb17a70e58aef105ae9da4954e16bf5f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:46:05 -0400 Subject: [PATCH 72/83] fix(remediation-followup): prove the seat patch matrix on a patch-capable host (Refs #220) #220.2 asked for a `patch` branch in evaluate(). The v2 host has no such permission action: its patch tool asserts `edit` once with every target (add, update, delete, move) as the resources, so the existing edit branch is the allow side. No product change beyond a comment saying so. What was missing is the proof, on both sides. The host offers `patch` only to a model id containing `gpt-`; the testkit's `fake/scripted` never gets the tool, so the patch legs in plugin.test.ts pass on "Tool is not available for this request", not on the policy. The testkit now takes a `model` id, and patch-scope.test.ts runs on `gpt-scripted`: - before(): in-scope patch passes; out-of-scope add/delete/move and control-file targets refuse (seat and user agent) - evaluate(): multi-resource edit is allowed for an in-scope seat, denied with the target's reason otherwise; an unknown `patch` action fails closed - real host: in-scope seat patches land, mixed/move/control-file/read-only patches are refused by the policy text and write nothing Mutation-checked: dropping before()'s patch loop or evaluate()'s deny turns the matching tests red. Refs #178 Co-Authored-By: Claude Opus 5.5 --- packages/opencode/src/policy.ts | 2 + packages/opencode/test/patch-scope.test.ts | 213 +++++++++++++++++++++ packages/testkit/src/host.ts | 11 +- 3 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 packages/opencode/test/patch-scope.test.ts diff --git a/packages/opencode/src/policy.ts b/packages/opencode/src/policy.ts index b55453d..def25da 100644 --- a/packages/opencode/src/policy.ts +++ b/packages/opencode/src/policy.ts @@ -228,6 +228,8 @@ export function createPolicyHooks(runtime: Runtime, seats?: SeatTracker) { }) => { const seat = seatOf(event.agent) const context = await runtime.policy() + // The host's patch tool asks for `edit` too, once, with every target of + // the patch as a resource: there is no `patch` action to branch on (#220). if (event.action === "edit" || event.action === "read") { const decisions = event.resources.map((resource) => event.action === "edit" diff --git a/packages/opencode/test/patch-scope.test.ts b/packages/opencode/test/patch-scope.test.ts new file mode 100644 index 0000000..ec1cef0 --- /dev/null +++ b/packages/opencode/test/patch-scope.test.ts @@ -0,0 +1,213 @@ +// Allowed-path `patch` for seats (#220.2). The v2 host has no `patch` +// permission action: its patch tool asserts `edit` once, with every target of +// the patch (add, update, delete and move) as the resources. The seat matrix +// for a patch is therefore the `edit` branch of the evaluate hook, proven here +// on the hook itself and end to end on the real host. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { createPolicyHooks } from "../src/policy.ts" +import type { Runtime } from "../src/runtime.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const exists = (file: string) => Bun.file(file).exists() + +let root: string +let state: string +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), "es-patch-scope-")) + state = await mkdtemp(path.join(tmpdir(), "es-patch-scope-state-")) +}) +afterAll(async () => { + await rm(root, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) +}) + +const runtime = () => + ({ + root, + factory: { read: async () => undefined }, + policy: async () => ({ root, stateDir: state }), + }) as unknown as Runtime + +const patchText = (...sections: string[]) => ["*** Begin Patch", ...sections, "*** End Patch"].join("\n") + +describe("before: every patch target is checked at the tool call (#220.2)", () => { + const before = (agent: string, text: string) => + createPolicyHooks(runtime()).before({ tool: "patch", agent, id: "1", input: { patchText: text } }) + + test("a patch whose targets are all inside the seat's scope passes", async () => { + await expect( + before("es-manager", patchText("*** Add File: docs/plan.md", "+# plan", "*** Delete File: README.md")), + ).resolves.toBeUndefined() + }) + + test("an out-of-scope add, delete or move target refuses the call", async () => { + for (const section of [ + ["*** Add File: src/escape.ts", "+export {}"], + ["*** Delete File: src/escape.ts"], + ["*** Update File: docs/plan.md", "*** Move to: src/escape.ts", "@@", "-a", "+b"], + ]) + await expect(before("es-manager", patchText("*** Add File: docs/ok.md", "+ok", ...section))).rejects.toThrow( + 'refused "src/escape.ts"', + ) + }) + + test("a control-file target refuses the call for a seat and for the user's own agent", async () => { + for (const agent of ["es-manager", "build"]) + await expect(before(agent, patchText("*** Add File: .factory/approvals/spec.json", "+{}"))).rejects.toThrow( + "is a factory control file", + ) + }) + + test("the user's own agent patches ordinary paths", async () => { + await expect(before("build", patchText("*** Add File: src/app.ts", "+export {}"))).resolves.toBeUndefined() + }) +}) + +describe("evaluate: the edit permission a host patch asks for (#220.2)", () => { + const evaluate = async (agent: string, action: string, resources: readonly string[]) => { + const hooks = createPolicyHooks(runtime()) + // The host's verdict for a seat with a write scope: no `edit` rule, so it asks. + const event: Parameters[0] = { agent, action, resources, effect: "ask" } + await hooks.evaluate(event) + return event + } + + test("every target inside the seat's scope: the seat's patch is allowed, never left asking", async () => { + const event = await evaluate("es-manager", "edit", ["docs/plan.md", "README.md", ".factory/notes/n.md"]) + expect(event.effect).toBe("allow") + expect(event.message).toBeUndefined() + }) + + test("one target outside the scope denies the whole patch, with that target's reason", async () => { + const event = await evaluate("es-manager", "edit", ["docs/plan.md", "src/escape.ts"]) + expect(event.effect).toBe("deny") + expect(event.message).toContain("src/escape.ts") + }) + + test("a control-file target denies the patch for a seat and for the user's own agent", async () => { + for (const agent of ["es-manager", "build"]) { + const event = await evaluate(agent, "edit", ["docs/plan.md", ".factory/gates.json"]) + expect([agent, event.effect]).toEqual([agent, "deny"]) + } + }) + + test("a seat's own file is allowed and its sibling's is not (research alpha/beta)", async () => { + expect((await evaluate("es-research-alpha", "edit", [".factory/research/alpha.md"])).effect).toBe("allow") + expect((await evaluate("es-research-alpha", "edit", [".factory/research/beta.md"])).effect).toBe("deny") + }) + + test("the user's own agent keeps the host's ask on ordinary paths", async () => { + expect((await evaluate("build", "edit", ["docs/plan.md", "src/app.ts"])).effect).toBe("ask") + }) + + test("a `patch` action the host does not emit fails closed for a seat", async () => { + const event = await evaluate("es-manager", "patch", ["docs/plan.md"]) + expect(event.effect).toBe("deny") + expect(event.message).toContain("cannot wait for a human approval") + }) +}) + +describe("a seat's patch on the real host (#220.2)", () => { + // The host offers `patch` only to a model whose id contains `gpt-` (and + // takes `edit`/`write` away from it); the default scripted model never sees + // the tool, so a patch leg there proves the host's refusal, not our policy. + const host = (model?: string) => + boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# demo\n", ".gitignore": "node_modules/\n" }, + ...(model ? { model } : {}), + }) + let h: Harness + beforeAll(async () => { + h = await host("gpt-scripted") + }, 60_000) + afterAll(() => h?.close()) + + const patch = (...sections: string[]) => call("patch", { patchText: patchText(...sections) }) + const outcome = async (agent: string, prompt: string) => { + const { tools } = await h.run(`go ${prompt}`, { agent }) + expect(tools).toHaveLength(1) + return { status: `${tools[0]?.name}:${tools[0]?.status}`, text: tools[0]?.text ?? "" } + } + + test("the manager's patch inside factory-docs lands, with no approval prompt", async () => { + const done = await outcome( + "es-manager", + patch("*** Add File: docs/plan.md", "+# plan", "*** Add File: .factory/notes/n.md", "+note"), + ) + expect(done.status).toBe("patch:completed") + expect(await readFile(path.join(h.directory, "docs/plan.md"), "utf8")).toBe("# plan\n") + expect(await readFile(path.join(h.directory, ".factory/notes/n.md"), "utf8")).toBe("note\n") + }) + + test("a research seat patches its own notes and is refused the dossier", async () => { + const own = await outcome("es-research-alpha", patch("*** Add File: .factory/research/alpha.md", "+# a")) + expect(own.status).toBe("patch:completed") + expect(await readFile(path.join(h.directory, ".factory/research/alpha.md"), "utf8")).toBe("# a\n") + const dossier = await outcome("es-research-alpha", patch("*** Add File: .factory/research/dossier.json", "+{}")) + expect(dossier.status).toBe("patch:error") + expect(dossier.text).toContain("is a factory control file") + expect(await exists(path.join(h.directory, ".factory/research/dossier.json"))).toBe(false) + }) + + test("one out-of-scope target refuses the whole patch: the in-scope file is not written either", async () => { + const mixed = await outcome( + "es-manager", + patch("*** Add File: docs/mixed.md", "+ok", "*** Add File: src/escape.ts", "+export {}"), + ) + expect(mixed.status).toBe("patch:error") + expect(mixed.text).toContain('refused "src/escape.ts"') + expect(await exists(path.join(h.directory, "docs/mixed.md"))).toBe(false) + expect(await exists(path.join(h.directory, "src/escape.ts"))).toBe(false) + }) + + test("a move out of the scope is refused and the source stays put", async () => { + const moved = await outcome( + "es-manager", + patch("*** Update File: README.md", "*** Move to: src/README.md", "@@", "-# demo", "+# moved"), + ) + expect(moved.status).toBe("patch:error") + expect(moved.text).toContain('refused "src/README.md"') + expect(await readFile(path.join(h.directory, "README.md"), "utf8")).toBe("# demo\n") + expect(await exists(path.join(h.directory, "src/README.md"))).toBe(false) + }) + + test("a read-only seat's patch is refused by its seat, not by a missing tool", async () => { + const qa = await outcome("es-qa-functional", patch("*** Add File: docs/qa.md", "+# qa")) + expect(qa.status).toBe("patch:error") + expect(qa.text).toContain("is read-only") + expect(await exists(path.join(h.directory, "docs/qa.md"))).toBe(false) + }) + + test("a control-file patch is refused by the policy for a seat and for the user's own agent", async () => { + for (const agent of ["es-manager", "build"]) { + const denied = await outcome(agent, patch("*** Add File: .factory/waivers/w.json", "+{}")) + expect([agent, denied.status]).toEqual([agent, "patch:error"]) + expect(denied.text).toContain("is a factory control file") + } + expect(await exists(path.join(h.directory, ".factory/waivers/w.json"))).toBe(false) + // The user's own agent still patches ordinary paths: the refusal above is the policy's. + const user = await outcome("build", patch("*** Add File: docs/user.md", "+# user")) + expect(user.status).toBe("patch:completed") + }) + + test("the default scripted model is never offered patch (why these tests name a gpt- model)", async () => { + const scripted = await host() + try { + const { tools } = await scripted.run(`go ${patch("*** Add File: docs/plan.md", "+# plan")}`, { + agent: "es-manager", + }) + expect(tools.map((tool) => `${tool.name}:${tool.status}`)).toEqual(["patch:error"]) + expect(tools[0]?.text).toContain("Tool is not available for this request: patch") + } finally { + await scripted.close() + } + }, 60_000) +}) diff --git a/packages/testkit/src/host.ts b/packages/testkit/src/host.ts index 4ac979c..62c48fd 100644 --- a/packages/testkit/src/host.ts +++ b/packages/testkit/src/host.ts @@ -43,6 +43,12 @@ export interface HarnessOptions { readonly files?: Record /** Initialise a git repository and commit `files` (author "test"). */ readonly git?: boolean + /** + * The fake provider's model id (default `scripted`). The host picks the + * write tools from it: an id containing `gpt-` is offered `patch` and loses + * `edit`/`write`; every other id gets `edit`/`write` and no `patch`. + */ + readonly model?: string } async function git(cwd: string, ...args: string[]) { @@ -104,13 +110,14 @@ export async function boot(options: HarnessOptions = {}): Promise { const llm = startFakeLLM(options.script) const previousConfigHome = process.env.XDG_CONFIG_HOME process.env.XDG_CONFIG_HOME = configDir + const model = options.model ?? "scripted" const config = { - model: "fake/scripted", + model: `fake/${model}`, providers: { fake: { package: "@opencode/ai/providers/openai-compatible", settings: { baseURL: llm.url, apiKey: "test" }, - models: { scripted: { limit: { context: 100000, output: 4000 } } }, + models: { [model]: { limit: { context: 100000, output: 4000 } } }, }, }, plugins: (options.plugins ?? []).map((entry) => From b186820f1ab956717f29ff02be344c0af1729c6b Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:50:49 -0400 Subject: [PATCH 73/83] fix(remediation-followup): proof-gap tests for expired waivers and forged trust (Refs #221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #221.1: an expired but validly-signed waiver planted at gate time waives nothing (no waiver/invalid, secret still fires, waived 0, passed false); a live-expiry control on the identical record waives, pinning the refusal to the expiry. recordWaiver refuses past expiries, so the record is hand-aged under the real human key. #221.3: a forged trust entry is not trust — replayed/made-up/missing signatures and a foreign-key signature all fail isTrusted and surface trust/untrusted at gate time without running the command; the signature's project+kind binding is pinned (moved entry verifies for neither), with a control that the untouched entry in the same file still verifies. Also makes the #227 stop-launch control test assert explicitly (resolves.toBeUndefined), closing Sonar S2699 on PR #290. TDD (test-only, no product change): mutation witnesses — weakening isWaiverActive (&& to ||) turns #221.1 red; trusting any hash-matching entry turns #221.3 red. Targeted suites green after restore. Refs #178, Refs #227 Co-Authored-By: Claude Opus 5.5 --- packages/core/test/gates.test.ts | 111 ++++++++++++++++++++- packages/opencode/test/stop-launch.test.ts | 4 +- 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/packages/core/test/gates.test.ts b/packages/core/test/gates.test.ts index cc058a9..2a3932c 100644 --- a/packages/core/test/gates.test.ts +++ b/packages/core/test/gates.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" -import { mkdir, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" import path from "node:path" import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" import { @@ -12,6 +13,7 @@ import { GatesConfigSchema, gitleaks, installed, + isTrusted, loadGatesConfig, parseOutput, recordWaiver, @@ -19,6 +21,7 @@ import { trustProject, validateArtifacts, } from "../src/index.ts" +import { canonicalPath } from "../src/trust/paths.ts" import { bwrapAvailable, resetBwrapProbe, run } from "../src/util/proc.ts" import { type Fixture, gitRepo } from "./helpers.ts" @@ -145,6 +148,77 @@ describe("trust", () => { expect((await gates({ scope: "full", config: changed })).findings[0]?.rule).toBe("trust/untrusted") }) + test("a forged trust entry is not trust: isTrusted verifies the human signature (#221.3)", async () => { + const cfg = config([{ id: "probe", kind: "test", command: "touch RAN" }]) + const wanted = config([{ id: "probe", kind: "test", command: "touch FORGED" }]) + const { hash } = await commandSetHash(fx.root, cfg.commands) + const forgedHash = (await commandSetHash(fx.root, wanted.commands)).hash + await trust(cfg) + expect(await isTrusted(fx.root, hash, fx.state)).toBe(true) + + const file = path.join(fx.state, "trust.json") + const genuine = await readFile(file, "utf8") + const store = JSON.parse(genuine) as { projects: Record }> } + const project = Object.keys(store.projects)[0]! + const entry = store.projects[project]!.gates + const plant = (projects: Record) => writeFile(file, JSON.stringify({ version: 3, projects })) + + // A second human key: a real Ed25519 signature, but not this machine's. + const foreign = await mkdtemp(path.join(tmpdir(), "es-gates-foreign-")) + await sealHumanKey(PASSPHRASE, foreign) + const foreignSigner = await unlockHumanKey(PASSPHRASE, foreign) + const retargeted = { + hash: forgedHash, + lines: ["probe: touch FORGED"], + approvedBy: "x", + approvedAt: entry.approvedAt, + } + const forgeries: Record> = { + // The genuine signature replayed over another command set. + "replayed signature": { [project]: { gates: { ...entry, hash: forgedHash, lines: retargeted.lines } } }, + "made-up signature": { + [project]: { + gates: { ...retargeted, signature: { alg: "ed25519", keyId: entry.signature.keyId, sig: "AAAA" } }, + }, + }, + "no signature": { [project]: { gates: retargeted } }, + "foreign key": { + [project]: { + gates: { + ...retargeted, + signature: foreignSigner.sign({ trust: 1, project, kind: "gates", ...retargeted }), + }, + }, + }, + } + try { + for (const [name, projects] of Object.entries(forgeries)) { + await plant(projects) + expect([name, await isTrusted(fx.root, forgedHash, fx.state)]).toEqual([name, false]) + const report = await gates({ scope: "full", config: wanted }) + expect([name, report.findings[0]?.rule]).toEqual([name, "trust/untrusted"]) + expect(await Bun.file(path.join(fx.root, "FORGED")).exists()).toBe(false) + } + + // The signature binds the project and the kind: the genuine entry moved + // to another project, or presented as hook trust, verifies for neither. + const other = await gitRepo("es-gates-other-") + try { + const otherProject = canonicalPath(other.root) + expect(project).toBe(canonicalPath(fx.root)) + await plant({ [project]: { gates: entry, hooks: entry }, [otherProject]: { gates: entry } }) + expect(await isTrusted(other.root, hash, fx.state)).toBe(false) + expect(await isTrusted(fx.root, hash, fx.state, "hooks")).toBe(false) + // Control: in the same planted file, the untouched entry still verifies. + expect(await isTrusted(fx.root, hash, fx.state)).toBe(true) + } finally { + await other.cleanup() + } + } finally { + await rm(foreign, { recursive: true, force: true }) + } + }) + test("a package script reachable from a command is part of the trusted hash", async () => { await write("package.json", JSON.stringify({ scripts: { typecheck: "tsc" } })) const cfg = config([{ id: "typecheck", kind: "typecheck", command: "npm run --silent typecheck" }]) @@ -195,6 +269,41 @@ describe("built-in checks", () => { expect(waived.waived).toBeGreaterThan(0) }) + test("an expired signed waiver planted at gate time waives nothing (#221.1)", async () => { + await write("src/config.ts", `export const token = "${FAKE_TOKEN}"\n`) + // recordWaiver refuses a past expiry, so the record is hand-aged: the + // same fields it signs, under the real human key, one day apart. + const plant = async (expiresAt: Date) => { + const unsigned = { + version: 2 as const, + id: "aged-token", + rule: "security/secret-github-token", + files: "src/config.ts", + reason: "fixture value, not a real token", + approvedBy: "tester", + approvedAt: new Date(expiresAt.getTime() - 30 * 86_400_000).toISOString(), + expiresAt: expiresAt.toISOString(), + channel: "cli" as const, + auditHead: null, + } + await write(".factory/waivers/aged-token.json", JSON.stringify({ ...unsigned, signature: signer.sign(unsigned) })) + return gates({ touched: ["src/config.ts"], config: config([]) }) + } + + const expired = await plant(new Date(Date.now() - 86_400_000)) + // The signature verifies, so this is not the forged path: no waiver/invalid. + expect(expired.findings.some((finding) => finding.rule === "waiver/invalid")).toBe(false) + expect(expired.findings.some((finding) => finding.rule === "security/secret-github-token")).toBe(true) + expect(expired.waived).toBe(0) + expect(expired.passed).toBe(false) + + // Control: the identical record with a live expiry does waive, so the + // refusal above is the expiry and nothing else about the planted file. + const live = await plant(new Date(Date.now() + 86_400_000)) + expect(live.findings.some((finding) => finding.rule === "security/secret-github-token")).toBe(false) + expect(live.waived).toBe(1) + }) + test("budgets: behaviour change without a test change, file length, complexity", async () => { await write("src/big.ts", Array.from({ length: 30 }, (_, i) => `export const v${i} = ${i}`).join("\n")) await write( diff --git a/packages/opencode/test/stop-launch.test.ts b/packages/opencode/test/stop-launch.test.ts index b1a1f80..1471155 100644 --- a/packages/opencode/test/stop-launch.test.ts +++ b/packages/opencode/test/stop-launch.test.ts @@ -51,9 +51,7 @@ describe("STOP/HALTED covers user-agent seat launches (#227)", () => { }) test("a user-agent launch is allowed when neither STOP nor HALTED applies", async () => { - const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) - void factory const { before } = createPolicyHooks(runtime("clean")) - await before(launch("build") as never) + await expect(before(launch("build") as never)).resolves.toBeUndefined() }) }) From 025c513c6008f8a3b1482629be8a119770f7085f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:54:07 -0400 Subject: [PATCH 74/83] fix(remediation-followup): per-verb non-TTY pipe tests (Refs #221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #221.2: approve, rebaseline and reseal had dedicated pipe tests; the rest of the confirmHuman funnel did not. New "per-verb pipe refusal" block: trust, waive, factory resume (seeded HALTED, stays halted), factory pr, gates install-git, lsp install typescript (refuses before consent or install), and audit open all refuse with exit 3 — which comes only from the NotInteractive catch, so each assertion proves the terminal gate was reached — and write nothing. es key seal gets the same pipe case (direct NotInteractive throw, no key minted). TDD (test-only, no product change): weakening the gate (throw to return-undefined) turns all 7 new tests red; full cli.test.ts 83/83 green after restore. Refs #178 Co-Authored-By: Claude Opus 5.5 --- packages/cli/test/cli.test.ts | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index aa6f48f..25b6bab 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -330,6 +330,63 @@ describe("human-only confirmation", () => { }) }) +describe("per-verb pipe refusal (#221.2)", () => { + // approve, rebaseline and reseal already have dedicated non-TTY pipe + // tests above. Every other verb in the confirmHuman funnel refuses the + // same way: exit 3 comes only from the NotInteractive catch in main, so + // asserting it proves the test reached the terminal gate — and each test + // also proves nothing was written. Each argv below is shaped to pass + // every check before the confirmation (usage → 2 and early-returns → 0/1 + // would fail the assertion instead of passing vacuously). + test("es trust refuses without a terminal and trusts nothing", async () => { + await mkdir(path.dirname(factoryLayout(root).gates), { recursive: true }) + await writeFile( + factoryLayout(root).gates, + JSON.stringify({ version: 1, commands: [{ id: "probe", kind: "test", command: "touch PROBE" }] }), + ) + expect((await run(["trust"])).code).toBe(3) + expect(await exists(path.join(state, "trust.json"))).toBe(false) + }) + + test("es waive refuses without a terminal and records nothing", async () => { + const result = await run(["waive", "lint/*", "--reason", "legacy code under migration"]) + expect(result.code).toBe(3) + expect(await exists(factoryLayout(root).waivers)).toBe(false) + }) + + test("es factory resume refuses without a terminal and stays halted", async () => { + const halted = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + await halted.begin("human:tester") + await halted.halt("system", "test halt") + expect((await run(["factory", "resume"])).code).toBe(3) + expect((await halted.read())?.stage).toBe("HALTED") + }) + + test("es factory pr refuses without a terminal and records nothing", async () => { + const result = await run(["factory", "pr", "https://github.com/example/repo/pull/1"]) + expect(result.code).toBe(3) + expect(await exists(factoryLayout(root).config)).toBe(false) + }) + + test("es gates install-git refuses without a terminal and installs nothing", async () => { + const result = await run(["gates", "install-git"]) + expect(result.code).toBe(3) + expect(await exists(path.join(root, ".factory", "git-hooks"))).toBe(false) + }) + + test("es lsp install refuses without a terminal before consent or install", async () => { + const result = await run(["lsp", "install", "typescript"]) + expect(result.code).toBe(3) + expect(await exists(path.join(state, "lsp-consent.json"))).toBe(false) + }) + + test("es audit open refuses without a terminal and provisions nothing", async () => { + const result = await run(["audit", "src", "--max-usd", "5", "--open-only"]) + expect(result.code).toBe(3) + expect(await exists(factoryLayout(root).dir)).toBe(false) + }) +}) + describe("es key", () => { test("seal asks twice, refuses short or mismatched passphrases, and status shows the fingerprint", async () => { const { humanKeyId } = await import("@heretek-ai/es-core") @@ -355,6 +412,9 @@ describe("es key", () => { expect((await runIn(["key", "seal"], seq(["short", "short"]))).code).toBe(1) expect((await runIn(["key", "seal"], seq(["long-enough-pass", "different-pass"]))).code).toBe(1) expect(await humanKeyId(fresh)).toBeUndefined() + // A pipe cannot seal either: keySeal throws NotInteractive (exit 3) before reading. + expect((await runIn(["key", "seal"], pipe)).code).toBe(3) + expect(await humanKeyId(fresh)).toBeUndefined() expect((await runIn(["key", "seal"], seq(["a-long-test-passphrase", "a-long-test-passphrase"]))).code).toBe(0) expect(await humanKeyId(fresh)).toBeDefined() expect((await runIn(["key", "status"], seq([]))).out).toContain("Human key") From 01870c583b694a384d68134319e0d79a3f7ed4bb Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 21:58:39 -0400 Subject: [PATCH 75/83] fix(remediation-followup): production MaskedInput render test (Refs #222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #222.1: masked-input.tsx was proven only by the #118 spike prototype and a scripted seam in approve-tui.test.ts. New masked-input.test.ts renders the shipped component headlessly (test/render/masked-input.tsx, same testRender+mockInput harness as panels): type 8 / backspace to 7 / retype to 8 / Enter submits exactly s3cr3t-P; bracketed paste with an embedded newline submits atomically stripped; Esc cancels with no submit. The submit buffer is zeroed (live-ref length 0) and the post-submit frame is bare; 11 captured frames contain no probe secret. TDD (test-only, no product change): echoing plaintext instead of bullets turns the leak audit red. Subprocess exit 0, 70 expects green after restore. #222.2 remainder: the attempts.ts bucket comment already matches (842a8ad); only the ADR I4 normative line still said "per surface" — now per-state-dir with the shared CLI/TUI bucket. #222.3 already holds: ChannelSchema is cli/tui/web and I5 records it; no "terminal" channel prose remains. Refs #178, Refs #118, Refs #119 Co-Authored-By: Claude Opus 5.5 --- docs/adr/0002-approvals-on-every-surface.md | 3 +- packages/opencode/test/masked-input.test.ts | 57 +++++++++ .../opencode/test/render/masked-input.tsx | 120 ++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 packages/opencode/test/masked-input.test.ts create mode 100644 packages/opencode/test/render/masked-input.tsx diff --git a/docs/adr/0002-approvals-on-every-surface.md b/docs/adr/0002-approvals-on-every-surface.md index 58445b6..f39b742 100644 --- a/docs/adr/0002-approvals-on-every-surface.md +++ b/docs/adr/0002-approvals-on-every-surface.md @@ -119,7 +119,8 @@ prompt-injected agent acting through a seat or user-agent shell. (TTL ≤ 120 s) binds the preview to `hashJson(subject)`, and signing re-derives the subject and refuses a mismatch. - **I4** Attempts are limited: at most N wrong passphrases per M minutes per - surface (recommended 5 per 10 min), then a lockout and an audit entry. + state dir (recommended 5 per 10 min; the CLI and the TUI share one bucket), + then a lockout and an audit entry. - **I5** The channel is recorded (`cli`, `tui` or `web`) and audited. - **I6** Mutating approval endpoints are unreachable from agent contexts: no agent tool and no MCP tool; the TUI path requires the TUI process's dialog; diff --git a/packages/opencode/test/masked-input.test.ts b/packages/opencode/test/masked-input.test.ts new file mode 100644 index 0000000..e09a721 --- /dev/null +++ b/packages/opencode/test/masked-input.test.ts @@ -0,0 +1,57 @@ +// Production MaskedInput, rendered for real (#222.1): OpenTUI's headless +// test renderer drives the shipped component from test/render/masked-input.tsx +// (run in a subprocess so the Solid transform preload stays out of the +// real-host tests that share this process). Keystrokes render only as +// bullets, submit delivers the exact code points, the buffer is zeroed, +// and no captured frame contains any probe secret. +import { expect, test } from "bun:test" +import path from "node:path" + +const pkg = path.resolve(import.meta.dir, "..") + +interface Scenario { + readonly frames: string[] + readonly submitted?: string + readonly cancelled: boolean + readonly refLength?: number +} + +test("MaskedInput masks, submits exactly, zeroes and never leaks a frame", async () => { + const proc = Bun.spawn( + ["bun", "--conditions=browser", "--preload", "@opentui/solid/preload", "test/render/masked-input.tsx"], + { cwd: pkg, stdout: "pipe", stderr: "pipe" }, + ) + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect({ code, stderr: code === 0 ? "" : stderr }).toEqual({ code: 0, stderr: "" }) + const { a, b, c } = JSON.parse(stdout.trim().split("\n").at(-1)!) as Record<"a" | "b" | "c", Scenario> + + // A: type 8, backspace to 7, retype to 8, Enter submits exactly s3cr3t-P. + expect(a.frames[1]).toContain("Passphrase: ••••••••") + expect(a.frames[2]).toContain("Passphrase: •••••••") + expect(a.frames[3]).toContain("Passphrase: ••••••••") + expect(a.submitted).toBe("s3cr3t-P") + expect(a.refLength).toBe(0) + // The buffer is cleared on screen too: a bare prompt, no bullets. + expect(a.frames.at(-1)).toContain("Passphrase:") + expect(a.frames.at(-1)).not.toContain("•") + + // B: bracketed paste with an embedded newline submits atomically, stripped. + expect(b.frames[1]).toContain(`Passphrase: ${"•".repeat(20)}`) + expect(b.submitted).toBe("pasted-passphrase-99") + expect(b.refLength).toBe(0) + + // C: typing then Esc cancels with no submit. + expect(c.frames[1]).toContain(`Passphrase: ${"•".repeat(13)}`) + expect(c.cancelled).toBe(true) + expect(c.submitted).toBeUndefined() + expect(c.frames.at(-1)).toContain("Cancelled.") + + // No captured frame contains any probe secret (typed, pasted or cancelled). + const probes = ["s3cr3t-P", "s3cr3t-p", "pasted-passphrase-99", "pasted-pass", "cancel-me-zz9"] + for (const scenario of [a, b, c]) + for (const frame of scenario.frames) for (const probe of probes) expect(frame).not.toContain(probe) +}, 120_000) diff --git a/packages/opencode/test/render/masked-input.tsx b/packages/opencode/test/render/masked-input.tsx new file mode 100644 index 0000000..9c6cadd --- /dev/null +++ b/packages/opencode/test/render/masked-input.tsx @@ -0,0 +1,120 @@ +// Renders the PRODUCTION MaskedInput headlessly with OpenTUI's test +// renderer (run by masked-input.test.ts in a subprocess with the Solid +// transform preloaded) and prints the captured frames plus outcomes as JSON. +// +// Probe secrets must appear in NONE of the frames: keystrokes render only +// as bullets, and the buffer is zeroed on submit/cancel. +// +// Synchronization is deliberately scheduler-free: fixed settles let the +// mock-PTY parser run, frames come from captureCharFrame (current state), +// and submit/cancel are observed by polling the callback flags — the +// renderer's waitFor* helpers break when the loop idles, which is exactly +// the dialog's settled state. +import { testRender } from "@opentui/solid" +import { MaskedInput } from "../../src/masked-input.tsx" + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +const bullets = (n: number) => `Passphrase: ${"•".repeat(n)}` + +async function scenario( + run: (mock: { + typeText: (text: string) => Promise + pressBackspace: () => void + pressEnter: () => void + pressEscape: () => void + pasteBracketedText: (text: string) => Promise + frame: () => string + }) => Promise, +): Promise<{ frames: string[]; submitted?: string; cancelled: boolean; refLength?: number }> { + let submitted: string | undefined + let submittedRef: string[] | undefined + let cancelled = false + let done = false + const setup = await testRender( + () => ( + { + submitted = codepoints.join("") + submittedRef = codepoints + done = true + }} + onCancel={() => { + cancelled = true + done = true + }} + /> + ), + { width: 60, height: 8 }, + ) + const line = () => + setup + .captureCharFrame() + .split("\n") + .find((l) => l.includes("Passphrase:")) ?? "" + try { + await sleep(400) + const frames = [setup.captureCharFrame()] + if (!frames[0]!.includes("Passphrase:")) throw new Error(`mount frame missing prompt: ${line()}`) + const mid = await run({ + typeText: (text) => setup.mockInput.typeText(text), + pressBackspace: () => setup.mockInput.pressBackspace(), + pressEnter: () => setup.mockInput.pressEnter(), + pressEscape: () => setup.mockInput.pressEscape(), + pasteBracketedText: (text) => setup.mockInput.pasteBracketedText(text), + frame: () => setup.captureCharFrame(), + }) + frames.push(...mid) + const deadline = Date.now() + 5_000 + while (!done) { + if (Date.now() > deadline) throw new Error(`settle never fired; last: ${line()}`) + await sleep(50) + } + await sleep(300) + frames.push(setup.captureCharFrame()) + return { frames, submitted, cancelled, refLength: submittedRef?.length } + } finally { + setup.renderer.destroy() + } +} + +const expectLine = (frame: string, want: string, what: string) => { + if (!frame.includes(want)) throw new Error(`${what}: missing ${JSON.stringify(want)} in ${JSON.stringify(frame)}`) + return frame +} + +// A: the spike sequence — type 8, backspace to 7, type to 8, Enter submits. +const a = await scenario(async (mock) => { + await mock.typeText("s3cr3t-p") + await sleep(400) + const f1 = expectLine(mock.frame(), bullets(8), "type-8") + mock.pressBackspace() + await sleep(400) + const f2 = expectLine(mock.frame(), bullets(7), "backspace-7") + await mock.typeText("P") + await sleep(400) + const f3 = expectLine(mock.frame(), bullets(8), "retype-8") + mock.pressEnter() + return [f1, f2, f3] +}) + +// B: bracketed paste with an embedded newline submits atomically, stripped. +const b = await scenario(async (mock) => { + await mock.pasteBracketedText("pasted-pass\nphrase-99") + await sleep(400) + const f1 = expectLine(mock.frame(), bullets(20), "paste-20") + mock.pressEnter() + return [f1] +}) + +// C: typing then Esc cancels with no submit. +const c = await scenario(async (mock) => { + await mock.typeText("cancel-me-zz9") + await sleep(400) + const f1 = expectLine(mock.frame(), bullets(13), "cancel-type-13") + mock.pressEscape() + return [f1] +}) + +process.stdout.write(`${JSON.stringify({ a, b, c })}\n`) +process.exit(0) From f4f502ea37d6c9d6a023305ac36cd456332ddbe5 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 22:02:48 -0400 Subject: [PATCH 76/83] fix(remediation-followup): visibility proof gaps (Refs #248) Six items, test-only, no product change: - STALE_AFTER_MS pinned to 600_000 with the stuck hint starting exactly at the boundary (one ms under stays "in progress"). - Dashboard lead order asserted by index (headline before header). - Non-RESEARCH stages print no research line (researchLine undefined). - status/runs (text, --json, --all) snapshot-pinned read-only over .factory and the state dir; a planted write turns the pin red. - DONE-without-PR headline plus SPEC/RELEASE wording. - Explicit es runs / es watch allow lines in the shell policy (read-only views for build, factory and user agents); otherRunHint window boundary (just inside / exactly at / outside), DONE, equal-timestamp, same-root and stale-local edges. TDD witnesses: planted-write mutation red, full files green after restore (liveness+policy 80/80, cli 85/85). Refs #178 Co-Authored-By: Claude Opus 5.5 --- packages/cli/test/cli.test.ts | 89 ++++++++++++++++++++++++++++- packages/core/test/liveness.test.ts | 44 ++++++++++++++ packages/core/test/policy.test.ts | 6 ++ 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/packages/cli/test/cli.test.ts b/packages/cli/test/cli.test.ts index 25b6bab..332f913 100644 --- a/packages/cli/test/cli.test.ts +++ b/packages/cli/test/cli.test.ts @@ -1,15 +1,17 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test" -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" import { exists, Factory, + type FactoryState, factoryLayout, gateRunner, HUMAN_VERBS, loadPrompt, parseArgs, + type RunIndexEntry, readApproval, recentAuditEntries, researchSourcesDir, @@ -22,6 +24,7 @@ import { type HarnessDriver, type HeadlessEvent, presentEvent, runHeadless } fro import { openProposalsPr } from "../src/improve.ts" import { commandHelp, main } from "../src/main.ts" import { createMcpServer } from "../src/mcp.ts" +import { otherRunHint, runsCommand, statusCommand } from "../src/status.ts" import { type ConfirmIO, confirmHuman, NotInteractive } from "../src/tty.ts" import { VERSION } from "../src/version.ts" @@ -387,6 +390,90 @@ describe("per-verb pipe refusal (#221.2)", () => { }) }) +describe("status and runs are read-only; other-run hints (#248)", () => { + // File list + sizes + content hashes under .factory and the state dir. + // Directory mtimes are excluded on purpose: the state lock legitimately + // mkdirs+removes its lock dir on reads, churning the parent mtime. + const snapshot = async () => { + const entries: string[] = [] + for (const [label, dir] of [ + ["root", path.join(root, ".factory")], + ["state", state], + ] as const) { + const walk = async (rel: string): Promise => { + const names = await readdir(path.join(dir, rel)).catch(() => [] as string[]) + for (const name of names) { + const child = rel ? `${rel}/${name}` : name + const info = await stat(path.join(dir, child)) + if (info.isDirectory()) await walk(child) + else if (info.isFile()) + entries.push(`${label}/${child}:${info.size}:${Bun.hash(await readFile(path.join(dir, child)))}`) + } + } + await walk("") + } + return entries.sort() + } + + test("status and runs (text, --json, --all) write nothing", async () => { + const factory = new Factory(root, { gates: gateRunner({ stateDir: state }), stateDir: state }) + const begun = await factory.begin("human:tester") + const printed: string[] = [] + const context = { root, stateDir: state, print: (text: string) => printed.push(text) } + const before = await snapshot() + expect(await statusCommand(context, { positionals: ["status"], flags: {} })).toBe(0) + expect(await statusCommand(context, { positionals: ["status"], flags: { json: true } })).toBe(0) + expect(await runsCommand(context, { positionals: ["runs"], flags: {} })).toBe(0) + expect(await runsCommand(context, { positionals: ["runs"], flags: { all: true, json: true } })).toBe(0) + // The commands really ran (not an early return): the GRILL headline and + // the begun run are in the output. + expect(printed.join("\n")).toContain("grill") + expect(printed.join("\n")).toContain(begun.runId) + expect(await snapshot()).toEqual(before) + }) + + test("otherRunHint: window boundary, DONE and equal-timestamp edges", async () => { + const NOW = new Date("2026-10-08T02:10:00.000Z") + const iso = (ms: number) => new Date(NOW.getTime() + ms).toISOString() + const entry = (overrides: Partial = {}): RunIndexEntry => ({ + runId: "run-other", + root: "/elsewhere", + stage: "BUILD", + createdAt: iso(-3_600_000), + updatedAt: iso(-60_000), + ...overrides, + }) + const here = (updatedAt: string): FactoryState => ({ + version: 2, + runId: "run-here", + stage: "BUILD", + createdAt: iso(-3_600_000), + updatedAt, + spend: { usd: 1, estimated: true, events: 1 }, + phases: [], + audits: [], + }) + const hint = (runs: RunIndexEntry[], updatedAt: string | undefined) => + otherRunHint(runs, { root, state: updatedAt === undefined ? undefined : here(updatedAt) }, NOW) + + // A more recently active run elsewhere is named; without a local run the + // copy points at it directly. + expect(hint([entry()], iso(-120_000))).toContain("run-other") + expect(hint([entry()], undefined)).toContain("A run is active elsewhere") + // The window is ten minutes: just inside counts, exactly at it and + // outside do not (OTHER_RUN_WINDOW_MS is pinned behaviorally here). + expect(hint([entry({ updatedAt: iso(-599_000) })], iso(-3_600_000))).toContain("run-other") + expect(hint([entry({ updatedAt: iso(-600_000) })], iso(-3_600_000))).toBeUndefined() + expect(hint([entry({ updatedAt: iso(-601_000) })], iso(-3_600_000))).toBeUndefined() + // DONE runs never hint, however fresh; an equally-fresh run is not + // "more recently active"; our own root is skipped; a stale local run is. + expect(hint([entry({ stage: "DONE" })], iso(-120_000))).toBeUndefined() + expect(hint([entry({ updatedAt: iso(-120_000) })], iso(-120_000))).toBeUndefined() + expect(hint([entry({ root })], iso(-120_000))).toBeUndefined() + expect(hint([entry()], iso(-30_000))).toBeUndefined() + }) +}) + describe("es key", () => { test("seal asks twice, refuses short or mismatched passphrases, and status shows the fingerprint", async () => { const { humanKeyId } = await import("@heretek-ai/es-core") diff --git a/packages/core/test/liveness.test.ts b/packages/core/test/liveness.test.ts index 7a99ba7..6a7701e 100644 --- a/packages/core/test/liveness.test.ts +++ b/packages/core/test/liveness.test.ts @@ -9,6 +9,7 @@ import { livenessLines, progressPrint, readLiveness, + researchLine, STALE_AFTER_MS, } from "../src/factory/liveness.ts" import { Factory } from "../src/factory/machine.ts" @@ -180,6 +181,31 @@ describe("headline", () => { const qa = state({ stage: "QA", activePhase: "p1" }) expect(headline(qa, await live(qa, T0))).toStartWith("QA on p1 in progress") }) + + test("STALE_AFTER_MS is ten minutes; the stuck hint starts exactly at the boundary (#248)", async () => { + expect(STALE_AFTER_MS).toBe(600_000) + const current = state() + const atBoundary = new Date(T0.getTime() + STALE_AFTER_MS) + expect(headline(current, await live(current, atBoundary))).toContain("No activity for 10m in RESEARCH") + const justUnder = new Date(T0.getTime() + STALE_AFTER_MS - 1_000) + expect(headline(current, await live(current, justUnder))).toBe( + "Research in progress: no seat is running; last activity 9m ago.", + ) + }) + + test("done without a PR, SPEC and RELEASE wording (#248)", async () => { + const done = state({ stage: "DONE" }) + expect(headline(done, await live(done, T0))).toBe("Done: the run is complete.") + const spec = state({ stage: "SPEC", runId: "run-20261008-020000-spec1" }) + await updateSeats(fx.root, spec.runId, (seats) => { + seats.push(seat({ lastActivityAt: T0.toISOString() })) + }) + expect(headline(spec, await live(spec, T0))).toBe("Spec writing is running: es-research-alpha started 0s ago.") + const release = state({ stage: "RELEASE", runId: "run-20261008-020000-rel1", updatedAt: T0.toISOString() }) + expect(headline(release, await live(release, T0))).toBe( + "Release in progress: no seat is running; last activity 0s ago.", + ) + }) }) describe("formatHuman", () => { @@ -210,6 +236,24 @@ describe("formatHuman", () => { expect(text).toContain("es-research-alpha running es_research_fetch 10s ago ses_ee6b70e18ffe") expect(text).toContain("es-research-beta completed 50s ago ses_beta") expect(text).toContain("research: 0 sources · alpha.md — · beta.md — · REPORT.md — · coverage not checked") + // The dashboard leads with the headline, then the run header (#248): + // order, not just containment. + expect(text.indexOf("Research is running: es-research-alpha")).toBeLessThan( + text.indexOf(`run run-20261008-020000-ab12 · RESEARCH · ${fx.root}`), + ) + }) + + test("non-RESEARCH stages print no research line (#248)", async () => { + const current = state({ stage: "BUILD", activePhase: "p1", updatedAt: at(20).toISOString() }) + const live = await readLiveness(fx.root, current, { now: at(60) }) + expect(researchLine(live)).toBeUndefined() + const text = formatHuman( + current, + live, + "\nrun run-20261008-020000-ab12 · stage BUILD\nbuild phase p1\n", + ) + expect(text).toContain("Building p1 in progress") + expect(text).not.toContain("research:") }) }) diff --git a/packages/core/test/policy.test.ts b/packages/core/test/policy.test.ts index c903d1b..2766d23 100644 --- a/packages/core/test/policy.test.ts +++ b/packages/core/test/policy.test.ts @@ -331,6 +331,12 @@ describe("shell policy", () => { expect(shell("build", "es factory stop").effect).toBe("allow") }) + test("es runs and es watch are explicitly allowed: read-only views, not absence-of-rule (#248)", () => { + for (const command of ["es runs", "es runs --all", "es watch", "es watch --interval 5"]) + for (const agent of ["build", "factory", undefined]) + expect([agent, command, shell(agent, command).effect]).toEqual([agent, command, "allow"]) + }) + test("a flag or `--` before the verb still reads as that verb, as the CLI parser does (#88)", () => { for (const command of [ "es --cwd . approve spec", From f964832bf9297f620c01a251c51fe2cde89d7be2 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 22:04:16 -0400 Subject: [PATCH 77/83] fix(remediation-followup): brainstorm nested-refusal and default-run pins (Refs #269) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #269.1: lens (es-lens-inversion) and critic seats cannot launch nested subagents — same delegate call the grill completes at depth 1 errors here with the host's tool-absent denial (empty spawns compiled to wildcard-deny; the tool is never offered). The grill-vs-lens pair in the same file is the live discriminator: subagent:completed for the grill, subagent:error for lens/critic. #269.2: the grill run's shortlist lands under runs/grill-fanout/ while the human default run is re-read and asserted unchanged (the brainstormer's b001/17 shortlist). Note: the issue asks for an "undefined" default check, but this file boots one shared harness and an earlier test completes default — so "untouched" is pinned as unchanged, with the reason in the comment. TDD (test-only, no product change): brainstorm.test.ts 5/5 green. Refs #178, Refs #108 Co-Authored-By: Claude Opus 5.5 --- packages/opencode/test/brainstorm.test.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/opencode/test/brainstorm.test.ts b/packages/opencode/test/brainstorm.test.ts index be763aa..48f927c 100644 --- a/packages/opencode/test/brainstorm.test.ts +++ b/packages/opencode/test/brainstorm.test.ts @@ -112,6 +112,18 @@ describe("brainstorm on the real host", () => { expect(critic.tools[0]?.status).toBe("error") }) + test("a lens seat and the critic cannot launch nested subagents (depth-1 refusal, #269)", async () => { + // Callable brainstorms run lenses at depth 1; nothing runs at depth 2: + // lens and critic seats have empty spawns, so the seat policy refuses + // the launch before anything starts. + for (const agent of ["es-lens-inversion", "es-brainstorm-critic"]) { + const { tools } = await h.run(`go ${delegate("es-lens-scamper", "Nested diverge.")}`, { agent }) + expect([agent, ...tools.map((tool) => `${tool.name}:${tool.status}`)]).toEqual([agent, "subagent:error"]) + // Empty spawns: the host never offers these seats the tool at all. + expect(tools[0]?.text).toContain('No tool named "subagent"') + } + }) + test("#108: the grill runs a 2-lens brainstorm at depth 1 and gets the shortlist JSON", async () => { const run = "grill-fanout" const planned = await h.run( @@ -172,7 +184,13 @@ describe("brainstorm on the real host", () => { const parsed = JSON.parse(json!) as { run: string; shortlist: Array<{ id: string; outlier: boolean }> } expect(parsed.run).toBe(run) expect(parsed.shortlist.map((entry) => entry.id).sort()).toEqual(["b001", "b002"]) - // The human's default run is untouched by the grill's run. + // The human's default run is untouched by the grill's run: the grill + // wrote run "grill-fanout", while default still holds the brainstormer's + // shortlist from the earlier test in this file (one shared harness, so + // "untouched" means unchanged, not absent). expect(await readResult(h.directory, run)).toBeDefined() + const untouched = await readResult(h.directory) + expect(untouched?.shortlist).toHaveLength(1) + expect(untouched?.ranked[0]).toEqual({ id: "b001", total: 17 }) }) }) From db6ccbc27ba8e70361700d26efcd72ef39141995 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 22:08:30 -0400 Subject: [PATCH 78/83] fix(remediation-followup): require SourceCache stateDir, close legacy path (Refs #251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #251 ctor: the stateDir parameter is required and the unsealed branches are gone from seal/sealed/writeIndex/readIndex. Production already always sealed (researchCache defaults to the global state dir; the src-wide misuse-guard test keeps it that way), so no product call site changes — tsc green in all 6 packages. Every legacy test construction is reworked so no test exercises the removed path (grep-verified zero one-arg or explicit-undefined sites): - research.test.ts: shared cache takes state; the forged-plant leg hand-writes bytes (as put() lays them out minus the seal) and gains an index-cannot-launder pin; the legacy-reader assertion is gone. - source-seal.test.ts H3: the pre-sealing dossier is laid out directly with its legacy ok-witness (writeDossier always verifies now and could never produce one); exportBrief still refuses it. - scout.test.ts, render.test.ts, fires-research.test.ts: pass state (fires-research strips the seal when planting, premise unchanged). - evidence-seal.test.ts: the unsealed-writes-no-sidecar leg is dropped with the mode it tested. - fleet evidence.test.ts rig: the "unsealed" explorer case is hand-written bytes; seal-status assertions unchanged. TDD witnesses: engine-seal/source-seal/evidence suites green (research+seal+render+scout+fleet 70/70, fires-research 6/6); the misuse-guard and full check suites re-verify at close-out. #251.1 (verifier fuzz corpus in CI) is out of scope here — it needs CI work. Refs #178, Refs #52 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/research/cache.ts | 13 +++----- packages/core/test/evidence-seal.test.ts | 5 +-- packages/core/test/render.test.ts | 6 ++-- packages/core/test/research.test.ts | 32 ++++++++++--------- packages/core/test/scout.test.ts | 2 +- packages/core/test/source-seal.test.ts | 20 ++++++------ packages/fleet/test/evidence.test.ts | 19 +++++++---- packages/opencode/test/fires-research.test.ts | 2 +- 8 files changed, 52 insertions(+), 47 deletions(-) diff --git a/packages/core/src/research/cache.ts b/packages/core/src/research/cache.ts index fb838d2..059cee8 100644 --- a/packages/core/src/research/cache.ts +++ b/packages/core/src/research/cache.ts @@ -49,23 +49,22 @@ const MAX_SOURCE_BYTES = 400_000 export class SourceCache { /** - * When `stateDir` is set, entries are sealed with the engine key on write - * (#52) and unsealed entries are refused on read. Without it the cache keeps - * its legacy behaviour (tests and auxiliary readers). + * Entries are sealed with the engine key on write (#52) and unsealed + * entries are refused on read. `stateDir` is required: there is no + * legacy unsealed mode (#251) — tests that need untrusted bytes write + * them to disk directly instead of constructing a cache without a key. */ constructor( readonly dir: string, - readonly stateDir?: string, + readonly stateDir: string, ) {} private async seal(meta: SourceMeta): Promise { - if (!this.stateDir) return meta const key = await ensureEngineKey(this.stateDir) return { ...meta, seal: sealMeta(meta as unknown as Record, key) } } private async sealed(meta: SourceMeta): Promise { - if (!this.stateDir) return true // Never mint a key to verify: without the engine's key nothing verifies (#59). const key = await readEngineKey(this.stateDir) return key !== undefined && verifyMetaSeal(meta as unknown as Record, key) @@ -84,14 +83,12 @@ export class SourceCache { */ private async writeIndex(index: Record): Promise { await atomicWrite(this.indexFile(), `${JSON.stringify(index, null, 2)}\n`) - if (!this.stateDir) return const key = await ensureEngineKey(this.stateDir) await atomicWrite(this.indexSealFile(), `${sealMeta(index as unknown as Record, key)}\n`) } private async readIndex(): Promise> { const index = (await readJson>(this.indexFile())) ?? {} - if (!this.stateDir) return index const seal = (await readFile(this.indexSealFile(), "utf8").catch(() => undefined))?.trim() if (seal === undefined || seal === "") return index const key = await readEngineKey(this.stateDir) diff --git a/packages/core/test/evidence-seal.test.ts b/packages/core/test/evidence-seal.test.ts index 29bc13b..44f969b 100644 --- a/packages/core/test/evidence-seal.test.ts +++ b/packages/core/test/evidence-seal.test.ts @@ -24,7 +24,7 @@ const put2 = async (cache: SourceCache) => { } describe("sealed URL index (#251.2)", () => { - test("a sealed cache writes an HMAC sidecar next to index.json; an unsealed one does not", async () => { + test("a sealed cache writes an HMAC sidecar next to index.json", async () => { const dir = await mkdtemp(path.join(tmpdir(), "es-index-")) const state = await mkdtemp(path.join(tmpdir(), "es-index-state-")) try { @@ -32,9 +32,6 @@ describe("sealed URL index (#251.2)", () => { await put2(sealed) const seal = await Bun.file(path.join(dir, "sealed", "index.seal")).text() expect(seal.trim()).toMatch(/^[0-9a-f]{64}$/) - const plain = new SourceCache(path.join(dir, "plain")) - await put2(plain) - expect(await Bun.file(path.join(dir, "plain", "index.seal")).exists()).toBe(false) } finally { await rm(dir, { recursive: true, force: true }) await rm(state, { recursive: true, force: true }) diff --git a/packages/core/test/render.test.ts b/packages/core/test/render.test.ts index 1cf7923..4bbebf9 100644 --- a/packages/core/test/render.test.ts +++ b/packages/core/test/render.test.ts @@ -166,11 +166,12 @@ describe("xss corpus", () => { describe("renderResearchRun", () => { test("reads the run (report, dossier, retractions, cache) in either mode", async () => { const dir = await mkdtemp(path.join(tmpdir(), "es-render-")) + const state = await mkdtemp(path.join(tmpdir(), "es-render-state-")) try { const { SourceCache, researchSourcesDir } = await import("../src/research/cache.ts") const { buildDossier, writeDossier } = await import("../src/claims/dossier.ts") const { normalizeClaim } = await import("../src/claims/claim.ts") - const cache = new SourceCache(researchSourcesDir(dir), undefined) + const cache = new SourceCache(researchSourcesDir(dir), state) const source = await cache.put({ url: "https://example.test/q", text: "Queues decouple workers from producers durably.", @@ -197,10 +198,11 @@ describe("renderResearchRun", () => { expect(html).toContain("Queues decouple workers.") // A source pruned after the dossier was written is shown, never hidden. const { SourceCache: Cache, researchSourcesDir: sourcesDir } = await import("../src/research/cache.ts") - await new Cache(sourcesDir(dir), undefined).prune(new Set()) + await new Cache(sourcesDir(dir), state).prune(new Set()) expect(await renderResearchRun(dir, { format: "md" })).toContain("not in the cache") } finally { await rm(dir, { recursive: true, force: true }) + await rm(state, { recursive: true, force: true }) } }) diff --git a/packages/core/test/research.test.ts b/packages/core/test/research.test.ts index 8fb4842..eeab990 100644 --- a/packages/core/test/research.test.ts +++ b/packages/core/test/research.test.ts @@ -15,6 +15,7 @@ import { isFresh, locateQuote, normalizeForQuote, + normalizeSourceText, normalizeWithMap, parseTags, pruneClaims, @@ -24,6 +25,7 @@ import { SourceCache, searxngProvider, selectProvider, + sha256, ttlDays, verifyQuote, } from "../src/index.ts" @@ -33,8 +35,8 @@ let cache: SourceCache let state: string beforeEach(async () => { dir = await mkdtemp(path.join(tmpdir(), "es-research-")) - cache = new SourceCache(path.join(dir, "sources")) state = await mkdtemp(path.join(tmpdir(), "es-research-state-")) + cache = new SourceCache(path.join(dir, "sources"), state) }) afterEach(() => Promise.all([rm(dir, { recursive: true, force: true }), rm(state, { recursive: true, force: true })])) @@ -149,25 +151,25 @@ describe("source cache", () => { expect((await sealed.get(entry.meta.sha256))?.text).toBe(SOURCE) expect((await sealed.byUrl("https://s.test"))?.meta.sha256).toBe(entry.meta.sha256) // A forged entry planted before the factory dir existed has no seal. + // No constructor can produce unsealed bytes anymore (#251), so the + // fixture is written to disk directly, laid out exactly as put() would + // leave it minus the seal. const planted = new SourceCache(path.join(dir, "planted"), state) - const forged = await new SourceCache(path.join(dir, "forged")).put({ - url: "https://evil.test", - text: SOURCE, - provider: "fetch", - }) const { atomicWrite } = await import("../src/util/fs.ts") - const { sha256 } = await import("../src/util/hash.ts") - const { readFile } = await import("node:fs/promises") - await atomicWrite(path.join(dir, "planted", `${forged.meta.sha256}.md`), await readFile(forged.path, "utf8")) + const normal = normalizeSourceText(SOURCE) + const forgedHash = sha256(normal) + await atomicWrite(path.join(dir, "planted", `${forgedHash}.md`), normal) await atomicWrite( - path.join(dir, "planted", `${forged.meta.sha256}.json`), - `${JSON.stringify({ ...forged.meta, sha256: forged.meta.sha256 })}\n`, + path.join(dir, "planted", `${forgedHash}.json`), + `${JSON.stringify({ sha256: forgedHash, url: "https://evil.test", retrieved: new Date().toISOString(), provider: "fetch", bytes: Buffer.byteLength(normal) })}\n`, ) - expect(sha256(SOURCE)).toBe(forged.meta.sha256) - expect(await planted.get(forged.meta.sha256)).toBeUndefined() + expect(sha256(SOURCE)).toBe(forgedHash) + expect(await planted.get(forgedHash)).toBeUndefined() + expect(await planted.byUrl("https://evil.test")).toBeUndefined() + // Even an index entry pointing at the forged bytes cannot launder them: + // byUrl resolves the mapping, then get() still refuses the bytes. + await writeFile(path.join(dir, "planted", "index.json"), JSON.stringify({ "https://evil.test": forgedHash })) expect(await planted.byUrl("https://evil.test")).toBeUndefined() - // Legacy readers without a state dir keep the old behaviour. - expect((await new SourceCache(path.join(dir, "forged")).get(forged.meta.sha256))?.text).toBe(SOURCE) }) }) diff --git a/packages/core/test/scout.test.ts b/packages/core/test/scout.test.ts index 0dd340d..b17b971 100644 --- a/packages/core/test/scout.test.ts +++ b/packages/core/test/scout.test.ts @@ -198,7 +198,7 @@ describe("the scout tools", () => { ) expect(osvCalls).toEqual([{ package: { name: "mit-lib", ecosystem: "npm" }, version: "1.0.0" }]) const hash = /sha256:([0-9a-f]{64})/.exec(advisories)![1]! - expect((await new SourceCache(researchSourcesDir(root)).get(hash))?.meta.provider).toBe("osv") + expect((await new SourceCache(researchSourcesDir(root), state).get(hash))?.meta.provider).toBe("osv") for (const name of NAMES) { await tool("es_scout_scan").execute({ candidate: name }, scout) // mit carries real advisories; the rest record an explicit skip (#257). diff --git a/packages/core/test/source-seal.test.ts b/packages/core/test/source-seal.test.ts index 944d7ce..8e8846c 100644 --- a/packages/core/test/source-seal.test.ts +++ b/packages/core/test/source-seal.test.ts @@ -9,7 +9,7 @@ import { workspacePackages } from "../../../scripts/packages.ts" import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" import { exportBrief } from "../src/claims/brief.ts" import { normalizeClaim } from "../src/claims/claim.ts" -import { buildDossier, writeDossier } from "../src/claims/dossier.ts" +import { buildDossier } from "../src/claims/dossier.ts" import { factoryLayout } from "../src/layout.ts" import { normalizeSourceText, researchSourcesDir, SourceCache } from "../src/research/cache.ts" import { scoutTools } from "../src/scout/ops.ts" @@ -109,7 +109,6 @@ describe("H3: brief export refuses planted entries", () => { test("exportBrief re-witnesses against the sealed cache: planted quotes fail", async () => { const text = "Independent benchmark reached 9.5 GB/s throughput on reference hardware." const hash = await plantUnsealed("https://evil.test/bench", text) - const legacy = new SourceCache(researchSourcesDir(root)) const claim = normalizeClaim({ tag: "VERIFIED", statement: "The benchmark is reproducible.", @@ -117,14 +116,15 @@ describe("H3: brief export refuses planted entries", () => { }) await mkdir(factoryLayout(root).research, { recursive: true }) await writeFile(factoryLayout(root).researchReport, "# R\n") - // A legacy dossier (written before sealing) witnesses fine unsealed. - await writeDossier( - factoryLayout(root).researchDossier, - buildDossier({ mode: "research", subject: "s", claims: [claim] }), - { - cache: legacy, - }, - ) + // A dossier as written before sealing (#251): the planted bytes + // witnessed fine unsealed, so the file carries an ok witness. + // writeDossier always verifies now and could never produce one, so the + // legacy artifact is laid out directly in its on-disk shape. + const { writeJson } = await import("../src/util/fs.ts") + await writeJson(factoryLayout(root).researchDossier, { + ...buildDossier({ mode: "research", subject: "s", claims: [claim] }), + claims: [{ ...claim, witness: { ok: true, checkedAt: "2026-01-01T00:00:00.000Z" } }], + }) const { brief } = await exportBrief(root, { signer, stateDir: state }) expect(brief.claims[0]!.witness.ok).toBe(false) expect(brief.sources[hash]).toBeUndefined() diff --git a/packages/fleet/test/evidence.test.ts b/packages/fleet/test/evidence.test.ts index b85eca4..22aba10 100644 --- a/packages/fleet/test/evidence.test.ts +++ b/packages/fleet/test/evidence.test.ts @@ -11,7 +11,9 @@ import { factoryLayout, normalizeClaim, normalizeForQuote, + normalizeSourceText, SourceCache, + sha256, writeJson, } from "@heretek-ai/es-core" import { @@ -35,12 +37,17 @@ async function rig(withRetraction = false) { await mkdir(sources, { recursive: true }) const sealed = new SourceCache(sources, state) const entry = await sealed.put({ url: "https://example.com/bun", text: SOURCE_TEXT, provider: "fetch" }) - const plain = new SourceCache(sources) - const unsealed = await plain.put({ - url: "https://example.com/plain", - text: `Plain source text. ${QUOTE}.`, - provider: "fetch", - }) + // No constructor can produce unsealed entries anymore (#251): the + // unsealed fixture is written to disk directly, laid out as put() would + // leave it minus the seal, so the explorer still has an "unsealed" case. + const plainText = normalizeSourceText(`Plain source text. ${QUOTE}.`) + const plainHash = sha256(plainText) + await writeFile(path.join(sources, `${plainHash}.md`), plainText) + await writeFile( + path.join(sources, `${plainHash}.json`), + `${JSON.stringify({ sha256: plainHash, url: "https://example.com/plain", retrieved: new Date().toISOString(), provider: "fetch", bytes: Buffer.byteLength(plainText) })}\n`, + ) + const unsealed = { meta: { sha256: plainHash } } const claim = normalizeClaim({ tag: "VERIFIED", statement: "Bun starts in about twelve milliseconds.", diff --git a/packages/opencode/test/fires-research.test.ts b/packages/opencode/test/fires-research.test.ts index 6c416f0..b585eec 100644 --- a/packages/opencode/test/fires-research.test.ts +++ b/packages/opencode/test/fires-research.test.ts @@ -248,7 +248,7 @@ describe("research-fires", () => { try { const cache = researchCache(h.directory, state) const text = "Planted findings about gate latency." - const planted = await new SourceCache(`${h.directory}.planted`, undefined).put({ + const planted = await new SourceCache(`${h.directory}.planted`, state).put({ url: "https://evil.test/planted", text, provider: "fetch", From c79812c03af5856f9d42cfaa5970ecbca957df96 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 22:10:01 -0400 Subject: [PATCH 79/83] fix(remediation-followup): pin the LSP fetch/install mitigation stack (Refs #275) #275.2 takes the "downgrade the claim" fork deliberately: sandboxing npm through bwrap is unsound here (the fetch needs the network; bwrap is not universally available, so a "preferred" sandbox would silently no-op). The header now states the actual stack, and a fake-npm test pins it: exact name@version fetch, SRI verification, install with --no-save/--ignore-scripts/--no-audit/--no-fund, a scrubbed child env (canary absent, registry override present), and pack/install confined to scratch/target cwds. No product change beyond the header comment. TDD: dropping --ignore-scripts turns the new pin red; lsp.test.ts 7/7 green after restore. Refs #178 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/lsp/install.ts | 5 +- packages/core/test/lsp.test.ts | 90 +++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/core/src/lsp/install.ts b/packages/core/src/lsp/install.ts index e48bb3a..5a19933 100644 --- a/packages/core/src/lsp/install.ts +++ b/packages/core/src/lsp/install.ts @@ -1,7 +1,10 @@ // Pinned, checksummed, ask-once installs of language servers. A human // consents once per server (CLI or TUI); the installer fetches the exact npm // tarball, verifies its sha512 against the pinned integrity, and installs it -// with scripts disabled into the user-global state dir. +// with scripts disabled into the user-global state dir. Installs are not +// bwrap-sandboxed (the fetch needs the network): consent + SRI pins + +// --ignore-scripts + a scrubbed environment is the whole mitigation stack, +// pinned by the installer tests (#275). import { createHash } from "node:crypto" import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises" import { tmpdir, userInfo } from "node:os" diff --git a/packages/core/test/lsp.test.ts b/packages/core/test/lsp.test.ts index 4441be9..4f64d95 100644 --- a/packages/core/test/lsp.test.ts +++ b/packages/core/test/lsp.test.ts @@ -1,7 +1,7 @@ // LSP runtime against real language servers (native TypeScript 7 `tsc --lsp`, // gopls) plus config resolution and the pinned installer. import { afterAll, beforeAll, describe, expect, test } from "bun:test" -import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" +import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" import { @@ -138,4 +138,92 @@ describe("pinned installer", () => { expect(await consent("fake-ls", state)).toBe(true) await expect(installServer(server, { stateDir: state, npm: fakeNpm })).rejects.toThrow("integrity mismatch") }) + + test("the fetch/install mitigation stack is pinned: SRI, scripts-disabled, scrubbed env, confined cwd (#275.2)", async () => { + // Installs are not bwrap-sandboxed (the fetch needs the network), so the + // narrower stack — consent, exact pins, --ignore-scripts, scrubbed env, + // scratch/target confinement — is pinned here instead of assumed. + const genuine = new TextEncoder().encode("genuine-tarball-bytes") + const fakeNpm = path.join(state, "npm-pin") + await writeFile( + fakeNpm, + [ + "#!/bin/sh", + 'here=$(dirname "$0")', + 'if [ "$1" = pack ]; then', + ' printf "%s\\n" "$@" > "$here/pack-args.log"', + ' printf "PWD=%s\\n" "$PWD" >> "$here/pack-args.log"', + ' env | sort > "$here/pack.env"', + " dest=$4", + ' printf "genuine-tarball-bytes" > "$dest/pkg-1.0.0.tgz"', + ' echo \'[{"filename":"pkg-1.0.0.tgz"}]\'', + " exit 0", + "fi", + 'if [ "$1" = install ]; then', + ' printf "%s\\n" "$@" > "$here/install-args.log"', + ' printf "PWD=%s\\n" "$PWD" >> "$here/install-args.log"', + ' env | sort > "$here/install.env"', + ' prefix=""', + ' prev=""', + ' for arg in "$@"; do', + ' if [ "$prev" = "--prefix" ]; then prefix=$arg; fi', + " prev=$arg", + " done", + ' mkdir -p "$prefix/node_modules/.bin"', + ' printf "#!/bin/sh\\nexit 0\\n" > "$prefix/node_modules/.bin/fake-pin"', + ' chmod +x "$prefix/node_modules/.bin/fake-pin"', + " exit 0", + "fi", + "exit 1", + "", + ].join("\n"), + ) + await chmod(fakeNpm, 0o755) + const server: ServerSpec = { + id: "fake-pin", + command: ["fake-pin"], + extensions: [".fake"], + rootMarkers: [], + install: { + kind: "npm", + packages: [{ name: "pkg", version: "1.0.0", integrity: sri(genuine) }], + bin: "fake-pin", + }, + source: "builtin", + } + await recordConsent("fake-pin", true, state) + // A canary the scrubbed env must not leak; a registry override must pass through. + process.env.ES_LSP_PIN_EVIL = "leak-me" + process.env.NPM_CONFIG_REGISTRY = "https://registry.test/" + let bin: string + try { + bin = await installServer(server, { stateDir: state, npm: fakeNpm }) + } finally { + delete process.env.ES_LSP_PIN_EVIL + delete process.env.NPM_CONFIG_REGISTRY + } + expect(bin).toContain(path.join("lsp", "fake-pin-1.0.0")) + const read = (name: string) => readFile(path.join(state, name), "utf8") + + const packArgs = await read("pack-args.log") + expect(packArgs).toContain("pkg@1.0.0") + // pack runs confined to a scratch dir, not the project root. + const packPwd = /PWD=(.*)/.exec(packArgs)![1]! + expect(path.basename(packPwd)).toMatch(/^es-lsp-/) + expect(packPwd).not.toBe(root) + const packEnv = await read("pack.env") + expect(packEnv).not.toContain("ES_LSP_PIN_EVIL") + expect(packEnv).toContain("NPM_CONFIG_REGISTRY=https://registry.test/") + + const installArgs = await read("install-args.log") + for (const flag of ["--no-save", "--ignore-scripts", "--no-audit", "--no-fund"]) expect(installArgs).toContain(flag) + expect(installArgs).toContain("--prefix") + expect(installArgs).toContain("pkg-1.0.0.tgz") + // install runs with cwd inside the state-dir target it writes. + const installPwd = /PWD=(.*)/.exec(installArgs)![1]! + expect(installPwd).toContain(path.join(state, "lsp", "fake-pin-1.0.0")) + const installEnv = await read("install.env") + expect(installEnv).not.toContain("ES_LSP_PIN_EVIL") + expect(installEnv).toContain("NPM_CONFIG_REGISTRY=https://registry.test/") + }) }) From 41207c3466177e11e77f93deb8c96b77fba6152f Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 22:11:20 -0400 Subject: [PATCH 80/83] fix(remediation-followup): document the user-global baseline boundary (Refs #240) #240.3 takes the "document the gap" fork deliberately: pinning the machine-scoped global layer into the per-project hash baseline would drift whenever the human edits global settings for another project. The PINNED comment states the design (global writes stay guarded by the human-only config-set gate), and a control-baseline test pins it: snapshots cover .factory/ files only, and a write to the global layer (hermetic XDG) leaves the snapshot equal and verification clean. TDD: pinning the global file turns the boundary test red; policy 61/61 green after restore. No behavior change. Refs #178 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/trust/control.ts | 9 ++++++++- packages/core/test/policy.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/core/src/trust/control.ts b/packages/core/src/trust/control.ts index 5ad65d5..ae545be 100644 --- a/packages/core/src/trust/control.ts +++ b/packages/core/src/trust/control.ts @@ -87,7 +87,14 @@ export function controlClass(relative: string): ControlClass | undefined { export const isControlPath = (relative: string) => controlClass(relative) !== undefined -/** Files whose hashes the baseline pins (existing, tracked control artifacts). */ +/** + * Files whose hashes the baseline pins (existing, tracked control artifacts). + * Project-root only by design (#240.3): the user-global layer + * (globalConfigFile, under XDG_CONFIG_HOME) is machine-scoped and shared + * across projects — pinning it here would drift whenever the human edits + * global settings for another project. Global writes stay guarded by the + * human-only `es config set --global` gate instead. + */ const PINNED = (root: string) => { const layout = factoryLayout(root) return [layout.gates, layout.frontier, layout.config] diff --git a/packages/core/test/policy.test.ts b/packages/core/test/policy.test.ts index 2766d23..7a1826a 100644 --- a/packages/core/test/policy.test.ts +++ b/packages/core/test/policy.test.ts @@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test" import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import path from "node:path" +import { globalConfigFile } from "../src/config.ts" import { controlClass, evaluateRead, @@ -10,6 +11,7 @@ import { HUMAN_VERBS, invokesHumanOnly, rebaseline, + snapshotControl, verifyControl, } from "../src/trust/index.ts" import { BOOLEAN_FLAGS, parseArgs } from "../src/util/args.ts" @@ -686,6 +688,32 @@ describe("control baseline", () => { expect((await verifyControl(root)).violations.join("\n")).toContain(".factory/config.json changed") }) + test("the user-global config sits outside the project baseline by design (#240.3)", async () => { + // The baseline pins project-root files only. The user-global layer + // (globalConfigFile, under XDG_CONFIG_HOME) is machine-scoped and + // shared across projects: pinning it here would drift whenever the + // human edits global settings for another project. Global writes stay + // guarded by the human-only `es config set --global` gate instead. + await rebaseline(root, "test") + const before = await snapshotControl(root) + expect(Object.keys(before).length).toBeGreaterThan(0) + expect(Object.keys(before).every((key) => key.startsWith(".factory/"))).toBe(true) + // A write to the global layer (hermetic: XDG pointed at the test state + // dir) changes nothing in the project baseline and verifies clean. + const realXdg = process.env.XDG_CONFIG_HOME + process.env.XDG_CONFIG_HOME = path.join(state, "xdg-config") + try { + const file = globalConfigFile() + await mkdir(path.dirname(file), { recursive: true }) + await writeFile(file, '{"research":{"depth":3}}\n') + expect(await snapshotControl(root)).toEqual(before) + expect((await verifyControl(root)).clean).toBe(true) + } finally { + if (realXdg === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = realXdg + } + }) + test("drift in a waiver file is reported (#221)", async () => { // Waiver records are pinned alongside the gates/config/frontier files, so // a hand edit to a granted waiver halts the run like any other drift. From 648c09328b83b18784e7ec112d4fdf6d04e1e962 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 22:11:47 -0400 Subject: [PATCH 81/83] fix(remediation-followup): lsp is v2's opencode.json key, not ours (Refs #277) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #277.3: the capability detail and the registry header said "lsp config key", reading as an ES config key — but ES layers strict-reject it (schema is .strict(), no top-level lsp; only lspAfterEdit). Both now name v2's opencode.json explicitly. Regenerated docs/CAPABILITIES.md via docs:gen (committed together); docs:check, capabilities suite, tsc and biome green. Refs #178 Co-Authored-By: Claude Opus 5.5 --- docs/CAPABILITIES.md | 2 +- packages/core/src/capabilities.ts | 3 ++- packages/core/src/lsp/registry.ts | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md index 034a3c6..11ebb50 100644 --- a/docs/CAPABILITIES.md +++ b/docs/CAPABILITIES.md @@ -18,7 +18,7 @@ and Antigravity adapters ship in 1.1–1.3). Generated: do not edit by hand. | subagents | ENFORCED | `packages/opencode/test/scoping.test.ts` | Depth 1; hidden seats are unlisted but invocable by ID, and only from the seats allowed to spawn them. Factory seats are not offered Code Mode execute. | | compaction | ENFORCED | `packages/opencode/test/plugin.test.ts` | Session compaction injects the compact factory-state block for factory seats. | | websearch | ENFORCED | `packages/opencode/test/research.test.ts` | The configured provider registers as the host websearch; results are cached and citable. Research, scout and auditor seats are cached-only (es_research_search/fetch) under an offline (--unshare-net) sandbox; host web caching applies only in a project that already has .factory/, and the cache is engine-sealed (planted entries refused). | -| lsp | ENFORCED | `packages/opencode/test/lsp.test.ts` | The lsp config key is read and served by our own manager (v2 accepts but does not run it). | +| lsp | ENFORCED | `packages/opencode/test/lsp.test.ts` | The `lsp` key in v2's `opencode.json` — not an ES config key (ES layers strict-reject it) — is read and served by our own manager (v2 accepts but does not run it). | | panels | ENFORCED | `packages/opencode/test/panels.test.ts` | Four session.panel dashboards (factory, LSP, hooks, brainstorm), rendered by OpenTUI and refreshed on server changes; Esc/q closes a panel, f toggles fullscreen. The factory dashboard leads with the run headline, seats and research progress; a prompt-footer indicator shows the stage and the running seat. | | research | ENFORCED | `packages/opencode/test/research.test.ts` | Engine-sealed cache (#52), fragment-level quote verifier (every ellipsis fragment >= 12 chars; code excerpts one contiguous span of at most 60 lines), auditor, providers and the websearch bridge. Research-only runs complete through the deep-researcher (thesis/antithesis/synthesis at depth 1) at DONE. | | brainstorm | ENFORCED | `packages/opencode/test/brainstorm.test.ts` | Lens fan-out, record/score/complete tools, dedupe, rubric, shortlist with a forced outlier. Runs are id-scoped (the human's /brainstorm is the default run); the grill and the factory run callable brainstorms at depth 1 and get the shortlist as JSON. | diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 0330fa6..acdfe25 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -97,7 +97,8 @@ export const CAPABILITY_MATRIX: Readonly Date: Sat, 10 Oct 2026 22:12:16 -0400 Subject: [PATCH 82/83] fix(remediation-followup): mirror the brainstorm wording fixes (Refs #263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #263: the "eight lenses / forced outlier" wording already landed in the brainstorm prompt + skill assets; the same claims in code said it unconditionally. Mirrored verbatim: registry brainstormer description (eight-lens registry; six by default, four lean for callable runs), capability detail + schema header/field (forced outlier whenever the shortlist has ≥2 slots). docs:gen regenerated CAPABILITIES.md (committed together); docs:check, assets + capabilities suites, tsc and biome green. No prompt touched, so no snapshot/version bump. Refs #178, Refs #191 Co-Authored-By: Claude Opus 5.5 --- docs/CAPABILITIES.md | 2 +- packages/core/src/agents/registry.ts | 3 ++- packages/core/src/capabilities.ts | 2 +- packages/core/src/schema/brainstorm.ts | 5 +++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md index 11ebb50..59a037f 100644 --- a/docs/CAPABILITIES.md +++ b/docs/CAPABILITIES.md @@ -21,7 +21,7 @@ and Antigravity adapters ship in 1.1–1.3). Generated: do not edit by hand. | lsp | ENFORCED | `packages/opencode/test/lsp.test.ts` | The `lsp` key in v2's `opencode.json` — not an ES config key (ES layers strict-reject it) — is read and served by our own manager (v2 accepts but does not run it). | | panels | ENFORCED | `packages/opencode/test/panels.test.ts` | Four session.panel dashboards (factory, LSP, hooks, brainstorm), rendered by OpenTUI and refreshed on server changes; Esc/q closes a panel, f toggles fullscreen. The factory dashboard leads with the run headline, seats and research progress; a prompt-footer indicator shows the stage and the running seat. | | research | ENFORCED | `packages/opencode/test/research.test.ts` | Engine-sealed cache (#52), fragment-level quote verifier (every ellipsis fragment >= 12 chars; code excerpts one contiguous span of at most 60 lines), auditor, providers and the websearch bridge. Research-only runs complete through the deep-researcher (thesis/antithesis/synthesis at depth 1) at DONE. | -| brainstorm | ENFORCED | `packages/opencode/test/brainstorm.test.ts` | Lens fan-out, record/score/complete tools, dedupe, rubric, shortlist with a forced outlier. Runs are id-scoped (the human's /brainstorm is the default run); the grill and the factory run callable brainstorms at depth 1 and get the shortlist as JSON. | +| brainstorm | ENFORCED | `packages/opencode/test/brainstorm.test.ts` | Lens fan-out, record/score/complete tools, dedupe, rubric, shortlist with a forced outlier whenever the shortlist has ≥2 slots. Runs are id-scoped (the human's /brainstorm is the default run); the grill and the factory run callable brainstorms at depth 1 and get the shortlist as JSON. | | harvest | ENFORCED | `packages/opencode/test/harvest.test.ts` | Fail-closed SPDX detection, provenance profiles, policy-enforced matrix, clean-room specs. Runs are id-scoped (the human's /harvest is the default run); es_harvest_target gives the grill, factory, scout and harvester deterministic verdicts, and prior-art search is shared with brainstorm callers. | | design | ENFORCED | `packages/opencode/test/design.test.ts` | Queereye interview, DTCG tokens, contrast gate, generated guide and drift check; phase-02 specs/webref/csf/tui renders and the phase-03 ledger bundle with its cite-gate receipt. | | claims | ENFORCED | `packages/core/test/research.test.ts` | Witnessed claims and dossiers: the evidence cache, dossiers and the claim ledger are tool-only; planted cache entries are refused (engine seal) and every VERIFIED quote and code span is re-checked against the bytes on disk. | diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index 75fcaeb..82c0790 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -177,7 +177,8 @@ export const AGENTS: readonly AgentSpec[] = [ mode: "primary", hidden: false, tier: "deep", - description: "Fans out divergent lenses on an idea and returns a diversified shortlist.", + description: + "Fans out divergent lenses (eight-lens registry; six by default, four lean for callable runs) on an idea and distills a diversified shortlist.", prompt: "brainstormer", tools: [ "es_status", diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index acdfe25..5614fae 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -116,7 +116,7 @@ export const CAPABILITY_MATRIX: Readonly Date: Sat, 10 Oct 2026 22:13:32 -0400 Subject: [PATCH 83/83] fix(remediation-followup): packaging pins for peer range and tarballs (Refs #212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #212a: no `opencode` peer range can exist — verified 2026-10-11 that no such npm package exists (registry 404; the host ships as @opencode/* scope + platform binaries). Pinned: no manifest field names a bare `opencode` dep, so the S7 triple-pin stays the mechanism by decision, not by oversight. #212c: published src trees are clean (only private web has src tests; fleet+web privacy already pinned), and a new layout test fails on any stray src test/spec file the manifest-string check cannot see — proven red with a planted file. scripts/pack-smoke.sh exit 0 on this tree and a tarball scan shows no test files (only product spec*.js). TDD: planted-file mutation red; pack-pins.test.ts 6/6 green after restore. #212.2/.4/.5 (names/versions, publish parsing, runtime matrix) route to S9 per the file's standing note. Refs #178 Co-Authored-By: Claude Opus 5.5 --- packages/core/test/pack-pins.test.ts | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/core/test/pack-pins.test.ts b/packages/core/test/pack-pins.test.ts index 4c87bc7..fd4fde2 100644 --- a/packages/core/test/pack-pins.test.ts +++ b/packages/core/test/pack-pins.test.ts @@ -26,6 +26,16 @@ describe("the opencode triple-pin moves together", () => { expect(new Set(cliPins).size).toBe(1) expect(cliPins[0]).toBe(plugin) }) + + test("no opencode peer range: no such npm package exists to peer on (#212a)", async () => { + // Verified 2026-10-11: `npm view opencode` is a registry 404 — the host + // ships as @opencode/*-scoped packages and platform binaries, so a peer + // on a bare `opencode` id could never resolve. The triple-pin above is + // the mechanism; this pins the decision against a future peer addition. + const opencode = await manifest("opencode") + for (const field of ["dependencies", "devDependencies", "peerDependencies"] as const) + expect(Object.keys(opencode[field] ?? {})).not.toContain("opencode") + }) }) describe("release tarball allowlists", () => { @@ -46,6 +56,29 @@ describe("release tarball allowlists", () => { expect(all.find((pkg) => pkg.dir === dir)?.private).toBe(true) } }) + + test("no test files sit under a released package's src dir (#212c)", async () => { + // The allowlists include whole `src` trees, so a stray src/*.test.ts + // would ship silently; the manifest-string check above cannot see it. + // (fleet and web are private and never packed; pack-smoke.sh 2026-10-11 + // additionally proved the tarballs ship no test files.) + const { readdir } = await import("node:fs/promises") + const strays: string[] = [] + for (const pkg of await releasePackages(root)) { + const walk = async (rel: string): Promise => { + const entries = await readdir(path.join(root, "packages", pkg.dir, rel), { withFileTypes: true }).catch( + () => [], + ) + for (const entry of entries) { + const child = `${rel}/${entry.name}` + if (entry.isDirectory()) await walk(child) + else if (/\.test\.[cm]?[jt]s$|\.spec\.[cm]?[jt]s$|__tests__/.test(child)) strays.push(`${pkg.dir}/${child}`) + } + } + await walk("src") + } + expect(strays).toEqual([]) + }) }) describe("the --release rendering contract", () => {