Merge pull request #245 from Heretek-AI/audit/202-release

[Audit] Release stage: OpenCode v2 integration tests
This commit is contained in:
BillyOutlast
2026-10-10 23:05:50 -04:00
committed by GitHub
3 changed files with 411 additions and 10 deletions
+140
View File
@@ -0,0 +1,140 @@
// Release base-branch safety against a local bare remote (issue #202):
// `Factory.release` pushes the run's `factory/...` integration branch only —
// the base branch SHA is identical on disk and on the remote afterwards — and
// a run with no PR opener refuses before anything is pushed.
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { type HumanSigner, recordApproval, sealHumanKey, unlockHumanKey } from "../src/approval/index.ts"
import { Factory, type GateRequest, type GateRunLike, type PrRequest } from "../src/factory/index.ts"
import { run } from "../src/util/proc.ts"
import * as Git from "../src/worktree/index.ts"
import { type Fixture, frontier, gitRepo, writeReport, writeSpecs } from "./helpers.ts"
const PASSPHRASE = "test-passphrase-release-202"
const green: GateRunLike = { passed: true, findings: [], summary: "green" }
let fx: Fixture
let signer: HumanSigner
let bare: string
/** Every request the pushing opener received (it pushes, then reports a fake URL). */
let opened: PrRequest[]
const gates = async (_request: GateRequest): Promise<GateRunLike> => green
const pushingOpener = (root: string) => ({
id: "local-push",
available: async () => true,
open: async (request: PrRequest) => {
opened.push(request)
// The same refspec `ghPrOpener` uses (`branch:branch`): only the run's
// branch travels; the base branch is never named.
await Git.git(root, ["push", "origin", `${request.branch}:${request.branch}`])
return { url: "https://example.test/pr/local-1" }
},
})
const make = (pr?: ConstructorParameters<typeof Factory>[1]["pr"]) =>
new Factory(fx.root, { gates, ...(pr === undefined ? {} : { pr }), stateDir: fx.state })
const sh = async (cwd: string, ...args: string[]) => {
const result = await run(["git", ...args], { cwd })
if (result.code !== 0) throw new Error(`git ${args.join(" ")} failed: ${(result.stderr || result.stdout).trim()}`)
return result.stdout.trim()
}
/** `ref -> sha` for every ref on the remote, via the local transport (no network). */
const remoteRefs = async (): Promise<Map<string, string>> => {
const out = await sh(fx.root, "ls-remote", bare)
const refs = new Map<string, string>()
for (const line of out.split("\n").filter(Boolean)) {
const [sha, ref] = line.split("\t")
refs.set(ref!, sha!)
}
return refs
}
async function toRelease(factory: Factory) {
await factory.begin("human:tester")
await factory.writeFrontier("grill", frontier())
await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer })
await factory.beginResearch("human:tester")
await writeReport(fx.root, fx.state)
await factory.completeResearch("factory")
await writeSpecs(fx.root, ["alpha"])
await recordApproval(fx.root, { stage: "spec", channel: "cli", approvedBy: "tester", signer })
await factory.startBuild("factory")
const dir = (await factory.activeWorktree())!
await mkdir(path.join(dir, "src"), { recursive: true })
await mkdir(path.join(dir, "test"), { recursive: true })
await writeFile(path.join(dir, "test/alpha.test.ts"), 'test("alpha", () => {})\n')
await factory.recordGateRun("es-programmer", {
passed: false,
findings: [],
summary: "red",
failedTests: ["test/alpha.test.ts"],
})
await writeFile(path.join(dir, "src/alpha.ts"), "export const alpha = 1\n")
await factory.complete("es-programmer")
await factory.qaVerdict("es-qa-functional", "pass", "criteria met")
await factory.qaVerdict("es-qa-adversarial", "pass", "nothing broke")
const state = (await factory.read())!
expect(state.stage).toBe("RELEASE")
return state
}
beforeEach(async () => {
fx = await gitRepo("es-release-")
await sealHumanKey(PASSPHRASE, fx.state)
signer = await unlockHumanKey(PASSPHRASE, fx.state)
opened = []
// A local bare remote stands in for GitHub: file transport, no network.
bare = await mkdtemp(path.join(tmpdir(), "es-release-bare-"))
await sh(fx.root, "init", "-q", "--bare", bare)
await sh(fx.root, "remote", "add", "origin", bare)
await sh(fx.root, "push", "-q", "origin", "main")
})
afterEach(async () => {
await fx.cleanup()
await rm(bare, { recursive: true, force: true })
})
describe("release against a local bare remote", () => {
test("release pushes factory/... only; the base SHA is unchanged everywhere", async () => {
const factory = make(pushingOpener(fx.root))
const before = await toRelease(factory)
expect(before.runBranch).toMatch(/^factory\//)
expect(before.runBranch).not.toBe(before.baseBranch)
const baseBefore = await Git.revParse(fx.root, "main")
const bareBaseBefore = await Git.revParse(bare, "main")
const done = await factory.release("factory")
expect(done.stage).toBe("DONE")
expect(done.release?.prUrl).toBe("https://example.test/pr/local-1")
expect(opened).toHaveLength(1)
expect(opened[0]?.branch).toBe(done.runBranch)
expect(opened[0]?.branch).toMatch(/^factory\//)
expect(opened[0]?.base).toBe("main")
// The remote gained exactly the run branch; nothing else moved.
const refs = await remoteRefs()
expect([...refs.keys()].sort()).toEqual(["refs/heads/main", `refs/heads/${done.runBranch}`].sort())
expect(refs.get("refs/heads/main")).toBe(bareBaseBefore)
expect(refs.get(`refs/heads/${done.runBranch}`)).toBe(await Git.revParse(fx.root, done.runBranch!))
// The base branch is byte-identical on disk and on the remote.
expect(await Git.revParse(fx.root, "main")).toBe(baseBefore)
expect(await Git.revParse(bare, "main")).toBe(baseBefore)
})
test("release with no PR opener refuses before anything is pushed", async () => {
const factory = make(undefined)
await toRelease(factory)
const refsBefore = await remoteRefs()
await expect(factory.release("factory")).rejects.toThrow("no PR opener")
expect((await factory.read())?.stage).toBe("RELEASE")
expect(await remoteRefs()).toEqual(refsBefore)
})
})
+46
View File
@@ -3,6 +3,10 @@
// argv through the shared builder. No network and no `gh` invocation: every
// refusal throws before the push, and the draft shape is asserted on the
// argv (the `gh pr create` spawn itself never runs in tests).
// Union of the audit (#202) and remediation suites at the #290 merge: the
// remediation side proves refusal order without any repo (plus the preset
// guard and argv shape); the audit side additionally proves against a real
// repo with a bare remote that refused attempts move no ref.
import { describe, expect, test } from "bun:test"
import { mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
@@ -20,6 +24,22 @@ const open = (root: string, overrides: { branch: string; base?: string; preset?:
...(overrides.preset !== undefined ? { preset: overrides.preset } : {}),
})
const gitSync = (cwd: string, ...args: string[]) => {
const result = Bun.spawnSync(["git", ...args], { cwd, stdout: "pipe", stderr: "pipe" })
if (result.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString().trim()}`)
return result.stdout.toString().trim()
}
/** An empty git repo without any remote: any push attempt would fail loudly. */
const lonelyRepo = async () => {
const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-"))
gitSync(root, "init", "-q", "-b", "main")
gitSync(root, "config", "user.name", "test")
gitSync(root, "config", "user.email", "test@test")
gitSync(root, "commit", "-q", "--allow-empty", "-m", "init")
return root
}
describe("ghPrOpener.open() branch guard", () => {
test("branch === base is refused", async () => {
const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-"))
@@ -68,6 +88,32 @@ describe("ghPrOpener.open() branch guard", () => {
await rm(root, { recursive: true, force: true })
}
})
test("refused attempts move no ref on a real repo with a bare remote", async () => {
// The audit side of the union: same refusal, proven by side effect
// (no ref created or moved) rather than by error precedence.
const root = await lonelyRepo()
// A bare remote stands in for GitHub: refused attempts must not create
// or move any ref on it.
const bare = await mkdtemp(path.join(tmpdir(), "es-pr-guard-bare-"))
try {
gitSync(root, "init", "-q", "--bare", bare)
gitSync(root, "remote", "add", "origin", bare)
gitSync(root, "push", "-q", "origin", "main")
const before = gitSync(root, "ls-remote", bare)
await expect(
ghPrOpener.open({ root, branch: "main", base: "main", title: "factory: test", body: "test body" }),
).rejects.toThrow(/refusing to push/)
await expect(
ghPrOpener.open({ root, branch: "feature/x", base: "main", title: "factory: test", body: "test body" }),
).rejects.toThrow(/refusing to push/)
expect(gitSync(root, "ls-remote", bare)).toBe(before)
expect(before.split("\n")).toHaveLength(1)
} finally {
await rm(root, { recursive: true, force: true })
await rm(bare, { recursive: true, force: true })
}
})
})
describe("release PR argv", () => {
+225 -10
View File
@@ -1,15 +1,30 @@
// Release base-branch safety against a local bare remote (#238): the
// factory release pushes a `factory/` branch only, `branch == base` (or a
// non-`factory/` branch) is refused before anything is pushed, and the base
// SHA on the remote is byte-identical afterwards. Local bare-remote fixture
// only (no real remote, no network); `gh` is never invoked — the authorized
// push runs the same `git push --set-upstream origin <branch>:<branch>`
// argv `pr.ts` uses, and the draft shape is asserted on the shared argv.
import { describe, expect, test } from "bun:test"
import { mkdtemp, rm, writeFile } from "node:fs/promises"
// Release base-branch safety: the factory release pushes a `factory/`
// branch only, `branch == base` (or a non-`factory/` branch) is refused
// before anything is pushed, and the base SHA on the remote is
// byte-identical afterwards.
// Union of the audit (#202) and remediation suites at the #290 merge: the
// remediation side proves refusal and push shape against a fast local
// bare-remote fixture (no host boot); the audit side drives a full run to
// RELEASE on the real host and proves the argv, the real push, the
// es_release refusal and recordPr-to-DONE through it.
import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { buildPrArgs, git } from "@heretek-ai/es-core"
import {
buildPrArgs,
Factory,
git,
type HumanSigner,
recordApproval,
researchSourcesDir,
SourceCache,
sealHumanKey,
stringifyFrontmatter,
unlockHumanKey,
writeJson,
} from "@heretek-ai/es-core"
import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
import { ghPrOpener } from "../src/pr.ts"
const BASE = "rewrite"
@@ -107,3 +122,203 @@ describe("release against a local bare remote", () => {
}
})
})
const pluginDir = path.resolve(import.meta.dir, "..")
const call = (name: string, args: Record<string, unknown> = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@`
const green = { passed: true, findings: [] as never[], summary: "green" }
const gitSync = (cwd: string, ...args: string[]) => {
const result = Bun.spawnSync(["git", ...args], { cwd, stdout: "pipe", stderr: "pipe" })
if (result.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString().trim()}`)
return result.stdout.toString().trim()
}
let h: Harness
let state: string
let signer: HumanSigner
let bare: string
let binDir: string
let captureFile: string
let baseBefore: string
const factory = () => new Factory(h.directory, { gates: async () => green, stateDir: state })
beforeAll(async () => {
state = await mkdtemp(path.join(tmpdir(), "es-release-state-"))
await sealHumanKey("test-passphrase-202", state)
signer = await unlockHumanKey("test-passphrase-202", state)
h = await boot({
git: true,
script: directiveScript,
plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }],
files: { "README.md": "# release safety\n" },
})
// A local bare remote stands in for GitHub: file transport, no network.
bare = await mkdtemp(path.join(tmpdir(), "es-release-bare-"))
gitSync(h.directory, "init", "-q", "--bare", bare)
gitSync(h.directory, "remote", "add", "origin", bare)
gitSync(h.directory, "push", "-q", "origin", "main")
baseBefore = gitSync(h.directory, "rev-parse", "main")
// A fake `gh`: `auth status` succeeds, `pr create` captures its argv and
// reports a PR URL. The capture path is baked in because the runner
// scrubs the environment (only PATH survives).
binDir = await mkdtemp(path.join(tmpdir(), "es-release-bin-"))
captureFile = path.join(binDir, "pr-argv.txt")
const gh = path.join(binDir, "gh")
await writeFile(
gh,
`#!/bin/sh\nif [ "$1" = "auth" ]; then exit 0; fi\ni=0\nfor a in "$@"; do printf 'arg%s=%s\\n' "$i" "$a" >> '${captureFile}'; i=$((i + 1)); done\nprintf '%s\\n' '---' >> '${captureFile}'\nprintf '%s\\n' 'https://example.test/pr/7'\nexit 0\n`,
)
await chmod(gh, 0o755)
}, 60_000)
afterAll(async () => {
await h?.close()
await rm(state, { recursive: true, force: true })
await rm(bare, { recursive: true, force: true })
await rm(binDir, { recursive: true, force: true })
})
/** Drive the shared-state run to RELEASE with direct core calls (one phase). */
const toRelease = async () => {
await factory().begin("human:tester")
await factory().writeFrontier("grill", {
version: "1.1",
idea: "a greeting library",
spendCeiling: { currency: "USD", maxAmount: 10 },
settled: true,
nodes: [{ id: "lang", question: "Language?", answer: "TypeScript", status: "settled" }],
})
await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", signer })
await factory().beginResearch("human:tester")
const cache = new SourceCache(researchSourcesDir(h.directory), state)
const source = await cache.put({
url: "https://example.test/greeting",
text: "Greeting libraries usually expose a single greet function.",
provider: "fetch",
})
await mkdir(path.join(h.directory, ".factory/research"), { recursive: true })
await writeFile(
path.join(h.directory, ".factory/research/REPORT.md"),
`# Research\n\n- A greet function is the conventional API [VERIFIED: sha256:${source.meta.sha256} "expose a single greet function"]\n`,
)
await factory().completeResearch("factory")
await writeJson(path.join(h.directory, ".factory/roadmap.json"), {
version: 1,
title: "Greeting library",
phases: [{ id: "alpha", title: "Greet" }],
})
await mkdir(path.join(h.directory, ".factory/specs/alpha"), { recursive: true })
await writeFile(
path.join(h.directory, ".factory/specs/alpha/GOAL.md"),
stringifyFrontmatter(
{
phase: "alpha",
title: "alpha",
acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }],
},
"Implement alpha.\n",
),
)
await recordApproval(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer })
await factory().startBuild("factory")
const wt = (await factory().activeWorktree())!
const rel = (file: string) => path.relative(h.directory, path.join(wt, file))
await mkdir(path.dirname(path.join(h.directory, rel("test/alpha.test.ts"))), { recursive: true })
await writeFile(path.join(h.directory, rel("test/alpha.test.ts")), 'test("alpha", () => {})\n')
await factory().recordGateRun("es-programmer", {
passed: false,
findings: [],
summary: "red",
failedTests: ["test/alpha.test.ts"],
})
await mkdir(path.dirname(path.join(h.directory, rel("src/alpha.ts"))), { recursive: true })
await writeFile(path.join(h.directory, rel("src/alpha.ts")), "export const alpha = 1\n")
await factory().complete("es-programmer")
await factory().qaVerdict("es-qa-functional", "pass", "criteria met")
await factory().qaVerdict("es-qa-adversarial", "pass", "nothing broke")
const runState = (await factory().read())!
expect(runState.stage).toBe("RELEASE")
return runState
}
describe("release on the real host against a local bare remote", () => {
test("the release argv names factory/... as a draft on the base; branch == base is refused", async () => {
const runState = await toRelease()
const branch = runState.runBranch!
const base = runState.baseBranch!
expect(branch).toMatch(/^factory\//)
expect(base).toBe("main")
// The argv the opener would use: a draft PR from the run branch onto base.
const argv = buildPrArgs({ branch, base, title: "factory: Greet", body: "release body for #202" })
expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"])
expect(argv[argv.indexOf("--head") + 1]).toBe(branch)
expect(argv[argv.indexOf("--base") + 1]).toBe("main")
// branch === base is refused before anything is pushed: the remote still
// holds only the base.
await expect(ghPrOpener.open({ root: h.directory, branch: base, base, title: "t", body: "b" })).rejects.toThrow(
/refusing to push/,
)
expect(gitSync(h.directory, "ls-remote", bare).split("\n")).toHaveLength(1)
}, 120_000)
test("the real opener pushes factory/... only; the base SHA is unchanged; es_release stays refused", async () => {
const runState = (await factory().read())!
const branch = runState.runBranch!
const savedPath = process.env.PATH
process.env.PATH = `${binDir}${path.delimiter}${savedPath}`
try {
expect(await ghPrOpener.available(h.directory)).toBe(true)
const opened = await ghPrOpener.open({
root: h.directory,
branch,
base: "main",
title: "factory: Greet",
body: "release body for #202",
})
expect(opened.url).toBe("https://example.test/pr/7")
// The draft PR went to the base from the run branch — nothing else.
const captured = (await readFile(captureFile, "utf8"))
.split("\n")
.filter((line: string) => line.startsWith("arg"))
.map((line: string) => line.slice(line.indexOf("=") + 1))
expect(captured).toContain("--draft")
expect(captured[captured.indexOf("--head") + 1]).toBe(branch)
expect(captured[captured.indexOf("--base") + 1]).toBe("main")
// The remote gained exactly the run branch; the base SHA is unchanged
// on disk and on the remote.
const refs = new Map(
gitSync(h.directory, "ls-remote", bare)
.split("\n")
.filter(Boolean)
.map((line: string) => {
const [sha, ref] = line.split("\t")
return [ref, sha] as const
}),
)
expect([...refs.keys()].sort()).toEqual(["refs/heads/main", `refs/heads/${branch}`].sort())
expect(refs.get("refs/heads/main")).toBe(baseBefore)
expect(refs.get(`refs/heads/${branch}`)).toBe(gitSync(h.directory, "rev-parse", branch))
expect(gitSync(h.directory, "rev-parse", "main")).toBe(baseBefore)
// With `pr: "off"` the host tool refuses instead of pushing: the stage
// stays RELEASE for the human, and the remote does not move.
const refsBeforeTool = gitSync(h.directory, "ls-remote", bare)
const refused = await h.run(`ship ${call("es_release", {})}`, { agent: "factory" })
expect(refused.tools[0]?.status).toBe("error")
expect((await factory().read())?.stage).toBe("RELEASE")
expect(gitSync(h.directory, "ls-remote", bare)).toBe(refsBeforeTool)
// The human records the PR; the run is DONE.
const done = await factory().recordPr("tester", "https://example.test/pr/9")
expect([done.stage, done.release?.prUrl]).toEqual(["DONE", "https://example.test/pr/9"])
} finally {
process.env.PATH = savedPath
}
}, 120_000)
})