From be2c12ad9fca1635132fc5ff6a801c8f5eb0dc3c Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 10 Oct 2026 13:38:15 -0400 Subject: [PATCH] test(audit): release stage and draft PR integration tests (#202) --- packages/core/test/factory-release.test.ts | 140 +++++++++++++ packages/opencode/test/pr-guard.test.ts | 76 +++++++ packages/opencode/test/release.test.ts | 224 +++++++++++++++++++++ 3 files changed, 440 insertions(+) create mode 100644 packages/core/test/factory-release.test.ts create mode 100644 packages/opencode/test/pr-guard.test.ts create mode 100644 packages/opencode/test/release.test.ts diff --git a/packages/core/test/factory-release.test.ts b/packages/core/test/factory-release.test.ts new file mode 100644 index 0000000..bec4bf6 --- /dev/null +++ b/packages/core/test/factory-release.test.ts @@ -0,0 +1,140 @@ +// Release base-branch safety against a local bare remote (issue #202): +// `Factory.release` pushes the run's `factory/...` integration branch only — +// the base branch SHA is identical on disk and on the remote afterwards — and +// a run with no PR opener refuses before anything is pushed. +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { type HumanSigner, recordApproval, sealHumanKey, unlockHumanKey } from "../src/approval/index.ts" +import { Factory, type GateRequest, type GateRunLike, type PrRequest } from "../src/factory/index.ts" +import { run } from "../src/util/proc.ts" +import * as Git from "../src/worktree/index.ts" +import { type Fixture, frontier, gitRepo, writeReport, writeSpecs } from "./helpers.ts" + +const PASSPHRASE = "test-passphrase-release-202" +const green: GateRunLike = { passed: true, findings: [], summary: "green" } + +let fx: Fixture +let signer: HumanSigner +let bare: string +/** Every request the pushing opener received (it pushes, then reports a fake URL). */ +let opened: PrRequest[] + +const gates = async (_request: GateRequest): Promise => green + +const pushingOpener = (root: string) => ({ + id: "local-push", + available: async () => true, + open: async (request: PrRequest) => { + opened.push(request) + // The same refspec `ghPrOpener` uses (`branch:branch`): only the run's + // branch travels; the base branch is never named. + await Git.git(root, ["push", "origin", `${request.branch}:${request.branch}`]) + return { url: "https://example.test/pr/local-1" } + }, +}) + +const make = (pr?: ConstructorParameters[1]["pr"]) => + new Factory(fx.root, { gates, ...(pr === undefined ? {} : { pr }), stateDir: fx.state }) + +const sh = async (cwd: string, ...args: string[]) => { + const result = await run(["git", ...args], { cwd }) + if (result.code !== 0) throw new Error(`git ${args.join(" ")} failed: ${(result.stderr || result.stdout).trim()}`) + return result.stdout.trim() +} + +/** `ref -> sha` for every ref on the remote, via the local transport (no network). */ +const remoteRefs = async (): Promise> => { + const out = await sh(fx.root, "ls-remote", bare) + const refs = new Map() + for (const line of out.split("\n").filter(Boolean)) { + const [sha, ref] = line.split("\t") + refs.set(ref!, sha!) + } + return refs +} + +async function toRelease(factory: Factory) { + await factory.begin("human:tester") + await factory.writeFrontier("grill", frontier()) + await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer }) + await factory.beginResearch("human:tester") + await writeReport(fx.root, fx.state) + await factory.completeResearch("factory") + await writeSpecs(fx.root, ["alpha"]) + await recordApproval(fx.root, { stage: "spec", channel: "cli", approvedBy: "tester", signer }) + await factory.startBuild("factory") + const dir = (await factory.activeWorktree())! + await mkdir(path.join(dir, "src"), { recursive: true }) + await mkdir(path.join(dir, "test"), { recursive: true }) + await writeFile(path.join(dir, "test/alpha.test.ts"), 'test("alpha", () => {})\n') + await factory.recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await writeFile(path.join(dir, "src/alpha.ts"), "export const alpha = 1\n") + await factory.complete("es-programmer") + await factory.qaVerdict("es-qa-functional", "pass", "criteria met") + await factory.qaVerdict("es-qa-adversarial", "pass", "nothing broke") + const state = (await factory.read())! + expect(state.stage).toBe("RELEASE") + return state +} + +beforeEach(async () => { + fx = await gitRepo("es-release-") + await sealHumanKey(PASSPHRASE, fx.state) + signer = await unlockHumanKey(PASSPHRASE, fx.state) + opened = [] + // A local bare remote stands in for GitHub: file transport, no network. + bare = await mkdtemp(path.join(tmpdir(), "es-release-bare-")) + await sh(fx.root, "init", "-q", "--bare", bare) + await sh(fx.root, "remote", "add", "origin", bare) + await sh(fx.root, "push", "-q", "origin", "main") +}) +afterEach(async () => { + await fx.cleanup() + await rm(bare, { recursive: true, force: true }) +}) + +describe("release against a local bare remote", () => { + test("release pushes factory/... only; the base SHA is unchanged everywhere", async () => { + const factory = make(pushingOpener(fx.root)) + const before = await toRelease(factory) + expect(before.runBranch).toMatch(/^factory\//) + expect(before.runBranch).not.toBe(before.baseBranch) + + const baseBefore = await Git.revParse(fx.root, "main") + const bareBaseBefore = await Git.revParse(bare, "main") + + const done = await factory.release("factory") + expect(done.stage).toBe("DONE") + expect(done.release?.prUrl).toBe("https://example.test/pr/local-1") + expect(opened).toHaveLength(1) + expect(opened[0]?.branch).toBe(done.runBranch) + expect(opened[0]?.branch).toMatch(/^factory\//) + expect(opened[0]?.base).toBe("main") + + // The remote gained exactly the run branch; nothing else moved. + const refs = await remoteRefs() + expect([...refs.keys()].sort()).toEqual(["refs/heads/main", `refs/heads/${done.runBranch}`].sort()) + expect(refs.get("refs/heads/main")).toBe(bareBaseBefore) + expect(refs.get(`refs/heads/${done.runBranch}`)).toBe(await Git.revParse(fx.root, done.runBranch!)) + + // The base branch is byte-identical on disk and on the remote. + expect(await Git.revParse(fx.root, "main")).toBe(baseBefore) + expect(await Git.revParse(bare, "main")).toBe(baseBefore) + }) + + test("release with no PR opener refuses before anything is pushed", async () => { + const factory = make(undefined) + await toRelease(factory) + const refsBefore = await remoteRefs() + await expect(factory.release("factory")).rejects.toThrow("no PR opener") + expect((await factory.read())?.stage).toBe("RELEASE") + expect(await remoteRefs()).toEqual(refsBefore) + }) +}) diff --git a/packages/opencode/test/pr-guard.test.ts b/packages/opencode/test/pr-guard.test.ts new file mode 100644 index 0000000..8498db0 --- /dev/null +++ b/packages/opencode/test/pr-guard.test.ts @@ -0,0 +1,76 @@ +// Cheap no-network guard tests for the release PR opener (bug #238): +// `ghPrOpener.open()` refuses `branch === base` and non-`factory/` branches +// before any push is attempted. +import { describe, expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { ghPrOpener } from "../src/pr.ts" + +const git = (cwd: string, ...args: string[]) => { + const result = Bun.spawnSync(["git", ...args], { cwd, stdout: "pipe", stderr: "pipe" }) + if (result.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString().trim()}`) + return result.stdout.toString().trim() +} + +/** An empty git repo without any remote: any push attempt would fail loudly. */ +const lonelyRepo = async () => { + const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-")) + git(root, "init", "-q", "-b", "main") + git(root, "config", "user.name", "test") + git(root, "config", "user.email", "test@test") + git(root, "commit", "-q", "--allow-empty", "-m", "init") + return root +} + +const request = (root: string, branch: string, base = "main") => ({ + root, + branch, + base, + title: "factory: test", + body: "test body", +}) + +describe("ghPrOpener branch guard", () => { + test("refuses branch === base", async () => { + const root = await lonelyRepo() + try { + await expect(ghPrOpener.open(request(root, "main", "main"))).rejects.toThrow(/refusing to push/) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + test("refuses non-factory/ branches", async () => { + const root = await lonelyRepo() + try { + for (const branch of ["feature/x", "improve/x", "main", "release-1", "factory"]) { + await expect(ghPrOpener.open(request(root, branch))).rejects.toThrow(/refusing to push/) + } + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + test("the refusal precedes any push attempt", async () => { + // No origin is configured, so a push attempt would fail with a remote + // error — but the guard throws first, before git is ever invoked. + const root = await lonelyRepo() + // A bare remote stands in for GitHub: refused attempts must not create + // or move any ref on it. + const bare = await mkdtemp(path.join(tmpdir(), "es-pr-guard-bare-")) + try { + git(root, "init", "-q", "--bare", bare) + git(root, "remote", "add", "origin", bare) + git(root, "push", "-q", "origin", "main") + const before = git(root, "ls-remote", bare) + await expect(ghPrOpener.open(request(root, "main", "main"))).rejects.toThrow(/refusing to push/) + await expect(ghPrOpener.open(request(root, "feature/x"))).rejects.toThrow(/refusing to push/) + expect(git(root, "ls-remote", bare)).toBe(before) + expect(before.split("\n")).toHaveLength(1) + } finally { + await rm(root, { recursive: true, force: true }) + await rm(bare, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/opencode/test/release.test.ts b/packages/opencode/test/release.test.ts new file mode 100644 index 0000000..57522d6 --- /dev/null +++ b/packages/opencode/test/release.test.ts @@ -0,0 +1,224 @@ +// Release base-branch safety on the real host (issue #202): with a local +// bare remote standing in for GitHub (file transport, no network) and a fake +// `gh` on PATH (no real GitHub CLI), the release opener pushes the run's +// `factory/...` integration branch only, opens the PR as a draft, and refuses +// `branch === base` before anything is pushed. The harness itself runs with +// `pr: "off"`, so the `es_release` tool refuses and the base SHA is unchanged. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { + buildPrArgs, + Factory, + type HumanSigner, + recordApproval, + researchSourcesDir, + SourceCache, + sealHumanKey, + stringifyFrontmatter, + unlockHumanKey, + writeJson, +} from "@heretek-ai/es-core" +import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit" +import { ghPrOpener } from "../src/pr.ts" + +const pluginDir = path.resolve(import.meta.dir, "..") +const call = (name: string, args: Record = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@` +const green = { passed: true, findings: [] as never[], summary: "green" } + +const git = (cwd: string, ...args: string[]) => { + const result = Bun.spawnSync(["git", ...args], { cwd, stdout: "pipe", stderr: "pipe" }) + if (result.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString().trim()}`) + return result.stdout.toString().trim() +} + +let h: Harness +let state: string +let signer: HumanSigner +let bare: string +let binDir: string +let captureFile: string +let baseBefore: string + +const factory = () => new Factory(h.directory, { gates: async () => green, stateDir: state }) + +beforeAll(async () => { + state = await mkdtemp(path.join(tmpdir(), "es-release-state-")) + await sealHumanKey("test-passphrase-202", state) + signer = await unlockHumanKey("test-passphrase-202", state) + h = await boot({ + git: true, + script: directiveScript, + plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }], + files: { "README.md": "# release safety\n" }, + }) + // A local bare remote stands in for GitHub: file transport, no network. + bare = await mkdtemp(path.join(tmpdir(), "es-release-bare-")) + git(h.directory, "init", "-q", "--bare", bare) + git(h.directory, "remote", "add", "origin", bare) + git(h.directory, "push", "-q", "origin", "main") + baseBefore = git(h.directory, "rev-parse", "main") + + // A fake `gh`: `auth status` succeeds, `pr create` captures its argv and + // reports a PR URL. The capture path is baked in because the runner + // scrubs the environment (only PATH survives). + binDir = await mkdtemp(path.join(tmpdir(), "es-release-bin-")) + captureFile = path.join(binDir, "pr-argv.txt") + const gh = path.join(binDir, "gh") + await writeFile( + gh, + `#!/bin/sh\nif [ "$1" = "auth" ]; then exit 0; fi\ni=0\nfor a in "$@"; do printf 'arg%s=%s\\n' "$i" "$a" >> '${captureFile}'; i=$((i + 1)); done\nprintf '%s\\n' '---' >> '${captureFile}'\nprintf '%s\\n' 'https://example.test/pr/7'\nexit 0\n`, + ) + await chmod(gh, 0o755) +}, 60_000) + +afterAll(async () => { + await h?.close() + await rm(state, { recursive: true, force: true }) + await rm(bare, { recursive: true, force: true }) + await rm(binDir, { recursive: true, force: true }) +}) + +/** Drive the shared-state run to RELEASE with direct core calls (one phase). */ +const toRelease = async () => { + await factory().begin("human:tester") + await factory().writeFrontier("grill", { + version: "1.1", + idea: "a greeting library", + spendCeiling: { currency: "USD", maxAmount: 10 }, + settled: true, + nodes: [{ id: "lang", question: "Language?", answer: "TypeScript", status: "settled" }], + }) + await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", signer }) + await factory().beginResearch("human:tester") + const cache = new SourceCache(researchSourcesDir(h.directory), state) + const source = await cache.put({ + url: "https://example.test/greeting", + text: "Greeting libraries usually expose a single greet function.", + provider: "fetch", + }) + await mkdir(path.join(h.directory, ".factory/research"), { recursive: true }) + await writeFile( + path.join(h.directory, ".factory/research/REPORT.md"), + `# Research\n\n- A greet function is the conventional API [VERIFIED: sha256:${source.meta.sha256} "expose a single greet function"]\n`, + ) + await factory().completeResearch("factory") + await writeJson(path.join(h.directory, ".factory/roadmap.json"), { + version: 1, + title: "Greeting library", + phases: [{ id: "alpha", title: "Greet" }], + }) + await mkdir(path.join(h.directory, ".factory/specs/alpha"), { recursive: true }) + await writeFile( + path.join(h.directory, ".factory/specs/alpha/GOAL.md"), + stringifyFrontmatter( + { + phase: "alpha", + title: "alpha", + acceptance: [{ kind: "file", id: "impl", description: "implementation exists", path: "src/alpha.ts" }], + }, + "Implement alpha.\n", + ), + ) + await recordApproval(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer }) + await factory().startBuild("factory") + const wt = (await factory().activeWorktree())! + const rel = (file: string) => path.relative(h.directory, path.join(wt, file)) + await mkdir(path.dirname(path.join(h.directory, rel("test/alpha.test.ts"))), { recursive: true }) + await writeFile(path.join(h.directory, rel("test/alpha.test.ts")), 'test("alpha", () => {})\n') + await factory().recordGateRun("es-programmer", { + passed: false, + findings: [], + summary: "red", + failedTests: ["test/alpha.test.ts"], + }) + await mkdir(path.dirname(path.join(h.directory, rel("src/alpha.ts"))), { recursive: true }) + await writeFile(path.join(h.directory, rel("src/alpha.ts")), "export const alpha = 1\n") + await factory().complete("es-programmer") + await factory().qaVerdict("es-qa-functional", "pass", "criteria met") + await factory().qaVerdict("es-qa-adversarial", "pass", "nothing broke") + const runState = (await factory().read())! + expect(runState.stage).toBe("RELEASE") + return runState +} + +describe("release on the real host against a local bare remote", () => { + test("the release argv names factory/... as a draft on the base; branch == base is refused", async () => { + const runState = await toRelease() + const branch = runState.runBranch! + const base = runState.baseBranch! + expect(branch).toMatch(/^factory\//) + expect(base).toBe("main") + + // The argv the opener would use: a draft PR from the run branch onto base. + const argv = buildPrArgs({ branch, base, title: "factory: Greet", body: "release body for #202" }) + expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"]) + expect(argv[argv.indexOf("--head") + 1]).toBe(branch) + expect(argv[argv.indexOf("--base") + 1]).toBe("main") + + // branch === base is refused before anything is pushed: the remote still + // holds only the base. + await expect(ghPrOpener.open({ root: h.directory, branch: base, base, title: "t", body: "b" })).rejects.toThrow( + /refusing to push/, + ) + expect(git(h.directory, "ls-remote", bare).split("\n")).toHaveLength(1) + }, 120_000) + + test("the real opener pushes factory/... only; the base SHA is unchanged; es_release stays refused", async () => { + const runState = (await factory().read())! + const branch = runState.runBranch! + const savedPath = process.env.PATH + process.env.PATH = `${binDir}${path.delimiter}${savedPath}` + try { + expect(await ghPrOpener.available(h.directory)).toBe(true) + const opened = await ghPrOpener.open({ + root: h.directory, + branch, + base: "main", + title: "factory: Greet", + body: "release body for #202", + }) + expect(opened.url).toBe("https://example.test/pr/7") + + // The draft PR went to the base from the run branch — nothing else. + const captured = (await readFile(captureFile, "utf8")) + .split("\n") + .filter((line: string) => line.startsWith("arg")) + .map((line: string) => line.slice(line.indexOf("=") + 1)) + expect(captured).toContain("--draft") + expect(captured[captured.indexOf("--head") + 1]).toBe(branch) + expect(captured[captured.indexOf("--base") + 1]).toBe("main") + + // The remote gained exactly the run branch; the base SHA is unchanged + // on disk and on the remote. + const refs = new Map( + git(h.directory, "ls-remote", bare) + .split("\n") + .filter(Boolean) + .map((line: string) => { + const [sha, ref] = line.split("\t") + return [ref, sha] as const + }), + ) + expect([...refs.keys()].sort()).toEqual(["refs/heads/main", `refs/heads/${branch}`].sort()) + expect(refs.get("refs/heads/main")).toBe(baseBefore) + expect(refs.get(`refs/heads/${branch}`)).toBe(git(h.directory, "rev-parse", branch)) + expect(git(h.directory, "rev-parse", "main")).toBe(baseBefore) + + // With `pr: "off"` the host tool refuses instead of pushing: the stage + // stays RELEASE for the human, and the remote does not move. + const refsBeforeTool = git(h.directory, "ls-remote", bare) + const refused = await h.run(`ship ${call("es_release", {})}`, { agent: "factory" }) + expect(refused.tools[0]?.status).toBe("error") + expect((await factory().read())?.stage).toBe("RELEASE") + expect(git(h.directory, "ls-remote", bare)).toBe(refsBeforeTool) + + // The human records the PR; the run is DONE. + const done = await factory().recordPr("tester", "https://example.test/pr/9") + expect([done.stage, done.release?.prUrl]).toEqual(["DONE", "https://example.test/pr/9"]) + } finally { + process.env.PATH = savedPath + } + }, 120_000) +})