mirror of
https://github.com/Heretek-AI/IUMBTEMS.git
synced 2026-10-12 02:03:00 +00:00
Merge pull request #297 from Heretek-AI/fixup/remediation-iii
Remediation III: fail-closed e2e gate, full trust hash, main retarget, claim wording (Refs #210, #217, #281, #223)
This commit is contained in:
@@ -24,7 +24,7 @@ name: E2E feedback (opencode + plugin)
|
||||
# use OPENCODE_API_KEY for Zen/Go keys)
|
||||
on:
|
||||
push:
|
||||
branches: [rewrite]
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
model:
|
||||
|
||||
@@ -4,13 +4,13 @@ name: Nightly (OpenCode v2 HEAD)
|
||||
# runs the real-host suite against the `v2` branch HEAD every night and turns
|
||||
# red when the host drifts. It is the mitigation, not a pin.
|
||||
#
|
||||
# Scheduled workflows only run from the default branch, so pre-cutover the job
|
||||
# is exercised through workflow_dispatch (and the push trigger on `rewrite`).
|
||||
# Scheduled workflows only run from the default branch, so the job
|
||||
# is exercised through workflow_dispatch (and the push trigger on `main`).
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 * * *"
|
||||
push:
|
||||
branches: [rewrite]
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -75,7 +75,7 @@ spike as proof).
|
||||
registry or capability rows. Prompt changes require the assets snapshot
|
||||
(`bun test -u packages/core/test/assets.test.ts`) and a version bump.
|
||||
- Stage explicit paths; never `git add -A`. Commit per milestone.
|
||||
- Push only `rewrite` during development; never push or merge to `main`
|
||||
without asking. The 1.0 cutover (tag legacy, reviewed merge, npm publish via
|
||||
OIDC) is a separate, human-approved step.
|
||||
- Push only your ticket branch during development; never push or merge to `main`
|
||||
without asking — release lands on `main` through a human-merged PR.
|
||||
Publishing to npm via OIDC is a separate, human-approved step.
|
||||
- After any push, check every workflow (`gh run list`) — not just the obvious one.
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
# Changelog
|
||||
|
||||
## 1.6.1 — 2026-10-11
|
||||
|
||||
**Trust hash covers the full gate command (#217).** `commandSetHash` now hashes
|
||||
the complete normalized `CommandCheck` — `kind`, `parser`, `related`,
|
||||
`touchedArgs`/`extensions` and `timeoutMs` alongside `id`/`command` — so any
|
||||
change to what a gate runs revokes trust. **Humans: re-run `es trust` after
|
||||
upgrading**; previously approved command sets hash differently and the gates
|
||||
refuse to run until the new set is approved. The e2e regression gate now fails
|
||||
closed when no history is sampled (#210). The self-dogfood preset releases
|
||||
onto `main` post-cutover: the `rewrite` pin, workflow triggers and runbook
|
||||
now target `main` (#281); the `rewrite` branch itself is untouched.
|
||||
|
||||
## 1.6.0 — 2026-10-09
|
||||
|
||||
Phase 5 self-improvement: runs already leave rich records (audit chain,
|
||||
|
||||
@@ -23,8 +23,10 @@
|
||||
## What it does
|
||||
|
||||
- **Factory stages, enforced by code.** Grill → Research → Spec → Build ⇄ QA →
|
||||
Release runs through a state machine; every `es_*` tool refuses an
|
||||
out-of-order step. Approvals (frontier, spec) are human-only and complete
|
||||
Release runs through a state machine; the phase-transition tools refuse an
|
||||
out-of-order step (status, validation, approval-request, gate-record,
|
||||
audit-open, research, scout, brainstorm, harvest, design and LSP tools are
|
||||
intentionally stage-free: seat-gated, not stage-gated). Approvals (frontier, spec) are human-only and complete
|
||||
in the TUI (masked passphrase dialog, in-process signing), in the browser
|
||||
(preview, then passphrase over loopback), or at a terminal
|
||||
(`es approve`); waivers and trust stay terminal-only (`es waive`,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
},
|
||||
"packages/cli": {
|
||||
"name": "@heretek-ai/es-cli",
|
||||
"version": "1.6.0",
|
||||
"version": "1.6.1",
|
||||
"bin": {
|
||||
"es": "./bin/es.js",
|
||||
"epistemic-swarm": "./bin/es.js",
|
||||
@@ -26,7 +26,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@heretek-ai/es-core",
|
||||
"version": "1.6.0",
|
||||
"version": "1.6.1",
|
||||
"dependencies": {
|
||||
"vscode-jsonrpc": "9.0.3",
|
||||
"vscode-languageserver-protocol": "3.18.4",
|
||||
@@ -50,7 +50,7 @@
|
||||
},
|
||||
"packages/opencode": {
|
||||
"name": "@heretek-ai/epistemic-swarm",
|
||||
"version": "1.6.0",
|
||||
"version": "1.6.1",
|
||||
"dependencies": {
|
||||
"@heretek-ai/es-core": "workspace:*",
|
||||
"@opencode/plugin": "2.0.24",
|
||||
|
||||
@@ -13,7 +13,7 @@ suite, and open a draft PR a human reviews. This runbook is that loop.
|
||||
PR through it.
|
||||
- A sealed human key (`es key seal` once); approvals, trust and resume all
|
||||
need the passphrase.
|
||||
- Work on `rewrite`, with a clean tree.
|
||||
- Work on `main`, with a clean tree.
|
||||
|
||||
## The flow
|
||||
|
||||
@@ -22,25 +22,25 @@ suite, and open a draft PR a human reviews. This runbook is that loop.
|
||||
This writes `.factory/config.json` (`audit.phase: required`), the
|
||||
self-dogfood `.factory/gates.json` (`bun install --frozen-lockfile`,
|
||||
`bun run check`, `bun run docs:check`), a `.factory/roadmap.json` seed
|
||||
pinning `baseBranch: rewrite`, and the issue body as
|
||||
pinning `baseBranch: main`, and the issue body as
|
||||
`.factory/notes/idea-<n>.md`.
|
||||
2. **Trust the gates**: `es trust` (human-only). The gate commands run
|
||||
project code; review the hashes before trusting.
|
||||
3. **Grill**: `/grill` (or `es factory begin` plus the grill flow) with a
|
||||
**$15 spend ceiling** (the preset default; raise deliberately). Keep
|
||||
`baseBranch: rewrite` when the grill rewrites the roadmap — the factory
|
||||
never pushes `main` or `rewrite` itself.
|
||||
`baseBranch: main` when the grill rewrites the roadmap — the factory
|
||||
never pushes `main` itself.
|
||||
4. **Approve the frontier and the spec** (`es approve frontier`, then
|
||||
`es approve spec`) — on any Phase 2/4 surface: terminal, TUI or browser.
|
||||
5. **Build ⇄ QA** runs in the phase worktree under bwrap; the runtime cap
|
||||
is mandatory and `.factory/STOP` halts the run at its next step.
|
||||
6. **Release**: the merge gate runs in full; on green the factory pushes
|
||||
only its run branch and opens a **draft PR against `rewrite`**.
|
||||
only its run branch and opens a **draft PR against `main`**.
|
||||
A human reviews and merges. Auto-merge never happens.
|
||||
|
||||
## Reviewing the PR
|
||||
|
||||
- Check the base is `rewrite` and the head is a `factory/…` branch.
|
||||
- Check the base is `main` and the head is a `factory/…` branch.
|
||||
- Read the gate logs under `.factory/runs/<run>/gates/` and the spend
|
||||
against the $15 ceiling.
|
||||
- Merge only when every check — including CodeRabbit and SonarCloud —
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@heretek-ai/es-cli",
|
||||
"version": "1.6.0",
|
||||
"version": "1.6.1",
|
||||
"esRelease": {
|
||||
"order": 2
|
||||
},
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
/** The `es` CLI version. Single source: kept equal to the package version by test/cli.test.ts. */
|
||||
export const VERSION = "1.6.0"
|
||||
export const VERSION = "1.6.1"
|
||||
|
||||
@@ -1105,9 +1105,9 @@ describe("es factory init --preset self-dogfood (#137)", () => {
|
||||
expect(result.out).toContain(".factory/roadmap.json")
|
||||
expect(result.out).toContain("es trust")
|
||||
expect(result.out).toContain("$15")
|
||||
expect(result.out).toContain("rewrite")
|
||||
expect(result.out).toContain('keep baseBranch "main"')
|
||||
const roadmap = JSON.parse(await readFile(factoryLayout(root).roadmap, "utf8"))
|
||||
expect(roadmap.baseBranch).toBe("rewrite")
|
||||
expect(roadmap.baseBranch).toBe("main")
|
||||
expect(await readFile(path.join(root, ".factory/notes/idea-137.md"), "utf8")).toContain("Stub issue")
|
||||
const entries = await recentAuditEntries(root, 5)
|
||||
const seeded = entries.find((entry) => entry.action === "factory.preset")!
|
||||
|
||||
@@ -9,8 +9,8 @@ a human reviews.
|
||||
- `gates.json` — the real merge gate: `bun install --frozen-lockfile`,
|
||||
`bun run check`, `bun run docs:check`. Trust them with `es trust`.
|
||||
- `idea-from-issue.sh` — pulls the issue body as the run idea.
|
||||
- Base branch: `rewrite`, never `main`. `es factory init` seeds
|
||||
`.factory/roadmap.json` with `baseBranch: rewrite`; keep that field when
|
||||
- Base branch: `main`. `es factory init` seeds
|
||||
`.factory/roadmap.json` with `baseBranch: main`; keep that field when
|
||||
the grill rewrites the roadmap.
|
||||
- Spend ceiling: $15 default, set on the frontier during the grill; the
|
||||
runtime cap is mandatory. See `docs/SELF-DOGFOOD.md`.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@heretek-ai/es-core",
|
||||
"version": "1.6.0",
|
||||
"version": "1.6.1",
|
||||
"esRelease": {
|
||||
"order": 1
|
||||
},
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// Pure verdict for the e2e regression gate (scripts/e2e-gate.ts, #210).
|
||||
// Fail-closed: a red run with zero sampled history exits 2 (distinct from
|
||||
// the exit-1 regression verdict, so humans can tell "no history" apart from
|
||||
// "no regression"), and failed IDs with no history at all (renamed or new
|
||||
// cases never sampled) count as regressions. Only a fully green run, or
|
||||
// failures confined to known-unstable cases, exits 0.
|
||||
export interface GateVerdictInput {
|
||||
/** Current-run failed case IDs. */
|
||||
readonly failed: readonly string[]
|
||||
/** Historically stable case IDs (>= 5 samples, >= 90% pass). */
|
||||
readonly stable: readonly string[]
|
||||
/** Number of prior runs actually sampled. */
|
||||
readonly sampled: number
|
||||
/** IDs seen in history (sampled at least once); defaults to `stable`. */
|
||||
readonly known?: readonly string[]
|
||||
}
|
||||
|
||||
export interface GateVerdict {
|
||||
/** 0 green, 1 regression, 2 red run with no history to judge it against. */
|
||||
readonly exit: 0 | 1 | 2
|
||||
readonly message: string
|
||||
}
|
||||
|
||||
export function decideGateVerdict(input: GateVerdictInput): GateVerdict {
|
||||
const { failed, stable, sampled } = input
|
||||
if (!failed.length) return { exit: 0, message: "no current failures; nothing to gate." }
|
||||
const known = new Set(input.known ?? stable)
|
||||
const stableSet = new Set(stable)
|
||||
if (sampled === 0)
|
||||
return {
|
||||
exit: 2,
|
||||
message:
|
||||
`fail-closed: sampled 0 prior run(s), so ${failed.length} current failure(s) ` +
|
||||
`cannot be judged against history (unknown IDs: ${failed.join(", ")}); refusing to green a red run.`,
|
||||
}
|
||||
const regressions = failed.filter((id) => stableSet.has(id) || !known.has(id))
|
||||
if (!regressions.length)
|
||||
return {
|
||||
exit: 0,
|
||||
message: `${failed.length} current failure(s) (${failed.join(", ")}) are not regressions of stable cases; passing.`,
|
||||
}
|
||||
const unknown = regressions.filter((id) => !known.has(id))
|
||||
const unknownNote = unknown.length ? ` (no history for: ${unknown.join(", ")})` : ""
|
||||
return {
|
||||
exit: 1,
|
||||
message: `REGRESSION in historically stable case(s): ${regressions.join(", ")}${unknownNote}`,
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,8 @@ const PRESETS = fileURLToPath(new URL("../../assets/presets/", import.meta.url))
|
||||
|
||||
/** The only preset so far: the factory builds this repo. */
|
||||
export const SELF_DOGFOOD_PRESET = "self-dogfood" as const
|
||||
/** Self-dogfood release PRs target `rewrite`, never `main`. */
|
||||
export const SELF_DOGFOOD_BASE_BRANCH = "rewrite" as const
|
||||
/** Self-dogfood release PRs target `main` (post-1.6.0 cutover, #281). */
|
||||
export const SELF_DOGFOOD_BASE_BRANCH = "main" as const
|
||||
/** Default spend ceiling (USD) the runbook sets on the grill frontier. */
|
||||
export const SELF_DOGFOOD_SPEND_CEILING_USD = 15 as const
|
||||
|
||||
@@ -139,8 +139,8 @@ export const readPresetSeed = (root: string): Promise<PresetSeed | undefined> =>
|
||||
|
||||
/**
|
||||
* The single argv builder every PR opener uses for `gh pr create --draft`.
|
||||
* For a preset-seeded run the base is pinned: `--base main` (or anything
|
||||
* other than the preset's base) is refused here, before anything is pushed.
|
||||
* For a preset-seeded run the base is pinned: anything other than the
|
||||
* preset's base is refused here, before anything is pushed.
|
||||
*/
|
||||
export function buildPrArgs(options: {
|
||||
branch: string
|
||||
@@ -151,7 +151,7 @@ export function buildPrArgs(options: {
|
||||
}): string[] {
|
||||
if (options.preset === SELF_DOGFOOD_PRESET && options.base !== SELF_DOGFOOD_BASE_BRANCH)
|
||||
throw new Error(
|
||||
`refusing --base "${options.base}" for a preset "${SELF_DOGFOOD_PRESET}" run (release PRs target "${SELF_DOGFOOD_BASE_BRANCH}", never "main")`,
|
||||
`refusing --base "${options.base}" for a preset "${SELF_DOGFOOD_PRESET}" run (release PRs target "${SELF_DOGFOOD_BASE_BRANCH}")`,
|
||||
)
|
||||
return [
|
||||
"gh",
|
||||
|
||||
@@ -9,6 +9,7 @@ import path from "node:path"
|
||||
import { type HumanSigner, type RecordSignature, verifyRecordSignature } from "../approval/keystore.ts"
|
||||
import { stateDir } from "../layout.ts"
|
||||
import type { CommandCheck } from "../schema/gates.ts"
|
||||
import { CommandCheckSchema } from "../schema/gates.ts"
|
||||
import { readJson, withLock, writeJson } from "../util/fs.ts"
|
||||
import { hashJson } from "../util/hash.ts"
|
||||
import { canonicalPath } from "./paths.ts"
|
||||
@@ -31,12 +32,26 @@ interface TrustFile {
|
||||
|
||||
const trustFile = (dir?: string) => path.join(dir ?? stateDir(), "trust.json")
|
||||
|
||||
/** Hash of the gate command set and the package.json scripts it can reach. */
|
||||
/** Hash of the gate command set and the package.json scripts it can reach.
|
||||
*
|
||||
* The hash covers the full normalized `CommandCheck` (post-`CommandCheckSchema`
|
||||
* defaults): `kind`, `related`, `touchedArgs`/`extensions`, `timeoutMs` and
|
||||
* `parser` change what runs, so flipping any of them revokes trust (#217).
|
||||
* Bare `{id, command}` inputs normalize to the same hash as their
|
||||
* schema-defaulted form (explicit defaults hash identically to omitted
|
||||
* ones). The hash shape itself changed in 1.6.1, so approvals recorded
|
||||
* under the old `[id, command]`-only hash do not carry over: re-approval
|
||||
* is required, and the gates refuse to run until it happens.
|
||||
*/
|
||||
export async function commandSetHash(
|
||||
dir: string,
|
||||
commands: readonly CommandCheck[],
|
||||
commands: readonly (CommandCheck | Parameters<typeof CommandCheckSchema.parse>[0])[],
|
||||
): Promise<{ hash: string; lines: string[] }> {
|
||||
const lines = commands.map((item) => `${item.id}: ${item.command}`)
|
||||
const normalized = commands.map((item) => CommandCheckSchema.parse(item))
|
||||
const lines = normalized.map(
|
||||
(item) =>
|
||||
`${item.id}: ${item.command} [kind=${item.kind} parser=${item.parser} related=${item.related} timeoutMs=${item.timeoutMs}${item.touchedArgs ? ` touchedArgs extensions=${item.extensions.join(",") || "*"}` : ""}]`,
|
||||
)
|
||||
let scripts: Record<string, string> = {}
|
||||
try {
|
||||
scripts = JSON.parse(await readFile(path.join(dir, "package.json"), "utf8")).scripts ?? {}
|
||||
@@ -50,10 +65,10 @@ export async function commandSetHash(
|
||||
return name === "test" && /^(npm|pnpm|yarn)\s+test\b/.test(command)
|
||||
}
|
||||
const reachable = Object.fromEntries(
|
||||
Object.entries(scripts).filter(([name]) => commands.some((item) => invokes(item.command, name))),
|
||||
Object.entries(scripts).filter(([name]) => normalized.some((item) => invokes(item.command, name))),
|
||||
)
|
||||
for (const [name, body] of Object.entries(reachable)) lines.push(` script ${name}: ${body}`)
|
||||
return { hash: hashJson({ commands: commands.map((item) => [item.id, item.command]), scripts: reachable }), lines }
|
||||
return { hash: hashJson({ commands: normalized, scripts: reachable }), lines }
|
||||
}
|
||||
|
||||
async function read(dir?: string): Promise<TrustFile> {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
// E2E regression gate verdict (#210): the gate must fail closed. A red run
|
||||
// with zero sampled history (expired artifacts) previously exited 0, and
|
||||
// renamed/never-sampled case IDs were never in `stable`, so a fully-red
|
||||
// renamed suite also exited 0. Unknown IDs are regressions; zero history
|
||||
// with failures is a distinct exit 2 ("no history", not "no regression").
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { decideGateVerdict } from "../src/evals/gate.ts"
|
||||
|
||||
describe("e2e gate verdict (#210)", () => {
|
||||
test("a fully green run stays green", () => {
|
||||
expect(decideGateVerdict({ failed: [], stable: [], sampled: 0 }).exit).toBe(0)
|
||||
expect(decideGateVerdict({ failed: [], stable: ["a"], sampled: 6 }).exit).toBe(0)
|
||||
})
|
||||
|
||||
test("zero sampled history with failures fails closed with exit 2", () => {
|
||||
const verdict = decideGateVerdict({ failed: ["case-a", "case-b"], stable: [], sampled: 0 })
|
||||
expect(verdict.exit).toBe(2)
|
||||
expect(verdict.message).toContain("sampled 0")
|
||||
expect(verdict.message).toContain("case-a")
|
||||
})
|
||||
|
||||
test("unknown (never-sampled) failed IDs are regressions", () => {
|
||||
const verdict = decideGateVerdict({ failed: ["renamed-case"], stable: ["old-case"], sampled: 6 })
|
||||
expect(verdict.exit).toBe(1)
|
||||
expect(verdict.message).toContain("renamed-case")
|
||||
})
|
||||
|
||||
test("a stable case failing now is a regression with exit 1", () => {
|
||||
const verdict = decideGateVerdict({ failed: ["stable-case"], stable: ["stable-case"], sampled: 6 })
|
||||
expect(verdict.exit).toBe(1)
|
||||
expect(verdict.message).toContain("stable-case")
|
||||
})
|
||||
|
||||
test("known-unstable failures still pass (regressions-only gate)", () => {
|
||||
const verdict = decideGateVerdict({
|
||||
failed: ["flaky-case"],
|
||||
stable: ["stable-case"],
|
||||
sampled: 6,
|
||||
known: ["stable-case", "flaky-case"],
|
||||
})
|
||||
expect(verdict.exit).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -29,10 +29,9 @@ describe("self-dogfood preset integration (#205)", () => {
|
||||
{ issue: 205, title: "Self-dogfood preset", body: "The factory builds this repo." },
|
||||
{ actor: "human:tester" },
|
||||
)
|
||||
// Current behavior (#281 owns the base-selection question): the seed
|
||||
// pins the release base to `rewrite`.
|
||||
expect(baseBranch).toBe("rewrite")
|
||||
expect(SELF_DOGFOOD_BASE_BRANCH).toBe("rewrite")
|
||||
// Post-cutover (#281): the seed pins the release base to `main`.
|
||||
expect(baseBranch).toBe("main")
|
||||
expect(SELF_DOGFOOD_BASE_BRANCH).toBe("main")
|
||||
expect(written).toEqual([
|
||||
".factory/config.json",
|
||||
".factory/gates.json",
|
||||
@@ -50,7 +49,7 @@ describe("self-dogfood preset integration (#205)", () => {
|
||||
"bun run docs:check",
|
||||
])
|
||||
const roadmap = RoadmapSchema.parse(JSON.parse(await readFile(layout.roadmap, "utf8")))
|
||||
expect(roadmap.baseBranch).toBe("rewrite")
|
||||
expect(roadmap.baseBranch).toBe("main")
|
||||
expect(roadmap.title).toContain("205")
|
||||
expect(await readFile(`${fx.root}/.factory/notes/idea-205.md`, "utf8")).toContain("# #205:")
|
||||
} finally {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Self-dogfood preset (#137): the factory builds this repo. Preset files
|
||||
// validate against their schemas, init seeds the run with the release base
|
||||
// pinned to `rewrite`, and the PR argv never names `main`.
|
||||
// pinned to `main`, and the PR argv always names that base.
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { access, mkdir, mkdtemp, readFile, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
@@ -48,7 +48,7 @@ describe("self-dogfood preset files", () => {
|
||||
})
|
||||
|
||||
describe("initFromPreset", () => {
|
||||
test("seeds control files, a rewrite-based roadmap and the issue idea", async () => {
|
||||
test("seeds control files, a main-based roadmap and the issue idea", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "es-preset-"))
|
||||
try {
|
||||
const { written, baseBranch } = await initFromPreset(root, "self-dogfood", {
|
||||
@@ -56,7 +56,7 @@ describe("initFromPreset", () => {
|
||||
title: "Self-dogfood preset",
|
||||
body: "The factory builds this repo.",
|
||||
})
|
||||
expect(baseBranch).toBe("rewrite")
|
||||
expect(baseBranch).toBe("main")
|
||||
expect(written).toEqual([
|
||||
".factory/config.json",
|
||||
".factory/gates.json",
|
||||
@@ -64,7 +64,7 @@ describe("initFromPreset", () => {
|
||||
".factory/notes/idea-137.md",
|
||||
])
|
||||
const layout = factoryLayout(root)
|
||||
expect(RoadmapSchema.parse(JSON.parse(await readFile(layout.roadmap, "utf8"))).baseBranch).toBe("rewrite")
|
||||
expect(RoadmapSchema.parse(JSON.parse(await readFile(layout.roadmap, "utf8"))).baseBranch).toBe("main")
|
||||
expect(GatesConfigSchema.parse(JSON.parse(await readFile(layout.gates, "utf8"))).commands).toHaveLength(3)
|
||||
expect(await readFile(path.join(root, ".factory/notes/idea-137.md"), "utf8")).toContain(
|
||||
"# #137: Self-dogfood preset",
|
||||
@@ -76,12 +76,12 @@ describe("initFromPreset", () => {
|
||||
})
|
||||
|
||||
describe("self-dogfood PR base", () => {
|
||||
test("the draft PR targets rewrite, never main", () => {
|
||||
expect(SELF_DOGFOOD_BASE_BRANCH).toBe("rewrite")
|
||||
const argv = buildPrArgs({ branch: "factory/x/integration", base: "rewrite", title: "t", body: "b" })
|
||||
test("the draft PR targets main", () => {
|
||||
expect(SELF_DOGFOOD_BASE_BRANCH).toBe("main")
|
||||
const argv = buildPrArgs({ branch: "factory/x/integration", base: "main", title: "t", body: "b" })
|
||||
expect(argv).toContain("--draft")
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["rewrite"])
|
||||
expect(argv).not.toContain("main")
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["main"])
|
||||
expect(argv).not.toContain("rewrite")
|
||||
})
|
||||
})
|
||||
|
||||
@@ -164,22 +164,25 @@ describe("preset-seeded runs pin the release base (S2.2)", () => {
|
||||
}
|
||||
expect(typeof core.buildPrArgs).toBe("function")
|
||||
expect(core.selfDogfoodPrArgs).toBeUndefined()
|
||||
const argv = core.buildPrArgs!({ branch: "factory/x/integration", base: "rewrite", title: "t", body: "b" })
|
||||
const argv = core.buildPrArgs!({ branch: "factory/x/integration", base: "main", title: "t", body: "b" })
|
||||
expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"])
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["rewrite"])
|
||||
expect(argv).not.toContain("main")
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["main"])
|
||||
expect(argv).not.toContain("rewrite")
|
||||
})
|
||||
|
||||
test("the shared argv refuses --base main for a preset run", async () => {
|
||||
test("the shared argv refuses a non-main base for a preset run", async () => {
|
||||
const { buildPrArgs } = (await import("../src/index.ts")) as unknown as {
|
||||
buildPrArgs: (options: { branch: string; base: string; title: string; body: string; preset?: string }) => string[]
|
||||
}
|
||||
expect(() =>
|
||||
buildPrArgs({ branch: "factory/x/integration", base: "main", title: "t", body: "b", preset: "self-dogfood" }),
|
||||
).toThrow(/refusing.*main/)
|
||||
buildPrArgs({ branch: "factory/x/integration", base: "rewrite", title: "t", body: "b", preset: "self-dogfood" }),
|
||||
).toThrow(/refusing.*rewrite/)
|
||||
expect(() =>
|
||||
buildPrArgs({ branch: "factory/x/integration", base: "other", title: "t", body: "b", preset: "self-dogfood" }),
|
||||
).toThrow(/rewrite/)
|
||||
).toThrow(/release PRs target "main"/)
|
||||
expect(
|
||||
buildPrArgs({ branch: "factory/x/integration", base: "main", title: "t", body: "b", preset: "self-dogfood" }),
|
||||
).toContain("--draft")
|
||||
})
|
||||
|
||||
test("the seed marker is part of the run-state schema, not a field the parse strips", () => {
|
||||
@@ -205,7 +208,7 @@ describe("a preset-seeded run end to end (S2.2)", () => {
|
||||
const { gitRepo, frontier, writeReport, writeSpecs } = await import("./helpers.ts")
|
||||
const { Factory, recordApproval, sealHumanKey, unlockHumanKey } = await import("../src/index.ts")
|
||||
const fx = await gitRepo("es-preset-e2e-")
|
||||
await Bun.spawn(["git", "branch", "rewrite"], { cwd: fx.root }).exited
|
||||
await Bun.spawn(["git", "branch", "main"], { cwd: fx.root }).exited
|
||||
await sealHumanKey(PASSPHRASE, fx.state)
|
||||
const signer = await unlockHumanKey(PASSPHRASE, fx.state)
|
||||
await initFromPreset(
|
||||
@@ -239,13 +242,13 @@ describe("a preset-seeded run end to end (S2.2)", () => {
|
||||
// The grill kept baseBranch when it rewrote the roadmap (before approval).
|
||||
const roadmapFile = factoryLayout(fx.root).roadmap
|
||||
const roadmap = JSON.parse(await readFile(roadmapFile, "utf8"))
|
||||
await Bun.write(roadmapFile, `${JSON.stringify({ ...roadmap, baseBranch: "rewrite" }, null, 2)}\n`)
|
||||
await Bun.write(roadmapFile, `${JSON.stringify({ ...roadmap, baseBranch: "main" }, null, 2)}\n`)
|
||||
}
|
||||
await recordApproval(fx.root, { stage: "spec", channel: "cli", approvedBy: "tester", signer })
|
||||
return { fx, factory, opened, begun, writeFrontier }
|
||||
}
|
||||
|
||||
test("a preset-seeded run releases its draft PR on rewrite", async () => {
|
||||
test("a preset-seeded run releases its draft PR on main", async () => {
|
||||
const { fx, factory, opened } = await seedAndReachSpec({ keepBaseBranch: true })
|
||||
try {
|
||||
const { buildPrArgs } = (await import("../src/index.ts")) as unknown as {
|
||||
@@ -258,7 +261,7 @@ describe("a preset-seeded run end to end (S2.2)", () => {
|
||||
}) => string[]
|
||||
}
|
||||
const built = await factory.startBuild("factory")
|
||||
expect(built.baseBranch).toBe("rewrite")
|
||||
expect(built.baseBranch).toBe("main")
|
||||
const dir = (await factory.activeWorktree())!
|
||||
await mkdir(path.join(dir, "test"), { recursive: true })
|
||||
await mkdir(path.join(dir, "src"), { recursive: true })
|
||||
@@ -276,12 +279,12 @@ describe("a preset-seeded run end to end (S2.2)", () => {
|
||||
const done = await factory.release("factory")
|
||||
expect(done.stage).toBe("DONE")
|
||||
expect(opened).toHaveLength(1)
|
||||
expect(opened[0]?.base).toBe("rewrite")
|
||||
expect(opened[0]?.base).toBe("main")
|
||||
expect(opened[0]?.preset).toBe("self-dogfood")
|
||||
const argv = buildPrArgs({ ...opened[0]! })
|
||||
expect(argv).toContain("--draft")
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["rewrite"])
|
||||
expect(argv).not.toContain("main")
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["main"])
|
||||
expect(argv).not.toContain("rewrite")
|
||||
} finally {
|
||||
await fx.cleanup()
|
||||
}
|
||||
@@ -295,7 +298,7 @@ describe("a preset-seeded run end to end (S2.2)", () => {
|
||||
expect(roadmap.baseBranch).toBeUndefined()
|
||||
await expect(factory.startBuild("factory")).rejects.toThrow(/baseBranch/)
|
||||
const state = await factory.read()
|
||||
expect(state?.baseBranch ?? "rewrite").toBe("rewrite")
|
||||
expect(state?.baseBranch ?? "main").toBe("main")
|
||||
} finally {
|
||||
await fx.cleanup()
|
||||
}
|
||||
|
||||
@@ -169,7 +169,9 @@ describe("trust", () => {
|
||||
const foreignSigner = await unlockHumanKey(PASSPHRASE, foreign)
|
||||
const retargeted = {
|
||||
hash: forgedHash,
|
||||
lines: ["probe: touch FORGED"],
|
||||
// The real display lines for the forged set (#217: the full normalized
|
||||
// check, not just `id: command`), so the forgery shapes match production.
|
||||
lines: (await commandSetHash(fx.root, wanted.commands)).lines,
|
||||
approvedBy: "x",
|
||||
approvedAt: entry.approvedAt,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// Stage-gating claim wording (#223.1): the "every tool refuses an
|
||||
// out-of-order step" claim is overbroad as worded — many tools are
|
||||
// intentionally stage-free (seat-gated, not stage-gated). The claim must be
|
||||
// narrowed to the phase-transition tools. The pre-QA audit-tiebreak design
|
||||
// itself is kept (pinned by codeaudit.test.ts: the manager breaks a split
|
||||
// verdict outside QA); only the wording changes here.
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { readFile } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
|
||||
const root = path.resolve(import.meta.dir, "../../..")
|
||||
const text = (rel: string) => readFile(path.join(root, rel), "utf8")
|
||||
|
||||
describe("stage-gating claim wording (#223.1)", () => {
|
||||
test("README narrows the refusal claim to phase-transition tools", async () => {
|
||||
const readme = await text("README.md")
|
||||
expect(readme).not.toContain("every `es_*` tool refuses an out-of-order step")
|
||||
expect(readme).toContain("phase-transition")
|
||||
})
|
||||
|
||||
test("the factory prompt already scopes refusal to phase-transition tools", async () => {
|
||||
const prompt = await text("packages/core/assets/prompts/factory.md")
|
||||
expect(prompt).toContain("phase-transition tools refuse an out-of-order step")
|
||||
})
|
||||
|
||||
test("SYSTEM_ARCHITECTURE scopes refusal to transitions", async () => {
|
||||
const arch = await text("SYSTEM_ARCHITECTURE.md")
|
||||
expect(arch).toContain("refuses out-of-order steps")
|
||||
expect(arch).not.toContain("every `es_*` tool refuses")
|
||||
})
|
||||
})
|
||||
@@ -206,6 +206,54 @@ describe("waivers and trust at gate time (#221 gaps)", () => {
|
||||
}, 60_000)
|
||||
})
|
||||
|
||||
describe("commandSetHash covers the full normalized CommandCheck (#217)", () => {
|
||||
const dirOf = () => mkdtemp(path.join(tmpdir(), "es-command-hash-"))
|
||||
type Variant = Parameters<typeof commandSetHash>[1][number]
|
||||
const base = { id: "probe", kind: "test", command: "touch RAN" } as const
|
||||
const table: [string, Variant][] = [
|
||||
["kind", { ...base, kind: "lint" }],
|
||||
["parser", { ...base, parser: "tsc" }],
|
||||
["touchedArgs", { ...base, touchedArgs: true }],
|
||||
["extensions", { ...base, touchedArgs: true, extensions: [".ts"] }],
|
||||
["related", { ...base, related: "vitest" }],
|
||||
["timeoutMs", { ...base, timeoutMs: 60_000 }],
|
||||
]
|
||||
test.each(table)("flipping %s revokes trust", async (_field: string, variant: Variant) => {
|
||||
const dir = await dirOf()
|
||||
try {
|
||||
const before = (await commandSetHash(dir, [base])).hash
|
||||
const after = (await commandSetHash(dir, [variant])).hash
|
||||
expect(after).not.toBe(before)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test("a bare id+command hashes the same as its schema-normalized form", async () => {
|
||||
const dir = await dirOf()
|
||||
try {
|
||||
const bare = (await commandSetHash(dir, [base])).hash
|
||||
const normalized = (
|
||||
await commandSetHash(dir, [
|
||||
{
|
||||
id: "probe",
|
||||
kind: "test",
|
||||
command: "touch RAN",
|
||||
parser: "generic",
|
||||
touchedArgs: false,
|
||||
extensions: [],
|
||||
related: "none",
|
||||
timeoutMs: 300_000,
|
||||
},
|
||||
])
|
||||
).hash
|
||||
expect(normalized).toBe(bare)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe("agent shells and sealed state through the host (#181 §2)", () => {
|
||||
test("an agent shell running human-gated verbs is refused", async () => {
|
||||
for (const command of ["es trust", "es waive lint/check --reason x", "es reseal --sign", "es rebaseline"]) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@heretek-ai/epistemic-swarm",
|
||||
"version": "1.6.0",
|
||||
"version": "1.6.1",
|
||||
"esRelease": {
|
||||
"order": 3
|
||||
},
|
||||
|
||||
@@ -18,7 +18,7 @@ const open = (root: string, overrides: { branch: string; base?: string; preset?:
|
||||
ghPrOpener.open({
|
||||
root,
|
||||
branch: overrides.branch,
|
||||
base: overrides.base ?? "rewrite",
|
||||
base: overrides.base ?? "main",
|
||||
title: "t",
|
||||
body: "b",
|
||||
...(overrides.preset !== undefined ? { preset: overrides.preset } : {}),
|
||||
@@ -44,7 +44,7 @@ describe("ghPrOpener.open() branch guard", () => {
|
||||
test("branch === base is refused", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-"))
|
||||
try {
|
||||
await expect(open(root, { branch: "rewrite", base: "rewrite" })).rejects.toThrow(/refusing to push "rewrite"/)
|
||||
await expect(open(root, { branch: "main", base: "main" })).rejects.toThrow(/refusing to push "main"/)
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
@@ -67,7 +67,7 @@ describe("ghPrOpener.open() branch guard", () => {
|
||||
// (`pr.ts` checks the branch before `git push --set-upstream`).
|
||||
const root = await mkdtemp(path.join(tmpdir(), "es-pr-guard-"))
|
||||
try {
|
||||
await open(root, { branch: "main", base: "rewrite" })
|
||||
await open(root, { branch: "main", base: "main" })
|
||||
expect.unreachable("a non-factory/ branch must be refused")
|
||||
} catch (error) {
|
||||
expect((error as Error).message).toMatch(/refusing to push "main"/)
|
||||
@@ -118,9 +118,9 @@ describe("ghPrOpener.open() branch guard", () => {
|
||||
|
||||
describe("release PR argv", () => {
|
||||
test("a factory branch opens a draft PR against the base", () => {
|
||||
const argv = buildPrArgs({ branch: "factory/run-1/integration", base: "rewrite", title: "t", body: "b" })
|
||||
const argv = buildPrArgs({ branch: "factory/run-1/integration", base: "main", title: "t", body: "b" })
|
||||
expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"])
|
||||
expect(argv.slice(argv.indexOf("--head") + 1, argv.indexOf("--head") + 2)).toEqual(["factory/run-1/integration"])
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["rewrite"])
|
||||
expect(argv.slice(argv.indexOf("--base") + 1, argv.indexOf("--base") + 2)).toEqual(["main"])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// Issue #205 opener half: `ghPrOpener.open` with `preset: self-dogfood` and
|
||||
// `--base main` refuses before anything is pushed. Local bare remote only, no
|
||||
// network and no `gh` invocation: the shared argv builder throws first, so the
|
||||
// test asserts the bare remote holds no new ref afterwards.
|
||||
// a non-`main` base refuses before anything is pushed (#281 retargeted the
|
||||
// preset base to `main`). Local bare remote only, no network and no `gh`
|
||||
// invocation: the shared argv builder throws first, so the test asserts the
|
||||
// bare remote holds no new ref afterwards.
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { mkdtemp, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
@@ -17,8 +18,8 @@ const sh = async (args: string[], cwd: string) => {
|
||||
const showRefs = async (bare: string, cwd: string) =>
|
||||
(await run(["git", "--git-dir", bare, "show-ref"], { cwd })).stdout.trim()
|
||||
|
||||
describe("self-dogfood opener refuses --base main without pushing (#205)", () => {
|
||||
test("preset run against main is refused and the remote is unchanged", async () => {
|
||||
describe("self-dogfood opener refuses a non-main base without pushing (#205, #281)", () => {
|
||||
test("preset run against the old base is refused and the remote is unchanged", async () => {
|
||||
const bare = await mkdtemp(path.join(tmpdir(), "es-205-bare-"))
|
||||
const repo = await mkdtemp(path.join(tmpdir(), "es-205-repo-"))
|
||||
try {
|
||||
@@ -44,12 +45,12 @@ describe("self-dogfood opener refuses --base main without pushing (#205)", () =>
|
||||
ghPrOpener.open({
|
||||
root: repo,
|
||||
branch: "factory/run-205/integration",
|
||||
base: "main",
|
||||
base: "rewrite",
|
||||
title: "dogfood",
|
||||
body: "body",
|
||||
preset: "self-dogfood",
|
||||
}),
|
||||
).rejects.toThrow(/refusing.*main|rewrite/)
|
||||
).rejects.toThrow(/refusing.*rewrite/)
|
||||
|
||||
// Refusal happened before the push: the bare remote is byte-identical.
|
||||
const after = await showRefs(bare, repo)
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
|
||||
import { ghPrOpener } from "../src/pr.ts"
|
||||
|
||||
const BASE = "rewrite"
|
||||
const BASE = "main"
|
||||
const BRANCH = "factory/run-test-1/integration"
|
||||
|
||||
interface Remote {
|
||||
@@ -81,8 +81,8 @@ describe("release against a local bare remote", () => {
|
||||
await expect(ghPrOpener.open({ root: fx.root, branch: BASE, base: BASE, title: "t", body: "b" })).rejects.toThrow(
|
||||
`refusing to push "${BASE}"`,
|
||||
)
|
||||
expect(await remoteBranches(fx.remote)).toEqual(["refs/heads/rewrite"])
|
||||
expect(await remoteSha(fx.remote, "refs/heads/rewrite")).toBe(fx.baseSha)
|
||||
expect(await remoteBranches(fx.remote)).toEqual(["refs/heads/main"])
|
||||
expect(await remoteSha(fx.remote, "refs/heads/main")).toBe(fx.baseSha)
|
||||
} finally {
|
||||
await fx.cleanup()
|
||||
}
|
||||
@@ -94,8 +94,8 @@ describe("release against a local bare remote", () => {
|
||||
await expect(
|
||||
ghPrOpener.open({ root: fx.root, branch: "feature/x", base: BASE, title: "t", body: "b" }),
|
||||
).rejects.toThrow(/refusing to push/)
|
||||
expect(await remoteBranches(fx.remote)).toEqual(["refs/heads/rewrite"])
|
||||
expect(await remoteSha(fx.remote, "refs/heads/rewrite")).toBe(fx.baseSha)
|
||||
expect(await remoteBranches(fx.remote)).toEqual(["refs/heads/main"])
|
||||
expect(await remoteSha(fx.remote, "refs/heads/main")).toBe(fx.baseSha)
|
||||
} finally {
|
||||
await fx.cleanup()
|
||||
}
|
||||
@@ -109,10 +109,10 @@ describe("release against a local bare remote", () => {
|
||||
// runs in tests: no `gh` invocation here).
|
||||
await git(fx.root, ["push", "--set-upstream", "origin", `${BRANCH}:${BRANCH}`])
|
||||
expect(await remoteBranches(fx.remote)).toEqual(
|
||||
["refs/heads/factory/run-test-1/integration", "refs/heads/rewrite"].sort(),
|
||||
["refs/heads/factory/run-test-1/integration", "refs/heads/main"].sort(),
|
||||
)
|
||||
expect(await remoteSha(fx.remote, `refs/heads/${BRANCH}`)).toBe(local)
|
||||
expect(await remoteSha(fx.remote, "refs/heads/rewrite")).toBe(fx.baseSha)
|
||||
expect(await remoteSha(fx.remote, "refs/heads/main")).toBe(fx.baseSha)
|
||||
// The release PR itself is a draft against the base.
|
||||
const argv = buildPrArgs({ branch: BRANCH, base: BASE, title: "t", body: "b" })
|
||||
expect(argv.slice(0, 4)).toEqual(["gh", "pr", "create", "--draft"])
|
||||
|
||||
+7
-8
@@ -13,6 +13,7 @@
|
||||
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import path from "node:path"
|
||||
import { decideGateVerdict } from "../packages/core/src/evals/gate.ts"
|
||||
|
||||
const root = path.resolve(import.meta.dir, "..")
|
||||
const resultsDir = path.join(root, "evals", "results")
|
||||
@@ -74,7 +75,7 @@ const ghOk = (args: string[]) => {
|
||||
}
|
||||
}
|
||||
|
||||
const branch = process.env.GITHUB_REF_NAME ?? "rewrite"
|
||||
const branch = process.env.GITHUB_REF_NAME ?? "main"
|
||||
const repo = process.env.GITHUB_REPOSITORY
|
||||
const repoArgs = repo ? ["--repo", repo] : []
|
||||
const listed = ghOut([
|
||||
@@ -139,13 +140,11 @@ try {
|
||||
.filter(([, cell]) => cell.samples >= 5 && cell.passes / cell.samples >= 0.9)
|
||||
.map(([id]) => id)
|
||||
console.log(`e2e-gate: sampled ${sampled} prior run(s); stable cases: ${stable.length ? stable.join(", ") : "(none yet)"}`)
|
||||
const regressions = failed.filter((id) => stable.includes(id))
|
||||
if (!regressions.length) {
|
||||
console.log(`e2e-gate: ${failed.length} current failure(s) (${failed.join(", ")}) are not regressions of stable cases; passing.`)
|
||||
process.exit(0)
|
||||
}
|
||||
console.log(`e2e-gate: REGRESSION in historically stable case(s): ${regressions.join(", ")}`)
|
||||
process.exit(1)
|
||||
// Fail-closed (#210): zero sampled history with failures exits 2 (not
|
||||
// green), and failed IDs never seen in history count as regressions.
|
||||
const verdict = decideGateVerdict({ failed, stable, sampled, known: [...tally.keys()] })
|
||||
console.log(`e2e-gate: ${verdict.message}`)
|
||||
process.exit(verdict.exit)
|
||||
} finally {
|
||||
await rm(scratch, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user