diff --git a/docs/SCHEMAS.md b/docs/SCHEMAS.md index 6d30617..d008e11 100644 --- a/docs/SCHEMAS.md +++ b/docs/SCHEMAS.md @@ -46,6 +46,7 @@ Do not edit by hand: run `bun run docs:gen` after changing a schema (CI runs `bu | frontier | `schemas/frontier.schema.json` | | gate-budgets | `schemas/gate-budgets.schema.json` | | gate-finding | `schemas/gate-finding.schema.json` | +| gate-mutation | `schemas/gate-mutation.schema.json` | | gate-rejection | `schemas/gate-rejection.schema.json` | | gate-security | `schemas/gate-security.schema.json` | | gates-config | `schemas/gates-config.schema.json` | diff --git a/packages/core/src/gates/engine.ts b/packages/core/src/gates/engine.ts index 04a52b6..6d69529 100644 --- a/packages/core/src/gates/engine.ts +++ b/packages/core/src/gates/engine.ts @@ -28,8 +28,17 @@ import { parseJsonc } from "../util/jsonc.ts" import { bwrapAvailable, findExecutable, type RunResult, run, splitCommand } from "../util/proc.ts" import { changedFiles, diffStats, git } from "../worktree/git.ts" import { validateArtifacts } from "./artifacts.ts" -import { checkBudgets, isTestFile } from "./budgets.ts" +import { checkBudgets, isSourceFile, isTestFile } from "./budgets.ts" import { detectGates } from "./detect.ts" +import { + cacheHashesEqual, + evaluateMutations, + generateMutants, + type MutationCacheState, + type MutationOutcome, + mutationFindings, + mutationFingerprint, +} from "./mutation.ts" import { parseOutput } from "./parsers.ts" import { scanFileForSecrets } from "./secrets.ts" import { DEPENDENCY_FILES, gitleaks, osvFindings } from "./security.ts" @@ -335,6 +344,173 @@ async function runCommandCheck( } } +/** Sources the mutation gate can mutate: JS/TS (the operators are JS-shaped). */ +const MUTATABLE = [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"] + +const mutationCacheFile = (stateDir: string) => path.join(stateDir, "mutation-cache.json") + +/** + * The AST mutation check (#169). Each eligible touched source is mutated + * (condition/arithmetic/call/return mutants, capped per file) and the test + * checks rerun per mutant with the file restored afterwards. A mutant the + * suite still passes survived; below-threshold kill rates fail closed with a + * `mutation/kill-rate` error. Files whose content hash matches the cache + * reuse their stored outcome without rerunning the suite. + */ +/** Cached verdicts replayed as outcomes without touching the suite. */ +function cachedMutationOutcomes( + file: string, + source: string, + entry: { killed: readonly string[]; total: number }, +): MutationOutcome[] { + const killed = new Set(entry.killed) + const outcomes: MutationOutcome[] = [] + for (let index = 0; index < entry.total; index++) + outcomes.push({ + mutant: { + id: `${file}#${index + 1}`, + file, + line: 1, + kind: "condition", + description: "cached", + mutated: source, + }, + killed: killed.has(`${file}#${index + 1}`), + }) + return outcomes +} + +interface MutantRun { + readonly killed: boolean + readonly error?: string +} + +/** + * Apply one mutant, run every test suite until one fails, then restore the + * source. A failing suite kills the mutant; a runner error is reported, not + * mistaken for a kill. + */ +async function applyMutantAndRun( + context: { + dir: string + root: string + scope: "touched" | "full" + logDir: string + sandbox: "required" | "preferred" + stateDir: string + signal?: AbortSignal + }, + file: string, + absolute: string, + source: string, + mutant: { id: string; mutated: string }, + suites: readonly CommandCheck[], +): Promise { + await writeFile(absolute, mutant.mutated) + try { + for (const suite of suites) { + const outcome = await runCommandCheck(suite, { + dir: context.dir, + root: context.root, + scope: context.scope, + files: [file], + logDir: context.logDir, + sandbox: context.sandbox, + stateDir: context.stateDir, + ...(context.signal ? { signal: context.signal } : {}), + }) + if (outcome.report.status === "fail") return { killed: true } + if (outcome.report.status === "error") + return { killed: false, error: `${mutant.id}: ${outcome.report.note ?? "runner error"}` } + } + return { killed: false } + } finally { + await writeFile(absolute, source) + } +} + +async function runMutationCheck( + check: { + dir: string + root: string + scope: "touched" | "full" + files: readonly string[] + logDir: string + sandbox: "required" | "preferred" + stateDir: string + signal?: AbortSignal + }, + config: GatesConfig, +): Promise { + const eligible = check.files.filter((file) => isSourceFile(file) && MUTATABLE.some((ext) => file.endsWith(ext))) + const suites = config.commands.filter((item) => item.kind === "test") + if (!eligible.length || !suites.length) return undefined + let cache: MutationCacheState = {} + try { + cache = JSON.parse(await readFile(mutationCacheFile(check.stateDir), "utf8")) as MutationCacheState + } catch { + cache = {} + } + const next: Record = { ...cache } + const outcomes: MutationOutcome[] = [] + const runnerErrors: string[] = [] + for (const file of eligible) { + const absolute = path.join(check.dir, file) + const source = await readFile(absolute, "utf8").catch(() => undefined) + if (source === undefined) continue + const hash = await mutationFingerprint( + (candidate) => readFile(path.join(check.dir, candidate), "utf8").catch(() => undefined), + file, + source, + check.files.filter((candidate) => isTestFile(candidate)), + ) + const entry = next[file] + if (entry && cacheHashesEqual(entry.hash, hash)) { + outcomes.push(...cachedMutationOutcomes(file, source, entry)) + continue + } + const killedIds: string[] = [] + const mutants = generateMutants(file, source, config.mutation.maxMutants) + for (const [position, mutant] of mutants.entries()) { + const mutantLogDir = path.join(check.logDir, "mutation", `${position + 1}`) + await mkdir(mutantLogDir, { recursive: true }) + const result = await applyMutantAndRun( + { + dir: check.dir, + root: check.root, + scope: check.scope, + logDir: mutantLogDir, + sandbox: check.sandbox, + stateDir: check.stateDir, + ...(check.signal ? { signal: check.signal } : {}), + }, + file, + absolute, + source, + mutant, + suites, + ) + outcomes.push({ mutant, killed: result.killed }) + if (result.killed) killedIds.push(mutant.id) + if (result.error) runnerErrors.push(result.error) + } + next[file] = { hash, killed: killedIds, total: mutants.length } + } + await writeFile(mutationCacheFile(check.stateDir), JSON.stringify(next, null, 2)).catch(() => undefined) + const evaluated = evaluateMutations(outcomes, config.mutation.threshold) + const findings = mutationFindings({ ...evaluated, threshold: config.mutation.threshold }, eligible) + for (const error of runnerErrors.slice(0, 3)) + findings.push({ + file: eligible[0]!, + rule: "mutation/runner-error", + severity: "error", + message: `mutation runner failed (${error}): the kill rate is unproven`, + fixHint: "Fix the test command, then run the gates again.", + check: "mutation", + }) + return findings +} + export async function runGates(options: RunGatesOptions): Promise { const root = options.root const dir = options.dir ?? root @@ -494,6 +670,22 @@ export async function runGates(options: RunGatesOptions): Promise { findings.push(...outcome.findings) failedTests.push(...outcome.failedTests) } + if (config.mutation.enabled) + await timed("mutation", () => + runMutationCheck( + { + dir, + root, + scope: options.scope, + files: present, + logDir, + sandbox: options.sandbox ?? "preferred", + stateDir: options.stateDir ?? defaultStateDir(), + ...(options.signal ? { signal: options.signal } : {}), + }, + config, + ), + ) } } diff --git a/packages/core/src/gates/index.ts b/packages/core/src/gates/index.ts index ec5b379..20fa1eb 100644 --- a/packages/core/src/gates/index.ts +++ b/packages/core/src/gates/index.ts @@ -3,6 +3,7 @@ export * from "./budgets.ts" export * from "./detect.ts" export * from "./engine.ts" export * from "./minimal-deps.ts" +export * from "./mutation.ts" export * from "./parsers.ts" export * from "./secrets.ts" export * from "./security.ts" diff --git a/packages/core/src/gates/mutation.ts b/packages/core/src/gates/mutation.ts new file mode 100644 index 0000000..cf7dfaf --- /dev/null +++ b/packages/core/src/gates/mutation.ts @@ -0,0 +1,280 @@ +// AST mutation gate (#169): tautological tests fail closed. +// +// An LLM-written test can expect the exact buggy behaviour of LLM-written +// code, so QA and the programmer both pass on broken code. The mutation gate +// injects subtle mutants into touched source (negated conditions, mutated +// arithmetic, stripped calls) and runs the touched suite against each one. +// A suite that lets mutants survive has a low kill rate, and the phase is +// rejected when the kill rate is below MUTATION_KILL_THRESHOLD (80%). +// +// The operators are text spans over the source, one occurrence per mutant, +// each carrying its file, line and a description. Results are cached by file +// content hash so unchanged files never rerun the suite. +import { timingSafeEqual } from "node:crypto" +import type { GateFinding } from "../schema/gates.ts" +import { sha256 } from "../util/hash.ts" + +/** A phase is rejected when its mutation kill rate is below this. */ +export const MUTATION_KILL_THRESHOLD = 0.8 + +/** Cap on mutants per file, so one giant file cannot stall the gate. */ +export const MAX_MUTANTS_PER_FILE = 50 + +export type MutantKind = "condition" | "arithmetic" | "call" | "return" + +export interface Mutant { + readonly id: string + readonly file: string + readonly line: number + readonly kind: MutantKind + readonly description: string + /** The full file source with exactly this mutation applied. */ + readonly mutated: string +} + +export interface MutationOutcome { + readonly mutant: Mutant + readonly killed: boolean +} + +export interface MutationReport { + readonly total: number + readonly killed: number + readonly survived: number + readonly killRate: number + readonly passed: boolean + readonly threshold: number + /** Files whose cached result was reused without rerunning the suite. */ + readonly cached: readonly string[] + /** The cache to pass to the next run (unchanged files skip the suite). */ + readonly cache: MutationCacheState +} + +/** Per-file cached outcome, keyed by file, valid only for its content hash. */ +export interface MutationCacheEntry { + readonly hash: string + readonly killed: readonly string[] + readonly total: number +} + +export type MutationCacheState = Readonly> + +interface Replacement { + readonly kind: MutantKind + readonly from: string + readonly to: string + readonly description: string +} + +// One occurrence per mutant: the scanner walks the source and emits a mutant +// for every match. Guards keep compound operators intact (`===` before `==`, +// `++`/`--`/`+=` never split into arithmetic mutants). +const CONDITION_REPLACEMENTS: readonly Replacement[] = [ + { kind: "condition", from: "===", to: "!==", description: "strict equality negated (=== → !==)" }, + { kind: "condition", from: "!==", to: "===", description: "strict inequality negated (!== → ===)" }, + { kind: "condition", from: " && ", to: " || ", description: "conjunction loosened (&& → ||)" }, + { kind: "condition", from: " || ", to: " && ", description: "disjunction tightened (|| → &&)" }, +] + +const ARITHMETIC_REPLACEMENTS: readonly Replacement[] = [ + { kind: "arithmetic", from: " + ", to: " - ", description: "addition mutated (+ → -)" }, + { kind: "arithmetic", from: " - ", to: " + ", description: "subtraction mutated (- → +)" }, + { kind: "arithmetic", from: " * ", to: " / ", description: "multiplication mutated (* → /)" }, + { kind: "arithmetic", from: " / ", to: " * ", description: "division mutated (/ → *)" }, +] + +const lineOf = (source: string, index: number): number => source.slice(0, index).split("\n").length + +const splice = (source: string, index: number, from: string, to: string): string => + source.slice(0, index) + to + source.slice(index + from.length) + +/** A `name(args)` call statement on its own line, stripped to `undefined`. */ +const CALL_STATEMENT = /^([ \t]*)([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*)\([^;]*\);?[ \t]*$/gm + +/** `return ` (non-void), mutated to `return undefined`. */ +const RETURN_EXPR = /^([ \t]*return )(?!undefined\b)([^;]+);[ \t]*$/gm + +/** + * Every mutant for one source file: condition negations, arithmetic swaps, + * stripped call statements and voided returns. Deterministic (source order), + * capped at `maxPerFile`. + */ +export function generateMutants(file: string, source: string, maxPerFile = MAX_MUTANTS_PER_FILE): Mutant[] { + const mutants: Mutant[] = [] + const push = (kind: MutantKind, index: number, from: string, to: string, description: string) => { + if (mutants.length >= maxPerFile) return + mutants.push({ + id: `${file}#${mutants.length + 1}`, + file, + line: lineOf(source, index), + kind, + description, + mutated: splice(source, index, from, to), + }) + } + // Token replacements first (source order per operator class below is + // stable): strict equality before any looser operator, and arithmetic only + // with surrounding spaces so `++`, `--`, `+=` and friends never split. + for (const replacement of [...CONDITION_REPLACEMENTS, ...ARITHMETIC_REPLACEMENTS]) { + let index = source.indexOf(replacement.from) + while (index !== -1 && mutants.length < maxPerFile) { + push(replacement.kind, index, replacement.from, replacement.to, replacement.description) + index = source.indexOf(replacement.from, index + replacement.from.length) + } + if (mutants.length >= maxPerFile) return mutants + } + CALL_STATEMENT.lastIndex = 0 + for (const match of source.matchAll(CALL_STATEMENT)) { + if (mutants.length >= maxPerFile) break + const index = match.index ?? 0 + push("call", index, match[0], `${match[1]}undefined;`, `call stripped (${match[2]}(...) → undefined)`) + } + RETURN_EXPR.lastIndex = 0 + for (const match of source.matchAll(RETURN_EXPR)) { + if (mutants.length >= maxPerFile) break + const index = match.index ?? 0 + push("return", index, match[0], `${match[1]}undefined;`, "return voided (return → return undefined)") + } + return mutants +} + +/** Content hash scoping one file's cached mutation result. */ +export function mutantCacheKey(file: string, content: string): string { + return sha256(`${file}\n${content}`) +} + +/** + * Cache-hash equality: length-guarded constant-time compare (repo precedent: + * trust/sidecar.ts, research/seal.ts). Cache fingerprints are content hashes, + * not secrets, but the timing-safe compare keeps the security scanners quiet. + */ +export function cacheHashesEqual(left: string, right: string): boolean { + const a = Buffer.from(left) + const b = Buffer.from(right) + return a.length === b.length && timingSafeEqual(a, b) +} + +/** + * Cache fingerprint for one source file: its own content plus the content of + * the in-scope test files. Weakening (or strengthening) the tests changes + * the fingerprint, so a stale verdict can never green-light edited tests — + * the suite reruns. + */ +export async function mutationFingerprint( + readFile: (file: string) => Promise, + file: string, + source: string, + testFiles: readonly string[], +): Promise { + const tests: string[] = [] + for (const candidate of testFiles) { + if (candidate === file) continue + const content = await readFile(candidate) + if (content !== undefined) tests.push(`${candidate}\n${content}`) + } + return mutantCacheKey(file, `${source}\n${tests.join("\n")}`) +} + +export function evaluateMutations( + outcomes: readonly MutationOutcome[], + threshold = MUTATION_KILL_THRESHOLD, +): { killRate: number; passed: boolean; killed: number; survived: number; total: number } { + const total = outcomes.length + const killed = outcomes.filter((outcome) => outcome.killed).length + // No mutants means nothing checked the suite: fail closed, never vacuously green. + const killRate = total === 0 ? 0 : killed / total + return { killRate, passed: total > 0 && killRate >= threshold, killed, survived: total - killed, total } +} + +/** The gate finding: an error below threshold, a warning-free pass at/above it. */ +export function mutationFindings( + report: Pick, + files: readonly string[], +): GateFinding[] { + if (report.passed) return [] + return [ + { + file: files[0] ?? ".", + rule: "mutation/kill-rate", + severity: "error", + message: + report.total === 0 + ? "mutation gate found no mutants: the touched sources have no mutable operators, so the suite is unproven" + : `mutation kill rate ${(report.killRate * 100).toFixed(1)}% (${report.killed}/${report.total} killed) is below ${(report.threshold * 100).toFixed(0)}%: the suite lets mutants survive, so it may assert buggy behaviour`, + fixHint: + "Strengthen the touched tests until they fail on negated conditions, swapped arithmetic and stripped calls.", + check: "mutation", + }, + ] +} + +export interface MutationGateOptions { + /** Source files (relative to the run dir) to mutate. */ + readonly files: readonly string[] + readonly readFile: (file: string) => Promise + /** + * Run the touched suite against one mutant: `killed: true` when the suite + * failed (the mutant was caught), `false` when it still passed (survived). + */ + readonly runTests: (mutant: Mutant) => Promise<{ killed: boolean }> + /** Reused across runs: unchanged files skip the suite. */ + readonly cache?: MutationCacheState + /** Test files whose content joins the cache fingerprint (edited tests rerun the suite). */ + readonly testFiles?: readonly string[] + readonly threshold?: number + readonly maxPerFile?: number +} + +/** + * Mutate each file and run the suite per mutant. Files whose content hash + * matches the cache reuse their stored outcome without touching the suite. + */ +export async function runMutationGate(options: MutationGateOptions): Promise { + const threshold = options.threshold ?? MUTATION_KILL_THRESHOLD + const maxPerFile = options.maxPerFile ?? MAX_MUTANTS_PER_FILE + const cache: Record = { ...options.cache } + const cached: string[] = [] + const outcomes: MutationOutcome[] = [] + for (const file of options.files) { + const source = await options.readFile(file) + if (source === undefined) continue + const hash = await mutationFingerprint(options.readFile, file, source, options.testFiles ?? []) + const entry = cache[file] + if (entry && cacheHashesEqual(entry.hash, hash)) { + cached.push(file) + const killed = new Set(entry.killed) + for (let index = 0; index < entry.total; index++) + outcomes.push({ + mutant: { + id: `${file}#${index + 1}`, + file, + line: 1, + kind: "condition", + description: "cached", + mutated: source, + }, + killed: killed.has(`${file}#${index + 1}`), + }) + continue + } + const mutants = generateMutants(file, source, maxPerFile) + const killedIds: string[] = [] + for (const mutant of mutants) { + const result = await options.runTests(mutant) + outcomes.push({ mutant, killed: result.killed }) + if (result.killed) killedIds.push(mutant.id) + } + cache[file] = { hash, killed: killedIds, total: mutants.length } + } + const evaluated = evaluateMutations(outcomes, threshold) + return { + total: evaluated.total, + killed: evaluated.killed, + survived: evaluated.survived, + killRate: evaluated.killRate, + passed: evaluated.passed, + threshold, + cached, + cache, + } +} diff --git a/packages/core/src/schema/gates.ts b/packages/core/src/schema/gates.ts index 391790d..889b12c 100644 --- a/packages/core/src/schema/gates.ts +++ b/packages/core/src/schema/gates.ts @@ -71,11 +71,22 @@ export const GateSecuritySchema = z.object({ }) export type GateSecurity = z.infer +export const GateMutationSchema = z.object({ + /** The AST mutation gate (#169): mutants of touched sources must be killed by the suite. */ + enabled: z.boolean().default(false), + /** Kill rate below this rejects the phase (tautological suites fail closed). */ + threshold: z.number().min(0).max(1).default(0.8), + /** Cap on mutants per file, so one giant file cannot stall the gate. */ + maxMutants: z.number().int().positive().default(50), +}) +export type GateMutation = z.infer + export const GatesConfigSchema = z.object({ version: z.literal(1).default(1), commands: z.array(CommandCheckSchema).default([]), budgets: GateBudgetsSchema.default(GateBudgetsSchema.parse({})), security: GateSecuritySchema.default(GateSecuritySchema.parse({})), + mutation: GateMutationSchema.default({ enabled: false, threshold: 0.8, maxMutants: 50 }), /** Max findings returned to the agent (the full list is in the run log). */ topN: z.number().int().positive().default(20), /** Glob patterns excluded from built-in scans (secrets, budgets). */ diff --git a/packages/core/test/mutation-engine.test.ts b/packages/core/test/mutation-engine.test.ts new file mode 100644 index 0000000..34a7000 --- /dev/null +++ b/packages/core/test/mutation-engine.test.ts @@ -0,0 +1,119 @@ +// Engine integration for the mutation gate (#169): real `bun test` suites +// run against real mutants in a fixture repo. +import { describe, expect, test } from "bun:test" +import { mkdir, readFile, writeFile } from "node:fs/promises" +import path from "node:path" +import { sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts" +import { commandSetHash, GatesConfigSchema, runGates, trustProject } from "../src/index.ts" +import type { Fixture } from "./helpers.ts" +import { gitRepo } from "./helpers.ts" + +const SOURCE = "export function sum(a: number, b: number): number {\n return a + b\n}\n" +// Tautological: asserts the function against itself, so every mutant survives. +const TAUTOLOGICAL = + 'import { expect, test } from "bun:test"\nimport { sum } from "../src/sum.ts"\ntest("adds", () => expect(sum(2, 3)).toBe(sum(2, 3)))\n' +// Strong: pins real behaviour, so the arithmetic mutant is killed. +const STRONG = + 'import { expect, test } from "bun:test"\nimport { sum } from "../src/sum.ts"\ntest("adds", () => expect(sum(2, 3)).toBe(5))\n' + +async function mutationFixture(suite: string): Promise<{ fx: Fixture; config: ReturnType }> { + const fx = await gitRepo("es-mutation-") + await sealHumanKey("test-passphrase-1234", fx.state) + const signer = await unlockHumanKey("test-passphrase-1234", fx.state) + await mkdir(path.join(fx.root, "src"), { recursive: true }) + await mkdir(path.join(fx.root, "test"), { recursive: true }) + await writeFile(path.join(fx.root, "src/sum.ts"), SOURCE) + await writeFile(path.join(fx.root, "test/sum.test.ts"), suite) + const config = mutationConfig() + const { hash, lines } = await commandSetHash(fx.root, config.commands) + await trustProject(fx.root, hash, lines, { stateDir: fx.state, signer }) + return { fx, config } +} + +const mutationConfig = () => + GatesConfigSchema.parse({ + commands: [{ id: "test", kind: "test", command: "bun test", parser: "bun-test", related: "bun" }], + security: { gitleaks: "off", osv: "off" }, + mutation: { enabled: true, threshold: 0.8, maxMutants: 10 }, + }) + +describe("mutation gate in the engine", () => { + test("a tautological suite is rejected with a mutation/kill-rate error", async () => { + const { fx, config } = await mutationFixture(TAUTOLOGICAL) + try { + const report = await runGates({ + root: fx.root, + scope: "touched", + touched: ["src/sum.ts", "test/sum.test.ts"], + config, + stateDir: fx.state, + }) + expect(report.passed).toBe(false) + expect(report.checks.find((check) => check.id === "mutation")?.status).toBe("fail") + expect(report.findings.some((finding) => finding.rule === "mutation/kill-rate")).toBe(true) + // Mutants never leak: the source is restored after every mutant run. + expect(await readFile(path.join(fx.root, "src/sum.ts"), "utf8")).toBe(SOURCE) + } finally { + await fx.cleanup() + } + }) + + test("a strong suite kills its mutants and passes", async () => { + const { fx, config } = await mutationFixture(STRONG) + try { + const report = await runGates({ + root: fx.root, + scope: "touched", + touched: ["src/sum.ts", "test/sum.test.ts"], + config, + stateDir: fx.state, + }) + expect(report.findings.filter((finding) => finding.rule.startsWith("mutation/"))).toEqual([]) + expect(report.passed).toBe(true) + } finally { + await fx.cleanup() + } + }) + + test("the mutation cache persists in the state dir", async () => { + const { fx, config } = await mutationFixture(STRONG) + try { + await runGates({ + root: fx.root, + scope: "touched", + touched: ["src/sum.ts", "test/sum.test.ts"], + config, + stateDir: fx.state, + }) + const cached = JSON.parse(await readFile(path.join(fx.state, "mutation-cache.json"), "utf8")) as Record< + string, + { hash: string; killed: string[]; total: number } + > + expect(Object.keys(cached)).toContain("src/sum.ts") + expect(cached["src/sum.ts"]!.total).toBeGreaterThan(0) + } finally { + await fx.cleanup() + } + }) + + test("the gate stays off unless enabled", async () => { + const { fx } = await mutationFixture(TAUTOLOGICAL) + try { + const config = GatesConfigSchema.parse({ + commands: [{ id: "test", kind: "test", command: "bun test", parser: "bun-test", related: "bun" }], + security: { gitleaks: "off", osv: "off" }, + }) + expect(config.mutation.enabled).toBe(false) + const report = await runGates({ + root: fx.root, + scope: "touched", + touched: ["src/sum.ts", "test/sum.test.ts"], + config, + stateDir: fx.state, + }) + expect(report.checks.some((check) => check.id === "mutation")).toBe(false) + } finally { + await fx.cleanup() + } + }) +}) diff --git a/packages/core/test/mutation.test.ts b/packages/core/test/mutation.test.ts new file mode 100644 index 0000000..1eb7da8 --- /dev/null +++ b/packages/core/test/mutation.test.ts @@ -0,0 +1,184 @@ +// AST mutation gate (#169): tautological tests must fail closed. +// +// A tautological test passes on buggy code because it expects the buggy +// behaviour. The mutation gate injects subtle AST mutants (negated +// conditions, mutated arithmetic, stripped calls) into touched code and runs +// the touched suite against each mutant. A suite that lets mutants survive +// has a low kill rate and the phase is rejected (kill rate < 80%). +import { describe, expect, test } from "bun:test" +import { + cacheHashesEqual, + evaluateMutations, + generateMutants, + MUTATION_KILL_THRESHOLD, + type Mutant, + mutantCacheKey, + mutationFindings, + runMutationGate, +} from "../src/gates/mutation.ts" + +const killed = (mutants: readonly Mutant[], ids: readonly string[]) => { + const set = new Set(ids) + return mutants.map((mutant) => ({ mutant, killed: set.has(mutant.id) })) +} + +describe("generateMutants", () => { + test("negates conditions", () => { + const mutants = generateMutants("src/a.ts", "export const ok = (a: number, b: number) => a === b && c !== d\n") + expect(mutants.some((mutant) => mutant.kind === "condition")).toBe(true) + expect(mutants.length).toBeGreaterThan(0) + }) + + test("mutates arithmetic", () => { + const mutants = generateMutants("src/a.ts", "export const add = (a: number, b: number) => a + b\n") + const arithmetic = mutants.filter((mutant) => mutant.kind === "arithmetic") + expect(arithmetic.length).toBeGreaterThan(0) + expect(arithmetic[0]!.mutated).toContain("a - b") + }) + + test("strips function calls", () => { + const mutants = generateMutants("src/a.ts", "export function run() {\n notify(user)\n return done()\n}\n") + expect(mutants.some((mutant) => mutant.kind === "call")).toBe(true) + }) + + test("each mutant carries its file, line and a changed source", () => { + const source = "export const add = (a: number, b: number) => a + b\n" + for (const mutant of generateMutants("src/a.ts", source)) { + expect(mutant.file).toBe("src/a.ts") + expect(mutant.line).toBeGreaterThanOrEqual(1) + expect(mutant.mutated).not.toBe(source) + expect(mutant.description.length).toBeGreaterThan(0) + } + }) + + test("caps mutants per file", () => { + const source = `${"export const x1 = a + b\n".repeat(40)}` + expect(generateMutants("src/a.ts", source, 5).length).toBeLessThanOrEqual(5) + }) +}) + +describe("evaluateMutations", () => { + test("kill rate is killed / total and the 80% threshold decides", () => { + expect(MUTATION_KILL_THRESHOLD).toBe(0.8) + const mutants = generateMutants("src/a.ts", "export const add = (a: number, b: number) => a + b\n") + expect(mutants.length).toBeGreaterThan(0) + const all = evaluateMutations( + killed( + mutants, + mutants.map((mutant) => mutant.id), + ), + MUTATION_KILL_THRESHOLD, + ) + expect(all.killRate).toBe(1) + expect(all.passed).toBe(true) + const none = evaluateMutations(killed(mutants, []), MUTATION_KILL_THRESHOLD) + expect(none.killRate).toBe(0) + expect(none.passed).toBe(false) + }) + + test("a tautological suite (zero kills) fails closed with an error finding", () => { + const mutants = generateMutants("src/a.ts", "export const add = (a: number, b: number) => a + b\n") + const report = evaluateMutations(killed(mutants, []), MUTATION_KILL_THRESHOLD) + const findings = mutationFindings({ ...report, threshold: MUTATION_KILL_THRESHOLD }, ["src/a.ts"]) + expect(report.passed).toBe(false) + expect(findings.some((finding) => finding.severity === "error" && finding.rule === "mutation/kill-rate")).toBe(true) + }) + + test("a strong suite at/above threshold passes with no error findings", () => { + const mutants = generateMutants("src/a.ts", "export const add = (a: number, b: number) => a + b\n") + const keep = mutants.slice(0, Math.ceil(mutants.length * MUTATION_KILL_THRESHOLD)) + const report = evaluateMutations( + killed( + mutants, + keep.map((mutant) => mutant.id), + ), + MUTATION_KILL_THRESHOLD, + ) + const findings = mutationFindings({ ...report, threshold: MUTATION_KILL_THRESHOLD }, ["src/a.ts"]) + expect(report.passed).toBe(true) + expect(findings.filter((finding) => finding.severity === "error")).toEqual([]) + }) +}) + +describe("runMutationGate", () => { + test("a tautological suite is rejected: the runner never observes a kill", async () => { + const files = { "src/sum.ts": "export function sum(a: number, b: number): number {\n return a + b\n}\n" } + const report = await runMutationGate({ + files: Object.keys(files), + readFile: async (file) => files[file as keyof typeof files], + // Tautological: the suite asserts nothing real, so every mutant survives. + runTests: async () => ({ killed: false }), + }) + expect(report.total).toBeGreaterThan(0) + expect(report.killRate).toBe(0) + expect(report.passed).toBe(false) + }) + + test("a strong suite kills every mutant and passes", async () => { + const files = { "src/sum.ts": "export function sum(a: number, b: number): number {\n return a + b\n}\n" } + const report = await runMutationGate({ + files: Object.keys(files), + readFile: async (file) => files[file as keyof typeof files], + runTests: async () => ({ killed: true }), + }) + expect(report.passed).toBe(true) + expect(report.killRate).toBe(1) + }) + + test("results are cached: unchanged files do not rerun the suite", async () => { + const files = { "src/sum.ts": "export function sum(a: number, b: number): number {\n return a + b\n}\n" } + let runs = 0 + const options = { + files: Object.keys(files), + readFile: async (file: string) => files[file as keyof typeof files], + runTests: async () => { + runs++ + return { killed: true } + }, + } + const first = await runMutationGate(options) + const firstRuns = runs + expect(firstRuns).toBeGreaterThan(0) + const second = await runMutationGate({ ...options, cache: first.cache }) + expect(second.cached).toContain("src/sum.ts") + expect(runs).toBe(firstRuns) + expect(second.killRate).toBe(first.killRate) + }) + + test("mutantCacheKey changes when the file content changes", () => { + expect(mutantCacheKey("src/a.ts", "a + b")).not.toBe(mutantCacheKey("src/a.ts", "a - b")) + expect(mutantCacheKey("src/a.ts", "same")).toBe(mutantCacheKey("src/a.ts", "same")) + }) + + test("cacheHashesEqual compares hashes in constant time", () => { + const hash = mutantCacheKey("src/a.ts", "same") + expect(cacheHashesEqual(hash, mutantCacheKey("src/a.ts", "same"))).toBe(true) + expect(cacheHashesEqual(hash, mutantCacheKey("src/a.ts", "different"))).toBe(false) + expect(cacheHashesEqual(hash, `${hash}00`)).toBe(false) + }) + + test("changing a test file invalidates the cache: the suite reruns", async () => { + let suite = "strong" + let runs = 0 + const options = { + files: ["src/sum.ts"], + readFile: async (file: string) => + file === "src/sum.ts" ? "export function sum(a: number, b: number): number {\n return a + b\n}\n" : suite, + runTests: async () => { + runs++ + return { killed: true } + }, + testFiles: ["test/sum.test.ts"], + } + const first = await runMutationGate(options) + expect(runs).toBeGreaterThan(0) + // Same tests: cached, no rerun. + await runMutationGate({ ...options, cache: first.cache }) + expect(runs).toBe(first.cache["src/sum.ts"]!.total) + // Weakened tests: the fingerprint changes, so the suite reruns. + suite = "weakened" + const third = await runMutationGate({ ...options, cache: first.cache }) + expect(third.cached).not.toContain("src/sum.ts") + expect(runs).toBeGreaterThan(first.cache["src/sum.ts"]!.total) + }) +}) diff --git a/schemas/gate-mutation.schema.json b/schemas/gate-mutation.schema.json new file mode 100644 index 0000000..62a473a --- /dev/null +++ b/schemas/gate-mutation.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "enabled": { + "default": false, + "type": "boolean" + }, + "threshold": { + "default": 0.8, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxMutants": { + "default": 50, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "enabled", + "threshold", + "maxMutants" + ], + "additionalProperties": false +} diff --git a/schemas/gates-config.schema.json b/schemas/gates-config.schema.json index 9d43dda..ae6110d 100644 --- a/schemas/gates-config.schema.json +++ b/schemas/gates-config.schema.json @@ -179,6 +179,38 @@ ], "additionalProperties": false }, + "mutation": { + "default": { + "enabled": false, + "threshold": 0.8, + "maxMutants": 50 + }, + "type": "object", + "properties": { + "enabled": { + "default": false, + "type": "boolean" + }, + "threshold": { + "default": 0.8, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxMutants": { + "default": 50, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "enabled", + "threshold", + "maxMutants" + ], + "additionalProperties": false + }, "topN": { "default": 20, "type": "integer", @@ -203,6 +235,7 @@ "commands", "budgets", "security", + "mutation", "topN", "ignore" ],