Merge pull request #280 from Heretek-AI/audit/193-queereye

[Audit] Queereye: OpenCode v2 integration tests
2026-10-12 02:03:00 +00:00 · 2026-10-10 22:34:59 -04:00
parent 2947eb31c7 58b177d62b
commit fc2a5e7379
2 changed files with 283 additions and 0 deletions
@@ -0,0 +1,144 @@
+// Issue #193 audit pins (bug-adjacent, behavior pinned — NOT fixed here).
+//
+// The defects below are tracked for the parent as bugs #266 (prototype-chain
+// aliases validate clean) and #270 (vacuous QA gates: cite-gate on missing
+// probes, spec-vs-render tautology). These tests pin the CURRENT behavior so
+// the audit verdicts are executable; product code is intentionally untouched.
+import { afterEach, beforeEach, describe, expect, test } from "bun:test"
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import path from "node:path"
+import {
+  AXES,
+  checkDrift,
+  designPaths,
+  designTools,
+  FORMS,
+  findOneOffs,
+  readProbes,
+  readTokens,
+  renderGuide,
+  resolveAlias,
+  validateTokens,
+  verifyCiteGate,
+} from "../src/index.ts"
+import { specVsRenderCheck } from "../src/queereye/csf.ts"
+import { EXEMPLARS } from "../src/queereye/specs.ts"
+import { isWebrefFresh } from "../src/queereye/webref.ts"
+
+let root: string
+beforeEach(async () => {
+  root = await mkdtemp(path.join(tmpdir(), "es-audit193-"))
+})
+afterEach(() => rm(root, { recursive: true, force: true }))
+
+const call = (name: string, input: Record<string, unknown> = {}, agent = "designer") =>
+  designTools({ root })
+    .find((tool) => tool.name === name)!
+    .execute(input, { agent })
+
+/** Skip the whole interview through the tools so tokens.json exists. */
+const completeInterview = async () => {
+  for (const axis of AXES) for (const slot of FORMS[axis].requiredSlots) await call("es_design_skip", { axis, slot })
+  await call("es_design_complete", {})
+}
+
+describe("audit #193 pin: prototype-chain aliases (#266)", () => {
+  test("an alias through the prototype chain validates clean today", () => {
+    const tree = {
+      color: {
+        $type: "color",
+        primitive: { ink: { $value: "#0f172a" } },
+        semantic: { evil: { $value: "{__proto__.toString}" } },
+      },
+    }
+    // FINDING (#266): the `in` operator in walkPath follows the prototype
+    // chain, so this resolves to Object.prototype.toString and validates
+    // with zero errors and zero one-offs.
+    expect(validateTokens(tree)).toEqual([])
+    expect(findOneOffs(tree)).toEqual([])
+    expect(typeof resolveAlias(tree, "__proto__.toString")).toBe("function")
+  })
+
+  test("inherited Object members validate clean as bare aliases today", () => {
+    for (const ref of ["toString", "valueOf"]) {
+      const tree = {
+        color: {
+          $type: "color",
+          primitive: { ink: { $value: "#0f172a" } },
+          semantic: { evil: { $value: `{${ref}}` } },
+        },
+      }
+      // FINDING (#266): same mechanism, single segment.
+      expect(validateTokens(tree)).toEqual([])
+    }
+  })
+
+  test("control: a genuinely dangling alias is still refused", () => {
+    const tree = {
+      color: {
+        $type: "color",
+        primitive: { ink: { $value: "#0f172a" } },
+        semantic: { evil: { $value: "{color.primitive.missing}" } },
+      },
+    }
+    expect(validateTokens(tree).some((error) => error.includes("does not resolve"))).toBe(true)
+  })
+})
+
+describe("audit #193 pin: cite-gate with deleted probes.json (#270.1)", () => {
+  test("deleted probes + guide re-rendered with [] verifies today", async () => {
+    await completeInterview()
+    await call("es_design_specs", {})
+    await call("es_design_harvest", {})
+    const receipt = await readFile(path.join(root, ".factory/design/cite-gate-demo.md"), "utf8")
+
+    await rm(designPaths(root).probes, { force: true })
+    expect(await readProbes(root)).toBeUndefined()
+    const tokens = await readTokens(root)
+    await writeFile(designPaths(root).guide, renderGuide(tokens, []))
+
+    // FINDING (#270.1): readProbes defaults a missing file to [] (store.ts),
+    // so the drift gate is green and the cite-gate verifies with zero
+    // contrast pairs — the contrast QA leg is vacuous on missing probes.
+    expect(await checkDrift(root)).toEqual({ drifted: [], missing: [], tokenErrors: [] })
+    expect(await verifyCiteGate(receipt, root)).toEqual([])
+  })
+
+  test("control: the receipt still binds tokens, so a stale tokens tree is refused", async () => {
+    await completeInterview()
+    await call("es_design_harvest", {})
+    const receipt = await readFile(path.join(root, ".factory/design/cite-gate-demo.md"), "utf8")
+    const tokens = (await readTokens(root)) as Record<string, unknown>
+    tokens.$edited = true
+    await writeFile(designPaths(root).tokens, `${JSON.stringify(tokens, null, 2)}\n`)
+    expect(await verifyCiteGate(receipt, root)).toContainEqual(expect.stringContaining("stale/forged receipt refused"))
+  })
+})
+
+describe("audit #193 pin: spec-vs-render check shape (#270.2)", () => {
+  test("the leg CAN fail when fed actual render output", () => {
+    const spec = EXEMPLARS.button as unknown as Record<string, unknown>
+    const slots = Object.keys((spec.slots ?? {}) as Record<string, unknown>).sort()
+    // Fed the real slot list it is green …
+    expect(specVsRenderCheck(spec, slots, isWebrefFresh())).toEqual([])
+    // … but a render missing slots fails the fill leg …
+    const errors = specVsRenderCheck(spec, ["label"], true)
+    expect(errors.some((error) => error.includes("not filled in render"))).toBe(true)
+    // … and a missing render output is refused outright.
+    expect(specVsRenderCheck(spec, null, true).some((error) => error.includes("must be supplied"))).toBe(true)
+  })
+
+  test("es_design_specs(check:true) feeds each spec its own slots, so the fill leg cannot fail", async () => {
+    await call("es_design_specs", {})
+    // FINDING (#270.2): ops.ts feeds Object.keys(spec.slots) as the
+    // "render output", so every expected slot is trivially filled and the
+    // tool check passes even though the leg is never exercised.
+    for (const spec of Object.values(EXEMPLARS)) {
+      const record = spec as unknown as Record<string, unknown>
+      const ownSlots = Object.keys((record.slots ?? {}) as Record<string, unknown>).sort()
+      expect(specVsRenderCheck(record, ownSlots, isWebrefFresh())).toEqual([])
+    }
+    expect(await call("es_design_specs", { check: true })).toContain("byte-match")
+  })
+})
@@ -0,0 +1,139 @@
+// Issue #193: the /design interview on the real testkit host with scripted
+// answers. The designer seat walks the 7-axis interview through its tools
+// (repairs included), completion writes DTCG tokens that validate with zero
+// one-offs, and a failing-contrast palette is refused by the gate.
+import { afterAll, beforeAll, describe, expect, test } from "bun:test"
+import { mkdtemp, rm } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import path from "node:path"
+import {
+  compileToCssVars,
+  designPaths,
+  findOneOffs,
+  gatePairs,
+  probeReport,
+  readProbes,
+  readTokens,
+  renderGuide,
+  validatePair,
+  validateTokens,
+  writeArtifacts,
+} from "@heretek-ai/es-core"
+import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
+
+const pluginDir = path.resolve(import.meta.dir, "..")
+const call = (name: string, args: Record<string, unknown> = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@`
+const answer = (axis: string, slot: string, value: string) => call("es_design_answer", { axis, slot, value })
+
+// Scripted answers: real words through every slot (not skip-all — that path
+// is design.test.ts). All values are repair-safe (recommended words only).
+const SCRIPTED: Array<[string, string, string]> = [
+  ["brand", "name", "Acme"],
+  ["brand", "voice", "minimal"],
+  ["brand", "values", "clarity"],
+  ["color", "primary", "ocean"],
+  ["color", "neutral", "slate"],
+  ["color", "accent", "teal"],
+  ["type", "family", "system"],
+  ["type", "scale", "1.250"],
+  ["type", "base_size", "16"],
+  ["layout", "density", "comfortable"],
+  ["layout", "radius", "soft"],
+  ["layout", "grid", "12"],
+  ["effects", "motion", "subtle"],
+  ["effects", "elevation", "restrained"],
+  ["effects", "decoration", "text-safe-only"],
+  ["dark_light", "modes", "both"],
+  ["dark_light", "surface", "dimmed brand"],
+  ["a11y", "text_level", "AA"],
+  ["a11y", "large_text_level", "AA-large"],
+  ["a11y", "reduced_motion", "yes"],
+]
+
+let state: string
+let h: Harness
+beforeAll(async () => {
+  state = await mkdtemp(path.join(tmpdir(), "es-design-interview-host-"))
+  h = await boot({
+    git: true,
+    script: directiveScript,
+    plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }],
+    files: { "README.md": "# d\n" },
+  })
+}, 60_000)
+
+afterAll(async () => {
+  await h?.close()
+  await rm(state, { recursive: true, force: true })
+})
+
+describe("design interview with scripted answers", () => {
+  test("vague and smuggled answers are repaired, scripted answers settle every slot", async () => {
+    const status = await h.run(call("es_design_status"), { agent: "designer" })
+    expect(status.tools[0]?.text).toContain("slots settled")
+
+    const vague = await h.run(answer("brand", "voice", "whatever"), { agent: "designer" })
+    expect(vague.tools[0]?.text).toContain("Not accepted (vague)")
+
+    const smuggled = await h.run(answer("color", "primary", "#ff0000"), { agent: "designer" })
+    expect(smuggled.tools[0]?.text).toContain("Not accepted (smuggled)")
+
+    const answered = await h.run(SCRIPTED.map(([axis, slot, value]) => answer(axis, slot, value)).join(" "), {
+      agent: "designer",
+    })
+    expect(answered.tools.filter((tool) => tool.status === "error")).toHaveLength(0)
+    expect(answered.tools).toHaveLength(SCRIPTED.length)
+    for (const tool of answered.tools) expect(tool.text).toContain("Accepted")
+    expect(answered.tools[answered.tools.length - 1]?.text).toContain("run es_design_complete")
+  })
+
+  test("completion writes DTCG tokens that validate with zero one-offs", async () => {
+    const completed = await h.run(call("es_design_complete"), { agent: "designer" })
+    expect(completed.tools[0]?.status).toBe("completed")
+    expect(completed.tools[0]?.text).toContain("reuse ratio")
+
+    const tokens = await readTokens(h.directory)
+    expect(validateTokens(tokens)).toEqual([])
+    expect(findOneOffs(tokens)).toEqual([])
+
+    const probes = await readProbes(h.directory)
+    expect(probes!.length).toBeGreaterThan(0)
+    expect(probes!.every((row) => row.pass)).toBe(true)
+
+    const guide = await Bun.file(designPaths(h.directory).guide).text()
+    expect(guide).toContain("# STYLE_GUIDE")
+
+    const checked = await h.run(call("es_design_check"), { agent: "designer" })
+    expect(checked.tools[0]?.text).toContain("byte-match")
+    expect(checked.tools[0]?.text).toContain("One-off mints: 0")
+  })
+
+  test("a failing-contrast palette is refused by the gate", async () => {
+    // The completion tool derives its probes from the ratio-first generator,
+    // so the interview's own palette always passes; refusal is pinned at the
+    // gate the completion tool enforces (es_design_complete: "Contrast gate
+    // failed") and surfaced through es_design_check's pair count.
+    const failing = gatePairs([{ name: "washed body", fg: "#eeeeee", bg: "#ffffff" }])
+    expect(failing).toHaveLength(1)
+    expect(failing[0]!.errors.join("; ")).toContain("below AA body text minimum 4.5:1")
+    expect(validatePair("#eeeeee", "#ffffff", { decorative: true }).errors).toContain(
+      "decorative exemption refused for a text pair",
+    )
+    // Control: a passing pair is green.
+    expect(validatePair("#595959", "#ffffff").errors).toEqual([])
+
+    // Host-level: plant a failing probe row (with its guide re-render) and
+    // the check surfaces the failure instead of passing silently.
+    const tokens = await readTokens(h.directory)
+    const bad = probeReport([{ name: "washed body", fg: "#eeeeee", bg: "#ffffff" }])
+    expect(bad[0]?.pass).toBe(false)
+    await writeArtifacts(h.directory, {
+      tokens,
+      probes: bad,
+      guide: renderGuide(tokens, bad),
+      css: compileToCssVars(tokens),
+    })
+    const surfaced = await h.run(call("es_design_check"), { agent: "designer" })
+    expect(surfaced.tools[0]?.text).toContain("Contrast pairs: 0/1 pass")
+  })
+})