mirror of
https://github.com/Heretek-AI/IUMBTEMS.git
synced 2026-10-12 02:03:00 +00:00
test(audit): evidence cache and quote verifier integration tests (#187)
This commit is contained in:
John Smith
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"description": "Research fetches a served fixture page and quotes its key sentence verbatim (no false VERIFIED).",
|
||||
"agent": "es-research-alpha",
|
||||
"prompt": "Fetch {{SERVE_URL}}/evidence-187.txt with es_research_fetch and report the key sentence about mechanical gates, quoted verbatim.",
|
||||
"maxSteps": 6,
|
||||
"checks": { "toolsUsed": ["es_research_fetch"], "contains": ["Mechanical gates catch regressions before review"] },
|
||||
"id": "evidence-187"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
The evidence cache seals every source with an engine key. Mechanical gates
|
||||
catch regressions before review, which keeps autonomous agents honest.
|
||||
Quote the gate sentence verbatim; a paraphrase is not evidence.
|
||||
@@ -0,0 +1,188 @@
|
||||
// Issue #187 §2 + bug #251 fuzz-adjacent edges (deterministic, no fuzz harness).
|
||||
//
|
||||
// Pins the CURRENT normalisation semantics of `verifyQuote` (NFKC,
|
||||
// typography folds, markdown stripping, whitespace collapse; no entity
|
||||
// decoding) and the CURRENT auditor downgrade rules (VERIFIED verbatim
|
||||
// grounds, paraphrases and planted entries stay ungrounded). Bugs #251
|
||||
// (fuzz corpus, legacy unsealed constructor) and #254 (wording) are for the
|
||||
// parent: this suite tests behaviour as-is and reports defects as findings,
|
||||
// never by changing product code.
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import path from "node:path"
|
||||
import { auditMarkdown, htmlToText, normalizeSourceText, SourceCache, sha256, verifyQuote } from "../src/index.ts"
|
||||
|
||||
let dir: string
|
||||
let state: string
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(path.join(tmpdir(), "es-evidence-187-"))
|
||||
state = await mkdtemp(path.join(tmpdir(), "es-evidence-187-state-"))
|
||||
})
|
||||
afterEach(() => Promise.all([rm(dir, { recursive: true, force: true }), rm(state, { recursive: true, force: true })]))
|
||||
|
||||
const sealed = () => new SourceCache(path.join(dir, "sources"), state)
|
||||
|
||||
const GATES = "Alpha gates keep agents honest under heavy load conditions today, witnesses agree."
|
||||
|
||||
describe("quote verifier fuzz-adjacent edges (#187 §2, gaps listed in #251)", () => {
|
||||
test("zero-width chars: ZWSP/FEFF fold to space and still match; ZWNJ/ZWJ/SHY/word-joiner refuse", () => {
|
||||
const plain = "hello world friends and more text here"
|
||||
// U+200B ZERO WIDTH SPACE and U+FEFF are in the typography fold to space.
|
||||
expect(verifyQuote(plain, "hello world friends and more").ok).toBe(true)
|
||||
expect(verifyQuote(plain, "hello world friends and more").ok).toBe(true)
|
||||
// An invisible join inside a word folds to a word break (current behaviour).
|
||||
expect(verifyQuote("helloworld friends and more text here", "hello world friends and more").ok).toBe(true)
|
||||
// U+200C/U+200D/U+00AD/U+2060 are NOT folded: the quote is refused.
|
||||
for (const invisible of ["", "", "", ""]) {
|
||||
const quote = `hello${invisible} world friends and more`
|
||||
expect([JSON.stringify(invisible), verifyQuote(plain, quote)]).toMatchObject([
|
||||
JSON.stringify(invisible),
|
||||
{ ok: false },
|
||||
])
|
||||
expect(verifyQuote(plain, quote).reason).toContain("not found verbatim")
|
||||
// And the reverse direction refuses too: the source keeps the char.
|
||||
expect(verifyQuote(`hello${invisible} world friends and more text here`, plain.slice(0, 30)).ok).toBe(false)
|
||||
}
|
||||
})
|
||||
|
||||
test("`..` stays literal, `...`/`…` split into ordered fragments, `....` leaves a stray dot", () => {
|
||||
// Two dots are code/punctuation, never an ellipsis.
|
||||
expect(verifyQuote("version 1..2 released today for all users", "version 1..2 released today").ok).toBe(true)
|
||||
expect(verifyQuote("version 1..2 released today for all users", "version 1..2 ... released today for all").ok).toBe(
|
||||
true,
|
||||
)
|
||||
// Three dots and the single-char ellipsis both split; order matters.
|
||||
expect(verifyQuote(GATES, "Alpha gates keep ... honest under heavy load").ok).toBe(true)
|
||||
expect(verifyQuote(GATES, "Alpha gates keep … honest under heavy load").ok).toBe(true)
|
||||
expect(verifyQuote(GATES, "honest under heavy load ... Alpha gates keep").ok).toBe(false)
|
||||
// Four dots are an ellipsis plus a literal dot: the stray dot is refused.
|
||||
const four = verifyQuote(GATES, "Alpha gates keep .... honest under heavy load")
|
||||
expect(four.ok).toBe(false)
|
||||
expect(four.reason).toContain("not found verbatim")
|
||||
})
|
||||
|
||||
test("entity-at-quote: the verifier does no entity decoding (the fetch layer decodes)", () => {
|
||||
const amp = htmlToText("<p>Fish & chips are tasty indeed yum.</p>")
|
||||
const numeric = htmlToText("<p>Fish & chips are tasty indeed yum.</p>")
|
||||
// Both spellings cache to the same decoded bytes.
|
||||
expect(amp.text).toBe("Fish & chips are tasty indeed yum.")
|
||||
expect(numeric.text).toBe(amp.text)
|
||||
// The decoded quote verifies; the still-encoded spellings are refused.
|
||||
expect(verifyQuote(amp.text, "Fish & chips are tasty").ok).toBe(true)
|
||||
for (const encoded of ["Fish & chips are tasty", "Fish & chips are tasty"]) {
|
||||
const check = verifyQuote(amp.text, encoded)
|
||||
expect([encoded, check.ok]).toEqual([encoded, false])
|
||||
expect(check.reason).toContain("not found verbatim")
|
||||
}
|
||||
})
|
||||
|
||||
test("markdown-rendered quotes: emphasis and links strip on both sides", () => {
|
||||
expect(verifyQuote("the **quick brown fox jumps** over fences high", "quick brown fox jumps over").ok).toBe(true)
|
||||
expect(verifyQuote("the quick brown fox jumps over fences high", "the **quick brown fox jumps** over").ok).toBe(
|
||||
true,
|
||||
)
|
||||
expect(verifyQuote("run __every__ `single` test file now please", "run every single test file now").ok).toBe(true)
|
||||
expect(verifyQuote("[Bun](https://bun.sh) is fast and furious indeed", "Bun is fast and furious").ok).toBe(true)
|
||||
expect(verifyQuote("Bun is fast and furious indeed ok", "[Bun](https://bun.sh) is fast and furious").ok).toBe(true)
|
||||
})
|
||||
|
||||
test("short-fragment refusal: every ellipsis fragment and every single quote needs 12+ characters", () => {
|
||||
expect(verifyQuote(GATES, "short").reason).toContain("too short")
|
||||
expect(verifyQuote(GATES, "abcdefghijk").ok).toBe(false)
|
||||
expect(verifyQuote("abcdefghijkl rest of the text here ok", "abcdefghijkl").ok).toBe(true)
|
||||
const short = verifyQuote(GATES, "Alpha gates keep ... hi ... honest under heavy load")
|
||||
expect(short.ok).toBe(false)
|
||||
expect(short.reason).toContain("fragment")
|
||||
expect(verifyQuote(GATES, "Alpha gates keep ... honest under heavy load").ok).toBe(true)
|
||||
})
|
||||
|
||||
test("unicode normalisation: NFKC, curly quotes, dashes and NBSP fold before matching", () => {
|
||||
expect(verifyQuote("filigree work is quite delicate indeed", "filigree work is quite").ok).toBe(true)
|
||||
expect(verifyQuote("café society is quite lively today", "café society is quite").ok).toBe(true)
|
||||
// Composed vs decomposed acute accents match after NFKC.
|
||||
expect(verifyQuote("café society is quite lively today", "café society is quite").ok).toBe(true)
|
||||
expect(verifyQuote("He said “hello world, friends” ok", 'said "hello world, friends"').ok).toBe(true)
|
||||
expect(verifyQuote("fast — reliable service for all users", "fast - reliable service").ok).toBe(true)
|
||||
expect(verifyQuote("hello world friends and more text", "hello world friends and more").ok).toBe(true)
|
||||
// Case still matters: normalisation is not case folding.
|
||||
expect(verifyQuote(GATES, "alpha gates keep agents honest").ok).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe("auditor downgrade rules (#187: verbatim grounds, paraphrase and plants do not)", () => {
|
||||
test("verbatim VERIFIED grounds; a paraphrase stays ungrounded (flagged, never re-tagged)", async () => {
|
||||
const cache = sealed()
|
||||
const source = await cache.put({
|
||||
url: "https://gates.test/run",
|
||||
text: "Mechanical gates catch regressions before review, which keeps autonomous agents honest under heavy load.",
|
||||
provider: "fetch",
|
||||
})
|
||||
const good = await auditMarkdown(
|
||||
`- Gates catch regressions early [VERIFIED: sha256:${source.meta.sha256} "catch regressions before review"]`,
|
||||
cache,
|
||||
)
|
||||
expect(good.coverage).toMatchObject({ claims: 1, grounded: 1, ungrounded: 0 })
|
||||
expect(good.passed).toBe(true)
|
||||
expect(good.cited).toEqual([source.meta.sha256])
|
||||
const paraphrased = await auditMarkdown(
|
||||
`- Gates catch failures ahead of inspection [VERIFIED: sha256:${source.meta.sha256} "catch failures ahead of inspection"]`,
|
||||
cache,
|
||||
)
|
||||
expect(paraphrased.coverage).toMatchObject({ claims: 1, grounded: 0, ungrounded: 1 })
|
||||
expect(paraphrased.passed).toBe(false)
|
||||
expect(paraphrased.claims[0]?.reason).toContain("not found verbatim")
|
||||
})
|
||||
|
||||
test("a planted unsealed entry is refused by the sealed cache; the legacy constructor still reads it (#251)", async () => {
|
||||
const text = "Planted findings about gate latency on reference hardware."
|
||||
const hash = sha256(normalizeSourceText(text))
|
||||
await mkdir(path.join(dir, "sources"), { recursive: true })
|
||||
await writeFile(path.join(dir, "sources", `${hash}.md`), normalizeSourceText(text))
|
||||
await writeFile(
|
||||
path.join(dir, "sources", `${hash}.json`),
|
||||
`${JSON.stringify({ sha256: hash, url: "https://evil.test/planted", retrieved: new Date().toISOString(), provider: "fetch", bytes: Buffer.byteLength(normalizeSourceText(text)) })}\n`,
|
||||
)
|
||||
expect(await sealed().get(hash)).toBeUndefined()
|
||||
const audit = await auditMarkdown(
|
||||
`- Planted latency wins [VERIFIED: sha256:${hash} "gate latency on reference"]`,
|
||||
sealed(),
|
||||
)
|
||||
expect(audit.coverage).toMatchObject({ claims: 1, grounded: 0, ungrounded: 1 })
|
||||
expect(audit.claims[0]?.reason).toContain("not in the cache")
|
||||
expect(audit.passed).toBe(false)
|
||||
// Current legacy behaviour (bug #251): without a state dir nothing is verified.
|
||||
expect((await new SourceCache(path.join(dir, "sources")).get(hash))?.text).toContain("gate latency")
|
||||
})
|
||||
|
||||
test("short and missing quotes stay ungrounded; substantive other tags ground, stubs are malformed", async () => {
|
||||
const cache = sealed()
|
||||
const source = await cache.put({
|
||||
url: "https://gates.test/short",
|
||||
text: "Mechanical gates catch regressions before review, which keeps autonomous agents honest under heavy load.",
|
||||
provider: "fetch",
|
||||
})
|
||||
const audit = await auditMarkdown(
|
||||
[
|
||||
`- Too short [VERIFIED: sha256:${source.meta.sha256} "tiny"]`,
|
||||
`- Missing quote [VERIFIED: no quote here]`,
|
||||
`- Fewer tools to install [INFERRED: one binary covers the runner and the bundler]`,
|
||||
`- Maybe faster [HYPOTHESIS: rerun the benchmark with n=30]`,
|
||||
`- Nothing else found [NEGATIVE_KNOWLEDGE: searched for gate failures on ARM]`,
|
||||
`- Stub [INFERRED: ok]`,
|
||||
"- No tag at all",
|
||||
].join("\n"),
|
||||
cache,
|
||||
)
|
||||
// "tiny" fails verification (ungrounded); "no quote here" has no
|
||||
// source/quote shape and the stub INFERRED body is malformed.
|
||||
expect(audit.coverage).toMatchObject({
|
||||
claims: 7,
|
||||
grounded: 3,
|
||||
ungrounded: 1,
|
||||
untagged: 1,
|
||||
malformed: 2,
|
||||
})
|
||||
expect(audit.passed).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,133 @@
|
||||
// Issue #187 §2: loopback HTTP source server on the testkit host — a
|
||||
// research seat fetches, quotes verbatim (VERIFIED survives the audit),
|
||||
// paraphrases (stays ungrounded: flagged, never re-tagged), and quotes a
|
||||
// planted file (refused). Real in-process host, scripted fake LLM,
|
||||
// 127.0.0.1 only; must also pass under scripts/v2-head.sh.
|
||||
import { afterAll, beforeAll, describe, expect, test } from "bun:test"
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import path from "node:path"
|
||||
import { normalizeSourceText, researchCache, sha256 } from "@heretek-ai/es-core"
|
||||
import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
|
||||
|
||||
const pluginDir = path.resolve(import.meta.dir, "..")
|
||||
const call = (name: string, args: Record<string, unknown> = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@`
|
||||
|
||||
const GATES_PAGE =
|
||||
"<html><head><title>Gates</title></head><body><main><p>Mechanical gates catch regressions before review, which keeps autonomous agents honest under heavy load.</p></main></body></html>"
|
||||
const ENTITY_PAGE = "<html><body><p>Fish & chips are tasty indeed yum.</p></body></html>"
|
||||
|
||||
let state: string
|
||||
let h: Harness
|
||||
let server: ReturnType<typeof Bun.serve>
|
||||
let previousLoopback: string | undefined
|
||||
|
||||
beforeAll(async () => {
|
||||
state = await mkdtemp(path.join(tmpdir(), "es-evidence-187-host-"))
|
||||
server = Bun.serve({
|
||||
port: 0,
|
||||
hostname: "127.0.0.1",
|
||||
fetch(req) {
|
||||
const url = new URL(req.url)
|
||||
if (url.pathname === "/gates") return new Response(GATES_PAGE, { headers: { "content-type": "text/html" } })
|
||||
if (url.pathname === "/entity") return new Response(ENTITY_PAGE, { headers: { "content-type": "text/html" } })
|
||||
return new Response("not found", { status: 404 })
|
||||
},
|
||||
})
|
||||
// The direct backend refuses loopback by default (SSRF guard); the
|
||||
// loopback-only fixture server above needs the explicit test allowance.
|
||||
previousLoopback = process.env.ES_RESEARCH_ALLOW_LOOPBACK
|
||||
process.env.ES_RESEARCH_ALLOW_LOOPBACK = "1"
|
||||
h = await boot({
|
||||
git: true,
|
||||
script: directiveScript,
|
||||
plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }],
|
||||
files: { "README.md": "# evidence 187\n" },
|
||||
})
|
||||
}, 60_000)
|
||||
|
||||
afterAll(async () => {
|
||||
await h?.close()
|
||||
server?.stop(true)
|
||||
if (previousLoopback === undefined) delete process.env.ES_RESEARCH_ALLOW_LOOPBACK
|
||||
else process.env.ES_RESEARCH_ALLOW_LOOPBACK = previousLoopback
|
||||
await rm(state, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
const gatesUrl = () => `http://127.0.0.1:${server.port}/gates`
|
||||
const note = (content: string) => writeFile(path.join(h.directory, ".factory/research/alpha.md"), content)
|
||||
|
||||
async function fetchHash(pathname: string): Promise<string> {
|
||||
const fetched = await h.run(call("es_research_fetch", { url: `http://127.0.0.1:${server.port}${pathname}` }), {
|
||||
agent: "es-research-alpha",
|
||||
})
|
||||
expect(fetched.tools[0]?.status).toBe("completed")
|
||||
const hash = /sha256:([0-9a-f]{64})/.exec(fetched.tools[0]?.text ?? "")?.[1]
|
||||
expect(hash).toBeDefined()
|
||||
return hash!
|
||||
}
|
||||
|
||||
describe("evidence on the real host (#187 §2)", () => {
|
||||
test("verbatim VERIFIED survives the audit; a paraphrase stays ungrounded and prunes away", async () => {
|
||||
const hash = await fetchHash("/gates")
|
||||
await mkdir(path.join(h.directory, ".factory/research"), { recursive: true })
|
||||
await note(
|
||||
`# Alpha\n\n- Gates catch regressions early [VERIFIED: sha256:${hash} "catch regressions before review"]\n- Gates catch failures ahead of inspection [VERIFIED: sha256:${hash} "catch failures ahead of inspection"]\n`,
|
||||
)
|
||||
const audit = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), {
|
||||
agent: "es-research-alpha",
|
||||
})
|
||||
expect(audit.tools[0]?.status).toBe("completed")
|
||||
expect(audit.tools[0]?.text).toContain("1/2 claims grounded")
|
||||
expect(audit.tools[0]?.text).toContain("not found verbatim")
|
||||
expect(audit.tools[0]?.text).not.toContain("Audit passed")
|
||||
const pruned = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md", prune: true }), {
|
||||
agent: "es-research-alpha",
|
||||
})
|
||||
expect(pruned.tools[0]?.text).toContain("Pruned 1 claim")
|
||||
const again = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), {
|
||||
agent: "es-research-alpha",
|
||||
})
|
||||
expect(again.tools[0]?.text).toContain("Audit passed")
|
||||
expect(again.tools[0]?.text).toContain("1/1 claims grounded")
|
||||
}, 120_000)
|
||||
|
||||
test("a planted unsealed entry is refused: the sealed cache misses and the audit stays ungrounded", async () => {
|
||||
const text = "Planted findings about gate latency on reference hardware."
|
||||
const hash = sha256(normalizeSourceText(text))
|
||||
await mkdir(path.join(h.directory, ".factory/research/sources"), { recursive: true })
|
||||
await writeFile(path.join(h.directory, `.factory/research/sources/${hash}.md`), normalizeSourceText(text))
|
||||
await writeFile(
|
||||
path.join(h.directory, `.factory/research/sources/${hash}.json`),
|
||||
`${JSON.stringify({ sha256: hash, url: "https://evil.test/planted", retrieved: new Date().toISOString(), provider: "fetch", bytes: Buffer.byteLength(normalizeSourceText(text)) })}\n`,
|
||||
)
|
||||
expect(await researchCache(h.directory, state).get(hash)).toBeUndefined()
|
||||
await note(`# Alpha\n\n- Planted latency wins [VERIFIED: sha256:${hash} "gate latency on reference"]\n`)
|
||||
const audit = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), {
|
||||
agent: "es-research-alpha",
|
||||
})
|
||||
expect(audit.tools[0]?.status).toBe("completed")
|
||||
expect(audit.tools[0]?.text).toContain("0/1 claims grounded")
|
||||
expect(audit.tools[0]?.text).toContain("not in the cache")
|
||||
expect(audit.tools[0]?.text).not.toContain("Audit passed")
|
||||
}, 120_000)
|
||||
|
||||
test("entity-at-quote on the host: the decoded & verifies, the encoded & is refused", async () => {
|
||||
const hash = await fetchHash("/entity")
|
||||
const cached = await researchCache(h.directory, state).get(hash)
|
||||
expect(cached?.text).toBe("Fish & chips are tasty indeed yum.")
|
||||
await note(
|
||||
`# Alpha\n\n- Fish and chips [VERIFIED: sha256:${hash} "Fish & chips are tasty"]\n- Fish encoded [VERIFIED: sha256:${hash} "Fish & chips are tasty"]\n`,
|
||||
)
|
||||
const audit = await h.run(call("es_research_audit", { path: ".factory/research/alpha.md" }), {
|
||||
agent: "es-research-alpha",
|
||||
})
|
||||
expect(audit.tools[0]?.status).toBe("completed")
|
||||
expect(audit.tools[0]?.text).toContain("1/2 claims grounded")
|
||||
expect(audit.tools[0]?.text).not.toContain("Audit passed")
|
||||
}, 120_000)
|
||||
|
||||
test("the fetch URL is the loopback fixture server (no external network)", () => {
|
||||
expect(gatesUrl()).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/gates$/)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user