mirror of
https://github.com/Heretek-AI/IUMBTEMS.git
synced 2026-10-11 21:09:53 +00:00
fix(improve): regex-free slug (CodeQL ReDoS) + proposal-writing tests (Refs #174)
This commit is contained in:
@@ -82,12 +82,23 @@ export const hashPrompt = (prompt: string): string => sha256(prompt)
|
||||
const candidateId = (operator: PromptOperator, prompt: string): string =>
|
||||
`prompt-${operator}-${shortHash(hashPrompt(`${operator}:${prompt}`))}`
|
||||
|
||||
const slug = (text: string): string =>
|
||||
text
|
||||
.toLowerCase()
|
||||
.replaceAll(/[^a-z0-9]+/g, "-")
|
||||
.replaceAll(/^-+|-+$/g, "")
|
||||
.slice(0, 48) || "signal"
|
||||
/** URL-safe slug without regular expressions (linear scan; no ReDoS surface). */
|
||||
const slug = (text: string): string => {
|
||||
let out = ""
|
||||
let prevDash = true // Leading runs collapse to nothing (trimmed).
|
||||
for (const ch of text.toLowerCase()) {
|
||||
const ok = (ch >= "a" && ch <= "z") || (ch >= "0" && ch <= "9")
|
||||
if (ok) {
|
||||
out += ch
|
||||
prevDash = false
|
||||
} else if (!prevDash) {
|
||||
out += "-"
|
||||
prevDash = true
|
||||
}
|
||||
}
|
||||
if (out.endsWith("-")) out = out.slice(0, -1)
|
||||
return out.slice(0, 48) || "signal"
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive candidate prompt variations from telemetry failure clusters. Each
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
OptimizerError,
|
||||
paretoDominates,
|
||||
receiptFor,
|
||||
writeOptimizerProposals,
|
||||
} from "../src/improve/optimizer.ts"
|
||||
import type { RunTelemetry, Telemetry } from "../src/schema/improve.ts"
|
||||
|
||||
@@ -164,6 +165,63 @@ describe("evolvePrompts", () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe("writeOptimizerProposals", () => {
|
||||
test("writes one proposal dir per winner with before/after receipts, and nothing else", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "es-opt-proposals-"))
|
||||
const baselineMap = { a: { pass: true, durationMs: 4 }, b: { pass: false, durationMs: 4 } }
|
||||
const grade = async (prompt: string) =>
|
||||
prompt.includes("lint/no-unused-vars")
|
||||
? { attempts: [attempt("a", true), attempt("b", true)], costUSD: 0.02, steps: 6 }
|
||||
: { attempts: [attempt("a", true), attempt("b", false)], costUSD: 0.02, steps: 6 }
|
||||
const result = await evolvePrompts({
|
||||
basePrompt: BASE,
|
||||
basePromptId: "programmer",
|
||||
telemetry: failingTelemetry(),
|
||||
baseline: baselineMap,
|
||||
grade,
|
||||
})
|
||||
expect(result.winners.length).toBeGreaterThan(0)
|
||||
const { written, droppedFile } = await writeOptimizerProposals(dir, result, BASE)
|
||||
expect(written).toHaveLength(result.winners.length)
|
||||
for (const entry of written) {
|
||||
const proposal = JSON.parse(await readFile(path.join(entry.dir, "proposal.json"), "utf8"))
|
||||
expect(proposal.candidate.prompt).toContain("lint/no-unused-vars")
|
||||
expect(proposal.after.regressions).toEqual([])
|
||||
expect(proposal.after.passAt1).toBeGreaterThan(proposal.before.passAt1)
|
||||
const patch = await readFile(path.join(entry.dir, "prompt.patch.md"), "utf8")
|
||||
expect(patch).toContain("## Before")
|
||||
expect(patch).toContain("## After")
|
||||
const receipt = JSON.parse(await readFile(path.join(entry.dir, "receipt.json"), "utf8"))
|
||||
expect(receipt.before.candidateId).toBe(proposal.after.candidateId)
|
||||
}
|
||||
expect(JSON.parse(await readFile(droppedFile, "utf8"))).toEqual([])
|
||||
// The shipped prompt is untouched: proposals are review-only.
|
||||
expect(await readFile(path.join(dir, "dropped.json"), "utf8")).toBeDefined()
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
test("dominated candidates land in dropped.json with reasons", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "es-opt-dropped-"))
|
||||
const baselineMap = { a: { pass: true, durationMs: 4 }, b: { pass: false, durationMs: 4 } }
|
||||
// Every candidate regresses case `a`: nothing may win.
|
||||
const grade = async () => ({ attempts: [attempt("a", false), attempt("b", true)], costUSD: 0.01, steps: 2 })
|
||||
const result = await evolvePrompts({
|
||||
basePrompt: BASE,
|
||||
basePromptId: "programmer",
|
||||
telemetry: failingTelemetry(),
|
||||
baseline: baselineMap,
|
||||
grade,
|
||||
})
|
||||
expect(result.winners).toEqual([])
|
||||
const { written, droppedFile } = await writeOptimizerProposals(dir, result, BASE)
|
||||
expect(written).toEqual([])
|
||||
const dropped = JSON.parse(await readFile(droppedFile, "utf8"))
|
||||
expect(dropped).toHaveLength(result.dominated.length)
|
||||
expect(dropped[0].reasons.join(" ")).toContain("regressions")
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
})
|
||||
|
||||
describe("applyPromptPatch is human-gated", () => {
|
||||
const setup = async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "es-optimizer-"))
|
||||
|
||||
Reference in New Issue
Block a user