fix(remediation-audit/director): eval runner input hardening (Refs #211)

packages/core/src/evals/grade.ts

@@ -48,7 +48,8 @@ export interface EvalTranscript {
   readonly tokens: EvalTokens
 }
 
-const num = (value: unknown): number => (typeof value === "number" && Number.isFinite(value) ? value : 0)
+const num = (value: unknown): number =>
+  typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0
 
 /**
  * Tools whose outputs are raw evidence, not the seat's own words. Quoting

packages/core/src/evals/run.ts

@@ -41,7 +41,8 @@ export const DEFAULT_CASE_STEPS = 6
  * makes one call per step), bounded by the run-wide ceiling.
  */
 export function caseStepCap(caseMaxSteps: number | undefined, ceiling: number): number {
-  return Math.min(caseMaxSteps ?? DEFAULT_CASE_STEPS, ceiling)
+  const own = typeof caseMaxSteps === "number" && Number.isFinite(caseMaxSteps) ? caseMaxSteps : DEFAULT_CASE_STEPS
+  return Math.min(own, ceiling)
 }
 
 export interface EvalWorkspace {
@@ -140,6 +141,11 @@ export async function runEvalCase(
   workspace: EvalWorkspace,
   options: EvalRunOptions,
 ): Promise<EvalResult> {
+  // Fail closed on flag injection (#211): the prompt is a positional argv, so
+  // a leading dash would parse as a CLI flag. Cases are repo files, but the
+  // runner must not depend on their goodwill.
+  if (testCase.prompt.startsWith("-"))
+    throw new Error(`Refusing eval case "${testCase.id}": the prompt must not start with "-".`)
   const child = spawn(
     options.binary,
     ["run", "--standalone", "--format", "json", "--agent", testCase.agent, testCase.prompt],
@@ -167,7 +173,8 @@ export async function runEvalCase(
         child.kill("SIGKILL")
       }
       if (event.type === "step_finish" && typeof event.part?.cost === "number" && Number.isFinite(event.part.cost)) {
-        spent += event.part.cost
+        // Clamp hostile negatives: a negative cost must never unbind spend (#211).
+        spent += Math.max(0, event.part.cost)
         if (options.budgetUSD !== undefined && spent > options.budgetUSD && !overBudget) {
           overBudget = true
           child.kill("SIGKILL")

packages/core/test/evals.test.ts

@@ -90,6 +90,8 @@ describe("transcripts", () => {
     expect(transcript.tokens).toEqual({ input: 150, output: 15, reasoning: 2, cacheRead: 4, cacheWrite: 6 })
     // Missing or non-numeric fields count as zero rather than poisoning the sum.
     expect(readTranscript([{ type: "step_finish", part: { cost: "n/a" } }]).costUSD).toBe(0)
+    // Negative costs clamp to zero: a hostile stream must not unbind spend (#211).
+    expect(readTranscript([{ type: "step_finish", part: { cost: -5 } }]).costUSD).toBe(0)
   })
 
   test("a completed call's output is graded text, except raw evidence and errored calls", () => {
@@ -133,6 +135,10 @@ describe("step caps", () => {
     expect(caseStepCap(undefined, 16)).toBe(DEFAULT_CASE_STEPS)
     expect(caseStepCap(undefined, 3)).toBe(3)
   })
+
+  test("a NaN maxSteps falls back to the default instead of disabling the cap (#211)", () => {
+    expect(caseStepCap(Number.NaN, 16)).toBe(DEFAULT_CASE_STEPS)
+  })
 })
 
 describe("fire graders", () => {