rea/scripts/verify-agent-experience.mjs

import { spawn } from "node:child_process";
import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";

import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js";
import {
  createAgentEvaluationFixtures,
  agentFixtureClaims,
  agentFixtureAnswerInstructions,
} from "../dist/evaluation/AgentEvaluationFixtures.js";
import {
  agentEvaluationPassed,
  summarizeFactualCorrectness,
} from "../dist/evaluation/AgentEvaluationReport.js";
import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";

const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const verifierRun = createVerifierRun();
const evaluationRoot = await mkdtemp(join(tmpdir(), "rea-agent-eval-"));
const fixtureRoot = join(evaluationRoot, "targets");
const skillDestination = join(
  evaluationRoot,
  ".agents/skills/reverse-engineer-anything",
);
const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000);
const codex = process.env.REA_CODEX_CLI ?? "codex";
const optionalModel = process.env.REA_AGENT_EVAL_MODEL;

if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10_000)
  throw new Error("REA_AGENT_EVAL_TIMEOUT_MS must be an integer >= 10000");
const codexVersion = await readCommandVersion(codex);

try {
  await mkdir(fixtureRoot, { recursive: true });
  await cp(
    join(repositoryRoot, "skills/reverse-engineer-anything"),
    skillDestination,
    { recursive: true },
  );
  const requestedScenarioIds = new Set(
    (process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
      .split(",")
      .map((value) => value.trim())
      .filter((value) => value.length > 0),
  );
  const targets = await createTargets(
    fixtureRoot,
    requestedScenarioIds.size === 0 || requestedScenarioIds.has("managed"),
  );
  const allScenarios = [
    {
      id: "native",
      expectedFirstTool: "open_binary",
      requiresEvidence: false,
      requiredAnswerTermGroups: [["unavailable", "could not", "provider"]],
      prompt: `Explain what the shipped native program at ${targets.native} does. Base the answer on artifact analysis, and clearly state any facts that remain unavailable.`,
    },
    {
      id: "asar",
      expectedFirstTool: "analyze_javascript_application",
      requiresEvidence: true,
      requiredAnswerTermGroups: [
        ["profileapi"],
        ["profile:read"],
        ["preload", "contextbridge"],
      ],
      prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
    },
    {
      id: "javascript-export-shape",
      expectedFirstTool: "analyze_javascript_application",
      requiresEvidence: true,
      requiredToolSubsequence: [
        "analyze_javascript_application",
        "analyze_javascript_application",
        "compare_javascript_export_shapes",
      ],
      requiredAnswerTermGroups: [
        ["/depth", "depth"],
        [
          "literal value `1`",
          "literal `1`",
          "depth: 1",
          '"depth": 1',
          "depth = 1",
        ],
        ["static", "inferred"],
        ["runtime", "behavioral probes"],
      ],
      prompt: `Without directly reading target files, compare the default export in parser.mjs between ${targets.javascriptShapeLeft} and ${targets.javascriptShapeRight}. Use REA to analyze each shipped artifact and then compare its exact static export return shapes. State the exact heading-shape change, cite produced Evidence, and distinguish static inference from runtime semantics.`,
    },
    {
      id: "managed",
      expectedFirstTool: "inspect_managed_artifact",
      requiresEvidence: true,
      requiredAnswerTermGroups: [["profile"], ["main", "entry point"]],
      requiredToolSubsequence: [
        "inspect_managed_artifact",
        "inspect_managed_members",
      ],
      prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`,
    },
    {
      id: "browser",
      expectedFirstTool: "list_browser_targets",
      requiresEvidence: false,
      requiredAnswerTermGroups: [
        ["unavailable", "could not", "connection", "endpoint"],
      ],
      prompt:
        "A user-owned page is already open at http://127.0.0.1:3000 through the approved local debugging endpoint http://127.0.0.1:9222. Inspect what is available without navigating or executing page code, and report any authority or availability limits.",
    },
    {
      id: "navigation-context",
      expectedFirstTool: "get_navigation_context",
      requiresEvidence: false,
      requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
      prompt:
        "Report the selected document, current address, and containing procedure from the current native-analysis session in one coherent request. If no native session is available, report that limitation plainly.",
    },
    {
      id: "address-context",
      expectedFirstTool: "inspect_address_context",
      requiresEvidence: false,
      requiredAnswerTermGroups: [["unavailable", "could not", "session"]],
      prompt:
        "Inspect explicit address 0x401000 in the current native-analysis session. Report its analyzed name, containing procedure, regular comment, inline comment, and matching bookmarks. If no native session is available, report that limitation plainly.",
    },
  ];
  const scenarios =
    requestedScenarioIds.size === 0
      ? allScenarios
      : allScenarios.filter(({ id }) => requestedScenarioIds.has(id));
  if (
    requestedScenarioIds.size > 0 &&
    scenarios.length !== requestedScenarioIds.size
  )
    throw new Error(
      `Unknown or duplicate REA_AGENT_EVAL_SCENARIOS value: ${[...requestedScenarioIds].join(",")}`,
    );
  const results = [];
  for (const scenario of scenarios) {
    process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
    const fixtureClaims = agentFixtureClaims(scenario.id, targets);
    const answerInstructions = agentFixtureAnswerInstructions(scenario.id);
    const execution = await runCodex(
      answerInstructions.length === 0
        ? scenario.prompt
        : `${scenario.prompt}\n\n${answerInstructions}`,
    );
    const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
    if (transcriptDirectory !== undefined) {
      const transcriptPath = resolve(
        transcriptDirectory,
        `${scenario.id}.jsonl`,
      );
      await mkdir(dirname(transcriptPath), { recursive: true });
      await writeFile(
        transcriptPath,
        `${execution.events.map((event) => JSON.stringify(event)).join("\n")}\n`,
      );
    }
    const metrics = evaluateCodexEvents(
      execution.events,
      scenario.expectedFirstTool,
      {
        requireEvidence: scenario.requiresEvidence,
        requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
        requiredToolSubsequence: scenario.requiredToolSubsequence,
        forbidInputValidationFailures: true,
        ...(fixtureClaims === undefined ? {} : { fixtureClaims }),
      },
    );
    const result = {
      id: scenario.id,
      expectedFirstTool: scenario.expectedFirstTool,
      requiresEvidence: scenario.requiresEvidence,
      requiredAnswerTermGroups: scenario.requiredAnswerTermGroups,
      requiredToolSubsequence: scenario.requiredToolSubsequence ?? [],
      configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [],
      exitCode: execution.exitCode,
      stderr: execution.stderr,
      ...metrics,
    };
    results.push(result);
    process.stderr.write(
      `${scenario.id}: first=${metrics.firstTool ?? "none"}, calls=${String(metrics.reaCalls.length)}, repeats=${String(metrics.repeatedCallCount)}, validation_failures=${String(metrics.inputValidationFailureCount)}, input_tokens=${String(metrics.inputTokens)}\n`,
    );
  }

  const factualCorrectness = summarizeFactualCorrectness(results);
  const summary = {
    schemaVersion: 3,
    evaluationScope: "routing_workflow_and_configured_fixture_claims",
    factualCorrectness,
    verifier_run: await completeVerifierRun(verifierRun),
    codex,
    codexVersion,
    model: optionalModel ?? null,
    scenarios: results,
    totals: {
      scenarios: results.length,
      naturalUse: results.filter(({ naturalUse }) => naturalUse).length,
      correctFirstTool: results.filter(
        ({ correctFirstTool }) => correctFirstTool,
      ).length,
      repeatedCallCount: results.reduce(
        (total, { repeatedCallCount }) => total + repeatedCallCount,
        0,
      ),
      inputValidationFailureCount: results.reduce(
        (total, { inputValidationFailureCount }) =>
          total + inputValidationFailureCount,
        0,
      ),
      requiredToolSubsequence: results.filter(
        ({ requiredToolSubsequenceMet }) => requiredToolSubsequenceMet,
      ).length,
      inputTokens: results.reduce(
        (total, { inputTokens }) => total + inputTokens,
        0,
      ),
      cachedInputTokens: results.reduce(
        (total, { cachedInputTokens }) => total + cachedInputTokens,
        0,
      ),
      answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) =>
        Boolean(answerHeuristicsMet),
      ).length,
      epistemicCuePresent: results.filter(({ epistemicCuePresent }) =>
        Boolean(epistemicCuePresent),
      ).length,
      factualScenariosPassed: factualCorrectness.counts.passed,
      factualScenariosFailed: factualCorrectness.counts.failed,
      factualScenariosNotAssessed: factualCorrectness.counts.notAssessed,
    },
  };
  const encoded = `${JSON.stringify(summary, null, 2)}\n`;
  process.stdout.write(encoded);
  const reportPath = process.env.REA_AGENT_EVAL_REPORT_PATH;
  if (reportPath !== undefined) {
    await mkdir(dirname(resolve(reportPath)), { recursive: true });
    await writeFile(resolve(reportPath), encoded);
  }

  const failed = results.filter((result) => !agentEvaluationPassed(result));
  if (failed.length > 0)
    throw new Error(
      `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
    );
} finally {
  if (process.env.REA_AGENT_EVAL_KEEP_FIXTURES !== "true")
    await rm(evaluationRoot, { recursive: true, force: true });
}

async function createTargets(root, includeManaged) {
  const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot);

  const managedOutput = join(root, "managed-output");
  if (includeManaged) {
    const managedProject = join(root, "managed-project");
    await mkdir(managedProject, { recursive: true });
    await Promise.all([
      writeFile(
        join(managedProject, "AgentEval.csproj"),
        '<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net8.0</TargetFramework><ImplicitUsings>enable</ImplicitUsings><Nullable>enable</Nullable></PropertyGroup></Project>\n',
      ),
      writeFile(
        join(managedProject, "Program.cs"),
        'namespace AgentEval; public sealed record Profile(string Id); public static class Program { public static void Main() => Console.WriteLine(new Profile("fixture")); }\n',
      ),
    ]);
    await runProcess(
      "dotnet",
      [
        "build",
        join(managedProject, "AgentEval.csproj"),
        "--configuration",
        "Release",
        "--output",
        managedOutput,
        "--nologo",
      ],
      root,
      120_000,
    );
  }
  return {
    native: "/bin/true",
    ...fixtures,
    managed: join(managedOutput, "AgentEval.dll"),
  };
}

async function runCodex(prompt) {
  const arguments_ = [
    "exec",
    "--ignore-user-config",
    "--ignore-rules",
    "--ephemeral",
    "--skip-git-repo-check",
    "--sandbox",
    "read-only",
    "--json",
    "--color",
    "never",
    "-C",
    evaluationRoot,
    "-c",
    'approval_policy="never"',
    "-c",
    `mcp_servers.rea.command=${JSON.stringify(process.execPath)}`,
    "-c",
    `mcp_servers.rea.args=${JSON.stringify([join(repositoryRoot, "scripts/rea.mjs"), "mcp"])}`,
    "-c",
    `mcp_servers.rea.startup_timeout_sec=${String(MCP_STARTUP_POLICY.codexStartupTimeoutSeconds)}`,
    ...(optionalModel === undefined ? [] : ["--model", optionalModel]),
    prompt,
  ];
  const child = spawn(codex, arguments_, {
    cwd: evaluationRoot,
    env: process.env,
    stdio: ["ignore", "pipe", "pipe"],
  });
  const events = [];
  const stderr = [];
  const lines = createInterface({ input: child.stdout });
  const linesClosed = new Promise((resolveClose) =>
    lines.once("close", resolveClose),
  );
  lines.on("line", (line) => {
    try {
      events.push(JSON.parse(line));
    } catch {
      stderr.push(`non-json stdout: ${line}`);
    }
  });
  child.stderr.on("data", (chunk) => {
    if (stderr.join("").length < 32_768) stderr.push(chunk.toString("utf8"));
  });
  const exitCode = await new Promise((resolveExit, reject) => {
    const timeout = setTimeout(() => {
      child.kill("SIGTERM");
      reject(
        new Error(`Codex evaluation timed out after ${String(timeoutMs)}ms`),
      );
    }, timeoutMs);
    child.once("error", (cause) => {
      clearTimeout(timeout);
      reject(cause);
    });
    child.once("exit", (code) => {
      clearTimeout(timeout);
      resolveExit(code ?? 1);
    });
  });
  await linesClosed;
  return { events, exitCode, stderr: stderr.join("").slice(0, 32_768) };
}

async function runProcess(command, arguments_, cwd, timeout) {
  const child = spawn(command, arguments_, {
    cwd,
    stdio: ["ignore", "ignore", "pipe"],
  });
  let stderr = "";
  child.stderr.on("data", (chunk) => {
    if (stderr.length < 32_768) stderr += chunk.toString("utf8");
  });
  const code = await new Promise((resolveExit, reject) => {
    const timer = setTimeout(() => {
      child.kill("SIGTERM");
      reject(new Error(`${command} timed out`));
    }, timeout);
    child.once("error", (cause) => {
      clearTimeout(timer);
      reject(cause);
    });
    child.once("exit", (value) => {
      clearTimeout(timer);
      resolveExit(value ?? 1);
    });
  });
  if (code !== 0)
    throw new Error(`${command} failed with ${String(code)}: ${stderr}`);
}

async function readCommandVersion(command) {
  const child = spawn(command, ["--version"], {
    cwd: repositoryRoot,
    stdio: ["ignore", "pipe", "pipe"],
  });
  let stdout = "";
  let stderr = "";
  child.stdout.on("data", (chunk) => {
    if (stdout.length < 4096) stdout += chunk.toString("utf8");
  });
  child.stderr.on("data", (chunk) => {
    if (stderr.length < 4096) stderr += chunk.toString("utf8");
  });
  const code = await new Promise((resolveExit, reject) => {
    const timer = setTimeout(() => {
      child.kill("SIGTERM");
      reject(new Error(`${command} --version timed out`));
    }, 10_000);
    child.once("error", (cause) => {
      clearTimeout(timer);
      reject(cause);
    });
    child.once("exit", (value) => {
      clearTimeout(timer);
      resolveExit(value ?? 1);
    });
  });
  if (code !== 0)
    throw new Error(
      `${command} --version failed with ${String(code)}: ${stderr}`,
    );
  const version = stdout.trim();
  if (version.length === 0)
    throw new Error(`${command} --version returned empty output`);
  return version;
}