Codewhale/scripts/review-eval.mjs

#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";

const SCHEMA = "codewhale.review-eval.corpus/1";
const RUN_SCHEMA = "codewhale.review-eval.run/1";
const GRADE_SCHEMA = "codewhale.review-eval.grade/1";
const DEFAULT_WINDOW_LINES = 3;

const CODE_EXT = /\.(?:rs|mjs|cjs|js|jsx|ts|tsx|py|go|java|kt|kts|swift|c|cc|cpp|h|hpp|cs|rb|php|scala|zig|lua|sh|bash|zsh|sql|toml|ya?ml|json|wl|cu)$/i;
const SKIP_PATH = /(?:^|\/)(?:CHANGELOG|Cargo\.lock|package-lock\.json|pnpm-lock\.yaml|bun\.lockb?)(?:\.|$)|(?:^|\/)(?:target|node_modules|dist|build|\.git)\//;

function usage() {
  return [
    "review-eval — escaped-defect corpus and reviewer replay for Codewhale review",
    "",
    "  node scripts/review-eval.mjs corpus  --repo <path> [--since YYYY-MM-DD] [--until YYYY-MM-DD]",
    "                                       [--limit N] [--cache <path>] [--out <path>] [--fetch]",
    "  node scripts/review-eval.mjs replay  --corpus <path> --reviewer cli|hosted-floor|skill --out <dir>",
    "                                       [--binary codewhale] [--model M] [--provider P] [--max-cases N]",
    "                                       [--money-cap-usd X] [--hosted-root <codewhale-platform>]",
    "  node scripts/review-eval.mjs grade   --corpus <path> --run <dir> [--window 3] [--out <path>]",
    "  node scripts/review-eval.mjs report  --grade <path> [--baseline <path>]",
    "  node scripts/review-eval.mjs all     --repo <path> --reviewer cli --out <dir> [corpus/replay/grade flags]",
    "",
    "Exit codes: 0 ok, 1 usage or IO error, 2 budget cap reached (partial run retained).",
  ].join("\n");
}

function parseArgs(argv) {
  const command = argv[0] && !argv[0].startsWith("-") ? argv[0] : "help";
  const flags = {};
  for (let index = command === "help" ? 0 : 1; index < argv.length; index += 1) {
    const token = argv[index];
    if (!token.startsWith("--")) continue;
    const [rawKey, inline] = token.slice(2).split("=", 2);
    const key = rawKey.replace(/-([a-z])/g, (_m, c) => c.toUpperCase());
    if (inline !== undefined) {
      flags[key] = inline;
      continue;
    }
    const next = argv[index + 1];
    if (next && !next.startsWith("--")) {
      flags[key] = next;
      index += 1;
    } else {
      flags[key] = true;
    }
  }
  return { command, flags };
}

function git(repo, args, { allowFailure = false } = {}) {
  const result = spawnSync("git", ["-C", repo, ...args], {
    encoding: "utf8",
    maxBuffer: 512 * 1024 * 1024,
  });
  if (result.status !== 0) {
    if (allowFailure) return "";
    throw new Error(`git ${args.join(" ")} failed: ${String(result.stderr || "").trim()}`);
  }
  return result.stdout;
}

function gitBuffer(repo, args) {
  return spawnSync("git", ["-C", repo, ...args], { maxBuffer: 512 * 1024 * 1024 }).stdout;
}

function nullSeparated(text) {
  return String(text || "").split("\u0000").filter((entry) => entry.length > 0);
}

function readCache(path) {
  if (!path || !existsSync(path)) return { commits: {}, prs: {} };
  try {
    const parsed = JSON.parse(readFileSync(path, "utf8"));
    return { commits: parsed.commits || {}, prs: parsed.prs || {} };
  } catch {
    return { commits: {}, prs: {} };
  }
}

function writeCache(path, cache) {
  if (!path) return;
  mkdirSync(dirname(resolve(path)), { recursive: true });
  writeFileSync(path, JSON.stringify(cache));
}

function parseHunks(diffText) {
  const hunks = [];
  let path = "";
  let oldPath = "";
  for (const line of String(diffText || "").split("\n")) {
    if (line.startsWith("diff --git ")) {
      path = "";
      oldPath = "";
      continue;
    }
    if (line.startsWith("--- ")) {
      oldPath = line.slice(4).replace(/^a\//, "");
      continue;
    }
    if (line.startsWith("+++ ")) {
      const next = line.slice(4).trim();
      path = next === "/dev/null" ? "" : next.replace(/^b\//, "");
      continue;
    }
    const header = line.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/);
    if (!header) continue;
    hunks.push({
      path: path || oldPath,
      oldStart: Number(header[1]),
      oldCount: Number(header[2] ?? 1),
      newStart: Number(header[3]),
      newCount: Number(header[4] ?? 1),
    });
  }
  return hunks;
}

function addedLineSet(repo, base, head) {
  const diff = git(repo, ["diff", "--no-color", "-U0", base, head], { allowFailure: true });
  const added = new Map();
  let path = "";
  let newLine = 0;
  for (const line of diff.split("\n")) {
    if (line.startsWith("+++ ")) {
      const next = line.slice(4).trim();
      path = next === "/dev/null" ? "" : next.replace(/^b\//, "");
      continue;
    }
    const header = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
    if (header) {
      newLine = Number(header[1]);
      continue;
    }
    if (!path) continue;
    if (line.startsWith("+") && !line.startsWith("+++")) {
      if (!added.has(path)) added.set(path, new Set());
      added.get(path).add(newLine);
      newLine += 1;
      continue;
    }
    if (line.startsWith(" ") || line === "") continue;
  }
  return added;
}

function looksReviewable(path) {
  if (!path || SKIP_PATH.test(path)) return false;
  if (/\.(?:md|mdx|txt|lock|snap)$/i.test(path)) return false;
  return CODE_EXT.test(path);
}

function commitSubject(repo, sha) {
  return git(repo, ["show", "-s", "--format=%s", sha], { allowFailure: true }).trim();
}

function prFromSubject(subject) {
  const squash = String(subject || "").match(/^(.*)\(#(\d+)\)\s*$/);
  if (squash) return { pr: Number(squash[2]), via: "squash" };
  const merge = String(subject || "").match(/^Merge pull request #(\d+) /);
  if (merge) return { pr: Number(merge[1]), via: "merge" };
  return null;
}

function prBaseFor(repo, record, sha) {
  if (record.base_sha) return record.base_sha;
  const anchor = record.merge_sha || sha;
  const base = git(repo, ["rev-parse", `${anchor}^1`], { allowFailure: true }).trim();
  if (base) record.base_sha = base;
  return base || "";
}

function prSize(repo, base, head) {
  if (!base || !head) return null;
  const numstat = git(repo, ["diff", "--numstat", base, head], { allowFailure: true });
  if (!numstat) return null;
  let files = 0;
  let lines = 0;
  for (const row of numstat.split("\n")) {
    const match = row.match(/^(\d+|-)\t(\d+|-)\t/);
    if (!match) continue;
    files += 1;
    lines += Number(match[1] === "-" ? 0 : match[1]) + Number(match[2] === "-" ? 0 : match[2]);
  }
  return { files, lines };
}

function firstParentMerge(repo, sha, branch) {
  const merges = git(repo, ["rev-list", "--merges", "--ancestry-path", `${sha}..${branch}`], {
    allowFailure: true,
  })
    .split("\n")
    .filter(Boolean);
  const oldest = merges.length ? merges[merges.length - 1] : "";
  return oldest ? [oldest] : [];
}

function prForCommit(repo, sha, branch, cache) {
  if (cache.commits[sha]) return cache.commits[sha];
  const own = prFromSubject(commitSubject(repo, sha));
  if (own) {
    cache.commits[sha] = own;
    return own;
  }
  const merges = firstParentMerge(repo, sha, branch)
  for (const merge of merges) {
    const found = prFromSubject(commitSubject(repo, merge));
    if (found) {
      const record = { pr: found.pr, via: "ancestry", merge_sha: merge };
      cache.commits[sha] = record;
      return record;
    }
  }
  cache.commits[sha] = { pr: null, via: null };
  return cache.commits[sha];
}

function fetchPullRef(repo, prNumber) {
  const ref = `refs/cw-eval/pr-${prNumber}`;
  const exists = spawnSync("git", ["-C", repo, "rev-parse", "--verify", "--quiet", ref], {
    encoding: "utf8",
  });
  if (exists.status === 0) return ref;
  const fetched = spawnSync(
    "git",
    ["-C", repo, "fetch", "--no-tags", "origin", `refs/pull/${prNumber}/head:${ref}`],
    { encoding: "utf8" },
  );
  if (fetched.status !== 0) return "";
  return ref;
}

function fileLinesAt(repo, sha, path) {
  const out = git(repo, ["show", `${sha}:${path}`], { allowFailure: true });
  return out === "" ? null : out.split("\n");
}

function locateAnchor(repo, headSha, path, text, nearLine) {
  const lines = fileLinesAt(repo, headSha, path);
  if (!lines) return null;
  const needle = String(text || "").trim();
  if (!needle) return null;
  const hits = [];
  for (let index = 0; index < lines.length; index += 1) {
    if (lines[index].trim() === needle) hits.push(index + 1);
  }
  if (!hits.length) return null;
  hits.sort((left, right) => Math.abs(left - nearLine) - Math.abs(right - nearLine));
  return { line: hits[0], ambiguous: hits.length > 1 };
}

function buildCorpus(flags) {
  const repo = resolve(String(flags.repo || "."));
  const branch = String(flags.branch || "origin/main");
  const since = String(flags.since || "2026-08-01");
  const until = flags.until ? String(flags.until) : "";
  const limit = flags.limit ? Number(flags.limit) : 0;
  const maxHunksPerFix = flags.maxHunksPerFix ? Number(flags.maxHunksPerFix) : 8;
  const maxPrFiles = flags.maxFiles ? Number(flags.maxFiles) : 40;
  const maxPrLines = flags.maxLines ? Number(flags.maxLines) : 4000;
  const cache = readCache(flags.cache ? resolve(String(flags.cache)) : "");
  const stats = {
    fix_commits_scanned: 0,
    hunks_scanned: 0,
    blamed_lines: 0,
    sized_prs: 0,
    pr_lines_total: 0,
    dropped: {
      no_blame: 0,
      no_pr: 0,
      not_reviewable: 0,
      pr_too_large: 0,
      pr_mismatch: 0,
      no_head_sha: 0,
      anchor_not_found: 0,
      anchor_not_introduced: 0,
      anchor_ambiguous: 0,
      same_pr: 0,
      duplicate_case: 0,
    },
  };
  const logArgs = ["log", "--no-merges", "--grep=^fix", "--since", since, "--format=%H"];
  if (until) logArgs.push("--until", until);
  logArgs.push(branch);
  const fixShas = git(repo, logArgs).split("\n").filter(Boolean);
  const cases = new Map();

  for (const fixSha of fixShas) {
    if (limit && stats.fix_commits_scanned >= limit) break;
    stats.fix_commits_scanned += 1;
    const diff = git(repo, ["show", "-U0", "--format=", "--no-color", fixSha], { allowFailure: true });
    const hunks = parseHunks(diff)
      .filter((hunk) => hunk.oldCount > 0 && looksReviewable(hunk.path))
      .slice(0, maxHunksPerFix);
    if (!hunks.length) continue;
    stats.hunks_scanned += hunks.length;
    const fixSubject = commitSubject(repo, fixSha);
    for (const hunk of hunks) {
      const blame = git(
        repo,
        ["blame", "--porcelain", `-L${hunk.oldStart},${hunk.oldStart + hunk.oldCount - 1}`, `${fixSha}^`, "--", hunk.path],
        { allowFailure: true },
      );
      if (!blame) {
        stats.dropped.no_blame += 1;
        continue;
      }
      const entries = [];
      let current = null;
      for (const line of blame.split("\n")) {
        const header = line.match(/^([0-9a-f]{40}) (\d+) (\d+)(?: (\d+))?$/);
        if (header) {
          current = {
            sha: header[1],
            originalLine: Number(header[2]),
            finalLine: Number(header[3]),
            text: "",
            filename: hunk.path,
          };
          entries.push(current);
          continue;
        }
        if (current && line.startsWith("filename ")) current.filename = line.slice(9);
        if (current && line.startsWith("\t")) {
          stats.blamed_lines += 1;
          current.text = line.slice(1);
        }
      }
      for (const entry of entries) {
        if (!entry.text.trim()) continue;
        const found = prForCommit(repo, entry.sha, branch, cache);
        if (!found || !found.pr) {
          stats.dropped.no_pr += 1;
          continue;
        }
        const pr = Number(found.pr);
        const key = `${pr}:${entry.filename}:${entry.originalLine}`;
        if (cases.has(key)) {
          stats.dropped.duplicate_case += 1;
          continue;
        }
        if (flags.fetch) fetchPullRef(repo, pr);
        const headRef = flags.fetch ? `refs/cw-eval/pr-${pr}` : "";
        const headSha = headRef
          ? git(repo, ["rev-parse", headRef], { allowFailure: true }).trim()
          : "";
        // A "(#NNNN)" subject or an ancestry walk can name a pull request in a
        // different repository of this history, so the introducing commit must
        // be proven to be on that pull request's head before it is a case. A
        // pull request whose head cannot be fetched is not replayable at all,
        // and admitting it would put a case in the corpus that no reviewer can
        // ever be shown.
        if (!headSha) {
          stats.dropped.no_head_sha += 1;
          cases.set(`bad:${pr}`, true);
          continue;
        }
        const contained = spawnSync(
          "git",
          ["-C", repo, "merge-base", "--is-ancestor", entry.sha, headSha],
        );
        if (contained.status !== 0) {
          stats.dropped.pr_mismatch += 1;
          cases.set(`bad:${pr}`, true);
          continue;
        }
        let size = null;
        if (headSha && !cases.has(`size:${pr}`)) {
          const base = prBaseFor(repo, found, entry.sha);
          size = prSize(repo, base, headSha);
          cases.set(`size:${pr}`, size);
          if (size && (size.files > maxPrFiles || size.lines > maxPrLines)) {
            stats.dropped.pr_too_large += 1;
            cases.set(`large:${pr}`, true);
          }
        }
        if (cases.get(`large:${pr}`)) continue;
        if (size) {
          stats.sized_prs += 1;
          stats.pr_lines_total += size.lines;
        }
        let introduced = null;
        if (headSha) {
          const base = prBaseFor(repo, found, entry.sha);
          if (base) introduced = addedLineSet(repo, base, headSha);
        }
        const anchor = headSha
          ? locateAnchor(repo, headSha, entry.filename, entry.text, entry.originalLine)
          : null;
        if (headSha && !anchor) {
          stats.dropped.anchor_not_found += 1;
          continue;
        }
        if (anchor?.ambiguous) stats.dropped.anchor_ambiguous += 1;
        if (introduced) {
          const lines = introduced.get(entry.filename);
          const target = anchor ? anchor.line : entry.originalLine;
          if (!lines || !lines.has(target)) {
            stats.dropped.anchor_not_introduced += 1;
            continue;
          }
        }
        cases.set(key, {
          id: `${pr}:${entry.filename}:${entry.originalLine}`,
          pr,
          pr_source: found.via,
          head_sha: headSha || null,
          merge_sha: found.merge_sha || null,
          introduced_by: entry.sha,
          fix: {
            sha: fixSha,
            subject: fixSubject,
            path: hunk.path,
            line: entry.originalLine,
          },
          anchor: anchor
            ? { path: entry.filename, line: anchor.line, text: entry.text }
            : { path: entry.filename, line: entry.originalLine, text: entry.text },
          anchor_ambiguous: Boolean(anchor?.ambiguous),
        });
      }
    }
  }
  writeCache(flags.cache ? resolve(String(flags.cache)) : "", cache);
  const byPr = new Map();
  for (const entry of cases.values()) {
    if (!entry || !entry.pr || typeof entry.pr !== "number") continue;
    const existing = byPr.get(entry.pr);
    if (!existing) {
      byPr.set(entry.pr, { ...entry, anchors: [entry.anchor], fixes: [entry.fix] });
      continue;
    }
    existing.anchors.push(entry.anchor);
    existing.fixes.push(entry.fix);
  }
  const corpus = {
    schema: SCHEMA,
    generated_at: new Date().toISOString(),
    repo: {
      path: repo,
      remote: git(repo, ["config", "--get", "remote.origin.url"], { allowFailure: true }).trim(),
      branch,
      head: git(repo, ["rev-parse", branch], { allowFailure: true }).trim(),
    },
    window: { since, until: until || null },
    stats,
    cases: [...byPr.values()].sort((left, right) => left.pr - right.pr),
  };
  const out = flags.out ? resolve(String(flags.out)) : join(process.cwd(), "review-eval-corpus.json");
  mkdirSync(dirname(out), { recursive: true });
  writeFileSync(out, JSON.stringify(corpus, null, 2));
  process.stdout.write(
    `${JSON.stringify(
      {
        out,
        cases: corpus.cases.length,
        prs: corpus.cases.length,
        stats,
      },
      null,
      2,
    )}\n`,
  );
  return corpus;
}

function parseCliPayload(stdout) {
  const text = String(stdout || "");
  const start = text.indexOf("{");
  if (start < 0) return null;
  for (let end = text.length; end > start; end -= 1) {
    if (text[end - 1] !== "}") continue;
    try {
      return JSON.parse(text.slice(start, end));
    } catch {
      continue;
    }
  }
  return null;
}

function claimsFromStructured(review) {
  if (!review || typeof review !== "object") return [];
  const claims = [];
  for (const issue of Array.isArray(review.issues) ? review.issues : []) {
    claims.push({
      path: typeof issue?.path === "string" ? issue.path : "",
      line: Number.isInteger(issue?.line) ? issue.line : null,
      severity: String(issue?.severity || ""),
      body: [issue?.title, issue?.description].filter(Boolean).join(" — "),
      origin: "issue",
    });
  }
  for (const suggestion of Array.isArray(review.suggestions) ? review.suggestions : []) {
    claims.push({
      path: typeof suggestion?.path === "string" ? suggestion.path : "",
      line: Number.isInteger(suggestion?.line) ? suggestion.line : null,
      severity: "suggestion",
      body: String(suggestion?.suggestion || ""),
      origin: "suggestion",
    });
  }
  return claims;
}

function prFiles(pr, repoSlug) {
  const result = spawnSync(
    "gh",
    ["api", "--paginate", `repos/${repoSlug}/pulls/${pr}/files?per_page=100`],
    { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 },
  );
  if (result.status !== 0) return [];
  const pages = [];
  let buffer = "";
  for (const line of String(result.stdout || "").split("\n")) {
    buffer += line;
    try {
      const parsed = JSON.parse(buffer);
      if (Array.isArray(parsed)) pages.push(...parsed);
      buffer = "";
    } catch {
      buffer += "\n";
    }
  }
  return pages;
}

function prView(pr, repoSlug) {
  const result = spawnSync(
    "gh",
    ["pr", "view", String(pr), "--repo", repoSlug, "--json", "number,title,body,headRefOid,baseRefOid,files"],
    { encoding: "utf8" },
  );
  if (result.status !== 0) return null;
  try {
    return JSON.parse(result.stdout);
  } catch {
    return null;
  }
}

function repoSlugFor(corpus) {
  const remote = String(corpus?.repo?.remote || "");
  const match = remote.match(/github\.com[:/]([^/]+\/[^/.]+)(?:\.git)?$/);
  return match ? match[1] : "";
}

function runCliCase(entry, corpus, flags) {
  const slug = repoSlugFor(corpus);
  const binary = String(flags.binary || "codewhale");
  const args = ["review", "--pr", String(entry.pr)];
  if (slug) args.push("--repo", slug);
  args.push("--json", "--max-passes", String(flags.maxPasses || 1));
  if (flags.model) args.push("--model", String(flags.model));
  if (flags.provider) args.push("--provider", String(flags.provider));
  const started = Date.now();
  const result = spawnSync(binary, args, {
    cwd: resolve(String(flags.workspace || corpus.repo.path)),
    encoding: "utf8",
    maxBuffer: 256 * 1024 * 1024,
    timeout: Number(flags.timeoutSeconds || 900) * 1000,
    env: process.env,
  });
  const wallMs = Date.now() - started;
  const payload = parseCliPayload(result.stdout);
  return {
    status: result.status,
    wall_ms: wallMs,
    payload,
    claims: payload?.review ? claimsFromStructured(payload.review) : [],
    usage: payload?.usage || null,
    stop_reason: payload?.stop_reason || null,
    summary: payload?.review?.summary || "",
    overall: payload?.review?.overall_assessment || "",
    stderr: String(result.stderr || "").slice(-4000),
    error: result.error ? String(result.error.message || result.error) : null,
  };
}

async function runHostedFloorCase(entry, corpus, flags, moduleCache) {
  const root = String(flags.hostedRoot || "");
  if (!root) throw new Error("hosted-floor replay needs --hosted-root <codewhale-platform checkout>");
  const modulePath = join(
    resolve(root),
    "services/control-plane/src/integrations/github/pr-review.mjs",
  );
  if (!existsSync(modulePath)) throw new Error(`hosted review module not found at ${modulePath}`);
  moduleCache.modules ??= new Map();
  if (!moduleCache.modules.has(modulePath)) {
    moduleCache.modules.set(modulePath, await import(`file://${modulePath}`));
  }
  const module = moduleCache.modules.get(modulePath);
  const view = prView(entry.pr, repoSlugFor(corpus));
  if (!view) return { status: 1, wall_ms: 0, claims: [], error: "gh_pr_view_failed" };
  const started = Date.now();
  const comments = await module.buildPullRequestReviewComments({
    files: prFiles(entry.pr, repoSlugFor(corpus)),
    pullRequest: { number: view.number, title: view.title, headSha: view.headRefOid },
  });
  return {
    status: 0,
    wall_ms: Date.now() - started,
    claims: comments.map((comment) => ({
      path: comment.path,
      line: comment.line,
      side: comment.side,
      severity: comment.severity,
      body: comment.body,
      origin: "hosted-floor",
    })),
    usage: null,
  };
}

function runSkillCase(entry, corpus, flags) {
  const skillPath = join(
    resolve(String(flags.workspace || corpus.repo.path)),
    "crates/tui/assets/skills/review/SKILL.md",
  );
  const skill = existsSync(skillPath) ? readFileSync(skillPath, "utf8") : "";
  const slug = repoSlugFor(corpus);
  const view = prView(entry.pr, slug);
  const diff = spawnSync("gh", ["pr", "diff", String(entry.pr), "--repo", slug], {
    encoding: "utf8",
    maxBuffer: 256 * 1024 * 1024,
  });
  const prompt = [
    skill,
    "",
    "---",
    `Pull request #${entry.pr} in ${slug}`,
    `Title: ${view?.title || ""}`,
    "Description:",
    String(view?.body || "").slice(0, 4000),
    "",
    "Diff:",
    String(diff.stdout || "").slice(0, Number(flags.maxChars || 200000)),
  ].join("\n");
  const binary = String(flags.binary || "codewhale");
  const args = ["exec", "--json"];
  if (flags.model) args.push("--model", String(flags.model));
  if (flags.provider) args.push("--provider", String(flags.provider));
  args.push(prompt);
  const started = Date.now();
  const result = spawnSync(binary, args, {
    cwd: resolve(String(flags.workspace || corpus.repo.path)),
    encoding: "utf8",
    maxBuffer: 256 * 1024 * 1024,
    timeout: Number(flags.timeoutSeconds || 900) * 1000,
  });
  const payload = parseCliPayload(result.stdout);
  const text = String(payload?.result || payload?.content || result.stdout || "");
  return {
    status: result.status,
    wall_ms: Date.now() - started,
    claims: claimsFromMarkdown(text),
    usage: payload?.usage || null,
    raw_excerpt: text.slice(0, 4000),
  };
}

function claimsFromMarkdown(text) {
  const claims = [];
  const linePattern = /`?([\w./-]+\.\w+):(\d+)`?/g;
  for (const block of String(text || "").split(/\n(?=\s*(?:\d+[.)]|[-*])\s)/)) {
    const match = block.match(linePattern);
    if (!match) continue;
    const [, path, line] = match[0].match(/`?([\w./-]+\.\w+):(\d+)`?/) || [];
    if (!path) continue;
    claims.push({ path, line: Number(line), severity: "", body: block.trim().slice(0, 1200), origin: "skill" });
  }
  return claims;
}

async function replay(flags) {
  const corpus = JSON.parse(readFileSync(resolve(String(flags.corpus)), "utf8"));
  const reviewer = String(flags.reviewer || "cli");
  const outDir = resolve(String(flags.out));
  mkdirSync(outDir, { recursive: true });
  const maxCases = flags.maxCases ? Number(flags.maxCases) : corpus.cases.length;
  const moneyCap = flags.moneyCapUsd ? Number(flags.moneyCapUsd) : 0;
  const moduleCache = {};
  const runs = [];
  let spend = 0;
  for (const entry of corpus.cases.slice(0, maxCases)) {
    if (moneyCap > 0 && spend >= moneyCap) {
      process.stderr.write(`money cap ${moneyCap} USD reached after ${runs.length} case(s)\n`);
      break;
    }
    const started = new Date().toISOString();
    let record;
    try {
      if (reviewer === "cli") record = runCliCase(entry, corpus, flags);
      else if (reviewer === "hosted-floor") record = await runHostedFloorCase(entry, corpus, flags, moduleCache);
      else if (reviewer === "skill") record = runSkillCase(entry, corpus, flags);
      else throw new Error(`unknown reviewer ${reviewer}`);
    } catch (error) {
      record = { status: 1, wall_ms: 0, claims: [], error: String(error?.message || error) };
    }
    const cost = estimateCost(record.usage, flags);
    spend += cost;
    const runRecord = {
      id: entry.id,
      pr: entry.pr,
      reviewer,
      started_at: started,
      status: record.status,
      wall_ms: record.wall_ms,
      claims: record.claims || [],
      usage: record.usage || null,
      estimated_cost_usd: Number(cost.toFixed(6)),
      stop_reason: record.stop_reason || null,
      summary: record.summary || "",
      overall: record.overall || "",
      error: record.error || null,
      stderr: record.stderr || "",
      raw_excerpt: record.raw_excerpt || "",
    };
    writeFileSync(join(outDir, `${entry.pr}.json`), JSON.stringify(runRecord, null, 2));
    runs.push(runRecord);
    process.stderr.write(
      `${reviewer} ${entry.id} status=${record.status} claims=${runRecord.claims.length} ${record.wall_ms}ms\n`,
    );
  }
  const manifest = {
    schema: RUN_SCHEMA,
    reviewer,
    corpus: resolve(String(flags.corpus)),
    model: flags.model || null,
    provider: flags.provider || null,
    generated_at: new Date().toISOString(),
    money_cap_usd: moneyCap || null,
    cases_attempted: runs.length,
    estimated_cost_usd: Number(spend.toFixed(6)),
    cases: runs.map((run) => run.id),
  };
  writeFileSync(join(outDir, "run.json"), JSON.stringify(manifest, null, 2));
  process.stdout.write(`${JSON.stringify(manifest, null, 2)}\n`);
  return moneyCap > 0 && spend >= moneyCap ? 2 : 0;
}

function estimateCost(usage, flags) {
  const inputRate = flags.inputUsdPerMTok ? Number(flags.inputUsdPerMTok) : 0;
  const outputRate = flags.outputUsdPerMTok ? Number(flags.outputUsdPerMTok) : 0;
  if (!usage) return 0;
  const input = Number(usage.input_tokens || 0) + Number(usage.cache_read_input_tokens || 0);
  const output = Number(usage.output_tokens || 0);
  return (input / 1_000_000) * inputRate + (output / 1_000_000) * outputRate;
}

function matchCase(entry, claims, windowLines) {
  let fileMatch = null;
  let lineMatch = null;
  for (const claim of claims) {
    if (!claim.path) continue;
    const anchors = Array.isArray(entry.anchors) ? entry.anchors : [entry.anchor];
    for (const anchor of anchors) {
      if (!anchor?.path) continue;
      if (claim.path === anchor.path || claim.path.endsWith(`/${anchor.path}`) || anchor.path.endsWith(`/${claim.path}`)) {
        fileMatch ??= claim;
        if (Number.isInteger(claim.line) && Math.abs(claim.line - anchor.line) <= windowLines) {
          lineMatch ??= claim;
        }
      }
    }
  }
  return { fileMatch, lineMatch };
}

function words(text) {
  return String(text || "").trim().split(/\s+/).filter(Boolean).length;
}

function grade(flags) {
  const corpus = JSON.parse(readFileSync(resolve(String(flags.corpus)), "utf8"));
  const runDir = resolve(String(flags.run));
  const manifestPath = join(runDir, "run.json");
  const manifest = existsSync(manifestPath) ? JSON.parse(readFileSync(manifestPath, "utf8")) : { reviewer: "unknown" };
  const windowLines = flags.window ? Number(flags.window) : DEFAULT_WINDOW_LINES;
  const rows = [];
  for (const entry of corpus.cases) {
    const file = join(runDir, `${entry.pr}.json`);
    if (!existsSync(file)) continue;
    const run = JSON.parse(readFileSync(file, "utf8"));
    const claims = Array.isArray(run.claims) ? run.claims : [];
    const { fileMatch, lineMatch } = matchCase(entry, claims, windowLines);
    const matchedClaims = claims.filter((claim) =>
      (Array.isArray(entry.anchors) ? entry.anchors : [entry.anchor]).some(
        (anchor) =>
          anchor?.path &&
          Number.isInteger(claim.line) &&
          (claim.path === anchor.path || claim.path.endsWith(`/${anchor.path}`)) &&
          Math.abs(claim.line - anchor.line) <= windowLines,
      ),
    );
    rows.push({
      id: entry.id,
      pr: entry.pr,
      status: run.status,
      claims: claims.length,
      words: claims.reduce((total, claim) => total + words(claim.body), 0),
      caught: Boolean(lineMatch),
      caught_file: Boolean(fileMatch),
      caught_by: lineMatch ? { path: lineMatch.path, line: lineMatch.line, severity: lineMatch.severity } : null,
      claims_in_region: matchedClaims.length,
      precise_share: claims.length ? Number((matchedClaims.length / claims.length).toFixed(3)) : null,
      wall_ms: run.wall_ms || 0,
      cost_usd: run.estimated_cost_usd || 0,
      error: run.error || null,
    });
  }
  const attempted = rows.length;
  const caught = rows.filter((row) => row.caught).length;
  const caughtFile = rows.filter((row) => row.caught_file).length;
  const claimsTotal = rows.reduce((total, row) => total + row.claims, 0);
  const wordsTotal = rows.reduce((total, row) => total + row.words, 0);
  const summary = {
    schema: GRADE_SCHEMA,
    reviewer: manifest.reviewer,
    model: manifest.model || null,
    corpus: resolve(String(flags.corpus)),
    run: runDir,
    window_lines: windowLines,
    graded_at: new Date().toISOString(),
    metrics: {
      cases_attempted: attempted,
      cases_with_claims: rows.filter((row) => row.claims > 0).length,
      recall_at_window: attempted ? Number((caught / attempted).toFixed(4)) : null,
      recall_same_file: attempted ? Number((caughtFile / attempted).toFixed(4)) : null,
      claims_total: claimsTotal,
      claims_in_region: rows.reduce((total, row) => total + row.claims_in_region, 0),
      words_total: wordsTotal,
      claims_per_case: attempted ? Number((claimsTotal / attempted).toFixed(2)) : null,
      words_per_case: attempted ? Math.round(wordsTotal / attempted) : null,
      wall_ms_total: rows.reduce((total, row) => total + row.wall_ms, 0),
      wall_ms_mean: attempted ? Math.round(rows.reduce((total, row) => total + row.wall_ms, 0) / attempted) : null,
      cost_usd_total: Number(rows.reduce((total, row) => total + row.cost_usd, 0).toFixed(6)),
      cost_usd_per_case: attempted
        ? Number((rows.reduce((total, row) => total + row.cost_usd, 0) / attempted).toFixed(6))
        : null,
      failures: rows.filter((row) => row.status !== 0).length,
    },
    rows,
  };
  const out = flags.out ? resolve(String(flags.out)) : join(runDir, "grade.json");
  writeFileSync(out, JSON.stringify(summary, null, 2));
  process.stdout.write(`${JSON.stringify({ out, ...summary.metrics }, null, 2)}\n`);
  return 0;
}

function report(flags) {
  const current = JSON.parse(readFileSync(resolve(String(flags.grade)), "utf8"));
  const lines = [];
  const header = ["metric", current.reviewer, flags.baseline ? "baseline" : "", flags.baseline ? "delta" : ""].filter(Boolean);
  lines.push(header.join(" | "));
  const baseline = flags.baseline
    ? JSON.parse(readFileSync(resolve(String(flags.baseline)), "utf8"))
    : null;
  const keys = [
    "cases_attempted",
    "recall_at_window",
    "recall_same_file",
    "claims_total",
    "claims_per_case",
    "words_per_case",
    "cost_usd_per_case",
    "wall_ms_mean",
    "failures",
  ];
  for (const key of keys) {
    const value = current.metrics?.[key];
    const base = baseline?.metrics?.[key];
    const delta = baseline && typeof value === "number" && typeof base === "number"
      ? (value - base).toFixed(4)
      : "";
    lines.push([key, value ?? "", baseline ? base ?? "" : "", delta].filter((cell) => cell !== "").join(" | "));
  }
  process.stdout.write(`${lines.join("\n")}\n`);
  return 0;
}

async function main() {
  const { command, flags } = parseArgs(process.argv.slice(2));
  if (command === "help" || flags.help) {
    process.stdout.write(`${usage()}\n`);
    return 0;
  }
  if (command === "corpus") {
    buildCorpus(flags);
    return 0;
  }
  if (command === "replay") return replay(flags);
  if (command === "grade") return grade(flags);
  if (command === "report") return report(flags);
  if (command === "all") {
    const outDir = resolve(String(flags.out));
    const corpusPath = join(outDir, "corpus.json");
    flags.out = corpusPath;
    buildCorpus({ ...flags, fetch: flags.fetch ?? true });
    const runDir = join(outDir, `run-${flags.reviewer || "cli"}`);
    const replayCode = await replay({ ...flags, corpus: corpusPath, out: runDir });
    grade({ ...flags, corpus: corpusPath, run: runDir, out: join(runDir, "grade.json") });
    return replayCode;
  }
  process.stderr.write(`${usage()}\n`);
  return 1;
}

main().then(
  (code) => process.exit(code ?? 0),
  (error) => {
    process.stderr.write(`${error?.stack || error}\n`);
    process.exit(1);
  },
);