explore/test/node/compare.test.mjs

import assert from "node:assert/strict";
import { join, resolve } from "node:path";
import { spawnSync } from "node:child_process";
import test from "node:test";

import { CompareError, generateComparison } from "../../dist/map/compare.js";
import { getCoverage } from "../../dist/map/store.js";
import { loadBuildWorkspace, validateComparisonDocument } from "../../dist/map/build-workspace.js";
import { createBuildWorkspaceFixture } from "../helpers/build-workspace-fixture.mjs";

const root = resolve(import.meta.dirname, "../..");
const cli = join(root, "dist", "cli.js");
const CAPTURED_AT = "2026-08-25T09:00:00.000Z";

/**
 * In-memory maps for the matcher matrix. Artifacts exist only as map records, which
 * is all validateComparisonDocument inspects; the CLI end-to-end case below uses the
 * on-disk workspace fixture for full artifact validation.
 */
function makeMap(buildId, screens, { version = "sha-" + buildId } = {}) {
  const artifacts = [];
  const observations = [];
  const variants = [];
  const screenRecords = [];
  for (const [position, input] of screens.entries()) {
    const captures = input.captures ?? [{ suffix: "primary", acc: input.acc }];
    const observationIds = [];
    for (const capture of captures) {
      const observationId = `observation-${buildId}-${input.id}-${capture.suffix}`;
      const variantId = `variant-${observationId}`;
      const artifactId = `artifact-${observationId}`;
      artifacts.push({
        id: artifactId,
        kind: "screenshot",
        path: `artifacts/screenshots/${input.id}-${capture.suffix}.png`,
        sha256: "0".repeat(64),
        bytes: 1,
        mimeType: "image/png",
        createdAt: CAPTURED_AT,
      });
      variants.push({
        id: variantId,
        screenId: input.id,
        key: observationId,
        firstSeenAt: CAPTURED_AT,
        lastSeenAt: CAPTURED_AT,
        observationIds: [observationId],
      });
      observations.push({
        id: observationId,
        screenId: input.id,
        variantId,
        fingerprints: {
          combined: `combined-${observationId}`,
          ...(capture.acc ? { accessibility: capture.acc } : {}),
          visual: `visual-${observationId}`,
        },
        capturedAt: CAPTURED_AT,
        surfaceScope: "app",
        elementIds: [],
        elements: [],
        ...(input.screenshot === false ? {} : { screenshotArtifactId: artifactId }),
      });
      observationIds.push(observationId);
    }
    screenRecords.push({
      id: input.id,
      title: input.title ?? input.id.replaceAll("-", " "),
      ...(input.route ? { route: input.route } : {}),
      start: position === 0,
      terminal: false,
      depth: position,
      firstSeenAt: CAPTURED_AT,
      lastSeenAt: CAPTURED_AT,
      visits: 1,
      variantIds: observationIds.map((id) => `variant-${id}`),
      observationIds,
      elementIds: [],
    });
  }
  return {
    contract: "expo.explore.local-map.v1",
    revision: 5,
    run: {
      id: `run-${buildId}`,
      name: `Compare fixture ${buildId}`,
      platform: "ios",
      controller: "agent-device",
      status: "complete",
      createdAt: CAPTURED_AT,
      updatedAt: CAPTURED_AT,
      build: { id: buildId, version, source: "local" },
      environment: { orientation: "portrait", colorScheme: "light" },
      limits: { maxScreens: 20, maxActions: 20, maxDurationMinutes: 5 },
    },
    artifacts,
    screens: screenRecords,
    variants,
    observations,
    elements: [],
    actionAttempts: [],
    transitions: [],
    journeys: [],
    frontier: [],
    gaps: [],
  };
}

function entryById(comparison, id) {
  const entry = comparison.entries.find((candidate) => candidate.id === id);
  assert.ok(entry, `missing entry ${id}: ${comparison.entries.map((e) => e.id).join(", ")}`);
  return entry;
}

test("matches screens by route, then id, then accessibility, then unique title", async () => {
  const baseline = makeMap("main-1111111", [
    { id: "home", route: "/", acc: "acc-home-v1" },
    { id: "settings-legacy", route: "/settings", acc: "acc-settings" },
    { id: "profile", acc: "acc-profile" },
    { id: "misc-a", title: "Payment methods", acc: "acc-pay-v1" },
    { id: "gone", title: "Legacy screen", acc: "acc-gone" },
  ]);
  const current = makeMap("pr-42-2222222", [
    { id: "home", route: "/", acc: "acc-home-v2" },
    { id: "settings", route: "/settings", acc: "acc-settings" },
    { id: "profile", acc: "acc-profile-v2" },
    { id: "misc-b", title: "Payment methods", acc: "acc-pay-v2" },
    { id: "fresh", title: "Brand new screen", acc: "acc-fresh" },
  ]);
  const { comparison, review, summary } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);

  const home = entryById(comparison, "match-home");
  assert.deepEqual(home.baselineScreenIds, ["home"]);
  assert.equal(home.status, "modified");
  assert.equal(home.confidence, 0.95);

  const settings = entryById(comparison, "match-settings");
  assert.deepEqual(settings.baselineScreenIds, ["settings-legacy"]);
  assert.equal(settings.status, "unchanged");

  const profile = entryById(comparison, "match-profile");
  assert.equal(profile.status, "modified");
  assert.equal(profile.confidence, 0.85);

  const pay = entryById(comparison, "match-misc-b");
  assert.deepEqual(pay.baselineScreenIds, ["misc-a"]);
  assert.equal(pay.confidence, 0.65);
  const payReview = review.entries.find((entry) => entry.entryId === "match-misc-b");
  assert.equal(payReview.needsReview, true);
  assert.deepEqual(payReview.matchedBy, ["title"]);

  entryById(comparison, "removed-gone");
  entryById(comparison, "added-fresh");
  assert.deepEqual(summary, {
    entries: 6,
    added: 1,
    removed: 1,
    modified: 3,
    unchanged: 1,
    needsReview: 1,
  });
});

test("matching representative accessibility fingerprints mark a route match unchanged", async () => {
  const baseline = makeMap("main-aaaaaaa", [
    { id: "home", route: "/", acc: "same-tree" },
  ]);
  const current = makeMap("pr-1-bbbbbbb", [
    {
      id: "home",
      route: "/",
      captures: [
        { suffix: "primary", acc: "same-tree" },
        { suffix: "revisit", acc: "other-tree" },
      ],
    },
  ]);
  const { comparison } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);
  assert.equal(entryById(comparison, "match-home").status, "unchanged");
});

test("unmatched screens with plausible counterparts carry review candidates", () => {
  const baseline = makeMap("main-ccccccc", [
    { id: "home", route: "/", acc: "acc-home" },
    { id: "plans", route: "/plans", title: "Plans", acc: "acc-plans" },
  ]);
  const current = makeMap("pr-2-ddddddd", [
    { id: "home", route: "/", acc: "acc-home" },
    { id: "plans-monthly", route: "/plans/monthly", title: "Plans monthly", acc: "acc-a" },
    { id: "plans-yearly", route: "/plans/yearly", title: "Plans yearly", acc: "acc-b" },
  ]);
  const { review } = generateComparison(baseline, current);
  const removedPlans = review.entries.find((entry) => entry.entryId === "removed-plans");
  assert.equal(removedPlans.needsReview, true);
  const candidateIds = (removedPlans.candidates ?? []).map((candidate) => candidate.screenId);
  assert.deepEqual(candidateIds.sort(), ["plans-monthly", "plans-yearly"]);
});

test("a claimed duplicate no longer blocks the remaining unambiguous title match", async () => {
  const baseline = makeMap("main-t111111", [
    { id: "x", route: "/a", title: "T", acc: "ax" },
    { id: "y", title: "T", acc: "ay" },
  ]);
  const current = makeMap("pr-7-t222222", [
    { id: "w", route: "/a", title: "T", acc: "aw" },
    { id: "z", title: "T", acc: "az" },
  ]);
  const { comparison, summary } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);
  assert.deepEqual(entryById(comparison, "match-w").baselineScreenIds, ["x"]);
  assert.deepEqual(entryById(comparison, "match-z").baselineScreenIds, ["y"]);
  assert.equal(summary.added, 0);
  assert.equal(summary.removed, 0);
});

test("a shared variant fingerprint does not hide a changed primary capture", async () => {
  const baseline = makeMap("main-v111111", [
    {
      id: "home",
      route: "/",
      captures: [
        { suffix: "primary", acc: "main-v1" },
        { suffix: "skeleton", acc: "shared-skeleton" },
      ],
    },
  ]);
  const current = makeMap("pr-5-v222222", [
    {
      id: "home",
      route: "/",
      captures: [
        { suffix: "primary", acc: "main-v2" },
        { suffix: "skeleton", acc: "shared-skeleton" },
      ],
    },
  ]);
  const { comparison } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);
  assert.equal(entryById(comparison, "match-home").status, "modified");
});

test("missing accessibility fingerprints report updated with a review flag, not a changed-tree claim", async () => {
  const baseline = makeMap("main-u111111", [{ id: "home", route: "/" }]);
  const current = makeMap("pr-6-u222222", [{ id: "home", route: "/" }]);
  const { comparison, review } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);
  const entry = entryById(comparison, "match-home");
  assert.equal(entry.status, "modified");
  assert.match(entry.rationale, /could not be verified/);
  const reviewEntry = review.entries.find((candidate) => candidate.entryId === "match-home");
  assert.equal(reviewEntry.needsReview, true);
});

test("a terminal run's duration limit uses its recorded lifetime, not file age", () => {
  const map = makeMap("main-d111111", [{ id: "home", route: "/" }]);
  map.run.limits.maxDurationMinutes = 5;
  assert.equal(getCoverage(map).limitReached, undefined);
  map.run.status = "exploring";
  assert.equal(getCoverage(map).limitReached, "duration");
});

test("route duplicated on one side falls through to id matching", async () => {
  const baseline = makeMap("main-eeeeeee", [
    { id: "sheet", route: "/items", acc: "acc-sheet" },
    { id: "items", route: "/items", acc: "acc-items" },
  ]);
  const current = makeMap("pr-3-fffffff", [
    { id: "items", route: "/items", acc: "acc-items-v2" },
    { id: "sheet", route: "/items", acc: "acc-sheet" },
  ]);
  const { comparison } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);
  assert.deepEqual(entryById(comparison, "match-items").baselineScreenIds, ["items"]);
  assert.deepEqual(entryById(comparison, "match-sheet").baselineScreenIds, ["sheet"]);
});

test("refuses maps without build ids, identical build ids, and screens without screenshots", () => {
  const noBuild = makeMap("main-2222222", [{ id: "home", route: "/" }]);
  delete noBuild.run.build.id;
  assert.throws(
    () => generateComparison(noBuild, makeMap("pr-9-3333333", [{ id: "home" }])),
    CompareError,
  );
  assert.throws(
    () => generateComparison(
      makeMap("same-id", [{ id: "home" }]),
      makeMap("same-id", [{ id: "home" }]),
    ),
    /same run\.build\.id/,
  );
  const noShot = makeMap("main-4444444", [{ id: "home", screenshot: false }]);
  assert.throws(
    () => generateComparison(noShot, makeMap("pr-8-5555555", [{ id: "home" }])),
    /no observation with a screenshot/,
  );
});

test("compare CLI emits a comparison, review, and catalog that reload as a workspace", async () => {
  const fixture = await createBuildWorkspaceFixture();
  // The catalog loader refuses mapDir paths that escape the catalog directory,
  // so generated outputs live in the workspace root next to the map bundles.
  const out = fixture.root;
  const result = spawnSync(process.execPath, [
    cli,
    "compare",
    "--baseline", join(fixture.root, "baseline"),
    "--current", join(fixture.root, "current"),
    "--output", join(out, "auto-comparison.json"),
    "--review-output", join(out, "auto-review.json"),
    "--catalog-output", join(out, "auto-builds.json"),
    "--catalog-name", "Compare fixture",
    "--json",
  ], { encoding: "utf8" });
  assert.equal(result.status, 0, result.stderr);
  const payload = JSON.parse(result.stdout);
  assert.equal(payload.ok, true);
  assert.equal(payload.baselineBuildId, "build-1.0.0");
  assert.equal(payload.currentBuildId, "build-2.0.0");
  assert.equal(
    payload.summary.entries,
    payload.summary.added + payload.summary.removed
      + payload.summary.modified + payload.summary.unchanged,
  );

  const workspace = await loadBuildWorkspace(join(out, "auto-builds.json"));
  assert.deepEqual([...workspace.builds.keys()].sort(), ["build-1.0.0", "build-2.0.0"]);
  const shared = workspace.comparison.entries.find((entry) => (
    entry.currentScreenIds.includes("shared-home")
  ));
  assert.deepEqual(shared.baselineScreenIds, ["shared-home"]);
});

test("fingerprints from different Explore versions are not comparable: the match needs review instead of reading as changed", async () => {
  // A map saved by Explore 0.20.x hashes the tree with the keyboard and the focus in it, unprefixed; a2 hashes leave them out.
  const baseline = makeMap("main-eeeeeee", [{ id: "home", route: "/", acc: "0".repeat(64) }]);
  const current = makeMap("pr-3-fffffff", [{ id: "home", route: "/", acc: `a2:${"1".repeat(64)}` }]);
  const { comparison, review } = generateComparison(baseline, current);
  await validateComparisonDocument(comparison, baseline, current);
  const home = review.entries.find((entry) => entry.entryId === "match-home");
  assert.equal(home.needsReview, true);
  assert.match(entryById(comparison, "match-home").rationale, /different Explore versions/);
  assert.ok(home.notes.some((note) => /compare the screenshots/.test(note)));
});