mirror of
https://github.com/expo/explore.git
synced 2026-11-15 17:32:23 -10:00
871 lines
40 KiB
JavaScript
871 lines
40 KiB
JavaScript
import assert from "node:assert/strict";
|
||
import test from "node:test";
|
||
|
||
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import { compareSurfaceEvidence } from "../../dist/review/evidence-comparator.js";
|
||
import { snapshotBackend } from "../../dist/review/evidence-assessment.js";
|
||
import { summarizeReviewerChanges } from "../../dist/review/reviewer-changes.js";
|
||
import { projectRuntimeEvidence } from "../../dist/review/runtime-evidence.js";
|
||
|
||
function element(options = {}) {
|
||
return {
|
||
elementId: options.elementId ?? "primary-cta",
|
||
...(options.identifier === undefined ? {} : { identifier: options.identifier }),
|
||
role: options.role ?? "button",
|
||
label: options.label ?? "Continue",
|
||
...(options.visible === undefined ? { visible: true } : { visible: options.visible }),
|
||
...(options.selector === undefined ? {} : { selector: options.selector }),
|
||
...(options.bounds === undefined ? {} : { bounds: options.bounds }),
|
||
occurrence: options.occurrence ?? 0,
|
||
controller: "agent-device",
|
||
actions: options.actions ?? ["press"],
|
||
risk: options.risk ?? "safe",
|
||
actionable: options.actionable ?? true,
|
||
};
|
||
}
|
||
|
||
function screen(id, title, route, observationIds = []) {
|
||
return {
|
||
id,
|
||
title,
|
||
route,
|
||
start: id === "profile",
|
||
terminal: false,
|
||
depth: id === "profile" ? 0 : 1,
|
||
firstSeenAt: "2026-09-01T01:00:00.000Z",
|
||
lastSeenAt: "2026-09-01T01:00:00.000Z",
|
||
visits: observationIds.length,
|
||
variantIds: id === "profile" ? ["ready"] : [],
|
||
observationIds,
|
||
elementIds: [],
|
||
};
|
||
}
|
||
|
||
function actionAttempt(side, sourceObservationId, sourceElement, target) {
|
||
return {
|
||
id: `${side}-attempt-${target.id}`,
|
||
sourceScreenId: "profile",
|
||
sourceObservationId,
|
||
targetScreenId: target.id,
|
||
action: {
|
||
kind: "press",
|
||
label: sourceElement.label,
|
||
elementId: sourceElement.elementId,
|
||
},
|
||
outcome: "transition",
|
||
risk: "safe",
|
||
startedAt: "2026-09-01T01:00:01.000Z",
|
||
completedAt: "2026-09-01T01:00:01.100Z",
|
||
durationMs: 100,
|
||
};
|
||
}
|
||
|
||
function mapFor(side, elements, options = {}) {
|
||
const observationId = `${side}-observation`;
|
||
const targets = options.targets ?? [];
|
||
const attempts = options.actionTarget === undefined
|
||
? []
|
||
: [actionAttempt(side, observationId, elements[0], options.actionTarget)];
|
||
const observation = {
|
||
id: observationId,
|
||
screenId: "profile",
|
||
variantId: "ready",
|
||
fingerprints: { combined: `${side}-combined`, accessibility: `${side}-a11y` },
|
||
capturedAt: "2026-09-01T01:00:00.000Z",
|
||
surfaceScope: "app",
|
||
elementIds: elements.map((candidate) => candidate.elementId),
|
||
elements,
|
||
};
|
||
return {
|
||
contract: "expo.explore.local-map.v1",
|
||
revision: 1,
|
||
run: {
|
||
id: `${side}-run`,
|
||
name: `${side} run`,
|
||
platform: "ios",
|
||
controller: "agent-device",
|
||
status: "complete",
|
||
createdAt: "2026-09-01T01:00:00.000Z",
|
||
updatedAt: "2026-09-01T01:01:00.000Z",
|
||
completedAt: "2026-09-01T01:01:00.000Z",
|
||
build: { id: `${side}-build`, source: "eas" },
|
||
environment: {
|
||
deviceModel: "iPhone 17",
|
||
osVersion: "26.5",
|
||
locale: "en-US",
|
||
orientation: "portrait",
|
||
colorScheme: "light",
|
||
},
|
||
limits: { maxScreens: 120, maxActions: 500, maxDurationMinutes: 120 },
|
||
},
|
||
artifacts: [],
|
||
screens: [
|
||
screen("profile", options.title ?? "Profile", "/profile", [observationId]),
|
||
...targets.map((target) => screen(target.id, target.title, target.route)),
|
||
],
|
||
variants: [],
|
||
observations: [observation],
|
||
elements: [],
|
||
actionAttempts: attempts,
|
||
transitions: [],
|
||
journeys: [],
|
||
frontier: [],
|
||
gaps: [],
|
||
};
|
||
}
|
||
|
||
function runtimeFor(side, map, expectedSamples = 3, capturedSamples = expectedSamples) {
|
||
const observation = map.observations[0];
|
||
const screens = new Map(map.screens.map((candidate) => [candidate.id, candidate]));
|
||
return projectRuntimeEvidence({
|
||
context: {
|
||
kind: "configured-capture-context",
|
||
side,
|
||
logicalBuild: `${side}-build`,
|
||
profile: "iphone-17-review",
|
||
surfaceId: "profile",
|
||
route: "/profile",
|
||
variantId: "ready",
|
||
observationId: observation.id,
|
||
expectedSampleCount: expectedSamples,
|
||
controllerPackageVersion: "0.20.2",
|
||
easCliVersion: "25.2.0",
|
||
},
|
||
runtime: {
|
||
kind: "eas-agent-device-raw",
|
||
observation: {
|
||
id: observation.id,
|
||
variantId: observation.variantId,
|
||
capturedAt: observation.capturedAt,
|
||
surfaceScope: observation.surfaceScope,
|
||
accessibility: {
|
||
truncated: false,
|
||
elements: observation.elements.map((candidate) => ({
|
||
id: candidate.elementId,
|
||
role: candidate.role,
|
||
label: candidate.label,
|
||
actionable: candidate.actionable,
|
||
landmark: candidate.role === "text" && candidate.label.includes("playfield"),
|
||
})),
|
||
},
|
||
},
|
||
interactions: map.actionAttempts
|
||
.filter((attempt) => attempt.sourceObservationId === observation.id)
|
||
.map((attempt) => {
|
||
const target = attempt.targetScreenId === undefined
|
||
? undefined
|
||
: screens.get(attempt.targetScreenId);
|
||
return {
|
||
id: attempt.id,
|
||
observationId: observation.id,
|
||
action: attempt.action.kind,
|
||
label: attempt.action.label,
|
||
outcome: attempt.outcome,
|
||
startedAt: attempt.startedAt,
|
||
completedAt: attempt.completedAt,
|
||
durationMs: attempt.durationMs,
|
||
...(target === undefined ? {} : {
|
||
target: { surfaceId: target.id, route: target.route },
|
||
}),
|
||
};
|
||
}),
|
||
captures: Array.from({ length: capturedSamples }, (_, index) => ({
|
||
id: `${side}-sample-${index + 1}`,
|
||
observationId: observation.id,
|
||
variantId: observation.variantId,
|
||
capturedAt: `2026-09-01T01:00:0${index + 1}.000Z`,
|
||
widthPixels: 1179,
|
||
heightPixels: 2556,
|
||
fingerprints: {
|
||
combined: `${side}-stable`,
|
||
visual: `${side}-visual`,
|
||
accessibility: `${side}-a11y`,
|
||
},
|
||
})),
|
||
environment: {
|
||
provider: "eas-simulator",
|
||
controller: "agent-device",
|
||
platform: "ios",
|
||
deviceModel: "iPhone 17",
|
||
osVersion: "26.5",
|
||
locale: "en-US",
|
||
orientation: "portrait",
|
||
colorScheme: "light",
|
||
sessionLifecycle: "stopped",
|
||
},
|
||
},
|
||
});
|
||
}
|
||
|
||
function side(sideName, elements, options = {}) {
|
||
const map = mapFor(sideName, elements, options);
|
||
return {
|
||
map,
|
||
observationId: `${sideName}-observation`,
|
||
runtimeEvidence: runtimeFor(
|
||
sideName,
|
||
map,
|
||
options.expectedSamples ?? 3,
|
||
options.capturedSamples ?? options.expectedSamples ?? 3,
|
||
),
|
||
...(options.annotations === undefined
|
||
? {}
|
||
: { reviewerAnnotations: options.annotations }),
|
||
};
|
||
}
|
||
|
||
function impact(options = {}) {
|
||
const presence = options.presence ?? "both";
|
||
const samples = options.samples ?? 3;
|
||
return {
|
||
id: "profile",
|
||
route: "/profile",
|
||
presence,
|
||
reasons: options.reasons ?? ["source:src/screens/profile.tsx"],
|
||
baseSamples: presence === "head-only" ? 0 : samples,
|
||
headSamples: presence === "base-only" ? 0 : samples,
|
||
};
|
||
}
|
||
|
||
function visual(status = "unchanged", options = {}) {
|
||
return {
|
||
status,
|
||
reason: options.reason ?? (
|
||
status === "modified" ? "repeated-cross-build-delta"
|
||
: status === "needs-review" ? "insufficient-samples"
|
||
: "matched"
|
||
),
|
||
baselineSamples: options.baselineSamples ?? 3,
|
||
currentSamples: options.currentSamples ?? 3,
|
||
requiredSamples: options.requiredSamples ?? 3,
|
||
withinSideNoisePercentage: 0,
|
||
...(status === "modified" ? { crossBuildMismatchPercentage: 8.4 } : {}),
|
||
};
|
||
}
|
||
|
||
function supportedChanges(result) {
|
||
return result.reviewerChanges
|
||
.filter((change) => change.assessment === "supported")
|
||
.map((change) => change.change);
|
||
}
|
||
|
||
test("observer payload changes do not become UI changes or invalidate runtime projections", () => {
|
||
const base = side('base', [element({ identifier: 'primary-cta', label: 'Continue' })]);
|
||
const head = side('head', [element({ identifier: 'primary-cta', label: 'Continue' })]);
|
||
// Runtime UI projections contain app elements only; raw observations also retain metadata.
|
||
for (const [name, evidence] of [['base', base], ['head', head]]) {
|
||
for (const identifier of ['expo-explore-route', 'expo-explore-observer']) {
|
||
evidence.map.observations[0].elements.push(element({ elementId: `${name}-${identifier}`, identifier,
|
||
role: 'text', actions: [], actionable: false,
|
||
label: JSON.stringify({ pathname: '/profile', parameters: { diagnostic: `${name}-session` } }) }));
|
||
}
|
||
}
|
||
const result = compareSurfaceEvidence({ impact: impact(), base, head, visualAssessment: visual('unchanged') });
|
||
assert.equal(result.status, 'unchanged');
|
||
assert.equal(result.needsReview, false);
|
||
assert.deepEqual(result.reviewReasons, []);
|
||
assert.deepEqual(supportedChanges(result).map(change => change.kind), ['no-visible-change']);
|
||
assert.doesNotMatch(JSON.stringify(result.reviewerChanges), /expo-explore|diagnostic|pathname/);
|
||
});
|
||
|
||
test("turns an annotated CTA label delta into a visible button-title reviewer fact", () => {
|
||
const baseElement = element({ identifier: "primary-cta", label: "A" });
|
||
const headElement = element({ identifier: "primary-cta", label: "B" });
|
||
const annotation = [{
|
||
identifier: "primary-cta",
|
||
role: "button",
|
||
textSource: "visible-title",
|
||
subjectLabel: "Primary CTA",
|
||
}];
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [baseElement], { annotations: annotation }),
|
||
head: side("head", [headElement], { annotations: annotation }),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
|
||
assert.equal(result.status, "modified");
|
||
assert.equal(result.needsReview, false);
|
||
assert.ok(supportedChanges(result).some((change) => (
|
||
change.kind === "visible-text-changed"
|
||
&& change.textKind === "title"
|
||
&& change.before === "A"
|
||
&& change.after === "B"
|
||
)));
|
||
assert.equal(
|
||
supportedChanges(result).some((change) => change.kind === "accessibility-label-changed"),
|
||
false,
|
||
);
|
||
const summary = summarizeReviewerChanges({ changes: result.reviewerChanges });
|
||
assert.ok(summary.facts.some((fact) => fact.bullet === "Changed button title from “A” to “B”."));
|
||
});
|
||
|
||
test("keeps an unannotated label delta in the accessibility category", () => {
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({ identifier: "save", label: "Submit form" })]),
|
||
head: side("head", [element({ identifier: "save", label: "Save profile" })]),
|
||
visualAssessment: visual(),
|
||
});
|
||
|
||
assert.ok(supportedChanges(result).some((change) => (
|
||
change.kind === "accessibility-label-changed"
|
||
&& change.before === "Submit form"
|
||
&& change.after === "Save profile"
|
||
)));
|
||
assert.equal(
|
||
supportedChanges(result).some((change) => change.kind === "visible-text-changed"),
|
||
false,
|
||
);
|
||
});
|
||
|
||
test("a label that varies within both builds, in the words that differ between them, is runtime state", () => {
|
||
const stamp = (label) => [element({ elementId: "stamp", identifier: "stamp", role: "text", label, actions: [], actionable: false })];
|
||
const sibling = (base, head) => ({ base: side("base", base), head: side("head", head) });
|
||
const input = {
|
||
impact: impact(),
|
||
base: side("base", stamp("edited 2m ago")),
|
||
head: side("head", stamp("edited 3m ago")),
|
||
visualAssessment: visual(),
|
||
};
|
||
const plain = supportedChanges(compareSurfaceEvidence(input));
|
||
assert.ok(plain.some((change) => change.kind === "accessibility-label-changed"), "without a sibling capture the label difference is a change");
|
||
|
||
const result = compareSurfaceEvidence({ ...input, siblings: [sibling(stamp("edited 1m ago"), stamp("edited 4m ago"))] });
|
||
const changes = supportedChanges(result);
|
||
assert.deepEqual(changes.map((change) => change.kind), ["unstable-text"]);
|
||
assert.deepEqual(changes[0], { kind: "unstable-text", subjectRole: "text", before: "edited 2m ago", after: "edited 3m ago", within: "edited 4m ago" });
|
||
const summary = summarizeReviewerChanges({ changes: result.reviewerChanges });
|
||
assert.deepEqual(summary.facts.map((fact) => [fact.templateId, fact.bullet]),
|
||
[["visible-text.unstable-within-build", "Text varies between captures of the same build: \u201cedited 2m ago\u201d / \u201cedited 4m ago\u201d / \u201cedited 3m ago\u201d."]]);
|
||
assert.equal(summary.needsReview, false);
|
||
|
||
// A stable label beside it is still a change.
|
||
const save = (label) => element({ identifier: "save", label });
|
||
const mixed = compareSurfaceEvidence({
|
||
...input,
|
||
base: side("base", [...stamp("edited 2m ago"), save("Submit")]),
|
||
head: side("head", [...stamp("edited 3m ago"), save("Save")]),
|
||
siblings: [sibling([...stamp("edited 1m ago"), save("Submit")], [...stamp("edited 4m ago"), save("Save")])],
|
||
});
|
||
assert.deepEqual(supportedChanges(mixed).map((change) => change.kind).sort(), ["accessibility-label-changed", "unstable-text"]);
|
||
// Pixels that only an unstable label explains still get the generic styling line.
|
||
const pixels = compareSurfaceEvidence({ ...input, visualAssessment: visual("modified"), siblings: [sibling(stamp("edited 1m ago"), stamp("edited 4m ago"))] });
|
||
assert.deepEqual(supportedChanges(pixels).map((change) => change.kind), ["unstable-text", "visual-changed"]);
|
||
});
|
||
|
||
test("a label that follows the appearance, or changed in a word that does not vary, is still a change", () => {
|
||
const label = (text) => [element({ identifier: "toggle", label: text })];
|
||
const compare = ([baseLight, baseDark], [headLight, headDark], options = {}) => supportedChanges(compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", label(baseLight), options.base),
|
||
head: side("head", label(headLight), options.head),
|
||
visualAssessment: visual(),
|
||
siblings: [{ base: side("base", label(baseDark)), head: side("head", label(headDark), options.headSibling) }],
|
||
})).map((change) => change.kind);
|
||
// The theme toggle's label follows the appearance; the head broke it in light only.
|
||
assert.deepEqual(compare(["Switch to dark mode", "Switch to light mode"], ["Switch to light mode", "Switch to light mode"]), ["accessibility-label-changed"]);
|
||
assert.deepEqual(compare(["Switch to dark mode", "Switch to light mode"], ["Enable dark mode", "Switch to light mode"]), ["accessibility-label-changed"]);
|
||
// The base varies, the head does not.
|
||
assert.deepEqual(compare(["Save", "Save draft"], ["Submit", "Submit"]), ["accessibility-label-changed"]);
|
||
// The time varies; the name beside it changed.
|
||
assert.deepEqual(compare(["Posted by Alice · 2m", "Posted by Alice · 3m"], ["Posted by Bob · 4m", "Posted by Bob · 5m"]), ["accessibility-label-changed"]);
|
||
// A sibling capture that did not resolve cleanly is no evidence.
|
||
assert.deepEqual(compare(["Saved 2m ago", "Saved 3m ago"], ["Saved 4m ago", "Saved 5m ago"], { headSibling: { capturedSamples: 1 } }), ["accessibility-label-changed"]);
|
||
assert.deepEqual(compare(["Saved 2m ago", "Saved 3m ago"], ["Saved 4m ago", "Saved 5m ago"]), ["unstable-text"]);
|
||
|
||
// Provenance on one side only stays inconclusive.
|
||
const annotation = [{ identifier: "toggle", role: "button", textSource: "visible-title" }];
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", label("Saved 2m ago"), { annotations: annotation }),
|
||
head: side("head", label("Saved 4m ago")),
|
||
visualAssessment: visual(),
|
||
siblings: [{ base: side("base", label("Saved 3m ago"), { annotations: annotation }), head: side("head", label("Saved 5m ago")) }],
|
||
});
|
||
assert.equal(result.needsReview, true);
|
||
assert.ok(result.reviewReasons.some((reason) => /provenance.*one side/i.test(reason)));
|
||
assert.equal(supportedChanges(result).some((change) => change.kind === "unstable-text"), false);
|
||
});
|
||
|
||
test("text without an identifier that varies within both builds is matched by its place, not its label", () => {
|
||
const bounds = { x: 0, y: 120, width: 390, height: 64 };
|
||
// A pattern row: a pressable with one accessibility label and no testID, exposed again by its container on iOS.
|
||
const row = (label) => [
|
||
element({ elementId: "row", role: "button", label, selector: `label=${label}`, bounds }),
|
||
element({ elementId: "row-cell", role: "other", label, selector: `label=${label}`, bounds, actions: [], actionable: false }),
|
||
];
|
||
const compare = (baseLabels, headLabels) => compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", row(baseLabels[0])),
|
||
head: side("head", row(headLabels[0])),
|
||
visualAssessment: visual(),
|
||
...(baseLabels[1] ? { siblings: [{ base: side("base", row(baseLabels[1])), head: side("head", row(headLabels[1])) }] } : {}),
|
||
});
|
||
assert.deepEqual(supportedChanges(compare(["Velvet Tresillo, 4 lanes, edited 2m ago"], ["Velvet Tresillo, 4 lanes, edited 4m ago"])).map((change) => change.kind).sort(),
|
||
["element-added", "element-removed"], "without a sibling capture the row left and arrived");
|
||
|
||
const result = compare(["Velvet Tresillo, 4 lanes, edited 2m ago", "Velvet Tresillo, 4 lanes, edited 3m ago"],
|
||
["Velvet Tresillo, 4 lanes, edited 4m ago", "Velvet Tresillo, 4 lanes, edited 5m ago"]);
|
||
assert.deepEqual(supportedChanges(result), [{ kind: "unstable-text", subjectRole: "button",
|
||
before: "Velvet Tresillo, 4 lanes, edited 2m ago", after: "Velvet Tresillo, 4 lanes, edited 4m ago", within: "Velvet Tresillo, 4 lanes, edited 5m ago" }]);
|
||
assert.equal(result.needsReview, false);
|
||
// A random name is runtime state the same way.
|
||
assert.deepEqual(supportedChanges(compare(["Velvet Tresillo", "Copper Pulse"], ["Hollow Echo", "Ghost Relay"])).map((change) => change.kind), ["unstable-text"]);
|
||
// A real change in the same row is still a change.
|
||
assert.deepEqual(supportedChanges(compare(["Velvet Tresillo, 4 lanes, edited 2m ago", "Velvet Tresillo, 4 lanes, edited 3m ago"],
|
||
["Velvet Tresillo, 5 lanes, edited 4m ago", "Velvet Tresillo, 5 lanes, edited 5m ago"])).map((change) => change.kind).sort(), ["element-added", "element-removed"]);
|
||
});
|
||
|
||
test("fails closed when visible-title provenance exists on only one side", () => {
|
||
const annotation = [{
|
||
identifier: "primary-cta",
|
||
role: "button",
|
||
textSource: "visible-title",
|
||
}];
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({ identifier: "primary-cta", label: "A" })], {
|
||
annotations: annotation,
|
||
}),
|
||
head: side("head", [element({ identifier: "primary-cta", label: "B" })]),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
|
||
assert.equal(result.needsReview, true);
|
||
assert.ok(result.reviewReasons.some((reason) => /provenance.*one side/i.test(reason)));
|
||
assert.equal(supportedChanges(result).some((change) => (
|
||
change.kind === "visible-text-changed" || change.kind === "accessibility-label-changed"
|
||
)), false);
|
||
});
|
||
|
||
test("matches by selector and occurrence after identifier matching and detects role changes", () => {
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({
|
||
elementId: "base-continue",
|
||
identifier: undefined,
|
||
selector: "app/profile/continue",
|
||
occurrence: 0,
|
||
role: "link",
|
||
label: "Continue",
|
||
})]),
|
||
head: side("head", [element({
|
||
elementId: "head-continue",
|
||
identifier: undefined,
|
||
selector: "app/profile/continue",
|
||
occurrence: 0,
|
||
role: "button",
|
||
label: "Continue",
|
||
})]),
|
||
visualAssessment: visual(),
|
||
});
|
||
|
||
assert.deepEqual(supportedChanges(result), [{
|
||
kind: "accessibility-role-changed",
|
||
subjectLabel: "Continue",
|
||
beforeRole: "link",
|
||
afterRole: "button",
|
||
}]);
|
||
});
|
||
|
||
test("reports added and removed elements after deterministic semantic matching", () => {
|
||
const commonBase = element({ elementId: "common-base", identifier: "common", label: "Common" });
|
||
const commonHead = element({ elementId: "common-head", identifier: "common", label: "Common" });
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [commonBase, element({
|
||
elementId: "old",
|
||
identifier: "old",
|
||
role: "image",
|
||
label: "Old avatar",
|
||
})]),
|
||
head: side("head", [commonHead, element({
|
||
elementId: "new",
|
||
identifier: "new",
|
||
role: "button",
|
||
label: "Upload avatar",
|
||
})]),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
|
||
assert.ok(supportedChanges(result).some((change) => (
|
||
change.kind === "element-removed" && change.elementLabel === "Old avatar"
|
||
)));
|
||
assert.ok(supportedChanges(result).some((change) => (
|
||
change.kind === "element-added" && change.elementLabel === "Upload avatar"
|
||
)));
|
||
});
|
||
|
||
test("compares only safe action transitions with canonical target routes", () => {
|
||
const baseTarget = { id: "old-target", title: "Cat details", route: "/cats/details" };
|
||
const headTarget = { id: "new-target", title: "Cat Care Guide", route: "/cats/care-guide" };
|
||
const baseElement = element({ identifier: "care-guide", label: "Care guide" });
|
||
const headElement = element({ identifier: "care-guide", label: "Care guide" });
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [baseElement], { targets: [baseTarget], actionTarget: baseTarget }),
|
||
head: side("head", [headElement], { targets: [headTarget], actionTarget: headTarget }),
|
||
visualAssessment: visual(),
|
||
});
|
||
|
||
assert.ok(supportedChanges(result).some((change) => (
|
||
change.kind === "action-target-changed"
|
||
&& change.beforeTarget === "Cat details"
|
||
&& change.afterTarget === "Cat Care Guide"
|
||
)));
|
||
});
|
||
|
||
test("equal-sized duplicate groups pair in capture order instead of failing closed", () => {
|
||
const duplicated = (prefix, count) => Array.from({ length: count }, (_, index) => (
|
||
element({ elementId: `${prefix}-${index}`, identifier: undefined, label: "Done", selector: undefined, occurrence: index })
|
||
));
|
||
// Repeated label-less controls are common (image grids, form fields); the trees differ elsewhere,
|
||
// so this exercises the pairing rather than the identical-tree shortcut.
|
||
const extra = element({ elementId: "extra", identifier: "extra", label: "Extra" });
|
||
const paired = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", duplicated("base", 2)),
|
||
head: side("head", [...duplicated("head", 2).reverse(), extra]),
|
||
visualAssessment: visual(),
|
||
});
|
||
assert.equal(paired.needsReview, false);
|
||
assert.ok(!paired.reviewReasons.some((reason) => /ambiguous/i.test(reason)));
|
||
assert.ok(supportedChanges(paired).some((change) => change.kind === "element-added"));
|
||
assert.ok(!supportedChanges(paired).some((change) => change.kind === "element-removed"));
|
||
|
||
// Different counts still fail closed: no guess about which duplicate appeared or vanished.
|
||
const unequal = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", duplicated("base", 2)),
|
||
head: side("head", duplicated("head", 3)),
|
||
visualAssessment: visual(),
|
||
});
|
||
assert.equal(unequal.needsReview, true);
|
||
assert.ok(unequal.reviewReasons.some((reason) => /role-label.*ambiguous/i.test(reason)));
|
||
assert.equal(supportedChanges(unequal).length, 0);
|
||
});
|
||
|
||
test("one label appearing or leaving is one fact, reported through its most specific role", () => {
|
||
// iOS exposes a control, its container and its text under the same label.
|
||
const layers = prefix => [
|
||
element({ elementId: `${prefix}-wrapper`, identifier: undefined, role: "other", label: "Care guide", selector: undefined, actionable: false, actions: [] }),
|
||
element({ elementId: `${prefix}-text`, identifier: undefined, role: "text", label: "Care guide", selector: undefined, actionable: false, actions: [] }),
|
||
element({ elementId: `${prefix}-button`, identifier: undefined, role: "button", label: "Care guide", selector: undefined }),
|
||
];
|
||
const added = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({ elementId: "keep", identifier: "keep", label: "Keep" })]),
|
||
head: side("head", [element({ elementId: "keep", identifier: "keep", label: "Keep" }), ...layers("head")]),
|
||
visualAssessment: visual(),
|
||
});
|
||
assert.deepEqual(supportedChanges(added).map(change => [change.kind, change.elementRole, change.elementLabel]),
|
||
[["element-added", "button", "Care guide"]]);
|
||
|
||
const removed = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({ elementId: "keep", identifier: "keep", label: "Keep" }), ...layers("base")]),
|
||
head: side("head", [element({ elementId: "keep", identifier: "keep", label: "Keep" })]),
|
||
visualAssessment: visual(),
|
||
});
|
||
assert.deepEqual(supportedChanges(removed).map(change => [change.kind, change.elementRole, change.elementLabel]),
|
||
[["element-removed", "button", "Care guide"]]);
|
||
});
|
||
|
||
test("a label that survives on the other revision is not reported as coming or going", () => {
|
||
// A list that shifts up re-exposes its rows: the same label appears under a different role.
|
||
const base = side("base", [
|
||
element({ elementId: "sweet", identifier: undefined, label: "SWEET BEE", selector: undefined }),
|
||
element({ elementId: "butterfly-text", identifier: undefined, role: "text", label: "BUTTERFLY", selector: undefined, actionable: false, actions: [] }),
|
||
element({ elementId: "butterfly-other", identifier: undefined, role: "other", label: "BUTTERFLY", selector: undefined, actionable: false, actions: [] }),
|
||
]);
|
||
const head = side("head", [
|
||
element({ elementId: "butterfly-other", identifier: undefined, role: "other", label: "BUTTERFLY", selector: undefined, actionable: false, actions: [] }),
|
||
]);
|
||
const result = compareSurfaceEvidence({ impact: impact(), base, head, visualAssessment: visual("modified") });
|
||
assert.deepEqual(supportedChanges(result).map(change => [change.kind, change.elementLabel]),
|
||
[["element-removed", "SWEET BEE"]]);
|
||
assert.equal(result.needsReview, false);
|
||
});
|
||
|
||
test("the generic styling fact only speaks when no element change explains the pixels", () => {
|
||
const withElement = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({ elementId: "keep", identifier: "keep", label: "Keep" })]),
|
||
head: side("head", [element({ elementId: "keep", identifier: "keep", label: "Keep" }),
|
||
element({ elementId: "added", identifier: "added", label: "Care guide" })]),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
assert.deepEqual(supportedChanges(withElement).map(change => change.kind), ["element-added"]);
|
||
|
||
// Restyling alone changes no element, so the visual fact is the only account of the difference.
|
||
const stylingOnly = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [element({ elementId: "keep", identifier: "keep", label: "Keep" })]),
|
||
head: side("head", [element({ elementId: "keep", identifier: "keep", label: "Keep" })]),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
assert.deepEqual(supportedChanges(stylingOnly).map(change => change.kind), ["visual-changed"]);
|
||
});
|
||
|
||
test("a duplicate stable key present on one side only is reported as a removal or addition", () => {
|
||
// iOS exposes a control and its wrapper under one label; renaming the control moves both to a new key.
|
||
const labelled = (prefix, label) => [
|
||
element({ elementId: `${prefix}-wrapper`, identifier: undefined, role: "other", label, selector: `label=${label}`, occurrence: 0, actionable: false, actions: [] }),
|
||
element({ elementId: `${prefix}-button`, identifier: undefined, role: "button", label, selector: `label=${label}`, occurrence: 0 }),
|
||
];
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", labelled("base", "Share App")),
|
||
head: side("head", labelled("head", "Share Example with friends")),
|
||
visualAssessment: visual(),
|
||
});
|
||
assert.equal(result.needsReview, false);
|
||
assert.ok(!result.reviewReasons.some((reason) => /ambiguous/i.test(reason)));
|
||
const kinds = supportedChanges(result).map((change) => change.kind);
|
||
assert.ok(kinds.includes("element-removed"));
|
||
assert.ok(kinds.includes("element-added"));
|
||
});
|
||
|
||
test("is deterministic when map element and annotation order changes", () => {
|
||
const baseElements = [
|
||
element({ elementId: "one", identifier: "one", label: "One" }),
|
||
element({ elementId: "two", identifier: "two", label: "Two" }),
|
||
];
|
||
const headElements = [
|
||
element({ elementId: "head-one", identifier: "one", label: "First" }),
|
||
element({ elementId: "head-two", identifier: "two", label: "Second" }),
|
||
];
|
||
const annotations = [
|
||
{ identifier: "one", role: "button", textSource: "visible-title" },
|
||
{ identifier: "two", role: "button", textSource: "visible-title" },
|
||
];
|
||
const forward = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", baseElements, { annotations }),
|
||
head: side("head", headElements, { annotations }),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
const reversed = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [...baseElements].reverse(), { annotations: [...annotations].reverse() }),
|
||
head: side("head", [...headElements].reverse(), { annotations: [...annotations].reverse() }),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
|
||
assert.deepEqual(reversed, forward);
|
||
});
|
||
|
||
test("emits a supported no-visible-change assessment for source-only changes", () => {
|
||
const unchanged = element({ identifier: "settings", label: "Settings" });
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [unchanged]),
|
||
head: side("head", [structuredClone(unchanged)]),
|
||
visualAssessment: visual(),
|
||
});
|
||
|
||
assert.equal(result.status, "unchanged");
|
||
assert.equal(result.needsReview, false);
|
||
assert.deepEqual(supportedChanges(result), [{ kind: "no-visible-change" }]);
|
||
const summary = summarizeReviewerChanges({ changes: result.reviewerChanges });
|
||
assert.equal(summary.status, "ready");
|
||
assert.equal(summary.noVisibleChange, true);
|
||
});
|
||
|
||
test("reports a semantic runtime delta as a fact and leaves source provenance to the verdict", () => {
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact({ samples: 1, reasons: [] }),
|
||
base: side("base", [element({ identifier: "save", label: "Save" })], { expectedSamples: 1 }),
|
||
head: side("head", [element({ identifier: "save", label: "Submit" })], { expectedSamples: 1 }),
|
||
visualAssessment: visual("unchanged", {
|
||
baselineSamples: 1,
|
||
currentSamples: 1,
|
||
requiredSamples: 1,
|
||
}),
|
||
});
|
||
|
||
assert.equal(result.status, "modified");
|
||
assert.equal(result.needsReview, false);
|
||
assert.equal(result.reviewReasons.some((reason) => /provenance/i.test(reason)), false);
|
||
});
|
||
|
||
test("never derives landmark semantics from pixels and validates explicit landmark scopes", () => {
|
||
const unchanged = element({ identifier: "game", label: "Game" });
|
||
const generic = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [unchanged]),
|
||
head: side("head", [structuredClone(unchanged)]),
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
assert.ok(supportedChanges(generic).some((change) => (
|
||
change.kind === "visual-changed" && change.scope.kind === "main-content"
|
||
)));
|
||
|
||
const unsupportedLandmark = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [unchanged]),
|
||
head: side("head", [structuredClone(unchanged)]),
|
||
visualAssessment: visual("modified"),
|
||
visualScope: { kind: "landmark", label: "Goldfish playfield" },
|
||
});
|
||
assert.equal(unsupportedLandmark.needsReview, true);
|
||
assert.equal(
|
||
supportedChanges(unsupportedLandmark).some((change) => change.kind === "visual-changed"),
|
||
false,
|
||
);
|
||
});
|
||
|
||
test("produces screen facts and one-sided RuntimeEvidenceComparison for added and removed surfaces", () => {
|
||
const head = side("head", [element()], { title: "Cat Care Guide" });
|
||
const added = compareSurfaceEvidence({
|
||
impact: impact({ presence: "head-only" }),
|
||
head,
|
||
});
|
||
assert.equal(added.status, "added");
|
||
assert.equal(added.runtimeEvidence.status, "added");
|
||
assert.equal("base" in added.runtimeEvidence, false);
|
||
assert.deepEqual(supportedChanges(added), [{
|
||
kind: "screen-added",
|
||
screenTitle: "Cat Care Guide",
|
||
}]);
|
||
|
||
const base = side("base", [element()], { title: "Sweet Bee" });
|
||
const removed = compareSurfaceEvidence({
|
||
impact: impact({ presence: "base-only" }),
|
||
base,
|
||
});
|
||
assert.equal(removed.status, "removed");
|
||
assert.equal(removed.runtimeEvidence.status, "removed");
|
||
assert.equal("head" in removed.runtimeEvidence, false);
|
||
assert.deepEqual(supportedChanges(removed), [{
|
||
kind: "screen-removed",
|
||
screenTitle: "Sweet Bee",
|
||
}]);
|
||
});
|
||
|
||
test("blocks publication for incomplete samples or inconclusive visual evidence", () => {
|
||
const unchanged = element({ identifier: "settings", label: "Settings" });
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [unchanged], { expectedSamples: 3, capturedSamples: 1 }),
|
||
head: side("head", [structuredClone(unchanged)], { expectedSamples: 3, capturedSamples: 1 }),
|
||
visualAssessment: visual("needs-review", {
|
||
baselineSamples: 1,
|
||
currentSamples: 1,
|
||
requiredSamples: 3,
|
||
}),
|
||
});
|
||
|
||
assert.equal(result.needsReview, true);
|
||
assert.ok(result.reviewReasons.some((reason) => /coverage is incomplete/i.test(reason)));
|
||
assert.ok(result.reviewReasons.some((reason) => /visual evidence needs review/i.test(reason)));
|
||
assert.equal(supportedChanges(result).some((change) => change.kind === "no-visible-change"), false);
|
||
});
|
||
|
||
test("the route observer's unidentified wrapper is capture metadata, not an added element", () => {
|
||
const destination = JSON.stringify({ provider: "expo-router", pathname: "/profile", segments: ["profile"], parameters: {} });
|
||
const content = (id) => element({ elementId: `${id}-continue`, identifier: "primary-cta", label: "Continue" });
|
||
// XCTest exposes an Other wrapper that carries the observer's label without its identifier.
|
||
const wrapper = element({ elementId: "head-wrapper", role: "other", label: destination, actions: [], actionable: false });
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: side("base", [content("base")]),
|
||
head: side("head", [content("head"), wrapper]),
|
||
visualAssessment: visual("unchanged"),
|
||
});
|
||
assert.equal(result.status, "unchanged");
|
||
assert.deepEqual(supportedChanges(result).map((change) => change.kind), ["no-visible-change"]);
|
||
});
|
||
|
||
test("snapshots from different controller backends compare content, not containers or roles", () => {
|
||
const text = (id, label, role) => element({ elementId: id, label, role, selector: `app/games/${label}`, actions: [], actionable: false });
|
||
// agent-device 0.21.x: the host AX bridge (base) and XCTest (head) describe the same text with
|
||
// different roles and containers, and head exposes merged labels the bridge splits.
|
||
const base = side("base", [text("b-score", "0", "text"), text("b-level", "Level 1", "text"), text("b-title", "Games", "text"),
|
||
text("b-mood", "Check Mood", "text"), text("b-coins", "3", "text")]);
|
||
const head = side("head", [text("h-score", "0", "other"), text("h-level", "Level 1", "other"), text("h-title", "Games", "other"),
|
||
text("h-container", "Games, Level 1", "other"), element({ elementId: "h-mood", role: "button", label: "Check Mood, 3" }),
|
||
element({ elementId: "h-account", identifier: "account-row", role: "button", label: "Account" })]);
|
||
const result = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: { ...base, snapshotBackend: "ax-bridge" },
|
||
head: { ...head, snapshotBackend: "tree" },
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
assert.deepEqual(supportedChanges(result).filter((change) => change.kind !== "visual-styling-changed"), [{
|
||
kind: "element-added", elementRole: "button", elementLabel: "Account",
|
||
}]);
|
||
// The same trees from one backend still report the role changes.
|
||
const same = compareSurfaceEvidence({
|
||
impact: impact(),
|
||
base: { ...base, snapshotBackend: "tree" },
|
||
head: { ...head, snapshotBackend: "tree" },
|
||
visualAssessment: visual("modified"),
|
||
});
|
||
assert.ok(supportedChanges(same).some((change) => change.kind === "accessibility-role-changed"));
|
||
});
|
||
|
||
test("a capture's snapshot backend is read from its saved controller response", async (t) => {
|
||
const directory = await mkdtemp(join(tmpdir(), "explore-snapshot-backend-"));
|
||
t.after(() => rm(directory, { recursive: true, force: true }));
|
||
await mkdir(join(directory, "artifacts/accessibility"), { recursive: true });
|
||
const map = (path) => ({ observations: [{ id: "o", accessibilityArtifactId: "a" }], artifacts: [{ id: "a", kind: "accessibility", path }] });
|
||
const write = (name, data) => writeFile(join(directory, "artifacts/accessibility", name), JSON.stringify({ success: true, data }));
|
||
await write("xctest.json", { snapshotQuality: { state: "healthy", backend: "tree" }, nodes: [{ type: "Other", label: "Games" }] });
|
||
await write("bridge.json", { nodes: [{ type: "Application", pid: 42, role: "UIApplication", subrole: "" }] });
|
||
await write("plain.json", { nodes: [{ type: "Other" }] });
|
||
assert.equal(snapshotBackend(directory, map("artifacts/accessibility/xctest.json"), "o"), "tree");
|
||
assert.equal(snapshotBackend(directory, map("artifacts/accessibility/bridge.json"), "o"), "ax-bridge");
|
||
assert.equal(snapshotBackend(directory, map("artifacts/accessibility/plain.json"), "o"), undefined);
|
||
assert.equal(snapshotBackend(directory, map("artifacts/accessibility/missing.json"), "o"), undefined);
|
||
assert.equal(snapshotBackend(directory, map("../outside.json"), "o"), undefined);
|
||
});
|
||
|
||
/**
|
||
* Tempo on the closed iPhone Duo in brentvatne/euxy#172 (review run 39362642343): a 1398 × 2034 screenshot of a
|
||
* 466 × 678 pt screen, and a footnote whose text changed below the fold of its scrolling sheet, so the screenshots match.
|
||
* The published review carries no accessibility tree, so the footnote's real bounds are unknown: these tests pin the
|
||
* rule (more than half of the changed element past the screen's edge), not where euxy's footnote was.
|
||
*/
|
||
function tempoSide(sideName, footnote, footnoteBounds, image = { width: 1398, height: 2034 }) {
|
||
const evidence = side(sideName, [
|
||
element({ elementId: "app", role: "other", label: "euxy", actions: [], actionable: false, bounds: { x: 0, y: 0, width: 466, height: 678, unit: "points" } }),
|
||
element({ elementId: "title", role: "text", label: "Base Resolution", actions: [], actionable: false, bounds: { x: 26, y: 560, width: 300, height: 24, unit: "points" } }),
|
||
element({ elementId: `${sideName}-footnote`, role: "text", label: footnote, actions: [], actionable: false, bounds: footnoteBounds }),
|
||
]);
|
||
if (image) {
|
||
evidence.map.artifacts.push({ id: `${sideName}-shot`, kind: "screenshot", path: `${sideName}.png`, sha256: "0".repeat(64), mimeType: "image/png", ...image, createdAt: "2026-09-01T01:00:00.000Z" });
|
||
evidence.map.observations[0].screenshotArtifactId = `${sideName}-shot`;
|
||
}
|
||
return evidence;
|
||
}
|
||
const NEW_LANES = "Changes apply live while playing. Base resolution sets the step grid for new lanes.";
|
||
const ADD_NEXT = "Changes apply live while playing. Base resolution sets the step grid for lanes you add next.";
|
||
// Its top 12 pt on the screen, 28 pt below the bottom edge: 70% past it.
|
||
const mostlyBelow = { x: 26, y: 666, width: 414, height: 40, unit: "points" };
|
||
const offScreenOf = (bounds, assessment = visual("unchanged"), image) => compareSurfaceEvidence({ impact: impact(),
|
||
base: tempoSide("base", NEW_LANES, bounds, image), head: tempoSide("head", ADD_NEXT, bounds, image), visualAssessment: assessment }).offScreen;
|
||
|
||
test("matching screenshots of a change whose elements lie mostly past the screen's edge are off-screen", () => {
|
||
assert.equal(offScreenOf(mostlyBelow), true);
|
||
// A turned frame (an opened iPhone Duo shown upright) keeps the tree on the portrait axes: the screen is still found.
|
||
assert.equal(offScreenOf(mostlyBelow, visual("unchanged"), { width: 2034, height: 1398 }), true);
|
||
});
|
||
|
||
test("a change on the screen, partly on it, with differing screenshots, or with no screen to measure against is never off-screen", () => {
|
||
const cases = {
|
||
"an on-screen text change": [{ x: 26, y: 600, width: 414, height: 40, unit: "points" }],
|
||
// Its top 28 pt on the screen, 12 pt below: mostly visible, so matching pixels prove nothing about it.
|
||
"a footnote mostly on the screen": [{ x: 26, y: 650, width: 414, height: 40, unit: "points" }],
|
||
// A control nudged 2 pt past the left edge, otherwise wholly visible.
|
||
"a control just crossing an edge": [{ x: -2, y: 600, width: 300, height: 40, unit: "points" }],
|
||
"screenshots that differ": [mostlyBelow, visual("modified")],
|
||
"no screenshot size": [mostlyBelow, visual("unchanged"), null],
|
||
"a changed element without bounds": [undefined],
|
||
};
|
||
for (const [name, args] of Object.entries(cases)) assert.equal(offScreenOf(...args), undefined, name);
|
||
});
|