mirror of
https://github.com/expo/explore.git
synced 2026-11-15 17:32:23 -10:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ef076fabf | ||
|
|
8b4ceb8602 | ||
|
|
0bbf44a0d0 |
No files matched your search
@@ -221,7 +221,7 @@ Capture handles iOS's confirmation to open a link in the expected app. After a f
|
||||
|
||||
### Stable captures (optional)
|
||||
|
||||
Screenshots are compared pixel by pixel, so anything that changes between runs shows up as a Screen variant or noise: the current time and relative dates, animations and loading indicators that can be caught mid-frame, screens that move on by themselves, and permission prompts that depend on earlier runs. If your app has these, consider making the capture profile's build fix them, for example with an environment variable in that EAS profile that the app reads. What to fix depends on your app; your coding agent can find candidates in the source, and each change is an app change for you to approve. The [example app's `explore` profile](../test/ci/apps/wallet/README.md#explore-build-profile) is one example.
|
||||
Screenshots are compared pixel by pixel, so anything that changes between runs shows up as a Screen variant or noise: the current time and relative dates, animations and loading indicators that can be caught mid-frame, screens that move on by themselves, and permission prompts that depend on earlier runs. When both builds capture light and dark, text that differs between each build's own two captures (a clock, a relative date, a random name) is reported as runtime state, never as the change that makes a screen Modified; text that only follows the appearance, or a word beside it that changed, is still a change. If your app has these, consider making the capture profile's build fix them, for example with an environment variable in that EAS profile that the app reads. What to fix depends on your app; your coding agent can find candidates in the source, and each change is an app change for you to approve. The [example app's `explore` profile](../test/ci/apps/wallet/README.md#explore-build-profile) is one example.
|
||||
|
||||
### Agent usage and reuse
|
||||
|
||||
|
||||
@@ -381,19 +381,21 @@ function localEvidencePath(side: PublicEvidenceSide, relativePath: string): stri
|
||||
*/
|
||||
export function decideReview(scope: SurfaceSourceScope, visual: PublicVisualVerdict, summary?: WhatChangedSummary): PublicSurfaceReview {
|
||||
const facts = summary?.facts.map((fact) => ({ category: fact.category, templateId: fact.templateId, bullet: truncate(fact.bullet) })) ?? [];
|
||||
const differs = visual.verdict === "modified" || facts.length > 0;
|
||||
// Text that also differs between two captures of the same build is runtime state: listed, never what makes a change.
|
||||
const changes = facts.filter((fact) => fact.templateId !== "visible-text.unstable-within-build");
|
||||
const differs = visual.verdict === "modified" || changes.length > 0;
|
||||
const inconclusive = !differs && (visual.verdict === "needs-review" || Boolean(summary?.needsReview));
|
||||
const issue = summary?.reviewIssues[0]?.reason;
|
||||
const evidenceText = facts[0]?.bullet ?? visualDescription(visual);
|
||||
const evidenceText = changes[0]?.bullet ?? visualDescription(visual);
|
||||
if (scope === "none") {
|
||||
if (!differs) return { status: "unchanged", description: "No visible change" };
|
||||
// The comparator's generic visual fact reads as a styling change; a variant names what differs.
|
||||
const state = facts.find((fact) => fact.category !== "visual")?.bullet
|
||||
const state = [...changes, ...facts].find((fact) => fact.category !== "visual")?.bullet
|
||||
?? `${percentage(visual.mismatchPercentage)}${visual.regions?.[0]?.location ? ` · ${visual.regions[0].location}` : ""}`;
|
||||
return { status: "variant", description: truncate(`Runtime state differs: ${state}`), ...(facts.length ? { facts } : {}) };
|
||||
}
|
||||
if (differs) {
|
||||
const description = !facts.length && issue ? `${visualDescription(visual)} · ${issue}` : evidenceText;
|
||||
const description = !changes.length && issue ? `${visualDescription(visual)} · ${issue}` : evidenceText;
|
||||
return { status: "modified", description: truncate(description), ...(facts.length ? { facts } : {}) };
|
||||
}
|
||||
if (inconclusive) {
|
||||
@@ -514,7 +516,19 @@ export async function assessPublicEvidence(options: {
|
||||
if (surface.baseSurface?.kind !== "screen" || surface.headSurface?.kind !== "screen"
|
||||
|| (!surface.baseCapture && !surface.headCapture)) continue;
|
||||
// Every appearance the capture rendered is compared like-for-like: `<id>@<scheme>` frames against each other.
|
||||
for (const { id, scheme, baseId } of appearancePairs(surface.id, colorSchemes, baseColorSchemes)) {
|
||||
const pairs = appearancePairs(surface.id, colorSchemes, baseColorSchemes);
|
||||
// The screen's other appearances, rendered by both builds and verified on both: a label that varies between a
|
||||
// build's own appearances is runtime state. An unverified capture may show another screen, so it is no evidence.
|
||||
const baseRoute = surface.baseSurface, headRoute = surface.headSurface;
|
||||
const siblings = (current: string) => pairs.flatMap((pair) => {
|
||||
if (pair.id === current || pair.baseId === null || captureDiagnostic(surface, "base", options.base, pair.baseId)
|
||||
|| captureDiagnostic(surface, "head", options.head, pair.id)) return [];
|
||||
try {
|
||||
return [{ base: matchedSide("base", options.base, { ...baseRoute, id: pair.baseId }, options.profile),
|
||||
head: matchedSide("head", options.head, { ...headRoute, id: pair.id }, options.profile) }];
|
||||
} catch { return []; }
|
||||
});
|
||||
for (const { id, scheme, baseId } of pairs) {
|
||||
// An appearance the base never rendered has nothing to compare against: the head frame is new evidence.
|
||||
if (baseId === null) {
|
||||
if (localImagePath(options.head, id)) {
|
||||
@@ -591,6 +605,7 @@ export async function assessPublicEvidence(options: {
|
||||
base: matchedSide("base", options.base, { ...surface.baseSurface, id: baseId }, options.profile),
|
||||
head: matchedSide("head", options.head, { ...surface.headSurface, id }, options.profile),
|
||||
visualAssessment: assessment,
|
||||
siblings: siblings(id),
|
||||
});
|
||||
const summary = summarizeReviewerChanges({ changes: result.reviewerChanges });
|
||||
accessibilityCounts.compared += 1;
|
||||
|
||||
@@ -70,6 +70,11 @@ export type CommonSurfaceComparatorInput = {
|
||||
visualAssessment: VisualEvidenceAssessment;
|
||||
/** Explicit stable runtime scope; semantic meaning is never inferred from diff pixels. */
|
||||
visualScope?: VisualReviewerChange["scope"];
|
||||
/**
|
||||
* The screen's other appearances, captured and verified on both builds. A label that varies between each build's own
|
||||
* captures, in the words that differ between the builds, is runtime state, and its difference is reported as such.
|
||||
*/
|
||||
siblings?: readonly { base: MatchedSurfaceSide; head: MatchedSurfaceSide }[];
|
||||
};
|
||||
|
||||
export type EvidenceComparatorInput =
|
||||
@@ -139,6 +144,7 @@ const CHANGE_PRIORITY: Record<SupportedReviewerChange["kind"], number> = {
|
||||
"element-added": 20,
|
||||
"element-removed": 20,
|
||||
"visible-text-changed": 30,
|
||||
"unstable-text": 35,
|
||||
"action-target-changed": 40,
|
||||
"accessibility-label-changed": 50,
|
||||
"accessibility-role-changed": 50,
|
||||
@@ -772,10 +778,76 @@ function actionTarget(side: ResolvedSide, element: ObservedElement): ActionTarge
|
||||
return target ? { kind: "target", target } : { kind: "none", evidenceRefs: [] };
|
||||
}
|
||||
|
||||
type ResolvedSibling = { base: ResolvedSide; head: ResolvedSide };
|
||||
|
||||
function resolveSiblings(input: CommonSurfaceComparatorInput, requiredSamples: 1 | 3): ResolvedSibling[] {
|
||||
const resolve = (side: "base" | "head", sibling: MatchedSurfaceSide) => {
|
||||
const screenId = sibling.map.observations.find((candidate) => candidate.id === sibling.observationId)?.screenId;
|
||||
const resolution = screenId === undefined ? undefined : resolveSide(side, sibling, { ...input.impact, id: screenId }, requiredSamples);
|
||||
// A sibling is evidence only when it resolves as cleanly as a comparison would.
|
||||
return resolution?.kind === "resolved" && resolution.side.issues.length === 0 ? resolution.side : undefined;
|
||||
};
|
||||
return (input.siblings ?? []).flatMap((sibling) => {
|
||||
const base = resolve("base", sibling.base);
|
||||
const head = resolve("head", sibling.head);
|
||||
return base && head ? [{ base, head }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
function sameBounds(left: ElementBounds | undefined, right: ElementBounds | undefined): boolean {
|
||||
return !!left && !!right && left.x === right.x && left.y === right.y && left.width === right.width && left.height === right.height;
|
||||
}
|
||||
|
||||
/** The element at the same place in another capture: the one of its role with its identifier or, without one, its bounds. */
|
||||
function counterpart(element: ObservedElement, other: ResolvedSide): ObservedElement | undefined {
|
||||
const identifier = compact(element.identifier ?? "");
|
||||
const bounds = boundsOf(element);
|
||||
const candidates = other.elements.filter((candidate) => candidate.role === element.role && (identifier
|
||||
? compact(candidate.identifier ?? "") === identifier
|
||||
: !compact(candidate.identifier ?? "") && sameBounds(bounds, boundsOf(candidate))));
|
||||
return candidates.length === 1 ? candidates[0] : undefined;
|
||||
}
|
||||
|
||||
/** The word positions at which two labels differ, or undefined when they have different numbers of words. */
|
||||
function changedWords(left: string, right: string): number[] | undefined {
|
||||
const leftWords = left.split(" ");
|
||||
const rightWords = right.split(" ");
|
||||
return leftWords.length === rightWords.length
|
||||
? leftWords.flatMap((word, index) => word === rightWords[index] ? [] : [index])
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the head showed in its other appearance, when a label's difference between the builds is runtime state: each
|
||||
* build shows the label differently in its own two appearances, the other appearance differs between the builds too,
|
||||
* and every word that differs between the builds is one that varies within a build. A label that follows the
|
||||
* appearance ("Switch to dark mode") does not vary within a build that broke it, or keeps a word that changed, so
|
||||
* its change is still a change.
|
||||
*/
|
||||
function unstableWithin(base: ObservedElement, head: ObservedElement, siblings: readonly ResolvedSibling[]): string | undefined {
|
||||
const baseLabel = compact(base.label);
|
||||
const headLabel = compact(head.label);
|
||||
for (const sibling of siblings) {
|
||||
const baseOther = compact(counterpart(base, sibling.base)?.label ?? "");
|
||||
const headOther = compact(counterpart(head, sibling.head)?.label ?? "");
|
||||
if (!baseLabel || !headLabel || !baseOther || !headOther) continue;
|
||||
const withinBase = changedWords(baseLabel, baseOther);
|
||||
const withinHead = changedWords(headLabel, headOther);
|
||||
const across = changedWords(baseLabel, headLabel);
|
||||
const acrossOther = changedWords(baseOther, headOther);
|
||||
if (!withinBase?.length || !withinHead?.length || !across?.length || !acrossOther?.length) continue;
|
||||
const varying = new Set([...withinBase, ...withinHead]);
|
||||
if ([...across, ...acrossOther].every((index) => varying.has(index))) return headOther;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function comparePair(
|
||||
pair: ElementPair,
|
||||
base: ResolvedSide,
|
||||
head: ResolvedSide,
|
||||
/** What the head showed in its other appearance when this element's label is runtime state. */
|
||||
within?: string,
|
||||
): { facts: SupportedInput[]; issues: IssueInput[] } {
|
||||
const facts: SupportedInput[] = [];
|
||||
const issues: IssueInput[] = [];
|
||||
@@ -784,6 +856,10 @@ function comparePair(
|
||||
const headLabel = compact(pair.head.label);
|
||||
const baseAnnotation = base.annotationsByElementId.get(pair.base.elementId);
|
||||
const headAnnotation = head.annotationsByElementId.get(pair.head.elementId);
|
||||
// Runtime state replaces a label change only where the label comparison itself holds: provenance still decides first.
|
||||
const labelChange = (change: SupportedReviewerChange): SupportedReviewerChange => within
|
||||
? { kind: "unstable-text", subjectRole: reviewerRole(pair.head.role), before: baseLabel, after: headLabel, within }
|
||||
: change;
|
||||
|
||||
if (baseLabel !== headLabel) {
|
||||
if (baseAnnotation && headAnnotation) {
|
||||
@@ -793,14 +869,14 @@ function comparePair(
|
||||
} else if (!baseLabel || !headLabel) {
|
||||
issues.push(inconclusive("Visible-title comparison is missing before or after text.", refs));
|
||||
} else {
|
||||
facts.push(supported({
|
||||
facts.push(supported(labelChange({
|
||||
kind: "visible-text-changed",
|
||||
subjectRole: subject.subjectRole,
|
||||
subjectLabel: subject.subjectLabel,
|
||||
textKind: pair.head.role === "button" ? "title" : "label",
|
||||
before: baseLabel,
|
||||
after: headLabel,
|
||||
}, refs));
|
||||
}), refs));
|
||||
}
|
||||
} else if (baseAnnotation || headAnnotation) {
|
||||
issues.push(inconclusive(
|
||||
@@ -810,12 +886,12 @@ function comparePair(
|
||||
} else if (!baseLabel || !headLabel) {
|
||||
issues.push(inconclusive("Accessibility label comparison is missing before or after text.", refs));
|
||||
} else {
|
||||
facts.push(supported({
|
||||
facts.push(supported(labelChange({
|
||||
kind: "accessibility-label-changed",
|
||||
subjectRole: reviewerRole(pair.head.role),
|
||||
before: baseLabel,
|
||||
after: headLabel,
|
||||
}, refs));
|
||||
}), refs));
|
||||
}
|
||||
}
|
||||
if (pair.base.role !== pair.head.role) {
|
||||
@@ -1110,6 +1186,27 @@ function commonResult(input: CommonSurfaceComparatorInput): EvidenceComparatorRe
|
||||
gone = base.elements.filter(element => content(element, headWords));
|
||||
arrived = head.elements.filter(element => content(element, baseWords));
|
||||
}
|
||||
// Text without an identifier is matched by its label, so a label that changed on its own leaves as one element and
|
||||
// arrives as another: the one of its role at the same place on both sides is runtime state when its label is.
|
||||
const siblings = mixedBackends ? [] : resolveSiblings(input, input.impact.baseSamples);
|
||||
const runtimeText = { base: new Set<string>(), head: new Set<string>() };
|
||||
for (const removed of siblings.length ? specificByLabel(gone) : []) {
|
||||
const added = counterpart(removed, head);
|
||||
if (!added || !arrived.includes(added) || counterpart(added, base) !== removed) continue;
|
||||
const within = unstableWithin(removed, added, siblings);
|
||||
if (!within) continue;
|
||||
runtimeText.base.add(compact(removed.label));
|
||||
runtimeText.head.add(compact(added.label));
|
||||
facts.push(supported({
|
||||
kind: "unstable-text",
|
||||
subjectRole: reviewerRole(added.role),
|
||||
before: compact(removed.label),
|
||||
after: compact(added.label),
|
||||
within,
|
||||
}, [elementRef("base", removed), elementRef("head", added)]));
|
||||
}
|
||||
gone = gone.filter((element) => !runtimeText.base.has(compact(element.label)));
|
||||
arrived = arrived.filter((element) => !runtimeText.head.has(compact(element.label)));
|
||||
for (const element of outermost(specificByLabel(gone))) {
|
||||
const label = compact(element.label);
|
||||
if (!label) {
|
||||
@@ -1141,7 +1238,7 @@ function commonResult(input: CommonSurfaceComparatorInput): EvidenceComparatorRe
|
||||
}
|
||||
}
|
||||
for (const pair of pairing.pairs) {
|
||||
const comparison = comparePair(pair, base, head);
|
||||
const comparison = comparePair(pair, base, head, unstableWithin(pair.base, pair.head, siblings));
|
||||
facts.push(...comparison.facts);
|
||||
issues.push(...comparison.issues);
|
||||
}
|
||||
@@ -1154,7 +1251,7 @@ function commonResult(input: CommonSurfaceComparatorInput): EvidenceComparatorRe
|
||||
const visual = visualFact(input, base, head);
|
||||
// Pixels always move when an element appears or its text changes. The generic styling line
|
||||
// only informs when nothing in the accessibility tree accounts for the difference.
|
||||
const explained = facts.length > 0 && input.visualScope?.kind !== "landmark";
|
||||
const explained = facts.some((fact) => fact.change.kind !== "unstable-text") && input.visualScope?.kind !== "landmark";
|
||||
if (visual.fact && !explained) facts.push(visual.fact);
|
||||
if (visual.issue) issues.push(visual.issue);
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ export type ReviewerChangeTemplateId =
|
||||
| "element.removed"
|
||||
| "visible-text.changed-short"
|
||||
| "visible-text.changed-long"
|
||||
| "visible-text.unstable-within-build"
|
||||
| "interaction.target-changed"
|
||||
| "accessibility.label-changed"
|
||||
| "accessibility.role-changed"
|
||||
@@ -95,6 +96,19 @@ export type VisibleTextReviewerChange = {
|
||||
after: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A label that differs between the two builds but also between two captures of the same build (its light and dark
|
||||
* appearances): a clock, a relative time, a random name. Runtime state, not a change.
|
||||
*/
|
||||
export type UnstableTextReviewerChange = {
|
||||
kind: "unstable-text";
|
||||
subjectRole: ReviewerSubjectRole;
|
||||
before: string;
|
||||
after: string;
|
||||
/** What the same build showed in its other capture. */
|
||||
within: string;
|
||||
};
|
||||
|
||||
export type InteractionReviewerChange = {
|
||||
kind: "action-target-changed";
|
||||
subjectRole: ReviewerSubjectRole;
|
||||
@@ -134,6 +148,7 @@ export type SupportedReviewerChange =
|
||||
| ScreenReviewerChange
|
||||
| ElementReviewerChange
|
||||
| VisibleTextReviewerChange
|
||||
| UnstableTextReviewerChange
|
||||
| InteractionReviewerChange
|
||||
| AccessibilityReviewerChange
|
||||
| VisualReviewerChange
|
||||
@@ -387,6 +402,28 @@ function buildFact(change: SupportedReviewerChange, evidenceRefs: string[]): Can
|
||||
},
|
||||
};
|
||||
}
|
||||
case "unstable-text": {
|
||||
const before = compact(change.before);
|
||||
const after = compact(change.after);
|
||||
const within = compact(change.within);
|
||||
if (!before || !after || !within) return invalidChange("An unstable-text change requires all three labels.", evidenceRefs);
|
||||
if (before === after) return invalidChange("An unstable-text change has identical labels.", evidenceRefs);
|
||||
const seen = [...new Set([before, within, after])];
|
||||
return {
|
||||
kind: "fact",
|
||||
fact: {
|
||||
category: "visible-text",
|
||||
templateId: "visible-text.unstable-within-build",
|
||||
bullet: `Text varies between captures of the same build: ${seen.map(quote).join(" / ")}.`,
|
||||
subjectRole: change.subjectRole,
|
||||
subjectLabel: after,
|
||||
before,
|
||||
after,
|
||||
priority: PRIORITY["visible-text"],
|
||||
evidenceRefs,
|
||||
},
|
||||
};
|
||||
}
|
||||
case "accessibility-label-changed": {
|
||||
const before = compact(change.before);
|
||||
const after = compact(change.after);
|
||||
|
||||
@@ -17,6 +17,7 @@ function element(options = {}) {
|
||||
label: options.label ?? "Continue",
|
||||
...(options.visible === undefined ? { visible: true } : { visible: options.visible }),
|
||||
...(options.selector === undefined ? {} : { selector: options.selector }),
|
||||
...(options.bounds === undefined ? {} : { bounds: options.bounds }),
|
||||
occurrence: options.occurrence ?? 0,
|
||||
controller: "agent-device",
|
||||
actions: options.actions ?? ["press"],
|
||||
@@ -320,6 +321,104 @@ test("keeps an unannotated label delta in the accessibility category", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("a label that varies within both builds, in the words that differ between them, is runtime state", () => {
|
||||
const stamp = (label) => [element({ elementId: "stamp", identifier: "stamp", role: "text", label, actions: [], actionable: false })];
|
||||
const sibling = (base, head) => ({ base: side("base", base), head: side("head", head) });
|
||||
const input = {
|
||||
impact: impact(),
|
||||
base: side("base", stamp("edited 2m ago")),
|
||||
head: side("head", stamp("edited 3m ago")),
|
||||
visualAssessment: visual(),
|
||||
};
|
||||
const plain = supportedChanges(compareSurfaceEvidence(input));
|
||||
assert.ok(plain.some((change) => change.kind === "accessibility-label-changed"), "without a sibling capture the label difference is a change");
|
||||
|
||||
const result = compareSurfaceEvidence({ ...input, siblings: [sibling(stamp("edited 1m ago"), stamp("edited 4m ago"))] });
|
||||
const changes = supportedChanges(result);
|
||||
assert.deepEqual(changes.map((change) => change.kind), ["unstable-text"]);
|
||||
assert.deepEqual(changes[0], { kind: "unstable-text", subjectRole: "text", before: "edited 2m ago", after: "edited 3m ago", within: "edited 4m ago" });
|
||||
const summary = summarizeReviewerChanges({ changes: result.reviewerChanges });
|
||||
assert.deepEqual(summary.facts.map((fact) => [fact.templateId, fact.bullet]),
|
||||
[["visible-text.unstable-within-build", "Text varies between captures of the same build: \u201cedited 2m ago\u201d / \u201cedited 4m ago\u201d / \u201cedited 3m ago\u201d."]]);
|
||||
assert.equal(summary.needsReview, false);
|
||||
|
||||
// A stable label beside it is still a change.
|
||||
const save = (label) => element({ identifier: "save", label });
|
||||
const mixed = compareSurfaceEvidence({
|
||||
...input,
|
||||
base: side("base", [...stamp("edited 2m ago"), save("Submit")]),
|
||||
head: side("head", [...stamp("edited 3m ago"), save("Save")]),
|
||||
siblings: [sibling([...stamp("edited 1m ago"), save("Submit")], [...stamp("edited 4m ago"), save("Save")])],
|
||||
});
|
||||
assert.deepEqual(supportedChanges(mixed).map((change) => change.kind).sort(), ["accessibility-label-changed", "unstable-text"]);
|
||||
// Pixels that only an unstable label explains still get the generic styling line.
|
||||
const pixels = compareSurfaceEvidence({ ...input, visualAssessment: visual("modified"), siblings: [sibling(stamp("edited 1m ago"), stamp("edited 4m ago"))] });
|
||||
assert.deepEqual(supportedChanges(pixels).map((change) => change.kind), ["unstable-text", "visual-changed"]);
|
||||
});
|
||||
|
||||
test("a label that follows the appearance, or changed in a word that does not vary, is still a change", () => {
|
||||
const label = (text) => [element({ identifier: "toggle", label: text })];
|
||||
const compare = ([baseLight, baseDark], [headLight, headDark], options = {}) => supportedChanges(compareSurfaceEvidence({
|
||||
impact: impact(),
|
||||
base: side("base", label(baseLight), options.base),
|
||||
head: side("head", label(headLight), options.head),
|
||||
visualAssessment: visual(),
|
||||
siblings: [{ base: side("base", label(baseDark)), head: side("head", label(headDark), options.headSibling) }],
|
||||
})).map((change) => change.kind);
|
||||
// The theme toggle's label follows the appearance; the head broke it in light only.
|
||||
assert.deepEqual(compare(["Switch to dark mode", "Switch to light mode"], ["Switch to light mode", "Switch to light mode"]), ["accessibility-label-changed"]);
|
||||
assert.deepEqual(compare(["Switch to dark mode", "Switch to light mode"], ["Enable dark mode", "Switch to light mode"]), ["accessibility-label-changed"]);
|
||||
// The base varies, the head does not.
|
||||
assert.deepEqual(compare(["Save", "Save draft"], ["Submit", "Submit"]), ["accessibility-label-changed"]);
|
||||
// The time varies; the name beside it changed.
|
||||
assert.deepEqual(compare(["Posted by Alice · 2m", "Posted by Alice · 3m"], ["Posted by Bob · 4m", "Posted by Bob · 5m"]), ["accessibility-label-changed"]);
|
||||
// A sibling capture that did not resolve cleanly is no evidence.
|
||||
assert.deepEqual(compare(["Saved 2m ago", "Saved 3m ago"], ["Saved 4m ago", "Saved 5m ago"], { headSibling: { capturedSamples: 1 } }), ["accessibility-label-changed"]);
|
||||
assert.deepEqual(compare(["Saved 2m ago", "Saved 3m ago"], ["Saved 4m ago", "Saved 5m ago"]), ["unstable-text"]);
|
||||
|
||||
// Provenance on one side only stays inconclusive.
|
||||
const annotation = [{ identifier: "toggle", role: "button", textSource: "visible-title" }];
|
||||
const result = compareSurfaceEvidence({
|
||||
impact: impact(),
|
||||
base: side("base", label("Saved 2m ago"), { annotations: annotation }),
|
||||
head: side("head", label("Saved 4m ago")),
|
||||
visualAssessment: visual(),
|
||||
siblings: [{ base: side("base", label("Saved 3m ago"), { annotations: annotation }), head: side("head", label("Saved 5m ago")) }],
|
||||
});
|
||||
assert.equal(result.needsReview, true);
|
||||
assert.ok(result.reviewReasons.some((reason) => /provenance.*one side/i.test(reason)));
|
||||
assert.equal(supportedChanges(result).some((change) => change.kind === "unstable-text"), false);
|
||||
});
|
||||
|
||||
test("text without an identifier that varies within both builds is matched by its place, not its label", () => {
|
||||
const bounds = { x: 0, y: 120, width: 390, height: 64 };
|
||||
// A pattern row: a pressable with one accessibility label and no testID, exposed again by its container on iOS.
|
||||
const row = (label) => [
|
||||
element({ elementId: "row", role: "button", label, selector: `label=${label}`, bounds }),
|
||||
element({ elementId: "row-cell", role: "other", label, selector: `label=${label}`, bounds, actions: [], actionable: false }),
|
||||
];
|
||||
const compare = (baseLabels, headLabels) => compareSurfaceEvidence({
|
||||
impact: impact(),
|
||||
base: side("base", row(baseLabels[0])),
|
||||
head: side("head", row(headLabels[0])),
|
||||
visualAssessment: visual(),
|
||||
...(baseLabels[1] ? { siblings: [{ base: side("base", row(baseLabels[1])), head: side("head", row(headLabels[1])) }] } : {}),
|
||||
});
|
||||
assert.deepEqual(supportedChanges(compare(["Velvet Tresillo, 4 lanes, edited 2m ago"], ["Velvet Tresillo, 4 lanes, edited 4m ago"])).map((change) => change.kind).sort(),
|
||||
["element-added", "element-removed"], "without a sibling capture the row left and arrived");
|
||||
|
||||
const result = compare(["Velvet Tresillo, 4 lanes, edited 2m ago", "Velvet Tresillo, 4 lanes, edited 3m ago"],
|
||||
["Velvet Tresillo, 4 lanes, edited 4m ago", "Velvet Tresillo, 4 lanes, edited 5m ago"]);
|
||||
assert.deepEqual(supportedChanges(result), [{ kind: "unstable-text", subjectRole: "button",
|
||||
before: "Velvet Tresillo, 4 lanes, edited 2m ago", after: "Velvet Tresillo, 4 lanes, edited 4m ago", within: "Velvet Tresillo, 4 lanes, edited 5m ago" }]);
|
||||
assert.equal(result.needsReview, false);
|
||||
// A random name is runtime state the same way.
|
||||
assert.deepEqual(supportedChanges(compare(["Velvet Tresillo", "Copper Pulse"], ["Hollow Echo", "Ghost Relay"])).map((change) => change.kind), ["unstable-text"]);
|
||||
// A real change in the same row is still a change.
|
||||
assert.deepEqual(supportedChanges(compare(["Velvet Tresillo, 4 lanes, edited 2m ago", "Velvet Tresillo, 4 lanes, edited 3m ago"],
|
||||
["Velvet Tresillo, 5 lanes, edited 4m ago", "Velvet Tresillo, 5 lanes, edited 5m ago"])).map((change) => change.kind).sort(), ["element-added", "element-removed"]);
|
||||
});
|
||||
|
||||
test("fails closed when visible-title provenance exists on only one side", () => {
|
||||
const annotation = [{
|
||||
identifier: "primary-cta",
|
||||
|
||||
@@ -5,7 +5,7 @@ import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { PNG as PngImage } from "pngjs";
|
||||
|
||||
import { assessPublicEvidence, presenceDescription, ignoreStatusBar, ignoreSystemBands, ignoreSystemRegions, readExportedMap } from "../../dist/review/evidence-assessment.js";
|
||||
import { assessPublicEvidence, decideReview, presenceDescription, ignoreStatusBar, ignoreSystemBands, ignoreSystemRegions, readExportedMap } from "../../dist/review/evidence-assessment.js";
|
||||
import { STATUS_BAR_BAND_PERCENT } from "../../dist/shared/png-evidence.js";
|
||||
import { homeIndicatorBandPercent, statusBarBandPercent, systemRegions } from "../../dist/contracts/capture.js";
|
||||
import { AGENT_DEVICE_VERSION } from "../../dist/capture/run.js";
|
||||
@@ -526,6 +526,48 @@ test("a presence verdict for a long route (an encoded share link) stays within t
|
||||
assert.equal(presenceDescription("Added ", "/home", " screen."), "Added /home screen.");
|
||||
});
|
||||
|
||||
test("unstable text never makes a screen Modified, and names a variant's runtime state", () => {
|
||||
const unstable = { category: "visible-text", templateId: "visible-text.unstable-within-build", bullet: 'Text varies between captures of the same build: "2m" / "4m" / "3m".' };
|
||||
const summary = (facts) => ({ facts, needsReview: false, reviewIssues: [] });
|
||||
const unchanged = { verdict: "unchanged", reason: "matched", mismatchPercentage: 0, threshold: 0.1, tool: "t" };
|
||||
const modified = { ...unchanged, verdict: "modified", reason: "repeated-cross-build-delta", mismatchPercentage: 0.4, regions: [{ location: "center" }] };
|
||||
assert.deepEqual(decideReview("screen", unchanged, summary([unstable])), { status: "unchanged", description: "No visible change" });
|
||||
const pixels = decideReview("screen", modified, summary([unstable]));
|
||||
assert.deepEqual([pixels.status, pixels.description, pixels.facts.length], ["modified", "Visual change · 0.4% · center", 1]);
|
||||
const real = decideReview("screen", modified, summary([{ category: "accessibility", templateId: "accessibility.label-changed", bullet: 'Changed accessibility label from "Submit" to "Save".' }, unstable]));
|
||||
assert.equal(real.description, 'Changed accessibility label from "Submit" to "Save".');
|
||||
assert.equal(decideReview("none", modified, summary([unstable])).description, 'Runtime state differs: Text varies between captures of the same build: "2m" / "4m" / "3m".');
|
||||
// A variant is named by what else differs, ahead of the runtime text.
|
||||
const target = { category: "interaction", templateId: "interaction.target-changed", bullet: "Changed where “Open” leads." };
|
||||
assert.equal(decideReview("none", modified, summary([unstable, target])).description, "Runtime state differs: Changed where “Open” leads.");
|
||||
});
|
||||
|
||||
test("text that varies between both builds' own verified appearances is runtime state; an unverified one is no evidence", async (t) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "explore-public-evidence-")); t.after(() => rm(root, { recursive: true, force: true }));
|
||||
const capture = async (unverifiedHeadDark) => {
|
||||
const base = await side(join(root, `${unverifiedHeadDark}`), "base", manifests.base);
|
||||
const head = await side(join(root, `${unverifiedHeadDark}`), "head", manifests.head);
|
||||
const labels = { base: ["edited 2m ago", "edited 3m ago"], head: ["edited 4m ago", "edited 5m ago"] };
|
||||
for (const [name, current] of [["base", base], ["head", head]]) {
|
||||
current.bundle.runtimeProfile = { deviceModel: "iPhone 17", osVersion: "26.0", locale: "en-US", orientation: "portrait", colorScheme: "light", colorSchemes: "light,dark" };
|
||||
await addDarkAppearance(current, manifests[name]);
|
||||
for (const [index, suffix] of ["", "@dark"].entries()) {
|
||||
const observation = current.map.observations.find((entry) => entry.id === `${name}-changed-observation${suffix}`);
|
||||
observation.elements[0].label = labels[name][index];
|
||||
}
|
||||
}
|
||||
if (unverifiedHeadDark) head.bundle.outcomes["changed@dark"] = { status: "captured", attempts: 1, landing: "unverified" };
|
||||
return assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", measure: measurer({}) });
|
||||
};
|
||||
const verified = await capture(false);
|
||||
assert.deepEqual(verified.surfaces.changed.review, { status: "unchanged", description: "No visible change" });
|
||||
assert.deepEqual(verified.surfaces["changed@dark"].review, { status: "unchanged", description: "No visible change" });
|
||||
// The head's dark capture may show another screen: the light label difference stays a change.
|
||||
const unverified = await capture(true);
|
||||
assert.equal(unverified.surfaces.changed.review.status, "modified");
|
||||
assert.deepEqual(unverified.surfaces.changed.review.facts.map((fact) => fact.templateId), ["accessibility.label-changed"]);
|
||||
});
|
||||
|
||||
test("a foldable's rail clock pill is left out of the verdict like a phone's status bar", () => {
|
||||
// Measured on EAS Simulator captures of an iPhone Duo (iOS 27.1): a 40×64 pt clock + Wi-Fi pill 28 pt from the right edge.
|
||||
const closed = systemRegions({ platform: "ios", device: "iPhone Duo", pose: "closed" });
|
||||
|
||||
Reference in new issue
Block a user