Compare commits

...
23 changed files with 792 additions and 32 deletions

No files matched your search

+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@expo/explore",
"version": "0.20.1",
"version": "0.20.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@expo/explore",
"version": "0.20.1",
"version": "0.20.2",
"license": "MIT",
"dependencies": {
"@expo/repack-app": "0.10.1",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@expo/explore",
"version": "0.20.1",
"version": "0.20.2",
"description": "Source-driven mobile app maps, deterministic CI capture, and visual reviews",
"license": "MIT",
"type": "module",
+35 -5
View File
@@ -5,6 +5,7 @@
* injected, so the translation is tested against recorded Argent output.
*/
import { CONTROLLER_SESSION } from "./common.js";
import { INPUT_METHOD_PACKAGES, KEYBOARD_WINDOW_IDENTIFIERS } from "./keyboard.js";
import type { DecodedFrame } from "../shared/png-evidence.js";
export type ArgentPlatform = "ios" | "android";
@@ -88,7 +89,7 @@ export function parseCommand(argv: readonly string[]): { positional: string[]; f
}
type ArgentNode = { role?: string; type?: string; label?: string; identifier?: string; value?: string; contentDescription?: string; disabled?: boolean;
frame?: Rect; traits?: string[]; hidden?: boolean; selected?: boolean; password?: boolean; bundleId?: string; children?: ArgentNode[] };
frame?: Rect; traits?: string[]; hidden?: boolean; selected?: boolean; focused?: boolean; password?: boolean; bundleId?: string; children?: ArgentNode[] };
export type ArgentTree = { schemaVersion?: number; source?: string; screen?: { width: number; height: number }; roots?: ArgentNode[];
truncated?: boolean; alertVisible?: boolean; keyboardVisible?: boolean; foregroundApp?: string };
@@ -137,7 +138,11 @@ export function snapshotFromTree(tree: ArgentTree, platform: ArgentPlatform, gen
if (root || !node.frame || ![node.frame.x, node.frame.y, node.frame.width, node.frame.height].every(finite)) return clip;
return clip && intersect(node.frame, clip);
};
const visit = (node: ArgentNode, depth: number, parentIndex: number | undefined, appRoot: boolean, clip: Rect | undefined, root: boolean) => {
const keyboard = platform === "ios" ? keyboardWindows(roots) : new Set<ArgentNode>();
/** `owner` marks every node of a window that is the system keyboard's, as agent-device marks an input method's nodes. */
type Owner = { bundleId?: string; systemKeyboard?: true };
const visit = (node: ArgentNode, depth: number, parentIndex: number | undefined, appRoot: boolean, clip: Rect | undefined, root: boolean, owner: Owner) => {
if (keyboard.has(node)) owner = { ...owner, systemKeyboard: true };
const frame = node.frame;
const inner = clipOf(node, clip, root);
const rect = frame && [frame.x, frame.y, frame.width, frame.height].every(finite) ? toPoints(frame) : undefined;
@@ -159,19 +164,42 @@ export function snapshotFromTree(tree: ArgentTree, platform: ArgentPlatform, gen
rect: appRoot ? { x: 0, y: 0, width: screen.width, height: screen.height } : rect,
...(label ? { label } : {}), ...(node.identifier ? { identifier: node.identifier } : {}),
...(node.value !== undefined ? { value: node.value } : {}), enabled, hittable: enabled && visible && !appRoot,
...(visible ? {} : { visible: false }), ...(node.selected || traits.includes("selected") ? { selected: true } : {}) });
...(visible ? {} : { visible: false }), ...(node.selected || traits.includes("selected") ? { selected: true } : {}),
// The field with the keyboard focus, as agent-device marks it, so capture can wait for the keyboard it raises.
...(node.focused === true || traits.includes("isEditing") ? { focused: true } : {}),
...owner });
refs[ref] = { rect: appRoot ? { x: 0, y: 0, width: screen.width, height: screen.height } : rect, type,
...(node.value !== undefined ? { value: node.value } : {}), ...(type === "SecureTextField" ? { secure: true } : {}) };
}
for (const child of node.children ?? []) visit(child, depth + 1, index, false, inner, false);
for (const child of node.children ?? []) visit(child, depth + 1, index, false, inner, false, owner);
};
for (const root of roots) {
const isAppRoot = platform === "ios" ? root.type === "Application" || nodes.length === 0 : root.bundleId === app;
visit(root, 0, undefined, isAppRoot, { x: 0, y: 0, width: 1, height: 1 }, true);
// An Android input method's window names its package on each node, as agent-device's do, which marks the keyboard.
const inputMethod = platform === "android" && root.bundleId && INPUT_METHOD_PACKAGES.has(root.bundleId) ? root.bundleId : undefined;
visit(root, 0, undefined, isAppRoot, { x: 0, y: 0, width: 1, height: 1 }, true, inputMethod ? { bundleId: inputMethod } : {});
}
return { nodes, refs, screen: { width: screen.width, height: screen.height } };
}
/**
* The windows iOS draws the software keyboard in. Argent lists each window as a child of its root, the app's first. A
* later window is the keyboard's when it holds XCTest's `Keyboard` element, which no app can give itself; an identifier
* never decides it alone, since an app can set any identifier on its own views (a second window of the app's with a
* `testID="inputView"` stays the app's). While such a window is up, UIKit also hosts the keyboard's input view
* placeholder in a window of its own: a later window whose only named node is an `inputView` belongs to it too.
*/
function keyboardWindows(roots: readonly ArgentNode[]): Set<ArgentNode> {
const holds = (node: ArgentNode, test: (node: ArgentNode) => boolean): boolean => test(node) || (node.children ?? []).some(child => holds(child, test));
const later = roots.flatMap(root => (root.children ?? []).slice(1));
const keyboard = later.filter(window => holds(window, node => node.type === "Keyboard"));
if (!keyboard.length) return new Set();
const named = (node: ArgentNode) => Boolean(node.label?.trim() || node.contentDescription?.trim() || node.value !== undefined
|| (node.identifier && !KEYBOARD_WINDOW_IDENTIFIERS.has(node.identifier)));
const placeholders = later.filter(window => !keyboard.includes(window) && holds(window, node => node.identifier === "inputView") && !holds(window, named));
return new Set([...keyboard, ...placeholders]);
}
function intersect(a: Rect, b: Rect): Rect | undefined {
const x = Math.max(a.x, b.x), y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.width, b.x + b.width), bottom = Math.min(a.y + a.height, b.y + b.height);
@@ -438,7 +466,9 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
converted.nodes.push({ depth: 1, index: converted.nodes.length, parentIndex: 0, type: "Alert", ref: `e${converted.nodes.length + 1}`,
rect: { x: 0, y: 0, width: converted.screen.width, height: converted.screen.height }, label: "System alert", enabled: true, hittable: false });
}
// Whether a keyboard shows, which Argent's Android tree does not hold: the input method's window is not in it.
return success({ nodes: converted.nodes, truncated: current.truncated === true, refsGeneration: state.generation,
...(typeof current.keyboardVisible === "boolean" ? { keyboardVisible: current.keyboardVisible } : {}),
...(current.foregroundApp ? { appBundleId: current.foregroundApp, appName: current.foregroundApp } : {}),
snapshotQuality: { state: "healthy", backend: "argent" } });
}
+8 -4
View File
@@ -5,6 +5,7 @@ import type {
ElementRole,
InputElement,
} from "../types.js";
import { INPUT_METHOD_PACKAGES, keyboardNodes } from "./keyboard.js";
export type ControllerParseResult = {
controller: Controller;
@@ -221,10 +222,7 @@ function nodeToElement(node: Record<string, unknown>): InputElement | undefined
const SYSTEM_UI_PACKAGES = new Set([
"android",
"com.android.systemui",
"com.google.android.inputmethod.latin",
"com.samsung.android.honeyboard",
"com.touchtype.swiftkey",
"com.microsoft.swiftkey",
...INPUT_METHOD_PACKAGES,
]);
export function isSystemUiNode(node: Record<string, unknown>): boolean {
@@ -240,6 +238,12 @@ function collectJsonElements(value: unknown, output: InputElement[], seen: Set<u
return;
}
const record = value as Record<string, unknown>;
// iOS's software keyboard is system UI too, like Android's: its keys and its Emoji and Dictate buttons are not the
// app's elements. A keyboard that rose on one side only made "Added button “Dictate”" lead a review whose real
// change was a field's label (brentvatne/euxy#172, review run 39362642343).
if (Array.isArray(record.nodes)) {
for (const node of keyboardNodes(record.nodes.filter((node): node is Record<string, unknown> => !!node && typeof node === "object" && !Array.isArray(node)))) seen.add(node);
}
const element = isSystemUiNode(record) ? undefined : nodeToElement(record);
if (element) output.push(element);
for (const [key, child] of Object.entries(record)) {
+146
View File
@@ -0,0 +1,146 @@
/**
* The system software keyboard in a controller snapshot (agent-device's flat `nodes`, which the Argent bridge also
* returns), and how long capture waits for it. Recognition is deliberately narrow: dropping one of the app's own elements
* hides a real change, which is worse than the keyboard noise this removes. A node is the keyboard's only when it, or a
* node above it, provably belongs to the system keyboard; nothing is inferred from where an element sits on the screen,
* and a `Key` alone proves nothing (React Native's `accessibilityRole="keyboardkey"` gives an app's PIN pad that type).
*/
type Rect = { x: number; y: number; width: number; height: number };
type SnapshotNode = Record<string, unknown>;
/** Android input methods: their windows name the IME's package, not the app's. */
export const INPUT_METHOD_PACKAGES: ReadonlySet<string> = new Set([
"com.google.android.inputmethod.latin",
"com.android.inputmethod.latin",
"com.samsung.android.honeyboard",
"com.touchtype.swiftkey",
"com.microsoft.swiftkey",
]);
/**
* UIKit's identifiers for the input view placeholder and the prediction bar. They mark a window as the keyboard's (see
* the Argent bridge's `keyboardWindows`); inside the app's own window an element with one of them is the app's.
*/
export const KEYBOARD_WINDOW_IDENTIFIERS: ReadonlySet<string> = new Set(["inputView", "SystemInputAssistantView"]);
/**
* The keyboard's native view classes, which agent-device's host accessibility bridge reports as `role`. The dock's Emoji
* and Dictate buttons (UIKeyboardDockItemButton) can sit outside the key plane's bounds, so they are named here.
*/
const KEYBOARD_VIEW_CLASSES = new Set(["UIKeyboardDockItemButton", "UIAccessibilityElementKBKey", "UIKeyboardLayoutStar",
"TUIPredictionView", "TUIPredictionViewCell", "_UIRemoteKeyboardPlaceholderView", "TUISystemInputAssistantView",
"TUISystemInputAssistantPageView"]);
const EDITABLE_TYPES = new Set(["textfield", "securetextfield", "searchfield", "textview"]);
const text = (value: unknown) => typeof value === "string" ? value : "";
function rectOf(node: SnapshotNode): Rect | undefined {
const rect = node.rect as Partial<Rect> | undefined;
return rect && [rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) && rect.width! > 0 && rect.height! > 0 ? rect as Rect : undefined;
}
/**
* A node that belongs to the system keyboard by what it is, not where it is: XCTest's `Keyboard` element (both
* controllers), a node of a window the Argent bridge found to be the keyboard's (`systemKeyboard`), one of the keyboard's
* native view classes, or a node of an Android input method's window.
*/
function keyboardRoot(node: SnapshotNode): boolean {
return node.systemKeyboard === true || text(node.type).toLowerCase() === "keyboard" || KEYBOARD_VIEW_CLASSES.has(text(node.role))
|| INPUT_METHOD_PACKAGES.has(text(node.bundleId ?? node.packageName ?? node.package));
}
/** Every node that belongs to the system keyboard: those `keyboardRoot` names and the nodes beneath them. */
export function keyboardNodes(nodes: readonly SnapshotNode[]): Set<SnapshotNode> {
const owned = new Set<SnapshotNode>();
if (!nodes.some(keyboardRoot)) return owned;
// Only an explicit parent links a node to a keyboard above it: a node without one is no node's descendant.
const byIndex = new Map(nodes.filter(node => typeof node.index === "number").map(node => [node.index as number, node]));
for (const node of nodes) {
const seen = new Set<SnapshotNode>();
for (let current: SnapshotNode | undefined = node; current && !seen.has(current);
current = typeof current.parentIndex === "number" ? byIndex.get(current.parentIndex) : undefined) {
seen.add(current);
if (keyboardRoot(current)) { owned.add(node); break; }
}
}
return owned;
}
function union(rects: readonly Rect[]): Rect | undefined {
if (!rects.length) return undefined;
const x = Math.min(...rects.map(rect => rect.x)), y = Math.min(...rects.map(rect => rect.y));
return { x, y, width: Math.max(...rects.map(rect => rect.x + rect.width)) - x, height: Math.max(...rects.map(rect => rect.y + rect.height)) - y };
}
/**
* Where the visible keyboard is, from the nodes that span it (the `Keyboard` element, the keyboard window's input view and
* prediction bar, the native views, an input method's window), or undefined when none shows. Containers inside the
* keyboard can report a scroll extent past the screen, so not every node it owns counts.
*/
export function keyboardBounds(nodes: readonly SnapshotNode[]): Rect | undefined {
return union(nodes.filter(node => node.visible !== false && keyboardRoot(node) && (node.systemKeyboard !== true
|| text(node.type).toLowerCase() === "keyboard" || KEYBOARD_WINDOW_IDENTIFIERS.has(text(node.identifier)))).flatMap(node => rectOf(node) ?? []));
}
/**
* A text input that has the keyboard focus: agent-device reports `focused` from XCTest's keyboard focus and Android's
* focused flag; the Argent bridge passes on Argent's own.
*/
export function focusedTextInput(nodes: readonly SnapshotNode[]): boolean {
return nodes.some(node => node.focused === true && node.visible !== false
&& (EDITABLE_TYPES.has(text(node.type).toLowerCase()) || /EditText$/.test(text(node.type)) || node.editable === true));
}
/** A snapshot as the wait reads it: Argent also says whether a keyboard shows, which its Android tree cannot. */
export type KeyboardRead = { nodes: readonly SnapshotNode[]; keyboardVisible?: boolean };
/** Where the keyboard is in one read: its bounds, or only that one shows when the controller cannot place it. */
function keyboardPosition(read: KeyboardRead): string | undefined {
const bounds = keyboardBounds(read.nodes);
return bounds ? JSON.stringify(bounds) : read.keyboardVisible === true ? "visible" : undefined;
}
/**
* How capture waits for a focused field's keyboard. Reads are 250 ms apart at most, and at least two follow the first,
* so a controller whose snapshot takes a second or more still sees the keyboard rise. Waiting for a keyboard that has not
* appeared ends after a second; one that has appeared but has not yet held still is read until it does, up to six reads
* or four seconds, so a slow controller cannot keep the capture waiting.
*/
export const KEYBOARD_POLL_MS = 250;
const KEYBOARD_WAIT_MS = 1_000;
const KEYBOARD_CAP_MS = 4_000;
const KEYBOARD_MIN_READS = 2;
const KEYBOARD_MAX_READS = 6;
export type KeyboardSettle = {
/** `shown`: in the same place in two reads running. `unfocused`: the field let go and no keyboard shows. */
readonly outcome: "shown" | "moving" | "absent" | "unfocused";
readonly reads: number;
/** Whether to read the screen again, `elapsedMs` after the first read. */
more(elapsedMs: number): boolean;
read(next: KeyboardRead): void;
};
/**
* The wait for a screen whose text field has the focus, or undefined when none has it. A keyboard already up counts only
* once a later read finds it in the same place: one caught rising or going down is still moving.
*/
export function keyboardSettle(first: KeyboardRead): KeyboardSettle | undefined {
if (!focusedTextInput(first.nodes)) return undefined;
let previous = keyboardPosition(first), reads = 0;
let outcome: KeyboardSettle["outcome"] = previous ? "moving" : "absent";
return {
get outcome() { return outcome; },
get reads() { return reads; },
more: elapsedMs => (outcome === "moving" || outcome === "absent") && reads < KEYBOARD_MAX_READS
&& (reads < KEYBOARD_MIN_READS || (elapsedMs < KEYBOARD_CAP_MS && (outcome === "moving" || elapsedMs < KEYBOARD_WAIT_MS))),
read: next => {
reads += 1;
const position = keyboardPosition(next);
outcome = position && position === previous ? "shown" : position ? "moving" : focusedTextInput(next.nodes) ? "absent" : "unfocused";
previous = position;
},
};
}
+59 -6
View File
@@ -17,6 +17,7 @@ import { type CaptureSetupSnapshot, credentialSavePromptDecline, describeSetupAc
import { silentProgress } from "../shared/progress.js";
import type { CaptureSetupReady } from "../config/explore-config.js";
import { READINESS_POLL_MS, defaultReadinessSeconds } from "./readiness.js";
import { KEYBOARD_POLL_MS, keyboardSettle, type KeyboardRead, type KeyboardSettle } from "./keyboard.js";
import { CAPTURE_OBSERVER_IDENTIFIER, captureObserverMarkers, parseCaptureDestination, captureDestinationMatches } from "./observer.js";
import { type CiAgentRuntimeOptions, runCiAgentNavigation, assertAgentCaptureContext } from "./agent/runtime.js";
import { createExplorationInputs } from "../config/exploration-inputs.js";
@@ -92,6 +93,11 @@ function snapshotNodes(raw: string): Record<string, unknown>[] {
return data.nodes;
}
/** Whether the controller says a keyboard shows: Argent does, and its Android tree has no input method window to show it. */
function keyboardVisibility(data: Record<string, unknown>): { keyboardVisible?: boolean } {
return typeof data.keyboardVisible === "boolean" ? { keyboardVisible: data.keyboardVisible } : {};
}
/** The visible element each required label matched, so an image assessor can find icon-only controls. */
export function verifiedCaptureLabels(nodes: readonly Record<string, unknown>[], requiredLabels: readonly string[],
/** The screen's portrait height, when the screenshot is that screen turned a quarter clockwise (an opened iPhone Duo). */
@@ -377,7 +383,7 @@ export async function capturePlannedRoutes(options: {
lastSnapshot = raw;
const generation = typeof data.refsGeneration === "number" ? data.refsGeneration : undefined;
const decline = declined < 2 && generation ? credentialSavePromptDecline(nodes) : undefined;
if (!decline) return { nodes, truncated: false, ...(generation ? { refsGeneration: generation } : {}) };
if (!decline) return { nodes, truncated: false, ...(generation ? { refsGeneration: generation } : {}), ...keyboardVisibility(data) };
controller(["press", `@${String(decline.ref).replace(/^@/, "")}~s${generation}`, "--level", "full", "--json"]);
recordDeclinedPrompt("deterministic");
controller(["wait", "500", "--level", "full", "--json"]);
@@ -507,6 +513,35 @@ export async function capturePlannedRoutes(options: {
.sort());
/** The controller's own quiescence gate; a timeout is reported by the snapshot comparison below. */
const settle = () => { controller(["wait", "stable", "500", "8000", "--level", "full", "--json"], true); };
/**
* A screen that focused a text field shows the keyboard that field raises, as its users see it. The keyboard rises
* after the screen has settled, so the controller's quiescence gate can pass in between: in brentvatne/euxy#172
* (review run 39362642343) the rename sheet's base was captured with its cursor showing and no keyboard, its head with
* the keyboard up, and the 48% mismatch was the keyboard. The screen is read again until the keyboard is in the same
* place twice running (see keyboardSettle); a keyboard that never comes is not waited for long, and none is dismissed.
*/
const awaitKeyboard = (route: SourceRoute, first: KeyboardRead): readonly Record<string, unknown>[] => {
const settle = keyboardSettle(first);
if (!settle) return first.nodes;
const startedAt = clock();
let current = first, pollStartedAt = startedAt;
// A read counts for at least one poll interval, so a clock that stands still (a fake controller) still expires.
const elapsed = () => Math.max(clock() - startedAt, settle.reads * KEYBOARD_POLL_MS);
while (settle.more(elapsed())) {
(capture.sleep ?? pause)(KEYBOARD_POLL_MS - (clock() - pollStartedAt));
pollStartedAt = clock();
current = snapshot();
settle.read(current);
}
recordKeyboardWait(route.id, settle, elapsed());
return current.nodes;
};
const recordKeyboardWait = (route: string | undefined, settle: KeyboardSettle, elapsedMs: number) => {
const diagnostics = join(store.root, "capture-diagnostics");
mkdirSync(diagnostics, { recursive: true, mode: 0o700 });
appendFileSync(join(diagnostics, "keyboard.jsonl"), `${JSON.stringify({ ...(route ? { route } : { phase: "agent-observe" }), outcome: settle.outcome,
reads: settle.reads, elapsedMs, at: new Date(clock()).toISOString() })}\n`, { mode: 0o600 });
};
const attempted = new Set<string>();
const captureCurrent = async (route: SourceRoute, attempt: number, before?: readonly Record<string, unknown>[],
unattributed?: NonNullable<CaptureBundle["outcomes"][string]["observed"]> & { message: string }) => {
@@ -514,9 +549,11 @@ export async function capturePlannedRoutes(options: {
attempted.add(route.id);
const temporary = await mkdtemp(join(store.root, ".capture-"));
try {
const accepted = before ?? waitForDestination(route, () => resolveDeepLinkConfirmation());
if (unattributed) assertNoAlert({ route, nodes: accepted });
else landing(accepted, route, inventory);
const arrived = before ?? waitForDestination(route, () => resolveDeepLinkConfirmation());
if (unattributed) assertNoAlert({ route, nodes: arrived });
else landing(arrived, route, inventory);
const accepted = unattributed ? arrived : awaitKeyboard(route, { nodes: arrived });
if (accepted !== arrived) landing(accepted, route, inventory);
assertSafeScreenshot(accepted);
const screenshot = join(temporary, "screen.png");
await writeFile(screenshot, "", { flag: "wx", mode: 0o600 });
@@ -663,7 +700,7 @@ export async function capturePlannedRoutes(options: {
const generation = typeof data.refsGeneration === "number" ? data.refsGeneration : undefined;
const decline = declined < 2 && generation ? credentialSavePromptDecline(nodes) : undefined;
if (!decline) {
const snapshot: CaptureSetupSnapshot = { nodes, truncated: false, ...(generation ? { refsGeneration: generation } : {}) };
const snapshot: CaptureSetupSnapshot = { nodes, truncated: false, ...(generation ? { refsGeneration: generation } : {}), ...keyboardVisibility(data) };
return { snapshot, raw };
}
await command(["press", `@${String(decline.ref).replace(/^@/, "")}~s${generation}`, "--level", "full", "--json"], signal);
@@ -769,16 +806,32 @@ export async function capturePlannedRoutes(options: {
}
}
};
/** The same wait for a focused field's keyboard as a route link's capture, before an observation's screenshot. */
const awaitKeyboardAsync = async (first: CaptureSetupSnapshot, signal: AbortSignal): Promise<CaptureSetupSnapshot> => {
const settle = keyboardSettle(first);
if (!settle) return first;
const startedAt = clock();
let current = first;
const elapsed = () => Math.max(clock() - startedAt, settle.reads * KEYBOARD_POLL_MS);
while (settle.more(elapsed())) {
await sleep(KEYBOARD_POLL_MS, undefined, { signal: liveSignal(signal) });
current = (await readSnapshot(signal)).snapshot;
settle.read(current);
}
recordKeyboardWait(undefined, settle, elapsed());
return current;
};
const observe = async (signal: AbortSignal, options?: { includeImage: boolean }) => {
await discard();
await appIdentity(signal);
const first = await readSnapshotThroughDeferral(signal);
const before = first.snapshot;
let before = first.snapshot;
if (first.retries > 0) {
await appendFile(join(dirname(tracePath), "accessibility-deferral.jsonl"), `${JSON.stringify({ retries: first.retries, waitedMs: first.deferredMs,
at: new Date(clock()).toISOString() })}\n`, { mode: 0o600 });
}
if (options?.includeImage === false) return { snapshot: before };
before = await awaitKeyboardAsync(before, signal);
try { assertSafeScreenshot(before.nodes); }
catch (error) {
if (error instanceof CiError && error.code === "CAPTURE_SENSITIVE_INPUT") return { snapshot: before, imageUnavailable: { code: error.code, message: error.message } };
+2
View File
@@ -6,6 +6,8 @@ export type CaptureSetupSnapshot = {
nodes: readonly Record<string, unknown>[];
refsGeneration?: number;
truncated?: boolean;
/** Whether the controller says a software keyboard shows, when it says (Argent does). */
keyboardVisible?: boolean;
};
type Awaitable<T> = T | Promise<T>;
+5
View File
@@ -251,6 +251,11 @@ export type PublicSurfaceReview = {
description: string;
/** Every deterministic accessibility fact for this surface, in reviewer priority order. */
facts?: Array<{ category: string; templateId: string; bullet: string }>;
/**
* The screenshots match because the change is past the edge of the screen: every element its facts name reaches
* beyond the captured screen. Only on a `modified` verdict whose screenshots match.
*/
offScreen?: true;
};
/** An evidence-bound explanation. It never changes a deterministic comparison verdict. */
export type PublicChangeNote = {
+1
View File
@@ -57,6 +57,7 @@ function assertReviewFields(surface: Surface, enforceComparisonEvidence: boolean
|| (["modified", "unchanged", "variant"].includes(review.status) && (!surface.base || !surface.head))
|| (surface.sourceScope !== undefined && !["added", "removed"].includes(review.status)
&& !SCOPE_VERDICTS[surface.sourceScope]?.includes(review.status))
|| (review.offScreen !== undefined && (review.offScreen !== true || review.status !== "modified" || surface.visual?.verdict !== "unchanged"))
|| (review.facts !== undefined && (!Array.isArray(review.facts) || review.facts.length > 100
|| review.facts.some((fact) => !fact || typeof fact.category !== "string" || typeof fact.templateId !== "string"
|| typeof fact.bullet !== "string" || !fact.bullet.trim() || fact.bullet.length > MAX_REVIEW_DESCRIPTION)))) {
+6 -2
View File
@@ -6,6 +6,7 @@ import {
type Observation,
type Screen,
} from "../types.js";
import { accessibilityFingerprintsComparable } from "./store.js";
/**
* Cross-run screen matching for PR-style comparisons (baseline = base branch map,
@@ -253,7 +254,8 @@ function matchEntry(match: Match): { entry: BuildComparisonEntry; review: Compar
// captured on both sides) hide a real change to the primary content.
const baselineHash = match.baseline.representative.fingerprints.accessibility;
const currentHash = match.current.representative.fingerprints.accessibility;
const verdict = baselineHash && currentHash
// Hashes from different fingerprint versions (an older Explore's map) say nothing about whether the trees differ.
const verdict = baselineHash && currentHash && accessibilityFingerprintsComparable(baselineHash, currentHash)
? (baselineHash === currentHash ? "unchanged" : "modified")
: "unknown";
const unchanged = verdict === "unchanged";
@@ -262,7 +264,9 @@ function matchEntry(match: Match): { entry: BuildComparisonEntry; review: Compar
? `Matched by ${signals}; the representative captures share an accessibility fingerprint, so the rendered tree is unchanged.`
: verdict === "modified"
? `Matched by ${signals}; the representative captures have different accessibility fingerprints, so the rendered tree changed.`
: `Matched by ${signals}; accessibility fingerprints are unavailable on at least one side, so the change state could not be verified from fingerprints.`;
: baselineHash && currentHash
? `Matched by ${signals}; the representative captures were fingerprinted by different Explore versions, so the change state could not be verified from fingerprints.`
: `Matched by ${signals}; accessibility fingerprints are unavailable on at least one side, so the change state could not be verified from fingerprints.`;
const entry: BuildComparisonEntry = {
id: `match-${match.current.screen.id}`,
relationship: "one-to-one",
+23 -5
View File
@@ -14,6 +14,7 @@ import { dirname, extname, isAbsolute, join, relative, resolve, sep } from "node
import { fileURLToPath } from "node:url";
import { Ajv2020, type ErrorObject, type ValidateFunction } from "ajv/dist/2020.js";
import { isSystemUiNode } from "../capture/controller-output.js";
import { keyboardNodes } from "../capture/keyboard.js";
import { validateNavigationGraph } from "../contracts/navigation-graph.js";
import {
CONTRACT,
@@ -67,11 +68,26 @@ export const normalizeSelector = (input?: string): string | undefined => {
return input.replace(/\s+/g, " ").trim() || undefined;
};
const VOLATILE_SNAPSHOT_NODE_KEYS = new Set(["ref"]);
/** A ref names a node within one read; which field has the focus can change between two reads of one screen. */
const VOLATILE_SNAPSHOT_NODE_KEYS = new Set(["ref", "focused"]);
/**
* The accessibility fingerprint's algorithm, written as its prefix. Version 2 leaves the system keyboard and the focus out
* of the tree; a hash from before carries no prefix. Two fingerprints of different versions say nothing about whether the
* trees differ, so a comparison treats them as not comparable rather than changed (see accessibilityFingerprintsComparable).
*/
const ACCESSIBILITY_FINGERPRINT_VERSION = "a2";
export function accessibilityFingerprintsComparable(left: string, right: string): boolean {
const version = (hash: string) => /^(a\d+):/.exec(hash)?.[1] ?? "a1";
return version(left) === version(right);
}
/** Canonical JSON of the node tree only: controller wrappers carry per-session diagnostics and
* ref generations that would otherwise make identical screens fingerprint differently, and Android
* system UI nodes (status bar clock and signal, keyboard) change between emulator sessions. */
* system UI nodes (status bar clock and signal, keyboard) change between emulator sessions. The
* system keyboard on either platform is left out as the elements leave it out, so a keyboard up on
* one side only does not make two captures of one screen differ. */
function normalizeJsonSnapshotNodes(input: string): string | undefined {
if (!/^\s*\{/.test(input)) return undefined;
let value: unknown;
@@ -86,10 +102,12 @@ function normalizeJsonSnapshotNodes(input: string): string | undefined {
if (!data || typeof data !== "object" || !Array.isArray((data as { nodes?: unknown }).nodes)) {
return undefined;
}
const nodes = (data as { nodes: unknown[] }).nodes;
const keyboard: Set<unknown> = keyboardNodes(nodes.filter((entry): entry is Record<string, unknown> => !!entry && typeof entry === "object" && !Array.isArray(entry)));
const canonical = (node: unknown): unknown => {
if (Array.isArray(node)) {
return node
.filter((entry) => !(entry && typeof entry === "object" && isSystemUiNode(entry as Record<string, unknown>)))
.filter((entry) => !(entry && typeof entry === "object" && (isSystemUiNode(entry as Record<string, unknown>) || keyboard.has(entry))))
.map(canonical);
}
if (!node || typeof node !== "object") return node;
@@ -100,7 +118,7 @@ function normalizeJsonSnapshotNodes(input: string): string | undefined {
.map(([key, entry]) => [key, canonical(entry)]),
);
};
return `nodes\0${JSON.stringify(canonical((data as { nodes: unknown[] }).nodes))}`;
return `nodes\0${JSON.stringify(canonical(nodes))}`;
}
export const normalizeSnapshot = (input: string): string =>
@@ -118,7 +136,7 @@ export async function fingerprintState(options: {
snapshotText?: string;
}): Promise<ObservationFingerprints> {
const accessibility = options.snapshotText
? sha256(`snapshot\0${normalizeSnapshot(options.snapshotText)}`)
? `${ACCESSIBILITY_FINGERPRINT_VERSION}:${sha256(`snapshot\0${normalizeSnapshot(options.snapshotText)}`)}`
: undefined;
const screenshot = options.screenshotPath ? await readFile(options.screenshotPath) : undefined;
const visual = screenshot ? sha256(Buffer.concat([Buffer.from("visual\0"), screenshot])) : undefined;
+29 -4
View File
@@ -143,6 +143,16 @@ function notComparedChange(surface: Surface): boolean {
return verified && (!surface.review || surface.visual?.reason === "diff-failed");
}
/**
* An accessibility-only change whose elements lie past the edge of the screen (the review's `offScreen`), so another
* target's row can say the change is there, off-screen, rather than missing. Like notComparedChange, a comparison that
* failed or a redirect is not one.
*/
function offScreenChange(surface: Surface): boolean {
return surface.review?.offScreen === true && isAccessibilityOnlyChange(surface) && surface.visual?.reason !== "diff-failed"
&& (surface.head ?? surface.base)?.kind !== "redirect";
}
function routeOf(surface: Surface): string {
return plainText(surface.head?.route || surface.base?.route || surface.id, MAX_ROUTE_LENGTH);
}
@@ -302,7 +312,8 @@ export function platformRecordOf(state: PublicCommentState, target?: CommentTarg
const group = groupOf(surface, evidenceBased);
if (group === "unlisted") continue;
const scheme = schemeOf(surface.id);
record.changes.push({ id: surface.id, route: routeOf(surface), group, description: descriptionOf(surface, group), ...(scheme ? { scheme } : {}) });
record.changes.push({ id: surface.id, route: routeOf(surface), group, description: descriptionOf(surface, group), ...(scheme ? { scheme } : {}),
...(group === "accessibility-only" && offScreenChange(surface) ? { offScreen: true as const } : {}) });
}
const navigation = state.map ? navigationOf(state.review, new Map(record.changes.map(change => [change.id, change.group]))) : undefined;
if (navigation) record.navigation = navigation;
@@ -562,10 +573,24 @@ function renderChangeRow(row: ChangeRow, ready: PlatformRecord[]): string {
text = `${route} · ${description}`;
}
const tag = row.group === "accessibility-only" ? "accessibility only" : row.group === "needs-review" ? "needs review" : undefined;
return `- ${[text, tag, ...whereNote(row, ready)].filter(Boolean).join(" · ")}`;
}
/**
* Where a screen's change shows. A target whose screenshots match because the changed elements lie past the edge of the
* screen has the change, off-screen, so it is named apart: "iPhone 17, Android only · off-screen on iPhone Duo (closed)"
* for Tempo's footnote below the fold of its sheet in brentvatne/euxy#172 (review run 39362642343). The targets that show
* it are still named the shorter way among the others (none when all of them do); without such a target the note is what
* it always was.
*/
function whereNote(row: ChangeRow, ready: PlatformRecord[]): string[] {
const offScreen = row.group === "modified"
? row.targets.filter(record => record.changes?.some(change => change.id === row.id && change.offScreen === true)) : [];
const shown = row.targets.filter(record => !offScreen.includes(record));
// A target with screens it has no verdict on may have changed this one too: name where it changed, not where it did not.
const where = ready.some(record => !row.targets.includes(record) && gaps(record).length)
? `on ${row.targets.map(labelOf).join(", ")}` : targetsNote(row.targets, ready);
return `- ${[text, tag, where].filter(Boolean).join(" · ")}`;
const unsure = ready.some(record => !row.targets.includes(record) && gaps(record).length);
const visible = unsure ? `on ${shown.map(labelOf).join(", ")}` : targetsNote(shown, ready.filter(record => !offScreen.includes(record)));
return [visible, offScreen.length ? `off-screen on ${offScreen.map(labelOf).join(", ")}` : undefined].filter((part): part is string => Boolean(part));
}
/** "Android only" / "not on Android": the targets a screen changed on, when not all of them, named the shorter way. */
+7 -3
View File
@@ -103,8 +103,11 @@ export type PlatformRecord = {
accessibility?: { compared: number; described: number; failed: number };
routes?: number;
};
/** The review's changed surfaces, one per surface; `scheme` for a secondary appearance (`<route>@<scheme>`). */
changes?: { id: string; route: string; group: ChangeGroup; description: string; scheme?: string }[];
/**
* The review's changed surfaces, one per surface; `scheme` for a secondary appearance (`<route>@<scheme>`). `offScreen`
* marks an accessibility-only change whose elements lie past the edge of the screen.
*/
changes?: { id: string; route: string; group: ChangeGroup; description: string; scheme?: string; offScreen?: true }[];
/** Screens whose source changed and whose captures were verified, but that got no verdict: the comparison did not run or failed. */
notCompared?: number;
/** Differences stated once rather than per screen (variants, app-wide source changes). */
@@ -270,7 +273,8 @@ export function parsePlatformRecord(value: unknown): PlatformRecord | undefined
if (Array.isArray(value.changes)) {
const changes = value.changes.flatMap((entry): NonNullable<PlatformRecord["changes"]> => isRecord(entry) && isString(entry.id, 220) && isString(entry.route, 2000)
&& CHANGE_GROUPS.includes(entry.group as ChangeGroup) && isString(entry.description, 2000)
? [{ id: entry.id, route: entry.route, group: entry.group as ChangeGroup, description: entry.description, ...(isString(entry.scheme, 20) ? { scheme: entry.scheme } : {}) }] : []);
? [{ id: entry.id, route: entry.route, group: entry.group as ChangeGroup, description: entry.description, ...(isString(entry.scheme, 20) ? { scheme: entry.scheme } : {}),
...(entry.offScreen === true && entry.group === "accessibility-only" ? { offScreen: true as const } : {}) }] : []);
if (changes.length <= 2000) record.changes = changes;
}
const navigation = parseNavigation(value.navigation);
+1
View File
@@ -716,6 +716,7 @@ export async function assessPublicEvidence(options: {
const summary = summarizeReviewerChanges({ changes: result.reviewerChanges });
accessibilityCounts.compared += 1;
review = decideReview(scope, visual, summary);
if (result.offScreen && review.status === "modified" && visual.verdict === "unchanged") review = { ...review, offScreen: true };
if (review.facts?.length) accessibilityCounts.described += 1;
} catch (error) {
accessibilityCounts.failed += 1;
+52
View File
@@ -93,6 +93,8 @@ export type EvidenceComparatorResult = {
visualAssessment?: VisualEvidenceAssessment;
/** Feed directly to summarizeReviewerChanges. */
reviewerChanges: ReviewerChangeInput[];
/** The screenshots match, and every element the changes name lies at least partly past the edge of the screen. */
offScreen?: true;
};
type SupportedInput = Extract<ReviewerChangeInput, { assessment: "supported" }>;
@@ -617,6 +619,54 @@ function outermost(elements: readonly ObservedElement[]): ObservedElement[] {
});
}
/**
* The screen in its accessibility tree's units: the element at the origin with the screenshot's shape (either way round,
* since a frame turned to show an opened iPhone Duo upright keeps the tree on the panel's portrait axes), the largest if
* several. Both controllers report the app's root that way. Without the screenshot's size there is no telling which
* element is the screen.
*/
function viewportOf(side: ResolvedSide): ElementBounds | undefined {
const image = side.map.artifacts.find(artifact => artifact.id === side.observation.screenshotArtifactId);
if (!image?.width || !image.height) return undefined;
const { width, height } = image;
const shaped = (bounds: ElementBounds) => Math.abs(bounds.width * height - bounds.height * width) <= 0.01 * width * height
|| Math.abs(bounds.width * width - bounds.height * height) <= 0.01 * width * height;
return side.observation.elements.flatMap(element => boundsOf(element) ?? [])
.filter(bounds => Math.abs(bounds.x) < 1 && Math.abs(bounds.y) < 1 && bounds.width > 0 && bounds.height > 0 && shaped(bounds))
.sort((a, b) => b.width * b.height - a.width * a.height)[0];
}
/**
* Whether the screenshots cannot show the changes because the elements they name lie past the edge of the screen: each
* change names at least one element, and more than half of each named element's area is beyond the screen on its side.
* Both controllers leave an element wholly outside the screen out of the tree (the Argent bridge as agent-device does),
* so a change past the edge can only show as an element straddling it, as Tempo's footnote below the fold of its sheet
* on the closed iPhone Duo in brentvatne/euxy#172 (review run 39362642343) would. Mostly outside, not merely crossing
* an edge: a control nudged a point or two past the edge is still on the screen, and its change stays "accessibility
* only". A change whose elements have no bounds never qualifies.
*/
function changesOffScreen(facts: readonly SupportedInput[], base: ResolvedSide, head: ResolvedSide): boolean {
const changes = facts.filter(fact => !["unstable-text", "no-visible-change", "visual-changed"].includes(fact.change.kind));
if (!changes.length) return false;
const viewports = { base: viewportOf(base), head: viewportOf(head) };
const outside = (reference: string) => {
const [, side, id] = /^(base|head):element:(.+)$/.exec(reference) ?? [];
if (side !== "base" && side !== "head") return undefined;
const viewport = viewports[side];
const element = (side === "base" ? base : head).observation.elements.find(candidate => candidate.elementId === id);
const bounds = element && boundsOf(element);
if (!viewport || !bounds) return false;
const visibleWidth = Math.max(0, Math.min(bounds.x + bounds.width, viewport.x + viewport.width) - Math.max(bounds.x, viewport.x));
const visibleHeight = Math.max(0, Math.min(bounds.y + bounds.height, viewport.y + viewport.height) - Math.max(bounds.y, viewport.y));
const area = bounds.width * bounds.height;
return area > 0 && visibleWidth * visibleHeight < area / 2;
};
return changes.every(fact => {
const named = fact.evidenceRefs.map(outside).filter((value): value is boolean => value !== undefined);
return named.length > 0 && named.every(Boolean);
});
}
function specificByLabel(elements: readonly ObservedElement[]): ObservedElement[] {
const byLabel = new Map<string, ObservedElement>();
const unlabelled: ObservedElement[] = [];
@@ -1276,6 +1326,7 @@ function commonResult(input: CommonSurfaceComparatorInput): EvidenceComparatorRe
const status: "modified" | "unchanged" = hasRuntimeDelta || reasons.length > 0
? "modified"
: "unchanged";
const offScreen = input.visualAssessment.status === "unchanged" && changesOffScreen(facts, base, head);
return {
surfaceId: input.impact.id,
status,
@@ -1288,6 +1339,7 @@ function commonResult(input: CommonSurfaceComparatorInput): EvidenceComparatorRe
),
visualAssessment: input.visualAssessment,
reviewerChanges,
...(offScreen ? { offScreen: true as const } : {}),
};
}
+2
View File
@@ -201,6 +201,8 @@ test("Android fill uses Argent's fill and requires it to verify the text", async
const snapshot = await run(["snapshot", ...flags("android")]);
const node = snapshot.json.data.nodes.find(item => item.type === "TextField");
assert.ok(node, "the rename fixture has an EditText");
assert.equal(node.focused, true, "the focused field is marked, as agent-device marks it");
assert.equal(snapshot.json.data.keyboardVisible, false, "whether a keyboard shows, which the Android tree cannot say, is passed on");
assert.equal((await run(["fill", node.ref, ...flags("android"), "--", "Groove"])).exitCode, 0);
assert.deepEqual(calls.find(call => call.tool === "fill").args, { udid: "emulator-5554", text: "Groove" });
const unverified = await run(["fill", node.ref, ...flags("android"), "--", "Groove"]);
+56
View File
@@ -69,6 +69,18 @@ const unreadableSurface = () => json({ success: false, error: { code: 'COMMAND_F
details: { hint: 'com.apple.SafariViewService hosts the surface presented over the app, and this capture targeted that surface. No snapshot backend could read this screen (XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen).' } } });
/**
* The iOS 26 keyboard as agent-device reports it: the Keyboard element with its keys beneath it, and the dock's Emoji and
* Dictate buttons, which its host bridge names by their view class (brentvatne/euxy#172, review run 39362642343).
*/
const softKeyboard = [
{ index: 900, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 }, visible: true },
{ index: 902, parentIndex: 900, type: 'Keyboard', rect: { x: 0, y: 583, width: 402, height: 233 }, visible: true },
{ index: 903, parentIndex: 902, type: 'Key', label: 'Q', rect: { x: 5, y: 590, width: 39, height: 54 }, visible: true },
{ index: 905, parentIndex: 900, type: 'Button', role: 'UIKeyboardDockItemButton', label: 'Emoji', rect: { x: 8, y: 806, width: 68, height: 68 }, visible: true },
{ index: 906, parentIndex: 900, type: 'Button', role: 'UIKeyboardDockItemButton', label: 'Dictate', identifier: 'dictation', rect: { x: 325, y: 806, width: 68, height: 68 }, visible: true },
];
function agentFixture(actions, observedRequests = []) {
return { instructions: 'Capture each selected route in its normal loaded state.', contexts: {},
provider: { provider: 'anthropic', model: 'test-model', preflight: async () => {}, run: async request => {
@@ -117,6 +129,8 @@ async function fixture(t, settings = {}) {
let enteredName = false; let scrolledToContinue = false; let finishHidden = false; let wedgeCleared = false; let deviceLost = false; let relaunchedAfterForeign = false; let serverStarts = 0; let unreadableLeft = settings.unreadableAfterFold ?? 0;
// startsErrored / startsUnnamed: how many session starts EAS fails, after naming the session (it ERRORS) or before.
let starts = 0; let currentSession = 'session-1'; const erroredSessions = new Set(); let foldUnverifiedLeft = settings.foldUnverified ?? 0;
// focusedField: the screen at `path` focuses its text field, and the keyboard is up from the `keyboardFrom`th read of it.
let focusedReads = 0;
// blackAfterFold: the first so many folds of an EAS agent-device session light the panel but draw nothing.
let folds = 0;
const deviceName = settings.android ? 'Pixel 7' : 'iPhone 17 Pro';
@@ -369,6 +383,8 @@ async function fixture(t, settings = {}) {
] : []),
...(settings.statefulSetup && current === '/transient' ? [{ type: 'Button', label: 'Finish', ref: 'e103', enabled: true, visible: !finishHidden }] : []),
...(alert ? [{ type: 'Alert', label: alert.message }, ...alert.items.map((label, index) => ({ type: 'Button', label, ref: `e${index + 1}`, enabled: true, visible: true }))] : []),
...(settings.focusedField?.path === current ? [{ type: 'TextField', label: 'Pattern name', focused: true, visible: true, rect: { x: 23, y: 386, width: 355, height: 50 } },
...(++focusedReads >= (settings.focusedField.keyboardFrom ?? Infinity) ? softKeyboard : [])] : []),
...(settings.missingObserver || settings.missingObserverAt === current ? [] : [observer]),
// A second active destination is ambiguous; the host AX bridge's identical copy is not.
...(settings.duplicateObserver ? [{ ...observer, label: JSON.stringify({ provider: 'expo-router', pathname: '/other', segments: ['other'], parameters: {} }) }] : []),
@@ -1181,6 +1197,46 @@ test('every app link settles before the screen is read', async t => {
assert.ok(opened >= 0 && settled > opened && settled < read, 'the settle runs between the link and the first read');
});
const keyboardLog = async f => (await readFile(join(f.options.mapDirectory, 'capture-diagnostics', 'keyboard.jsonl'), 'utf8'))
.trim().split('\n').map(line => JSON.parse(line));
test('a screen whose text field has the focus is captured once its keyboard is up and still, and the keyboard is not one of its elements', async t => {
// euxy#172 (review run 39362642343): the rename sheet focuses its field, and the keyboard rises after the sheet settles.
const f = await fixture(t, { routes: [route('rename')], focusedField: { path: '/rename', keyboardFrom: 2 } });
const result = await runCiMap(f.options);
assert.equal(result.outcomes[f.ids.rename].status, 'captured');
assert.equal(result.outcomes[f.ids.rename].attempts, 1);
assert.deepEqual((await keyboardLog(f)).map(line => [line.route, line.outcome, line.reads]), [[f.ids.rename, 'shown', 2]],
'read until the keyboard was up in the same place twice');
const calls = f.requests.filter(request => request.command === 'fake-controller').map(request => request.args[0]);
const shot = calls.lastIndexOf('screenshot');
assert.deepEqual(calls.slice(shot - 3, shot), ['snapshot', 'snapshot', 'snapshot'], 'the destination read, then the keyboard reads, then the screenshot');
const map = await readExportedMap(f.options.outputDirectory);
const labels = map.observations.flatMap(observation => observation.elements.map(element => element.label));
assert.ok(labels.includes('Pattern name'));
for (const label of ['Q', 'Emoji', 'Dictate']) assert.equal(labels.includes(label), false, `${label} is the keyboard's, not the app's`);
});
test('a focused field whose keyboard never rises is captured after about a second and at least two reads, without it', async t => {
const f = await fixture(t, { routes: [route('rename')], focusedField: { path: '/rename' } });
const result = await runCiMap(f.options);
assert.equal(result.outcomes[f.ids.rename].status, 'captured');
const [line] = await keyboardLog(f);
assert.equal(line.outcome, 'absent');
assert.ok(line.reads >= 2 && line.reads <= 6 && line.elapsedMs >= 1000, `gave up after ${line.reads} reads in ${line.elapsedMs} ms`);
assert.equal(f.operations().includes('keyboard'), false, 'the keyboard is never dismissed or summoned');
// A screen without a focused field is not read again.
const plain = await fixture(t, { routes: [route('home')] });
await runCiMap(plain.options);
assert.equal(existsSync(join(plain.options.mapDirectory, 'capture-diagnostics', 'keyboard.jsonl')), false);
});
test('a keyboard already up when the screen is read is captured once a later read finds it in the same place', async t => {
const f = await fixture(t, { routes: [route('rename')], focusedField: { path: '/rename', keyboardFrom: 1 } });
assert.equal((await runCiMap(f.options)).outcomes[f.ids.rename].status, 'captured');
assert.deepEqual((await keyboardLog(f)).map(line => [line.outcome, line.reads]), [['shown', 1]]);
});
test('an application Open button is never an implicit confirmation action', async t => {
const f = await fixture(t, { routes: [route('home')], appOpenButton: true });
const result = await runCiMap(f.options);
+12
View File
@@ -331,3 +331,15 @@ test("compare CLI emits a comparison, review, and catalog that reload as a works
));
assert.deepEqual(shared.baselineScreenIds, ["shared-home"]);
});
test("fingerprints from different Explore versions are not comparable: the match needs review instead of reading as changed", async () => {
// A map saved by Explore 0.20.x hashes the tree with the keyboard and the focus in it, unprefixed; a2 hashes leave them out.
const baseline = makeMap("main-eeeeeee", [{ id: "home", route: "/", acc: "0".repeat(64) }]);
const current = makeMap("pr-3-fffffff", [{ id: "home", route: "/", acc: `a2:${"1".repeat(64)}` }]);
const { comparison, review } = generateComparison(baseline, current);
await validateComparisonDocument(comparison, baseline, current);
const home = review.entries.find((entry) => entry.entryId === "match-home");
assert.equal(home.needsReview, true);
assert.match(entryById(comparison, "match-home").rationale, /different Explore versions/);
assert.ok(home.notes.some((note) => /compare the screenshots/.test(note)));
});
+45
View File
@@ -823,3 +823,48 @@ test("a capture's snapshot backend is read from its saved controller response",
assert.equal(snapshotBackend(directory, map("artifacts/accessibility/missing.json"), "o"), undefined);
assert.equal(snapshotBackend(directory, map("../outside.json"), "o"), undefined);
});
/**
* Tempo on the closed iPhone Duo in brentvatne/euxy#172 (review run 39362642343): a 1398 × 2034 screenshot of a
* 466 × 678 pt screen, and a footnote whose text changed below the fold of its scrolling sheet, so the screenshots match.
* The published review carries no accessibility tree, so the footnote's real bounds are unknown: these tests pin the
* rule (more than half of the changed element past the screen's edge), not where euxy's footnote was.
*/
function tempoSide(sideName, footnote, footnoteBounds, image = { width: 1398, height: 2034 }) {
const evidence = side(sideName, [
element({ elementId: "app", role: "other", label: "euxy", actions: [], actionable: false, bounds: { x: 0, y: 0, width: 466, height: 678, unit: "points" } }),
element({ elementId: "title", role: "text", label: "Base Resolution", actions: [], actionable: false, bounds: { x: 26, y: 560, width: 300, height: 24, unit: "points" } }),
element({ elementId: `${sideName}-footnote`, role: "text", label: footnote, actions: [], actionable: false, bounds: footnoteBounds }),
]);
if (image) {
evidence.map.artifacts.push({ id: `${sideName}-shot`, kind: "screenshot", path: `${sideName}.png`, sha256: "0".repeat(64), mimeType: "image/png", ...image, createdAt: "2026-09-01T01:00:00.000Z" });
evidence.map.observations[0].screenshotArtifactId = `${sideName}-shot`;
}
return evidence;
}
const NEW_LANES = "Changes apply live while playing. Base resolution sets the step grid for new lanes.";
const ADD_NEXT = "Changes apply live while playing. Base resolution sets the step grid for lanes you add next.";
// Its top 12 pt on the screen, 28 pt below the bottom edge: 70% past it.
const mostlyBelow = { x: 26, y: 666, width: 414, height: 40, unit: "points" };
const offScreenOf = (bounds, assessment = visual("unchanged"), image) => compareSurfaceEvidence({ impact: impact(),
base: tempoSide("base", NEW_LANES, bounds, image), head: tempoSide("head", ADD_NEXT, bounds, image), visualAssessment: assessment }).offScreen;
test("matching screenshots of a change whose elements lie mostly past the screen's edge are off-screen", () => {
assert.equal(offScreenOf(mostlyBelow), true);
// A turned frame (an opened iPhone Duo shown upright) keeps the tree on the portrait axes: the screen is still found.
assert.equal(offScreenOf(mostlyBelow, visual("unchanged"), { width: 2034, height: 1398 }), true);
});
test("a change on the screen, partly on it, with differing screenshots, or with no screen to measure against is never off-screen", () => {
const cases = {
"an on-screen text change": [{ x: 26, y: 600, width: 414, height: 40, unit: "points" }],
// Its top 28 pt on the screen, 12 pt below: mostly visible, so matching pixels prove nothing about it.
"a footnote mostly on the screen": [{ x: 26, y: 650, width: 414, height: 40, unit: "points" }],
// A control nudged 2 pt past the left edge, otherwise wholly visible.
"a control just crossing an edge": [{ x: -2, y: 600, width: 300, height: 40, unit: "points" }],
"screenshots that differ": [mostlyBelow, visual("modified")],
"no screenshot size": [mostlyBelow, visual("unchanged"), null],
"a changed element without bounds": [undefined],
};
for (const [name, args] of Object.entries(cases)) assert.equal(offScreenOf(...args), undefined, name);
});
+201
View File
@@ -0,0 +1,201 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import { tsImport } from "tsx/esm/api";
const { snapshotFromTree } = await tsImport("../../src/capture/argent-driver.ts", import.meta.url);
const { focusedTextInput, keyboardBounds, keyboardNodes, keyboardSettle } = await tsImport("../../src/capture/keyboard.ts", import.meta.url);
const { parseControllerOutput } = await tsImport("../../src/capture/controller-output.ts", import.meta.url);
const { normalizeSnapshot } = await tsImport("../../src/map/store.ts", import.meta.url);
const fixture = name => JSON.parse(readFileSync(new URL(`../fixtures/argent-0.29/${name}`, import.meta.url), "utf8"));
// euxy's rename sheet on iOS 26 under Argent 0.29, its field focused and the keyboard up: the head of brentvatne/euxy#172
// (review run 39362642343). Its root holds three windows: the app's, the input view placeholder's, and the keyboard's.
const RENAME_KEYBOARD_UP = fixture("fixture-ios-ui-tree-rename.json");
// The same sheet a moment earlier, as the base was captured: the field focused and its cursor showing, no keyboard yet.
const RENAME_KEYBOARD_DOWN = { ...RENAME_KEYBOARD_UP, keyboardVisible: false,
roots: [{ ...RENAME_KEYBOARD_UP.roots[0], children: RENAME_KEYBOARD_UP.roots[0].children.slice(0, 1) }] };
const ANDROID_RENAME = fixture("fixture-android-ui-tree-rename.json");
const nodesOf = tree => snapshotFromTree(tree, "ios", 1).nodes;
const labelsOf = nodes => parseControllerOutput("agent-device", JSON.stringify({ success: true, data: { nodes } })).elements.map(element => element.label);
const nameOf = node => node.label ?? node.identifier;
const rect = (x, y, width, height) => ({ x, y, width, height });
test("the iOS keyboard Argent reports is its own windows: the input view placeholder's and the keyboard's, Emoji and Dictate included", () => {
const nodes = nodesOf(RENAME_KEYBOARD_UP);
assert.deepEqual(keyboardBounds(nodes), rect(0, 539, 402, 335), "the input view and prediction bar down to the bottom edge");
const owned = [...keyboardNodes(nodes)];
const names = owned.map(nameOf).filter(Boolean);
for (const name of ["inputView", "SystemInputAssistantView", "Typing Predictions", "Q", "shift", "Emoji", "Dictate"]) assert.ok(names.includes(name), name);
assert.ok(owned.every(node => node.systemKeyboard === true), "only nodes of the keyboard's windows");
// Every element of the app's window stays, those behind the keyboard's band among them.
const app = nodesOf(RENAME_KEYBOARD_DOWN).map(nameOf).filter(Boolean);
for (const name of app) assert.ok(nodes.some(node => nameOf(node) === name && !keyboardNodes(nodes).has(node)), name);
assert.ok(app.includes("AdditionalDimmingOverlay") && app.includes("Tab Bar") && app.includes("Clear name"));
});
test("a keyboard up on one side only leaves the elements and the accessibility fingerprint unchanged", () => {
// In euxy#172 the review led with "Added button “Dictate”", “Emoji” and “inputView” over the label change.
const up = nodesOf(RENAME_KEYBOARD_UP), down = nodesOf(RENAME_KEYBOARD_DOWN);
assert.deepEqual(labelsOf(up), labelsOf(down));
for (const name of ["Dictate", "Emoji", "inputView", "Q"]) assert.equal(labelsOf(up).includes(name), false, name);
// The bridge numbers nodes in order, so the app's own nodes keep their indices when the keyboard's windows follow them.
const snapshot = nodes => JSON.stringify({ success: true, data: { nodes: nodes.map(({ ref: _ref, ...node }) => node) } });
assert.equal(normalizeSnapshot(snapshot(up)), normalizeSnapshot(snapshot(down)));
});
test("the text field with the focus is passed on from Argent", () => {
const up = nodesOf(RENAME_KEYBOARD_UP), down = nodesOf(RENAME_KEYBOARD_DOWN);
assert.equal(up.filter(node => node.focused === true).map(node => node.type).join(), "TextField");
assert.equal(focusedTextInput(up), true);
assert.equal(focusedTextInput(down), true);
assert.equal(keyboardBounds(down), undefined);
assert.equal(focusedTextInput(nodesOf(fixture("fixture-ios-ui-tree-home.json"))), false);
});
test("an element the app identifies as inputView in its own window is the app's", () => {
// A chat composer with testID="inputView", no keyboard up.
const app = RENAME_KEYBOARD_DOWN.roots[0].children[0];
const composer = { role: "AXGroup", type: "Other", identifier: "inputView", frame: rect(0, 0.8, 1, 0.1),
children: [{ role: "AXButton", type: "Button", label: "Send", frame: rect(0.8, 0.82, 0.15, 0.05), children: [] }] };
const tree = { ...RENAME_KEYBOARD_DOWN, roots: [{ ...RENAME_KEYBOARD_DOWN.roots[0], children: [{ ...app, children: [...app.children, composer] }] }] };
const nodes = nodesOf(tree);
assert.equal(keyboardNodes(nodes).size, 0);
assert.equal(keyboardBounds(nodes), undefined);
assert.ok(labelsOf(nodes).includes("inputView") && labelsOf(nodes).includes("Send"));
});
test("a second window of the app's own, its testID inputView included, is the app's; the keyboard's windows need a Keyboard", () => {
// The app's second UIWindow (a toast or a floating composer) with a view identified as inputView and a Save button.
const window = { role: "AXGroup", type: "Application", frame: rect(0, 0, 1, 1), children: [
{ role: "AXGroup", type: "Other", identifier: "inputView", frame: rect(0, 0.8, 1, 0.1),
children: [{ role: "AXButton", type: "Button", label: "Save", frame: rect(0.8, 0.82, 0.15, 0.05), children: [] }] }] };
const [app, placeholder, keyboard] = RENAME_KEYBOARD_UP.roots[0].children;
const withWindows = windows => nodesOf({ ...RENAME_KEYBOARD_UP, roots: [{ ...RENAME_KEYBOARD_UP.roots[0], children: [app, ...windows] }] });
for (const windows of [[window], [window, keyboard], [window, placeholder, keyboard]]) {
const labels = labelsOf(withWindows(windows));
assert.ok(labels.includes("Save") && labels.includes("inputView"), `kept beside ${windows.length - 1} keyboard windows`);
}
// Without the keyboard's own window, an input view placeholder window proves nothing.
assert.ok(labelsOf(withWindows([placeholder])).includes("inputView"));
assert.equal(keyboardNodes(withWindows([placeholder])).size, 0);
// With it, the placeholder is the keyboard's too, as in the real tree.
assert.equal(labelsOf(withWindows([placeholder, keyboard])).includes("inputView"), false);
});
test("agent-device's keyboard is its Keyboard element and what is under it, and its host bridge's keyboard view classes", () => {
// XCTest: the Keyboard directly under the Application, beside the app's own controls in the same band.
const xctest = [
{ index: 0, type: "Application", label: "Example", rect: rect(0, 0, 402, 874) },
{ index: 1, parentIndex: 0, type: "TextField", label: "Pattern name", focused: true, rect: rect(23, 386, 355, 50) },
{ index: 2, parentIndex: 0, type: "Keyboard", rect: rect(0, 583, 402, 233) },
{ index: 3, parentIndex: 2, type: "Key", label: "Q", rect: rect(5, 590, 39, 54) },
{ index: 4, parentIndex: 0, type: "Button", label: "Home tab", rect: rect(0, 800, 134, 74) },
{ index: 5, parentIndex: 0, type: "Other", label: "Formatting toolbar", rect: rect(0, 540, 402, 43) },
{ index: 6, parentIndex: 0, type: "Button", label: "Emoji", rect: rect(8, 300, 68, 40) },
];
assert.deepEqual(keyboardBounds(xctest), rect(0, 583, 402, 233));
assert.deepEqual(labelsOf(xctest), ["Example", "Pattern name", "Home tab", "Formatting toolbar", "Emoji"]);
// The host bridge names the dock's buttons by class, wherever they sit.
const hostBridge = [
{ type: "TextField", role: "RCTUITextField", label: "Pattern name", focused: true, rect: rect(23, 386, 355, 50) },
{ type: "Button", role: "UIKeyboardDockItemButton", label: "Dictate", rect: rect(325, 806, 68, 68) },
{ type: "Other", role: "UIAccessibilityElementKBKey", label: "Q", rect: rect(5, 590, 39, 54) },
{ type: "Button", role: "RCTViewComponentView", label: "Profile tab", rect: rect(268, 800, 134, 74) },
];
assert.ok(keyboardBounds(hostBridge));
assert.deepEqual(labelsOf(hostBridge), ["Pattern name", "Profile tab"]);
});
test("nothing without a parent link is taken for the keyboard's, and keys alone are no keyboard", () => {
// agent-device's flat shape: no parentIndex. The Keyboard element goes; nothing can be proven to be under it.
const flat = [
{ type: "Keyboard", rect: rect(0, 583, 402, 233) },
{ type: "Key", label: "Q", rect: rect(5, 590, 39, 54) },
{ type: "Button", label: "Profile tab", rect: rect(268, 800, 134, 74) },
];
assert.deepEqual(labelsOf(flat), ["Q", "Profile tab"]);
// An app's own PIN pad (accessibilityRole="keyboardkey") and its footer action, with no system keyboard up.
const pinPad = [
{ index: 0, type: "Application", label: "Example", rect: rect(0, 0, 402, 874) },
{ index: 1, parentIndex: 0, type: "SecureTextField", label: "PIN", focused: true, rect: rect(23, 200, 355, 50) },
...["1", "2", "3"].map((digit, offset) => ({ index: 2 + offset, parentIndex: 0, type: "Key", label: digit, rect: rect(offset * 134, 600, 134, 70) })),
{ index: 5, parentIndex: 0, type: "Button", label: "App footer action", rect: rect(0, 800, 402, 74) },
];
assert.equal(keyboardNodes(pinPad).size, 0);
assert.equal(keyboardBounds(pinPad), undefined);
assert.deepEqual(labelsOf(pinPad), ["Example", "PIN", "1", "2", "3", "App footer action"]);
});
test("in landscape the keyboard is told apart the same way, the app's bottom bar kept", () => {
const landscape = [
{ index: 0, type: "Application", label: "Example", rect: rect(0, 0, 874, 402) },
{ index: 1, parentIndex: 0, type: "TextField", label: "Search", focused: true, rect: rect(100, 60, 674, 40) },
{ index: 2, parentIndex: 0, type: "Keyboard", rect: rect(0, 230, 874, 172) },
{ index: 3, parentIndex: 2, type: "Key", label: "Q", rect: rect(60, 236, 70, 38) },
{ index: 4, parentIndex: 0, type: "TabBar", label: "Tab Bar", rect: rect(0, 352, 874, 50) },
];
assert.deepEqual(keyboardBounds(landscape), rect(0, 230, 874, 172));
assert.deepEqual(labelsOf(landscape), ["Example", "Search", "Tab Bar"]);
});
test("an Android input method's window is the keyboard under Argent, AOSP's included, and Argent says when one shows", () => {
const ime = pkg => ({ role: "FrameLayout", type: "android.widget.FrameLayout", bundleId: pkg, frame: rect(0, 0.6, 1, 0.4),
children: [{ role: "Button", type: "android.widget.Button", contentDescription: "q", frame: rect(0.01, 0.65, 0.09, 0.06), children: [] }] });
const before = snapshotFromTree(ANDROID_RENAME, "android", 1, "dev.brent.euxy").nodes;
assert.equal(focusedTextInput(before), true, "Argent marks the focused EditText");
assert.equal(keyboardBounds(before), undefined);
for (const pkg of ["com.google.android.inputmethod.latin", "com.android.inputmethod.latin"]) {
const after = snapshotFromTree({ ...ANDROID_RENAME, keyboardVisible: true, roots: [...ANDROID_RENAME.roots, ime(pkg)] }, "android", 1, "dev.brent.euxy").nodes;
assert.deepEqual(keyboardBounds(after), rect(0, 960, 720, 640), pkg);
assert.deepEqual(labelsOf(after), labelsOf(before), pkg);
}
});
test("the wait reads until the keyboard is in one place twice, at least twice, at most six times", () => {
const field = { type: "TextField", label: "Pattern name", focused: true, rect: rect(23, 386, 355, 50) };
const keyboardAt = y => [field, { index: 9, type: "Keyboard", rect: rect(0, y, 402, 874 - y) }];
// Already up when the screen was read: one more read must find it in the same place.
const up = keyboardSettle({ nodes: keyboardAt(583) });
assert.equal(up.more(0), true, "a keyboard already up may still be rising");
up.read({ nodes: keyboardAt(583) });
assert.deepEqual([up.outcome, up.reads, up.more(250)], ["shown", 1, false]);
// Caught rising: it moves between reads, then holds.
const rising = keyboardSettle({ nodes: [field] });
for (const y of [700, 583, 583]) rising.read({ nodes: keyboardAt(y) });
assert.deepEqual([rising.outcome, rising.reads], ["shown", 3]);
// A slow controller: the second read is owed even after a second has passed.
const slow = keyboardSettle({ nodes: [field] });
slow.read({ nodes: [field] });
assert.equal(slow.more(2_500), true);
slow.read({ nodes: [field] });
assert.deepEqual([slow.outcome, slow.more(5_000)], ["absent", false], "no keyboard came: captured without one");
// A slow first read without a keyboard, then the keyboard first seen still moving: read on until it holds.
const late = keyboardSettle({ nodes: [field] });
late.read({ nodes: [field] });
late.read({ nodes: keyboardAt(700) });
assert.deepEqual([late.outcome, late.more(2_500)], ["moving", true], "a moving keyboard is not given up on after a second");
late.read({ nodes: keyboardAt(583) });
late.read({ nodes: keyboardAt(583) });
assert.deepEqual([late.outcome, late.reads], ["shown", 4]);
// But not past four seconds, however few reads that was.
const stalled = keyboardSettle({ nodes: [field] });
stalled.read({ nodes: [field] });
stalled.read({ nodes: keyboardAt(700) });
assert.equal(stalled.more(4_000), false);
// Six reads at most, however fast they come.
const busy = keyboardSettle({ nodes: [field] });
for (let read = 0; busy.more(0); read++) busy.read({ nodes: keyboardAt(read % 2 ? 583 : 700) });
assert.deepEqual([busy.outcome, busy.reads], ["moving", 6]);
// The field let go and no keyboard shows: nothing more to wait for.
const released = keyboardSettle({ nodes: [field] });
released.read({ nodes: [{ ...field, focused: false }] });
assert.deepEqual([released.outcome, released.more(0)], ["unfocused", false]);
// Argent on Android says a keyboard shows without placing it: it shows twice running.
const android = keyboardSettle({ nodes: [field] });
android.read({ nodes: [field], keyboardVisible: true });
android.read({ nodes: [field], keyboardVisible: true });
assert.equal(android.outcome, "shown");
assert.equal(keyboardSettle({ nodes: [{ ...field, focused: false }] }), undefined, "a screen without a focused field is not waited on");
});
+74
View File
@@ -679,6 +679,80 @@ test('a config without targets writes the same records and comment as before tar
assert.deepEqual(Object.keys(readCommentRecords(after).current), ['ios', 'android']);
});
/**
* brentvatne/euxy#172, review run 39362642343: five targets, two strings changed. Tempo's footnote sits low in a
* scrolling sheet. On the closed iPhone Duo the accessibility tree has the new text but the screenshots match: the
* footnote runs past the bottom of the screen (`offScreen`, from the elements' bounds). On the half-open and opened Duo
* neither the tree nor the pixels have it. Android captures no dark appearance.
*/
const euxyTargets = [{ id: 'iphone', platform: 'ios', label: 'iPhone 17' }, { id: 'duo-closed', platform: 'ios', label: 'iPhone Duo (closed)' },
{ id: 'duo-half-open', platform: 'ios', label: 'iPhone Duo (half-open)' }, { id: 'duo-open', platform: 'ios', label: 'iPhone Duo (open)' },
{ id: 'android', platform: 'android', label: 'Android' }];
const footnote = 'Added text “Changes apply live while playing. Base resolution sets the step grid for lanes you add next.”.';
const footnoteText = 'Added text “Changes apply live while playing\\. Base resolution sets the step grid for lanes you add next\\.”';
function euxyReview(target, options = {}) {
const own = { sourceImpact: ['own'], sourceScope: 'screen' };
const matched = { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: '[email protected]' };
const changed = (description, mismatchPercentage) => ({ ...own,
review: { status: 'modified', description, facts: [{ category: 'element', templateId: 'element.added', bullet: description }] },
visual: { verdict: 'modified', reason: 'repeated-cross-build-delta', mismatchPercentage, threshold: 0.1, tool: '[email protected]' } });
const inTreeOnly = { ...changed(footnote, 0), visual: matched };
if (options.offScreen !== false) inTreeOnly.review.offScreen = true;
if (options.diffFailed) inTreeOnly.visual = { ...matched, reason: 'diff-failed' };
const unseen = { ...own, review: { status: 'unchanged', description: 'No visible change' }, visual: matched };
const tempo = target === 'duo-closed' ? inTreeOnly : ['duo-half-open', 'duo-open'].includes(target) ? unseen : changed(footnote, 0.11);
const rename = changed('Added text “Pattern name”.', 0.16);
const schemes = target === 'android' ? [''] : ['', '@dark'];
return review(schemes.flatMap(scheme => [
surface(`tempo${scheme}`, 'modified', { route: '/tempo', extra: structuredClone(tempo) }),
surface(`rename-pattern${scheme}`, 'modified', { route: '/rename-pattern', extra: structuredClone(rename) }),
]));
}
async function euxyRows(reviewOf = euxyReview) {
const { platformSectionMerge } = await import('../../dist/report/pr-comment-body.js');
let body;
for (const target of euxyTargets) body = platformSectionMerge(state({ artifactOnly: true, target, targets: euxyTargets, review: reviewOf(target.id),
build: { ...state().build, platform: target.platform }, runtime: { ...state().runtime, platform: target.platform, deviceModel: target.label } }), target.id).merge(body);
return { body, rows: listOf(body).map(line => line.replace(/\(<[^>]+>\)/, '')) };
}
test('a change whose elements run past the screen on a device is named off-screen there, beside the devices that show it (euxy#172)', async () => {
const { body, rows } = await euxyRows();
assert.deepEqual(rows, [
`- [\`/tempo\`] · ${footnoteText} · iPhone 17, Android only · off-screen on iPhone Duo \\(closed\\)`,
`- [\`/rename-pattern\`] · Added text “Pattern name”`,
`- [\`/tempo\`] (dark) · ${footnoteText} · iPhone 17 only · off-screen on iPhone Duo \\(closed\\)`,
`- [\`/rename-pattern\`] (dark) · Added text “Pattern name” · not on Android`,
], 'the half-open and opened Duo have no trace of it, so they stay among the devices it is not on');
const { readCommentRecords } = await import('../../dist/report/pr-comment-body.js');
assert.deepEqual(readCommentRecords(body).current['duo-closed'].changes.filter(change => change.offScreen).map(change => change.id), ['tempo', 'tempo@dark'],
'the mark survives the comment round trip');
});
test('without geometry that puts it past the screen, or with a failed comparison, a device keeps today\'s wording', async () => {
// The review euxy#172 published: the closed Duo's change is accessibility-only, with nothing saying where its elements are.
const { rows } = await euxyRows(target => euxyReview(target, { offScreen: false }));
assert.equal(rows[0], `- [\`/tempo\`] · ${footnoteText} · not on iPhone Duo \\(half\\-open\\), iPhone Duo \\(open\\)`);
assert.equal(rows[2], `- [\`/tempo\`] (dark) · ${footnoteText} · iPhone 17, iPhone Duo \\(closed\\) only`);
// A comparison that failed is no verdict at all: the closed Duo is not counted among the devices with the change.
const failed = await euxyRows(target => euxyReview(target, { diffFailed: true }));
assert.match(failed.rows[0], / · on iPhone 17, Android$/);
assert.doesNotMatch(failed.rows.join('\n'), /off-screen/);
});
test('a change one platform has and another lacks keeps "only" and "not on"', async () => {
// An iOS-only branch in a shared screen file: Android's capture of the screen shows no change, and says nothing off-screen.
const iosOnly = target => {
const reviewed = euxyReview(target);
if (target === 'android') for (const entry of reviewed.surfaces) Object.assign(entry, { review: { status: 'unchanged', description: 'No visible change' },
visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: '[email protected]' } });
return reviewed;
};
const { rows } = await euxyRows(iosOnly);
assert.equal(rows[0], `- [\`/tempo\`] · ${footnoteText} · iPhone 17 only · off-screen on iPhone Duo \\(closed\\)`);
assert.match(rows.find(row => row.startsWith('- [`/rename-pattern`] ·')), / · not on Android$/);
});
test('several targets share a comment: named in config order in the subtitle, each linking its report, keyed by id in the records and history', async () => {
const { platformSectionMerge, readCommentRecords } = await import('../../dist/report/pr-comment-body.js');
const [iphone, closed, open, android] = duoTargets;
@@ -320,3 +320,15 @@ test("an excluded surface carries no capture evidence", () => {
excluded.surfaces[1].excluded = false;
assert.throws(() => assertPublicReview(excluded), /excluded surface carries capture evidence/);
});
test("a review says its change is off-screen only on a modified verdict whose screenshots match", () => {
const offScreen = (status, verdict) => {
const input = review();
input.surfaces[0].review = { status, description: status === "modified" ? "Added text “lanes you add next”." : "No visible change", offScreen: true };
input.surfaces[0].visual = { verdict, reason: verdict === "unchanged" ? "matched" : "repeated-cross-build-delta", mismatchPercentage: verdict === "unchanged" ? 0 : 3, threshold: 0.1, tool: "[email protected]" };
return input;
};
assert.doesNotThrow(() => assertPublicReview(offScreen("modified", "unchanged")));
assert.throws(() => assertPublicReview(offScreen("modified", "modified")), /inconsistent/);
assert.throws(() => assertPublicReview(offScreen("unchanged", "unchanged")), /inconsistent/);
});
+13
View File
@@ -97,3 +97,16 @@ test("accessibility fingerprints ignore Android system UI nodes", async () => {
assert.equal(sameApp.accessibility, base.accessibility);
assert.notEqual(changedApp.accessibility, base.accessibility);
});
test("accessibility fingerprints are versioned, and leave out the system keyboard and which field has the focus", async () => {
const snapshot = (focused, keyboard) => JSON.stringify({ success: true, data: { nodes: [
{ index: 0, type: "Application", label: "euxy", rect: { x: 0, y: 0, width: 402, height: 874 } },
{ index: 1, parentIndex: 0, type: "TextField", label: "Pattern name", ...(focused ? { focused: true } : {}), rect: { x: 23, y: 386, width: 355, height: 50 } },
...(keyboard ? [{ index: 2, parentIndex: 0, type: "Keyboard", rect: { x: 0, y: 583, width: 402, height: 233 } },
{ index: 3, parentIndex: 2, type: "Key", label: "Q", rect: { x: 5, y: 590, width: 39, height: 54 } }] : []),
] } });
const resting = await fingerprintState({ snapshotText: snapshot(false, false) });
assert.match(resting.accessibility, /^a2:[0-9a-f]{64}$/);
assert.equal((await fingerprintState({ snapshotText: snapshot(true, false) })).accessibility, resting.accessibility, "focus alone does not change the tree");
assert.equal((await fingerprintState({ snapshotText: snapshot(true, true) })).accessibility, resting.accessibility, "nor does the keyboard");
});