mirror of
https://github.com/expo/explore.git
synced 2026-11-15 17:32:23 -10:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d6b97331b3 |
No files matched your search
+1
-1
@@ -46,7 +46,7 @@ Each layer imports only from the layers below it.
|
||||
4. `capture` imports `capture/agent`, `config` and `map`. `capture/agent` imports `capture` and `config`.
|
||||
5. `review` imports `capture` and `map`.
|
||||
6. `ci` imports `config`, `source`, `capture`, `capture/agent`, `review` and `report`. `binary` imports `config`, `capture`, `capture/agent`, `map` and `report`.
|
||||
7. `cli` imports `ci`, `binary`, `map`, `config` and `report`. `src/cli.ts` imports `cli`, `map` and `shared`.
|
||||
7. `cli` imports `ci`, `binary`, `map`, `config`, `report` and `review`. `src/cli.ts` imports `cli`, `map` and `shared`.
|
||||
|
||||
Every folder may import `contracts/`, `shared/` and `src/types.ts`.
|
||||
The exception is `contracts/`: its value imports stay inside `contracts/` or go to `src/types.ts`.
|
||||
|
||||
@@ -267,6 +267,14 @@ npx @expo/explore explore:binary ./my-app.app --flows flows.json
|
||||
- The report lists every flow and checkpoint of the file, in order, even those a run never reached: each one without a picture says why, so two runs still pair up step by step.
|
||||
- Each flow is also a journey of the map: the path its steps took, from the screen it started on. When the app moved without a tap the map could record (it left the app, or changed on its own), the path goes on as a second journey of the flow.
|
||||
- Relaunching ends the app's process, and the app keeps its data, so a flow that signs in leaves the next flow signed in. On `explore:session`, the app is relaunched before the first flow too.
|
||||
|
||||
To compare two runs of the same flows file, such as two builds, pass both output directories:
|
||||
|
||||
```sh
|
||||
npx @expo/explore compare:flows --before explore-app-1 --after explore-app-2 --output flows.json
|
||||
```
|
||||
|
||||
Each checkpoint comes back `unchanged`, `changed` (with where the screenshots differ, the status bar and home indicator set aside, as in a review), `missing` (no picture on one side, with why), `different-size` (another device or orientation, so not compared), or `added` / `removed` (in only one run's flows file). The result also lists how the two runs were captured differently (the app, device, OS version, locale, orientation, or appearance), since a change can come from those.
|
||||
- A `{{secret:NAME}}` or a described step limited by `platforms` to another platform needs no value, and no model, on this one.
|
||||
|
||||
</details>
|
||||
|
||||
@@ -42,6 +42,7 @@ To compare runs, or an iPhone with an Android phone, screen for screen, pass `--
|
||||
When the user describes a flow, or a set of flows, to look at ("sign in, then rename a pattern"), write them as a flows file and pass `--flows flows.json`, not `--replay`. The format is in the README ("Follow named flows, step by step"): `{ "flows": [{ "name": "Sign in", "steps": [{ "checkpoint": "Welcome" }, { "do": "Sign in with {{secret:TEST_EMAIL}}" }, { "checkpoint": "Home" }] }] }`.
|
||||
- Each step is an exact action (the `--replay` format), a step described in words (`do`) that a model carries out, or a named `checkpoint`. Use exact actions where you know the labels, and `do` where you don't.
|
||||
- Each flow starts from a relaunched app, and its checkpoints are photographed under their names. `report/exploration.json` lists each flow under `exploration.flows`, with its status and its checkpoints in order, so two runs pair up by flow and checkpoint name. A checkpoint on a screen that withholds its picture (a secret on it) is listed with why it has none, and its flow goes on.
|
||||
- To compare two runs of the same flows file, run `npx @expo/explore compare:flows --before DIR --after DIR` with the two output directories. Each checkpoint is `unchanged`, `changed` (with regions), `missing`, `different-size`, `added` or `removed`, and `conditions` lists what differed between the runs (device, OS, locale, appearance).
|
||||
- The file is checked before a device starts (`FLOWS_PLAN_INVALID`, `FLOWS_PLAN_EMPTY`, `FLOWS_PLAN_OVER_BUDGET`). `--provider`, `--model` and `--hints` steer the model that carries out `do` steps, and are refused when there are none (`FLOWS_OPTIONS_IGNORED`). `--flows` and `--replay` cannot be combined.
|
||||
|
||||
## Sign-in and other app state
|
||||
|
||||
+2
-1
@@ -11,7 +11,7 @@ import { help } from "./cli/help.js";
|
||||
import { listScreens, getScreen, listElements, listObservations, getObservation, nextFrontier, listAttempts, getAttempt, listTransitions, getTransition, neighbors, pathQuery, listJourneys, getJourney, listFrontier, listGaps, summary, search, status, validate, exportArtifacts } from "./cli/map-commands.js";
|
||||
import { compareCommand, exportWeb, view } from "./cli/catalog-commands.js";
|
||||
import { ciReport, ciPublish, ciFailure, ciDeploy, ciPullRequest, ciFindBaseline, ciCompareRefs, ciAfterCapture, setup, doctor, sourceVerify, captureReview, captureBaseline, configPlatforms, configTargetList } from "./cli/ci-commands.js";
|
||||
import { exploreBinary, exploreSession } from "./cli/binary-commands.js";
|
||||
import { compareFlows, exploreBinary, exploreSession } from "./cli/binary-commands.js";
|
||||
import { reports, reportUpload } from "./cli/report-commands.js";
|
||||
|
||||
const handlers: Record<string, () => Promise<void> | void> = {
|
||||
@@ -57,6 +57,7 @@ const handlers: Record<string, () => Promise<void> | void> = {
|
||||
"config:targets": configTargetList,
|
||||
doctor,
|
||||
compare: compareCommand,
|
||||
"compare:flows": compareFlows,
|
||||
view,
|
||||
help,
|
||||
"--help": help,
|
||||
|
||||
@@ -19,6 +19,7 @@ const aliases: Record<string, string> = {
|
||||
"frontier list": "frontier:list",
|
||||
"gap list": "gap:list",
|
||||
"export web": "export:web",
|
||||
"compare flows": "compare:flows",
|
||||
"ci report": "ci:report",
|
||||
"ci publish": "ci:publish",
|
||||
"ci failure": "ci:failure",
|
||||
@@ -104,6 +105,7 @@ const flagKinds: Record<string, Record<string, "value" | "boolean">> = {
|
||||
"ci:find-baseline": { sha:"value", target:"value", platform:"value" },
|
||||
"ci:compare-refs": { project:"value", base:"value", head:"value", title:"value", output:"value" },
|
||||
"ci:after-capture": { output:"value", checkout:"value" },
|
||||
"compare:flows": { before: "value", after: "value", output: "value" },
|
||||
compare: {
|
||||
baseline: "value",
|
||||
current: "value",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { stat } from "node:fs/promises";
|
||||
import { readFile, stat, writeFile } from "node:fs/promises";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { DEFAULT_BINARY_PROJECT, runBinaryExploration, runSessionExploration } from "../binary/run.js";
|
||||
import { openViewerInBrowser, startViewerServer, waitForViewerShutdown } from "../map/viewer-server.js";
|
||||
@@ -6,10 +6,14 @@ import { writeExplorationReport } from "../binary/report.js";
|
||||
import { assertHostingExport, deployExplorationReport, zipReportDirectory } from "../report/preview-deploy.js";
|
||||
import { keepReports } from "./report-commands.js";
|
||||
import { warnIgnoredExpiry } from "../config/explore-config.js";
|
||||
import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger } from "./args.js";
|
||||
import { argumentError, flag, flags, has, nonNegativeInteger, oneOf, output, positional, positiveInteger, required } from "./args.js";
|
||||
import { readReplayPlan } from "../binary/replay.js";
|
||||
import { readFlowPlan } from "../binary/flows.js";
|
||||
import type { ExplorationFlow } from "../contracts/standalone-report.js";
|
||||
import { EXPLORATION_REPORT_KIND, type ExplorationFlow, type ExplorationReport } from "../contracts/standalone-report.js";
|
||||
import { compareFlowRuns, type FlowRun } from "../review/flow-comparison.js";
|
||||
import { diffScreenshots } from "../review/visual-evidence.js";
|
||||
import { agentDeviceLauncher, ensureController } from "../shared/controller-cache.js";
|
||||
import { CiError } from "../shared/ci-errors.js";
|
||||
|
||||
|
||||
/** `--alias NAME` plus a fixed token from EXPLORE_PREVIEW_TOKEN: the hosted link stays the same from one run to the next (validated by the deploy). */
|
||||
@@ -156,3 +160,32 @@ export async function exploreSession(): Promise<void> {
|
||||
openViewerInBrowser(viewer.url);
|
||||
await waitForViewerShutdown(viewer);
|
||||
}
|
||||
|
||||
/** A flows run's report: the report directory explore:binary or explore:session wrote, or the output directory holding it. */
|
||||
async function flowRun(path: string, name: "before" | "after"): Promise<FlowRun> {
|
||||
const refuse = (message: string) => { throw new CiError({ code: "FLOW_COMPARE_INPUT_INVALID", stage: "preflight", message,
|
||||
nextAction: `Pass --${name} the output directory of an explore:binary or explore:session run that used --flows (or its report directory).` }); };
|
||||
for (const directory of [resolve(path), resolve(path, "report")]) {
|
||||
let text: string;
|
||||
try { text = await readFile(join(directory, "exploration.json"), "utf8"); } catch { continue; }
|
||||
let report: ExplorationReport;
|
||||
try { report = JSON.parse(text) as ExplorationReport; } catch { return refuse(`${join(directory, "exploration.json")} is not JSON.`); }
|
||||
if (report?.kind !== EXPLORATION_REPORT_KIND) return refuse(`${join(directory, "exploration.json")} is not an exploration report.`);
|
||||
if (!Array.isArray(report.exploration?.flows)) return refuse(`The run in ${directory} did not follow a flows file, so it has no checkpoints to compare.`);
|
||||
return { report, directory };
|
||||
}
|
||||
return refuse(`No exploration.json was found in ${resolve(path)} or its report folder.`);
|
||||
}
|
||||
|
||||
/** `compare:flows --before DIR --after DIR [--output FILE]`: two runs of one flows file, checkpoint by checkpoint. */
|
||||
export async function compareFlows(): Promise<void> {
|
||||
const [before, after] = await Promise.all([flowRun(required("before"), "before"), flowRun(required("after"), "after")]);
|
||||
let command: string | undefined;
|
||||
const comparison = await compareFlowRuns(before, after, async (beforePath, afterPath) => {
|
||||
if (!command) { await ensureController("agent-device", { environment: process.env }); command = agentDeviceLauncher(process.env); }
|
||||
return diffScreenshots({ baselinePath: beforePath, currentPath: afterPath, command });
|
||||
});
|
||||
const path = flag("output");
|
||||
if (path) await writeFile(resolve(path), JSON.stringify(comparison, null, 2));
|
||||
output({ ok: true, ...(path ? { output: resolve(path) } : {}), comparison });
|
||||
}
|
||||
@@ -84,6 +84,8 @@ const CI = `CI (the installed workflows run these; they manage remote EAS Simula
|
||||
capture:baseline --project DIRECTORY --sha COMMIT --output TEMP_DIRECTORY [--config FILE] [--target ID | --platform ios|android] [--controller agent-device|argent] [--simulator eas|local] [--repack eas|local]
|
||||
source:verify --base-project DIRECTORY --head-project DIRECTORY
|
||||
--base-sha COMMIT --head-sha COMMIT --output DIRECTORY [--config FILE] [--scope all|changes] [--target ID | --platform ios|android]
|
||||
compare:flows --before DIRECTORY --after DIRECTORY [--output FILE]
|
||||
(two runs of one --flows file, checkpoint by checkpoint: unchanged, changed with where, or why not compared)
|
||||
compare --baseline DIRECTORY --current DIRECTORY --output FILE
|
||||
[--review-output FILE] [--catalog-output FILE] [--catalog-name NAME]
|
||||
[--base-surface-coverage FILE] [--head-surface-coverage FILE]
|
||||
|
||||
@@ -189,7 +189,7 @@ export const DUO_PANELS = {
|
||||
inner: { width: 951, height: 669, statusBar: { x: 883, y: 34, width: 40, height: 64 } },
|
||||
} as const;
|
||||
const DUO_STATUS_BAR_PADDING_POINTS = 4;
|
||||
const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? "");
|
||||
export const isFoldable = (platform: "ios" | "android" | undefined, device: string | undefined) => platform !== "android" && /\bduo\b/i.test(device ?? "");
|
||||
|
||||
/**
|
||||
* The areas the verdict leaves out for a capture: the platform's status-bar band and, on iOS, its home-indicator
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { ExplorationReport } from "./standalone-report.js";
|
||||
|
||||
export const FLOW_COMPARISON_KIND = "explore.flow-comparison" as const;
|
||||
|
||||
/** One run's side of a checkpoint: its screenshot (a path inside that run's report directory), or why it has none. */
|
||||
export type CheckpointSide = { observationId: string; screenshot: string } | { missing: string };
|
||||
|
||||
/**
|
||||
* A checkpoint of one flow in two runs, paired by flow name and checkpoint name. `before` or `after` is absent when
|
||||
* that run's flows file has no such flow or checkpoint.
|
||||
*/
|
||||
export type CheckpointPair = { flow: string; checkpoint: string; before?: CheckpointSide; after?: CheckpointSide };
|
||||
|
||||
/** What a run was captured on: two runs that differ here can still be compared, but a difference may come from it. */
|
||||
export type RunConditions = { app: string; platform: string; device: string; osVersion?: string; locale?: string; orientation?: string; colorScheme?: string };
|
||||
export type RunConditionDifference = { field: keyof RunConditions; before: string; after: string };
|
||||
|
||||
export function runConditions(report: ExplorationReport): RunConditions {
|
||||
const environment = report.map.run.environment ?? {};
|
||||
return { app: report.exploration.app.bundleId, platform: report.exploration.app.platform, device: report.exploration.device.model,
|
||||
...(report.exploration.device.osVersion ? { osVersion: report.exploration.device.osVersion } : {}),
|
||||
...(environment.locale ? { locale: environment.locale } : {}), ...(environment.orientation ? { orientation: environment.orientation } : {}),
|
||||
...(environment.colorScheme ? { colorScheme: environment.colorScheme } : {}) };
|
||||
}
|
||||
|
||||
export function conditionDifferences(before: RunConditions, after: RunConditions): RunConditionDifference[] {
|
||||
const fields = ["app", "platform", "device", "osVersion", "locale", "orientation", "colorScheme"] as const;
|
||||
return fields.flatMap(field => (before[field] ?? "unknown") === (after[field] ?? "unknown") ? []
|
||||
: [{ field, before: before[field] ?? "unknown", after: after[field] ?? "unknown" }]);
|
||||
}
|
||||
|
||||
const keyOf = (name: string) => name.trim().toLowerCase();
|
||||
|
||||
function side(report: ExplorationReport, checkpoint: NonNullable<ExplorationReport["exploration"]["flows"]>[number]["checkpoints"][number]): CheckpointSide {
|
||||
if (checkpoint.observationId === undefined) return { missing: checkpoint.missing };
|
||||
const observation = report.map.observations.find(item => item.id === checkpoint.observationId);
|
||||
const artifact = observation?.screenshotArtifactId ? report.map.artifacts.find(item => item.id === observation.screenshotArtifactId) : undefined;
|
||||
const asset = artifact ? report.assets.find(item => item.sha256 === artifact.sha256) : undefined;
|
||||
return asset ? { observationId: checkpoint.observationId, screenshot: asset.path } : { missing: "The report has no screenshot for this checkpoint." };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pairs the checkpoints of two runs of a flows file: the after run's flows and checkpoints in its order, then those only
|
||||
* the before run has. Names match as the flows file compares them, ignoring case and surrounding space.
|
||||
*/
|
||||
export function pairCheckpoints(before: ExplorationReport, after: ExplorationReport): CheckpointPair[] {
|
||||
const beforeFlows = before.exploration.flows ?? [], afterFlows = after.exploration.flows ?? [];
|
||||
const flowNames = [...afterFlows, ...beforeFlows.filter(flow => !afterFlows.some(other => keyOf(other.name) === keyOf(flow.name)))].map(flow => flow.name);
|
||||
return flowNames.flatMap(name => {
|
||||
const was = beforeFlows.find(flow => keyOf(flow.name) === keyOf(name)), now = afterFlows.find(flow => keyOf(flow.name) === keyOf(name));
|
||||
const wasPoints = was?.checkpoints ?? [], nowPoints = now?.checkpoints ?? [];
|
||||
const names = [...nowPoints, ...wasPoints.filter(point => !nowPoints.some(other => keyOf(other.name) === keyOf(point.name)))].map(point => point.name);
|
||||
return names.map(checkpoint => {
|
||||
const old = wasPoints.find(point => keyOf(point.name) === keyOf(checkpoint)), current = nowPoints.find(point => keyOf(point.name) === keyOf(checkpoint));
|
||||
return { flow: name, checkpoint, ...(old ? { before: side(before, old) } : {}), ...(current ? { after: side(after, current) } : {}) };
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import { open } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { isFoldable, systemRegions } from "../contracts/capture.js";
|
||||
import { FLOW_COMPARISON_KIND, conditionDifferences, pairCheckpoints, runConditions, type CheckpointPair, type RunConditionDifference, type RunConditions } from "../contracts/flow-comparison.js";
|
||||
import type { ExplorationReport } from "../contracts/standalone-report.js";
|
||||
import { ignoreSystemRegions, systemBandFrameLoader } from "./evidence-assessment.js";
|
||||
import type { ScreenshotDiffMeasurement } from "./visual-evidence.js";
|
||||
|
||||
/**
|
||||
* `changed` and `unchanged` compare the two screenshots; `different-size` could not (another device or orientation);
|
||||
* `missing` has no screenshot on one side; `added` and `removed` are in only one run's flows file.
|
||||
*/
|
||||
export type CheckpointVerdict = "unchanged" | "changed" | "different-size" | "missing" | "added" | "removed";
|
||||
export type ComparedCheckpoint = CheckpointPair & {
|
||||
verdict: CheckpointVerdict;
|
||||
/** Share of the screenshot that differs once the status bar and home indicator are set aside. */
|
||||
mismatchPercentage?: number;
|
||||
/** Where the screenshots differ, in percent of the image (0–100). */
|
||||
regions?: Array<{ x: number; y: number; width: number; height: number }>;
|
||||
};
|
||||
|
||||
export type FlowComparison = {
|
||||
kind: typeof FLOW_COMPARISON_KIND;
|
||||
before: RunConditions & { report: string };
|
||||
after: RunConditions & { report: string };
|
||||
/** How the runs were captured differently; a checkpoint that changed may owe its change to these. */
|
||||
conditions: RunConditionDifference[];
|
||||
notes: string[];
|
||||
counts: Record<CheckpointVerdict, number>;
|
||||
checkpoints: ComparedCheckpoint[];
|
||||
};
|
||||
|
||||
export type FlowRun = { report: ExplorationReport; directory: string };
|
||||
export type MeasureScreenshots = (beforePath: string, afterPath: string) => Promise<ScreenshotDiffMeasurement>;
|
||||
|
||||
/** A PNG's pixel size, from its header. */
|
||||
async function pngSize(path: string): Promise<{ width: number; height: number }> {
|
||||
const file = await open(path, "r");
|
||||
try {
|
||||
const header = Buffer.alloc(24);
|
||||
await file.read(header, 0, 24, 0);
|
||||
return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) };
|
||||
} finally { await file.close(); }
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares two runs of the same flows file checkpoint by checkpoint, with the review's screenshot diff and the
|
||||
* status-bar and home-indicator regions it sets aside. `measure` is only called for pairs with a screenshot on both
|
||||
* sides of the same size.
|
||||
*/
|
||||
export async function compareFlowRuns(before: FlowRun, after: FlowRun, measure: MeasureScreenshots, threshold = 0.1): Promise<FlowComparison> {
|
||||
const was = runConditions(before.report), now = runConditions(after.report);
|
||||
const notes: string[] = [];
|
||||
// A Duo's status bar sits on whichever panel was in use, and a run does not record the pose.
|
||||
const foldable = isFoldable(before.report.exploration.app.platform, was.device);
|
||||
if (foldable) notes.push("The status bar is not set aside: the run does not record which panel of the foldable was in use.");
|
||||
const system = foldable ? [] : systemRegions({ platform: before.report.exploration.app.platform, device: was.device });
|
||||
const checkpoints: ComparedCheckpoint[] = [];
|
||||
for (const pair of pairCheckpoints(before.report, after.report)) {
|
||||
if (!pair.before) { checkpoints.push({ ...pair, verdict: "added" }); continue; }
|
||||
if (!pair.after) { checkpoints.push({ ...pair, verdict: "removed" }); continue; }
|
||||
if (!("screenshot" in pair.before) || !("screenshot" in pair.after)) { checkpoints.push({ ...pair, verdict: "missing" }); continue; }
|
||||
const beforePath = resolve(before.directory, pair.before.screenshot), afterPath = resolve(after.directory, pair.after.screenshot);
|
||||
const [a, b] = await Promise.all([pngSize(beforePath), pngSize(afterPath)]);
|
||||
if (a.width !== b.width || a.height !== b.height) { checkpoints.push({ ...pair, verdict: "different-size" }); continue; }
|
||||
const raw = await measure(beforePath, afterPath);
|
||||
const { measurement } = ignoreSystemRegions(raw, system, systemBandFrameLoader(raw, threshold));
|
||||
const different = measurement.regions.reduce((sum, region) => sum + region.differentPixels, 0);
|
||||
const mismatchPercentage = raw.totalPixels ? Math.round((different / raw.totalPixels) * 10000) / 100 : 0;
|
||||
// As in a review, a mismatch that rounds to 0.00% is rendering noise.
|
||||
const changed = measurement.regions.length > 0 && mismatchPercentage > 0;
|
||||
checkpoints.push({ ...pair, verdict: changed ? "changed" : "unchanged", mismatchPercentage,
|
||||
...(changed ? { regions: measurement.regions.map(region => region.normalizedRect) } : {}) });
|
||||
}
|
||||
const counts = { unchanged: 0, changed: 0, "different-size": 0, missing: 0, added: 0, removed: 0 };
|
||||
for (const item of checkpoints) counts[item.verdict] += 1;
|
||||
return { kind: FLOW_COMPARISON_KIND, before: { ...was, report: before.directory }, after: { ...now, report: after.directory },
|
||||
conditions: conditionDifferences(was, now), notes, counts, checkpoints };
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { tsImport } from "tsx/esm/api";
|
||||
|
||||
const { pairCheckpoints, conditionDifferences, runConditions } = await tsImport("../../src/contracts/flow-comparison.ts", import.meta.url);
|
||||
const { compareFlowRuns } = await tsImport("../../src/review/flow-comparison.ts", import.meta.url);
|
||||
const { PNG } = createRequire(import.meta.url)("pngjs");
|
||||
|
||||
const png = (width, height) => PNG.sync.write(new PNG({ width, height }));
|
||||
|
||||
/**
|
||||
* A minimal exploration report of a flows run, written to a directory. `flows` maps each flow name to its checkpoints,
|
||||
* each `[name, size]` for a picture of that size or `[name, { missing }]` for one without.
|
||||
*/
|
||||
async function run(flows, options = {}) {
|
||||
const directory = await mkdtemp(join(tmpdir(), "explore-flow-compare-"));
|
||||
await mkdir(join(directory, "assets"));
|
||||
const report = { kind: "explore.exploration-report", assets: [],
|
||||
map: { run: { environment: { deviceModel: options.device ?? "iPhone 17 Pro", locale: options.locale ?? "en_US", orientation: "portrait", colorScheme: options.colorScheme ?? "light" } }, observations: [], artifacts: [] },
|
||||
exploration: { app: { name: "Example", bundleId: options.bundleId ?? "dev.example.app", platform: options.platform ?? "ios" }, device: { model: options.device ?? "iPhone 17 Pro", osVersion: options.osVersion ?? "26.0" }, flows: [] } };
|
||||
let index = 0;
|
||||
for (const [name, checkpoints] of Object.entries(flows)) {
|
||||
const flow = { name, status: "completed", checkpoints: [] };
|
||||
for (const [checkpoint, shot] of checkpoints) {
|
||||
if (shot.missing) { flow.checkpoints.push({ name: checkpoint, missing: shot.missing }); continue; }
|
||||
const sha = String(++index).padStart(64, "0"), id = `obs_${index}`;
|
||||
await writeFile(join(directory, "assets", `${sha}.png`), png(...shot));
|
||||
report.assets.push({ sha256: sha, path: `assets/${sha}.png` });
|
||||
report.map.artifacts.push({ id: `art_${index}`, kind: "screenshot", sha256: sha });
|
||||
report.map.observations.push({ id, screenshotArtifactId: `art_${index}` });
|
||||
flow.checkpoints.push({ name: checkpoint, observationId: id });
|
||||
}
|
||||
report.exploration.flows.push(flow);
|
||||
}
|
||||
return { report, directory };
|
||||
}
|
||||
|
||||
/** A diff that finds the given regions (in percent), each 1% of the pixels. */
|
||||
const measuring = (regionsFor = () => []) => async (baselinePath, currentPath) => {
|
||||
const regions = regionsFor(baselinePath, currentPath).map((normalizedRect, index) => ({ index, normalizedRect, rect: normalizedRect, differentPixels: 100, shareOfDiffPercentage: 0, densityPercentage: 0, shape: "", size: "" }));
|
||||
return { baselinePath, currentPath, totalPixels: 10_000, differentPixels: regions.length * 100, mismatchPercentage: regions.length, match: regions.length === 0, regions };
|
||||
};
|
||||
|
||||
test("checkpoints pair by flow and checkpoint name, ignoring case, in the after run's order", async () => {
|
||||
const before = await run({ "Sign in": [["Welcome", [4, 8]], ["Home", [4, 8]]], Settings: [["Settings", [4, 8]]] });
|
||||
const after = await run({ "sign in": [["home", [4, 8]], ["welcome", { missing: "The screenshot was blocked." }]], Profile: [["Profile", [4, 8]]] });
|
||||
const pairs = pairCheckpoints(before.report, after.report);
|
||||
assert.deepEqual(pairs.map(pair => [pair.flow, pair.checkpoint, !!pair.before, !!pair.after]), [
|
||||
["sign in", "home", true, true], ["sign in", "welcome", true, true], ["Profile", "Profile", false, true], ["Settings", "Settings", true, false]]);
|
||||
assert.deepEqual(pairs[1].after, { missing: "The screenshot was blocked." });
|
||||
assert.match(pairs[0].before.screenshot, /^assets\/0+2\.png$/);
|
||||
});
|
||||
|
||||
test("a run's conditions that differ are listed, not refused", async () => {
|
||||
const before = await run({ A: [["Home", [4, 8]]] }), after = await run({ A: [["Home", [4, 8]]] }, { osVersion: "27.0", colorScheme: "dark" });
|
||||
assert.deepEqual(conditionDifferences(runConditions(before.report), runConditions(after.report)),
|
||||
[{ field: "osVersion", before: "26.0", after: "27.0" }, { field: "colorScheme", before: "light", after: "dark" }]);
|
||||
});
|
||||
|
||||
test("each pair gets a verdict: unchanged, changed with where, missing, different-size, added or removed", async () => {
|
||||
const before = await run({ Flow: [["Same", [4, 8]], ["Moved", [4, 8]], ["Gone", [4, 8]], ["Blocked", [4, 8]], ["Rotated", [4, 8]]] });
|
||||
const after = await run({ Flow: [["Same", [4, 8]], ["Moved", [4, 8]], ["New", [4, 8]], ["Blocked", { missing: "A secret was on screen." }], ["Rotated", [8, 4]]] });
|
||||
const measured = [];
|
||||
const result = await compareFlowRuns(before, after, measuring((baseline, current) => {
|
||||
measured.push([baseline, current]);
|
||||
return baseline.endsWith(before.report.assets[1].path) ? [{ x: 10, y: 40, width: 20, height: 10 }] : [];
|
||||
}));
|
||||
assert.deepEqual(result.checkpoints.map(item => [item.checkpoint, item.verdict]),
|
||||
[["Same", "unchanged"], ["Moved", "changed"], ["New", "added"], ["Blocked", "missing"], ["Rotated", "different-size"], ["Gone", "removed"]]);
|
||||
assert.equal(measured.length, 2, "only pairs with a picture of the same size on both sides are diffed");
|
||||
assert.deepEqual(result.checkpoints[1].regions, [{ x: 10, y: 40, width: 20, height: 10 }]);
|
||||
assert.equal(result.checkpoints[1].mismatchPercentage, 1);
|
||||
assert.deepEqual(result.counts, { unchanged: 1, changed: 1, "different-size": 1, missing: 1, added: 1, removed: 1 });
|
||||
assert.deepEqual(result.conditions, []);
|
||||
});
|
||||
|
||||
test("a difference only in the status bar is set aside, as in a review", async () => {
|
||||
const before = await run({ Flow: [["Home", [4, 8]]] }), after = await run({ Flow: [["Home", [4, 8]]] });
|
||||
const result = await compareFlowRuns(before, after, measuring(() => [{ x: 80, y: 1, width: 10, height: 2 }]));
|
||||
assert.equal(result.checkpoints[0].verdict, "unchanged");
|
||||
});
|
||||
|
||||
test("on a foldable, the status bar is not set aside and the result says why", async () => {
|
||||
const before = await run({ Flow: [["Home", [4, 8]]] }, { device: "iPhone Duo" }), after = await run({ Flow: [["Home", [4, 8]]] }, { device: "iPhone Duo" });
|
||||
// Where the folded panel's status bar is: guessing that panel would hide a change the open panel shows there.
|
||||
const result = await compareFlowRuns(before, after, measuring(() => [{ x: 86, y: 13, width: 4, height: 4 }]));
|
||||
assert.equal(result.checkpoints[0].verdict, "changed");
|
||||
assert.match(result.notes[0], /foldable/);
|
||||
});
|
||||
|
||||
test("compare:flows reads two output directories, needs no diff when no pair has two pictures, and refuses a run without flows", async () => {
|
||||
const { spawnSync } = await import("node:child_process");
|
||||
const root = join(import.meta.dirname, "../..");
|
||||
const cli = join(root, "dist", "cli.js");
|
||||
const before = await run({ Flow: [["Home", { missing: "Not reached." }]] }), after = await run({ Flow: [["Home", [4, 8]]] });
|
||||
// explore:binary leaves the report in <output>/report.
|
||||
const output = await mkdtemp(join(tmpdir(), "explore-flow-compare-out-"));
|
||||
await mkdir(join(output, "report", "assets"), { recursive: true });
|
||||
await writeFile(join(output, "report", "exploration.json"), JSON.stringify(after.report));
|
||||
await writeFile(join(before.directory, "exploration.json"), JSON.stringify(before.report));
|
||||
const compared = spawnSync(process.execPath, [cli, "compare:flows", "--before", before.directory, "--after", output], { cwd: root, encoding: "utf8" });
|
||||
assert.equal(compared.status, 0, compared.stderr);
|
||||
const result = JSON.parse(compared.stdout);
|
||||
assert.deepEqual(result.comparison.checkpoints.map(item => [item.checkpoint, item.verdict]), [["Home", "missing"]]);
|
||||
const plain = await run({});
|
||||
delete plain.report.exploration.flows;
|
||||
await writeFile(join(plain.directory, "exploration.json"), JSON.stringify(plain.report));
|
||||
const refused = spawnSync(process.execPath, [cli, "compare:flows", "--before", plain.directory, "--after", output], { cwd: root, encoding: "utf8" });
|
||||
assert.notEqual(refused.status, 0);
|
||||
assert.match(JSON.parse(refused.stderr).error, /did not follow a flows file/);
|
||||
});
|
||||
Reference in new issue
Block a user