Compare commits

..
31 changed files with 904 additions and 116 deletions

No files matched your search

+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@expo/explore",
"version": "0.20.5",
"version": "0.20.6",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@expo/explore",
"version": "0.20.5",
"version": "0.20.6",
"license": "MIT",
"dependencies": {
"@expo/repack-app": "0.10.1",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@expo/explore",
"version": "0.20.5",
"version": "0.20.6",
"description": "Source-driven mobile app maps, deterministic CI capture, and visual reviews",
"license": "MIT",
"type": "module",
+9 -1
View File
@@ -1,4 +1,4 @@
import type { CaptureOutcome } from "../contracts/capture.js";
import type { CaptureBundle, CaptureOutcome } from "../contracts/capture.js";
export type ColorScheme = "light" | "dark";
/** `auto` follows the app's `userInterfaceStyle`; a list captures exactly those schemes, first one primary. */
@@ -118,3 +118,11 @@ export function appearancePairs(routeId: string, headSchemes: readonly ColorSche
baseId: baseSchemes.includes(scheme) ? schemeScreenId(routeId, scheme === baseSchemes[0] ? undefined : scheme) : null,
}));
}
/** A `<id>@<scheme>` outcome recording that its whole appearance pass failed (`sessionFailures`), whatever the primary did. */
export function failedPassOutcome(bundle: Pick<CaptureBundle, "outcomes" | "sessionFailures">, id: string): boolean {
const { scheme } = routeIdOfScreen(id);
const own = bundle.outcomes[id];
return scheme !== undefined && own?.status === "failed" && (bundle.sessionFailures ?? []).some(failure => failure.scheme === scheme
&& failure.error.code === own.error?.code && failure.error.message === own.error?.message);
}
+3 -1
View File
@@ -92,7 +92,9 @@ export function startArgentBridgeRunner(options: {
stop(`the Argent bridge thread is unavailable (${text}); running the bridge per command`);
return options.fallback(request);
}
const result = JSON.parse(text) as { exitCode: number; stdout: string };
const result = JSON.parse(text) as { exitCode: number; stdout: string; notes?: string[] };
// The thread cannot reach the capture's log while this one waits for it, so its lines come with the answer.
for (const note of result.notes ?? []) options.progress?.(note);
return commandOutcome(request, { status: result.exitCode, stdout: result.stdout, stderr: "" });
};
return { runner, get active() { return worker !== undefined; }, close: () => { if (worker) { void worker.terminate(); worker = undefined; } } };
+17 -2
View File
@@ -61,6 +61,20 @@ async function materializedFile(data: unknown): Promise<string | undefined> {
return undefined;
}
/**
* An error's message with its cause's, as Argent's client words a failed fetch: "fetch failed: connect ECONNREFUSED …".
* A host that resolves to several addresses (localhost: ::1 and 127.0.0.1) fails with an AggregateError whose message is
* empty, so the causes' codes are named too.
*/
export function failureText(error: unknown): string {
if (!(error instanceof Error)) return String(error);
const cause = error.cause;
if (!(cause instanceof Error)) return error.message;
const codes = [cause, ...(cause instanceof AggregateError ? cause.errors : [])]
.map(item => (item as { code?: unknown } | undefined)?.code).filter((code): code is string => typeof code === "string" && !cause.message.includes(code));
return [`${error.message}:`, cause.message, ...(codes.length ? [`(${[...new Set(codes)].join(", ")})`] : [])].filter(Boolean).join(" ");
}
async function loadClient(environment: NodeJS.ProcessEnv, clientModule?: string): Promise<ArgentClient> {
const module = clientModule ?? join(dirname(await ensureController("argent", { environment })), "client.js");
const loaded = await import(pathToFileURL(module).href) as { createArgentClient?: () => ArgentClient };
@@ -74,8 +88,9 @@ export function argentClientCall(client: ArgentClient, signal: () => AbortSignal
const active = signal();
if (active.aborted) throw new ArgentCallError(`argent run ${tool} was not sent: the command was given up.`, false);
let answer: { data: unknown };
try { answer = await client.callTool(tool, args, { signal: AbortSignal.any([active, AbortSignal.timeout(ARGENT_CALL_TIMEOUT_MS)]) }); }
catch (error) { throw new ArgentCallError(`argent run ${tool} failed: ${(error instanceof Error ? error.message : String(error)).slice(0, 800)}`, true); }
const timeoutMs = Math.max(1, Math.ceil(Math.min(ARGENT_CALL_TIMEOUT_MS, options?.timeoutMs ?? ARGENT_CALL_TIMEOUT_MS)));
try { answer = await client.callTool(tool, args, { signal: AbortSignal.any([active, AbortSignal.timeout(timeoutMs)]) }); }
catch (error) { throw new ArgentCallError(`argent run ${tool} failed: ${failureText(error).slice(0, 800)}`, true); }
if (!options?.out) return answer.data;
const file = await materializedFile(answer.data);
if (!file) {
+20 -3
View File
@@ -57,9 +57,23 @@ function summarize(tool: string, value: unknown): unknown {
screen: tree.screen, nodes, roots: (tree.roots ?? []).map(root => [root.type ?? null, root.bundleId ?? null, (root.children ?? []).length]) };
}
/**
* What a failed `argent run` printed, for its error: the last few lines, and ahead of them the line saying the
* tool-server is not serving its tools yet, which the bridge waits out (argent-driver's TOOLS_NOT_SERVED) and which a
* stack trace can push out of the last lines.
*/
export function argentRunFailureDetail(stderr: string, stdout: string): string {
const lines = `${stderr}\n${stdout}`.trim().split("\n").filter(Boolean);
const notServed = lines.find(line => /GET \/tools failed: \d{3}|ECONNREFUSED/.test(line));
return [...new Set([...(notServed ? [notServed] : []), ...lines.slice(-4)])].join(" | ").slice(0, 800);
}
/** A command's answer, with the lines it has for the capture's log (`notes`), which its host prints. */
export type ArgentBridgeResult = BridgeResult & { notes?: string[] };
export type ArgentBridge = {
/** Runs one agent-device-shaped command against the session. */
run(argv: readonly string[]): Promise<BridgeResult>;
run(argv: readonly string[]): Promise<ArgentBridgeResult>;
};
/**
@@ -77,8 +91,10 @@ export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentC
await appendFile(logPath, `${JSON.stringify({ at: new Date().toISOString(), host, command: argv[0], ...entry })}\n`, { mode: 0o600 }).catch(() => undefined);
};
const state = await readState(statePath);
const notes: string[] = [];
const result = await runBridgeCommand(argv, {
state, platform, version: ARGENT_VERSION,
progress: message => { notes.push(message); void log({ note: message }); },
...(environment.EXPLORE_ARGENT_BUNDLE_ID ? { bundleId: environment.EXPLORE_ARGENT_BUNDLE_ID } : {}),
...(environment.EXPLORE_ARGENT_DEVICE ? { device: environment.EXPLORE_ARGENT_DEVICE } : {}),
// An opened foldable's inner panel shows its UI in landscape (Argent's fold: "Unfolded, the UI is landscape").
@@ -101,7 +117,8 @@ export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentC
// The same URL Argent's own client checks: ARGENT_TOOLS_URL may carry a path, which /tools is appended to.
const response = await fetch(`${(toolsUrl ?? "").replace(/\/+$/, "")}/tools`, { headers: token ? { authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(5_000) });
await response.body?.cancel();
ok = response.status < 500;
// A tool-server still starting can answer 404 to /tools (euxy main baseline run 39506374824): it is not serving yet.
ok = response.status < 500 && response.status !== 404;
} catch { ok = false; }
if (!ok) await log({ tool: "reachable", ok });
return ok;
@@ -131,7 +148,7 @@ export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentC
});
await writeState(statePath, state);
await log({ result: result.exitCode === 0 ? "ok" : result.stdout.slice(0, 300) });
return result;
return notes.length ? { ...result, notes } : result;
},
};
}
+5 -5
View File
@@ -8,7 +8,7 @@
import { spawnSync } from "node:child_process";
import { stat } from "node:fs/promises";
import { ArgentCallError } from "./argent-driver.js";
import { argentBridgeSettings, createArgentBridge } from "./argent-bridge.js";
import { argentBridgeSettings, argentRunFailureDetail, createArgentBridge } from "./argent-bridge.js";
import { ensureController } from "../shared/controller-cache.js";
const ARGENT_CALL_TIMEOUT_MS = 180_000;
@@ -34,13 +34,12 @@ async function main(): Promise<number> {
return 1;
}
const callArgent = async (tool: string, args: Record<string, unknown>, options?: { out?: string }): Promise<unknown> => {
const callArgent = async (tool: string, args: Record<string, unknown>, options?: { out?: string; timeoutMs?: number }): Promise<unknown> => {
const run = spawnSync(process.execPath, [cli, "run", tool, "--args", JSON.stringify(args), ...(options?.out ? ["--out", options.out] : [])],
{ env: environment, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: ARGENT_CALL_TIMEOUT_MS });
{ env: environment, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: Math.min(ARGENT_CALL_TIMEOUT_MS, options?.timeoutMs ?? ARGENT_CALL_TIMEOUT_MS) });
if (run.error) throw new ArgentCallError(`argent run ${tool}: ${run.error.message}`, true);
if (run.status !== 0) {
const detail = `${run.stderr ?? ""}\n${run.stdout ?? ""}`.trim().split("\n").filter(Boolean).slice(-4).join(" | ").slice(0, 800);
throw new ArgentCallError(`argent run ${tool} failed: ${detail}`, true);
throw new ArgentCallError(`argent run ${tool} failed: ${argentRunFailureDetail(run.stderr ?? "", run.stdout ?? "")}`, true);
}
// With --out the client saves the tool's file and prints where ("Wrote: <path>"), not the tool's JSON.
if (options?.out) {
@@ -53,6 +52,7 @@ async function main(): Promise<number> {
};
const result = await createArgentBridge(settings, callArgent, "process").run(argv);
for (const note of result.notes ?? []) process.stderr.write(`[explore] ${note}\n`);
process.stdout.write(result.stdout);
return result.exitCode;
}
+55 -7
View File
@@ -15,7 +15,8 @@ export class ArgentCallError extends Error {
constructor(message: string, readonly sent: boolean) { super(message); }
}
export type ArgentCall = (tool: string, args: Record<string, unknown>, options?: { out?: string }) => Promise<unknown>;
/** `timeoutMs` shortens the call's own timeout, so a wait for a tool-server still starting stays within its window. */
export type ArgentCall = (tool: string, args: Record<string, unknown>, options?: { out?: string; timeoutMs?: number }) => Promise<unknown>;
type Rect = { x: number; y: number; width: number; height: number };
@@ -31,6 +32,8 @@ export type ArgentBridgeState = {
/** iOS asks "Open in “<app>”?" for the first custom-scheme link of a session; later links open directly. */
deepLinkPromptAccepted?: boolean;
recording?: string;
/** The tool-server has answered a call this session, so a GET /tools it fails is no longer taken for it still starting. */
toolsServed?: boolean;
};
export type ArgentBridgeContext = {
@@ -43,6 +46,8 @@ export type ArgentBridgeContext = {
device?: string;
version: string;
sleep: (milliseconds: number) => Promise<void>;
/** A monotonic clock in milliseconds (performance.now when absent), which bounds the wait for a tool-server to start. */
now?: () => number;
/** Moves a file the Argent client downloaded (a screen recording) to where Explore asked for it. */
moveFile: (from: string, to: string) => Promise<void>;
/** Whether the tool-server answers, checked before a UI action is sent (see `remoteUnavailable`). */
@@ -57,6 +62,8 @@ export type ArgentBridgeContext = {
landscape?: boolean;
/** Turns a portrait screen recording a quarter clockwise, to the landscape UI it shows. */
orientVideo?: (path: string) => Promise<void>;
/** A line for the capture's log: the bridge waiting on a tool-server that is still starting. */
progress?: (message: string) => void;
};
export type BridgeResult = { exitCode: number; stdout: string };
@@ -259,6 +266,20 @@ const SERVICE_DEPENDENCY_FAILED = "Service dependency failed:";
* comes after a pause, so a device still coming back from the restart has the time it needs.
*/
const DEPENDENCY_RETRY_DELAYS_MS = [3_000, 6_000] as const;
/**
* A tool-server that is not serving its tools yet. Argent's client asks GET /tools before every tool call and stops with
* "GET /tools failed: <status>" when that is not answered, so the tool itself was never sent; a refused connection
* reached nothing at all. Either is safe to send again, a UI action too. In brentvatne/euxy's main baseline run
* 39506374824 a replacement EAS session's tool-server answered 404 to /tools for the first list-devices of its install,
* and the whole target failed. An unknown tool's 404 comes from POST /tools/<name> instead, and still fails at once.
*/
const TOOLS_NOT_SERVED = /GET \/tools failed: (?:404|503)\b|ECONNREFUSED/;
/**
* How long a tool-server that has not answered yet gets to start serving its tools, requests included: a deadline, not
* a number of tries, since each try can itself take until the call's timeout. The pauses between tries grow to 10 s.
*/
const STARTUP_WINDOW_MS = 75_000;
const STARTUP_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 10_000] as const;
const SPRINGBOARD = "com.apple.springboard";
const ANDROID_SYSTEM_UI = "com.android.systemui";
const OPEN_IN_APP_PROMPT = /^Open in “.+”\?$/;
@@ -345,14 +366,40 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
/**
* Reads are resent after a transport failure (they change nothing); a UI action is sent once, after a health check.
* Any tool, a UI action too, is resent when Argent could not start a service it depends on, which it raises before
* the tool runs (see SERVICE_DEPENDENCY_FAILED).
* the tool runs (see SERVICE_DEPENDENCY_FAILED), and while a tool-server that has not answered yet is still starting
* (see TOOLS_NOT_SERVED).
*/
const call = async (tool: string, args: Record<string, unknown> = {}, options?: { out?: string }): Promise<Record<string, unknown>> => {
// Tools that name devices their own way (or none) take no udid.
const withDevice = tool === "list-devices" || tool === "stop-all-simulator-servers" ? args : { udid: state.udid, ...args };
for (let readRetry = 0, dependencyRetry = 0; ;) {
try { return record(await context.call(tool, withDevice, options)); }
catch (error) {
const clock = context.now ?? (() => performance.now());
// The window opens with the first call to a tool-server that has not answered yet, and closes 75 s later. A device
// already named came from the tool-server's own answer.
const served = () => state.toolsServed === true || state.udid !== undefined;
const startupDeadline = served() ? undefined : clock() + STARTUP_WINDOW_MS;
for (let readRetry = 0, dependencyRetry = 0, startupRetry = 0; ;) {
const starting = !served() && startupDeadline !== undefined;
try {
// A call to a tool-server still starting gets only the time left in the window.
const answer = record(await context.call(tool, withDevice, starting ? { ...options, timeoutMs: Math.max(1, Math.ceil(startupDeadline - clock())) } : options));
state.toolsServed = true;
return answer;
} catch (error) {
// Until the session's tool-server has answered once, a GET /tools it does not serve is it still starting.
if (starting && error instanceof ArgentCallError) {
const notServed = TOOLS_NOT_SERVED.test(error.message);
const left = startupDeadline - clock();
// A try the window's end cut short is the tool-server not starting in time, whatever the request reported.
if (left <= 0) {
throw new BridgeFailure("TOOL_SERVER_NOT_READY", `Argent's tool-server did not start serving its tools within ${STARTUP_WINDOW_MS / 1_000} s: ${error.message}`);
}
if (notServed) {
const delay = Math.min(STARTUP_RETRY_DELAYS_MS[Math.min(startupRetry++, STARTUP_RETRY_DELAYS_MS.length - 1)]!, left);
context.progress?.(`Argent's tool-server is not serving its tools yet (${error.message}); sending ${tool} again in ${Math.ceil(delay / 1_000)} s (${Math.ceil(left / 1_000)} s left of ${STARTUP_WINDOW_MS / 1_000} s)`);
await context.sleep(delay);
continue;
}
}
if (error instanceof ArgentCallError && error.message.includes(`[Tool:${tool}] ${SERVICE_DEPENDENCY_FAILED}`)
&& dependencyRetry < DEPENDENCY_RETRY_DELAYS_MS.length) {
await context.sleep(DEPENDENCY_RETRY_DELAYS_MS[dependencyRetry++]!);
@@ -417,8 +464,9 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
return target;
};
/**
* A UI action is only sent when the tool-server answers first, so a failure after this point is never resent. A
* session's tool-server can still be starting when EAS reports the session ready, so it is given about 25 s to answer.
* A UI action is only sent when the tool-server answers first, so a failure after this point is never resent unless
* the tool provably never ran (see call). A session's tool-server can still be starting when EAS reports the session
* ready, so it is given about 25 s to answer.
*/
const beforeAction = async () => {
for (const delay of [1_000, 2_000, 3_000, 4_000, 5_000, 5_000, 5_000]) {
+12 -2
View File
@@ -1,5 +1,5 @@
import type { CaptureBundle, CaptureOutcome } from "../contracts/capture.js";
import { bundleColorSchemes, routeIdOfScreen, schemeScreenId } from "./appearance.js";
import { bundleColorSchemes, failedPassOutcome, routeIdOfScreen, schemeScreenId } from "./appearance.js";
import { CiError } from "../shared/ci-errors.js";
type CaptureSide = CaptureBundle["side"];
@@ -59,8 +59,10 @@ export function captureHealthIssues(bundle: CaptureBundle, expectedIds: readonly
for (const id of [...extras].sort()) {
// A secondary frame is expected evidence; its own outcome is the failure of that appearance pass, or its verified
// frame when the primary pass has none (the route is then published from this appearance alone).
// A pass that failed as a whole (sessionFailures) records its failure under every route's `<target>@<scheme>`.
const primaryCaptured = bundle.outcomes[routeIdOfScreen(id).routeId]?.status === "captured";
if (!expected.has(id) && !(expectedSecondary.has(id) && !selected.has(id) && hasVerifiedOutcome(bundle.outcomes[id]) !== primaryCaptured)) {
if (!expected.has(id) && !(expectedSecondary.has(id) && !selected.has(id)
&& (hasVerifiedOutcome(bundle.outcomes[id]) !== primaryCaptured || failedPassOutcome(bundle, id)))) {
add(id, "CAPTURE_TARGET_UNEXPECTED", "Capture evidence contains a target absent from the source plan.", inventoryAction);
}
}
@@ -121,6 +123,14 @@ export function captureHealthIssues(bundle: CaptureBundle, expectedIds: readonly
const INTEGRITY_CODES = new Set(["CAPTURE_SIDE_MISMATCH", "CAPTURE_COMMIT_INVALID", "CAPTURE_SELECTION_DUPLICATE", "CAPTURE_TARGET_UNEXPECTED",
"CAPTURE_TARGET_MISSING", "CAPTURE_OUTCOME_MISSING", "CAPTURE_IMAGE_BINDING_INVALID", "CAPTURE_IMAGE_PROVENANCE_INVALID"]);
/**
* Whether a failure casts doubt on everything a capture took, so its other appearances cannot be kept as a partial
* report: a checkout that changed under it, or evidence that failed its integrity checks.
*/
export function capturedEvidenceInDoubt(code: string): boolean {
return /^(?:SOURCE|CHECKOUT)_/.test(code) || code === "CAPTURE_INCOMPLETE" || INTEGRITY_CODES.has(code);
}
export function sortedCaptureHealthIssues(inputs: readonly CaptureHealthInput[]): CaptureHealthIssue[] {
const compare = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
return inputs.flatMap(({ bundle, selectedIds, side }) => captureHealthIssues(bundle, selectedIds, side))
+79 -39
View File
@@ -1,5 +1,5 @@
import { appearanceArgs, argentColorSchemes, bundleColorSchemes, mergeSchemeOutcomes, resolveColorSchemes, routeIdOfScreen, schemeScreenId, type AppearanceSetting, type ColorScheme } from "./appearance.js";
import { assertNoCaptureHealthIssues, captureHealthIssues, partitionCaptureHealthIssues } from "./health.js";
import { assertNoCaptureHealthIssues, capturedEvidenceInDoubt, captureHealthIssues, partitionCaptureHealthIssues } from "./health.js";
import { type RouteResetSetting } from "./route-reset.js";
import { defaultReadinessSeconds } from "./readiness.js";
import { captureLaunch, ffmpegRecordingTools, launchDirectory, type LaunchSetting, type RecordingTools } from "./launch-capture.js";
@@ -13,7 +13,7 @@ import { sourceRouteNames } from "../contracts/source-route.js";
import type { SourceCapturePlan } from "../source/capture-plan.js";
import { CONTRACT, type ExploreMap, type Platform, type RunStatus } from "../types.js";
import { ExploreStore, validateMap } from "../map/store.js";
import { CiError, serializeCiError, toCiError } from "../shared/ci-errors.js";
import { CiError, environmentSecrets, serializeCiError, toCiError } from "../shared/ci-errors.js";
import type { CaptureSetupAction, DevicePose } from "../config/explore-config.js";
import { type CiAgentRuntimeOptions } from "./agent/runtime.js";
import { EAS_CLI_VERSION, CONTROLLER_SESSION, isRecord, parseJson, baseEnvironment, nonEmpty } from "./common.js";
@@ -360,6 +360,8 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
+ (routes.length - missingRoutes.length ? `, ${routes.length - missingRoutes.length} reused from the baseline` : "")
+ (missingRoutes.length ? `, ${missingRoutes.length} to capture` : ""));
let captureFailure: unknown;
// Later appearance passes that failed after the primary one captured (see the pass loop).
const sessionFailures: NonNullable<CaptureBundle["sessionFailures"]> = [];
const finish = async (): Promise<CiMapResult> => {
if (!(await store.exists())) await store.initialize(map, false);
// A session making only secondary passes owns the `<id>@<scheme>` frames of those passes, not the primary ones, even
@@ -368,10 +370,11 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
? selectedIds.flatMap(id => options.appearancePass!.capture.filter(scheme => scheme !== colorSchemes[0]).map(scheme => schemeScreenId(id, scheme)))
: selectedIds;
const incomplete = captureFailure !== undefined || ownFrameIds.some(id => outcomes[id]?.status !== "captured" || outcomes[id]?.landing !== "verified")
|| map.gaps.some(gap => gap.status === "open");
|| map.gaps.some(gap => gap.status === "open") || sessionFailures.length > 0;
map.run.status = incomplete || map.screens.length === 0 ? "partial" : "complete";
if (routes.length === 0) map.run.stopReason = "No source targets require capture.";
else if (captureFailure) map.run.stopReason = "Agent exploration did not finish; see capture diagnostics and bundle outcomes.";
else if (sessionFailures.length) map.run.stopReason = sessionFailures.map(failure => `The ${failure.scheme} pass failed (${failure.error.code}): ${failure.error.message.replace(/\.+$/, "")}.`).join(" ");
else if (incomplete) map.run.stopReason = "Some source targets could not be captured; see bundle outcomes.";
map.run.completedAt = new Date((options.now ?? Date.now)()).toISOString();
await store.write(map);
@@ -432,7 +435,8 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
...(Object.keys(launchRecords).length ? { launch: launchRecords } : {}),
...(options.agent ? { captureContexts: Object.fromEntries(selectedIds.filter(id => outcomes[id]?.status === "captured" && options.agent!.contexts[id])
.map(id => [id, options.agent!.contexts[id]!])), agentAcceptance: { policy: AGENT_ACCEPTANCE_POLICY,
frames: Object.fromEntries(Object.keys(images).map(id => [id, acceptedFrames[id]!])) } } : {}) };
frames: Object.fromEntries(Object.keys(images).map(id => [id, acceptedFrames[id]!])) } } : {}),
...(sessionFailures.length ? { sessionFailures } : {}) };
if (options.agent) assertAgentAcceptance(bundle, map, { required: true });
// The side that produces the evidence checks it the way the review will: an integrity defect fails here, naming
// its codes, instead of on the consuming side after both captures. Unavailable screens are not defects.
@@ -521,6 +525,10 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
await rm(simulatorProject, { recursive: true, force: true });
await localRecord?.close();
};
const earlyCleanupFailure = (original: unknown, cleanup: unknown) => new CiError({ code: "SESSION_CLEANUP_FAILED", stage: "cleanup",
message: "A simulator Explore started before the app was ready could not be confirmed stopped.",
nextAction: "Stop the EAS Simulator session or local device this capture started (eas simulator:list), and verify its status.",
details: { originalFailure: serializeCiError(original, { secrets: [options.expoToken] }) }, cause: cleanup });
// What this side starts on this machine, so a later review cleans up after it if this process is stopped hard.
const localRecord = local ? await openLocalCaptureRecord(sourceEnvironment, simulatorProject) : undefined;
if (local) try {
@@ -546,7 +554,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
} });
localOsVersion = created.osVersion;
}
} catch (error) { await stopEarlyDevice().catch(() => undefined); throw error; }
} catch (error) { await stopEarlyDevice().catch(cleanup => { throw earlyCleanupFailure(error, cleanup); }); throw error; }
// The session is named before its app exists: an EAS session starts while the app is repacked (startSession).
// A session EAS failed to start is replaced once (see the session block), under a name of its own.
const sessionNameFor = (retry: number) => simulatorSessionName({ mapName: options.name, mapId: options.buildId, gitCommit: options.gitCommit, easBuildId: local ? "local" : "installed",
@@ -659,7 +667,8 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
options.signal?.throwIfAborted();
} catch (error) {
// A build or repack that fails leaves nothing running: a device started for it goes, with the session's files and the app.
await stopEarlyDevice().catch(() => undefined);
// One that does not go is the failure to report: a session left running is never hidden behind the build's error.
await stopEarlyDevice().catch(cleanup => { throw earlyCleanupFailure(error, cleanup); });
if (appDirectory) await rm(appDirectory, { recursive: true, force: true });
throw error;
}
@@ -714,7 +723,14 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
const live = liveSimulatorSessionsNamed(easRead(["simulator:list", "--name", sessionName, "--status", "new", "--status", "queued", "--status", "starting",
"--status", "in-progress", "--limit", "100", "--json", "--non-interactive"]), sessionName, profile.projectIdentity);
if (live) {
for (const id of live) assertSimulatorStopped(eas(["simulator:stop", "--id", id, "--json", "--non-interactive"], 2 * 60_000).stdout, id);
// No session file names these sessions, so one that does not stop is reported as left running, never as a
// capture failure a split side could keep as a partial report.
for (const id of live) {
try { assertSimulatorStopped(eas(["simulator:stop", "--id", id, "--json", "--non-interactive"], 2 * 60_000).stdout, id); }
catch (error) { throw new CiError({ code: "SESSION_CLEANUP_FAILED", stage: "cleanup",
message: `EAS Simulator session ${id}, started by a request that failed, could not be confirmed stopped.`,
nextAction: `Stop EAS Simulator session ${id} and verify its status.`, details: { sessionId: id }, cause: error }); }
}
reason = `EAS could not start the simulator session (${simulatorStartFailure(startError, [options.expoToken]).message})`;
}
}
@@ -991,41 +1007,60 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
const passes: CaptureBundle["outcomes"][] = [];
const recordingTools = launchSetting === "off" ? undefined : options.recordingTools === undefined ? ffmpegRecordingTools() : options.recordingTools ?? undefined;
const capturedSchemes = options.appearancePass ? colorSchemes.filter(scheme => options.appearancePass!.capture.includes(scheme)) : colorSchemes;
const activeRuntime = runtime;
for (const [index, colorScheme] of colorSchemes.entries()) {
// A pass another session makes leaves its slot empty, so the passes stay aligned with the appearances they merge by.
if (!capturedSchemes.includes(colorScheme)) { passes.push({}); continue; }
// Argent cannot switch the appearance yet (software-mansion/argent#1509): the host does, simctl or adb on this
// machine and serve-sim on an EAS iOS session; elsewhere its single pass runs as the device starts.
if (controller !== "argent") runner({ command: controllerCommand, args: [...appearanceArgs(colorScheme), "--session", CONTROLLER_SESSION, ...deviceArgs(runtime), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
else if (sessionId && controlActions.has("appearance.set")) simulatorControl(sessionEas, sessionId, "appearance.set", [colorScheme], controlOptions);
else if (localSimulator) setLocalSimulatorAppearance({ runner, udid: localSimulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
else if (localEmulator) setLocalEmulatorAppearance({ runner, serial: localEmulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
// A scheme change recreates the Android activity in place, and the sheets it re-presents lose their inset; a
// relaunch starts the app in the new scheme, the same way the first pass did.
if (launchSetting !== "off") {
// Observing the launch relaunches the app before setup runs, so no UI-created state is lost.
say(`observing the app launch${colorSchemes.length > 1 ? ` (${colorScheme})` : ""}`);
try {
// A restored baseline's launch evidence is replaced only by a fresh observation.
const record = await captureLaunch({ bundleId: options.bundleId, keepVideo: launchSetting === "video", platform: options.platform,
directory: join(launchWorkDirectory, launchDirectory(colorScheme)), relativeDirectory: launchDirectory(colorScheme), ...(options.now ? { now: options.now } : {}),
...(recordingTools ? { recordingTools } : {}),
controller: (args, controllerOptions) => runner({ command: controllerCommand, args: [...args, "--session", CONTROLLER_SESSION, ...deviceArgs(runtime!), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: controllerOptions?.timeoutMs ?? 60_000, ...(controllerOptions?.allowFailure ? { allowFailure: true } : {}) }) });
launch[colorScheme] = { root: launchWorkDirectory, record };
} catch (error) {
// The launch is supplementary evidence: a failure to observe it leaves the route capture intact.
say(`the app launch could not be observed (${(error as Error).message}); the report has no launch evidence for ${colorScheme}`);
await rm(join(launchWorkDirectory, launchDirectory(colorScheme)), { recursive: true, force: true });
}
} else if (index > 0) runner({ command: controllerCommand, args: ["open", options.bundleId, "--relaunch", "--session", CONTROLLER_SESSION,
...deviceArgs(runtime), "--platform", options.platform, "--level", "full", "--json"], cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
runtime.colorScheme = colorScheme;
if (colorSchemes.length > 1) say(`appearance ${colorScheme} (${index + 1}/${colorSchemes.length})`);
passes.push(await capturePlannedRoutes({ capture: { ...options, ...(argentBridge && !options.agent ? { runner: argentBridge.runner } : {}), agentDeviceCommand: controllerCommand, setup, progress: say }, routes: missingRoutes, inventory, scheme, appName: app.name, runtime,
connection, environment: controllerEnvironment, store, map, acceptedFrames, outcomes: index === 0 ? outcomes : {},
...(colorSchemes.length > 1 ? { appearance: { scheme: colorScheme, primary: index === 0 } } : {}) }));
const passOutcomes: CaptureBundle["outcomes"] = index === 0 ? outcomes : {};
const capturePass = async () => {
// Argent cannot switch the appearance yet (software-mansion/argent#1509): the host does, simctl or adb on this
// machine and serve-sim on an EAS iOS session; elsewhere its single pass runs as the device starts.
if (controller !== "argent") runner({ command: controllerCommand, args: [...appearanceArgs(colorScheme), "--session", CONTROLLER_SESSION, ...deviceArgs(activeRuntime), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
else if (sessionId && controlActions.has("appearance.set")) simulatorControl(sessionEas, sessionId, "appearance.set", [colorScheme], controlOptions);
else if (localSimulator) setLocalSimulatorAppearance({ runner, udid: localSimulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
else if (localEmulator) setLocalEmulatorAppearance({ runner, serial: localEmulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
// A scheme change recreates the Android activity in place, and the sheets it re-presents lose their inset; a
// relaunch starts the app in the new scheme, the same way the first pass did.
if (launchSetting !== "off") {
// Observing the launch relaunches the app before setup runs, so no UI-created state is lost.
say(`observing the app launch${colorSchemes.length > 1 ? ` (${colorScheme})` : ""}`);
try {
// A restored baseline's launch evidence is replaced only by a fresh observation.
const record = await captureLaunch({ bundleId: options.bundleId, keepVideo: launchSetting === "video", platform: options.platform,
directory: join(launchWorkDirectory, launchDirectory(colorScheme)), relativeDirectory: launchDirectory(colorScheme), ...(options.now ? { now: options.now } : {}),
...(recordingTools ? { recordingTools } : {}),
controller: (args, controllerOptions) => runner({ command: controllerCommand, args: [...args, "--session", CONTROLLER_SESSION, ...deviceArgs(activeRuntime), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: controllerOptions?.timeoutMs ?? 60_000, ...(controllerOptions?.allowFailure ? { allowFailure: true } : {}) }) });
launch[colorScheme] = { root: launchWorkDirectory, record };
} catch (error) {
// The launch is supplementary evidence: a failure to observe it leaves the route capture intact.
say(`the app launch could not be observed (${(error as Error).message}); the report has no launch evidence for ${colorScheme}`);
await rm(join(launchWorkDirectory, launchDirectory(colorScheme)), { recursive: true, force: true });
}
} else if (index > 0) runner({ command: controllerCommand, args: ["open", options.bundleId, "--relaunch", "--session", CONTROLLER_SESSION,
...deviceArgs(activeRuntime), "--platform", options.platform, "--level", "full", "--json"], cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
activeRuntime.colorScheme = colorScheme;
if (colorSchemes.length > 1) say(`appearance ${colorScheme} (${index + 1}/${colorSchemes.length})`);
return await capturePlannedRoutes({ capture: { ...options, ...(argentBridge && !options.agent ? { runner: argentBridge.runner } : {}), agentDeviceCommand: controllerCommand, setup, progress: say }, routes: missingRoutes, inventory, scheme, appName: app.name, runtime: activeRuntime,
connection, environment: controllerEnvironment, store, map, acceptedFrames, outcomes: passOutcomes,
...(colorSchemes.length > 1 ? { appearance: { scheme: colorScheme, primary: index === 0 } } : {}) });
};
// A later pass of a session whose primary pass captured fails only its own appearance, as a secondary session does
// when the passes are split over sessions (captureSideInSessions): in brentvatne/euxy's main baseline run
// 39506374824 a failed dark session failed a target whose light session had captured 16 of 16 routes.
if (index === 0 || !capturedSchemes.includes(colorSchemes[0]!) || options.agent) { passes.push(await capturePass()); continue; }
try { passes.push(await capturePass()); }
catch (error) {
const failure = toCiError(error, { code: `CAPTURE_${runtimeStage.toUpperCase()}_FAILED`, stage: runtimeStage,
nextAction: "Inspect the capture diagnostic, correct the failing prerequisite, and rerun Explore." });
if (options.signal?.aborted || capturedEvidenceInDoubt(failure.code)) throw failure;
const { code, stage, message, nextAction, retryable } = serializeCiError(failure, { secrets: [options.expoToken, ...environmentSecrets(sourceEnvironment)] });
say(`the ${colorScheme} pass failed (${code}: ${message}); keeping the ${colorSchemes[0]} captures, and the report marks every ${colorScheme} frame it did not take failed`);
sessionFailures.push({ scheme: colorScheme, error: { code, stage, message, nextAction, retryable } });
// The routes the pass finished keep what it recorded.
passes.push(passOutcomes);
}
}
runtime.colorScheme = colorSchemes[0]!;
// A session without the primary pass keeps each outcome as its pass recorded it, under `<id>@<scheme>`: the export
@@ -1034,6 +1069,11 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
if (options.appearancePass && !capturedSchemes.includes(colorSchemes[0]!)) {
for (const [index, pass] of passes.entries()) for (const [id, outcome] of Object.entries(pass)) outcomes[schemeScreenId(id, colorSchemes[index])] = outcome;
} else Object.assign(outcomes, mergeSchemeOutcomes(passes, colorSchemes));
// A failed pass records its failure under each route it did not finish, as the export merge does for a failed session.
for (const { scheme: failedScheme, error } of sessionFailures) {
const pass = passes[colorSchemes.indexOf(failedScheme)] ?? {};
for (const id of selectedIds) if (!pass[id]) outcomes[schemeScreenId(id, failedScheme)] = { status: "failed", attempts: 0, landing: "unverified", reason: error.message, error };
}
} catch (error) { primaryError = toCiError(error, { code: `CAPTURE_${runtimeStage.toUpperCase()}_FAILED`, stage: runtimeStage,
nextAction: "Inspect the capture diagnostic, correct the failing prerequisite, and rerun Explore." }); }
finally {
+35 -4
View File
@@ -1,4 +1,4 @@
import { routeIdOfScreen } from "../capture/appearance.js";
import { bundleColorSchemes, routeIdOfScreen, schemeScreenId } from "../capture/appearance.js";
import { createHash } from "node:crypto";
import { cp, lstat, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
import { isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -54,7 +54,8 @@ function requiresAgentAcceptance(recipe: Recipe, bundle: CaptureBundle): boolean
catch { return false; }
}
async function validateEvidence(directory: string, bundle: CaptureBundle): Promise<void> {
/** Checks an export the way a cache entry is checked before it is written or restored: its provenance, profile and frames. */
export async function validateEvidence(directory: string, bundle: CaptureBundle): Promise<void> {
const map = await readExportedMap(directory, { checkArtifacts: true });
const execution = parseRuntimeExecutionContext(JSON.parse(await readFile(join(directory, "execution.json"), "utf8")));
if (!bundle.runtimeProfile || !/^[a-f0-9]{40}$/.test(bundle.gitCommit)
@@ -113,6 +114,31 @@ async function fileInventory(directory: string, path = ""): Promise<FileRecord[]
return files;
}
/**
* The routes a review can reuse from a capture: a verified frame in every appearance it renders. A baseline whose dark
* session failed (brentvatne/euxy's main baseline run 39506374824) holds its light frames and a failed `<route>@dark`;
* reusing the light frame alone would leave the review's base with no dark frame, and dark would read as unavailable
* because of the base. Such a route is captured by the review's base instead, in every appearance.
*/
export function reusableSurfaceIds(bundle: CaptureBundle): string[] {
const secondarySchemes = bundleColorSchemes(bundle).slice(1);
return bundle.selectedSurfaceIds.filter(id => hasVerifiedCapture(bundle, id)
&& secondarySchemes.every(scheme => bundle.images[schemeScreenId(id, scheme)] && bundle.outcomes[schemeScreenId(id, scheme)] === undefined));
}
/**
* Why a capture of `commit` in `sourceDirectory` is not offered for reuse, when it is there and captured some routes:
* none has every appearance (its dark session failed). Undefined otherwise, the cache's usual misses included.
*/
export async function incompleteCaptureReason(sourceDirectory: string, commit: string): Promise<string | undefined> {
try {
const bundle = await readCaptureBundle(sourceDirectory);
if (bundle.gitCommit !== commit || reusableSurfaceIds(bundle).length || !bundle.selectedSurfaceIds.some(id => hasVerifiedCapture(bundle, id))) return undefined;
const failed = (bundle.sessionFailures ?? []).map(failure => failure.scheme);
return failed.length ? `its ${failed.join(" and ")} session failed, so no route has every appearance` : "no route was captured in every appearance";
} catch { return undefined; }
}
/** Cache full exported evidence, including accessibility maps and all referenced screenshots. */
export async function writeCaptureArtifactCache(directory: string, recipe: Recipe): Promise<boolean> {
try {
@@ -122,6 +148,9 @@ export async function writeCaptureArtifactCache(directory: string, recipe: Recip
if (!bundle.runtimeProfile) return false;
assertAgentAcceptance(bundle, undefined, { required: requiresAgentAcceptance(recipe, bundle) });
await validateEvidence(directory, bundle);
// A capture with no route a review could reuse is not offered: a review that found it would wait for it, and capture
// its sides in turn instead of together, only to capture every route of the base itself.
if (!reusableSurfaceIds(bundle).length) return false;
const metadata: CacheMetadata = {
contract: CONTRACT, recipe: recipeKey(recipe), commit: bundle.gitCommit, files: await fileInventory(directory),
};
@@ -181,7 +210,8 @@ export async function restoreCaptureArtifactCache(options: Recipe & {
stage = await mkdtemp(`${options.directory}.cache-`);
const stagedEvidence = join(stage, "evidence");
await cp(source, stagedEvidence, { recursive: true, force: false, errorOnExist: true });
const available = options.selectedIds.filter(id => hasVerifiedCapture(bundle, id)
const reusable = new Set(reusableSurfaceIds(bundle));
const available = options.selectedIds.filter(id => reusable.has(id)
&& (!options.captureContexts || options.captureContexts[id] && bundle.captureContexts?.[id]
&& canonicalJson(options.captureContexts[id]) === canonicalJson(bundle.captureContexts[id])));
if (!available.length) return false;
@@ -192,7 +222,8 @@ export async function restoreCaptureArtifactCache(options: Recipe & {
const captureContexts = bundle.captureContexts ? Object.fromEntries(available.map(id => [id, bundle.captureContexts![id]])) : undefined;
const agentAcceptance = bundle.agentAcceptance ? { policy: bundle.agentAcceptance.policy,
frames: Object.fromEntries(available.map(id => [id, bundle.agentAcceptance!.frames[id]!])) } : undefined;
await writeFile(join(stagedEvidence, "bundle.json"), JSON.stringify({ ...bundle, side: "base", selectedSurfaceIds: available, images, outcomes,
// Only complete routes are restored, so a failed pass of the baseline's is no part of the restored bundle.
await writeFile(join(stagedEvidence, "bundle.json"), JSON.stringify({ ...bundle, side: "base", selectedSurfaceIds: available, images, outcomes, sessionFailures: undefined,
...(captureContexts ? { captureContexts } : {}), ...(agentAcceptance ? { agentAcceptance } : {}) }));
await cp(stagedEvidence, options.directory, { recursive: true, force: false, errorOnExist: true });
return true;
+2 -2
View File
@@ -1,6 +1,6 @@
import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
import { runCiMap, type CiMapOptions, type CiMapResult } from "../capture/run.js";
import { CiError, serializeCiError, type CiErrorData } from "../shared/ci-errors.js";
import { CiError, environmentSecrets, serializeCiError, type CiErrorData } from "../shared/ci-errors.js";
import type { Progress } from "../shared/progress.js";
type WorkerMessage = { type: "progress"; message: string } | { type: "done"; result: CiMapResult } | { type: "error"; error: CiErrorData };
@@ -41,5 +41,5 @@ if (!isMainThread && parentPort) {
port.unref();
runCiMap({ ...options, signal: controller.signal, progress: message => port.postMessage({ type: "progress", message } satisfies WorkerMessage) })
.then(result => port.postMessage({ type: "done", result } satisfies WorkerMessage))
.catch((error: unknown) => port.postMessage({ type: "error", error: serializeCiError(error, { secrets: [options.expoToken] }) } satisfies WorkerMessage));
.catch((error: unknown) => port.postMessage({ type: "error", error: serializeCiError(error, { secrets: [options.expoToken, ...environmentSecrets(options.environment ?? {})] }) } satisfies WorkerMessage));
}
+38 -11
View File
@@ -1,7 +1,7 @@
import { copyFile, mkdir, readdir, readFile, writeFile, appendFile, lstat } from "node:fs/promises";
import { dirname, join, relative } from "node:path";
import { bundleColorSchemes, mergeSchemeOutcomes, routeIdOfScreen, type ColorScheme } from "../capture/appearance.js";
import { bundleColorSchemes, mergeSchemeOutcomes, routeIdOfScreen, schemeScreenId, type ColorScheme } from "../capture/appearance.js";
import type { CaptureOutcome } from "../contracts/capture.js";
import type { CaptureBundle } from "../contracts/capture.js";
import type { ExploreMap } from "../types.js";
@@ -22,12 +22,21 @@ async function copyTree(source: string, destination: string, skip: (relativePath
}
}
/**
* A secondary appearance session's result: its export, or the error it failed with, when it left none. In brentvatne/euxy's
* main baseline run 39506374824 the dark session's install failed after the light session had captured 16 of 16 routes.
*/
export type SecondaryCaptureExport = { directory: string; scheme: ColorScheme } | { scheme: ColorScheme; error: NonNullable<CaptureOutcome["error"]> };
/**
* Fold the exports of a side's secondary appearance sessions into its primary export, in place: the frames, outcomes,
* launch evidence and map records of each `<id>@<scheme>` screen, and their files. Each session captured its passes as
* one run would (`appearancePass`), so records only append; a secondary export never replaces a primary record.
* one run would (`appearancePass`), so records only append; a secondary export never replaces a primary record. A session
* that failed is listed in the bundle's `sessionFailures`, adds a failed `<id>@<scheme>` outcome carrying its error for
* every selected route, and leaves the run partial, so its appearance reads as unavailable with its own reason (a route
* the primary lost too would otherwise show the primary's) instead of going missing.
*/
export async function mergeCaptureExports(primary: string, secondaries: readonly { directory: string; scheme: ColorScheme }[]): Promise<void> {
export async function mergeCaptureExports(primary: string, secondaries: readonly SecondaryCaptureExport[]): Promise<void> {
if (secondaries.length === 0) return;
const bundle = await readJson<CaptureBundle>(join(primary, "bundle.json"));
const map = await readJson<ExploreMap>(join(primary, "screen-map.json"));
@@ -37,15 +46,28 @@ export async function mergeCaptureExports(primary: string, secondaries: readonly
const schemes = bundleColorSchemes(bundle);
const passes: Record<string, CaptureOutcome>[] = schemes.map(() => ({}));
for (const [id, outcome] of Object.entries(bundle.outcomes)) if (!routeIdOfScreen(id).scheme) passes[0]![id] = outcome;
for (const { directory: secondary, scheme } of secondaries) {
const other = await readJson<CaptureBundle>(join(secondary, "bundle.json"));
const failedSessions: string[] = [];
const failedOutcomes: Record<string, CaptureOutcome> = {};
for (const secondary of secondaries) {
const { scheme } = secondary;
if ("error" in secondary) {
failedSessions.push(`The ${scheme} session failed (${secondary.error.code}): ${secondary.error.message.replace(/\.+$/, "")}.`);
(bundle.sessionFailures ??= []).push({ scheme, error: secondary.error });
// A primary that renders no such appearance has no slot for its frames; the failure stays on the bundle alone.
if (!schemes.includes(scheme)) continue;
for (const id of bundle.selectedSurfaceIds) {
failedOutcomes[schemeScreenId(id, scheme)] = { status: "failed", attempts: 0, landing: "unverified", reason: secondary.error.message, error: secondary.error };
}
continue;
}
const other = await readJson<CaptureBundle>(join(secondary.directory, "bundle.json"));
const pass = passes[schemes.indexOf(scheme)];
// A session that could not show its appearance (Argent without appearance.set captures one) has nothing to add, as
// one run would have no pass for it.
if (!pass && !bundleColorSchemes(other).includes(scheme)) continue;
if (!pass) throw new Error(`The ${scheme} session's export is not an appearance of the primary export (${schemes.join(", ")})`);
for (const [id, outcome] of Object.entries(other.outcomes)) pass[routeIdOfScreen(id).routeId] = outcome;
const otherMap = await readJson<ExploreMap>(join(secondary, "screen-map.json"));
const otherMap = await readJson<ExploreMap>(join(secondary.directory, "screen-map.json"));
for (const [id, image] of Object.entries(other.images)) if (routeIdOfScreen(id).scheme && !bundle.images[id]) bundle.images[id] = image;
for (const [scheme, record] of Object.entries(other.launch ?? {})) { bundle.launch ??= {}; bundle.launch[scheme] ??= record; }
for (const [id, context] of Object.entries(other.captureContexts ?? {})) { bundle.captureContexts ??= {}; bundle.captureContexts[id] ??= context; }
@@ -61,9 +83,9 @@ export async function mergeCaptureExports(primary: string, secondaries: readonly
}
}
// Files: artifacts and launch evidence are new paths; diagnostics append to the primary's lines.
await copyTree(secondary, primary, rel => rel === "bundle.json" || rel === "screen-map.json" || rel === "screen-map.schema.json" || rel === "manifest.json"
await copyTree(secondary.directory, primary, rel => rel === "bundle.json" || rel === "screen-map.json" || rel === "screen-map.schema.json" || rel === "manifest.json"
|| rel === "execution.json" || rel.startsWith("capture-diagnostics"));
const diagnostics = join(secondary, "capture-diagnostics");
const diagnostics = join(secondary.directory, "capture-diagnostics");
for (const entry of await readdir(diagnostics, { withFileTypes: true }).catch(() => [])) {
if (!entry.isFile()) continue;
const to = join(primary, "capture-diagnostics", entry.name);
@@ -72,12 +94,17 @@ export async function mergeCaptureExports(primary: string, secondaries: readonly
else if (!(await lstat(to).catch(() => undefined))) await copyFile(join(diagnostics, entry.name), to);
}
}
// An open gap a secondary pass left makes the run partial, as it does one run's (a complete run has none).
if (map.gaps.some(gap => gap.status === "open")) map.run.status = "partial";
// An open gap a secondary pass left makes the run partial, as it does one run's (a complete run has none), and so does
// a secondary session that failed. A partial run names why, as the map's schema requires.
if (map.gaps.some(gap => gap.status === "open") || failedSessions.length) {
map.run.status = "partial";
map.run.stopReason = [map.run.stopReason ?? (failedSessions.length ? undefined : "Some source targets could not be captured; see bundle outcomes."),
...failedSessions].filter(Boolean).join(" ");
}
const manifest = await readJson<{ files: unknown[] }>(join(primary, "manifest.json"));
manifest.files = map.artifacts.map(artifact => ({ path: artifact.path, sha256: artifact.sha256, bytes: artifact.bytes, mimeType: artifact.mimeType }));
await writeFile(join(primary, "screen-map.json"), JSON.stringify(map));
bundle.outcomes = mergeSchemeOutcomes(passes, schemes);
bundle.outcomes = { ...mergeSchemeOutcomes(passes, schemes), ...failedOutcomes };
await writeFile(join(primary, "bundle.json"), JSON.stringify(bundle));
await writeFile(join(primary, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
}
+12 -2
View File
@@ -293,6 +293,17 @@ function sessionStateFiles(output: string): string[] {
catch { return []; }
}
/**
* A finished target keeps only its report: the captures, maps and checkouts are the bulk of the run. A session file
* stays, since it names a session no capture confirmed stopped; the review stops it on its way out (stopRecordedSessions).
*/
export async function pruneTargetOutput(output: string): Promise<void> {
const sessions = new Set(sessionStateFiles(output));
for (const entry of await readdir(output)) {
if (entry !== "report" && entry !== "report.zip" && !sessions.has(entry)) await rm(join(output, entry), { recursive: true, force: true });
}
}
function recordedSessionIds(outputs: string[]): string[] {
return outputs.flatMap(output => sessionStateFiles(output).flatMap(name => {
try { const id = readFileSync(join(output, name), "utf8").trim(); return /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(id) ? [id] : []; } catch { return []; }
@@ -526,8 +537,7 @@ export async function runLocalReview(options: LocalReviewOptions): Promise<Local
if (side === "head" && head.uncommitted) continue;
await keepCapture(join(output, `${side}-export`), captureCacheDirectory(environment, target.id, side === "base" ? base.sha : head.sha));
}
// The report is all that is kept; the captures, maps and checkouts are the bulk of the run.
for (const entry of await readdir(output)) if (entry !== "report" && entry !== "report.zip") await rm(join(output, entry), { recursive: true, force: true });
await pruneTargetOutput(output);
abort.signal.throwIfAborted();
return report;
};
+60 -14
View File
@@ -1,7 +1,7 @@
import { systemRegions } from "../contracts/capture.js";
import type { RunTarget } from "../types.js";
import { execFile, spawnSync } from "node:child_process";
import { mkdir, mkdtemp, open, readFile, realpath, rm, writeFile } from "node:fs/promises";
import { lstat, mkdir, mkdtemp, open, readFile, realpath, rm, writeFile } from "node:fs/promises";
import { join, relative, resolve, isAbsolute } from "node:path";
import { tmpdir } from "node:os";
import { setTimeout as delay } from "node:timers/promises";
@@ -12,12 +12,12 @@ import type { RepackShare } from "../capture/eas-build.js";
import { resolveColorSchemes, routeIdOfScreen, type ColorScheme } from "../capture/appearance.js";
import { baseEnvironment } from "../capture/common.js";
import { DEFAULT_MAX_SESSIONS, planSideSessions, sessionShare } from "./session-plan.js";
import { mergeCaptureExports } from "./export-merge.js";
import { mergeCaptureExports, type SecondaryCaptureExport } from "./export-merge.js";
import { abandonSharedBuilds } from "../capture/shared-build.js";
import { prepareCaptureReview, readCaptureBundle } from "../review/capture-review.js";
import { quarantineReviewContract } from "../contracts/review-validation.js";
import { agentCaptureConstraints } from "../capture/agent/source-context.js";
import { captureArtifactCacheMayMatch, captureRecipeConfig, restoreCaptureArtifactCache, writeCaptureArtifactCache } from "./artifact-cache.js";
import { captureArtifactCacheMayMatch, captureRecipeConfig, incompleteCaptureReason, restoreCaptureArtifactCache, writeCaptureArtifactCache } from "./artifact-cache.js";
import { runCiMapOnWorker } from "./capture-worker.js";
import { DEFAULT_BASELINE_WAIT_MINUTES, awaitPendingBaseline } from "./baseline-wait.js";
import { writeCiReportInput, readAppDirectory } from "../report/artifact.js";
@@ -30,10 +30,10 @@ import { viewerLinks } from "../report/viewer-links.js";
import { DEFAULT_EXPLORE_CONFIG_PATH, captureRepack, type ExploreConfig, type ExplorePlatform } from "../config/explore-config.js";
import { prepareCiExploration, type PrepareCiExplorationOptions } from "../capture/agent/exploration.js";
import { generateAgentChangeNotes } from "../capture/agent/change-notes.js";
import { CiError, serializeCiError, toCiError } from "../shared/ci-errors.js";
import { CiError, environmentSecrets, serializeCiError, toCiError } from "../shared/ci-errors.js";
import type { SourceCapturePlan } from "../source/capture-plan.js";
import { createSourceChangeSummaries } from "../source/change-summary.js";
import { assertNoCaptureHealthIssues, partitionCaptureHealthIssues, renderCaptureIssueAnnotation, renderComparisonDiagnosticAnnotation, renderNotesUnavailableAnnotation, renderReviewGapAnnotation, sortedCaptureHealthIssues } from "../capture/health.js";
import { assertNoCaptureHealthIssues, capturedEvidenceInDoubt, partitionCaptureHealthIssues, renderCaptureIssueAnnotation, renderComparisonDiagnosticAnnotation, renderNotesUnavailableAnnotation, renderReviewGapAnnotation, sortedCaptureHealthIssues } from "../capture/health.js";
import { EXPLORE_PACKAGE, packageBin } from "../shared/package-paths.js";
import { agentDeviceLauncher, ensureArgentToolServer, ensureController } from "../shared/controller-cache.js";
import { silentProgress, type Progress } from "../shared/progress.js";
@@ -130,14 +130,37 @@ async function expectedColorSchemes(config: ExploreConfig, project: string, envi
}
}
/**
* Failures that fail the side even when only a secondary appearance session had them. A session Explore could not
* confirm stopped, or whose identity it could not verify, must never be hidden behind a partial report; a checkout that
* changed, evidence that failed its integrity checks, or a failure Explore did not classify means the primary's frames
* may not be what they claim either.
*/
const SIDE_FAILURE_CODES = new Set(["SESSION_CLEANUP_FAILED", "SESSION_HANDOFF_UNVERIFIED", "CAPTURE_WORKER_EXITED", "EXPLORE_FAILED"]);
/**
* Whether a secondary appearance session's failure can be reported as that appearance's frames failing, keeping the
* primary's: only when the capture reported it itself (a worker that died before its cleanup ran rejects otherwise), its
* session is proven stopped (the capture removes its session file only after a confirmed stop), and nothing in it
* casts doubt on the primary's evidence.
*/
async function secondaryFailureKept(error: unknown, sessionStatePath: string | undefined): Promise<boolean> {
if (!(error instanceof CiError)) return false;
if (SIDE_FAILURE_CODES.has(error.code) || capturedEvidenceInDoubt(error.code)
|| (error.details as { allocation?: unknown } | undefined)?.allocation === "unknown") return false;
return !(sessionStatePath && await lstat(sessionStatePath).then(() => true, () => false));
}
/**
* Run one side's capture: in one session, or when `passes` splits its appearances, one worker session per pass. The
* primary pass exports to `mapOptions.outputDirectory`; the others beside it, named by their appearances, then merge.
* `runPass` replaces the worker capture of each pass (tests).
*/
async function captureSideInSessions(mapOptions: CiMapOptions, passes: readonly (readonly ColorScheme[])[] | undefined,
progress: Progress, signal: AbortSignal | undefined, worker: boolean): Promise<Awaited<ReturnType<typeof runCiMap>>> {
const runOnWorker = (passOptions: CiMapOptions) =>
runCiMapOnWorker((({ progress: _progress, runner: _runner, agent: _agent, signal: _signal, ...rest }) => rest)(passOptions), progress, signal);
export async function captureSideInSessions(mapOptions: CiMapOptions, passes: readonly (readonly ColorScheme[])[] | undefined,
progress: Progress, signal: AbortSignal | undefined, worker: boolean,
runPass?: (passOptions: CiMapOptions) => Promise<Awaited<ReturnType<typeof runCiMap>>>): Promise<Awaited<ReturnType<typeof runCiMap>>> {
const runOnWorker = runPass ?? ((passOptions: CiMapOptions) =>
runCiMapOnWorker((({ progress: _progress, runner: _runner, agent: _agent, signal: _signal, ...rest }) => rest)(passOptions), progress, signal));
if (!passes || passes.length <= 1) return worker ? runOnWorker(mapOptions) : runCiMap(mapOptions);
const schemes = passes.flat();
progress(`${mapOptions.side}: capturing ${passes.map(pass => pass.join("+")).join(" and ")} in separate sessions`);
@@ -158,9 +181,29 @@ async function captureSideInSessions(mapOptions: CiMapOptions, passes: readonly
}));
// The build folder holds the repacked app, which the sessions have installed.
if (ownBuildShare) await rm(ownBuildShare, { recursive: true, force: true });
for (const outcome of settled) if (outcome.status === "rejected") throw outcome.reason;
await mergeCaptureExports(mapOptions.outputDirectory, passes.slice(1).map(pass => ({ directory: `${mapOptions.outputDirectory}-${pass.join("-")}`, scheme: pass[0]! })));
return (settled[0] as PromiseFulfilledResult<Awaited<ReturnType<typeof runCiMap>>>).value;
// Only the primary appearance's session fails the side, besides a stopped review and a secondary failure that cannot be
// kept (secondaryFailureKept). In brentvatne/euxy's main baseline run 39506374824 the light session captured 16 of 16
// routes and the dark session's install failed, which failed the whole target and lost its baseline. Such a failure is
// kept in the report instead: every frame of its appearance is failed with its error, and the run is partial.
const suffixOf = (pass: readonly ColorScheme[]) => `-${pass.join("-")}`;
for (const [index, outcome] of settled.entries()) {
if (outcome.status !== "rejected" || index === 0) continue;
const statePath = mapOptions.sessionStatePath ? `${mapOptions.sessionStatePath}${suffixOf(passes[index]!)}` : undefined;
if (signal?.aborted || !await secondaryFailureKept(outcome.reason, statePath)) throw outcome.reason;
}
const primary = settled[0]!;
if (primary.status === "rejected") throw primary.reason;
const secrets = [mapOptions.expoToken, ...environmentSecrets(mapOptions.environment ?? {})];
const secondaries = passes.slice(1).map((pass, index): SecondaryCaptureExport => {
const outcome = settled[index + 1]!;
if (outcome.status === "fulfilled") return { directory: `${mapOptions.outputDirectory}${suffixOf(pass)}`, scheme: pass[0]! };
const error = serializeCiError(outcome.reason, { secrets });
progress(`${mapOptions.side}: the ${pass.join("+")} session failed (${error.code}: ${error.message}); keeping the ${passes[0]!.join("+")} captures, and the report marks every ${pass.join("+")} frame failed`);
return { scheme: pass[0]!, error: { code: error.code, stage: error.stage, message: error.message, nextAction: error.nextAction, retryable: error.retryable } };
});
await mergeCaptureExports(mapOptions.outputDirectory, secondaries);
const result = primary.value;
return secondaries.some(secondary => "error" in secondary) ? { ...result, runStatus: "partial" } : result;
}
/**
@@ -435,7 +478,7 @@ export async function runCiReview(options: {
if (!options.runner) await prefetchControllers(config, environment, progress);
const pr = options.presentation;
const now = options.now ?? Date.now;
const secretValues = Object.entries(environment).filter(([key]) => /(?:TOKEN|SECRET|PASSWORD|API_KEY)/i.test(key)).flatMap(([, value]) => value ? [value] : []);
const secretValues = environmentSecrets(environment);
let stage = "plan";
let usagePhase: AgentUsagePhase = "preflight";
let preserveReport: (() => Promise<void>) | undefined;
@@ -500,7 +543,10 @@ export async function runCiReview(options: {
commit: pr.base.sha, selectedIds, directory: join(output, "base-cache"), runtimeProfile,
...(exploration.agent ? { captureContexts: exploration.agent.contexts } : {}) });
if (restored) reuseDirectory = join(output, "base-cache");
progress(restored ? `base: found the baseline captures of ${pr.base.sha.slice(0, 8)}` : `base: no compatible baseline of ${pr.base.sha.slice(0, 8)}; capturing the base too`);
const incomplete = restored ? undefined : await incompleteCaptureReason(cacheDirectory, pr.base.sha);
progress(restored ? `base: found the baseline captures of ${pr.base.sha.slice(0, 8)}`
: incomplete ? `base: the baseline of ${pr.base.sha.slice(0, 8)} has nothing to reuse (${incomplete}); capturing the base too`
: `base: no compatible baseline of ${pr.base.sha.slice(0, 8)}; capturing the base too`);
}
const mapOptions = { ...captureOptions(config, environment, options.runner, { now: options.now, sleep: options.sleep }), progress,
...(options.useLocalEasSession ? { useLocalEasSession: true } : {}),
+6
View File
@@ -93,6 +93,12 @@ export type CaptureBundle = {
}>;
/** The exact observed frames explicitly accepted by the agent, never replacement screenshots. */
agentAcceptance?: { policy: typeof AGENT_ACCEPTANCE_POLICY; frames: Record<string, AgentAcceptedFrame> };
/**
* The appearance passes that failed while the primary one captured (a secondary session that failed, or a later pass of
* one session), each with its error. Every selected route then has a failed `<id>@<scheme>` outcome carrying it, when
* the bundle renders that appearance. Absent when every pass finished.
*/
sessionFailures?: Array<{ scheme: "light" | "dark"; error: NonNullable<CaptureOutcome["error"]> }>;
};
/** Bind acceptance receipts to captured images and, when supplied, their exported observations. */
+31 -7
View File
@@ -321,21 +321,34 @@ export function platformRecordOf(state: PublicCommentState, target?: CommentTarg
if (notCompared) record.notCompared = notCompared;
record.context = splashLines(state.review);
const diagnostics = [...(state.evidence?.diagnostics ?? [])];
// A failed appearance pass is one issue, named first, rather than one per frame it left unavailable: in brentvatne/
// euxy's main baseline run 39506374824 the dark session's failure would have listed all 16 routes, crowding out the rest.
const sessionFailures = state.evidence?.sessionFailures ?? [];
if (sessionFailures.length) record.sessionFailures = sessionFailures.map(({ side, scheme, frames }) => ({ side, scheme, frames }));
const sessionLines = sessionFailures.map(failure => ({ side: failure.side, route: `${failure.scheme} session`, code: failure.code,
message: `${failure.message.replace(/\.+$/, "")}.${failure.frames ? ` ${plural(failure.frames, `${failure.scheme} frame`)} ${failure.frames === 1 ? "is" : "are"} unavailable.` : ""}`,
nextAction: failure.nextAction }));
const ofFailedPass = (surfaceId: string, side: "base" | "head", outcome: NonNullable<Surface["headOutcome"]>) => sessionFailures.some(failure => failure.side === side
&& schemeOf(surfaceId) === failure.scheme && outcome.error?.code === failure.code && outcome.error.message === failure.message);
for (const surface of state.review.surfaces) {
for (const side of ["base", "head"] as const) {
const outcome = surface[`${side}Outcome`];
if (!outcome || (outcome.status === "captured" && outcome.landing === "verified") || diagnostics.some(item => item.surfaceId === surface.id && item.side === side)) continue;
if (ofFailedPass(surface.id, side, outcome)) continue;
diagnostics.push({ surfaceId: surface.id, side, code: outcome.error?.code ?? (outcome.status === "captured" ? "CAPTURE_LANDING_UNVERIFIED" : "CAPTURE_UNAVAILABLE"),
stage: outcome.error?.stage ?? "capture", message: outcome.error?.message ?? outcome.reason ?? (outcome.status === "captured" ? "Destination route was not verified." : `Capture ${outcome.status}.`),
nextAction: outcome.error?.nextAction ?? "Check this side's capture diagnostics, correct the route configuration or capture failure, and rerun capture.", retryable: outcome.error?.retryable ?? false });
}
}
if (diagnostics.length) {
record.diagnostics = diagnostics.slice(0, 20).map(diagnostic => {
if (diagnostics.length || sessionLines.length) {
record.diagnostics = [...sessionLines, ...diagnostics.map(diagnostic => {
const surface = state.review!.surfaces.find(surface => surface.id === diagnostic.surfaceId);
return { ...(diagnostic.side ? { side: diagnostic.side } : {}), route: surface ? routeOf(surface) : diagnostic.surfaceId, code: diagnostic.code, message: diagnostic.message, nextAction: diagnostic.nextAction };
});
record.diagnosticCount = diagnostics.length;
// A secondary appearance's issue names its appearance, as its change row does: its light frame may be fine.
const scheme = schemeOf(diagnostic.surfaceId);
return { ...(diagnostic.side ? { side: diagnostic.side } : {}), route: `${surface ? routeOf(surface) : diagnostic.surfaceId}${surface && scheme ? ` (${scheme})` : ""}`,
code: diagnostic.code, message: diagnostic.message, nextAction: diagnostic.nextAction };
})].slice(0, 20);
record.diagnosticCount = sessionLines.length + diagnostics.length;
}
}
if (state.failure) record.failure = { step: state.failure.step, message: state.failure.message };
@@ -480,9 +493,11 @@ function title(records: PlatformRecord[], rows: ChangeRow[]): string {
}
// With screens left uncompared, nothing changed only among the ones compared.
const summary = rows.length ? `${plural(rows.length, "screen")} changed` : ready.some(record => gaps(record).length) ? "No compared screens changed" : "No screens changed";
// An appearance a pass never captured is said in the heading too: the verdict above says nothing about it.
const missing = [...new Set(ready.flatMap(record => (record.sessionFailures ?? []).map(failure => failure.scheme)))].map(scheme => `${scheme} not captured`);
const report = firstReport(ready);
const open = report ? link("Review in Explore", report) : ready.some(record => record.links.deploying) ? "report deploying…" : undefined;
return `### ${summary}${open ? ` · ${open}` : ""}`;
return `### ${[summary, ...missing, ...(open ? [open] : [])].join(" · ")}`;
}
/** Every target by name, each linking its own report, then the commits and when the comment last changed. */
@@ -512,12 +527,21 @@ function statusLine(record: PlatformRecord): string | undefined {
}
}
/** "dark not captured" for an appearance whose pass failed on head (or both sides), "… on base" when only the base's did. */
function missingAppearances(record: PlatformRecord): string[] {
const failures = record.sessionFailures ?? [];
return [...new Set(failures.map(failure => failure.scheme))].map(scheme => {
const sides = new Set(failures.filter(failure => failure.scheme === scheme).map(failure => failure.side));
return `${scheme} not captured${sides.size === 1 && sides.has("base") ? " on base" : ""}`;
});
}
/** What a finished target has no verdict on: captures missing or unattributed, comparisons that did not run or failed. */
function gaps(record: PlatformRecord): string[] {
const unavailable = record.health ? record.health.base.unavailable + record.health.head.unavailable : 0;
const unverified = record.health ? record.health.base.unverified + record.health.head.unverified : 0;
const failed = (record.validation?.visual?.failed ?? 0) + (record.validation?.accessibility?.failed ?? 0);
return [unavailable ? `${plural(unavailable, "capture")} unavailable` : undefined, unverified ? `${unverified} unattributed` : undefined,
return [...missingAppearances(record), unavailable ? `${plural(unavailable, "capture")} unavailable` : undefined, unverified ? `${unverified} unattributed` : undefined,
record.notCompared ? `${record.notCompared} not compared` : undefined, failed ? `${plural(failed, "comparison")} failed` : undefined]
.filter((part): part is string => Boolean(part));
}
+7
View File
@@ -115,6 +115,8 @@ export type PlatformRecord = {
navigation?: CommentNavigation;
diagnostics?: { side?: "base" | "head"; route: string; code: string; message: string; nextAction: string }[];
diagnosticCount?: number;
/** Appearance passes that failed while the primary one captured: that appearance was not captured on that side. */
sessionFailures?: { side: "base" | "head"; scheme: "light" | "dark"; frames: number }[];
failure?: { step: string; message: string };
/** The PR moved while this run worked; its report stays reachable but it is not the current review. */
superseded?: boolean;
@@ -287,6 +289,11 @@ export function parsePlatformRecord(value: unknown): PlatformRecord | undefined
}
if (isCount(value.notCompared) && value.notCompared > 0) record.notCompared = value.notCompared;
if (isCount(value.diagnosticCount)) record.diagnosticCount = value.diagnosticCount;
if (Array.isArray(value.sessionFailures)) {
const failures = value.sessionFailures.flatMap((entry): NonNullable<PlatformRecord["sessionFailures"]> => isRecord(entry) && isSide(entry.side)
&& (entry.scheme === "light" || entry.scheme === "dark") && isCount(entry.frames) ? [{ side: entry.side, scheme: entry.scheme, frames: entry.frames }] : []).slice(0, 4);
if (failures.length) record.sessionFailures = failures;
}
if (isRecord(value.failure) && isString(value.failure.step, 200) && isString(value.failure.message)) record.failure = { step: value.failure.step, message: value.failure.message };
if (value.superseded === true) record.superseded = true;
return record;
+9 -2
View File
@@ -1,5 +1,5 @@
import type { RunTarget } from "../types.js";
import { appearancePairs, bundleColorSchemes, routeIdOfScreen } from "../capture/appearance.js";
import { appearancePairs, bundleColorSchemes, failedPassOutcome, routeIdOfScreen } from "../capture/appearance.js";
import { createHash } from "node:crypto";
import { lstat, readFile, realpath } from "node:fs/promises";
import { isAbsolute, relative, resolve, sep } from "node:path";
@@ -95,9 +95,11 @@ function secondaryPassOutcome(bundle: CaptureBundle, selected: ReadonlySet<strin
const { routeId, scheme } = routeIdOfScreen(id);
const own = bundle.outcomes[id];
const verified = own?.status === "captured" && own.landing === "verified";
return scheme !== undefined && selected.has(routeId) && verified !== (bundle.outcomes[routeId]?.status === "captured");
return scheme !== undefined && selected.has(routeId)
&& (verified !== (bundle.outcomes[routeId]?.status === "captured") || failedPassOutcome(bundle, id));
}
export async function readCaptureBundle(directory: string): Promise<CaptureBundle> {
const bundle = JSON.parse(await readFile(resolve(await realpath(directory), "bundle.json"), "utf8")) as CaptureBundle;
if (!bundle || bundle.contract !== "explore.bundle" || typeof bundle.gitCommit !== "string"
@@ -110,6 +112,11 @@ export async function readCaptureBundle(directory: string): Promise<CaptureBundl
throw new Error("Capture bundle completion timestamp is invalid");
}
if (bundle.appIcon !== undefined) assertAppIconRecord(bundle.appIcon);
if (bundle.sessionFailures !== undefined && (!Array.isArray(bundle.sessionFailures) || bundle.sessionFailures.some(failure => !failure
|| !["light", "dark"].includes(failure.scheme) || !failure.error || typeof failure.error.code !== "string" || typeof failure.error.message !== "string"
|| typeof failure.error.stage !== "string" || typeof failure.error.nextAction !== "string" || typeof failure.error.retryable !== "boolean"))) {
throw new Error("Capture bundle session failures are invalid");
}
if (bundle.launch !== undefined) assertLaunchRecords(bundle.launch);
const selected = new Set(bundle.selectedSurfaceIds);
if (selected.size !== bundle.selectedSurfaceIds.length
+13 -1
View File
@@ -1,5 +1,5 @@
import { unattributedIssue } from "../capture/health.js";
import { appearancePairs, bundleColorSchemes, type ColorScheme } from "../capture/appearance.js";
import { appearancePairs, bundleColorSchemes, failedPassOutcome, type ColorScheme } from "../capture/appearance.js";
import { launchSurfaceId, systemRegions as defaultSystemRegions, type SystemRegion } from "../contracts/capture.js";
import { readFileSync, statSync } from "node:fs";
import { stat } from "node:fs/promises";
@@ -79,8 +79,16 @@ export type PublicEvidenceSummary = {
comparison: { planned: number; eligible: number; skipped: number };
/** Every unavailable or unverified requested side, plus operational comparison errors. */
diagnostics: PublicEvidenceDiagnostic[];
/**
* Each appearance pass of a side that failed while its primary one captured (the bundle's `sessionFailures`), with how
* many of its frames the failure left unavailable: the comment names it once, not once per frame.
*/
sessionFailures?: PublicSessionFailure[];
};
export type PublicSessionFailure = { side: "base" | "head"; scheme: ColorScheme; frames: number;
code: string; stage: string; message: string; nextAction: string; retryable: boolean };
const MAX_SUMMARY_ERRORS = 5;
function recordError(errors: string[], surfaceId: string, error: unknown): void {
@@ -756,6 +764,9 @@ export async function assessPublicEvidence(options: {
}
}
const sessionFailures = (["base", "head"] as const).flatMap(side => (options[side].bundle.sessionFailures ?? []).map(({ scheme, error }): PublicSessionFailure => ({
side, scheme, frames: Object.keys(options[side].bundle.outcomes).filter(id => id.endsWith(`@${scheme}`) && failedPassOutcome(options[side].bundle, id)
&& options[side].bundle.outcomes[id]!.error?.message === error.message).length, ...error })));
return {
surfaces,
summary: {
@@ -763,6 +774,7 @@ export async function assessPublicEvidence(options: {
accessibility: accessibilityAvailable ? accessibilityCounts : null,
review: reviewCounts,
capture, comparison, diagnostics,
...(sessionFailures.length ? { sessionFailures } : {}),
},
};
}
+5
View File
@@ -55,6 +55,11 @@ export function ciErrorForCommand(error: unknown, command: string): CiError {
export type CiErrorRenderOptions = { secrets?: readonly string[] };
/** The values of every variable that names a secret (a token, secret, password or API key), to redact from what Explore writes. */
export function environmentSecrets(environment: NodeJS.ProcessEnv): string[] {
return Object.entries(environment).filter(([key]) => /(?:TOKEN|SECRET|PASSWORD|API_KEY)/i.test(key)).flatMap(([, value]) => value ? [value] : []);
}
function redact(value: string, secrets: readonly string[]): string {
let result = value;
// Controller failures may embed JSON or URLs inside their message rather than
+49
View File
@@ -4,7 +4,11 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { createServer } from "node:http";
import { startArgentBridgeRunner } from "../../dist/capture/argent-bridge-host.js";
import { argentClientCall } from "../../dist/capture/argent-bridge-worker.js";
import { argentRunFailureDetail } from "../../dist/capture/argent-bridge.js";
import { runBridgeCommand } from "../../dist/capture/argent-driver.js";
import { ARGENT_VERSION } from "../../dist/shared/argent-version.js";
const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
@@ -133,3 +137,48 @@ test("a recording runs per command, and a screenshot is taken only from a file t
assert.match(failed.error.message, /did not write/);
await assert.rejects(readFile(join(root, "out", "two.png")), { code: "ENOENT" });
});
test("the thread's lines about a tool-server still starting reach the capture's log with the command's answer", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
// Argent's client asks GET /tools before each call; the first one is answered 404, as on euxy's main baseline run 39506374824.
const clientModule = await fakeClient(root, `
if (name === "list-devices" && calls.filter(call => call.name === "list-devices").length === 1) throw new Error("GET /tools failed: 404 Not Found");
if (name === "list-devices") return { data: { devices: [{ udid: "UDID-1", name: "iPhone 17", state: "Booted", platform: "ios", runtime: "iOS 26.0" }] } };
if (name === "await-screen-idle") return { data: { settled: true } };
throw new Error("unexpected tool " + name);`);
const messages = [];
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/tools", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule, progress: message => messages.push(message),
fallback: () => assert.fail("the thread serves every command") });
t.after(() => bridge.close());
const waited = JSON.parse(bridge.runner({ command: "/opt/argent-controller", args: ["wait", "stable", "--session", "explore", "--platform", "ios", "--json"], cwd: root, environment, timeoutMs: 20_000 }).stdout);
assert.deepEqual(waited, { success: true, data: { stable: true } });
assert.deepEqual(messages, ["Argent's tool-server is not serving its tools yet (argent run list-devices failed: GET /tools failed: 404 Not Found); sending list-devices again in 1 s (75 s left of 75 s)"]);
});
test("a refused connection to a localhost tool-server, whose cause is an AggregateError with no message, is the tool-server still starting", async () => {
// A port nothing listens on: bound, then closed.
const port = await new Promise(resolve => { const server = createServer().listen(0, "127.0.0.1", () => { const { port } = server.address(); server.close(() => resolve(port)); }); });
const client = { async callTool() { const response = await fetch(`http://localhost:${port}/tools`); return { data: await response.json() }; } };
const call = argentClientCall(client, () => new AbortController().signal);
const refused = await call("list-devices", {}).then(() => assert.fail("nothing listens"), error => error);
assert.match(refused.message, /^argent run list-devices failed: fetch failed: .*ECONNREFUSED/);
// Through the bridge, the refusal is waited out like a 404 from /tools, until the window ends.
let now = 0, tries = 0;
const result = await runBridgeCommand(["snapshot", "--platform", "ios", "--json"], { state: { generation: 0 }, platform: "ios", version: "0.29.0",
now: () => now, sleep: async milliseconds => { now += milliseconds; }, reachable: async () => true, moveFile: async () => {},
call: async (tool, args, options) => { tries++; return call(tool, args, options); } });
assert.equal(JSON.parse(result.stdout).error.code, "TOOL_SERVER_NOT_READY");
assert.equal(tries, 11);
});
test("a failed argent run keeps the line saying /tools is not served ahead of the stack trace that follows it", () => {
const stderr = ["[argent] Unhandled rejection: Error: GET /tools failed: 404 Not Found", " at fetchTools (file:///argent/dist/cli-cmds.mjs:10065:21)",
" at async fetchTool (file:///argent/dist/cli-cmds.mjs:10070:19)", " at async run (file:///argent/dist/cli-cmds.mjs:31166:17)",
" at async main (file:///argent/dist/cli.js:12:3)"].join("\n");
const detail = argentRunFailureDetail(stderr, "");
assert.ok(detail.startsWith("[argent] Unhandled rejection: Error: GET /tools failed: 404 Not Found | at fetchTools "), detail);
assert.ok(detail.endsWith("at async main (file:///argent/dist/cli.js:12:3)"), "the last frames follow it");
assert.equal(argentRunFailureDetail("Tool \"gesture-tap\" not found. Run `argent tools` to list available tools.\n", ""), "Tool \"gesture-tap\" not found. Run `argent tools` to list available tools.");
assert.equal(argentRunFailureDetail("GET /tools failed: 503 Service Unavailable\n", ""), "GET /tools failed: 503 Service Unavailable", "a line named once");
});
+81
View File
@@ -302,6 +302,87 @@ test("an unreachable tool-server before a UI action is the retryable 'Remote dae
assert.ok(!calls.some(call => call.tool === "gesture-tap"));
});
// brentvatne/euxy's main baseline run 39506374824: a replacement session's tool-server answered 404 to /tools for the
// first list-devices of its install, before it served its tools.
const TOOLS_404 = "argent run list-devices failed: GET /tools failed: 404 Not Found";
/** A clock the harness's pauses (and, given `requestMs`, its requests) move forward. */
function withClock(context, requestMs = () => 0) {
const clock = { now: 0, pauses: [], timeouts: [] };
context.now = () => clock.now;
context.sleep = async milliseconds => { clock.pauses.push(milliseconds); clock.now += milliseconds; };
const call = context.call;
context.call = async (tool, args, options) => {
if (options?.timeoutMs !== undefined) clock.timeouts.push(options.timeoutMs);
clock.now += Math.min(requestMs(tool), options?.timeoutMs ?? Infinity);
return call(tool, args, options);
};
return clock;
}
test("a tool-server that does not serve /tools yet is asked again, with a log line, until it does", async () => {
const { run, calls, context } = harness("ios", { "list-devices": [new ArgentCallError(TOOLS_404, true), new ArgentCallError(TOOLS_404, true), IOS_DEVICES], "reinstall-app": {} }, { bundleId: BUNDLE });
const clock = withClock(context);
const lines = [];
context.progress = line => lines.push(line);
assert.deepEqual((await run(["install", "/work/bundle/euxy.app", ...flags("ios")])).json.data, { installed: true, appBundleId: BUNDLE });
assert.equal(calls.filter(call => call.tool === "list-devices").length, 3);
assert.deepEqual(clock.pauses, [1_000, 2_000]);
assert.deepEqual(lines, [
"Argent's tool-server is not serving its tools yet (argent run list-devices failed: GET /tools failed: 404 Not Found); sending list-devices again in 1 s (75 s left of 75 s)",
"Argent's tool-server is not serving its tools yet (argent run list-devices failed: GET /tools failed: 404 Not Found); sending list-devices again in 2 s (74 s left of 75 s)"]);
assert.equal(context.state.toolsServed, true);
assert.deepEqual(clock.timeouts, [75_000, 74_000, 72_000], "each try gets only what is left of the window");
assert.equal(calls.filter(call => call.tool === "reinstall-app")[0].options, undefined, "a served tool-server's calls keep their own timeout");
});
test("a refused connection or a 503 from /tools is the tool-server still starting too", async () => {
for (const failure of ["fetch failed: connect ECONNREFUSED 127.0.0.1:9", "GET /tools failed: 503 Service Unavailable"]) {
const starting = new ArgentCallError(`argent run list-devices failed: ${failure}`, true);
const { run, calls, context } = harness("ios", { "list-devices": [starting, starting, starting, IOS_DEVICES], "ui-tree": IOS_HOME });
withClock(context);
assert.equal((await run(["snapshot", ...flags("ios")])).exitCode, 0, failure);
assert.equal(calls.filter(call => call.tool === "list-devices").length, 4, "asked past the two read retries");
}
});
test("a tool-server that never serves /tools fails with a clear error 75 s after the first try", async () => {
const { run, calls, context } = harness("ios", { "list-devices": new ArgentCallError(TOOLS_404, true), "reinstall-app": {} }, { bundleId: BUNDLE });
const clock = withClock(context);
const failed = await run(["install", "/work/bundle/euxy.app", ...flags("ios")]);
assert.deepEqual(failed.json.error, { code: "TOOL_SERVER_NOT_READY", message: `Argent's tool-server did not start serving its tools within 75 s: ${TOOLS_404}` });
assert.equal(clock.now, 75_000);
assert.equal(calls.filter(call => call.tool === "list-devices").length, 11);
assert.ok(!calls.some(call => call.tool === "reinstall-app"));
});
test("slow answers count against the 75 s too: each try is cut to the time left, and the wait ends on time", async () => {
const slow = new ArgentCallError("argent run list-devices failed: GET /tools failed: 503 Service Unavailable", true);
const { run, calls, context } = harness("ios", { "list-devices": slow, "reinstall-app": {} }, { bundleId: BUNDLE });
// Each 503 takes 30 s to come back, as a gateway's might.
const clock = withClock(context, () => 30_000);
assert.equal((await run(["install", "/work/bundle/euxy.app", ...flags("ios")])).json.error.code, "TOOL_SERVER_NOT_READY");
assert.equal(clock.now, 75_000, "not eleven 30 s tries");
assert.deepEqual(clock.timeouts, [75_000, 44_000, 12_000]);
assert.equal(calls.filter(call => call.tool === "list-devices").length, 3);
});
test("an unknown tool's 404 after the tool-server is up fails at once, and a later /tools 404 keeps only the read retries", async () => {
const unknown = 'argent run gesture-tap failed: Tool "gesture-tap" not found';
const { run, calls, context } = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": IOS_HOME, "gesture-tap": new ArgentCallError(unknown, true) });
const pauses = [];
context.sleep = async milliseconds => { pauses.push(milliseconds); };
assert.deepEqual((await run(["press", "10", "10", ...flags("ios")])).json.error, { code: "ARGENT_TOOL_FAILED", message: unknown });
assert.equal(calls.filter(call => call.tool === "gesture-tap").length, 1);
assert.deepEqual(pauses, []);
const served = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": new ArgentCallError("argent run ui-tree failed: GET /tools failed: 404 Not Found", true) },
{ state: { generation: 0, udid: UDID, toolsServed: true } });
served.context.sleep = async milliseconds => { pauses.push(milliseconds); };
assert.equal((await served.run(["snapshot", ...flags("ios")])).json.error.code, "ARGENT_TOOL_FAILED");
assert.equal(served.calls.filter(call => call.tool === "ui-tree").length, 3);
assert.deepEqual(pauses, [1_000, 2_000]);
});
test("a UI action whose answer is lost is reported once and never retried", async () => {
const { run, calls } = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": IOS_HOME, "gesture-tap": new ArgentCallError("argent run gesture-tap failed: socket hang up", true) });
const pressed = await run(["press", "10", "10", ...flags("ios")]);
+68 -8
View File
@@ -9,7 +9,9 @@ import { AGENT_DEVICE_VERSION, EAS_CLI_VERSION } from "../../dist/capture/run.js
import { readExportedMap } from "../../dist/review/evidence-assessment.js";
import { ExploreStore, recordObservation } from "../../dist/map/store.js";
const { captureRecipeConfig, restoreCaptureArtifactCache, writeCaptureArtifactCache } = await tsImport("../../src/ci/artifact-cache.ts", import.meta.url);
const { captureRecipeConfig, incompleteCaptureReason, restoreCaptureArtifactCache, validateEvidence, writeCaptureArtifactCache } = await tsImport("../../src/ci/artifact-cache.ts", import.meta.url);
const { mergeCaptureExports } = await tsImport("../../src/ci/export-merge.ts", import.meta.url);
const { captureHealthIssues, partitionCaptureHealthIssues } = await tsImport("../../src/capture/health.ts", import.meta.url);
const { readCaptureBundle } = await tsImport("../../src/review/capture-review.ts", import.meta.url);
const { CAPTURE_OBSERVER_HASH } = await tsImport("../../src/capture/observer.ts", import.meta.url);
const { AGENT_ACCEPTANCE_POLICY } = await tsImport("../../src/contracts/capture.ts", import.meta.url);
@@ -36,7 +38,7 @@ function acceptedFrames(bundle) {
}])) };
}
async function fixture(t, { missingAccessibility = false, partial = false } = {}) {
async function fixture(t, { missingAccessibility = false, partial = false, appearances = false, darkFrames = appearances } = {}) {
const root = await mkdtemp(join(tmpdir(), "explore-capture-artifact-cache-"));
t.after(() => rm(root, { recursive: true, force: true }));
const store = new ExploreStore(join(root, "active"));
@@ -47,7 +49,7 @@ async function fixture(t, { missingAccessibility = false, partial = false } = {}
id: "capture-run", name: "Cache evidence", platform: "ios", controller: "agent-device", status: "draft",
createdAt: timestamp, updatedAt: timestamp,
build: { id: "eas-build", gitCommit: COMMIT, source: "eas" },
environment: { ...RUNTIME, authState: "logged-out" },
environment: { ...RUNTIME, authState: "logged-out", ...(appearances ? { colorSchemes: ["light", "dark"] } : {}) },
limits: { maxScreens: 20, maxActions: 100, maxDurationMinutes: 60 },
},
screens: [], variants: [], observations: [], elements: [], artifacts: [],
@@ -58,7 +60,7 @@ async function fixture(t, { missingAccessibility = false, partial = false } = {}
const snapshotPath = join(root, "sample.json");
await writeFile(screenshotPath, PNG);
const images = {};
for (const id of ["home", "settings"]) {
for (const id of darkFrames ? ["home", "settings", "home@dark", "settings@dark"] : ["home", "settings"]) {
const snapshotText = JSON.stringify({ nodes: [{ type: "text", label: id }] });
await writeFile(snapshotPath, snapshotText);
const result = await recordObservation({
@@ -81,15 +83,17 @@ async function fixture(t, { missingAccessibility = false, partial = false } = {}
await store.exportTo(sourceDirectory, map);
const bundle = {
contract: "explore.bundle", side: "head", gitCommit: COMMIT,
easBuildId: "eas-build", easProjectId: "eas-project", profile: RECIPE.profile, runtimeProfile: RUNTIME,
easBuildId: "eas-build", easProjectId: "eas-project", profile: RECIPE.profile, runtimeProfile: appearances ? { ...RUNTIME, colorSchemes: "light,dark" } : RUNTIME,
selectedSurfaceIds: ["home", "settings"], images,
outcomes: Object.fromEntries(Object.entries(images).map(([id, image]) => [id, { status: "captured", attempts: 1, landing: "verified", screenshotSha256: image.sha256 }])),
// A dark frame verified beside its light one has no outcome of its own, as the passes merge them.
outcomes: Object.fromEntries(Object.entries(images).filter(([id]) => !id.endsWith("@dark"))
.map(([id, image]) => [id, { status: "captured", attempts: 1, landing: "verified", screenshotSha256: image.sha256 }])),
};
await writeFile(join(sourceDirectory, "bundle.json"), JSON.stringify(bundle));
await writeFile(join(sourceDirectory, "execution.json"), JSON.stringify(EXECUTION));
// Construct actual exported, validated evidence; this fixture cannot pass with stub JSON maps.
assert.equal((await readExportedMap(sourceDirectory)).observations.length, 2);
const options = { ...RECIPE, sourceDirectory, directory: join(root, "restored"), commit: COMMIT, selectedIds: ["home", "settings"], runtimeProfile: RUNTIME };
assert.equal((await readExportedMap(sourceDirectory)).observations.length, darkFrames ? 4 : 2);
const options = { ...RECIPE, sourceDirectory, directory: join(root, "restored"), commit: COMMIT, selectedIds: ["home", "settings"], runtimeProfile: bundle.runtimeProfile };
return { root, sourceDirectory, map, bundle, options };
}
@@ -302,6 +306,62 @@ test("partial captures reuse only the screens with complete successful evidence"
assert.equal(await restoreCaptureArtifactCache({ ...f.options, directory: join(f.root, "failed-only"), selectedIds: ["settings"] }), false);
});
test("a baseline whose dark session failed is cached, and a review captures the routes without a dark frame itself", async t => {
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route and the dark session failed.
const failedDark = { status: "failed", attempts: 0, landing: "unverified", reason: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
error: { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found", nextAction: "Rerun the capture.", retryable: false } };
const f = await fixture(t, { partial: true, appearances: true });
const complete = structuredClone(f.bundle);
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), true);
assert.equal(await restoreCaptureArtifactCache(f.options), true, "a baseline with every appearance is reused whole");
assert.deepEqual(Object.keys((await readCaptureBundle(f.options.directory)).images).sort(), ["home", "home@dark", "settings", "settings@dark"]);
for (const id of ["home", "settings"]) { delete f.bundle.images[`${id}@dark`]; f.bundle.outcomes[`${id}@dark`] = failedDark; }
f.bundle.sessionFailures = [{ scheme: "dark", error: failedDark.error }];
await writeFile(join(f.sourceDirectory, "bundle.json"), JSON.stringify(f.bundle));
// No route has its dark frame: offering it would make a review wait for it and capture its sides in turn, only to
// capture every route of the base itself.
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), false, "a baseline with nothing to reuse is not offered");
assert.equal(await restoreCaptureArtifactCache({ ...f.options, directory: join(f.root, "no-dark") }), false);
assert.equal(await incompleteCaptureReason(f.sourceDirectory, COMMIT), "its dark session failed, so no route has every appearance");
assert.equal(await incompleteCaptureReason(f.sourceDirectory, "b".repeat(40)), undefined, "another commit's baseline is the usual miss");
const mixed = structuredClone(complete);
delete mixed.images["settings@dark"]; mixed.outcomes["settings@dark"] = failedDark; delete mixed.sessionFailures;
await writeFile(join(f.sourceDirectory, "bundle.json"), JSON.stringify(mixed));
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), true);
assert.equal(await restoreCaptureArtifactCache({ ...f.options, directory: join(f.root, "mixed") }), true);
const restored = await readCaptureBundle(join(f.root, "mixed"));
assert.deepEqual(restored.selectedSurfaceIds, ["home"], "settings has no dark frame, so the review's base captures it in both appearances");
assert.deepEqual(Object.keys(restored.images).sort(), ["home", "home@dark"]);
assert.deepEqual(Object.keys(restored.outcomes), ["home"]);
});
test("a light session's real export merged with a failed dark session is valid evidence, a valid map, and healthy but for dark", async t => {
// The light session of a side split over sessions: it renders light and dark, and took light alone.
const f = await fixture(t, { appearances: true, darkFrames: false });
const error = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture.", retryable: true };
// The light session lost /settings.
delete f.bundle.images.settings;
f.bundle.outcomes.settings = { status: "failed", attempts: 2, landing: "unverified", reason: "The route observer never answered." };
await writeFile(join(f.sourceDirectory, "bundle.json"), JSON.stringify(f.bundle));
await mergeCaptureExports(f.sourceDirectory, [{ scheme: "dark", error }]);
const map = await readExportedMap(f.sourceDirectory);
assert.equal(map.run.status, "partial");
assert.equal(map.run.stopReason, `The dark session failed (CAPTURE_SESSION_FAILED): ${error.message}.`);
const bundle = await readCaptureBundle(f.sourceDirectory);
assert.deepEqual(Object.keys(bundle.outcomes).sort(), ["home", "home@dark", "settings", "settings@dark"]);
// Each dark frame carries the dark session's reason, the one light lost too: the review shows it, not light's.
assert.equal(bundle.outcomes["settings@dark"].reason, error.message);
assert.equal(bundle.outcomes.settings.reason, "The route observer never answered.");
await validateEvidence(f.sourceDirectory, bundle);
const issues = partitionCaptureHealthIssues(captureHealthIssues(bundle, ["home", "settings"]));
assert.deepEqual(issues.integrity, []);
assert.deepEqual(issues.unavailable.map(issue => [issue.surfaceId, issue.code]), [["home@dark", "CAPTURE_SESSION_FAILED"], ["settings", "CAPTURE_FAILED"]]);
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), false, "valid, but nothing in it a review could reuse");
});
test("partial caches retain verified targets without restoring captured but unverified screenshots", async t => {
const f = await fixture(t, { partial: true });
f.bundle.outcomes.settings.landing = "unverified";
+47
View File
@@ -562,6 +562,36 @@ test('an app that follows the system appearance is captured light then dark, dar
assert.equal(execution.colorScheme, 'light');
});
test('a single session whose dark pass fails keeps its light captures and marks every dark frame failed, as split sessions do', async t => {
// brentvatne/euxy's main baseline run 39506374824 split the appearances over two sessions; one session must read the same.
const f = await fixture(t, { userInterfaceStyle: 'automatic' });
const lines = [];
const result = await runCiMap({ ...f.options, progress: line => lines.push(line), runner: request => {
if (request.args[0] === 'settings' && request.args[2] === 'dark') throw new Error('Controller settings exited with status 1: the device stopped answering');
return f.options.runner(request);
} });
assert.equal(result.runStatus, 'partial');
const bundle = JSON.parse(await readFile(join(f.options.outputDirectory, 'bundle.json'), 'utf8'));
assert.deepEqual(Object.keys(bundle.images).sort(), [f.ids.home, f.ids.settings].sort());
for (const id of Object.values(f.ids)) {
assert.equal(bundle.outcomes[id].status, 'captured');
assert.equal(bundle.outcomes[`${id}@dark`].status, 'failed');
assert.match(bundle.outcomes[`${id}@dark`].error.message, /the device stopped answering/);
}
assert.deepEqual(bundle.sessionFailures.map(failure => failure.scheme), ['dark']);
const map = JSON.parse(await readFile(join(f.options.outputDirectory, 'screen-map.json'), 'utf8'));
assert.equal(map.run.status, 'partial');
assert.match(map.run.stopReason, /^The dark pass failed \(/);
assert.ok(lines.some(line => /the dark pass failed .*keeping the light captures/.test(line)), lines.join('\n'));
assert.equal(f.operations().filter(operation => operation === 'simulator:stop').length, 1);
// A checkout that changed under the dark pass casts doubt on the light frames too: the capture fails.
const g = await fixture(t, { userInterfaceStyle: 'automatic' });
await assert.rejects(runCiMap({ ...g.options, runner: request => {
if (request.args[0] === 'settings' && request.args[2] === 'dark') throw new CiError({ code: 'SOURCE_CHECKOUT_CHANGED', stage: 'source', message: 'The application checkout differs from its recorded commit.', nextAction: 'Rerun.' });
return g.options.runner(request);
} }), error => error.code === 'SOURCE_CHECKOUT_CHANGED');
});
test('an app pinned to one appearance is captured once, in that appearance', async t => {
const f = await fixture(t, { userInterfaceStyle: 'dark' });
const result = await runCiMap(f.options);
@@ -780,6 +810,15 @@ test('a start that fails before naming a session stops any session of that name
assert.equal(always.requests.filter(request => request.args[0] === 'simulator:stop' && value(request, '--id') === 'orphan-1').length, 6);
});
test('a session a failed start left that does not stop is reported as left running, not as a start failure', async t => {
// No session file names it, so a split side would otherwise keep its other appearance and hide it.
const f = await fixture(t, { startsUnnamed: 1, orphanFromFailedStart: true });
await assert.rejects(runCiMap({ ...f.options, sleep: () => {}, runner: request => {
if (request.args[0] === 'simulator:stop' && value(request, '--id') === 'orphan-1') throw new Error('Controller simulator:stop exited with status 1: network unreachable');
return f.options.runner(request);
} }), error => error.code === 'SESSION_CLEANUP_FAILED' && /orphan-1/.test(error.message));
});
test('a session of that name Explore cannot prove is its own is never stopped or replaced', async t => {
const f = await fixture(t, { startsUnnamed: 1, orphanFromFailedStart: true, orphanUnproven: true });
await assert.rejects(runCiMap({ ...f.options, sleep: () => {} }), error => error.code === 'SIMULATOR_START_FAILED');
@@ -1943,6 +1982,14 @@ test('a failed hosted repack job stops the capture as EAS_REPACK_FAILED with the
assert.deepEqual(f.operations().filter(operation => operation.startsWith('simulator:')).filter(operation => operation !== 'simulator:availability'), ['simulator:start', 'simulator:stop']);
});
test('a session started beside a repack that failed, and that does not stop, is the failure reported', async t => {
const f = await fixture(t, { workflowFailure: true });
await assert.rejects(runCiMap({ ...f.options, runner: request => {
if (request.args[0] === 'simulator:stop') throw new Error('Controller simulator:stop exited with status 1: network unreachable');
return f.options.runner(request);
} }), error => error.code === 'SESSION_CLEANUP_FAILED' && error.details.originalFailure.code === 'EAS_REPACK_FAILED');
});
const runs = f => f.operations().filter(operation => operation === 'workflow:run').length;
const sessionOperations = f => f.operations().filter(operation => /^simulator:(start|stop)$/.test(operation));
// The upload step of brentvatne/euxy#176 (review run 39506405687), refused once the build had repacked.
+112 -1
View File
@@ -6,6 +6,8 @@ import test from "node:test";
import { mergeSchemeOutcomes } from "../../dist/capture/appearance.js";
import { mergeCaptureExports } from "../../dist/ci/export-merge.js";
import { captureSideInSessions } from "../../dist/ci/pipeline.js";
import { CiError } from "../../dist/shared/ci-errors.js";
async function writeExport(root, { screens, images, outcomes, launch, diagnostics }) {
await mkdir(join(root, "artifacts"), { recursive: true });
@@ -20,7 +22,8 @@ async function writeExport(root, { screens, images, outcomes, launch, diagnostic
screens: screens.map(id => ({ id, colorScheme: id.endsWith("@dark") ? "dark" : "light" })), variants: [], observations: screens.map(id => ({ screenId: id })),
elements: [], actionAttempts: [], transitions: [], journeys: [], frontier: [], gaps: [] };
await writeFile(join(root, "screen-map.json"), JSON.stringify(map));
await writeFile(join(root, "bundle.json"), JSON.stringify({ contract: "explore.bundle", side: "head", runtimeProfile: { colorSchemes: "light,dark" }, images, outcomes, ...(launch ? { launch } : {}) }));
await writeFile(join(root, "bundle.json"), JSON.stringify({ contract: "explore.bundle", side: "head", runtimeProfile: { colorSchemes: "light,dark" },
selectedSurfaceIds: [...new Set(Object.keys(outcomes).map(id => id.replace(/@(light|dark)$/, "")))], images, outcomes, ...(launch ? { launch } : {}) }));
await writeFile(join(root, "manifest.json"), JSON.stringify({ contract: "explore.map", runId: "run", revision: 1, files: artifacts.map(({ id: _id, ...file }) => file) }));
await writeFile(join(root, "capture-diagnostics", "controller-timing.jsonl"), diagnostics);
return root;
@@ -94,3 +97,111 @@ test("a dark session that could not show dark (Argent without appearance.set) ad
await mergeCaptureExports(primary, [{ directory: gapped, scheme: "dark" }]);
assert.equal(JSON.parse(await readFile(mapPath, "utf8")).run.status, "partial", "a complete run has no open gaps");
});
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route, then the dark session's install failed.
const DARK_FAILURE = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture on a new session.", retryable: true };
const failedDark = { status: "failed", attempts: 0, landing: "unverified", reason: DARK_FAILURE.message, error: DARK_FAILURE };
test("a dark session that left no export records its failure as every route's own dark outcome, and the run is partial", async () => {
const root = await mkdtemp(join(tmpdir(), "explore-merge-"));
const lost = { status: "failed", landing: "unverified", attempts: 3, reason: "The route observer never answered." };
const primary = await writeExport(join(root, "head-export"), { screens: ["home"], images: { home: { path: "artifacts/home.png" } },
outcomes: { home: { status: "captured", landing: "verified", attempts: 1 }, settings: lost }, diagnostics: "" });
const mapPath = join(primary, "screen-map.json");
await writeFile(mapPath, JSON.stringify({ ...JSON.parse(await readFile(mapPath, "utf8")), run: { id: "run", status: "complete", environment: { colorSchemes: ["light", "dark"] } } }));
await mergeCaptureExports(primary, [{ scheme: "dark", error: DARK_FAILURE }]);
const bundle = JSON.parse(await readFile(join(primary, "bundle.json"), "utf8"));
// A route the light session lost keeps its own reason, and its dark frame carries the dark session's, not the light one's.
assert.deepEqual(bundle.outcomes, { home: { status: "captured", landing: "verified", attempts: 1 }, settings: lost, "home@dark": failedDark, "settings@dark": failedDark });
assert.deepEqual(bundle.sessionFailures, [{ scheme: "dark", error: DARK_FAILURE }]);
assert.deepEqual(Object.keys(bundle.images), ["home"]);
const map = JSON.parse(await readFile(mapPath, "utf8"));
assert.equal(map.run.status, "partial");
assert.equal(map.run.stopReason, `The dark session failed (CAPTURE_SESSION_FAILED): ${DARK_FAILURE.message}.`);
});
test("a dark session that failed beside a primary that renders no dark has no frames to fail, and stays on the bundle", async () => {
const root = await mkdtemp(join(tmpdir(), "explore-merge-"));
const primary = await writeExport(join(root, "head-export"), { screens: ["home"], images: { home: { path: "artifacts/home.png" } },
outcomes: { home: { status: "captured", landing: "verified", attempts: 1 } }, diagnostics: "" });
const bundlePath = join(primary, "bundle.json");
// The light session could not switch appearance (Argent without appearance.set): it captured light alone.
await writeFile(bundlePath, JSON.stringify({ ...JSON.parse(await readFile(bundlePath, "utf8")), runtimeProfile: { colorScheme: "light" } }));
await mergeCaptureExports(primary, [{ scheme: "dark", error: DARK_FAILURE }]);
const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
assert.deepEqual(Object.keys(bundle.outcomes), ["home"]);
assert.deepEqual(bundle.sessionFailures, [{ scheme: "dark", error: DARK_FAILURE }], "the comment still names the failed session");
assert.equal(JSON.parse(await readFile(join(primary, "screen-map.json"), "utf8")).run.status, "partial");
});
/** The side's sessions as captureSideInSessions runs them: the light one exports, the dark one does what `dark` says. */
async function sideInSessions({ light, dark, leftSessionFile = false, environment = {}, signal } = {}) {
const root = await mkdtemp(join(tmpdir(), "explore-sessions-"));
const progress = [];
const mapOptions = { side: "head", outputDirectory: join(root, "head-export"), mapDirectory: join(root, "head-map"), buildShare: join(root, "builds"),
sessionStatePath: join(root, "head-session"), expoToken: "expo-secret-token", environment };
const runPass = async options => {
const scheme = options.appearancePass.capture[0];
const outcome = scheme === "light" ? light : dark;
// A capture removes its session file only once its session is confirmed stopped.
if (scheme === "dark" && leftSessionFile) await writeFile(options.sessionStatePath, "01a15b61-4df9\n");
if (outcome instanceof Error) throw outcome;
const id = scheme === "light" ? "home" : "home@dark";
await writeExport(options.outputDirectory, { screens: [id], images: { [id]: { path: `artifacts/${id}.png` } },
outcomes: { [id]: { status: "captured", landing: "verified", attempts: 1 } }, diagnostics: "" });
return { runStatus: "complete", outputDirectory: options.outputDirectory, scheme };
};
const run = () => captureSideInSessions(mapOptions, [["light"], ["dark"]], line => progress.push(line), signal, true, runPass);
return { root, progress, run, bundle: async () => JSON.parse(await readFile(join(mapOptions.outputDirectory, "bundle.json"), "utf8")) };
}
test("a side keeps its light captures when only its dark session fails, and the report marks every dark frame failed", async () => {
const side = await sideInSessions({ dark: new CiError({ ...DARK_FAILURE, message: `${DARK_FAILURE.message} (tokens expo-secret-token, gh-secret)` }),
environment: { GITHUB_TOKEN: "gh-secret", PATH: "/usr/bin" } });
const result = await side.run();
assert.equal(result.scheme, "light", "the primary session's result");
assert.equal(result.runStatus, "partial");
const bundle = await side.bundle();
assert.deepEqual(Object.keys(bundle.images), ["home"]);
assert.equal(bundle.outcomes["home@dark"].status, "failed");
assert.equal(bundle.outcomes["home@dark"].error.code, "CAPTURE_SESSION_FAILED");
assert.doesNotMatch(JSON.stringify(bundle), /expo-secret-token|gh-secret/, "the session's error is stored without any secret the environment holds");
assert.ok(side.progress.some(line => /^head: the dark session failed \(CAPTURE_SESSION_FAILED: install: argent run list-devices failed/.test(line)
&& line.endsWith("keeping the light captures, and the report marks every dark frame failed")), side.progress.join("\n"));
});
test("a side whose light session fails still fails, whatever the dark session did", async () => {
const light = new CiError({ ...DARK_FAILURE, code: "CAPTURE_INSTALL_FAILED" });
await assert.rejects((await sideInSessions({ light })).run(), { code: "CAPTURE_INSTALL_FAILED" });
await assert.rejects((await sideInSessions({ light, dark: new CiError({ ...DARK_FAILURE }) })).run(), { code: "CAPTURE_INSTALL_FAILED" });
});
test("a dark session not proven stopped fails the side, so a running session is never hidden behind a partial report", async () => {
const cleanup = new CiError({ code: "SESSION_CLEANUP_FAILED", stage: "cleanup", message: "The exact EAS session could not be confirmed stopped.",
nextAction: "Stop EAS Simulator session 01a15b61 and verify its status.", retryable: false });
await assert.rejects((await sideInSessions({ dark: cleanup })).run(), { code: "SESSION_CLEANUP_FAILED" });
await assert.rejects((await sideInSessions({ light: new CiError({ ...DARK_FAILURE, code: "CAPTURE_INSTALL_FAILED" }), dark: cleanup })).run(), { code: "SESSION_CLEANUP_FAILED" },
"the leaked session is reported before the light session's own failure");
const handoff = new CiError({ code: "SESSION_HANDOFF_UNVERIFIED", stage: "session", message: "EAS did not provide an exact verifiable simulator session.",
nextAction: "Verify a session's exact ID and ownership before cleanup." });
await assert.rejects((await sideInSessions({ dark: handoff })).run(), { code: "SESSION_HANDOFF_UNVERIFIED" });
const unverified = new CiError({ code: "SIMULATOR_START_FAILED", stage: "session", message: "EAS simulator:start failed before a session identity could be verified.",
nextAction: "Verify a session's exact ID and ownership before cleanup.", details: { allocation: "unknown" } });
await assert.rejects((await sideInSessions({ dark: unverified })).run(), { code: "SIMULATOR_START_FAILED" });
// A worker that died before its capture's own cleanup ran: it exited, or failed with a raw error.
const exited = new CiError({ code: "CAPTURE_WORKER_EXITED", stage: "capture-head", message: "The capture worker for the head side exited with code 1 before it reported a result.",
nextAction: "Inspect the job log for the worker's failure, then rerun Explore." });
await assert.rejects((await sideInSessions({ dark: exited })).run(), { code: "CAPTURE_WORKER_EXITED" });
await assert.rejects((await sideInSessions({ dark: new Error("Worker terminated due to reaching memory limit: JS heap out of memory") })).run(), /memory limit/);
// Any failure that left the session's file behind: the session was never confirmed stopped.
await assert.rejects((await sideInSessions({ dark: new CiError({ ...DARK_FAILURE }), leftSessionFile: true })).run(), { code: "CAPTURE_SESSION_FAILED" });
});
test("a dark session whose checkout changed, whose evidence failed its checks, or whose review was stopped fails the side", async () => {
for (const code of ["SOURCE_CHECKOUT_CHANGED", "CHECKOUT_MODIFIED", "CAPTURE_INCOMPLETE", "CAPTURE_IMAGE_BINDING_INVALID", "EXPLORE_FAILED"]) {
await assert.rejects((await sideInSessions({ dark: new CiError({ ...DARK_FAILURE, code }) })).run(), { code }, code);
}
const stop = new AbortController(); stop.abort(new CiError({ code: "LOCAL_REVIEW_INTERRUPTED", stage: "capture", message: "Interrupted.", nextAction: "Rerun." }));
await assert.rejects((await sideInSessions({ dark: new CiError({ ...DARK_FAILURE }), signal: stop.signal })).run(), { code: "CAPTURE_SESSION_FAILED" });
});
+14 -1
View File
@@ -5,7 +5,7 @@ import { spawn, spawnSync } from 'node:child_process';
import { lstat, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { appEnvironment, captureCacheDirectory, defaultBase, installDependencies, keepCapture, localInstallCommand, localReviewConfig, reportRef, reviewHeadSource, runLocalReview, snapshotWorkingTree, stopRecordedSessions } from '../../dist/ci/local-review.js';
import { appEnvironment, captureCacheDirectory, defaultBase, installDependencies, keepCapture, localInstallCommand, localReviewConfig, reportRef, pruneTargetOutput, reviewHeadSource, runLocalReview, snapshotWorkingTree, stopRecordedSessions } from '../../dist/ci/local-review.js';
import { mkdir, readdir } from 'node:fs/promises';
import { startLocalReportServer } from '../../dist/report/local-report-server.js';
import { loadReport } from '../../dist/report/agent-api.js';
@@ -364,6 +364,19 @@ test('an interrupted review deletes the local simulators it recorded and stops i
await assert.rejects(lstat(join(output, 'head-session')));
});
test('a finished target keeps its report and any session file, so the review still stops a session no capture confirmed stopped', async t => {
const output = await realpath(await mkdtemp(join(tmpdir(), 'explore-pruned-')));
t.after(() => rm(output, { recursive: true, force: true }));
for (const name of ['report.zip', 'head-session-dark', 'head-export', 'base-export-dark', 'head-map']) await writeFile(join(output, name), 'x\n');
await mkdir(join(output, 'report'));
await pruneTargetOutput(output);
assert.deepEqual((await readdir(output)).sort(), ['head-session-dark', 'report', 'report.zip']);
const eas = [];
await writeFile(join(output, 'head-session-dark'), '01a15b61-4df9\n');
assert.deepEqual(await stopRecordedSessions([output], {}, args => { eas.push(args); return { status: 0 }; }, true), ['01a15b61-4df9']);
assert.deepEqual(eas, [['simulator:stop', '--id', '01a15b61-4df9', '--json', '--non-interactive']]);
});
test('a local review captures every target of the config at once, two of them on one platform', async t => {
const targets = { phone: { platform: 'ios' }, 'duo-open': { platform: 'ios', device: 'iPhone Duo', pose: 'open' } };
const f = await fixture(t, 'expo-router', { targets });
+32
View File
@@ -325,6 +325,38 @@ test('a failed dark pass on a route that captured in light reads as a valid bund
assert.equal(dark.headOutcome?.error?.code, 'CAPTURE_OBSERVER_MISSING');
});
test('a route light lost and whose dark session failed reviews dark with the dark session\'s reason, not light\'s', async t => {
// brentvatne/euxy's main baseline run 39506374824: the dark session's install failed; here light also lost the route.
const f = await captureFixture(t);
const error = { code: 'CAPTURE_SESSION_FAILED', stage: 'session', message: 'install: argent run list-devices failed: GET /tools failed: 404 Not Found',
nextAction: 'Rerun the capture on a new session.', retryable: true };
for (const side of ['base', 'head']) {
const bundle = f.bundles[side];
bundle.runtimeProfile = { ...runtimeProfile, colorSchemes: 'light,dark' };
if (side === 'base') {
await writeFile(join(f.root, side, 'home-dark.png'), png);
bundle.images[`${f.id}@dark`] = { ...bundle.images[f.id], path: 'home-dark.png' };
} else {
delete bundle.images[f.id];
bundle.outcomes[f.id] = { status: 'failed', attempts: 2, landing: 'unverified', reason: 'The route observer never answered.' };
bundle.outcomes[`${f.id}@dark`] = { status: 'failed', attempts: 0, landing: 'unverified', reason: error.message, error };
bundle.sessionFailures = [{ scheme: 'dark', error }];
}
await writeFile(join(f.root, side, 'bundle.json'), JSON.stringify(bundle));
}
await readCaptureBundle(f.options.headDirectory);
const result = await prepareCaptureReview(f.options);
assertPublicReview(result.review);
const [light, dark] = result.review.surfaces;
assert.equal(light.headOutcome.reason, 'The route observer never answered.');
assert.equal(dark.headOutcome.error.code, 'CAPTURE_SESSION_FAILED');
assert.equal(dark.headOutcome.reason, error.message);
// Without the session's record, a failed dark outcome beside a failed light one is not one the passes produce.
delete f.bundles.head.sessionFailures;
await writeFile(join(f.root, 'head', 'bundle.json'), JSON.stringify(f.bundles.head));
await assert.rejects(readCaptureBundle(f.options.headDirectory), /inventory is inconsistent/);
});
test('a head that gained a light appearance keeps its route in source navigation on the dark-only base', async t => {
const f = await captureFixture(t);
f.bundles.base.runtimeProfile = { ...runtimeProfile, colorSchemes: 'dark' };
+40
View File
@@ -268,6 +268,46 @@ test('capture problems are counted on the target\'s line and listed, collapsed,
assert.doesNotMatch(body, /needs review|Modified|old capture|old destination|token=private|\*\*iOS\*\*|Validation/);
});
test('a target whose dark session failed reads as ready, says dark was not captured, and names the session once', async () => {
const { readCommentRecords } = await import('../../dist/report/pr-comment-body.js');
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route, the dark session's install failed.
const error = { code: 'CAPTURE_SESSION_FAILED', stage: 'session', message: 'install: argent run list-devices failed: GET /tools failed: 404 Not Found',
nextAction: 'Rerun the capture on a new session.', retryable: true };
const unchanged = { review: { status: 'unchanged', description: 'No visible change' }, visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: '[email protected]' } };
const surfaces = [];
// 24 routes: more dark frames than the comment lists issues, and one light route the light session lost.
for (let index = 0; index < 24; index++) {
const id = `route-${index}`;
const light = surface(id, 'modified', { extra: { colorScheme: 'light', ...unchanged } });
if (index === 23) { delete light.headImage; delete light.review; light.headOutcome = { status: 'failed', attempts: 2, landing: 'unverified', reason: 'The route observer never answered.' }; }
const dark = surface(`${id}@dark`, 'modified', { route: `/${id}`, extra: { colorScheme: 'dark', headOutcome: { status: 'failed', attempts: 0, landing: 'unverified', reason: error.message, error } } });
delete dark.headImage;
surfaces.push(light, dark);
}
const health = { planned: 24, captured: 24, verified: 24, unverified: 0, unavailable: 0 };
const evidence = { visual: null, accessibility: null, capture: { base: health, head: { ...health, captured: 23, verified: 23, unavailable: 1 } },
comparison: { planned: 24, eligible: 23, skipped: 0 }, diagnostics: [], sessionFailures: [{ side: 'head', scheme: 'dark', frames: 24, ...error }] };
const body = renderPublicComment(state({ review: review(surfaces), evidence, capture: { planned: { base: 24, head: 24 }, done: { base: 24, head: 23 }, baseFromCache: true } }));
assert.doesNotMatch(body, /failed\*\*|capture failed|🟥/, 'one appearance missing is not a failed review');
assert.match(titleOf(body), /^### No compared screens changed · dark not captured · /);
assert.equal(statusOf(body, 'iOS'), `iOS · dark not captured · 1 capture unavailable · ${runLink}`);
assert.deepEqual(summaries(body), ['Capture issues (2)'], 'the session once, and the light route it lost');
assert.match(body, /\n- Head · dark session · CAPTURE\\_SESSION\\_FAILED: install\\: argent run list\\-devices failed\\: GET \/tools failed\\: 404 Not Found\\. 24 dark frames are unavailable\\. Next action: Rerun the capture on a new session\\\.\n/);
assert.match(body, /\n- Head · \/route\\-23 · CAPTURE\\_UNAVAILABLE: The route observer never answered\\. /, 'the light route is not crowded out');
assert.doesNotMatch(body, /\(dark\\\) · CAPTURE\\_SESSION\\_FAILED/, 'no dark frame repeats the session');
const record = readCommentRecords(body).current.ios;
assert.deepEqual(record.sessionFailures, [{ side: 'head', scheme: 'dark', frames: 24 }], 'the record keeps it, so a republished comment says it too');
});
test('a dark issue that is not the session\'s is still listed with its appearance', () => {
const error = { code: 'CAPTURE_OBSERVER_MISSING', stage: 'capture', message: 'The capture build has not exposed its active navigation destination.', nextAction: 'Rerun capture.', retryable: true };
const dark = surface('settings@dark', 'modified', { route: '/settings', extra: { colorScheme: 'dark', headOutcome: { status: 'failed', attempts: 2, landing: 'unverified', reason: error.message, error } } });
delete dark.headImage;
const body = renderPublicComment(state({ review: review([surface('settings', 'modified', { extra: { colorScheme: 'light' } }), dark]), evidence: undefined }));
assert.match(body, /\n- Head · \/settings \\\(dark\\\) · CAPTURE\\_OBSERVER\\_MISSING/);
assert.doesNotMatch(titleOf(body), /not captured/);
});
test('zero eligible pairs are counted as unattributed instead of successful comparisons', () => {
const flagged = evidenceReview();
for (const surface of flagged.surfaces) {
+17
View File
@@ -684,3 +684,20 @@ test("on a foldable a region across the pill's edge is kept whole, and one acros
assert.deepEqual([clipped.ignoredStatusBar, clipped.ignoredHomeIndicator, clipped.measurement.match, clipped.measurement.differentPixels], [0, 1, false, 80]);
assert.deepEqual(clipped.measurement.regions.map((entry) => [entry.rect.y, entry.rect.height, entry.differentPixels, entry.shareOfDiffPercentage]), [[840, 8, 80, 100]]);
});
test("a failed appearance pass is summarized once per side, with the frames it left unavailable", async (t) => {
// brentvatne/euxy's main baseline run 39506374824: the dark session's install failed after light captured every route.
const root = await mkdtemp(join(tmpdir(), "explore-public-evidence-")); t.after(() => rm(root, { recursive: true, force: true }));
const base = await side(root, "base", manifests.base);
const head = await side(root, "head", manifests.head);
const error = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture on a new session.", retryable: true };
head.bundle.runtimeProfile = { deviceModel: "iPhone 17", osVersion: "26.0", locale: "en-US", orientation: "portrait", colorScheme: "light", colorSchemes: "light,dark" };
for (const id of head.bundle.selectedSurfaceIds) head.bundle.outcomes[`${id}@dark`] = { status: "failed", attempts: 0, landing: "unverified", reason: error.message, error };
head.bundle.sessionFailures = [{ scheme: "dark", error }];
const result = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", measure: measurer({}) });
assert.deepEqual(result.summary.sessionFailures, [{ side: "head", scheme: "dark", frames: head.bundle.selectedSurfaceIds.length, ...error }]);
delete head.bundle.sessionFailures;
const complete = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", measure: measurer({}) });
assert.equal(complete.summary.sessionFailures, undefined);
});
+23
View File
@@ -740,6 +740,29 @@ test("a planned screen whose capture failed is unavailable, with the report's re
assert.equal(isChangedStatus(screen.status), false);
});
test("a dark frame whose session failed is unavailable with the session's error, and its light frame is unaffected", () => {
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route, the dark session's install failed.
const review = navigationReview();
const scan = review.surfaces.find(surface => surface.id === "scan-mood");
const error = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture on a new session.", retryable: true };
scan.colorScheme = "light";
review.surfaces.push({ id: "scan-mood@dark", status: "unchanged", colorScheme: "dark",
base: entry("scan-mood@dark", scan.base.route, "mood"), head: entry("scan-mood@dark", scan.head.route, "mood"),
baseImage: { ...scan.baseImage, url: `${scan.baseImage.url}-dark`, sha256: "f".repeat(64) }, baseOutcome: { status: "captured", attempts: 1, landing: "verified" },
headOutcome: { status: "failed", attempts: 0, landing: "unverified", reason: error.message, error } });
verifyImages(review);
const diff = diffFor(review);
const dark = diff.screens.get("scan-mood@dark");
assert.equal(dark.status, "unavailable");
assert.match(dark.captureNotes.after, /^Capture failed: install: argent run list-devices failed: GET \/tools failed: 404 Not Found/);
assert.equal(dark.afterScreenshot, undefined);
assert.notEqual(diff.screens.get("scan-mood").status, "unavailable", "the light frame keeps its own verdict");
const capture = publicReviewProjection(review).runtimeEvidence["public:scan-mood@dark"].current.groups.capture;
assert.equal(capture.find(row => row.label === "Capture error").value, `CAPTURE_SESSION_FAILED: ${error.message}`);
assert.equal(capture.find(row => row.label === "Next action").value, error.nextAction);
});
test("source navigation names routes, so a route whose only base frame is a secondary appearance stays known on base", () => {
const review = navigationReview();
const scan = review.surfaces.find(surface => surface.id === "scan-mood");