Compare commits

..
Author SHA1 Message Date
Brent Vatne 2eb266159a A hosted repack keeps to one budget, waits on its own run, and joins the run attempt's shared repack by its id
Reviews of 362d5f2c found a capture could prepare its app for over an hour, start a second workflow run beside one
eas-cli's --wait gave up on, reuse a first attempt's repack with old EAS environment values, lose a followed build
off the 30-build list, and start a repack beside the leader's retry or the other followers' own.

Preparing the app now has one 30-minute budget from the session's start, a session whose app took longer than its
20-minute allowance is replaced, and the run is started without --wait and read by its id, then cancelled when its
state stays unreadable, the budget ends or the capture is stopped (CI captures now abort on SIGINT and SIGTERM).
The share is scoped to the run attempt and keyed by the EAS environment's variables too, a followed build is read
with build:view, a failed one leaves a 60-second look for the leader's retry, later targets wait a minute longer
each so that a missing leader is replaced by one repack, and a native build a job made itself is not shared.
Failures are classified from the failed step's logs and EAS's exact words, so a registry 401, a bundling error or a
build timeout is not retried. CI-FLOW.md §4.1 states what the key does and does not protect against.
2026-10-20 16:29:25 -07:00
Brent Vatne 362d5f2c4d A failed hosted repack starts again, and a run's matrix jobs share one repack of each commit
In brentvatne/euxy#176 (review run 39506405687) EAS ended a repack's build while it saved the app, and the target
failed with EAS_REPACK_FAILED; the four iOS matrix jobs had each made the same repack of build a81d5d80.

A repack EAS failed around (an upload, spin-up or timeout) starts again up to twice, one whose run does not say why
once, and one the app's JavaScript broke, a refused token or a cancelled run never, within 20 minutes of the first.

The targets that repack one app on EAS now meet on EAS: each keys its staged upload (every file's bytes, the native
build, profile, environment, Explore release and the GitHub run), and a build whose message carries that key is used
instead of repacking again. The first target in config order repacks at once; the others wait up to 3 minutes for its
repack to appear, follow it, and repack themselves when it fails or never comes. CI-FLOW.md §4.1 documents it.
2026-10-20 15:58:41 -07:00
31 changed files with 116 additions and 904 deletions

No files matched your search

+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@expo/explore",
"version": "0.20.6",
"version": "0.20.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@expo/explore",
"version": "0.20.6",
"version": "0.20.5",
"license": "MIT",
"dependencies": {
"@expo/repack-app": "0.10.1",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@expo/explore",
"version": "0.20.6",
"version": "0.20.5",
"description": "Source-driven mobile app maps, deterministic CI capture, and visual reviews",
"license": "MIT",
"type": "module",
+1 -9
View File
@@ -1,4 +1,4 @@
import type { CaptureBundle, CaptureOutcome } from "../contracts/capture.js";
import type { CaptureOutcome } from "../contracts/capture.js";
export type ColorScheme = "light" | "dark";
/** `auto` follows the app's `userInterfaceStyle`; a list captures exactly those schemes, first one primary. */
@@ -118,11 +118,3 @@ export function appearancePairs(routeId: string, headSchemes: readonly ColorSche
baseId: baseSchemes.includes(scheme) ? schemeScreenId(routeId, scheme === baseSchemes[0] ? undefined : scheme) : null,
}));
}
/** A `<id>@<scheme>` outcome recording that its whole appearance pass failed (`sessionFailures`), whatever the primary did. */
export function failedPassOutcome(bundle: Pick<CaptureBundle, "outcomes" | "sessionFailures">, id: string): boolean {
const { scheme } = routeIdOfScreen(id);
const own = bundle.outcomes[id];
return scheme !== undefined && own?.status === "failed" && (bundle.sessionFailures ?? []).some(failure => failure.scheme === scheme
&& failure.error.code === own.error?.code && failure.error.message === own.error?.message);
}
+1 -3
View File
@@ -92,9 +92,7 @@ export function startArgentBridgeRunner(options: {
stop(`the Argent bridge thread is unavailable (${text}); running the bridge per command`);
return options.fallback(request);
}
const result = JSON.parse(text) as { exitCode: number; stdout: string; notes?: string[] };
// The thread cannot reach the capture's log while this one waits for it, so its lines come with the answer.
for (const note of result.notes ?? []) options.progress?.(note);
const result = JSON.parse(text) as { exitCode: number; stdout: string };
return commandOutcome(request, { status: result.exitCode, stdout: result.stdout, stderr: "" });
};
return { runner, get active() { return worker !== undefined; }, close: () => { if (worker) { void worker.terminate(); worker = undefined; } } };
+2 -17
View File
@@ -61,20 +61,6 @@ async function materializedFile(data: unknown): Promise<string | undefined> {
return undefined;
}
/**
* An error's message with its cause's, as Argent's client words a failed fetch: "fetch failed: connect ECONNREFUSED …".
* A host that resolves to several addresses (localhost: ::1 and 127.0.0.1) fails with an AggregateError whose message is
* empty, so the causes' codes are named too.
*/
export function failureText(error: unknown): string {
if (!(error instanceof Error)) return String(error);
const cause = error.cause;
if (!(cause instanceof Error)) return error.message;
const codes = [cause, ...(cause instanceof AggregateError ? cause.errors : [])]
.map(item => (item as { code?: unknown } | undefined)?.code).filter((code): code is string => typeof code === "string" && !cause.message.includes(code));
return [`${error.message}:`, cause.message, ...(codes.length ? [`(${[...new Set(codes)].join(", ")})`] : [])].filter(Boolean).join(" ");
}
async function loadClient(environment: NodeJS.ProcessEnv, clientModule?: string): Promise<ArgentClient> {
const module = clientModule ?? join(dirname(await ensureController("argent", { environment })), "client.js");
const loaded = await import(pathToFileURL(module).href) as { createArgentClient?: () => ArgentClient };
@@ -88,9 +74,8 @@ export function argentClientCall(client: ArgentClient, signal: () => AbortSignal
const active = signal();
if (active.aborted) throw new ArgentCallError(`argent run ${tool} was not sent: the command was given up.`, false);
let answer: { data: unknown };
const timeoutMs = Math.max(1, Math.ceil(Math.min(ARGENT_CALL_TIMEOUT_MS, options?.timeoutMs ?? ARGENT_CALL_TIMEOUT_MS)));
try { answer = await client.callTool(tool, args, { signal: AbortSignal.any([active, AbortSignal.timeout(timeoutMs)]) }); }
catch (error) { throw new ArgentCallError(`argent run ${tool} failed: ${failureText(error).slice(0, 800)}`, true); }
try { answer = await client.callTool(tool, args, { signal: AbortSignal.any([active, AbortSignal.timeout(ARGENT_CALL_TIMEOUT_MS)]) }); }
catch (error) { throw new ArgentCallError(`argent run ${tool} failed: ${(error instanceof Error ? error.message : String(error)).slice(0, 800)}`, true); }
if (!options?.out) return answer.data;
const file = await materializedFile(answer.data);
if (!file) {
+3 -20
View File
@@ -57,23 +57,9 @@ function summarize(tool: string, value: unknown): unknown {
screen: tree.screen, nodes, roots: (tree.roots ?? []).map(root => [root.type ?? null, root.bundleId ?? null, (root.children ?? []).length]) };
}
/**
* What a failed `argent run` printed, for its error: the last few lines, and ahead of them the line saying the
* tool-server is not serving its tools yet, which the bridge waits out (argent-driver's TOOLS_NOT_SERVED) and which a
* stack trace can push out of the last lines.
*/
export function argentRunFailureDetail(stderr: string, stdout: string): string {
const lines = `${stderr}\n${stdout}`.trim().split("\n").filter(Boolean);
const notServed = lines.find(line => /GET \/tools failed: \d{3}|ECONNREFUSED/.test(line));
return [...new Set([...(notServed ? [notServed] : []), ...lines.slice(-4)])].join(" | ").slice(0, 800);
}
/** A command's answer, with the lines it has for the capture's log (`notes`), which its host prints. */
export type ArgentBridgeResult = BridgeResult & { notes?: string[] };
export type ArgentBridge = {
/** Runs one agent-device-shaped command against the session. */
run(argv: readonly string[]): Promise<ArgentBridgeResult>;
run(argv: readonly string[]): Promise<BridgeResult>;
};
/**
@@ -91,10 +77,8 @@ export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentC
await appendFile(logPath, `${JSON.stringify({ at: new Date().toISOString(), host, command: argv[0], ...entry })}\n`, { mode: 0o600 }).catch(() => undefined);
};
const state = await readState(statePath);
const notes: string[] = [];
const result = await runBridgeCommand(argv, {
state, platform, version: ARGENT_VERSION,
progress: message => { notes.push(message); void log({ note: message }); },
...(environment.EXPLORE_ARGENT_BUNDLE_ID ? { bundleId: environment.EXPLORE_ARGENT_BUNDLE_ID } : {}),
...(environment.EXPLORE_ARGENT_DEVICE ? { device: environment.EXPLORE_ARGENT_DEVICE } : {}),
// An opened foldable's inner panel shows its UI in landscape (Argent's fold: "Unfolded, the UI is landscape").
@@ -117,8 +101,7 @@ export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentC
// The same URL Argent's own client checks: ARGENT_TOOLS_URL may carry a path, which /tools is appended to.
const response = await fetch(`${(toolsUrl ?? "").replace(/\/+$/, "")}/tools`, { headers: token ? { authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(5_000) });
await response.body?.cancel();
// A tool-server still starting can answer 404 to /tools (euxy main baseline run 39506374824): it is not serving yet.
ok = response.status < 500 && response.status !== 404;
ok = response.status < 500;
} catch { ok = false; }
if (!ok) await log({ tool: "reachable", ok });
return ok;
@@ -148,7 +131,7 @@ export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentC
});
await writeState(statePath, state);
await log({ result: result.exitCode === 0 ? "ok" : result.stdout.slice(0, 300) });
return notes.length ? { ...result, notes } : result;
return result;
},
};
}
+5 -5
View File
@@ -8,7 +8,7 @@
import { spawnSync } from "node:child_process";
import { stat } from "node:fs/promises";
import { ArgentCallError } from "./argent-driver.js";
import { argentBridgeSettings, argentRunFailureDetail, createArgentBridge } from "./argent-bridge.js";
import { argentBridgeSettings, createArgentBridge } from "./argent-bridge.js";
import { ensureController } from "../shared/controller-cache.js";
const ARGENT_CALL_TIMEOUT_MS = 180_000;
@@ -34,12 +34,13 @@ async function main(): Promise<number> {
return 1;
}
const callArgent = async (tool: string, args: Record<string, unknown>, options?: { out?: string; timeoutMs?: number }): Promise<unknown> => {
const callArgent = async (tool: string, args: Record<string, unknown>, options?: { out?: string }): Promise<unknown> => {
const run = spawnSync(process.execPath, [cli, "run", tool, "--args", JSON.stringify(args), ...(options?.out ? ["--out", options.out] : [])],
{ env: environment, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: Math.min(ARGENT_CALL_TIMEOUT_MS, options?.timeoutMs ?? ARGENT_CALL_TIMEOUT_MS) });
{ env: environment, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: ARGENT_CALL_TIMEOUT_MS });
if (run.error) throw new ArgentCallError(`argent run ${tool}: ${run.error.message}`, true);
if (run.status !== 0) {
throw new ArgentCallError(`argent run ${tool} failed: ${argentRunFailureDetail(run.stderr ?? "", run.stdout ?? "")}`, true);
const detail = `${run.stderr ?? ""}\n${run.stdout ?? ""}`.trim().split("\n").filter(Boolean).slice(-4).join(" | ").slice(0, 800);
throw new ArgentCallError(`argent run ${tool} failed: ${detail}`, true);
}
// With --out the client saves the tool's file and prints where ("Wrote: <path>"), not the tool's JSON.
if (options?.out) {
@@ -52,7 +53,6 @@ async function main(): Promise<number> {
};
const result = await createArgentBridge(settings, callArgent, "process").run(argv);
for (const note of result.notes ?? []) process.stderr.write(`[explore] ${note}\n`);
process.stdout.write(result.stdout);
return result.exitCode;
}
+7 -55
View File
@@ -15,8 +15,7 @@ export class ArgentCallError extends Error {
constructor(message: string, readonly sent: boolean) { super(message); }
}
/** `timeoutMs` shortens the call's own timeout, so a wait for a tool-server still starting stays within its window. */
export type ArgentCall = (tool: string, args: Record<string, unknown>, options?: { out?: string; timeoutMs?: number }) => Promise<unknown>;
export type ArgentCall = (tool: string, args: Record<string, unknown>, options?: { out?: string }) => Promise<unknown>;
type Rect = { x: number; y: number; width: number; height: number };
@@ -32,8 +31,6 @@ export type ArgentBridgeState = {
/** iOS asks "Open in “<app>”?" for the first custom-scheme link of a session; later links open directly. */
deepLinkPromptAccepted?: boolean;
recording?: string;
/** The tool-server has answered a call this session, so a GET /tools it fails is no longer taken for it still starting. */
toolsServed?: boolean;
};
export type ArgentBridgeContext = {
@@ -46,8 +43,6 @@ export type ArgentBridgeContext = {
device?: string;
version: string;
sleep: (milliseconds: number) => Promise<void>;
/** A monotonic clock in milliseconds (performance.now when absent), which bounds the wait for a tool-server to start. */
now?: () => number;
/** Moves a file the Argent client downloaded (a screen recording) to where Explore asked for it. */
moveFile: (from: string, to: string) => Promise<void>;
/** Whether the tool-server answers, checked before a UI action is sent (see `remoteUnavailable`). */
@@ -62,8 +57,6 @@ export type ArgentBridgeContext = {
landscape?: boolean;
/** Turns a portrait screen recording a quarter clockwise, to the landscape UI it shows. */
orientVideo?: (path: string) => Promise<void>;
/** A line for the capture's log: the bridge waiting on a tool-server that is still starting. */
progress?: (message: string) => void;
};
export type BridgeResult = { exitCode: number; stdout: string };
@@ -266,20 +259,6 @@ const SERVICE_DEPENDENCY_FAILED = "Service dependency failed:";
* comes after a pause, so a device still coming back from the restart has the time it needs.
*/
const DEPENDENCY_RETRY_DELAYS_MS = [3_000, 6_000] as const;
/**
* A tool-server that is not serving its tools yet. Argent's client asks GET /tools before every tool call and stops with
* "GET /tools failed: <status>" when that is not answered, so the tool itself was never sent; a refused connection
* reached nothing at all. Either is safe to send again, a UI action too. In brentvatne/euxy's main baseline run
* 39506374824 a replacement EAS session's tool-server answered 404 to /tools for the first list-devices of its install,
* and the whole target failed. An unknown tool's 404 comes from POST /tools/<name> instead, and still fails at once.
*/
const TOOLS_NOT_SERVED = /GET \/tools failed: (?:404|503)\b|ECONNREFUSED/;
/**
* How long a tool-server that has not answered yet gets to start serving its tools, requests included: a deadline, not
* a number of tries, since each try can itself take until the call's timeout. The pauses between tries grow to 10 s.
*/
const STARTUP_WINDOW_MS = 75_000;
const STARTUP_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 10_000] as const;
const SPRINGBOARD = "com.apple.springboard";
const ANDROID_SYSTEM_UI = "com.android.systemui";
const OPEN_IN_APP_PROMPT = /^Open in “.+”\?$/;
@@ -366,40 +345,14 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
/**
* Reads are resent after a transport failure (they change nothing); a UI action is sent once, after a health check.
* Any tool, a UI action too, is resent when Argent could not start a service it depends on, which it raises before
* the tool runs (see SERVICE_DEPENDENCY_FAILED), and while a tool-server that has not answered yet is still starting
* (see TOOLS_NOT_SERVED).
* the tool runs (see SERVICE_DEPENDENCY_FAILED).
*/
const call = async (tool: string, args: Record<string, unknown> = {}, options?: { out?: string }): Promise<Record<string, unknown>> => {
// Tools that name devices their own way (or none) take no udid.
const withDevice = tool === "list-devices" || tool === "stop-all-simulator-servers" ? args : { udid: state.udid, ...args };
const clock = context.now ?? (() => performance.now());
// The window opens with the first call to a tool-server that has not answered yet, and closes 75 s later. A device
// already named came from the tool-server's own answer.
const served = () => state.toolsServed === true || state.udid !== undefined;
const startupDeadline = served() ? undefined : clock() + STARTUP_WINDOW_MS;
for (let readRetry = 0, dependencyRetry = 0, startupRetry = 0; ;) {
const starting = !served() && startupDeadline !== undefined;
try {
// A call to a tool-server still starting gets only the time left in the window.
const answer = record(await context.call(tool, withDevice, starting ? { ...options, timeoutMs: Math.max(1, Math.ceil(startupDeadline - clock())) } : options));
state.toolsServed = true;
return answer;
} catch (error) {
// Until the session's tool-server has answered once, a GET /tools it does not serve is it still starting.
if (starting && error instanceof ArgentCallError) {
const notServed = TOOLS_NOT_SERVED.test(error.message);
const left = startupDeadline - clock();
// A try the window's end cut short is the tool-server not starting in time, whatever the request reported.
if (left <= 0) {
throw new BridgeFailure("TOOL_SERVER_NOT_READY", `Argent's tool-server did not start serving its tools within ${STARTUP_WINDOW_MS / 1_000} s: ${error.message}`);
}
if (notServed) {
const delay = Math.min(STARTUP_RETRY_DELAYS_MS[Math.min(startupRetry++, STARTUP_RETRY_DELAYS_MS.length - 1)]!, left);
context.progress?.(`Argent's tool-server is not serving its tools yet (${error.message}); sending ${tool} again in ${Math.ceil(delay / 1_000)} s (${Math.ceil(left / 1_000)} s left of ${STARTUP_WINDOW_MS / 1_000} s)`);
await context.sleep(delay);
continue;
}
}
for (let readRetry = 0, dependencyRetry = 0; ;) {
try { return record(await context.call(tool, withDevice, options)); }
catch (error) {
if (error instanceof ArgentCallError && error.message.includes(`[Tool:${tool}] ${SERVICE_DEPENDENCY_FAILED}`)
&& dependencyRetry < DEPENDENCY_RETRY_DELAYS_MS.length) {
await context.sleep(DEPENDENCY_RETRY_DELAYS_MS[dependencyRetry++]!);
@@ -464,9 +417,8 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
return target;
};
/**
* A UI action is only sent when the tool-server answers first, so a failure after this point is never resent unless
* the tool provably never ran (see call). A session's tool-server can still be starting when EAS reports the session
* ready, so it is given about 25 s to answer.
* A UI action is only sent when the tool-server answers first, so a failure after this point is never resent. A
* session's tool-server can still be starting when EAS reports the session ready, so it is given about 25 s to answer.
*/
const beforeAction = async () => {
for (const delay of [1_000, 2_000, 3_000, 4_000, 5_000, 5_000, 5_000]) {
+2 -12
View File
@@ -1,5 +1,5 @@
import type { CaptureBundle, CaptureOutcome } from "../contracts/capture.js";
import { bundleColorSchemes, failedPassOutcome, routeIdOfScreen, schemeScreenId } from "./appearance.js";
import { bundleColorSchemes, routeIdOfScreen, schemeScreenId } from "./appearance.js";
import { CiError } from "../shared/ci-errors.js";
type CaptureSide = CaptureBundle["side"];
@@ -59,10 +59,8 @@ export function captureHealthIssues(bundle: CaptureBundle, expectedIds: readonly
for (const id of [...extras].sort()) {
// A secondary frame is expected evidence; its own outcome is the failure of that appearance pass, or its verified
// frame when the primary pass has none (the route is then published from this appearance alone).
// A pass that failed as a whole (sessionFailures) records its failure under every route's `<target>@<scheme>`.
const primaryCaptured = bundle.outcomes[routeIdOfScreen(id).routeId]?.status === "captured";
if (!expected.has(id) && !(expectedSecondary.has(id) && !selected.has(id)
&& (hasVerifiedOutcome(bundle.outcomes[id]) !== primaryCaptured || failedPassOutcome(bundle, id)))) {
if (!expected.has(id) && !(expectedSecondary.has(id) && !selected.has(id) && hasVerifiedOutcome(bundle.outcomes[id]) !== primaryCaptured)) {
add(id, "CAPTURE_TARGET_UNEXPECTED", "Capture evidence contains a target absent from the source plan.", inventoryAction);
}
}
@@ -123,14 +121,6 @@ export function captureHealthIssues(bundle: CaptureBundle, expectedIds: readonly
const INTEGRITY_CODES = new Set(["CAPTURE_SIDE_MISMATCH", "CAPTURE_COMMIT_INVALID", "CAPTURE_SELECTION_DUPLICATE", "CAPTURE_TARGET_UNEXPECTED",
"CAPTURE_TARGET_MISSING", "CAPTURE_OUTCOME_MISSING", "CAPTURE_IMAGE_BINDING_INVALID", "CAPTURE_IMAGE_PROVENANCE_INVALID"]);
/**
* Whether a failure casts doubt on everything a capture took, so its other appearances cannot be kept as a partial
* report: a checkout that changed under it, or evidence that failed its integrity checks.
*/
export function capturedEvidenceInDoubt(code: string): boolean {
return /^(?:SOURCE|CHECKOUT)_/.test(code) || code === "CAPTURE_INCOMPLETE" || INTEGRITY_CODES.has(code);
}
export function sortedCaptureHealthIssues(inputs: readonly CaptureHealthInput[]): CaptureHealthIssue[] {
const compare = (a: string, b: string) => a < b ? -1 : a > b ? 1 : 0;
return inputs.flatMap(({ bundle, selectedIds, side }) => captureHealthIssues(bundle, selectedIds, side))
+39 -79
View File
@@ -1,5 +1,5 @@
import { appearanceArgs, argentColorSchemes, bundleColorSchemes, mergeSchemeOutcomes, resolveColorSchemes, routeIdOfScreen, schemeScreenId, type AppearanceSetting, type ColorScheme } from "./appearance.js";
import { assertNoCaptureHealthIssues, capturedEvidenceInDoubt, captureHealthIssues, partitionCaptureHealthIssues } from "./health.js";
import { assertNoCaptureHealthIssues, captureHealthIssues, partitionCaptureHealthIssues } from "./health.js";
import { type RouteResetSetting } from "./route-reset.js";
import { defaultReadinessSeconds } from "./readiness.js";
import { captureLaunch, ffmpegRecordingTools, launchDirectory, type LaunchSetting, type RecordingTools } from "./launch-capture.js";
@@ -13,7 +13,7 @@ import { sourceRouteNames } from "../contracts/source-route.js";
import type { SourceCapturePlan } from "../source/capture-plan.js";
import { CONTRACT, type ExploreMap, type Platform, type RunStatus } from "../types.js";
import { ExploreStore, validateMap } from "../map/store.js";
import { CiError, environmentSecrets, serializeCiError, toCiError } from "../shared/ci-errors.js";
import { CiError, serializeCiError, toCiError } from "../shared/ci-errors.js";
import type { CaptureSetupAction, DevicePose } from "../config/explore-config.js";
import { type CiAgentRuntimeOptions } from "./agent/runtime.js";
import { EAS_CLI_VERSION, CONTROLLER_SESSION, isRecord, parseJson, baseEnvironment, nonEmpty } from "./common.js";
@@ -360,8 +360,6 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
+ (routes.length - missingRoutes.length ? `, ${routes.length - missingRoutes.length} reused from the baseline` : "")
+ (missingRoutes.length ? `, ${missingRoutes.length} to capture` : ""));
let captureFailure: unknown;
// Later appearance passes that failed after the primary one captured (see the pass loop).
const sessionFailures: NonNullable<CaptureBundle["sessionFailures"]> = [];
const finish = async (): Promise<CiMapResult> => {
if (!(await store.exists())) await store.initialize(map, false);
// A session making only secondary passes owns the `<id>@<scheme>` frames of those passes, not the primary ones, even
@@ -370,11 +368,10 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
? selectedIds.flatMap(id => options.appearancePass!.capture.filter(scheme => scheme !== colorSchemes[0]).map(scheme => schemeScreenId(id, scheme)))
: selectedIds;
const incomplete = captureFailure !== undefined || ownFrameIds.some(id => outcomes[id]?.status !== "captured" || outcomes[id]?.landing !== "verified")
|| map.gaps.some(gap => gap.status === "open") || sessionFailures.length > 0;
|| map.gaps.some(gap => gap.status === "open");
map.run.status = incomplete || map.screens.length === 0 ? "partial" : "complete";
if (routes.length === 0) map.run.stopReason = "No source targets require capture.";
else if (captureFailure) map.run.stopReason = "Agent exploration did not finish; see capture diagnostics and bundle outcomes.";
else if (sessionFailures.length) map.run.stopReason = sessionFailures.map(failure => `The ${failure.scheme} pass failed (${failure.error.code}): ${failure.error.message.replace(/\.+$/, "")}.`).join(" ");
else if (incomplete) map.run.stopReason = "Some source targets could not be captured; see bundle outcomes.";
map.run.completedAt = new Date((options.now ?? Date.now)()).toISOString();
await store.write(map);
@@ -435,8 +432,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
...(Object.keys(launchRecords).length ? { launch: launchRecords } : {}),
...(options.agent ? { captureContexts: Object.fromEntries(selectedIds.filter(id => outcomes[id]?.status === "captured" && options.agent!.contexts[id])
.map(id => [id, options.agent!.contexts[id]!])), agentAcceptance: { policy: AGENT_ACCEPTANCE_POLICY,
frames: Object.fromEntries(Object.keys(images).map(id => [id, acceptedFrames[id]!])) } } : {}),
...(sessionFailures.length ? { sessionFailures } : {}) };
frames: Object.fromEntries(Object.keys(images).map(id => [id, acceptedFrames[id]!])) } } : {}) };
if (options.agent) assertAgentAcceptance(bundle, map, { required: true });
// The side that produces the evidence checks it the way the review will: an integrity defect fails here, naming
// its codes, instead of on the consuming side after both captures. Unavailable screens are not defects.
@@ -525,10 +521,6 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
await rm(simulatorProject, { recursive: true, force: true });
await localRecord?.close();
};
const earlyCleanupFailure = (original: unknown, cleanup: unknown) => new CiError({ code: "SESSION_CLEANUP_FAILED", stage: "cleanup",
message: "A simulator Explore started before the app was ready could not be confirmed stopped.",
nextAction: "Stop the EAS Simulator session or local device this capture started (eas simulator:list), and verify its status.",
details: { originalFailure: serializeCiError(original, { secrets: [options.expoToken] }) }, cause: cleanup });
// What this side starts on this machine, so a later review cleans up after it if this process is stopped hard.
const localRecord = local ? await openLocalCaptureRecord(sourceEnvironment, simulatorProject) : undefined;
if (local) try {
@@ -554,7 +546,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
} });
localOsVersion = created.osVersion;
}
} catch (error) { await stopEarlyDevice().catch(cleanup => { throw earlyCleanupFailure(error, cleanup); }); throw error; }
} catch (error) { await stopEarlyDevice().catch(() => undefined); throw error; }
// The session is named before its app exists: an EAS session starts while the app is repacked (startSession).
// A session EAS failed to start is replaced once (see the session block), under a name of its own.
const sessionNameFor = (retry: number) => simulatorSessionName({ mapName: options.name, mapId: options.buildId, gitCommit: options.gitCommit, easBuildId: local ? "local" : "installed",
@@ -667,8 +659,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
options.signal?.throwIfAborted();
} catch (error) {
// A build or repack that fails leaves nothing running: a device started for it goes, with the session's files and the app.
// One that does not go is the failure to report: a session left running is never hidden behind the build's error.
await stopEarlyDevice().catch(cleanup => { throw earlyCleanupFailure(error, cleanup); });
await stopEarlyDevice().catch(() => undefined);
if (appDirectory) await rm(appDirectory, { recursive: true, force: true });
throw error;
}
@@ -723,14 +714,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
const live = liveSimulatorSessionsNamed(easRead(["simulator:list", "--name", sessionName, "--status", "new", "--status", "queued", "--status", "starting",
"--status", "in-progress", "--limit", "100", "--json", "--non-interactive"]), sessionName, profile.projectIdentity);
if (live) {
// No session file names these sessions, so one that does not stop is reported as left running, never as a
// capture failure a split side could keep as a partial report.
for (const id of live) {
try { assertSimulatorStopped(eas(["simulator:stop", "--id", id, "--json", "--non-interactive"], 2 * 60_000).stdout, id); }
catch (error) { throw new CiError({ code: "SESSION_CLEANUP_FAILED", stage: "cleanup",
message: `EAS Simulator session ${id}, started by a request that failed, could not be confirmed stopped.`,
nextAction: `Stop EAS Simulator session ${id} and verify its status.`, details: { sessionId: id }, cause: error }); }
}
for (const id of live) assertSimulatorStopped(eas(["simulator:stop", "--id", id, "--json", "--non-interactive"], 2 * 60_000).stdout, id);
reason = `EAS could not start the simulator session (${simulatorStartFailure(startError, [options.expoToken]).message})`;
}
}
@@ -1007,60 +991,41 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
const passes: CaptureBundle["outcomes"][] = [];
const recordingTools = launchSetting === "off" ? undefined : options.recordingTools === undefined ? ffmpegRecordingTools() : options.recordingTools ?? undefined;
const capturedSchemes = options.appearancePass ? colorSchemes.filter(scheme => options.appearancePass!.capture.includes(scheme)) : colorSchemes;
const activeRuntime = runtime;
for (const [index, colorScheme] of colorSchemes.entries()) {
// A pass another session makes leaves its slot empty, so the passes stay aligned with the appearances they merge by.
if (!capturedSchemes.includes(colorScheme)) { passes.push({}); continue; }
const passOutcomes: CaptureBundle["outcomes"] = index === 0 ? outcomes : {};
const capturePass = async () => {
// Argent cannot switch the appearance yet (software-mansion/argent#1509): the host does, simctl or adb on this
// machine and serve-sim on an EAS iOS session; elsewhere its single pass runs as the device starts.
if (controller !== "argent") runner({ command: controllerCommand, args: [...appearanceArgs(colorScheme), "--session", CONTROLLER_SESSION, ...deviceArgs(activeRuntime), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
else if (sessionId && controlActions.has("appearance.set")) simulatorControl(sessionEas, sessionId, "appearance.set", [colorScheme], controlOptions);
else if (localSimulator) setLocalSimulatorAppearance({ runner, udid: localSimulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
else if (localEmulator) setLocalEmulatorAppearance({ runner, serial: localEmulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
// A scheme change recreates the Android activity in place, and the sheets it re-presents lose their inset; a
// relaunch starts the app in the new scheme, the same way the first pass did.
if (launchSetting !== "off") {
// Observing the launch relaunches the app before setup runs, so no UI-created state is lost.
say(`observing the app launch${colorSchemes.length > 1 ? ` (${colorScheme})` : ""}`);
try {
// A restored baseline's launch evidence is replaced only by a fresh observation.
const record = await captureLaunch({ bundleId: options.bundleId, keepVideo: launchSetting === "video", platform: options.platform,
directory: join(launchWorkDirectory, launchDirectory(colorScheme)), relativeDirectory: launchDirectory(colorScheme), ...(options.now ? { now: options.now } : {}),
...(recordingTools ? { recordingTools } : {}),
controller: (args, controllerOptions) => runner({ command: controllerCommand, args: [...args, "--session", CONTROLLER_SESSION, ...deviceArgs(activeRuntime), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: controllerOptions?.timeoutMs ?? 60_000, ...(controllerOptions?.allowFailure ? { allowFailure: true } : {}) }) });
launch[colorScheme] = { root: launchWorkDirectory, record };
} catch (error) {
// The launch is supplementary evidence: a failure to observe it leaves the route capture intact.
say(`the app launch could not be observed (${(error as Error).message}); the report has no launch evidence for ${colorScheme}`);
await rm(join(launchWorkDirectory, launchDirectory(colorScheme)), { recursive: true, force: true });
}
} else if (index > 0) runner({ command: controllerCommand, args: ["open", options.bundleId, "--relaunch", "--session", CONTROLLER_SESSION,
...deviceArgs(activeRuntime), "--platform", options.platform, "--level", "full", "--json"], cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
activeRuntime.colorScheme = colorScheme;
if (colorSchemes.length > 1) say(`appearance ${colorScheme} (${index + 1}/${colorSchemes.length})`);
return await capturePlannedRoutes({ capture: { ...options, ...(argentBridge && !options.agent ? { runner: argentBridge.runner } : {}), agentDeviceCommand: controllerCommand, setup, progress: say }, routes: missingRoutes, inventory, scheme, appName: app.name, runtime: activeRuntime,
connection, environment: controllerEnvironment, store, map, acceptedFrames, outcomes: passOutcomes,
...(colorSchemes.length > 1 ? { appearance: { scheme: colorScheme, primary: index === 0 } } : {}) });
};
// A later pass of a session whose primary pass captured fails only its own appearance, as a secondary session does
// when the passes are split over sessions (captureSideInSessions): in brentvatne/euxy's main baseline run
// 39506374824 a failed dark session failed a target whose light session had captured 16 of 16 routes.
if (index === 0 || !capturedSchemes.includes(colorSchemes[0]!) || options.agent) { passes.push(await capturePass()); continue; }
try { passes.push(await capturePass()); }
catch (error) {
const failure = toCiError(error, { code: `CAPTURE_${runtimeStage.toUpperCase()}_FAILED`, stage: runtimeStage,
nextAction: "Inspect the capture diagnostic, correct the failing prerequisite, and rerun Explore." });
if (options.signal?.aborted || capturedEvidenceInDoubt(failure.code)) throw failure;
const { code, stage, message, nextAction, retryable } = serializeCiError(failure, { secrets: [options.expoToken, ...environmentSecrets(sourceEnvironment)] });
say(`the ${colorScheme} pass failed (${code}: ${message}); keeping the ${colorSchemes[0]} captures, and the report marks every ${colorScheme} frame it did not take failed`);
sessionFailures.push({ scheme: colorScheme, error: { code, stage, message, nextAction, retryable } });
// The routes the pass finished keep what it recorded.
passes.push(passOutcomes);
}
// Argent cannot switch the appearance yet (software-mansion/argent#1509): the host does, simctl or adb on this
// machine and serve-sim on an EAS iOS session; elsewhere its single pass runs as the device starts.
if (controller !== "argent") runner({ command: controllerCommand, args: [...appearanceArgs(colorScheme), "--session", CONTROLLER_SESSION, ...deviceArgs(runtime), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
else if (sessionId && controlActions.has("appearance.set")) simulatorControl(sessionEas, sessionId, "appearance.set", [colorScheme], controlOptions);
else if (localSimulator) setLocalSimulatorAppearance({ runner, udid: localSimulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
else if (localEmulator) setLocalEmulatorAppearance({ runner, serial: localEmulator, scheme: colorScheme, cwd: simulatorProject, environment: restrictedEnvironment });
// A scheme change recreates the Android activity in place, and the sheets it re-presents lose their inset; a
// relaunch starts the app in the new scheme, the same way the first pass did.
if (launchSetting !== "off") {
// Observing the launch relaunches the app before setup runs, so no UI-created state is lost.
say(`observing the app launch${colorSchemes.length > 1 ? ` (${colorScheme})` : ""}`);
try {
// A restored baseline's launch evidence is replaced only by a fresh observation.
const record = await captureLaunch({ bundleId: options.bundleId, keepVideo: launchSetting === "video", platform: options.platform,
directory: join(launchWorkDirectory, launchDirectory(colorScheme)), relativeDirectory: launchDirectory(colorScheme), ...(options.now ? { now: options.now } : {}),
...(recordingTools ? { recordingTools } : {}),
controller: (args, controllerOptions) => runner({ command: controllerCommand, args: [...args, "--session", CONTROLLER_SESSION, ...deviceArgs(runtime!), "--platform", options.platform],
cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: controllerOptions?.timeoutMs ?? 60_000, ...(controllerOptions?.allowFailure ? { allowFailure: true } : {}) }) });
launch[colorScheme] = { root: launchWorkDirectory, record };
} catch (error) {
// The launch is supplementary evidence: a failure to observe it leaves the route capture intact.
say(`the app launch could not be observed (${(error as Error).message}); the report has no launch evidence for ${colorScheme}`);
await rm(join(launchWorkDirectory, launchDirectory(colorScheme)), { recursive: true, force: true });
}
} else if (index > 0) runner({ command: controllerCommand, args: ["open", options.bundleId, "--relaunch", "--session", CONTROLLER_SESSION,
...deviceArgs(runtime), "--platform", options.platform, "--level", "full", "--json"], cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
runtime.colorScheme = colorScheme;
if (colorSchemes.length > 1) say(`appearance ${colorScheme} (${index + 1}/${colorSchemes.length})`);
passes.push(await capturePlannedRoutes({ capture: { ...options, ...(argentBridge && !options.agent ? { runner: argentBridge.runner } : {}), agentDeviceCommand: controllerCommand, setup, progress: say }, routes: missingRoutes, inventory, scheme, appName: app.name, runtime,
connection, environment: controllerEnvironment, store, map, acceptedFrames, outcomes: index === 0 ? outcomes : {},
...(colorSchemes.length > 1 ? { appearance: { scheme: colorScheme, primary: index === 0 } } : {}) }));
}
runtime.colorScheme = colorSchemes[0]!;
// A session without the primary pass keeps each outcome as its pass recorded it, under `<id>@<scheme>`: the export
@@ -1069,11 +1034,6 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
if (options.appearancePass && !capturedSchemes.includes(colorSchemes[0]!)) {
for (const [index, pass] of passes.entries()) for (const [id, outcome] of Object.entries(pass)) outcomes[schemeScreenId(id, colorSchemes[index])] = outcome;
} else Object.assign(outcomes, mergeSchemeOutcomes(passes, colorSchemes));
// A failed pass records its failure under each route it did not finish, as the export merge does for a failed session.
for (const { scheme: failedScheme, error } of sessionFailures) {
const pass = passes[colorSchemes.indexOf(failedScheme)] ?? {};
for (const id of selectedIds) if (!pass[id]) outcomes[schemeScreenId(id, failedScheme)] = { status: "failed", attempts: 0, landing: "unverified", reason: error.message, error };
}
} catch (error) { primaryError = toCiError(error, { code: `CAPTURE_${runtimeStage.toUpperCase()}_FAILED`, stage: runtimeStage,
nextAction: "Inspect the capture diagnostic, correct the failing prerequisite, and rerun Explore." }); }
finally {
+4 -35
View File
@@ -1,4 +1,4 @@
import { bundleColorSchemes, routeIdOfScreen, schemeScreenId } from "../capture/appearance.js";
import { routeIdOfScreen } from "../capture/appearance.js";
import { createHash } from "node:crypto";
import { cp, lstat, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises";
import { isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -54,8 +54,7 @@ function requiresAgentAcceptance(recipe: Recipe, bundle: CaptureBundle): boolean
catch { return false; }
}
/** Checks an export the way a cache entry is checked before it is written or restored: its provenance, profile and frames. */
export async function validateEvidence(directory: string, bundle: CaptureBundle): Promise<void> {
async function validateEvidence(directory: string, bundle: CaptureBundle): Promise<void> {
const map = await readExportedMap(directory, { checkArtifacts: true });
const execution = parseRuntimeExecutionContext(JSON.parse(await readFile(join(directory, "execution.json"), "utf8")));
if (!bundle.runtimeProfile || !/^[a-f0-9]{40}$/.test(bundle.gitCommit)
@@ -114,31 +113,6 @@ async function fileInventory(directory: string, path = ""): Promise<FileRecord[]
return files;
}
/**
* The routes a review can reuse from a capture: a verified frame in every appearance it renders. A baseline whose dark
* session failed (brentvatne/euxy's main baseline run 39506374824) holds its light frames and a failed `<route>@dark`;
* reusing the light frame alone would leave the review's base with no dark frame, and dark would read as unavailable
* because of the base. Such a route is captured by the review's base instead, in every appearance.
*/
export function reusableSurfaceIds(bundle: CaptureBundle): string[] {
const secondarySchemes = bundleColorSchemes(bundle).slice(1);
return bundle.selectedSurfaceIds.filter(id => hasVerifiedCapture(bundle, id)
&& secondarySchemes.every(scheme => bundle.images[schemeScreenId(id, scheme)] && bundle.outcomes[schemeScreenId(id, scheme)] === undefined));
}
/**
* Why a capture of `commit` in `sourceDirectory` is not offered for reuse, when it is there and captured some routes:
* none has every appearance (its dark session failed). Undefined otherwise, the cache's usual misses included.
*/
export async function incompleteCaptureReason(sourceDirectory: string, commit: string): Promise<string | undefined> {
try {
const bundle = await readCaptureBundle(sourceDirectory);
if (bundle.gitCommit !== commit || reusableSurfaceIds(bundle).length || !bundle.selectedSurfaceIds.some(id => hasVerifiedCapture(bundle, id))) return undefined;
const failed = (bundle.sessionFailures ?? []).map(failure => failure.scheme);
return failed.length ? `its ${failed.join(" and ")} session failed, so no route has every appearance` : "no route was captured in every appearance";
} catch { return undefined; }
}
/** Cache full exported evidence, including accessibility maps and all referenced screenshots. */
export async function writeCaptureArtifactCache(directory: string, recipe: Recipe): Promise<boolean> {
try {
@@ -148,9 +122,6 @@ export async function writeCaptureArtifactCache(directory: string, recipe: Recip
if (!bundle.runtimeProfile) return false;
assertAgentAcceptance(bundle, undefined, { required: requiresAgentAcceptance(recipe, bundle) });
await validateEvidence(directory, bundle);
// A capture with no route a review could reuse is not offered: a review that found it would wait for it, and capture
// its sides in turn instead of together, only to capture every route of the base itself.
if (!reusableSurfaceIds(bundle).length) return false;
const metadata: CacheMetadata = {
contract: CONTRACT, recipe: recipeKey(recipe), commit: bundle.gitCommit, files: await fileInventory(directory),
};
@@ -210,8 +181,7 @@ export async function restoreCaptureArtifactCache(options: Recipe & {
stage = await mkdtemp(`${options.directory}.cache-`);
const stagedEvidence = join(stage, "evidence");
await cp(source, stagedEvidence, { recursive: true, force: false, errorOnExist: true });
const reusable = new Set(reusableSurfaceIds(bundle));
const available = options.selectedIds.filter(id => reusable.has(id)
const available = options.selectedIds.filter(id => hasVerifiedCapture(bundle, id)
&& (!options.captureContexts || options.captureContexts[id] && bundle.captureContexts?.[id]
&& canonicalJson(options.captureContexts[id]) === canonicalJson(bundle.captureContexts[id])));
if (!available.length) return false;
@@ -222,8 +192,7 @@ export async function restoreCaptureArtifactCache(options: Recipe & {
const captureContexts = bundle.captureContexts ? Object.fromEntries(available.map(id => [id, bundle.captureContexts![id]])) : undefined;
const agentAcceptance = bundle.agentAcceptance ? { policy: bundle.agentAcceptance.policy,
frames: Object.fromEntries(available.map(id => [id, bundle.agentAcceptance!.frames[id]!])) } : undefined;
// Only complete routes are restored, so a failed pass of the baseline's is no part of the restored bundle.
await writeFile(join(stagedEvidence, "bundle.json"), JSON.stringify({ ...bundle, side: "base", selectedSurfaceIds: available, images, outcomes, sessionFailures: undefined,
await writeFile(join(stagedEvidence, "bundle.json"), JSON.stringify({ ...bundle, side: "base", selectedSurfaceIds: available, images, outcomes,
...(captureContexts ? { captureContexts } : {}), ...(agentAcceptance ? { agentAcceptance } : {}) }));
await cp(stagedEvidence, options.directory, { recursive: true, force: false, errorOnExist: true });
return true;
+2 -2
View File
@@ -1,6 +1,6 @@
import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
import { runCiMap, type CiMapOptions, type CiMapResult } from "../capture/run.js";
import { CiError, environmentSecrets, serializeCiError, type CiErrorData } from "../shared/ci-errors.js";
import { CiError, serializeCiError, type CiErrorData } from "../shared/ci-errors.js";
import type { Progress } from "../shared/progress.js";
type WorkerMessage = { type: "progress"; message: string } | { type: "done"; result: CiMapResult } | { type: "error"; error: CiErrorData };
@@ -41,5 +41,5 @@ if (!isMainThread && parentPort) {
port.unref();
runCiMap({ ...options, signal: controller.signal, progress: message => port.postMessage({ type: "progress", message } satisfies WorkerMessage) })
.then(result => port.postMessage({ type: "done", result } satisfies WorkerMessage))
.catch((error: unknown) => port.postMessage({ type: "error", error: serializeCiError(error, { secrets: [options.expoToken, ...environmentSecrets(options.environment ?? {})] }) } satisfies WorkerMessage));
.catch((error: unknown) => port.postMessage({ type: "error", error: serializeCiError(error, { secrets: [options.expoToken] }) } satisfies WorkerMessage));
}
+11 -38
View File
@@ -1,7 +1,7 @@
import { copyFile, mkdir, readdir, readFile, writeFile, appendFile, lstat } from "node:fs/promises";
import { dirname, join, relative } from "node:path";
import { bundleColorSchemes, mergeSchemeOutcomes, routeIdOfScreen, schemeScreenId, type ColorScheme } from "../capture/appearance.js";
import { bundleColorSchemes, mergeSchemeOutcomes, routeIdOfScreen, type ColorScheme } from "../capture/appearance.js";
import type { CaptureOutcome } from "../contracts/capture.js";
import type { CaptureBundle } from "../contracts/capture.js";
import type { ExploreMap } from "../types.js";
@@ -22,21 +22,12 @@ async function copyTree(source: string, destination: string, skip: (relativePath
}
}
/**
* A secondary appearance session's result: its export, or the error it failed with, when it left none. In brentvatne/euxy's
* main baseline run 39506374824 the dark session's install failed after the light session had captured 16 of 16 routes.
*/
export type SecondaryCaptureExport = { directory: string; scheme: ColorScheme } | { scheme: ColorScheme; error: NonNullable<CaptureOutcome["error"]> };
/**
* Fold the exports of a side's secondary appearance sessions into its primary export, in place: the frames, outcomes,
* launch evidence and map records of each `<id>@<scheme>` screen, and their files. Each session captured its passes as
* one run would (`appearancePass`), so records only append; a secondary export never replaces a primary record. A session
* that failed is listed in the bundle's `sessionFailures`, adds a failed `<id>@<scheme>` outcome carrying its error for
* every selected route, and leaves the run partial, so its appearance reads as unavailable with its own reason (a route
* the primary lost too would otherwise show the primary's) instead of going missing.
* one run would (`appearancePass`), so records only append; a secondary export never replaces a primary record.
*/
export async function mergeCaptureExports(primary: string, secondaries: readonly SecondaryCaptureExport[]): Promise<void> {
export async function mergeCaptureExports(primary: string, secondaries: readonly { directory: string; scheme: ColorScheme }[]): Promise<void> {
if (secondaries.length === 0) return;
const bundle = await readJson<CaptureBundle>(join(primary, "bundle.json"));
const map = await readJson<ExploreMap>(join(primary, "screen-map.json"));
@@ -46,28 +37,15 @@ export async function mergeCaptureExports(primary: string, secondaries: readonly
const schemes = bundleColorSchemes(bundle);
const passes: Record<string, CaptureOutcome>[] = schemes.map(() => ({}));
for (const [id, outcome] of Object.entries(bundle.outcomes)) if (!routeIdOfScreen(id).scheme) passes[0]![id] = outcome;
const failedSessions: string[] = [];
const failedOutcomes: Record<string, CaptureOutcome> = {};
for (const secondary of secondaries) {
const { scheme } = secondary;
if ("error" in secondary) {
failedSessions.push(`The ${scheme} session failed (${secondary.error.code}): ${secondary.error.message.replace(/\.+$/, "")}.`);
(bundle.sessionFailures ??= []).push({ scheme, error: secondary.error });
// A primary that renders no such appearance has no slot for its frames; the failure stays on the bundle alone.
if (!schemes.includes(scheme)) continue;
for (const id of bundle.selectedSurfaceIds) {
failedOutcomes[schemeScreenId(id, scheme)] = { status: "failed", attempts: 0, landing: "unverified", reason: secondary.error.message, error: secondary.error };
}
continue;
}
const other = await readJson<CaptureBundle>(join(secondary.directory, "bundle.json"));
for (const { directory: secondary, scheme } of secondaries) {
const other = await readJson<CaptureBundle>(join(secondary, "bundle.json"));
const pass = passes[schemes.indexOf(scheme)];
// A session that could not show its appearance (Argent without appearance.set captures one) has nothing to add, as
// one run would have no pass for it.
if (!pass && !bundleColorSchemes(other).includes(scheme)) continue;
if (!pass) throw new Error(`The ${scheme} session's export is not an appearance of the primary export (${schemes.join(", ")})`);
for (const [id, outcome] of Object.entries(other.outcomes)) pass[routeIdOfScreen(id).routeId] = outcome;
const otherMap = await readJson<ExploreMap>(join(secondary.directory, "screen-map.json"));
const otherMap = await readJson<ExploreMap>(join(secondary, "screen-map.json"));
for (const [id, image] of Object.entries(other.images)) if (routeIdOfScreen(id).scheme && !bundle.images[id]) bundle.images[id] = image;
for (const [scheme, record] of Object.entries(other.launch ?? {})) { bundle.launch ??= {}; bundle.launch[scheme] ??= record; }
for (const [id, context] of Object.entries(other.captureContexts ?? {})) { bundle.captureContexts ??= {}; bundle.captureContexts[id] ??= context; }
@@ -83,9 +61,9 @@ export async function mergeCaptureExports(primary: string, secondaries: readonly
}
}
// Files: artifacts and launch evidence are new paths; diagnostics append to the primary's lines.
await copyTree(secondary.directory, primary, rel => rel === "bundle.json" || rel === "screen-map.json" || rel === "screen-map.schema.json" || rel === "manifest.json"
await copyTree(secondary, primary, rel => rel === "bundle.json" || rel === "screen-map.json" || rel === "screen-map.schema.json" || rel === "manifest.json"
|| rel === "execution.json" || rel.startsWith("capture-diagnostics"));
const diagnostics = join(secondary.directory, "capture-diagnostics");
const diagnostics = join(secondary, "capture-diagnostics");
for (const entry of await readdir(diagnostics, { withFileTypes: true }).catch(() => [])) {
if (!entry.isFile()) continue;
const to = join(primary, "capture-diagnostics", entry.name);
@@ -94,17 +72,12 @@ export async function mergeCaptureExports(primary: string, secondaries: readonly
else if (!(await lstat(to).catch(() => undefined))) await copyFile(join(diagnostics, entry.name), to);
}
}
// An open gap a secondary pass left makes the run partial, as it does one run's (a complete run has none), and so does
// a secondary session that failed. A partial run names why, as the map's schema requires.
if (map.gaps.some(gap => gap.status === "open") || failedSessions.length) {
map.run.status = "partial";
map.run.stopReason = [map.run.stopReason ?? (failedSessions.length ? undefined : "Some source targets could not be captured; see bundle outcomes."),
...failedSessions].filter(Boolean).join(" ");
}
// An open gap a secondary pass left makes the run partial, as it does one run's (a complete run has none).
if (map.gaps.some(gap => gap.status === "open")) map.run.status = "partial";
const manifest = await readJson<{ files: unknown[] }>(join(primary, "manifest.json"));
manifest.files = map.artifacts.map(artifact => ({ path: artifact.path, sha256: artifact.sha256, bytes: artifact.bytes, mimeType: artifact.mimeType }));
await writeFile(join(primary, "screen-map.json"), JSON.stringify(map));
bundle.outcomes = { ...mergeSchemeOutcomes(passes, schemes), ...failedOutcomes };
bundle.outcomes = mergeSchemeOutcomes(passes, schemes);
await writeFile(join(primary, "bundle.json"), JSON.stringify(bundle));
await writeFile(join(primary, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
}
+2 -12
View File
@@ -293,17 +293,6 @@ function sessionStateFiles(output: string): string[] {
catch { return []; }
}
/**
* A finished target keeps only its report: the captures, maps and checkouts are the bulk of the run. A session file
* stays, since it names a session no capture confirmed stopped; the review stops it on its way out (stopRecordedSessions).
*/
export async function pruneTargetOutput(output: string): Promise<void> {
const sessions = new Set(sessionStateFiles(output));
for (const entry of await readdir(output)) {
if (entry !== "report" && entry !== "report.zip" && !sessions.has(entry)) await rm(join(output, entry), { recursive: true, force: true });
}
}
function recordedSessionIds(outputs: string[]): string[] {
return outputs.flatMap(output => sessionStateFiles(output).flatMap(name => {
try { const id = readFileSync(join(output, name), "utf8").trim(); return /^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(id) ? [id] : []; } catch { return []; }
@@ -537,7 +526,8 @@ export async function runLocalReview(options: LocalReviewOptions): Promise<Local
if (side === "head" && head.uncommitted) continue;
await keepCapture(join(output, `${side}-export`), captureCacheDirectory(environment, target.id, side === "base" ? base.sha : head.sha));
}
await pruneTargetOutput(output);
// The report is all that is kept; the captures, maps and checkouts are the bulk of the run.
for (const entry of await readdir(output)) if (entry !== "report" && entry !== "report.zip") await rm(join(output, entry), { recursive: true, force: true });
abort.signal.throwIfAborted();
return report;
};
+14 -60
View File
@@ -1,7 +1,7 @@
import { systemRegions } from "../contracts/capture.js";
import type { RunTarget } from "../types.js";
import { execFile, spawnSync } from "node:child_process";
import { lstat, mkdir, mkdtemp, open, readFile, realpath, rm, writeFile } from "node:fs/promises";
import { mkdir, mkdtemp, open, readFile, realpath, rm, writeFile } from "node:fs/promises";
import { join, relative, resolve, isAbsolute } from "node:path";
import { tmpdir } from "node:os";
import { setTimeout as delay } from "node:timers/promises";
@@ -12,12 +12,12 @@ import type { RepackShare } from "../capture/eas-build.js";
import { resolveColorSchemes, routeIdOfScreen, type ColorScheme } from "../capture/appearance.js";
import { baseEnvironment } from "../capture/common.js";
import { DEFAULT_MAX_SESSIONS, planSideSessions, sessionShare } from "./session-plan.js";
import { mergeCaptureExports, type SecondaryCaptureExport } from "./export-merge.js";
import { mergeCaptureExports } from "./export-merge.js";
import { abandonSharedBuilds } from "../capture/shared-build.js";
import { prepareCaptureReview, readCaptureBundle } from "../review/capture-review.js";
import { quarantineReviewContract } from "../contracts/review-validation.js";
import { agentCaptureConstraints } from "../capture/agent/source-context.js";
import { captureArtifactCacheMayMatch, captureRecipeConfig, incompleteCaptureReason, restoreCaptureArtifactCache, writeCaptureArtifactCache } from "./artifact-cache.js";
import { captureArtifactCacheMayMatch, captureRecipeConfig, restoreCaptureArtifactCache, writeCaptureArtifactCache } from "./artifact-cache.js";
import { runCiMapOnWorker } from "./capture-worker.js";
import { DEFAULT_BASELINE_WAIT_MINUTES, awaitPendingBaseline } from "./baseline-wait.js";
import { writeCiReportInput, readAppDirectory } from "../report/artifact.js";
@@ -30,10 +30,10 @@ import { viewerLinks } from "../report/viewer-links.js";
import { DEFAULT_EXPLORE_CONFIG_PATH, captureRepack, type ExploreConfig, type ExplorePlatform } from "../config/explore-config.js";
import { prepareCiExploration, type PrepareCiExplorationOptions } from "../capture/agent/exploration.js";
import { generateAgentChangeNotes } from "../capture/agent/change-notes.js";
import { CiError, environmentSecrets, serializeCiError, toCiError } from "../shared/ci-errors.js";
import { CiError, serializeCiError, toCiError } from "../shared/ci-errors.js";
import type { SourceCapturePlan } from "../source/capture-plan.js";
import { createSourceChangeSummaries } from "../source/change-summary.js";
import { assertNoCaptureHealthIssues, capturedEvidenceInDoubt, partitionCaptureHealthIssues, renderCaptureIssueAnnotation, renderComparisonDiagnosticAnnotation, renderNotesUnavailableAnnotation, renderReviewGapAnnotation, sortedCaptureHealthIssues } from "../capture/health.js";
import { assertNoCaptureHealthIssues, partitionCaptureHealthIssues, renderCaptureIssueAnnotation, renderComparisonDiagnosticAnnotation, renderNotesUnavailableAnnotation, renderReviewGapAnnotation, sortedCaptureHealthIssues } from "../capture/health.js";
import { EXPLORE_PACKAGE, packageBin } from "../shared/package-paths.js";
import { agentDeviceLauncher, ensureArgentToolServer, ensureController } from "../shared/controller-cache.js";
import { silentProgress, type Progress } from "../shared/progress.js";
@@ -130,37 +130,14 @@ async function expectedColorSchemes(config: ExploreConfig, project: string, envi
}
}
/**
* Failures that fail the side even when only a secondary appearance session had them. A session Explore could not
* confirm stopped, or whose identity it could not verify, must never be hidden behind a partial report; a checkout that
* changed, evidence that failed its integrity checks, or a failure Explore did not classify means the primary's frames
* may not be what they claim either.
*/
const SIDE_FAILURE_CODES = new Set(["SESSION_CLEANUP_FAILED", "SESSION_HANDOFF_UNVERIFIED", "CAPTURE_WORKER_EXITED", "EXPLORE_FAILED"]);
/**
* Whether a secondary appearance session's failure can be reported as that appearance's frames failing, keeping the
* primary's: only when the capture reported it itself (a worker that died before its cleanup ran rejects otherwise), its
* session is proven stopped (the capture removes its session file only after a confirmed stop), and nothing in it
* casts doubt on the primary's evidence.
*/
async function secondaryFailureKept(error: unknown, sessionStatePath: string | undefined): Promise<boolean> {
if (!(error instanceof CiError)) return false;
if (SIDE_FAILURE_CODES.has(error.code) || capturedEvidenceInDoubt(error.code)
|| (error.details as { allocation?: unknown } | undefined)?.allocation === "unknown") return false;
return !(sessionStatePath && await lstat(sessionStatePath).then(() => true, () => false));
}
/**
* Run one side's capture: in one session, or when `passes` splits its appearances, one worker session per pass. The
* primary pass exports to `mapOptions.outputDirectory`; the others beside it, named by their appearances, then merge.
* `runPass` replaces the worker capture of each pass (tests).
*/
export async function captureSideInSessions(mapOptions: CiMapOptions, passes: readonly (readonly ColorScheme[])[] | undefined,
progress: Progress, signal: AbortSignal | undefined, worker: boolean,
runPass?: (passOptions: CiMapOptions) => Promise<Awaited<ReturnType<typeof runCiMap>>>): Promise<Awaited<ReturnType<typeof runCiMap>>> {
const runOnWorker = runPass ?? ((passOptions: CiMapOptions) =>
runCiMapOnWorker((({ progress: _progress, runner: _runner, agent: _agent, signal: _signal, ...rest }) => rest)(passOptions), progress, signal));
async function captureSideInSessions(mapOptions: CiMapOptions, passes: readonly (readonly ColorScheme[])[] | undefined,
progress: Progress, signal: AbortSignal | undefined, worker: boolean): Promise<Awaited<ReturnType<typeof runCiMap>>> {
const runOnWorker = (passOptions: CiMapOptions) =>
runCiMapOnWorker((({ progress: _progress, runner: _runner, agent: _agent, signal: _signal, ...rest }) => rest)(passOptions), progress, signal);
if (!passes || passes.length <= 1) return worker ? runOnWorker(mapOptions) : runCiMap(mapOptions);
const schemes = passes.flat();
progress(`${mapOptions.side}: capturing ${passes.map(pass => pass.join("+")).join(" and ")} in separate sessions`);
@@ -181,29 +158,9 @@ export async function captureSideInSessions(mapOptions: CiMapOptions, passes: re
}));
// The build folder holds the repacked app, which the sessions have installed.
if (ownBuildShare) await rm(ownBuildShare, { recursive: true, force: true });
// Only the primary appearance's session fails the side, besides a stopped review and a secondary failure that cannot be
// kept (secondaryFailureKept). In brentvatne/euxy's main baseline run 39506374824 the light session captured 16 of 16
// routes and the dark session's install failed, which failed the whole target and lost its baseline. Such a failure is
// kept in the report instead: every frame of its appearance is failed with its error, and the run is partial.
const suffixOf = (pass: readonly ColorScheme[]) => `-${pass.join("-")}`;
for (const [index, outcome] of settled.entries()) {
if (outcome.status !== "rejected" || index === 0) continue;
const statePath = mapOptions.sessionStatePath ? `${mapOptions.sessionStatePath}${suffixOf(passes[index]!)}` : undefined;
if (signal?.aborted || !await secondaryFailureKept(outcome.reason, statePath)) throw outcome.reason;
}
const primary = settled[0]!;
if (primary.status === "rejected") throw primary.reason;
const secrets = [mapOptions.expoToken, ...environmentSecrets(mapOptions.environment ?? {})];
const secondaries = passes.slice(1).map((pass, index): SecondaryCaptureExport => {
const outcome = settled[index + 1]!;
if (outcome.status === "fulfilled") return { directory: `${mapOptions.outputDirectory}${suffixOf(pass)}`, scheme: pass[0]! };
const error = serializeCiError(outcome.reason, { secrets });
progress(`${mapOptions.side}: the ${pass.join("+")} session failed (${error.code}: ${error.message}); keeping the ${passes[0]!.join("+")} captures, and the report marks every ${pass.join("+")} frame failed`);
return { scheme: pass[0]!, error: { code: error.code, stage: error.stage, message: error.message, nextAction: error.nextAction, retryable: error.retryable } };
});
await mergeCaptureExports(mapOptions.outputDirectory, secondaries);
const result = primary.value;
return secondaries.some(secondary => "error" in secondary) ? { ...result, runStatus: "partial" } : result;
for (const outcome of settled) if (outcome.status === "rejected") throw outcome.reason;
await mergeCaptureExports(mapOptions.outputDirectory, passes.slice(1).map(pass => ({ directory: `${mapOptions.outputDirectory}-${pass.join("-")}`, scheme: pass[0]! })));
return (settled[0] as PromiseFulfilledResult<Awaited<ReturnType<typeof runCiMap>>>).value;
}
/**
@@ -478,7 +435,7 @@ export async function runCiReview(options: {
if (!options.runner) await prefetchControllers(config, environment, progress);
const pr = options.presentation;
const now = options.now ?? Date.now;
const secretValues = environmentSecrets(environment);
const secretValues = Object.entries(environment).filter(([key]) => /(?:TOKEN|SECRET|PASSWORD|API_KEY)/i.test(key)).flatMap(([, value]) => value ? [value] : []);
let stage = "plan";
let usagePhase: AgentUsagePhase = "preflight";
let preserveReport: (() => Promise<void>) | undefined;
@@ -543,10 +500,7 @@ export async function runCiReview(options: {
commit: pr.base.sha, selectedIds, directory: join(output, "base-cache"), runtimeProfile,
...(exploration.agent ? { captureContexts: exploration.agent.contexts } : {}) });
if (restored) reuseDirectory = join(output, "base-cache");
const incomplete = restored ? undefined : await incompleteCaptureReason(cacheDirectory, pr.base.sha);
progress(restored ? `base: found the baseline captures of ${pr.base.sha.slice(0, 8)}`
: incomplete ? `base: the baseline of ${pr.base.sha.slice(0, 8)} has nothing to reuse (${incomplete}); capturing the base too`
: `base: no compatible baseline of ${pr.base.sha.slice(0, 8)}; capturing the base too`);
progress(restored ? `base: found the baseline captures of ${pr.base.sha.slice(0, 8)}` : `base: no compatible baseline of ${pr.base.sha.slice(0, 8)}; capturing the base too`);
}
const mapOptions = { ...captureOptions(config, environment, options.runner, { now: options.now, sleep: options.sleep }), progress,
...(options.useLocalEasSession ? { useLocalEasSession: true } : {}),
-6
View File
@@ -93,12 +93,6 @@ export type CaptureBundle = {
}>;
/** The exact observed frames explicitly accepted by the agent, never replacement screenshots. */
agentAcceptance?: { policy: typeof AGENT_ACCEPTANCE_POLICY; frames: Record<string, AgentAcceptedFrame> };
/**
* The appearance passes that failed while the primary one captured (a secondary session that failed, or a later pass of
* one session), each with its error. Every selected route then has a failed `<id>@<scheme>` outcome carrying it, when
* the bundle renders that appearance. Absent when every pass finished.
*/
sessionFailures?: Array<{ scheme: "light" | "dark"; error: NonNullable<CaptureOutcome["error"]> }>;
};
/** Bind acceptance receipts to captured images and, when supplied, their exported observations. */
+7 -31
View File
@@ -321,34 +321,21 @@ export function platformRecordOf(state: PublicCommentState, target?: CommentTarg
if (notCompared) record.notCompared = notCompared;
record.context = splashLines(state.review);
const diagnostics = [...(state.evidence?.diagnostics ?? [])];
// A failed appearance pass is one issue, named first, rather than one per frame it left unavailable: in brentvatne/
// euxy's main baseline run 39506374824 the dark session's failure would have listed all 16 routes, crowding out the rest.
const sessionFailures = state.evidence?.sessionFailures ?? [];
if (sessionFailures.length) record.sessionFailures = sessionFailures.map(({ side, scheme, frames }) => ({ side, scheme, frames }));
const sessionLines = sessionFailures.map(failure => ({ side: failure.side, route: `${failure.scheme} session`, code: failure.code,
message: `${failure.message.replace(/\.+$/, "")}.${failure.frames ? ` ${plural(failure.frames, `${failure.scheme} frame`)} ${failure.frames === 1 ? "is" : "are"} unavailable.` : ""}`,
nextAction: failure.nextAction }));
const ofFailedPass = (surfaceId: string, side: "base" | "head", outcome: NonNullable<Surface["headOutcome"]>) => sessionFailures.some(failure => failure.side === side
&& schemeOf(surfaceId) === failure.scheme && outcome.error?.code === failure.code && outcome.error.message === failure.message);
for (const surface of state.review.surfaces) {
for (const side of ["base", "head"] as const) {
const outcome = surface[`${side}Outcome`];
if (!outcome || (outcome.status === "captured" && outcome.landing === "verified") || diagnostics.some(item => item.surfaceId === surface.id && item.side === side)) continue;
if (ofFailedPass(surface.id, side, outcome)) continue;
diagnostics.push({ surfaceId: surface.id, side, code: outcome.error?.code ?? (outcome.status === "captured" ? "CAPTURE_LANDING_UNVERIFIED" : "CAPTURE_UNAVAILABLE"),
stage: outcome.error?.stage ?? "capture", message: outcome.error?.message ?? outcome.reason ?? (outcome.status === "captured" ? "Destination route was not verified." : `Capture ${outcome.status}.`),
nextAction: outcome.error?.nextAction ?? "Check this side's capture diagnostics, correct the route configuration or capture failure, and rerun capture.", retryable: outcome.error?.retryable ?? false });
}
}
if (diagnostics.length || sessionLines.length) {
record.diagnostics = [...sessionLines, ...diagnostics.map(diagnostic => {
if (diagnostics.length) {
record.diagnostics = diagnostics.slice(0, 20).map(diagnostic => {
const surface = state.review!.surfaces.find(surface => surface.id === diagnostic.surfaceId);
// A secondary appearance's issue names its appearance, as its change row does: its light frame may be fine.
const scheme = schemeOf(diagnostic.surfaceId);
return { ...(diagnostic.side ? { side: diagnostic.side } : {}), route: `${surface ? routeOf(surface) : diagnostic.surfaceId}${surface && scheme ? ` (${scheme})` : ""}`,
code: diagnostic.code, message: diagnostic.message, nextAction: diagnostic.nextAction };
})].slice(0, 20);
record.diagnosticCount = sessionLines.length + diagnostics.length;
return { ...(diagnostic.side ? { side: diagnostic.side } : {}), route: surface ? routeOf(surface) : diagnostic.surfaceId, code: diagnostic.code, message: diagnostic.message, nextAction: diagnostic.nextAction };
});
record.diagnosticCount = diagnostics.length;
}
}
if (state.failure) record.failure = { step: state.failure.step, message: state.failure.message };
@@ -493,11 +480,9 @@ function title(records: PlatformRecord[], rows: ChangeRow[]): string {
}
// With screens left uncompared, nothing changed only among the ones compared.
const summary = rows.length ? `${plural(rows.length, "screen")} changed` : ready.some(record => gaps(record).length) ? "No compared screens changed" : "No screens changed";
// An appearance a pass never captured is said in the heading too: the verdict above says nothing about it.
const missing = [...new Set(ready.flatMap(record => (record.sessionFailures ?? []).map(failure => failure.scheme)))].map(scheme => `${scheme} not captured`);
const report = firstReport(ready);
const open = report ? link("Review in Explore", report) : ready.some(record => record.links.deploying) ? "report deploying…" : undefined;
return `### ${[summary, ...missing, ...(open ? [open] : [])].join(" · ")}`;
return `### ${summary}${open ? ` · ${open}` : ""}`;
}
/** Every target by name, each linking its own report, then the commits and when the comment last changed. */
@@ -527,21 +512,12 @@ function statusLine(record: PlatformRecord): string | undefined {
}
}
/** "dark not captured" for an appearance whose pass failed on head (or both sides), "… on base" when only the base's did. */
function missingAppearances(record: PlatformRecord): string[] {
const failures = record.sessionFailures ?? [];
return [...new Set(failures.map(failure => failure.scheme))].map(scheme => {
const sides = new Set(failures.filter(failure => failure.scheme === scheme).map(failure => failure.side));
return `${scheme} not captured${sides.size === 1 && sides.has("base") ? " on base" : ""}`;
});
}
/** What a finished target has no verdict on: captures missing or unattributed, comparisons that did not run or failed. */
function gaps(record: PlatformRecord): string[] {
const unavailable = record.health ? record.health.base.unavailable + record.health.head.unavailable : 0;
const unverified = record.health ? record.health.base.unverified + record.health.head.unverified : 0;
const failed = (record.validation?.visual?.failed ?? 0) + (record.validation?.accessibility?.failed ?? 0);
return [...missingAppearances(record), unavailable ? `${plural(unavailable, "capture")} unavailable` : undefined, unverified ? `${unverified} unattributed` : undefined,
return [unavailable ? `${plural(unavailable, "capture")} unavailable` : undefined, unverified ? `${unverified} unattributed` : undefined,
record.notCompared ? `${record.notCompared} not compared` : undefined, failed ? `${plural(failed, "comparison")} failed` : undefined]
.filter((part): part is string => Boolean(part));
}
-7
View File
@@ -115,8 +115,6 @@ export type PlatformRecord = {
navigation?: CommentNavigation;
diagnostics?: { side?: "base" | "head"; route: string; code: string; message: string; nextAction: string }[];
diagnosticCount?: number;
/** Appearance passes that failed while the primary one captured: that appearance was not captured on that side. */
sessionFailures?: { side: "base" | "head"; scheme: "light" | "dark"; frames: number }[];
failure?: { step: string; message: string };
/** The PR moved while this run worked; its report stays reachable but it is not the current review. */
superseded?: boolean;
@@ -289,11 +287,6 @@ export function parsePlatformRecord(value: unknown): PlatformRecord | undefined
}
if (isCount(value.notCompared) && value.notCompared > 0) record.notCompared = value.notCompared;
if (isCount(value.diagnosticCount)) record.diagnosticCount = value.diagnosticCount;
if (Array.isArray(value.sessionFailures)) {
const failures = value.sessionFailures.flatMap((entry): NonNullable<PlatformRecord["sessionFailures"]> => isRecord(entry) && isSide(entry.side)
&& (entry.scheme === "light" || entry.scheme === "dark") && isCount(entry.frames) ? [{ side: entry.side, scheme: entry.scheme, frames: entry.frames }] : []).slice(0, 4);
if (failures.length) record.sessionFailures = failures;
}
if (isRecord(value.failure) && isString(value.failure.step, 200) && isString(value.failure.message)) record.failure = { step: value.failure.step, message: value.failure.message };
if (value.superseded === true) record.superseded = true;
return record;
+2 -9
View File
@@ -1,5 +1,5 @@
import type { RunTarget } from "../types.js";
import { appearancePairs, bundleColorSchemes, failedPassOutcome, routeIdOfScreen } from "../capture/appearance.js";
import { appearancePairs, bundleColorSchemes, routeIdOfScreen } from "../capture/appearance.js";
import { createHash } from "node:crypto";
import { lstat, readFile, realpath } from "node:fs/promises";
import { isAbsolute, relative, resolve, sep } from "node:path";
@@ -95,11 +95,9 @@ function secondaryPassOutcome(bundle: CaptureBundle, selected: ReadonlySet<strin
const { routeId, scheme } = routeIdOfScreen(id);
const own = bundle.outcomes[id];
const verified = own?.status === "captured" && own.landing === "verified";
return scheme !== undefined && selected.has(routeId)
&& (verified !== (bundle.outcomes[routeId]?.status === "captured") || failedPassOutcome(bundle, id));
return scheme !== undefined && selected.has(routeId) && verified !== (bundle.outcomes[routeId]?.status === "captured");
}
export async function readCaptureBundle(directory: string): Promise<CaptureBundle> {
const bundle = JSON.parse(await readFile(resolve(await realpath(directory), "bundle.json"), "utf8")) as CaptureBundle;
if (!bundle || bundle.contract !== "explore.bundle" || typeof bundle.gitCommit !== "string"
@@ -112,11 +110,6 @@ export async function readCaptureBundle(directory: string): Promise<CaptureBundl
throw new Error("Capture bundle completion timestamp is invalid");
}
if (bundle.appIcon !== undefined) assertAppIconRecord(bundle.appIcon);
if (bundle.sessionFailures !== undefined && (!Array.isArray(bundle.sessionFailures) || bundle.sessionFailures.some(failure => !failure
|| !["light", "dark"].includes(failure.scheme) || !failure.error || typeof failure.error.code !== "string" || typeof failure.error.message !== "string"
|| typeof failure.error.stage !== "string" || typeof failure.error.nextAction !== "string" || typeof failure.error.retryable !== "boolean"))) {
throw new Error("Capture bundle session failures are invalid");
}
if (bundle.launch !== undefined) assertLaunchRecords(bundle.launch);
const selected = new Set(bundle.selectedSurfaceIds);
if (selected.size !== bundle.selectedSurfaceIds.length
+1 -13
View File
@@ -1,5 +1,5 @@
import { unattributedIssue } from "../capture/health.js";
import { appearancePairs, bundleColorSchemes, failedPassOutcome, type ColorScheme } from "../capture/appearance.js";
import { appearancePairs, bundleColorSchemes, type ColorScheme } from "../capture/appearance.js";
import { launchSurfaceId, systemRegions as defaultSystemRegions, type SystemRegion } from "../contracts/capture.js";
import { readFileSync, statSync } from "node:fs";
import { stat } from "node:fs/promises";
@@ -79,16 +79,8 @@ export type PublicEvidenceSummary = {
comparison: { planned: number; eligible: number; skipped: number };
/** Every unavailable or unverified requested side, plus operational comparison errors. */
diagnostics: PublicEvidenceDiagnostic[];
/**
* Each appearance pass of a side that failed while its primary one captured (the bundle's `sessionFailures`), with how
* many of its frames the failure left unavailable: the comment names it once, not once per frame.
*/
sessionFailures?: PublicSessionFailure[];
};
export type PublicSessionFailure = { side: "base" | "head"; scheme: ColorScheme; frames: number;
code: string; stage: string; message: string; nextAction: string; retryable: boolean };
const MAX_SUMMARY_ERRORS = 5;
function recordError(errors: string[], surfaceId: string, error: unknown): void {
@@ -764,9 +756,6 @@ export async function assessPublicEvidence(options: {
}
}
const sessionFailures = (["base", "head"] as const).flatMap(side => (options[side].bundle.sessionFailures ?? []).map(({ scheme, error }): PublicSessionFailure => ({
side, scheme, frames: Object.keys(options[side].bundle.outcomes).filter(id => id.endsWith(`@${scheme}`) && failedPassOutcome(options[side].bundle, id)
&& options[side].bundle.outcomes[id]!.error?.message === error.message).length, ...error })));
return {
surfaces,
summary: {
@@ -774,7 +763,6 @@ export async function assessPublicEvidence(options: {
accessibility: accessibilityAvailable ? accessibilityCounts : null,
review: reviewCounts,
capture, comparison, diagnostics,
...(sessionFailures.length ? { sessionFailures } : {}),
},
};
}
-5
View File
@@ -55,11 +55,6 @@ export function ciErrorForCommand(error: unknown, command: string): CiError {
export type CiErrorRenderOptions = { secrets?: readonly string[] };
/** The values of every variable that names a secret (a token, secret, password or API key), to redact from what Explore writes. */
export function environmentSecrets(environment: NodeJS.ProcessEnv): string[] {
return Object.entries(environment).filter(([key]) => /(?:TOKEN|SECRET|PASSWORD|API_KEY)/i.test(key)).flatMap(([, value]) => value ? [value] : []);
}
function redact(value: string, secrets: readonly string[]): string {
let result = value;
// Controller failures may embed JSON or URLs inside their message rather than
-49
View File
@@ -4,11 +4,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { createServer } from "node:http";
import { startArgentBridgeRunner } from "../../dist/capture/argent-bridge-host.js";
import { argentClientCall } from "../../dist/capture/argent-bridge-worker.js";
import { argentRunFailureDetail } from "../../dist/capture/argent-bridge.js";
import { runBridgeCommand } from "../../dist/capture/argent-driver.js";
import { ARGENT_VERSION } from "../../dist/shared/argent-version.js";
const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
@@ -137,48 +133,3 @@ test("a recording runs per command, and a screenshot is taken only from a file t
assert.match(failed.error.message, /did not write/);
await assert.rejects(readFile(join(root, "out", "two.png")), { code: "ENOENT" });
});
test("the thread's lines about a tool-server still starting reach the capture's log with the command's answer", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
// Argent's client asks GET /tools before each call; the first one is answered 404, as on euxy's main baseline run 39506374824.
const clientModule = await fakeClient(root, `
if (name === "list-devices" && calls.filter(call => call.name === "list-devices").length === 1) throw new Error("GET /tools failed: 404 Not Found");
if (name === "list-devices") return { data: { devices: [{ udid: "UDID-1", name: "iPhone 17", state: "Booted", platform: "ios", runtime: "iOS 26.0" }] } };
if (name === "await-screen-idle") return { data: { settled: true } };
throw new Error("unexpected tool " + name);`);
const messages = [];
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/tools", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule, progress: message => messages.push(message),
fallback: () => assert.fail("the thread serves every command") });
t.after(() => bridge.close());
const waited = JSON.parse(bridge.runner({ command: "/opt/argent-controller", args: ["wait", "stable", "--session", "explore", "--platform", "ios", "--json"], cwd: root, environment, timeoutMs: 20_000 }).stdout);
assert.deepEqual(waited, { success: true, data: { stable: true } });
assert.deepEqual(messages, ["Argent's tool-server is not serving its tools yet (argent run list-devices failed: GET /tools failed: 404 Not Found); sending list-devices again in 1 s (75 s left of 75 s)"]);
});
test("a refused connection to a localhost tool-server, whose cause is an AggregateError with no message, is the tool-server still starting", async () => {
// A port nothing listens on: bound, then closed.
const port = await new Promise(resolve => { const server = createServer().listen(0, "127.0.0.1", () => { const { port } = server.address(); server.close(() => resolve(port)); }); });
const client = { async callTool() { const response = await fetch(`http://localhost:${port}/tools`); return { data: await response.json() }; } };
const call = argentClientCall(client, () => new AbortController().signal);
const refused = await call("list-devices", {}).then(() => assert.fail("nothing listens"), error => error);
assert.match(refused.message, /^argent run list-devices failed: fetch failed: .*ECONNREFUSED/);
// Through the bridge, the refusal is waited out like a 404 from /tools, until the window ends.
let now = 0, tries = 0;
const result = await runBridgeCommand(["snapshot", "--platform", "ios", "--json"], { state: { generation: 0 }, platform: "ios", version: "0.29.0",
now: () => now, sleep: async milliseconds => { now += milliseconds; }, reachable: async () => true, moveFile: async () => {},
call: async (tool, args, options) => { tries++; return call(tool, args, options); } });
assert.equal(JSON.parse(result.stdout).error.code, "TOOL_SERVER_NOT_READY");
assert.equal(tries, 11);
});
test("a failed argent run keeps the line saying /tools is not served ahead of the stack trace that follows it", () => {
const stderr = ["[argent] Unhandled rejection: Error: GET /tools failed: 404 Not Found", " at fetchTools (file:///argent/dist/cli-cmds.mjs:10065:21)",
" at async fetchTool (file:///argent/dist/cli-cmds.mjs:10070:19)", " at async run (file:///argent/dist/cli-cmds.mjs:31166:17)",
" at async main (file:///argent/dist/cli.js:12:3)"].join("\n");
const detail = argentRunFailureDetail(stderr, "");
assert.ok(detail.startsWith("[argent] Unhandled rejection: Error: GET /tools failed: 404 Not Found | at fetchTools "), detail);
assert.ok(detail.endsWith("at async main (file:///argent/dist/cli.js:12:3)"), "the last frames follow it");
assert.equal(argentRunFailureDetail("Tool \"gesture-tap\" not found. Run `argent tools` to list available tools.\n", ""), "Tool \"gesture-tap\" not found. Run `argent tools` to list available tools.");
assert.equal(argentRunFailureDetail("GET /tools failed: 503 Service Unavailable\n", ""), "GET /tools failed: 503 Service Unavailable", "a line named once");
});
-81
View File
@@ -302,87 +302,6 @@ test("an unreachable tool-server before a UI action is the retryable 'Remote dae
assert.ok(!calls.some(call => call.tool === "gesture-tap"));
});
// brentvatne/euxy's main baseline run 39506374824: a replacement session's tool-server answered 404 to /tools for the
// first list-devices of its install, before it served its tools.
const TOOLS_404 = "argent run list-devices failed: GET /tools failed: 404 Not Found";
/** A clock the harness's pauses (and, given `requestMs`, its requests) move forward. */
function withClock(context, requestMs = () => 0) {
const clock = { now: 0, pauses: [], timeouts: [] };
context.now = () => clock.now;
context.sleep = async milliseconds => { clock.pauses.push(milliseconds); clock.now += milliseconds; };
const call = context.call;
context.call = async (tool, args, options) => {
if (options?.timeoutMs !== undefined) clock.timeouts.push(options.timeoutMs);
clock.now += Math.min(requestMs(tool), options?.timeoutMs ?? Infinity);
return call(tool, args, options);
};
return clock;
}
test("a tool-server that does not serve /tools yet is asked again, with a log line, until it does", async () => {
const { run, calls, context } = harness("ios", { "list-devices": [new ArgentCallError(TOOLS_404, true), new ArgentCallError(TOOLS_404, true), IOS_DEVICES], "reinstall-app": {} }, { bundleId: BUNDLE });
const clock = withClock(context);
const lines = [];
context.progress = line => lines.push(line);
assert.deepEqual((await run(["install", "/work/bundle/euxy.app", ...flags("ios")])).json.data, { installed: true, appBundleId: BUNDLE });
assert.equal(calls.filter(call => call.tool === "list-devices").length, 3);
assert.deepEqual(clock.pauses, [1_000, 2_000]);
assert.deepEqual(lines, [
"Argent's tool-server is not serving its tools yet (argent run list-devices failed: GET /tools failed: 404 Not Found); sending list-devices again in 1 s (75 s left of 75 s)",
"Argent's tool-server is not serving its tools yet (argent run list-devices failed: GET /tools failed: 404 Not Found); sending list-devices again in 2 s (74 s left of 75 s)"]);
assert.equal(context.state.toolsServed, true);
assert.deepEqual(clock.timeouts, [75_000, 74_000, 72_000], "each try gets only what is left of the window");
assert.equal(calls.filter(call => call.tool === "reinstall-app")[0].options, undefined, "a served tool-server's calls keep their own timeout");
});
test("a refused connection or a 503 from /tools is the tool-server still starting too", async () => {
for (const failure of ["fetch failed: connect ECONNREFUSED 127.0.0.1:9", "GET /tools failed: 503 Service Unavailable"]) {
const starting = new ArgentCallError(`argent run list-devices failed: ${failure}`, true);
const { run, calls, context } = harness("ios", { "list-devices": [starting, starting, starting, IOS_DEVICES], "ui-tree": IOS_HOME });
withClock(context);
assert.equal((await run(["snapshot", ...flags("ios")])).exitCode, 0, failure);
assert.equal(calls.filter(call => call.tool === "list-devices").length, 4, "asked past the two read retries");
}
});
test("a tool-server that never serves /tools fails with a clear error 75 s after the first try", async () => {
const { run, calls, context } = harness("ios", { "list-devices": new ArgentCallError(TOOLS_404, true), "reinstall-app": {} }, { bundleId: BUNDLE });
const clock = withClock(context);
const failed = await run(["install", "/work/bundle/euxy.app", ...flags("ios")]);
assert.deepEqual(failed.json.error, { code: "TOOL_SERVER_NOT_READY", message: `Argent's tool-server did not start serving its tools within 75 s: ${TOOLS_404}` });
assert.equal(clock.now, 75_000);
assert.equal(calls.filter(call => call.tool === "list-devices").length, 11);
assert.ok(!calls.some(call => call.tool === "reinstall-app"));
});
test("slow answers count against the 75 s too: each try is cut to the time left, and the wait ends on time", async () => {
const slow = new ArgentCallError("argent run list-devices failed: GET /tools failed: 503 Service Unavailable", true);
const { run, calls, context } = harness("ios", { "list-devices": slow, "reinstall-app": {} }, { bundleId: BUNDLE });
// Each 503 takes 30 s to come back, as a gateway's might.
const clock = withClock(context, () => 30_000);
assert.equal((await run(["install", "/work/bundle/euxy.app", ...flags("ios")])).json.error.code, "TOOL_SERVER_NOT_READY");
assert.equal(clock.now, 75_000, "not eleven 30 s tries");
assert.deepEqual(clock.timeouts, [75_000, 44_000, 12_000]);
assert.equal(calls.filter(call => call.tool === "list-devices").length, 3);
});
test("an unknown tool's 404 after the tool-server is up fails at once, and a later /tools 404 keeps only the read retries", async () => {
const unknown = 'argent run gesture-tap failed: Tool "gesture-tap" not found';
const { run, calls, context } = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": IOS_HOME, "gesture-tap": new ArgentCallError(unknown, true) });
const pauses = [];
context.sleep = async milliseconds => { pauses.push(milliseconds); };
assert.deepEqual((await run(["press", "10", "10", ...flags("ios")])).json.error, { code: "ARGENT_TOOL_FAILED", message: unknown });
assert.equal(calls.filter(call => call.tool === "gesture-tap").length, 1);
assert.deepEqual(pauses, []);
const served = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": new ArgentCallError("argent run ui-tree failed: GET /tools failed: 404 Not Found", true) },
{ state: { generation: 0, udid: UDID, toolsServed: true } });
served.context.sleep = async milliseconds => { pauses.push(milliseconds); };
assert.equal((await served.run(["snapshot", ...flags("ios")])).json.error.code, "ARGENT_TOOL_FAILED");
assert.equal(served.calls.filter(call => call.tool === "ui-tree").length, 3);
assert.deepEqual(pauses, [1_000, 2_000]);
});
test("a UI action whose answer is lost is reported once and never retried", async () => {
const { run, calls } = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": IOS_HOME, "gesture-tap": new ArgentCallError("argent run gesture-tap failed: socket hang up", true) });
const pressed = await run(["press", "10", "10", ...flags("ios")]);
+8 -68
View File
@@ -9,9 +9,7 @@ import { AGENT_DEVICE_VERSION, EAS_CLI_VERSION } from "../../dist/capture/run.js
import { readExportedMap } from "../../dist/review/evidence-assessment.js";
import { ExploreStore, recordObservation } from "../../dist/map/store.js";
const { captureRecipeConfig, incompleteCaptureReason, restoreCaptureArtifactCache, validateEvidence, writeCaptureArtifactCache } = await tsImport("../../src/ci/artifact-cache.ts", import.meta.url);
const { mergeCaptureExports } = await tsImport("../../src/ci/export-merge.ts", import.meta.url);
const { captureHealthIssues, partitionCaptureHealthIssues } = await tsImport("../../src/capture/health.ts", import.meta.url);
const { captureRecipeConfig, restoreCaptureArtifactCache, writeCaptureArtifactCache } = await tsImport("../../src/ci/artifact-cache.ts", import.meta.url);
const { readCaptureBundle } = await tsImport("../../src/review/capture-review.ts", import.meta.url);
const { CAPTURE_OBSERVER_HASH } = await tsImport("../../src/capture/observer.ts", import.meta.url);
const { AGENT_ACCEPTANCE_POLICY } = await tsImport("../../src/contracts/capture.ts", import.meta.url);
@@ -38,7 +36,7 @@ function acceptedFrames(bundle) {
}])) };
}
async function fixture(t, { missingAccessibility = false, partial = false, appearances = false, darkFrames = appearances } = {}) {
async function fixture(t, { missingAccessibility = false, partial = false } = {}) {
const root = await mkdtemp(join(tmpdir(), "explore-capture-artifact-cache-"));
t.after(() => rm(root, { recursive: true, force: true }));
const store = new ExploreStore(join(root, "active"));
@@ -49,7 +47,7 @@ async function fixture(t, { missingAccessibility = false, partial = false, appea
id: "capture-run", name: "Cache evidence", platform: "ios", controller: "agent-device", status: "draft",
createdAt: timestamp, updatedAt: timestamp,
build: { id: "eas-build", gitCommit: COMMIT, source: "eas" },
environment: { ...RUNTIME, authState: "logged-out", ...(appearances ? { colorSchemes: ["light", "dark"] } : {}) },
environment: { ...RUNTIME, authState: "logged-out" },
limits: { maxScreens: 20, maxActions: 100, maxDurationMinutes: 60 },
},
screens: [], variants: [], observations: [], elements: [], artifacts: [],
@@ -60,7 +58,7 @@ async function fixture(t, { missingAccessibility = false, partial = false, appea
const snapshotPath = join(root, "sample.json");
await writeFile(screenshotPath, PNG);
const images = {};
for (const id of darkFrames ? ["home", "settings", "home@dark", "settings@dark"] : ["home", "settings"]) {
for (const id of ["home", "settings"]) {
const snapshotText = JSON.stringify({ nodes: [{ type: "text", label: id }] });
await writeFile(snapshotPath, snapshotText);
const result = await recordObservation({
@@ -83,17 +81,15 @@ async function fixture(t, { missingAccessibility = false, partial = false, appea
await store.exportTo(sourceDirectory, map);
const bundle = {
contract: "explore.bundle", side: "head", gitCommit: COMMIT,
easBuildId: "eas-build", easProjectId: "eas-project", profile: RECIPE.profile, runtimeProfile: appearances ? { ...RUNTIME, colorSchemes: "light,dark" } : RUNTIME,
easBuildId: "eas-build", easProjectId: "eas-project", profile: RECIPE.profile, runtimeProfile: RUNTIME,
selectedSurfaceIds: ["home", "settings"], images,
// A dark frame verified beside its light one has no outcome of its own, as the passes merge them.
outcomes: Object.fromEntries(Object.entries(images).filter(([id]) => !id.endsWith("@dark"))
.map(([id, image]) => [id, { status: "captured", attempts: 1, landing: "verified", screenshotSha256: image.sha256 }])),
outcomes: Object.fromEntries(Object.entries(images).map(([id, image]) => [id, { status: "captured", attempts: 1, landing: "verified", screenshotSha256: image.sha256 }])),
};
await writeFile(join(sourceDirectory, "bundle.json"), JSON.stringify(bundle));
await writeFile(join(sourceDirectory, "execution.json"), JSON.stringify(EXECUTION));
// Construct actual exported, validated evidence; this fixture cannot pass with stub JSON maps.
assert.equal((await readExportedMap(sourceDirectory)).observations.length, darkFrames ? 4 : 2);
const options = { ...RECIPE, sourceDirectory, directory: join(root, "restored"), commit: COMMIT, selectedIds: ["home", "settings"], runtimeProfile: bundle.runtimeProfile };
assert.equal((await readExportedMap(sourceDirectory)).observations.length, 2);
const options = { ...RECIPE, sourceDirectory, directory: join(root, "restored"), commit: COMMIT, selectedIds: ["home", "settings"], runtimeProfile: RUNTIME };
return { root, sourceDirectory, map, bundle, options };
}
@@ -306,62 +302,6 @@ test("partial captures reuse only the screens with complete successful evidence"
assert.equal(await restoreCaptureArtifactCache({ ...f.options, directory: join(f.root, "failed-only"), selectedIds: ["settings"] }), false);
});
test("a baseline whose dark session failed is cached, and a review captures the routes without a dark frame itself", async t => {
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route and the dark session failed.
const failedDark = { status: "failed", attempts: 0, landing: "unverified", reason: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
error: { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found", nextAction: "Rerun the capture.", retryable: false } };
const f = await fixture(t, { partial: true, appearances: true });
const complete = structuredClone(f.bundle);
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), true);
assert.equal(await restoreCaptureArtifactCache(f.options), true, "a baseline with every appearance is reused whole");
assert.deepEqual(Object.keys((await readCaptureBundle(f.options.directory)).images).sort(), ["home", "home@dark", "settings", "settings@dark"]);
for (const id of ["home", "settings"]) { delete f.bundle.images[`${id}@dark`]; f.bundle.outcomes[`${id}@dark`] = failedDark; }
f.bundle.sessionFailures = [{ scheme: "dark", error: failedDark.error }];
await writeFile(join(f.sourceDirectory, "bundle.json"), JSON.stringify(f.bundle));
// No route has its dark frame: offering it would make a review wait for it and capture its sides in turn, only to
// capture every route of the base itself.
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), false, "a baseline with nothing to reuse is not offered");
assert.equal(await restoreCaptureArtifactCache({ ...f.options, directory: join(f.root, "no-dark") }), false);
assert.equal(await incompleteCaptureReason(f.sourceDirectory, COMMIT), "its dark session failed, so no route has every appearance");
assert.equal(await incompleteCaptureReason(f.sourceDirectory, "b".repeat(40)), undefined, "another commit's baseline is the usual miss");
const mixed = structuredClone(complete);
delete mixed.images["settings@dark"]; mixed.outcomes["settings@dark"] = failedDark; delete mixed.sessionFailures;
await writeFile(join(f.sourceDirectory, "bundle.json"), JSON.stringify(mixed));
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), true);
assert.equal(await restoreCaptureArtifactCache({ ...f.options, directory: join(f.root, "mixed") }), true);
const restored = await readCaptureBundle(join(f.root, "mixed"));
assert.deepEqual(restored.selectedSurfaceIds, ["home"], "settings has no dark frame, so the review's base captures it in both appearances");
assert.deepEqual(Object.keys(restored.images).sort(), ["home", "home@dark"]);
assert.deepEqual(Object.keys(restored.outcomes), ["home"]);
});
test("a light session's real export merged with a failed dark session is valid evidence, a valid map, and healthy but for dark", async t => {
// The light session of a side split over sessions: it renders light and dark, and took light alone.
const f = await fixture(t, { appearances: true, darkFrames: false });
const error = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture.", retryable: true };
// The light session lost /settings.
delete f.bundle.images.settings;
f.bundle.outcomes.settings = { status: "failed", attempts: 2, landing: "unverified", reason: "The route observer never answered." };
await writeFile(join(f.sourceDirectory, "bundle.json"), JSON.stringify(f.bundle));
await mergeCaptureExports(f.sourceDirectory, [{ scheme: "dark", error }]);
const map = await readExportedMap(f.sourceDirectory);
assert.equal(map.run.status, "partial");
assert.equal(map.run.stopReason, `The dark session failed (CAPTURE_SESSION_FAILED): ${error.message}.`);
const bundle = await readCaptureBundle(f.sourceDirectory);
assert.deepEqual(Object.keys(bundle.outcomes).sort(), ["home", "home@dark", "settings", "settings@dark"]);
// Each dark frame carries the dark session's reason, the one light lost too: the review shows it, not light's.
assert.equal(bundle.outcomes["settings@dark"].reason, error.message);
assert.equal(bundle.outcomes.settings.reason, "The route observer never answered.");
await validateEvidence(f.sourceDirectory, bundle);
const issues = partitionCaptureHealthIssues(captureHealthIssues(bundle, ["home", "settings"]));
assert.deepEqual(issues.integrity, []);
assert.deepEqual(issues.unavailable.map(issue => [issue.surfaceId, issue.code]), [["home@dark", "CAPTURE_SESSION_FAILED"], ["settings", "CAPTURE_FAILED"]]);
assert.equal(await writeCaptureArtifactCache(f.sourceDirectory, RECIPE), false, "valid, but nothing in it a review could reuse");
});
test("partial caches retain verified targets without restoring captured but unverified screenshots", async t => {
const f = await fixture(t, { partial: true });
f.bundle.outcomes.settings.landing = "unverified";
-47
View File
@@ -562,36 +562,6 @@ test('an app that follows the system appearance is captured light then dark, dar
assert.equal(execution.colorScheme, 'light');
});
test('a single session whose dark pass fails keeps its light captures and marks every dark frame failed, as split sessions do', async t => {
// brentvatne/euxy's main baseline run 39506374824 split the appearances over two sessions; one session must read the same.
const f = await fixture(t, { userInterfaceStyle: 'automatic' });
const lines = [];
const result = await runCiMap({ ...f.options, progress: line => lines.push(line), runner: request => {
if (request.args[0] === 'settings' && request.args[2] === 'dark') throw new Error('Controller settings exited with status 1: the device stopped answering');
return f.options.runner(request);
} });
assert.equal(result.runStatus, 'partial');
const bundle = JSON.parse(await readFile(join(f.options.outputDirectory, 'bundle.json'), 'utf8'));
assert.deepEqual(Object.keys(bundle.images).sort(), [f.ids.home, f.ids.settings].sort());
for (const id of Object.values(f.ids)) {
assert.equal(bundle.outcomes[id].status, 'captured');
assert.equal(bundle.outcomes[`${id}@dark`].status, 'failed');
assert.match(bundle.outcomes[`${id}@dark`].error.message, /the device stopped answering/);
}
assert.deepEqual(bundle.sessionFailures.map(failure => failure.scheme), ['dark']);
const map = JSON.parse(await readFile(join(f.options.outputDirectory, 'screen-map.json'), 'utf8'));
assert.equal(map.run.status, 'partial');
assert.match(map.run.stopReason, /^The dark pass failed \(/);
assert.ok(lines.some(line => /the dark pass failed .*keeping the light captures/.test(line)), lines.join('\n'));
assert.equal(f.operations().filter(operation => operation === 'simulator:stop').length, 1);
// A checkout that changed under the dark pass casts doubt on the light frames too: the capture fails.
const g = await fixture(t, { userInterfaceStyle: 'automatic' });
await assert.rejects(runCiMap({ ...g.options, runner: request => {
if (request.args[0] === 'settings' && request.args[2] === 'dark') throw new CiError({ code: 'SOURCE_CHECKOUT_CHANGED', stage: 'source', message: 'The application checkout differs from its recorded commit.', nextAction: 'Rerun.' });
return g.options.runner(request);
} }), error => error.code === 'SOURCE_CHECKOUT_CHANGED');
});
test('an app pinned to one appearance is captured once, in that appearance', async t => {
const f = await fixture(t, { userInterfaceStyle: 'dark' });
const result = await runCiMap(f.options);
@@ -810,15 +780,6 @@ test('a start that fails before naming a session stops any session of that name
assert.equal(always.requests.filter(request => request.args[0] === 'simulator:stop' && value(request, '--id') === 'orphan-1').length, 6);
});
test('a session a failed start left that does not stop is reported as left running, not as a start failure', async t => {
// No session file names it, so a split side would otherwise keep its other appearance and hide it.
const f = await fixture(t, { startsUnnamed: 1, orphanFromFailedStart: true });
await assert.rejects(runCiMap({ ...f.options, sleep: () => {}, runner: request => {
if (request.args[0] === 'simulator:stop' && value(request, '--id') === 'orphan-1') throw new Error('Controller simulator:stop exited with status 1: network unreachable');
return f.options.runner(request);
} }), error => error.code === 'SESSION_CLEANUP_FAILED' && /orphan-1/.test(error.message));
});
test('a session of that name Explore cannot prove is its own is never stopped or replaced', async t => {
const f = await fixture(t, { startsUnnamed: 1, orphanFromFailedStart: true, orphanUnproven: true });
await assert.rejects(runCiMap({ ...f.options, sleep: () => {} }), error => error.code === 'SIMULATOR_START_FAILED');
@@ -1982,14 +1943,6 @@ test('a failed hosted repack job stops the capture as EAS_REPACK_FAILED with the
assert.deepEqual(f.operations().filter(operation => operation.startsWith('simulator:')).filter(operation => operation !== 'simulator:availability'), ['simulator:start', 'simulator:stop']);
});
test('a session started beside a repack that failed, and that does not stop, is the failure reported', async t => {
const f = await fixture(t, { workflowFailure: true });
await assert.rejects(runCiMap({ ...f.options, runner: request => {
if (request.args[0] === 'simulator:stop') throw new Error('Controller simulator:stop exited with status 1: network unreachable');
return f.options.runner(request);
} }), error => error.code === 'SESSION_CLEANUP_FAILED' && error.details.originalFailure.code === 'EAS_REPACK_FAILED');
});
const runs = f => f.operations().filter(operation => operation === 'workflow:run').length;
const sessionOperations = f => f.operations().filter(operation => /^simulator:(start|stop)$/.test(operation));
// The upload step of brentvatne/euxy#176 (review run 39506405687), refused once the build had repacked.
+1 -112
View File
@@ -6,8 +6,6 @@ import test from "node:test";
import { mergeSchemeOutcomes } from "../../dist/capture/appearance.js";
import { mergeCaptureExports } from "../../dist/ci/export-merge.js";
import { captureSideInSessions } from "../../dist/ci/pipeline.js";
import { CiError } from "../../dist/shared/ci-errors.js";
async function writeExport(root, { screens, images, outcomes, launch, diagnostics }) {
await mkdir(join(root, "artifacts"), { recursive: true });
@@ -22,8 +20,7 @@ async function writeExport(root, { screens, images, outcomes, launch, diagnostic
screens: screens.map(id => ({ id, colorScheme: id.endsWith("@dark") ? "dark" : "light" })), variants: [], observations: screens.map(id => ({ screenId: id })),
elements: [], actionAttempts: [], transitions: [], journeys: [], frontier: [], gaps: [] };
await writeFile(join(root, "screen-map.json"), JSON.stringify(map));
await writeFile(join(root, "bundle.json"), JSON.stringify({ contract: "explore.bundle", side: "head", runtimeProfile: { colorSchemes: "light,dark" },
selectedSurfaceIds: [...new Set(Object.keys(outcomes).map(id => id.replace(/@(light|dark)$/, "")))], images, outcomes, ...(launch ? { launch } : {}) }));
await writeFile(join(root, "bundle.json"), JSON.stringify({ contract: "explore.bundle", side: "head", runtimeProfile: { colorSchemes: "light,dark" }, images, outcomes, ...(launch ? { launch } : {}) }));
await writeFile(join(root, "manifest.json"), JSON.stringify({ contract: "explore.map", runId: "run", revision: 1, files: artifacts.map(({ id: _id, ...file }) => file) }));
await writeFile(join(root, "capture-diagnostics", "controller-timing.jsonl"), diagnostics);
return root;
@@ -97,111 +94,3 @@ test("a dark session that could not show dark (Argent without appearance.set) ad
await mergeCaptureExports(primary, [{ directory: gapped, scheme: "dark" }]);
assert.equal(JSON.parse(await readFile(mapPath, "utf8")).run.status, "partial", "a complete run has no open gaps");
});
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route, then the dark session's install failed.
const DARK_FAILURE = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture on a new session.", retryable: true };
const failedDark = { status: "failed", attempts: 0, landing: "unverified", reason: DARK_FAILURE.message, error: DARK_FAILURE };
test("a dark session that left no export records its failure as every route's own dark outcome, and the run is partial", async () => {
const root = await mkdtemp(join(tmpdir(), "explore-merge-"));
const lost = { status: "failed", landing: "unverified", attempts: 3, reason: "The route observer never answered." };
const primary = await writeExport(join(root, "head-export"), { screens: ["home"], images: { home: { path: "artifacts/home.png" } },
outcomes: { home: { status: "captured", landing: "verified", attempts: 1 }, settings: lost }, diagnostics: "" });
const mapPath = join(primary, "screen-map.json");
await writeFile(mapPath, JSON.stringify({ ...JSON.parse(await readFile(mapPath, "utf8")), run: { id: "run", status: "complete", environment: { colorSchemes: ["light", "dark"] } } }));
await mergeCaptureExports(primary, [{ scheme: "dark", error: DARK_FAILURE }]);
const bundle = JSON.parse(await readFile(join(primary, "bundle.json"), "utf8"));
// A route the light session lost keeps its own reason, and its dark frame carries the dark session's, not the light one's.
assert.deepEqual(bundle.outcomes, { home: { status: "captured", landing: "verified", attempts: 1 }, settings: lost, "home@dark": failedDark, "settings@dark": failedDark });
assert.deepEqual(bundle.sessionFailures, [{ scheme: "dark", error: DARK_FAILURE }]);
assert.deepEqual(Object.keys(bundle.images), ["home"]);
const map = JSON.parse(await readFile(mapPath, "utf8"));
assert.equal(map.run.status, "partial");
assert.equal(map.run.stopReason, `The dark session failed (CAPTURE_SESSION_FAILED): ${DARK_FAILURE.message}.`);
});
test("a dark session that failed beside a primary that renders no dark has no frames to fail, and stays on the bundle", async () => {
const root = await mkdtemp(join(tmpdir(), "explore-merge-"));
const primary = await writeExport(join(root, "head-export"), { screens: ["home"], images: { home: { path: "artifacts/home.png" } },
outcomes: { home: { status: "captured", landing: "verified", attempts: 1 } }, diagnostics: "" });
const bundlePath = join(primary, "bundle.json");
// The light session could not switch appearance (Argent without appearance.set): it captured light alone.
await writeFile(bundlePath, JSON.stringify({ ...JSON.parse(await readFile(bundlePath, "utf8")), runtimeProfile: { colorScheme: "light" } }));
await mergeCaptureExports(primary, [{ scheme: "dark", error: DARK_FAILURE }]);
const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
assert.deepEqual(Object.keys(bundle.outcomes), ["home"]);
assert.deepEqual(bundle.sessionFailures, [{ scheme: "dark", error: DARK_FAILURE }], "the comment still names the failed session");
assert.equal(JSON.parse(await readFile(join(primary, "screen-map.json"), "utf8")).run.status, "partial");
});
/** The side's sessions as captureSideInSessions runs them: the light one exports, the dark one does what `dark` says. */
async function sideInSessions({ light, dark, leftSessionFile = false, environment = {}, signal } = {}) {
const root = await mkdtemp(join(tmpdir(), "explore-sessions-"));
const progress = [];
const mapOptions = { side: "head", outputDirectory: join(root, "head-export"), mapDirectory: join(root, "head-map"), buildShare: join(root, "builds"),
sessionStatePath: join(root, "head-session"), expoToken: "expo-secret-token", environment };
const runPass = async options => {
const scheme = options.appearancePass.capture[0];
const outcome = scheme === "light" ? light : dark;
// A capture removes its session file only once its session is confirmed stopped.
if (scheme === "dark" && leftSessionFile) await writeFile(options.sessionStatePath, "01a15b61-4df9\n");
if (outcome instanceof Error) throw outcome;
const id = scheme === "light" ? "home" : "home@dark";
await writeExport(options.outputDirectory, { screens: [id], images: { [id]: { path: `artifacts/${id}.png` } },
outcomes: { [id]: { status: "captured", landing: "verified", attempts: 1 } }, diagnostics: "" });
return { runStatus: "complete", outputDirectory: options.outputDirectory, scheme };
};
const run = () => captureSideInSessions(mapOptions, [["light"], ["dark"]], line => progress.push(line), signal, true, runPass);
return { root, progress, run, bundle: async () => JSON.parse(await readFile(join(mapOptions.outputDirectory, "bundle.json"), "utf8")) };
}
test("a side keeps its light captures when only its dark session fails, and the report marks every dark frame failed", async () => {
const side = await sideInSessions({ dark: new CiError({ ...DARK_FAILURE, message: `${DARK_FAILURE.message} (tokens expo-secret-token, gh-secret)` }),
environment: { GITHUB_TOKEN: "gh-secret", PATH: "/usr/bin" } });
const result = await side.run();
assert.equal(result.scheme, "light", "the primary session's result");
assert.equal(result.runStatus, "partial");
const bundle = await side.bundle();
assert.deepEqual(Object.keys(bundle.images), ["home"]);
assert.equal(bundle.outcomes["home@dark"].status, "failed");
assert.equal(bundle.outcomes["home@dark"].error.code, "CAPTURE_SESSION_FAILED");
assert.doesNotMatch(JSON.stringify(bundle), /expo-secret-token|gh-secret/, "the session's error is stored without any secret the environment holds");
assert.ok(side.progress.some(line => /^head: the dark session failed \(CAPTURE_SESSION_FAILED: install: argent run list-devices failed/.test(line)
&& line.endsWith("keeping the light captures, and the report marks every dark frame failed")), side.progress.join("\n"));
});
test("a side whose light session fails still fails, whatever the dark session did", async () => {
const light = new CiError({ ...DARK_FAILURE, code: "CAPTURE_INSTALL_FAILED" });
await assert.rejects((await sideInSessions({ light })).run(), { code: "CAPTURE_INSTALL_FAILED" });
await assert.rejects((await sideInSessions({ light, dark: new CiError({ ...DARK_FAILURE }) })).run(), { code: "CAPTURE_INSTALL_FAILED" });
});
test("a dark session not proven stopped fails the side, so a running session is never hidden behind a partial report", async () => {
const cleanup = new CiError({ code: "SESSION_CLEANUP_FAILED", stage: "cleanup", message: "The exact EAS session could not be confirmed stopped.",
nextAction: "Stop EAS Simulator session 01a15b61 and verify its status.", retryable: false });
await assert.rejects((await sideInSessions({ dark: cleanup })).run(), { code: "SESSION_CLEANUP_FAILED" });
await assert.rejects((await sideInSessions({ light: new CiError({ ...DARK_FAILURE, code: "CAPTURE_INSTALL_FAILED" }), dark: cleanup })).run(), { code: "SESSION_CLEANUP_FAILED" },
"the leaked session is reported before the light session's own failure");
const handoff = new CiError({ code: "SESSION_HANDOFF_UNVERIFIED", stage: "session", message: "EAS did not provide an exact verifiable simulator session.",
nextAction: "Verify a session's exact ID and ownership before cleanup." });
await assert.rejects((await sideInSessions({ dark: handoff })).run(), { code: "SESSION_HANDOFF_UNVERIFIED" });
const unverified = new CiError({ code: "SIMULATOR_START_FAILED", stage: "session", message: "EAS simulator:start failed before a session identity could be verified.",
nextAction: "Verify a session's exact ID and ownership before cleanup.", details: { allocation: "unknown" } });
await assert.rejects((await sideInSessions({ dark: unverified })).run(), { code: "SIMULATOR_START_FAILED" });
// A worker that died before its capture's own cleanup ran: it exited, or failed with a raw error.
const exited = new CiError({ code: "CAPTURE_WORKER_EXITED", stage: "capture-head", message: "The capture worker for the head side exited with code 1 before it reported a result.",
nextAction: "Inspect the job log for the worker's failure, then rerun Explore." });
await assert.rejects((await sideInSessions({ dark: exited })).run(), { code: "CAPTURE_WORKER_EXITED" });
await assert.rejects((await sideInSessions({ dark: new Error("Worker terminated due to reaching memory limit: JS heap out of memory") })).run(), /memory limit/);
// Any failure that left the session's file behind: the session was never confirmed stopped.
await assert.rejects((await sideInSessions({ dark: new CiError({ ...DARK_FAILURE }), leftSessionFile: true })).run(), { code: "CAPTURE_SESSION_FAILED" });
});
test("a dark session whose checkout changed, whose evidence failed its checks, or whose review was stopped fails the side", async () => {
for (const code of ["SOURCE_CHECKOUT_CHANGED", "CHECKOUT_MODIFIED", "CAPTURE_INCOMPLETE", "CAPTURE_IMAGE_BINDING_INVALID", "EXPLORE_FAILED"]) {
await assert.rejects((await sideInSessions({ dark: new CiError({ ...DARK_FAILURE, code }) })).run(), { code }, code);
}
const stop = new AbortController(); stop.abort(new CiError({ code: "LOCAL_REVIEW_INTERRUPTED", stage: "capture", message: "Interrupted.", nextAction: "Rerun." }));
await assert.rejects((await sideInSessions({ dark: new CiError({ ...DARK_FAILURE }), signal: stop.signal })).run(), { code: "CAPTURE_SESSION_FAILED" });
});
+1 -14
View File
@@ -5,7 +5,7 @@ import { spawn, spawnSync } from 'node:child_process';
import { lstat, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { appEnvironment, captureCacheDirectory, defaultBase, installDependencies, keepCapture, localInstallCommand, localReviewConfig, reportRef, pruneTargetOutput, reviewHeadSource, runLocalReview, snapshotWorkingTree, stopRecordedSessions } from '../../dist/ci/local-review.js';
import { appEnvironment, captureCacheDirectory, defaultBase, installDependencies, keepCapture, localInstallCommand, localReviewConfig, reportRef, reviewHeadSource, runLocalReview, snapshotWorkingTree, stopRecordedSessions } from '../../dist/ci/local-review.js';
import { mkdir, readdir } from 'node:fs/promises';
import { startLocalReportServer } from '../../dist/report/local-report-server.js';
import { loadReport } from '../../dist/report/agent-api.js';
@@ -364,19 +364,6 @@ test('an interrupted review deletes the local simulators it recorded and stops i
await assert.rejects(lstat(join(output, 'head-session')));
});
test('a finished target keeps its report and any session file, so the review still stops a session no capture confirmed stopped', async t => {
const output = await realpath(await mkdtemp(join(tmpdir(), 'explore-pruned-')));
t.after(() => rm(output, { recursive: true, force: true }));
for (const name of ['report.zip', 'head-session-dark', 'head-export', 'base-export-dark', 'head-map']) await writeFile(join(output, name), 'x\n');
await mkdir(join(output, 'report'));
await pruneTargetOutput(output);
assert.deepEqual((await readdir(output)).sort(), ['head-session-dark', 'report', 'report.zip']);
const eas = [];
await writeFile(join(output, 'head-session-dark'), '01a15b61-4df9\n');
assert.deepEqual(await stopRecordedSessions([output], {}, args => { eas.push(args); return { status: 0 }; }, true), ['01a15b61-4df9']);
assert.deepEqual(eas, [['simulator:stop', '--id', '01a15b61-4df9', '--json', '--non-interactive']]);
});
test('a local review captures every target of the config at once, two of them on one platform', async t => {
const targets = { phone: { platform: 'ios' }, 'duo-open': { platform: 'ios', device: 'iPhone Duo', pose: 'open' } };
const f = await fixture(t, 'expo-router', { targets });
-32
View File
@@ -325,38 +325,6 @@ test('a failed dark pass on a route that captured in light reads as a valid bund
assert.equal(dark.headOutcome?.error?.code, 'CAPTURE_OBSERVER_MISSING');
});
test('a route light lost and whose dark session failed reviews dark with the dark session\'s reason, not light\'s', async t => {
// brentvatne/euxy's main baseline run 39506374824: the dark session's install failed; here light also lost the route.
const f = await captureFixture(t);
const error = { code: 'CAPTURE_SESSION_FAILED', stage: 'session', message: 'install: argent run list-devices failed: GET /tools failed: 404 Not Found',
nextAction: 'Rerun the capture on a new session.', retryable: true };
for (const side of ['base', 'head']) {
const bundle = f.bundles[side];
bundle.runtimeProfile = { ...runtimeProfile, colorSchemes: 'light,dark' };
if (side === 'base') {
await writeFile(join(f.root, side, 'home-dark.png'), png);
bundle.images[`${f.id}@dark`] = { ...bundle.images[f.id], path: 'home-dark.png' };
} else {
delete bundle.images[f.id];
bundle.outcomes[f.id] = { status: 'failed', attempts: 2, landing: 'unverified', reason: 'The route observer never answered.' };
bundle.outcomes[`${f.id}@dark`] = { status: 'failed', attempts: 0, landing: 'unverified', reason: error.message, error };
bundle.sessionFailures = [{ scheme: 'dark', error }];
}
await writeFile(join(f.root, side, 'bundle.json'), JSON.stringify(bundle));
}
await readCaptureBundle(f.options.headDirectory);
const result = await prepareCaptureReview(f.options);
assertPublicReview(result.review);
const [light, dark] = result.review.surfaces;
assert.equal(light.headOutcome.reason, 'The route observer never answered.');
assert.equal(dark.headOutcome.error.code, 'CAPTURE_SESSION_FAILED');
assert.equal(dark.headOutcome.reason, error.message);
// Without the session's record, a failed dark outcome beside a failed light one is not one the passes produce.
delete f.bundles.head.sessionFailures;
await writeFile(join(f.root, 'head', 'bundle.json'), JSON.stringify(f.bundles.head));
await assert.rejects(readCaptureBundle(f.options.headDirectory), /inventory is inconsistent/);
});
test('a head that gained a light appearance keeps its route in source navigation on the dark-only base', async t => {
const f = await captureFixture(t);
f.bundles.base.runtimeProfile = { ...runtimeProfile, colorSchemes: 'dark' };
-40
View File
@@ -268,46 +268,6 @@ test('capture problems are counted on the target\'s line and listed, collapsed,
assert.doesNotMatch(body, /needs review|Modified|old capture|old destination|token=private|\*\*iOS\*\*|Validation/);
});
test('a target whose dark session failed reads as ready, says dark was not captured, and names the session once', async () => {
const { readCommentRecords } = await import('../../dist/report/pr-comment-body.js');
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route, the dark session's install failed.
const error = { code: 'CAPTURE_SESSION_FAILED', stage: 'session', message: 'install: argent run list-devices failed: GET /tools failed: 404 Not Found',
nextAction: 'Rerun the capture on a new session.', retryable: true };
const unchanged = { review: { status: 'unchanged', description: 'No visible change' }, visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: '[email protected]' } };
const surfaces = [];
// 24 routes: more dark frames than the comment lists issues, and one light route the light session lost.
for (let index = 0; index < 24; index++) {
const id = `route-${index}`;
const light = surface(id, 'modified', { extra: { colorScheme: 'light', ...unchanged } });
if (index === 23) { delete light.headImage; delete light.review; light.headOutcome = { status: 'failed', attempts: 2, landing: 'unverified', reason: 'The route observer never answered.' }; }
const dark = surface(`${id}@dark`, 'modified', { route: `/${id}`, extra: { colorScheme: 'dark', headOutcome: { status: 'failed', attempts: 0, landing: 'unverified', reason: error.message, error } } });
delete dark.headImage;
surfaces.push(light, dark);
}
const health = { planned: 24, captured: 24, verified: 24, unverified: 0, unavailable: 0 };
const evidence = { visual: null, accessibility: null, capture: { base: health, head: { ...health, captured: 23, verified: 23, unavailable: 1 } },
comparison: { planned: 24, eligible: 23, skipped: 0 }, diagnostics: [], sessionFailures: [{ side: 'head', scheme: 'dark', frames: 24, ...error }] };
const body = renderPublicComment(state({ review: review(surfaces), evidence, capture: { planned: { base: 24, head: 24 }, done: { base: 24, head: 23 }, baseFromCache: true } }));
assert.doesNotMatch(body, /failed\*\*|capture failed|🟥/, 'one appearance missing is not a failed review');
assert.match(titleOf(body), /^### No compared screens changed · dark not captured · /);
assert.equal(statusOf(body, 'iOS'), `iOS · dark not captured · 1 capture unavailable · ${runLink}`);
assert.deepEqual(summaries(body), ['Capture issues (2)'], 'the session once, and the light route it lost');
assert.match(body, /\n- Head · dark session · CAPTURE\\_SESSION\\_FAILED: install\\: argent run list\\-devices failed\\: GET \/tools failed\\: 404 Not Found\\. 24 dark frames are unavailable\\. Next action: Rerun the capture on a new session\\\.\n/);
assert.match(body, /\n- Head · \/route\\-23 · CAPTURE\\_UNAVAILABLE: The route observer never answered\\. /, 'the light route is not crowded out');
assert.doesNotMatch(body, /\(dark\\\) · CAPTURE\\_SESSION\\_FAILED/, 'no dark frame repeats the session');
const record = readCommentRecords(body).current.ios;
assert.deepEqual(record.sessionFailures, [{ side: 'head', scheme: 'dark', frames: 24 }], 'the record keeps it, so a republished comment says it too');
});
test('a dark issue that is not the session\'s is still listed with its appearance', () => {
const error = { code: 'CAPTURE_OBSERVER_MISSING', stage: 'capture', message: 'The capture build has not exposed its active navigation destination.', nextAction: 'Rerun capture.', retryable: true };
const dark = surface('settings@dark', 'modified', { route: '/settings', extra: { colorScheme: 'dark', headOutcome: { status: 'failed', attempts: 2, landing: 'unverified', reason: error.message, error } } });
delete dark.headImage;
const body = renderPublicComment(state({ review: review([surface('settings', 'modified', { extra: { colorScheme: 'light' } }), dark]), evidence: undefined }));
assert.match(body, /\n- Head · \/settings \\\(dark\\\) · CAPTURE\\_OBSERVER\\_MISSING/);
assert.doesNotMatch(titleOf(body), /not captured/);
});
test('zero eligible pairs are counted as unattributed instead of successful comparisons', () => {
const flagged = evidenceReview();
for (const surface of flagged.surfaces) {
-17
View File
@@ -684,20 +684,3 @@ test("on a foldable a region across the pill's edge is kept whole, and one acros
assert.deepEqual([clipped.ignoredStatusBar, clipped.ignoredHomeIndicator, clipped.measurement.match, clipped.measurement.differentPixels], [0, 1, false, 80]);
assert.deepEqual(clipped.measurement.regions.map((entry) => [entry.rect.y, entry.rect.height, entry.differentPixels, entry.shareOfDiffPercentage]), [[840, 8, 80, 100]]);
});
test("a failed appearance pass is summarized once per side, with the frames it left unavailable", async (t) => {
// brentvatne/euxy's main baseline run 39506374824: the dark session's install failed after light captured every route.
const root = await mkdtemp(join(tmpdir(), "explore-public-evidence-")); t.after(() => rm(root, { recursive: true, force: true }));
const base = await side(root, "base", manifests.base);
const head = await side(root, "head", manifests.head);
const error = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture on a new session.", retryable: true };
head.bundle.runtimeProfile = { deviceModel: "iPhone 17", osVersion: "26.0", locale: "en-US", orientation: "portrait", colorScheme: "light", colorSchemes: "light,dark" };
for (const id of head.bundle.selectedSurfaceIds) head.bundle.outcomes[`${id}@dark`] = { status: "failed", attempts: 0, landing: "unverified", reason: error.message, error };
head.bundle.sessionFailures = [{ scheme: "dark", error }];
const result = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", measure: measurer({}) });
assert.deepEqual(result.summary.sessionFailures, [{ side: "head", scheme: "dark", frames: head.bundle.selectedSurfaceIds.length, ...error }]);
delete head.bundle.sessionFailures;
const complete = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", measure: measurer({}) });
assert.equal(complete.summary.sessionFailures, undefined);
});
-23
View File
@@ -740,29 +740,6 @@ test("a planned screen whose capture failed is unavailable, with the report's re
assert.equal(isChangedStatus(screen.status), false);
});
test("a dark frame whose session failed is unavailable with the session's error, and its light frame is unaffected", () => {
// brentvatne/euxy's main baseline run 39506374824: the light session captured every route, the dark session's install failed.
const review = navigationReview();
const scan = review.surfaces.find(surface => surface.id === "scan-mood");
const error = { code: "CAPTURE_SESSION_FAILED", stage: "session", message: "install: argent run list-devices failed: GET /tools failed: 404 Not Found",
nextAction: "Rerun the capture on a new session.", retryable: true };
scan.colorScheme = "light";
review.surfaces.push({ id: "scan-mood@dark", status: "unchanged", colorScheme: "dark",
base: entry("scan-mood@dark", scan.base.route, "mood"), head: entry("scan-mood@dark", scan.head.route, "mood"),
baseImage: { ...scan.baseImage, url: `${scan.baseImage.url}-dark`, sha256: "f".repeat(64) }, baseOutcome: { status: "captured", attempts: 1, landing: "verified" },
headOutcome: { status: "failed", attempts: 0, landing: "unverified", reason: error.message, error } });
verifyImages(review);
const diff = diffFor(review);
const dark = diff.screens.get("scan-mood@dark");
assert.equal(dark.status, "unavailable");
assert.match(dark.captureNotes.after, /^Capture failed: install: argent run list-devices failed: GET \/tools failed: 404 Not Found/);
assert.equal(dark.afterScreenshot, undefined);
assert.notEqual(diff.screens.get("scan-mood").status, "unavailable", "the light frame keeps its own verdict");
const capture = publicReviewProjection(review).runtimeEvidence["public:scan-mood@dark"].current.groups.capture;
assert.equal(capture.find(row => row.label === "Capture error").value, `CAPTURE_SESSION_FAILED: ${error.message}`);
assert.equal(capture.find(row => row.label === "Next action").value, error.nextAction);
});
test("source navigation names routes, so a route whose only base frame is a secondary appearance stays known on base", () => {
const review = navigationReview();
const scan = review.surfaces.find(surface => surface.id === "scan-mood");