Compare commits

...
Author SHA1 Message Date
Brent VatneandDevin AI 69d52661c1 Faster capture: parallel compare with early-abort region search, one Argent bridge thread per capture
- Compare screenshots in parallel and stop region-shift searches early
- Keep one Argent bridge on a worker thread for the whole capture
- Region shifts: stop a whole-pixel fit only past the margin of the fit a tie chooses, so the search finds what the exhaustive one did
- Compare: a pair whose turn comes after the budget is not started, every review in a process shares one limit, and a region-shift worker that fails measures on the calling thread
- Argent bridge thread: start again when the session's environment changes, stop serving when it crashes, and wait on the value it compared
- Argent bridge thread: recordings run per command in the session's directory, and a screenshot is taken only from a file that is a PNG or JPEG

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
2026-10-18 21:23:48 -07:00
13 changed files with 823 additions and 185 deletions

No files matched your search

+99
View File
@@ -0,0 +1,99 @@
/**
* Runs the Argent bridge's commands on a worker thread of this process, through a synchronous runner, so the
* capture's command-at-a-time control flow keeps one bridge and one tool-server connection for the whole run
* instead of starting a process per command. Commands for any other executable go to the fallback runner.
*/
import { Worker } from "node:worker_threads";
import { commandOutcome } from "./controller.js";
import type { CommandRequest, CommandResult, CommandRunner } from "./run.js";
import { BRIDGE_HEADER_BYTES, BRIDGE_KIND_UNAVAILABLE, BRIDGE_SLOT_KIND, BRIDGE_SLOT_LENGTH, BRIDGE_SLOT_READY, type BridgeRequest, type BridgeWorkerData } from "./argent-bridge-worker.js";
/** Room for one answer: an accessibility tree of a large screen is a few megabytes. */
const BRIDGE_BUFFER_BYTES = 64 * 1024 * 1024;
/** How long a command gets when its request names no timeout. */
const DEFAULT_COMMAND_TIMEOUT_MS = 10 * 60_000;
/** After a timeout, how long the thread gets to confirm it dropped the command. */
const ABORT_GRACE_MS = 15_000;
export type ArgentBridgeRunner = {
runner: CommandRunner;
/** Whether commands still go to the thread (false after it proved unavailable or stopped answering). */
readonly active: boolean;
close(): void;
};
export function startArgentBridgeRunner(options: {
/** The bridge executable the callers name; its commands run on the thread. */
command: string;
/** The bridge's environment, as argentControllerInvocation builds it. */
environment: NodeJS.ProcessEnv;
fallback: CommandRunner;
/** Tests: a module exporting `createArgentClient` in place of the pinned Argent release's client. */
clientModule?: string;
progress?: (message: string) => void;
}): ArgentBridgeRunner {
const shared = new SharedArrayBuffer(BRIDGE_HEADER_BYTES + BRIDGE_BUFFER_BYTES);
const header = new Int32Array(shared, 0, BRIDGE_HEADER_BYTES / 4);
const bytes = new Uint8Array(shared, BRIDGE_HEADER_BYTES);
const workerData: BridgeWorkerData = { shared, ...(options.clientModule ? { clientModule: options.clientModule } : {}) };
let worker: Worker | undefined;
// The environment the thread runs with: a copy, so a caller that changes the session's (a restarted local
// tool-server's URL and token) is seen by the next command, which starts a thread with the new one.
let environment: NodeJS.ProcessEnv = {};
const start = (from: NodeJS.ProcessEnv) => {
environment = { ...from };
try {
const started = new Worker(new URL("./argent-bridge-worker.js", import.meta.url), { workerData, env: environment });
// A thread that crashes stops serving; the commands after it run per command.
started.on("error", error => { if (worker === started) stop(`the Argent bridge thread failed (${error.message}); running the bridge per command`); });
worker = started;
} catch (error) { options.progress?.(`the Argent bridge thread could not start (${(error as Error).message}); running the bridge per command`); }
};
let sequence = 0;
const stop = (reason: string) => {
if (!worker) return;
options.progress?.(reason);
void worker.terminate();
worker = undefined;
};
const changed = (next: NodeJS.ProcessEnv) => {
const keys = new Set([...Object.keys(environment), ...Object.keys(next)]);
return [...keys].some(key => environment[key] !== next[key]);
};
const waitFor = (id: number, timeoutMs: number): boolean => {
const deadline = Date.now() + timeoutMs;
for (;;) {
// The value waited on is the one just compared: an answer stored after the comparison wakes the wait at once.
const ready = Atomics.load(header, BRIDGE_SLOT_READY);
if (ready === id) return true;
const remaining = deadline - Date.now();
if (remaining <= 0) return false;
Atomics.wait(header, BRIDGE_SLOT_READY, ready, remaining);
}
};
start(options.environment);
const runner: CommandRunner = (request: CommandRequest): CommandResult => {
// A recording runs per command, in the session's own directory: Argent's client saves one under the project its
// working directory is in, and this thread's is the caller's checkout. The state file carries it between hosts.
if (!worker || request.command !== options.command || request.args[0] === "record") return options.fallback(request);
if (changed(request.environment)) { void worker.terminate(); worker = undefined; start(request.environment); }
if (!worker) return options.fallback(request);
const id = ++sequence;
Atomics.store(header, BRIDGE_SLOT_READY, 0);
worker.postMessage({ id, argv: request.args, ...(request.timeoutMs !== undefined ? { timeoutMs: request.timeoutMs } : {}) } satisfies BridgeRequest);
const timeoutMs = request.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS;
if (!waitFor(id, timeoutMs)) {
worker.postMessage({ abort: id } satisfies BridgeRequest);
if (!waitFor(id, ABORT_GRACE_MS)) stop("the Argent bridge thread stopped answering; running the bridge per command");
throw Object.assign(new Error(`${request.command} ${request.args[0] ?? ""} timed out after ${timeoutMs} ms`), { code: "ETIMEDOUT" });
}
const text = Buffer.from(bytes.subarray(0, Atomics.load(header, BRIDGE_SLOT_LENGTH))).toString("utf8");
if (Atomics.load(header, BRIDGE_SLOT_KIND) === BRIDGE_KIND_UNAVAILABLE) {
stop(`the Argent bridge thread is unavailable (${text}); running the bridge per command`);
return options.fallback(request);
}
const result = JSON.parse(text) as { exitCode: number; stdout: string };
return commandOutcome(request, { status: result.exitCode, stdout: result.stdout, stderr: "" });
};
return { runner, get active() { return worker !== undefined; }, close: () => { if (worker) { void worker.terminate(); worker = undefined; } } };
}
+144
View File
@@ -0,0 +1,144 @@
/**
* The Argent bridge on a worker thread: one bridge and one tool-server connection for a whole capture, instead of a
* process (and a TLS handshake) per command. The capture thread posts a command and blocks on the shared buffer;
* this thread runs it through Argent's own client library (the code `argent run` uses) and writes the answer back.
*/
import { isMainThread, parentPort, workerData } from "node:worker_threads";
import { copyFile, mkdir, open, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { pathToFileURL } from "node:url";
import { ArgentCallError, type ArgentCall } from "./argent-driver.js";
import { argentBridgeSettings, createArgentBridge, type ArgentBridge } from "./argent-bridge.js";
import { ensureController } from "../shared/controller-cache.js";
/** The shared buffer: four Int32 header slots, then the UTF-8 answer. */
export const BRIDGE_HEADER_BYTES = 16;
export const BRIDGE_SLOT_READY = 0;
export const BRIDGE_SLOT_LENGTH = 1;
export const BRIDGE_SLOT_KIND = 2;
export const BRIDGE_KIND_RESULT = 0;
/** The bridge cannot run on this thread (Argent's client did not load); the host falls back to the process bridge. */
export const BRIDGE_KIND_UNAVAILABLE = 1;
export type BridgeWorkerData = { shared: SharedArrayBuffer; clientModule?: string };
export type BridgeRequest = { id: number; argv: readonly string[]; timeoutMs?: number } | { abort: number };
const ARGENT_CALL_TIMEOUT_MS = 180_000;
/** Argent's programmatic client, as `dist/client.js` of the pinned release exports it. */
type ArgentClient = { callTool(name: string, args: Record<string, unknown>, options?: { signal?: AbortSignal }): Promise<{ data: unknown; note?: string }> };
/** Whether some bytes begin as a PNG or a JPEG does: `--out` is a screenshot, and nothing else is taken for one. */
function isImage(head: Uint8Array): boolean {
const png = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], jpeg = [0xff, 0xd8, 0xff];
return [png, jpeg].some(signature => signature.every((byte, index) => head[index] === byte));
}
/** Whether a path names a file that is an image by its content. */
async function imageFile(path: string): Promise<boolean> {
const file = await open(path, "r").catch(() => undefined);
if (!file) return false;
try {
if (!(await file.stat()).isFile()) return false;
const head = new Uint8Array(8);
const { bytesRead } = await file.read(head, 0, head.length, 0);
return isImage(head.subarray(0, bytesRead));
} finally { await file.close(); }
}
/** The first string in a tool's answer that names an image file: the materialized screenshot (`path` first). */
async function materializedFile(data: unknown): Promise<string | undefined> {
const candidates: string[] = [];
const walk = (value: unknown, key?: string) => {
if (typeof value === "string") { if (key === "path") candidates.unshift(value); else candidates.push(value); }
else if (Array.isArray(value)) value.forEach(item => walk(item));
else if (value && typeof value === "object") Object.entries(value).forEach(([k, v]) => walk(v, k));
};
walk(data);
for (const candidate of candidates) {
if (candidate.startsWith("/") && await imageFile(candidate)) return candidate;
}
return undefined;
}
async function loadClient(environment: NodeJS.ProcessEnv, clientModule?: string): Promise<ArgentClient> {
const module = clientModule ?? join(dirname(await ensureController("argent", { environment })), "client.js");
const loaded = await import(pathToFileURL(module).href) as { createArgentClient?: () => ArgentClient };
if (typeof loaded.createArgentClient !== "function") throw new Error(`${module} does not export createArgentClient`);
return loaded.createArgentClient();
}
/** The in-thread Argent call: the client's `callTool`, with `--out` served from the artifact it materialized. */
export function argentClientCall(client: ArgentClient, signal: () => AbortSignal): ArgentCall {
return async (tool, args, options) => {
const active = signal();
if (active.aborted) throw new ArgentCallError(`argent run ${tool} was not sent: the command was given up.`, false);
let answer: { data: unknown };
try { answer = await client.callTool(tool, args, { signal: AbortSignal.any([active, AbortSignal.timeout(ARGENT_CALL_TIMEOUT_MS)]) }); }
catch (error) { throw new ArgentCallError(`argent run ${tool} failed: ${(error instanceof Error ? error.message : String(error)).slice(0, 800)}`, true); }
if (!options?.out) return answer.data;
const file = await materializedFile(answer.data);
if (!file) {
const url = (answer.data as { url?: unknown } | null)?.url;
if (typeof url !== "string") throw new ArgentCallError(`argent run ${tool} did not write ${options.out}`, true);
const response = await fetch(url, { signal: signal() });
if (!response.ok) throw new ArgentCallError(`argent run ${tool} did not write ${options.out}: ${response.status}`, true);
const body = Buffer.from(await response.arrayBuffer());
if (!isImage(body)) throw new ArgentCallError(`argent run ${tool} did not write ${options.out}: the answer is not an image`, true);
await mkdir(dirname(options.out), { recursive: true });
await writeFile(options.out, body);
return { path: options.out };
}
await mkdir(dirname(options.out), { recursive: true });
if (file !== options.out) await copyFile(file, options.out);
return { path: options.out };
};
}
if (!isMainThread && parentPort && (workerData as Partial<BridgeWorkerData> | null)?.shared instanceof SharedArrayBuffer) {
const port = parentPort;
const { shared, clientModule } = workerData as BridgeWorkerData;
const header = new Int32Array(shared, 0, BRIDGE_HEADER_BYTES / 4);
const bytes = new Uint8Array(shared, BRIDGE_HEADER_BYTES);
const answer = (id: number, kind: number, text: string) => {
let encoded = Buffer.from(text, "utf8");
if (encoded.length > bytes.length) {
encoded = Buffer.from(JSON.stringify({ exitCode: 1, stdout: JSON.stringify({ success: false,
error: { code: "COMMAND_FAILED", message: `The controller's answer (${encoded.length} bytes) exceeds the bridge buffer.` } }) }));
}
bytes.set(encoded);
Atomics.store(header, BRIDGE_SLOT_LENGTH, encoded.length);
Atomics.store(header, BRIDGE_SLOT_KIND, kind);
Atomics.store(header, BRIDGE_SLOT_READY, id);
Atomics.notify(header, BRIDGE_SLOT_READY);
};
const controllers = new Map<number, AbortController>();
let current: AbortController | undefined;
const settings = argentBridgeSettings(process.env);
const bridge: Promise<ArgentBridge> = settings && settings.toolsUrl
? loadClient(process.env, clientModule).then(client => createArgentBridge(settings, argentClientCall(client, () => current?.signal ?? new AbortController().signal), "thread"))
: Promise.reject(new Error("The Argent bridge needs EXPLORE_ARGENT_STATE, EXPLORE_ARGENT_PLATFORM and ARGENT_TOOLS_URL."));
bridge.catch(() => undefined);
let queue: Promise<unknown> = Promise.resolve();
port.on("message", (request: BridgeRequest) => {
if ("abort" in request) { controllers.get(request.abort)?.abort(new Error("The capture gave up waiting for this command.")); return; }
const controller = new AbortController();
controllers.set(request.id, controller);
queue = queue.then(async () => {
current = controller;
try {
const ready = await bridge.catch(error => { throw Object.assign(error as Error, { unavailable: true }); });
if (controller.signal.aborted) throw controller.signal.reason;
const result = await ready.run(request.argv);
answer(request.id, BRIDGE_KIND_RESULT, JSON.stringify(result));
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if ((error as { unavailable?: boolean }).unavailable) answer(request.id, BRIDGE_KIND_UNAVAILABLE, message);
else answer(request.id, BRIDGE_KIND_RESULT, JSON.stringify({ exitCode: 1, stdout: JSON.stringify({ success: false, error: { code: "COMMAND_FAILED", message } }) }));
} finally {
controllers.delete(request.id);
current = undefined;
}
});
});
}
+137
View File
@@ -0,0 +1,137 @@
/**
* The Argent bridge behind both of its hosts: the command-line worker Explore spawns per command, and the
* in-process worker thread that keeps one bridge (and one tool-server connection) for a whole capture. Each command
* reads the shared state file, runs through `runBridgeCommand`, and writes the state back, so the two hosts can
* take turns on one session.
*/
import { spawnSync } from "node:child_process";
import { appendFile, copyFile, mkdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { createRequire } from "node:module";
import type { DecodedFrame } from "../shared/png-evidence.js";
import { runBridgeCommand, type ArgentBridgeState, type ArgentCall, type ArgentPlatform, type ArgentTree, type BridgeResult } from "./argent-driver.js";
import { ARGENT_VERSION } from "../shared/argent-version.js";
export type ArgentBridgeSettings = {
environment: NodeJS.ProcessEnv;
statePath: string;
platform: ArgentPlatform;
toolsUrl: string | undefined;
};
/** Reads the settings the bridge takes from its environment; absent when they are incomplete. */
export function argentBridgeSettings(environment: NodeJS.ProcessEnv): ArgentBridgeSettings | undefined {
const statePath = environment.EXPLORE_ARGENT_STATE;
const platform = environment.EXPLORE_ARGENT_PLATFORM;
if (!statePath || (platform !== "ios" && platform !== "android")) return undefined;
return { environment, statePath, platform, toolsUrl: environment.ARGENT_TOOLS_URL };
}
async function readState(path: string): Promise<ArgentBridgeState> {
try { return JSON.parse(await readFile(path, "utf8")) as ArgentBridgeState; }
catch { return { generation: 0 }; }
}
/** Written whole and renamed into place, so a command stopped mid-write never leaves a torn state. */
async function writeState(path: string, state: ArgentBridgeState): Promise<void> {
const temporary = `${path}.${process.pid}.tmp`;
await writeFile(temporary, JSON.stringify(state), { mode: 0o600 });
await rename(temporary, path);
}
/** A call's arguments for the debug log: text that could be typed into a field is logged by length only. */
function loggedArgs(args: Record<string, unknown>): Record<string, unknown> {
return Object.fromEntries(Object.entries(args).map(([key, value]) => key === "text" && typeof value === "string" ? [key, `<${value.length} characters>`]
: key === "steps" && Array.isArray(value) ? [key, `<${value.length} steps>`] : [key, value]));
}
/** What a call answered, for the debug log: a tree's shape, not its contents. */
function summarize(tool: string, value: unknown): unknown {
if (!value || typeof value !== "object") return value;
if (tool !== "ui-tree") return Object.keys(value);
const tree = value as ArgentTree;
let nodes = 0;
const walk = (node: { children?: unknown[] }) => { nodes++; (node.children ?? []).forEach(child => walk(child as { children?: unknown[] })); };
(tree.roots ?? []).forEach(walk);
return { foregroundApp: tree.foregroundApp, alertVisible: tree.alertVisible, keyboardVisible: tree.keyboardVisible, truncated: tree.truncated,
screen: tree.screen, nodes, roots: (tree.roots ?? []).map(root => [root.type ?? null, root.bundleId ?? null, (root.children ?? []).length]) };
}
export type ArgentBridge = {
/** Runs one agent-device-shaped command against the session. */
run(argv: readonly string[]): Promise<BridgeResult>;
};
/**
* One bridge over the given Argent call. `host` names the process running it in the debug log.
*/
export function createArgentBridge(settings: ArgentBridgeSettings, call: ArgentCall, host: "process" | "thread"): ArgentBridge {
const { environment, statePath, platform, toolsUrl } = settings;
const logPath = environment.EXPLORE_ARGENT_LOG_DIR
? join(environment.EXPLORE_ARGENT_LOG_DIR, `argent-${platform}-${environment.EXPLORE_ARGENT_LOG_NAME ?? "bridge"}.jsonl`) : undefined;
return {
async run(argv) {
// EXPLORE_ARGENT_LOG_DIR (opt-in): every Argent call this bridge makes, how long it took, and the shape of what it read.
const log = async (entry: Record<string, unknown>) => {
if (!logPath) return;
await appendFile(logPath, `${JSON.stringify({ at: new Date().toISOString(), host, command: argv[0], ...entry })}\n`, { mode: 0o600 }).catch(() => undefined);
};
const state = await readState(statePath);
const result = await runBridgeCommand(argv, {
state, platform, version: ARGENT_VERSION,
...(environment.EXPLORE_ARGENT_BUNDLE_ID ? { bundleId: environment.EXPLORE_ARGENT_BUNDLE_ID } : {}),
...(environment.EXPLORE_ARGENT_DEVICE ? { device: environment.EXPLORE_ARGENT_DEVICE } : {}),
// An opened foldable's inner panel shows its UI in landscape (Argent's fold: "Unfolded, the UI is landscape").
...(platform === "ios" && ["open", "half-open"].includes(environment.EXPLORE_ARGENT_POSE ?? "") ? { landscape: true } : {}),
call: async (tool, args, options) => {
const started = Date.now();
try {
const value = await call(tool, args, options);
await log({ tool, args: loggedArgs(args), ms: Date.now() - started, summary: summarize(tool, value) });
return value;
} catch (error) {
await log({ tool, args: loggedArgs(args), ms: Date.now() - started, error: (error instanceof Error ? error.message : String(error)).slice(0, 400) });
throw error;
}
},
reachable: async () => {
let ok = false;
try {
const token = environment.ARGENT_AUTH_TOKEN;
// The same URL Argent's own client checks: ARGENT_TOOLS_URL may carry a path, which /tools is appended to.
const response = await fetch(`${(toolsUrl ?? "").replace(/\/+$/, "")}/tools`, { headers: token ? { authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(5_000) });
await response.body?.cancel();
ok = response.status < 500;
} catch { ok = false; }
if (!ok) await log({ tool: "reachable", ok });
return ok;
},
sleep: milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)),
orientVideo: async path => {
const probed = spawnSync("ffprobe", ["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", path], { encoding: "utf8", timeout: 30_000 });
const [width, height] = (probed.stdout ?? "").trim().split("x").map(Number);
if (probed.status !== 0 || !width || !height || width >= height) return;
const turned = `${path}.upright.mp4`;
const ffmpeg = spawnSync("ffmpeg", ["-nostdin", "-y", "-loglevel", "error", "-i", path, "-vf", "transpose=clock", "-an", turned], { encoding: "utf8", timeout: 120_000 });
if (ffmpeg.status === 0) await rename(turned, path);
else await rm(turned, { force: true });
},
scratchImage: { path: join(dirname(statePath), "argent-scratch.png"), read: async path => {
const { PNG } = createRequire(import.meta.url)("pngjs") as { PNG: { sync: { read(bytes: Buffer, options: { checkCRC: boolean }): DecodedFrame } } };
return PNG.sync.read(await readFile(path), { checkCRC: false });
} },
moveFile: async (from, to) => {
const source = await realpath(from);
const roots = await Promise.all([environment.HOME, process.cwd()].filter((root): root is string => !!root).map(root => realpath(root).catch(() => root)));
if (!roots.some(root => source.startsWith(`${root}/`))) throw new Error(`Argent returned a recording outside its own directories: ${from}`);
await mkdir(dirname(to), { recursive: true });
try { await rename(from, to); }
catch { await copyFile(from, to); await rm(from, { force: true }); }
},
});
await writeState(statePath, state);
await log({ result: result.exitCode === 0 ? "ok" : result.stdout.slice(0, 300) });
return result;
},
};
}
+9 -106
View File
@@ -2,71 +2,30 @@
* The Argent controller's command line: Explore runs it where it would run agent-device, with the same
* arguments. Each command becomes `argent run <tool> --args <json>` calls, as EAS Simulator recommends
* driving an `argent` session, with the session's ARGENT_TOOLS_URL and ARGENT_AUTH_TOKEN in the environment.
* A capture keeps the same bridge on a worker thread instead (argent-bridge-host); this entry serves the callers
* that run one command at a time.
*/
import { spawnSync } from "node:child_process";
import { appendFile, copyFile, mkdir, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { createRequire } from "node:module";
import type { DecodedFrame } from "../shared/png-evidence.js";
import { ArgentCallError, runBridgeCommand, type ArgentBridgeState, type ArgentPlatform, type ArgentTree } from "./argent-driver.js";
import { ARGENT_VERSION } from "../shared/argent-version.js";
import { stat } from "node:fs/promises";
import { ArgentCallError } from "./argent-driver.js";
import { argentBridgeSettings, createArgentBridge } from "./argent-bridge.js";
import { ensureController } from "../shared/controller-cache.js";
const ARGENT_CALL_TIMEOUT_MS = 180_000;
async function readState(path: string): Promise<ArgentBridgeState> {
try { return JSON.parse(await readFile(path, "utf8")) as ArgentBridgeState; }
catch { return { generation: 0 }; }
}
/** Written whole and renamed into place, so a command stopped mid-write never leaves a torn state. */
async function writeState(path: string, state: ArgentBridgeState): Promise<void> {
const temporary = `${path}.${process.pid}.tmp`;
await writeFile(temporary, JSON.stringify(state), { mode: 0o600 });
await rename(temporary, path);
}
/** A call's arguments for the debug log: text that could be typed into a field is logged by length only. */
function loggedArgs(args: Record<string, unknown>): Record<string, unknown> {
return Object.fromEntries(Object.entries(args).map(([key, value]) => key === "text" && typeof value === "string" ? [key, `<${value.length} characters>`]
: key === "steps" && Array.isArray(value) ? [key, `<${value.length} steps>`] : [key, value]));
}
/** What a call answered, for the debug log: a tree's shape, not its contents. */
function summarize(tool: string, value: unknown): unknown {
if (!value || typeof value !== "object") return value;
if (tool !== "ui-tree") return Object.keys(value);
const tree = value as ArgentTree;
let nodes = 0;
const walk = (node: { children?: unknown[] }) => { nodes++; (node.children ?? []).forEach(child => walk(child as { children?: unknown[] })); };
(tree.roots ?? []).forEach(walk);
return { foregroundApp: tree.foregroundApp, alertVisible: tree.alertVisible, keyboardVisible: tree.keyboardVisible, truncated: tree.truncated,
screen: tree.screen, nodes, roots: (tree.roots ?? []).map(root => [root.type ?? null, root.bundleId ?? null, (root.children ?? []).length]) };
}
async function main(): Promise<number> {
const environment = process.env;
const statePath = environment.EXPLORE_ARGENT_STATE;
const platform = environment.EXPLORE_ARGENT_PLATFORM as ArgentPlatform | undefined;
const toolsUrl = environment.ARGENT_TOOLS_URL;
if (!statePath || (platform !== "ios" && platform !== "android")) {
const settings = argentBridgeSettings(environment);
if (!settings) {
process.stdout.write(JSON.stringify({ success: false, error: { code: "INVALID_ARGS", message: "The Argent controller needs EXPLORE_ARGENT_STATE and EXPLORE_ARGENT_PLATFORM." } }));
return 1;
}
const argv = process.argv.slice(2);
if (argv[0] !== "--version" && !toolsUrl) {
if (argv[0] !== "--version" && !settings.toolsUrl) {
// Without a URL the Argent client would start a local tool-server; Explore only drives the session's own.
process.stdout.write(JSON.stringify({ success: false, error: { code: "INVALID_ARGS", message: "ARGENT_TOOLS_URL is not set for the Argent controller." } }));
return 1;
}
// EXPLORE_ARGENT_LOG_DIR (opt-in): every Argent call this bridge makes, how long it took, and the shape of what it read.
const logPath = environment.EXPLORE_ARGENT_LOG_DIR
? join(environment.EXPLORE_ARGENT_LOG_DIR, `argent-${platform}-${environment.EXPLORE_ARGENT_LOG_NAME ?? "bridge"}.jsonl`) : undefined;
const log = async (entry: Record<string, unknown>) => {
if (!logPath) return;
await appendFile(logPath, `${JSON.stringify({ at: new Date().toISOString(), command: argv[0], ...entry })}\n`, { mode: 0o600 }).catch(() => undefined);
};
const state = await readState(statePath);
let cli: string;
try { cli = argv[0] === "--version" ? "" : await ensureController("argent", { environment, progress: message => process.stderr.write(`[explore] ${message}\n`) }); }
catch (error) {
@@ -93,65 +52,9 @@ async function main(): Promise<number> {
catch { throw new ArgentCallError(`argent run ${tool} did not answer with JSON`, true); }
};
const result = await runBridgeCommand(argv, {
state, platform, version: ARGENT_VERSION,
...(environment.EXPLORE_ARGENT_BUNDLE_ID ? { bundleId: environment.EXPLORE_ARGENT_BUNDLE_ID } : {}),
...(environment.EXPLORE_ARGENT_DEVICE ? { device: environment.EXPLORE_ARGENT_DEVICE } : {}),
// An opened foldable's inner panel shows its UI in landscape (Argent's fold: "Unfolded, the UI is landscape").
...(platform === "ios" && ["open", "half-open"].includes(environment.EXPLORE_ARGENT_POSE ?? "") ? { landscape: true } : {}),
call: async (tool, args, options) => {
const started = Date.now();
try {
const value = await callArgent(tool, args, options);
await log({ tool, args: loggedArgs(args), ms: Date.now() - started, summary: summarize(tool, value) });
return value;
} catch (error) {
await log({ tool, args: loggedArgs(args), ms: Date.now() - started, error: (error instanceof Error ? error.message : String(error)).slice(0, 400) });
throw error;
}
},
reachable: async () => {
let ok = false;
try {
const token = environment.ARGENT_AUTH_TOKEN;
// The same URL Argent's own client checks: ARGENT_TOOLS_URL may carry a path, which /tools is appended to.
const response = await fetch(`${(toolsUrl ?? "").replace(/\/+$/, "")}/tools`, { headers: token ? { authorization: `Bearer ${token}` } : {}, signal: AbortSignal.timeout(5_000) });
await response.body?.cancel();
ok = response.status < 500;
} catch { ok = false; }
if (!ok) await log({ tool: "reachable", ok });
return ok;
},
sleep: milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)),
orientVideo: async path => {
// Turned with ffmpeg, which observing a launch needs anyway; without it the recording is left as it is.
const probed = spawnSync("ffprobe", ["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", path], { encoding: "utf8", timeout: 30_000 });
const [width, height] = (probed.stdout ?? "").trim().split("x").map(Number);
if (probed.status !== 0 || !width || !height || width >= height) return;
const turned = `${path}.upright.mp4`;
const ffmpeg = spawnSync("ffmpeg", ["-nostdin", "-y", "-loglevel", "error", "-i", path, "-vf", "transpose=clock", "-an", turned], { encoding: "utf8", timeout: 120_000 });
if (ffmpeg.status === 0) await rename(turned, path);
else await rm(turned, { force: true });
},
scratchImage: { path: join(dirname(statePath), "argent-scratch.png"), read: async path => {
const { PNG } = createRequire(import.meta.url)("pngjs") as { PNG: { sync: { read(bytes: Buffer, options: { checkCRC: boolean }): DecodedFrame } } };
return PNG.sync.read(await readFile(path), { checkCRC: false });
} },
moveFile: async (from, to) => {
// Argent's client downloads a recording under its HOME or the working directory; nothing else is moved.
const source = await realpath(from);
const roots = await Promise.all([environment.HOME, process.cwd()].filter((root): root is string => !!root).map(root => realpath(root).catch(() => root)));
if (!roots.some(root => source.startsWith(`${root}/`))) throw new Error(`Argent returned a recording outside its own directories: ${from}`);
await mkdir(dirname(to), { recursive: true });
try { await rename(from, to); }
catch { await copyFile(from, to); await rm(from, { force: true }); }
},
});
await writeState(statePath, state);
await log({ result: result.exitCode === 0 ? "ok" : result.stdout.slice(0, 300) });
const result = await createArgentBridge(settings, callArgent, "process").run(argv);
process.stdout.write(result.stdout);
return result.exitCode;
}
process.exitCode = await main();
+8 -3
View File
@@ -14,20 +14,25 @@ export function defaultRunner(request: CommandRequest): CommandResult {
...(request.timeoutMs ? { timeout: request.timeoutMs } : {}),
});
if (result.error) throw result.error;
return commandOutcome(request, { status: result.status, stdout: result.stdout ?? "", stderr: result.stderr ?? "" });
}
/** A finished command's output, or the error its exit status and output describe. */
export function commandOutcome(request: CommandRequest, result: { status: number | null; stdout: string; stderr: string }): CommandResult {
if (result.status !== 0 && !request.allowFailure) {
const sessionIndex = request.args.indexOf("--session");
// Our own session or a caller's session attached by explore:session: both answer --json failures with a structured error.
const structuredFailure = (sessionIndex >= 0 && request.args[sessionIndex + 1] === CONTROLLER_SESSION) || request.args.includes("--json")
? controllerCommandFailure(result.stdout ?? "", request.args[0] ?? "command") : undefined;
? controllerCommandFailure(result.stdout, request.args[0] ?? "command") : undefined;
if (structuredFailure) throw structuredFailure;
const subcommand = typeof request.args[0] === "string" ? ` ${request.args[0]}` : "";
const detail = failureDetail(`${result.stderr ?? ""}\n${result.stdout ?? ""}`);
const detail = failureDetail(`${result.stderr}\n${result.stdout}`);
throw new Error(
`${request.command}${subcommand} exited with status ${result.status ?? "unknown"}`
+ (detail ? `: ${detail}` : ""),
);
}
return { stdout: result.stdout ?? "", stderr: result.stderr ?? "" };
return { stdout: result.stdout, stderr: result.stderr };
}
/** Keep the CLI tool host responsive while a remote controller request is in flight. */
+10 -2
View File
@@ -24,6 +24,7 @@ import { silentProgress, type Progress } from "../shared/progress.js";
import { assertCaptureEasProfile, assertSimulatorAvailable, fingerprintFrom, hasEasEnvironmentFetchFailure, nativeBuildFrom, existingBuild, repackProfileFrom, prepareExactSourceBuild, type RepackMode } from "./eas-build.js";
import { SIMULATOR_DOTENV_FILE_NAME, SIMULATOR_DOTENV_HEADER, endedOwnSimulatorSession, liveSimulatorSessionsNamed, simulatorSessionName, simulatorStartFailure, readSimulatorDotenvHandoff, assertSimulatorSessionIdentity, assertSimulatorStopped, collectRuntime, controllerVersion, simulatorOsVersion, type ActiveSimulatorRuntime, type SimulatorConnection } from "./simulator.js";
import { argentControllerInvocation } from "./controller-invocation.js";
import { startArgentBridgeRunner, type ArgentBridgeRunner } from "./argent-bridge-host.js";
import { sharedBuild } from "./shared-build.js";
import { LOCAL_EMULATOR_STATE_PREFIX, LOCAL_SIMULATOR_STATE_PREFIX, agentDeviceClaims, assertLocalEmulatorCompatible, createLocalSimulator, startLocalEmulator, waitForLocalEmulator, waitForLocalSimulator, deleteLocalEmulator,
deleteLocalSimulator, hostEmulatorAbi, installOnLocalEmulator, installOnLocalSimulator, approveLinkScheme, localPoseShown,
@@ -218,7 +219,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
});
const routes = inventory.filter(route => selected.has(route.id));
const setup = revisionSetup(options.setup ?? [], plan, side);
const runner = options.runner ?? defaultRunner;
let runner = options.runner ?? defaultRunner;
const sourceEnvironment = options.environment ?? process.env;
const controller = options.controller ?? "agent-device";
const local = options.simulator === "local";
@@ -481,6 +482,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
let localOsVersion: string | undefined;
// Argent's tool-server on this machine, when Argent drives a local device.
let localArgent: LocalArgentServer | undefined;
let argentBridge: ArgentBridgeRunner | undefined;
const localDevice = () => localSimulator ?? localEmulator;
// A local device starts booting now and finishes while the app is built and repacked; the session waits for it.
let localEmulatorStart: Awaited<ReturnType<typeof startLocalEmulator>> | undefined;
@@ -756,6 +758,11 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
AGENT_DEVICE_SESSION: CONTROLLER_SESSION, AGENT_DEVICE_SESSION_LOCK: "reject", AGENT_DEVICE_PLATFORM: options.platform };
// Every controller call below goes to the session's controller; for Argent that is the bridge.
const controllerCommand = argent?.command ?? options.agentDeviceCommand;
// The capture's commands keep one bridge on a worker thread (EXPLORE_ARGENT_BRIDGE=process: one process per command, as before).
if (argent && !options.runner && sourceEnvironment.EXPLORE_ARGENT_BRIDGE !== "process") {
argentBridge = startArgentBridgeRunner({ command: argent.command, environment: argent.environment, fallback: defaultRunner, progress: say });
runner = argentBridge.runner;
}
if (!local && localApp) {
// The session started empty while the app was repacked; the controller installs it now.
say(`installing the app on session ${sessionId}`);
@@ -944,7 +951,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
...deviceArgs(runtime), "--platform", options.platform, "--level", "full", "--json"], cwd: simulatorProject, environment: controllerEnvironment, timeoutMs: 60_000 });
runtime.colorScheme = colorScheme;
if (colorSchemes.length > 1) say(`appearance ${colorScheme} (${index + 1}/${colorSchemes.length})`);
passes.push(await capturePlannedRoutes({ capture: { ...options, agentDeviceCommand: controllerCommand, setup, progress: say }, routes: missingRoutes, inventory, scheme, appName: app.name, runtime,
passes.push(await capturePlannedRoutes({ capture: { ...options, ...(argentBridge && !options.agent ? { runner: argentBridge.runner } : {}), agentDeviceCommand: controllerCommand, setup, progress: say }, routes: missingRoutes, inventory, scheme, appName: app.name, runtime,
connection, environment: controllerEnvironment, store, map, acceptedFrames, outcomes: index === 0 ? outcomes : {},
...(colorSchemes.length > 1 ? { appearance: { scheme: colorScheme, primary: index === 0 } } : {}) }));
}
@@ -954,6 +961,7 @@ async function executeCiMap(options: CiMapOptions, setStage: (stage: string) =>
nextAction: "Inspect the capture diagnostic, correct the failing prerequisite, and rerun Explore." }); }
finally {
if (appDirectory) await rm(appDirectory, { recursive: true, force: true });
argentBridge?.close();
if (localArgent) stopLocalArgentServer(localArgent);
if (localEmulator) {
try {
+99 -38
View File
@@ -1,5 +1,5 @@
import { unattributedIssue } from "../capture/health.js";
import { appearancePairs, bundleColorSchemes } from "../capture/appearance.js";
import { appearancePairs, bundleColorSchemes, type ColorScheme } from "../capture/appearance.js";
import { launchSurfaceId, systemRegions as defaultSystemRegions, type SystemRegion } from "../contracts/capture.js";
import { readFileSync, statSync } from "node:fs";
import { stat } from "node:fs/promises";
@@ -27,7 +27,9 @@ import {
} from "./visual-evidence.js";
import { AGENT_DEVICE_VERSION } from "../shared/agent-device-version.js";
import { STATUS_BAR_BAND_PERCENT, decodePng, differentPixelsInRect, maxColorDistance, type DecodedFrame } from "../shared/png-evidence.js";
import { measureRegionMoves, type RegionMove, type RegionShift } from "./region-shift.js";
import type { RegionMove, RegionShift } from "./region-shift.js";
import { measureRegionMovesInWorker } from "./region-shift-worker.js";
import { availableParallelism } from "node:os";
export const DEFAULT_VISUAL_THRESHOLD = 0.1;
export const DEFAULT_EVIDENCE_TIME_BUDGET_MS = 10 * 60_000;
@@ -194,7 +196,7 @@ export function publishMeasuredRegions(all: readonly VisualDiffRegion[], shifts:
async function withRegionShifts(basePath: string, headPath: string, all: readonly VisualDiffRegion[], published: NonNullable<PublicVisualVerdict["regions"]>):
Promise<Pick<PublicVisualVerdict, "regions" | "allRegionsMoved">> {
try {
const { shifts, moves, width, height } = await measureRegionMoves(basePath, headPath,
const { shifts, moves, width, height } = await measureRegionMovesInWorker(basePath, headPath,
all.map(region => ({ ...region.rect, dominantChange: region.dominantChange })));
if (moves.length) return publishMeasuredRegions(all, shifts, moves, width, height);
const shiftOf = new Map(all.map((region, index) => [region.rect, shifts[index]] as const));
@@ -406,6 +408,40 @@ function impactFor(surface: PlanSurface, presence: ComparatorSurfaceImpact["pres
};
}
/** The screenshot pairs `assessPublicEvidence` compares: both sides captured and landed, in the loop's order. */
function comparablePairs(options: { plan: SourceCapturePlan; base: PublicEvidenceSide; head: PublicEvidenceSide }, colorSchemes: readonly ColorScheme[],
baseColorSchemes: readonly ColorScheme[]): Array<{ id: string; basePath: string; headPath: string }> {
const pairs: Array<{ id: string; basePath: string; headPath: string }> = [];
for (const surface of options.plan.surfaces) {
if (surface.baseSurface?.kind !== "screen" || surface.headSurface?.kind !== "screen" || (!surface.baseCapture && !surface.headCapture)) continue;
for (const { id, baseId } of appearancePairs(surface.id, colorSchemes, baseColorSchemes)) {
if (baseId === null) continue;
const basePath = localImagePath(options.base, baseId);
const headPath = localImagePath(options.head, id);
if (!basePath || !headPath || captureDiagnostic(surface, "base", options.base, baseId) || captureDiagnostic(surface, "head", options.head, id)) continue;
pairs.push({ id, basePath, headPath });
}
}
return pairs;
}
/** Runs tasks at most `size` at a time, each started in the order it was queued. */
function concurrencyLimit(size: number): <T>(task: () => Promise<T>) => Promise<T> {
let active = 0;
const queue: Array<() => void> = [];
const next = () => { if (active < size) queue.shift()?.(); };
return task => new Promise((resolve, reject) => {
queue.push(() => {
active += 1;
task().then(resolve, reject).finally(() => { active -= 1; next(); });
});
next();
});
}
/** The limit the reviews in this process share when none is given: a local review compares its targets at once. */
let processLimit: ReturnType<typeof concurrencyLimit> | undefined;
function localImagePath(side: PublicEvidenceSide, id: string): string | undefined {
const frame = side.bundle.images[id];
return frame ? localEvidencePath(side, frame.path) : undefined;
@@ -499,6 +535,8 @@ export async function assessPublicEvidence(options: {
timeBudgetMs?: number;
now?: () => number;
measure?: (baselinePath: string, currentPath: string) => Promise<ScreenshotDiffMeasurement>;
/** How many screenshot pairs compare at once. Absent: one per available CPU, shared by every review in this process. */
concurrency?: number;
}): Promise<PublicEvidenceResult> {
const threshold = options.threshold ?? DEFAULT_VISUAL_THRESHOLD;
const system = options.systemRegions ?? defaultSystemRegions({ platform: options.platform });
@@ -517,6 +555,34 @@ export async function assessPublicEvidence(options: {
const measure = options.measure ?? (async (baselinePath: string, currentPath: string) => (
await diffScreenshots({ baselinePath, currentPath, command: options.agentDeviceCommand, threshold })
));
type ComparedPair = { visual: PublicVisualVerdict; assessment: VisualEvidenceAssessment };
/** One pair's measurement, verdict and region moves: the work that runs ahead of the surface loop. */
const comparePair = async (basePath: string, headPath: string): Promise<ComparedPair> => {
const raw = await measure(basePath, headPath);
const { measurement, ignoredStatusBar: ignoredTop, ignoredHomeIndicator: ignoredBottom } = ignoreSystemRegions(raw, system, systemBandFrameLoader(raw, threshold));
const assessment = assessVisualMeasurements({
baselineSamples: 1,
currentSamples: 1,
requiredSamples: 1,
baselineWithin: [],
currentWithin: [],
crossBuild: [measurement],
representative: measurement,
});
const visual = visualVerdict(assessment, threshold);
// Measuring moves is extra: once the comparison budget is spent, regions go out unmeasured.
if (visual.verdict === "modified" && visual.regions?.length && now() <= deadline) {
Object.assign(visual, await withRegionShifts(basePath, headPath, measurement.regions, visual.regions));
}
if (ignoredTop > 0) visual.ignoredStatusBarRegions = ignoredTop;
if (ignoredBottom > 0) visual.ignoredHomeIndicatorRegions = ignoredBottom;
if (ignoredTop > 0 || ignoredBottom > 0) visual.systemRegions = system.map((area) => ({ ...area }));
return { visual, assessment };
};
/** A pair compared if the budget has not expired when its turn to start comes; undefined when it has. */
const compareInBudget = async (basePath: string, headPath: string): Promise<ComparedPair | undefined> => (
now() > deadline ? undefined : await comparePair(basePath, headPath)
);
const visualCounts = { compared: 0, modified: 0, unchanged: 0, needsReview: 0, failed: 0, errors: [] as string[] };
const accessibilityCounts = { compared: 0, described: 0, failed: 0, skipped: 0, errors: [] as string[] };
const reviewCounts = { modified: 0, needsReview: 0, variants: 0, unchanged: 0 };
@@ -533,6 +599,20 @@ export async function assessPublicEvidence(options: {
const colorSchemes = bundleColorSchemes(options.head.bundle);
const baseColorSchemes = bundleColorSchemes(options.base.bundle);
// The pairs the loop below compares start now, a few at a time: each is a comparator process and a pixel search
// that take seconds on a large screenshot, so they overlap instead of queueing behind one another. The loop reads
// them in its own order, so the result does not depend on which finishes first. A pair whose turn comes after the
// budget expired is not started; one started in time is waited for and used.
const ahead = new Map<string, Promise<ComparedPair | undefined>>();
if (visualAvailable) {
const limit = options.concurrency !== undefined ? concurrencyLimit(Math.max(1, Math.floor(options.concurrency)))
: processLimit ??= concurrencyLimit(availableParallelism());
for (const { id, basePath, headPath } of comparablePairs(options, colorSchemes, baseColorSchemes)) {
const started = limit(() => compareInBudget(basePath, headPath));
started.catch(() => undefined);
ahead.set(id, started);
}
}
for (const surface of options.plan.surfaces) {
for (const side of ["base", "head"] as const) {
if (!surface[`${side}Capture`] || surface[`${side}Surface`]?.kind !== "screen") continue;
@@ -598,48 +678,29 @@ export async function assessPublicEvidence(options: {
continue;
}
let visual: PublicVisualVerdict;
let assessment: VisualEvidenceAssessment | undefined;
if (now() > deadline) {
let compared: ComparedPair | undefined;
try {
compared = await (ahead.get(id) ?? compareInBudget(basePath, headPath));
} catch (error) {
visualCounts.failed += 1;
const diagnostic = comparisonError(id, error, { code: "VISUAL_COMPARISON_FAILED", stage: "compare",
message: "Screenshot comparison failed.", nextAction: "Check the screenshot files and the agent-device comparison diagnostics, then rerun comparison.", retryable: true });
recordError(visualCounts.errors, id, diagnostic.message);
continue;
}
if (!compared) {
visualCounts.failed += 1;
const diagnostic = comparisonError(id, undefined, { code: "VISUAL_COMPARISON_BUDGET_EXHAUSTED", stage: "compare",
message: "The time budget expired before screenshot comparison.",
nextAction: "Increase the evidence comparison time budget or reduce the number of selected routes, then rerun comparison.", retryable: true });
recordError(visualCounts.errors, id, diagnostic.message);
continue;
} else {
try {
const raw = await measure(basePath, headPath);
const { measurement, ignoredStatusBar: ignoredTop, ignoredHomeIndicator: ignoredBottom } = ignoreSystemRegions(raw, system, systemBandFrameLoader(raw, threshold));
assessment = assessVisualMeasurements({
baselineSamples: 1,
currentSamples: 1,
requiredSamples: 1,
baselineWithin: [],
currentWithin: [],
crossBuild: [measurement],
representative: measurement,
});
visual = visualVerdict(assessment, threshold);
// Measuring moves is extra: once the comparison budget is spent, regions go out unmeasured.
if (visual.verdict === "modified" && visual.regions?.length && now() <= deadline) {
Object.assign(visual, await withRegionShifts(basePath, headPath, measurement.regions, visual.regions));
}
if (ignoredTop > 0) visual.ignoredStatusBarRegions = ignoredTop;
if (ignoredBottom > 0) visual.ignoredHomeIndicatorRegions = ignoredBottom;
if (ignoredTop > 0 || ignoredBottom > 0) visual.systemRegions = system.map((area) => ({ ...area }));
visualCounts.compared += 1;
if (visual.verdict === "modified") visualCounts.modified += 1;
else if (visual.verdict === "unchanged") visualCounts.unchanged += 1;
else visualCounts.needsReview += 1;
} catch (error) {
visualCounts.failed += 1;
const diagnostic = comparisonError(id, error, { code: "VISUAL_COMPARISON_FAILED", stage: "compare",
message: "Screenshot comparison failed.", nextAction: "Check the screenshot files and the agent-device comparison diagnostics, then rerun comparison.", retryable: true });
recordError(visualCounts.errors, id, diagnostic.message);
continue;
}
}
const { visual, assessment } = compared;
visualCounts.compared += 1;
if (visual.verdict === "modified") visualCounts.modified += 1;
else if (visual.verdict === "unchanged") visualCounts.unchanged += 1;
else visualCounts.needsReview += 1;
const scope = surfaceSourceScope(surface) ?? "none";
let review: PublicSurfaceReview | undefined;
+33
View File
@@ -0,0 +1,33 @@
import { Worker, isMainThread, parentPort, workerData } from "node:worker_threads";
import { measureRegionMoves, type Rect } from "./region-shift.js";
type MeasuredRegions = Awaited<ReturnType<typeof measureRegionMoves>>;
type RegionMoveWork = { regionMoveWork: true; basePath: string; headPath: string; regions: ReadonlyArray<Rect & { dominantChange?: string }> };
type RegionMoveReply = { measured: MeasuredRegions } | { error: string };
/**
* Measures region shifts and large moves on a worker thread, so several screenshot pairs search at once while the
* main thread stays free: the search is CPU-bound and takes seconds on a tablet-sized screenshot. Falls back to the
* calling thread when no worker can start.
*/
export async function measureRegionMovesInWorker(basePath: string, headPath: string, regions: RegionMoveWork["regions"]): Promise<MeasuredRegions> {
const work: RegionMoveWork = { regionMoveWork: true, basePath, headPath, regions };
let worker: Worker;
try { worker = new Worker(new URL("./region-shift-worker.js", import.meta.url), { workerData: work }); }
catch { return measureRegionMoves(basePath, headPath, regions); }
return new Promise((resolve, reject) => {
let settled = false;
const finish = (outcome: () => void) => { if (!settled) { settled = true; outcome(); } };
worker.once("message", (reply: RegionMoveReply) => finish(() => "error" in reply ? reject(new Error(reply.error)) : resolve(reply.measured)));
// A worker that fails to load or dies before answering is a worker that did not start: the calling thread measures.
worker.once("error", () => finish(() => resolve(measureRegionMoves(basePath, headPath, regions))));
worker.once("exit", () => finish(() => resolve(measureRegionMoves(basePath, headPath, regions))));
});
}
if (!isMainThread && parentPort && (workerData as Partial<RegionMoveWork> | null)?.regionMoveWork === true) {
const port = parentPort;
const work = workerData as RegionMoveWork;
measureRegionMoves(work.basePath, work.headPath, work.regions)
.then(measured => port.postMessage({ measured } satisfies RegionMoveReply), (error: unknown) => port.postMessage({ error: (error as Error).message } satisfies RegionMoveReply));
}
+42 -9
View File
@@ -4,7 +4,7 @@ import type { DecodedFrame } from "../shared/png-evidence.js";
/** How far a region's content moved from Before to After, in screenshot pixels (right and down are positive). */
export type RegionShift = { x: number; y: number };
type Rect = { x: number; y: number; width: number; height: number };
export type Rect = { x: number; y: number; width: number; height: number };
/**
* The farthest a region is searched for, in screenshot pixels: layout drift, not a different screen. Drift adds up
@@ -63,16 +63,38 @@ function at(source: Image, x: number, y: number, channel?: number): number {
return value(x0, y0) * (1 - fx) * (1 - fy) + value(x1, y0) * fx * (1 - fy) + value(x0, y1) * (1 - fx) * fy + value(x1, y1) * fx * fy;
}
/** How badly After's region matches Before shifted by (sx, sy). A whole-pixel shift reads Before's pixels directly. */
function mismatch(base: Image, head: Image, rect: Rect, sx: number, sy: number): number {
/**
* How badly After's region matches Before shifted by (sx, sy): the summed luminance difference, row by row. A sum
* that passes `limit` stops early and returns a value above it: such a fit is neither the best, nor a tie, nor an
* ambiguity, so its exact error is never read. Whole-pixel shifts read the pixel directly (the bilinear weights are
* exactly 1 and 0 there); fractional ones interpolate like `at`.
*/
function mismatch(base: Image, head: Image, rect: Rect, sx: number, sy: number, limit = Infinity): number {
const { width, height, luminance } = base;
const maxX = width - 1, maxY = height - 1;
const headLuminance = head.luminance, headWidth = head.width;
const whole = Number.isInteger(sx) && Number.isInteger(sy);
let error = 0;
for (let y = rect.y; y < rect.y + rect.height; y += 1) {
const by = Math.min(base.height - 1, Math.max(0, y - sy));
for (let x = rect.x; x < rect.x + rect.width; x += 1) {
const before = whole ? base.luminance[by * base.width + Math.min(base.width - 1, Math.max(0, x - sx))]! : at(base, x - sx, y - sy);
error += Math.abs(before - head.luminance[y * head.width + x]!);
const headRow = y * headWidth;
if (whole) {
const row = Math.min(maxY, Math.max(0, y - sy)) * width;
for (let x = rect.x; x < rect.x + rect.width; x += 1) {
error += Math.abs(luminance[row + Math.min(maxX, Math.max(0, x - sx))]! - headLuminance[headRow + x]!);
}
} else {
const cy = Math.min(maxY, Math.max(0, y - sy));
const y0 = Math.floor(cy), y1 = Math.min(maxY, y0 + 1), fy = cy - y0;
const row0 = y0 * width, row1 = y1 * width;
for (let x = rect.x; x < rect.x + rect.width; x += 1) {
const cx = Math.min(maxX, Math.max(0, x - sx));
const x0 = Math.floor(cx), x1 = Math.min(maxX, x0 + 1), fx = cx - x0;
const sample = luminance[row0 + x0]! * (1 - fx) * (1 - fy) + luminance[row0 + x1]! * fx * (1 - fy)
+ luminance[row1 + x0]! * (1 - fx) * fy + luminance[row1 + x1]! * fx * fy;
error += Math.abs(sample - headLuminance[headRow + x]!);
}
}
if (error > limit) return error;
}
return error;
}
@@ -113,18 +135,29 @@ function regionShift(base: Image, head: Image, region: Rect): RegionShift | unde
const unshifted = mismatch(base, head, rect, 0, 0);
if (unshifted === 0) return undefined;
// Whole pixels first, every pixel counting: sampling every other row aliases on thin shapes (text, hairlines).
// A fit worse than the best so far by more than the ambiguity margin cannot become the best, tie with it, or make
// it ambiguous; its measurement stops as soon as that is known. The margin is taken from the worst fit the tie
// could choose, since a tie goes to the smallest move, not the least error.
const whole: Fit[] = [];
let least = unshifted;
for (let sy = -MAX_SHIFT_Y; sy <= MAX_SHIFT_Y; sy += 1) {
for (let sx = -MAX_SHIFT_X; sx <= MAX_SHIFT_X; sx += 1) whole.push({ sx, sy, error: mismatch(base, head, rect, sx, sy) });
for (let sx = -MAX_SHIFT_X; sx <= MAX_SHIFT_X; sx += 1) {
const error = sx === 0 && sy === 0 ? unshifted : mismatch(base, head, rect, sx, sy, (least * TIE + 1e-6) * AMBIGUOUS + 1e-6);
least = Math.min(least, error);
whole.push({ sx, sy, error });
}
}
const coarse = bestOf(whole);
if (whole.some(fit => Math.abs(fit.sy - coarse.sy) >= 2 && fit.error <= coarse.error * AMBIGUOUS + 1e-6)) return undefined;
// Then sixths of a pixel around it, from whole-number steps so a move up and a move down round alike.
const fine: Fit[] = [];
least = coarse.error;
for (let iy = -STEPS_PER_PIXEL; iy <= STEPS_PER_PIXEL; iy += 1) {
for (let ix = -STEPS_PER_PIXEL; ix <= STEPS_PER_PIXEL; ix += 1) {
const sx = coarse.sx + ix / STEPS_PER_PIXEL, sy = coarse.sy + iy / STEPS_PER_PIXEL;
fine.push({ sx, sy, error: ix === 0 && iy === 0 ? coarse.error : mismatch(base, head, rect, sx, sy) });
const error = ix === 0 && iy === 0 ? coarse.error : mismatch(base, head, rect, sx, sy, least * TIE + 1e-6);
least = Math.min(least, error);
fine.push({ sx, sy, error });
}
}
const best = bestOf(fine);
+28 -24
View File
@@ -1,4 +1,4 @@
import { spawnSync } from "node:child_process";
import { execFile } from "node:child_process";
import { lstat, readFile, realpath } from "node:fs/promises";
import { isAbsolute } from "node:path";
@@ -133,30 +133,34 @@ export async function diffScreenshots(options: {
|| !isAbsolute(options.command)
|| /[\u0000-\u001f\u007f]/.test(options.command)
) throw new Error("agent-device command must be an absolute path");
const result = spawnSync(
options.command,
[
"--json",
"diff",
"screenshot",
"--baseline",
baselinePath,
currentPath,
"--threshold",
String(threshold),
],
{
encoding: "utf8",
shell: false,
timeout: options.timeoutMs ?? 60_000,
env: {
PATH: process.env.PATH,
const result = await new Promise<{ error: (Error & { code?: unknown }) | null; stdout: string; stderr: string }>(resolve => {
execFile(
options.command,
[
"--json",
"diff",
"screenshot",
"--baseline",
baselinePath,
currentPath,
"--threshold",
String(threshold),
],
{
encoding: "utf8",
shell: false,
timeout: options.timeoutMs ?? 60_000,
env: {
PATH: process.env.PATH,
},
maxBuffer: 4 * 1024 * 1024,
},
maxBuffer: 4 * 1024 * 1024,
},
);
if (result.error) throw new Error(`Could not run screenshot diff: ${result.error.message}`);
if (result.status !== 0) {
(error, stdout, stderr) => resolve({ error, stdout, stderr }),
);
});
// execFile reports a non-zero exit as an error carrying the exit status; a spawn failure or timeout carries none.
if (result.error && typeof result.error.code !== "number") throw new Error(`Could not run screenshot diff: ${result.error.message}`);
if (result.error) {
const detail = (result.stderr || result.stdout || "unknown error").trim().slice(0, 500);
throw new Error(`Screenshot diff failed: ${detail}`);
}
+135
View File
@@ -0,0 +1,135 @@
import assert from "node:assert/strict";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { startArgentBridgeRunner } from "../../dist/capture/argent-bridge-host.js";
import { ARGENT_VERSION } from "../../dist/shared/argent-version.js";
const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
async function fakeClient(root, body) {
const module = join(root, "client.mjs");
await writeFile(module, `const calls = [];\nexport function createArgentClient() { return { calls, async callTool(name, args, options) { calls.push({ name, args }); ${body} } }; }\n`);
return module;
}
test("the bridge thread answers the capture's commands through Argent's client and falls back for other executables", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
await writeFile(join(root, "shot.png"), PNG);
const clientModule = await fakeClient(root, `
if (name === "list-devices") return { data: { devices: [{ udid: "UDID-1", name: "iPhone 17", state: "Booted", platform: "ios", runtime: "iOS 26.0" }] } };
if (name === "screenshot") return { data: { path: ${JSON.stringify(join(root, "shot.png"))} } };
if (name === "await-screen-idle") { if (options?.signal?.aborted) throw options.signal.reason; await new Promise((resolve, reject) => { const timer = setTimeout(resolve, 5_000); options?.signal?.addEventListener("abort", () => { clearTimeout(timer); reject(options.signal.reason); }); }); return { data: { idle: true } }; }
throw new Error("unexpected tool " + name);`);
const fallbackCalls = [];
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/tools", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule, fallback: request => { fallbackCalls.push(request.args[0]); return { stdout: "{}", stderr: "" }; } });
t.after(() => bridge.close());
const request = args => ({ command: "/opt/argent-controller", args, cwd: root, environment, timeoutMs: 20_000 });
const version = bridge.runner(request(["--version"]));
assert.equal(version.stdout.trim(), ARGENT_VERSION);
const shot = JSON.parse(bridge.runner(request(["screenshot", join(root, "out", "screen.png"), "--session", "explore", "--platform", "ios", "--json"])).stdout);
assert.equal(shot.success, true);
assert.deepEqual(await readFile(join(root, "out", "screen.png")), PNG, "the materialized artifact is copied where the command asked");
assert.ok(bridge.active);
bridge.runner({ command: "/usr/bin/other", args: ["noop"], cwd: root, environment });
assert.deepEqual(fallbackCalls, ["noop"], "other executables go to the fallback runner");
const started = Date.now();
assert.throws(() => bridge.runner({ ...request(["wait", "stable", "500", "8000", "--session", "explore", "--platform", "ios", "--json"]), timeoutMs: 300 }), /timed out after 300 ms/);
assert.ok(Date.now() - started < 5_000, "a timed-out command is aborted, not waited out");
assert.ok(bridge.active, "the thread answered the abort, so it keeps serving");
assert.equal(bridge.runner(request(["--version"])).stdout.trim(), ARGENT_VERSION);
});
test("a thread whose Argent client cannot load hands every command to the fallback runner", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
const fallbackCalls = [];
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/tools", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule: join(root, "missing.mjs"), fallback: request => { fallbackCalls.push(request.args[0]); return { stdout: "{}", stderr: "" }; } });
t.after(() => bridge.close());
bridge.runner({ command: "/opt/argent-controller", args: ["snapshot", "--json"], cwd: root, environment, timeoutMs: 20_000 });
bridge.runner({ command: "/opt/argent-controller", args: ["screenshot", "x.png", "--json"], cwd: root, environment, timeoutMs: 20_000 });
assert.deepEqual(fallbackCalls, ["snapshot", "screenshot"]);
assert.equal(bridge.active, false);
});
test("a session whose tool-server was replaced is driven at the new one: the thread starts again with the changed environment", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
await writeFile(join(root, "shot.png"), PNG);
// The fake client notes which tool-server it was given, as Argent's reads ARGENT_TOOLS_URL from its environment.
const clientModule = await fakeClient(root, `
if (name === "list-devices") return { data: { devices: [{ udid: "UDID-1", name: "iPhone 17", state: "Booted", platform: "ios", runtime: "iOS 26.0" }] } };
if (name === "screenshot") { (await import("node:fs")).appendFileSync(${JSON.stringify(join(root, "servers.txt"))}, process.env.ARGENT_TOOLS_URL + "\\n"); return { data: { path: ${JSON.stringify(join(root, "shot.png"))} } }; }
throw new Error("unexpected tool " + name);`);
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/old", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule, fallback: () => assert.fail("the thread serves every command") });
t.after(() => bridge.close());
const screenshot = () => JSON.parse(bridge.runner({ command: "/opt/argent-controller", args: ["screenshot", join(root, "out", "screen.png"), "--session", "explore", "--platform", "ios", "--json"],
cwd: root, environment, timeoutMs: 20_000 }).stdout);
assert.equal(screenshot().success, true);
// As the capture's local recovery does: the same environment object, pointed at the restarted tool-server.
Object.assign(environment, { ARGENT_TOOLS_URL: "http://127.0.0.1:9/new", ARGENT_AUTH_TOKEN: "new-token" });
assert.equal(screenshot().success, true);
assert.deepEqual((await readFile(join(root, "servers.txt"), "utf8")).trim().split("\n"), ["http://127.0.0.1:9/old", "http://127.0.0.1:9/new"]);
assert.ok(bridge.active);
});
test("a thread that crashes stops serving, and the commands after it run per command", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
await writeFile(join(root, "shot.png"), PNG);
// An exception nothing catches, thrown on the thread well after its answer.
const clientModule = await fakeClient(root, `
if (name === "list-devices") return { data: { devices: [{ udid: "UDID-1", name: "iPhone 17", state: "Booted", platform: "ios", runtime: "iOS 26.0" }] } };
if (name === "screenshot") { setTimeout(() => { throw new Error("the client crashed"); }, 300); return { data: { path: ${JSON.stringify(join(root, "shot.png"))} } }; }
throw new Error("unexpected tool " + name);`);
const fallbackCalls = [], messages = [];
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/tools", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule, progress: message => messages.push(message),
fallback: request => { fallbackCalls.push(request.args[0]); return { stdout: "{}", stderr: "" }; } });
t.after(() => bridge.close());
const request = args => ({ command: "/opt/argent-controller", args, cwd: root, environment, timeoutMs: 20_000 });
assert.equal(JSON.parse(bridge.runner(request(["screenshot", join(root, "out", "screen.png"), "--session", "explore", "--platform", "ios", "--json"])).stdout).success, true);
await new Promise(resolve => setTimeout(resolve, 1_500));
assert.equal(bridge.active, false);
assert.match(messages.join("\n"), /the Argent bridge thread failed \(the client crashed\)/);
bridge.runner(request(["snapshot", "--json"]));
assert.deepEqual(fallbackCalls, ["snapshot"]);
});
test("a recording runs per command, and a screenshot is taken only from a file that is an image", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-argent-bridge-")); t.after(() => rm(root, { recursive: true, force: true }));
await writeFile(join(root, "shot.png"), PNG);
await writeFile(join(root, "notes.txt"), "not a screenshot\n");
const answer = join(root, "answer.json");
const clientModule = await fakeClient(root, `
if (name === "list-devices") return { data: { devices: [{ udid: "UDID-1", name: "iPhone 17", state: "Booted", platform: "ios", runtime: "iOS 26.0" }] } };
if (name === "screenshot") return { data: JSON.parse((await import("node:fs")).readFileSync(${JSON.stringify(answer)}, "utf8")) };
throw new Error("unexpected tool " + name);`);
const fallbackCalls = [];
const environment = { EXPLORE_ARGENT_STATE: join(root, "state.json"), EXPLORE_ARGENT_PLATFORM: "ios", ARGENT_TOOLS_URL: "http://127.0.0.1:9/tools", HOME: root, PATH: process.env.PATH };
const bridge = startArgentBridgeRunner({ command: "/opt/argent-controller", environment, clientModule,
fallback: request => { fallbackCalls.push(request.args.slice(0, 2).join(" ")); return { stdout: "{}", stderr: "" }; } });
t.after(() => bridge.close());
const request = args => ({ command: "/opt/argent-controller", args, cwd: root, environment, timeoutMs: 20_000 });
bridge.runner(request(["record", "start", join(root, "launch.mp4"), "--scope", "app", "--json"]));
bridge.runner(request(["record", "stop", "--json"]));
assert.deepEqual(fallbackCalls, ["record start", "record stop"], "recordings go to the per-command bridge");
assert.ok(bridge.active);
const screenshot = out => JSON.parse(bridge.runner({ ...request(["screenshot", out, "--session", "explore", "--platform", "ios", "--json"]), allowFailure: true }).stdout);
// A path to a file that is not an image is not a screenshot, whatever its key; the image beside it is.
await writeFile(answer, JSON.stringify({ path: join(root, "notes.txt"), image: join(root, "shot.png") }));
assert.equal(screenshot(join(root, "out", "one.png")).success, true);
assert.deepEqual(await readFile(join(root, "out", "one.png")), PNG);
await writeFile(answer, JSON.stringify({ path: join(root, "notes.txt") }));
const failed = screenshot(join(root, "out", "two.png"));
assert.equal(failed.success, false);
assert.match(failed.error.message, /did not write/);
await assert.rejects(readFile(join(root, "out", "two.png")), { code: "ENOENT" });
});
+45 -2
View File
@@ -340,8 +340,10 @@ test("expired comparison budgets produce diagnostics rather than review verdicts
const root = await mkdtemp(join(tmpdir(), "explore-public-evidence-")); t.after(() => rm(root, { recursive: true, force: true }));
const base = await side(root, "base", manifests.base);
const head = await side(root, "head", manifests.head);
let clock = 0;
const result = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", timeBudgetMs: 0, now: () => clock++, measure: async () => { assert.fail('expired comparisons do not run'); } });
let clock = 0, measured = 0;
const result = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", timeBudgetMs: 0, now: () => clock++,
measure: async (...args) => { measured += 1; return measurer({})(...args); } });
assert.equal(measured, 0, "expired comparisons do not run");
assert.equal(result.summary.visual.compared, 0);
assert.equal(result.summary.visual.failed, 3);
assert.equal(result.summary.visual.needsReview, 0);
@@ -526,6 +528,47 @@ test("a presence verdict for a long route (an encoded share link) stays within t
assert.equal(presenceDescription("Added ", "/home", " screen."), "Added /home screen.");
});
test("screenshot pairs compare a few at a time, in the loop's order, with the same verdicts as one at a time", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-public-evidence-")); t.after(() => rm(root, { recursive: true, force: true }));
const base = await side(root, "base", manifests.base);
const head = await side(root, "head", manifests.head);
const run = async (concurrency) => {
let active = 0, peak = 0;
const started = [];
const measure = async (baselinePath, currentPath) => {
started.push(baselinePath);
active += 1; peak = Math.max(peak, active);
await new Promise(resolve => setTimeout(resolve, 20));
active -= 1;
return measurer({ changed: 3.2, stable: 0.4 })(baselinePath, currentPath);
};
const result = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device", measure, concurrency });
return { peak, started, surfaces: result.surfaces, summary: result.summary };
};
const sequential = await run(1);
const parallel = await run(4);
assert.equal(sequential.peak, 1);
assert.ok(parallel.peak > 1, `comparisons overlap (peak ${parallel.peak})`);
assert.deepEqual(parallel.started, sequential.started, "pairs start in the loop's order");
assert.deepEqual(parallel.surfaces, sequential.surfaces);
assert.deepEqual(parallel.summary, sequential.summary);
});
test("a pair whose turn comes after the budget expired is not started, and one started in time is used", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-public-evidence-")); t.after(() => rm(root, { recursive: true, force: true }));
const base = await side(root, "base", manifests.base);
const head = await side(root, "head", manifests.head);
// The first comparison takes past the budget: it was started in time, the two queued behind it were not.
let time = 0, measured = 0;
const measure = async (...args) => { measured += 1; time += 1_000; return measurer({})(...args); };
const result = await assessPublicEvidence({ plan: plan(), base, head, profile: "explore:bundle", agentDeviceCommand: "/nonexistent/agent-device",
timeBudgetMs: 100, now: () => time, measure, concurrency: 1 });
assert.equal(measured, 1);
assert.equal(result.summary.visual.compared, 1);
assert.equal(result.summary.visual.failed, 2);
assert.ok(result.summary.diagnostics.every(item => item.code === 'VISUAL_COMPARISON_BUDGET_EXHAUSTED'));
});
test("unstable text never makes a screen Modified, and names a variant's runtime state", () => {
const unstable = { category: "visible-text", templateId: "visible-text.unstable-within-build", bullet: 'Text varies between captures of the same build: "2m" / "4m" / "3m".' };
const summary = (facts) => ({ facts, needsReview: false, reviewIssues: [] });
+34 -1
View File
@@ -1,7 +1,12 @@
import assert from "node:assert/strict";
import test from "node:test";
import { Buffer } from "node:buffer";
import { regionMoves, regionShifts } from "../../dist/review/region-shift.js";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { PNG } from "pngjs";
import { measureRegionMoves, regionMoves, regionShifts } from "../../dist/review/region-shift.js";
import { measureRegionMovesInWorker } from "../../dist/review/region-shift-worker.js";
/** A 120×160 frame of a light background with dark text-like bars, drawn `offsetY` pixels lower (fractions blend rows). */
function frame(offsetY, { barColor = 40, extraBar = false } = {}) {
@@ -102,6 +107,18 @@ test("a half-pixel move up and down is judged alike", () => {
assert.deepEqual([up?.y, down?.y], [-0.5, 0.5]);
});
test("a move whose best fit ties with a smaller one is still a move: the search stops no fit that could make it ambiguous", () => {
// Two and a half pixels lower, with a sparse speckle of noise. The tie goes to a fit a little worse than the least error,
// and an ambiguity is judged against that one, so a fit stopped short against the least error must not pass for one.
const moved = frame(2.5), speckled = { ...moved, data: Buffer.from(moved.data) };
for (let y = 0; y < moved.height; y += 1) for (let x = 0; x < moved.width; x += 1) {
if ((x * 3 + y * 5) % 11 !== 0) continue;
const offset = (y * moved.width + x) * 4, value = Math.max(0, Math.min(255, moved.data[offset] + ((x + y) % 2 ? 10 : -10)));
speckled.data.fill(value, offset, offset + 3);
}
assert.deepEqual(regionShifts(frame(0), speckled, [region]), [{ x: 0, y: 2.5 }]);
});
/**
* A light frame with a dark card of irregular blocks at (x, y): the blocks make every offset look different, as a
* real card's text and icons do. `tile` repeats one such block pattern every `tile` pixels across the whole row
@@ -138,6 +155,22 @@ test("a card that moved far along a row is one move, from the regions where it l
[{ rect: { x: 330, y: 80, width: 150, height: 120 }, shift: { x: 270, y: 0 }, members: [1, 0] }], "the region it arrived in first");
});
test("a large move is measured on a worker thread the same as on the calling thread", async (t) => {
const root = await mkdtemp(join(tmpdir(), "explore-region-shift-")); t.after(() => rm(root, { recursive: true, force: true }));
const write = async (name, frame) => {
const png = new PNG({ width: frame.width, height: frame.height }); frame.data.copy(png.data);
await writeFile(join(root, name), PNG.sync.write(png)); return join(root, name);
};
const basePath = await write("base.png", cardFrame()), headPath = await write("head.png", cardFrame({ x: 330 }));
const regions = [
{ x: 60, y: 80, width: 150, height: 120, dominantChange: "brighter" },
{ x: 330, y: 80, width: 150, height: 120, dominantChange: "darker" },
];
const measured = await measureRegionMovesInWorker(basePath, headPath, regions);
assert.deepEqual(measured, await measureRegionMoves(basePath, headPath, regions));
assert.deepEqual(measured.moves, [{ rect: { x: 330, y: 80, width: 150, height: 120 }, shift: { x: 270, y: 0 }, members: [1, 0] }]);
});
test("a move whose old and new places overlap takes in the fragments of it the comparison found in between", () => {
// A 300-wide card moved 120 right: the comparison sees where it only was, where it only is, and a fragment of the
// overlap whose pixels differ because the card's own content moved over itself.