Compare commits

..
Author SHA1 Message Date
Cursor Agent 5c137ce50e chore: merge main into codex/audit-desktop-build 2026-11-11 02:26:33 +00:00
Cursor Agent c98594638d test(desktop): match reader source inspection diagnostics
Unreadable reader sources now report the source inspection failure, including the operating-system reason, instead of the old file wording.
2026-11-10 23:21:11 +00:00
morluto 1bfc8e0dec fix(desktop)!: preserve build evidence and validate task inputs
BREAKING CHANGE: Witnessing requires the current generated verifier. Re-export bundles with older evaluators using the original capture Evidence and task inputs.
2026-11-10 17:33:06 -05:00
52 changed files with 1506 additions and 750 deletions

No files matched your search

+5
View File
@@ -262,6 +262,11 @@ in the bundle differs from what the export writes for it. A reset
relaunches the app, and REA waits until the computer is ready again before
answering, sending progress notifications while it waits.
Re-export bundles whose generated verifier uses an older evaluator before
witnessing them with current REA. The verifier must match the current export
exactly; re-exporting uses the existing capture Evidence and task inputs,
which remain unchanged.
Only a check that fails on what the app's view shows counts as a failure. A
check that fails because its file or table could not be read, such as an
SQLite file locked past the reader's wait, or because what it looks for may
@@ -5,8 +5,6 @@ import { expect, it, vi } from "vitest";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { apiContractSchema } from "../../domain/apiContract/apiContract.js";
import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
import { parseEvidence } from "../../domain/evidence.js";
import { projectAnalysisView } from "../../domain/analysisView/analysisView.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { inspectApiContractEvidence } from "./ApiContractService.js";
import * as readOwner from "./ApiContractRead.js";
@@ -319,89 +317,3 @@ it("cancels a cooperative document traversal after the actual verified file read
await resources.close();
}
});
it.each(["between-paths", "after-projection"] as const)(
"preserves authenticated OpenAPI facts on cancellation %s",
async (stage) => {
const bytes = JSON.stringify({
openapi: "3.1.2",
info: { title: "Cancellation", version: "1" },
paths: { "/first": { get: {} }, "/second": { post: {} } },
});
const { path } = await inputFile(bytes);
const resources = new ArtifactResourceScope();
const controller = new AbortController();
const realParse = parser.parseApiContractSteps;
try {
const baseline = await inspectApiContractEvidence({ path }, resources);
if (!baseline.ok) throw baseline.error;
const complete = apiContractSchema.parse(
baseline.value.normalized_result,
);
const scheduled = vi
.spyOn(parser, "parseApiContractSteps")
.mockImplementationOnce(function* (...args) {
const steps = realParse(...args);
if (stage === "between-paths") {
expect(steps.next().done).toBe(false);
expect(steps.next().done).toBe(false);
setImmediate(() => controller.abort(stage));
return yield* steps;
}
const result = yield* steps;
setImmediate(() => controller.abort(stage));
return result;
});
try {
const result = await inspectApiContractEvidence({ path }, resources, {
signal: controller.signal,
});
if (result.ok) throw new Error("Expected cancellation");
expect(result.error._tag).toBe("AnalysisCancelledError");
expect(result.error.cause).toBe(stage);
const evidence = parseEvidence(result.error.partialObservation);
const retained = apiContractSchema.parse(evidence.normalized_result);
expect(evidence.subject).toEqual(baseline.value.subject);
expect(evidence.parameters).toEqual(baseline.value.parameters);
expect(retained.artifact).toEqual(complete.artifact);
expect(retained.contract_id).toBe(complete.contract_id);
if (stage === "between-paths") {
expect(retained.operations).toEqual(complete.operations.slice(0, 1));
expect(retained.source_coverage.status).toBe("partial");
expect(evidence.evidence_id).not.toBe(baseline.value.evidence_id);
const page = projectAnalysisView(
{
evidenceId: evidence.evidence_id,
operation: evidence.operation,
predicateType: evidence.predicate_type,
normalizedResult: evidence.normalized_result,
limitations: evidence.limitations,
},
{
kind: "api-contract",
collection: "operations",
offset: 0,
limit: 1,
},
);
expect(page).toMatchObject({
ok: true,
value: {
parent_evidence_id: evidence.evidence_id,
items: [{ path: "/first", state: "declared" }],
coverage: { exhausted: true },
source_coverage: { status: "partial" },
},
});
} else {
expect(retained).toEqual(complete);
expect(evidence.evidence_id).toBe(baseline.value.evidence_id);
}
} finally {
scheduled.mockRestore();
}
} finally {
await resources.close();
}
},
);
@@ -11,8 +11,6 @@ import { err, ok, type Result } from "../../domain/result.js";
import {
API_CONTRACT_OPERATION as operation,
API_CONTRACT_PROVIDER,
apiContractSchema,
type ApiContract,
type ApiContractArtifact,
} from "../../domain/apiContract/apiContract.js";
import {
@@ -72,32 +70,8 @@ function* documentSteps(document: unknown, artifact: ApiContractArtifact) {
return yield* parseApiContractSteps(document, artifact);
}
type OpenApiInput = Extract<
InspectApiContractInput,
{ format: "openapi-json" }
>;
const documentEvidence = (input: OpenApiInput, result: ApiContract): Evidence =>
createEvidence(
{ path: input.path, format: "file", sha256: result.artifact.sha256 },
API_CONTRACT_PROVIDER,
{
operation,
predicateType: "rea.api-contract-inventory",
parameters: {
path: input.path,
format: input.format,
max_file_bytes: input.max_file_bytes,
},
result,
confidence: "observed",
limitations: result.limitations,
locations: [{ kind: "artifact-path", path: input.path }],
},
);
const importDocument = async (
input: OpenApiInput,
input: Extract<InspectApiContractInput, { format: "openapi-json" }>,
bytes: Buffer,
sha256: string,
signal?: AbortSignal,
@@ -112,16 +86,26 @@ const importDocument = async (
}),
signal,
);
if (!parsed.ok) {
const partial = apiContractSchema.safeParse(
parsed.error.partialObservation,
);
if (partial.success)
parsed.error.partialObservation = documentEvidence(input, partial.data);
return parsed;
}
if (!parsed.ok) return parsed;
const result = parsed.value;
return completeProviderEvidence(
documentEvidence(input, parsed.value),
createEvidence(
{ path: input.path, format: "file", sha256 },
API_CONTRACT_PROVIDER,
{
operation,
predicateType: "rea.api-contract-inventory",
parameters: {
path: input.path,
format: input.format,
max_file_bytes: input.max_file_bytes,
},
result,
confidence: "observed",
limitations: result.limitations,
locations: [{ kind: "artifact-path", path: input.path }],
},
),
signal,
);
};
@@ -0,0 +1,38 @@
import { chmod, mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { seededLockFiles } from "./DesktopBuildBundle.js";
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
"keeps lock inspection failures separate from empty and partially examined seeds",
async () => {
const root = await createTestTempDirectory("rea-seeded-locks-");
expect(await seededLockFiles(root)).toEqual({ locks: [], unreadable: [] });
const envs = join(root, "envs");
const denied = join(envs, "denied", "seed");
const available = join(envs, "available", "seed");
await mkdir(denied, { recursive: true });
await mkdir(available, { recursive: true });
await writeFile(join(available, "SingletonLock"), "controlled lock");
await chmod(denied, 0);
try {
const found = await seededLockFiles(root);
expect(found.locks).toEqual(["envs/available/seed/SingletonLock"]);
expect(found.unreadable).toEqual([
{ path: denied, reason: expect.stringContaining("EACCES") },
]);
} finally {
await chmod(denied, 0o700);
}
await chmod(envs, 0);
try {
expect(await seededLockFiles(root)).toEqual({
locks: [],
unreadable: [{ path: envs, reason: expect.stringContaining("EACCES") }],
});
} finally {
await chmod(envs, 0o700);
}
},
);
+44 -6
View File
@@ -20,6 +20,9 @@ import {
import { isLockFile } from "../../domain/desktop/appExportUnowned.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js";
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { bindArtifactFileCleanup } from "../../artifacts/ArtifactFileOperation.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
const START = "start_build_computer";
const MAX_KEY_BYTES = 1024 * 1024;
@@ -76,6 +79,7 @@ const readAptKey = async (path: string): Promise<Buffer> => {
const writeBuildBundle = async (
request: BuildRequest,
payload: ReaPayload,
resources: ArtifactResourceScope,
): Promise<{
readonly workspace: PrivateRuntimeRoot;
readonly bundle: string;
@@ -103,7 +107,23 @@ const writeBuildBundle = async (
}
return { workspace, bundle, name };
} catch (cause: unknown) {
await workspace.close();
const released = await resources.release({
kind: "private-runtime-root",
resource: workspace.path,
root: workspace,
});
if (released.kind === "failed") {
const cleanup = resources.cleanupFailure({ kind: "failed", cause }, [
released,
]);
const error = new ProviderAdapterError(AGENTENV_PROVIDER, START, {
cause,
userMessage: cause instanceof Error ? cause.message : String(cause),
cleanup: cleanup.cleanup,
});
bindArtifactFileCleanup(cleanup, error);
throw error;
}
throw cause;
}
};
@@ -115,7 +135,8 @@ const writeBuildBundle = async (
export const writeRequestBundle = async (
request: BuildRequest,
environment: Readonly<NodeJS.ProcessEnv>,
) => writeBuildBundle(request, await reaPayload(environment));
resources: ArtifactResourceScope,
) => writeBuildBundle(request, await reaPayload(environment), resources);
/**
* The lock files in a bundle folder's seeds, as `envs/<name>/seed/<home
@@ -125,16 +146,33 @@ export const writeRequestBundle = async (
*/
export const seededLockFiles = async (
bundle: string,
): Promise<readonly string[]> => {
): Promise<{
readonly locks: readonly string[];
readonly unreadable: readonly {
readonly path: string;
readonly reason: string;
}[];
}> => {
const environments = join(bundle, "envs");
const names = await readdir(environments).catch(() => []);
const unreadable: { path: string; reason: string }[] = [];
const unavailable = (path: string, cause: unknown) => {
if (!(cause instanceof Error && "code" in cause && cause.code === "ENOENT"))
unreadable.push({
path,
reason: cause instanceof Error ? cause.message : String(cause),
});
return [];
};
const names = await readdir(environments).catch((cause: unknown) =>
unavailable(environments, cause),
);
const locks: string[] = [];
for (const name of names) {
const seed = join(environments, name, "seed");
const entries = await readdir(seed, {
recursive: true,
withFileTypes: true,
}).catch(() => []);
}).catch((cause: unknown) => unavailable(seed, cause));
const homes = new Set(
entries
.filter((entry) => entry.isFile())
@@ -147,5 +185,5 @@ export const seededLockFiles = async (
for (const home of homes)
if (isLockFile(home, [], homes)) locks.push(`envs/${name}/seed/${home}`);
}
return locks.sort();
return { locks: locks.sort(), unreadable };
};
@@ -1,11 +1,157 @@
import { expect, it, vi } from "vitest";
import { AgentEnvComputer } from "../../desktop/AgentEnvComputer.js";
import { access, chmod, mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { request } from "../../domain/desktop/appBuild.fixture.js";
import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js";
import {
AgentEnvComputer,
ComputerStartFailure,
} from "../../desktop/AgentEnvComputer.js";
import { ComputerHelper } from "../../desktop/AgentEnvComputerHelper.js";
import { ScreenJudgeProcess } from "../../desktop/ScreenJudgeProcess.js";
import { BuildComputerLongRuns } from "../../desktop/BuildComputerLongRuns.js";
import { DesktopBuildComputers } from "./DesktopBuildComputers.js";
import { DesktopRemoteTaskReaders } from "./DesktopRemoteTaskReaders.js";
it("resolves the original failed start's cleanup after its retained computer closes", async () => {
const release = vi
.fn()
.mockResolvedValueOnce({
verified: false,
reason: "container busy",
resources: ["sandbox"],
})
.mockResolvedValue({ verified: true, teardown: "agentenv" });
const spawn = vi.spyOn(ComputerHelper, "spawn").mockResolvedValue({
release,
output: () => undefined,
} as never);
const connect = vi
.spyOn(AgentEnvComputer, "connect")
.mockRejectedValue(new Error("readiness failed"));
const computers = new DesktopBuildComputers({
REA_AGENT_ENV_COMMAND: "/runner",
REA_DOCKER_COMMAND: "/docker",
});
try {
const started = await computers.start(
{ source: { kind: "bundle", path: "/bundle" } },
new AbortController().signal,
);
if (started.ok) throw new Error("Expected failed start");
const id = started.error.diagnostics?.computer_id;
if (typeof id !== "string")
throw new Error("Expected retained computer id");
expect(started.error.cleanupIncomplete).toBe(true);
expect(started.error.cleanupResources).toEqual(["sandbox"]);
expect((await computers.close(id)).ok).toBe(true);
expect(started.error.cleanupIncomplete).toBe(false);
expect(started.error.cleanupResources).toEqual([]);
expect(started.error.userMessage).toBe("readiness failed");
} finally {
await computers.closeAll();
spawn.mockRestore();
connect.mockRestore();
}
});
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
"preserves a readiness failure while reporting that seed locks could not be checked",
async () => {
const bundle = await createTestTempDirectory("rea-unreadable-build-seed-");
const envs = join(bundle, "envs");
await mkdir(envs);
await chmod(envs, 0);
const original = new ComputerStartFailure(
"Application readiness failed",
"Repair the app",
{ reason: "not_ready" },
);
const spawn = vi.spyOn(ComputerHelper, "spawn").mockResolvedValue({
release: async () => ({ verified: true, teardown: "agentenv" }),
output: () => undefined,
} as never);
const connect = vi
.spyOn(AgentEnvComputer, "connect")
.mockRejectedValue(original);
const computers = new DesktopBuildComputers({
REA_AGENT_ENV_COMMAND: "/runner",
REA_DOCKER_COMMAND: "/docker",
});
try {
const started = await computers.start(
{ source: { kind: "bundle", path: bundle } },
new AbortController().signal,
);
expect(started.ok).toBe(false);
if (started.ok) throw new Error("Expected readiness failure");
expect(started.error.userMessage).toContain(original.message);
expect(started.error.userMessage).toContain("could not check");
expect(started.error.cause).toMatchObject({
reason: "not_ready",
cause: original,
});
} finally {
await chmod(envs, 0o700);
await computers.closeAll();
spawn.mockRestore();
connect.mockRestore();
}
},
);
it("retains a failed bundle workspace and its write error until shutdown retries cleanup", async () => {
const folder = await createTestTempDirectory("rea-build-bundle-cleanup-");
const payload = join(folder, "payload.tar");
await writeFile(payload, "controlled payload");
const create = PrivateRuntimeRoot.create.bind(PrivateRuntimeRoot);
let workspace: PrivateRuntimeRoot | undefined;
const created = vi
.spyOn(PrivateRuntimeRoot, "create")
.mockImplementation(async (options) => {
workspace = await create(options);
await writeFile(join(workspace.path, "bundle"), "blocks bundle creation");
vi.spyOn(workspace, "close").mockRejectedValueOnce(
new Error("workspace removal failed"),
);
return workspace;
});
const computers = new DesktopBuildComputers({
REA_AGENT_ENV_COMMAND: "/runner",
REA_DOCKER_COMMAND: "/docker",
REA_BUILD_COMPUTER_REA_ARCHIVE: payload,
});
try {
const started = await computers.start(
{
source: request({
kind: "archive",
package: "mousepad",
extra_packages: [],
}),
},
new AbortController().signal,
);
expect(started.ok).toBe(false);
if (started.ok || workspace === undefined)
throw new Error("Expected bundle creation failure");
expect(started.error.userMessage).toContain("ENOTDIR");
expect(started.error.cleanup?.reason).toContain("workspace removal failed");
expect(started.error.cleanupResources).toEqual([workspace.path]);
await expect(access(workspace.path)).resolves.toBeUndefined();
await computers.closeAll();
await expect(access(workspace.path)).rejects.toMatchObject({
code: "ENOENT",
});
expect(started.error.cleanupIncomplete).toBe(false);
} finally {
created.mockRestore();
await computers.closeAll().catch(() => computers.closeAll());
await workspace?.close();
}
});
it.each([false, true])(
"releases the computer even if judge cleanup fails (computer also fails: %s)",
async (computerFails) => {
@@ -106,11 +106,23 @@ interface HeldComputer {
readonly turns: DesktopBuildTurns;
readonly cleanup: {
resources: readonly string[];
errors: Set<WeakRef<ProviderCleanupError>>;
errors: Set<WeakRef<AnalysisError>>;
};
closing: boolean;
}
/** Bind every returned teardown failure to the computer retained for retry. */
const bindCleanupFailure = (held: HeldComputer, error: AnalysisError): void => {
error.bindCleanupResourceOwner({
get cleanupResources() {
return held.cleanup.resources;
},
});
for (const reference of held.cleanup.errors)
if (reference.deref() === undefined) held.cleanup.errors.delete(reference);
held.cleanup.errors.add(new WeakRef(error));
};
/** What the build step tools work on: a live computer, its capture, its witness and the turns they take. */
export interface BuildComputerSteps {
readonly computer: AgentEnvComputer;
@@ -325,13 +337,21 @@ const withSeededLocks = async (
): Promise<unknown> => {
if (source.kind !== "bundle" || !computerFailedOnReadiness(cause))
return cause;
const locks = await seededLockFiles(source.path);
return locks.length === 0
const { locks, unreadable } = await seededLockFiles(source.path);
const notices = [
...(locks.length === 0 ? [] : [seededLockStartNotice(locks)]),
...(unreadable.length === 0
? []
: [
`REA could not check these seed folders for lock files: ${unreadable.map(({ path, reason }) => `${path}: ${reason}`).join("; ")}. Other lock files remain unknown.`,
]),
];
return notices.length === 0
? cause
: new ComputerStartFailure(
`${cause.message} ${seededLockStartNotice(locks)}`,
`${cause.message} ${notices.join(" ")}`,
cause.remediation,
{ cause: cause.cause, reason: cause.reason },
{ cause, reason: cause.reason },
);
};
@@ -466,7 +486,7 @@ export class DesktopBuildComputers {
target =
source.kind === "bundle"
? { bundle: source.path, name: source.path }
: await writeRequestBundle(source, environment);
: await writeRequestBundle(source, environment, this.#work.resources);
} catch (cause: unknown) {
return err(failure(START, cause, signal));
}
@@ -545,9 +565,16 @@ export class DesktopBuildComputers {
const failed = signal.aborted
? cause
: await withSeededLocks(source, cause);
return err(
startFailure(failed, signal, owner?.output(), { id, release }),
);
const error = startFailure(failed, signal, owner?.output(), {
id,
release,
});
const retained = this.#computers.get(id);
if (!release.verified && retained !== undefined) {
retained.cleanup.resources = release.resources;
bindCleanupFailure(retained, error);
}
return err(error);
}
}
@@ -699,15 +726,7 @@ export class DesktopBuildComputers {
{ reason: release.reason },
{ operation: CLOSE, remediationAction: release.reason },
);
failure.bindCleanupResourceOwner({
get cleanupResources() {
return held.cleanup.resources;
},
});
for (const reference of held.cleanup.errors)
if (reference.deref() === undefined)
held.cleanup.errors.delete(reference);
held.cleanup.errors.add(new WeakRef(failure));
bindCleanupFailure(held, failure);
return err(failure);
}
@@ -1,3 +1,7 @@
import * as buildEnvironment from "../../desktop/BuildComputerEnvironment.js";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { renderSmokeTask } from "../../domain/desktop/appExport.js";
import { MEASURE_BUILD_TASK_CONTRACT } from "../../contracts/desktopBuildMeasureContract.js";
import {
guestFileAnswerCeiling,
guestFileAnswerWorkingBytes,
@@ -18,7 +22,11 @@ import {
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import type { SolveReport } from "../../desktop/AgentEnvSolveHelper.js";
import { measureBuildTask, type Solver } from "./DesktopBuildSolves.js";
import {
measureBuildTask,
measureInFreshComputers,
type Solver,
} from "./DesktopBuildSolves.js";
/** A bundle holding stand-in steps and verifiers for these tasks, and a README. */
const bundleWith = async (...tasks: string[]) => {
@@ -422,3 +430,124 @@ it("retains completed solves when the destination becomes unreadable before publ
"invalid",
);
});
it.each(["../../outside", "nested/task", "nested\\task", ".", ".."])(
"refuses invalid selected task names before any file read or solve: %s",
async (task) => {
const read = vi
.spyOn(hostFiles, "readHostFile")
.mockRejectedValue(new Error("invalid task must not reach filesystem"));
const solve = vi.fn<Solver>();
try {
const result = await measureBuildTask(
request("/unused-bundle", task),
solve,
new AbortController().signal,
);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError", issues: [{ path: ["task"] }] },
});
expect(read).not.toHaveBeenCalled();
expect(solve).not.toHaveBeenCalled();
} finally {
read.mockRestore();
}
},
);
it("keeps the portable smoke name selectable independently of authoring reservations", async () => {
const bundle = await bundleWith("smoke");
const { solver, calls } = counting();
const result = await measureBuildTask(
request(bundle, "smoke"),
solver,
new AbortController().signal,
);
expect(result.ok).toBe(true);
expect(calls).toEqual(["smoke", "smoke", "smoke"]);
});
it("refuses invalid measurement identity before resolving the fresh-computer environment", async () => {
const environment = vi
.spyOn(buildEnvironment, "resolveBuildComputerEnvironment")
.mockRejectedValue(
new Error("invalid task must not resolve an environment"),
);
const read = vi.spyOn(hostFiles, "readHostFile");
const resources = new ArtifactResourceScope();
try {
const result = await measureInFreshComputers(
request("/unused-bundle", "../../outside"),
{
environment: {},
defaults: {},
retainCleanup: () => undefined,
resources,
},
new AbortController().signal,
);
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisInputError", issues: [{ path: ["task"] }] },
});
expect(environment).not.toHaveBeenCalled();
expect(read).not.toHaveBeenCalled();
} finally {
environment.mockRestore();
read.mockRestore();
await resources.close();
}
});
it("advertises the same portable lookup names without changing their spelling", () => {
for (const task of ["add-eggs", "123-task", "a".repeat(63), "smoke"])
expect(
MEASURE_BUILD_TASK_CONTRACT.inputSchema.parse(
request("/unused-bundle", task),
).task,
).toBe(task);
for (const task of [
"../../outside",
"nested/task",
"nested\\task",
".",
"..",
"con",
"123",
"a".repeat(64),
])
expect(
MEASURE_BUILD_TASK_CONTRACT.inputSchema.safeParse(
request("/unused-bundle", task),
).success,
).toBe(false);
});
it("keeps a generated deploy-only smoke task inapplicable without its measurement verifier", async () => {
const bundle = await createTestTempDirectory("measured-smoke-");
await mkdir(join(bundle, "tasks"));
await mkdir(join(bundle, "artifacts", "check_app"), { recursive: true });
await writeFile(
join(bundle, "tasks", "smoke.json"),
renderSmokeTask({ name: "app" }),
);
await writeFile(
join(bundle, "artifacts", "check_app", "verify.py"),
"# generated deployment check\n",
);
const solve = vi.fn<Solver>();
const result = await measureBuildTask(
request(bundle, "smoke"),
solve,
new AbortController().signal,
);
expect(result).toMatchObject({
ok: false,
error: {
_tag: "AnalysisInputError",
userMessage: expect.stringContaining("artifacts/smoke-check/verify.py"),
},
});
expect(solve).not.toHaveBeenCalled();
});
+37 -10
View File
@@ -1,3 +1,7 @@
import {
parsePortableBundleName,
PORTABLE_BUNDLE_NAME_CONSTRAINT,
} from "../../domain/portableBundleName.js";
import { createHash } from "node:crypto";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import {
@@ -89,6 +93,24 @@ const refused = (userMessage: string, remediationAction: string) =>
conformsToInputContract: true,
});
/** Refuse malformed lookup identity before reading files or resolving a solver. */
const admitTaskName = (name: string): Result<string, AnalysisError> => {
const parsed = parsePortableBundleName(name);
if (parsed.ok) return parsed;
const message = `Task ${JSON.stringify(name)} must be ${PORTABLE_BUNDLE_NAME_CONSTRAINT}.`;
return err(
new AnalysisInputError(
MEASURE,
{
userMessage: message,
remediationAction:
"Select the exact portable task name written under the bundle's tasks directory.",
},
[{ path: ["task"], reason: "invalid_value", message }],
),
);
};
/** The records a difficulty file holds, each read only as far as the README names it. */
const recordsSchema = z.looseObject({
tasks: z.record(
@@ -271,13 +293,16 @@ export const measureBuildTask = async (
progress: ProgressReporter = silentProgressReporter,
resources: ArtifactResourceScope = new ArtifactResourceScope(),
): Promise<Result<MeasuredTask, AnalysisError>> => {
const taskName = admitTaskName(request.task);
if (!taskName.ok) return taskName;
const task = taskName.value;
const solves: SolveRecord[] = [];
const retainSolves = (failure: AnalysisError): AnalysisError => {
if (solves.length > 0)
failure.retainPartialObservation({
kind: "partial-task-measurement",
bundle: request.bundle,
task: request.task,
task: task,
solves: [...solves],
});
return failure;
@@ -285,7 +310,7 @@ export const measureBuildTask = async (
let agentenv: string | null = null;
let stopped: StopReason = "decided";
try {
const digest = await taskDigest(request.bundle, request.task, signal);
const digest = await taskDigest(request.bundle, task, signal);
if (!digest.ok) return digest;
// Refuse records REA could not keep before any solve is paid for.
const records = await readRecords(request.bundle, signal);
@@ -301,15 +326,15 @@ export const measureBuildTask = async (
phase: MEASURE,
completed: solves.length,
total: null,
message: `solve ${String(solves.length + 1)} of ${request.task} with ${request.model}`,
message: `solve ${String(solves.length + 1)} of ${task} with ${request.model}`,
})
.catch(() => undefined);
const report = await solver(request, signal);
const report = await solver({ ...request, task }, signal);
if (!("outcome" in report))
return err(
retainSolves(
refused(
`A solve of ${request.task} did not run: ${report.error}`,
`A solve of ${task} did not run: ${report.error}`,
"Fix what the error names, such as the model endpoint agentenv needs (LITELLM_BASE_URL and LITELLM_API_KEY where REA starts, or [model] in .agentenv/config.toml), and measure again.",
),
),
@@ -329,7 +354,7 @@ export const measureBuildTask = async (
});
const file = await recordMeasure(
request.bundle,
request.task,
task,
record,
signal,
resources,
@@ -338,7 +363,7 @@ export const measureBuildTask = async (
const { versions: _versions, ...reported } = record;
return ok({
bundle: request.bundle,
task: request.task,
task: task,
...reported,
difficulty_file: file.value,
});
@@ -374,7 +399,7 @@ export const measureBuildTask = async (
cleanup,
})
: new ProviderAdapterError(AGENTENV_PROVIDER, MEASURE, {
userMessage: `Measuring ${request.task} could not close an input file.`,
userMessage: `Measuring ${task} could not close an input file.`,
cause,
cleanup,
});
@@ -387,7 +412,7 @@ export const measureBuildTask = async (
: signal.aborted
? new AnalysisCancelledError(MEASURE, { cause: signal.reason })
: new ProviderAdapterError(AGENTENV_PROVIDER, MEASURE, {
userMessage: `Measuring ${request.task} failed after ${String(solves.length)} solve(s): ${errorText(cause)}`,
userMessage: `Measuring ${task} failed after ${String(solves.length)} solve(s): ${errorText(cause)}`,
remediationAction:
"Check the bundle files and host I/O error, then retry or retain the completed solve results.",
cause,
@@ -408,13 +433,15 @@ export const measureInFreshComputers = async (
signal: AbortSignal,
progress?: ProgressReporter,
): Promise<Result<MeasuredTask, AnalysisError>> => {
const taskName = admitTaskName(request.task);
if (!taskName.ok) return taskName;
const environment = await resolveBuildComputerEnvironment(
host.environment,
process.platform,
host.defaults,
);
return measureBuildTask(
request,
{ ...request, task: taskName.value },
(solve, turn) =>
runSolveHelper({
...solve,
@@ -1,4 +1,4 @@
import { mkdir, stat, writeFile } from "node:fs/promises";
import { access, mkdir, stat, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
@@ -14,7 +14,6 @@ import {
renderTaskVerifier,
taskVerifierArtifact,
} from "../../domain/desktop/appTaskPython.js";
import { isPortableBundleName } from "../environment/environmentBundle.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { BuildComputerActionLog } from "./DesktopBuildActionLog.js";
import {
@@ -52,6 +51,77 @@ const witnessAt = async (bundle: string) =>
);
describe("bundle task reads", () => {
it("keeps the selected record directory through failed-save retries", async () => {
const directory = await createTestTempDirectory("witness-original-");
const other = await createTestTempDirectory("witness-other-");
let endAttempts = 0;
const witness = new DesktopBuildWitness(
{
...computer,
appData: async () => [
{
kind: "data",
data: {
state: {
"Documents/today.txt": {
format: "text",
view: { lines: ["eggs"], line_count: 1, truncated: false },
error: null,
},
},
},
},
],
startRecording: async () => ({
recording_id: "rec",
started_at: "2026-11-10T00:00:00.000Z",
}),
endRecording: async () => {
if (++endAttempts === 1) throw new Error("recording busy");
return { recording_id: "rec", segments: [] };
},
recordingSegments: async () => [],
},
new BuildComputerActionLog(),
{ ...SETTLE_TIMING, windowMs: 0 },
);
const signal = new AbortController().signal;
const first = await witness.check(
{ task, stage: "seed", directory },
signal,
);
expect(first.ok).toBe(false);
if (first.ok) throw new Error("Expected recording failure");
expect(first.error.userMessage).toContain("recording busy");
const refused = await witness.check(
{ task, stage: "seed", directory: other },
signal,
);
expect(refused.ok).toBe(false);
if (refused.ok) throw new Error("Expected directory refusal");
expect(refused.error.userMessage).toContain(directory);
expect(endAttempts).toBe(1);
await expect(access(join(directory, "witness.json"))).rejects.toThrow();
await expect(access(join(other, "witness.json"))).rejects.toThrow();
const saved = await witness.check(
{ task, stage: "seed", directory },
signal,
);
expect(saved.ok).toBe(true);
if (!saved.ok) throw saved.error;
expect(saved.value.witness?.record?.path).toBe(
join(directory, "witness.json"),
);
const savedAgain = await witness.check(
{ task, stage: "seed", directory: other },
signal,
);
expect(savedAgain.ok).toBe(false);
expect(endAttempts).toBe(2);
});
it("returns a cancelled result when reading the bundle's task is aborted", async () => {
const bundle = await createTestTempDirectory("witness-bundle-");
const directory = await createTestTempDirectory("witness-record-");
@@ -87,7 +157,7 @@ describe("bundle task reads", () => {
it("reads a task whose prompt is larger than the old 1 MiB file cap", async () => {
const prompt = `Add ${"eggs ".repeat(Math.ceil((1.1 * 1024 * 1024) / 5))}end`;
const given = { ...task, prompt };
const parsed = parseDesktopTask(given, isPortableBundleName);
const parsed = parseDesktopTask(given);
expect(parsed.ok).toBe(true);
if (!parsed.ok) return;
const large = parsed.value;
+27 -11
View File
@@ -35,7 +35,6 @@ import type {
RecordingManifest,
} from "../../desktop/AgentEnvComputer.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import { isPortableBundleName } from "../environment/environmentBundle.js";
import {
silentProgressReporter,
type ProgressReporter,
@@ -104,6 +103,18 @@ const refused = (userMessage: string, remediationAction: string) =>
const taskDigest = (task: DesktopTask) =>
createHash("sha256").update(JSON.stringify(task)).digest("hex");
/** A retry cannot redirect an ended witness to another folder. */
const recordDirectoryRefusal = (
expected: string,
requested: string | undefined,
): AnalysisError | undefined =>
requested !== undefined && requested !== expected
? refused(
`This witness's record belongs to ${expected}, not ${requested}.`,
"Ask again without directory or with the original folder; another witness needs a fresh computer.",
)
: undefined;
/** Whether a folder can take the record: absent, or empty. */
const usableFolder = async (directory: string): Promise<boolean> =>
readdir(directory).then(
@@ -228,7 +239,7 @@ export class DesktopBuildWitness {
signal: AbortSignal,
progress: ProgressReporter = silentProgressReporter,
): Promise<Result<CheckedTask, AnalysisError>> {
const parsed = parseDesktopTask(input.task, isPortableBundleName);
const parsed = parseDesktopTask(input.task);
if (!parsed.ok)
return err(
refused(
@@ -238,8 +249,15 @@ export class DesktopBuildWitness {
);
const task = parsed.value;
const { stage } = input;
if (stage !== undefined && this.#witness?.state === "saving")
return this.#resave(this.#witness, task, stage, signal);
if (stage !== undefined && this.#witness?.state === "saving") {
const directoryRefusal = recordDirectoryRefusal(
this.#witness.directory,
input.directory,
);
return directoryRefusal === undefined
? this.#resave(this.#witness, task, stage, signal)
: err(directoryRefusal);
}
const saved =
stage === undefined
? undefined
@@ -439,13 +457,11 @@ export class DesktopBuildWitness {
)
return undefined;
// A folder the record was not saved to gets nothing written, so it is refused.
if (directory !== undefined && directory !== witness.directory)
return err(
refused(
`This witness's record was saved to ${witness.directory}, not ${directory}.`,
"Ask again without directory or with the folder it was saved to; another witness needs a fresh computer.",
),
);
const directoryRefusal = recordDirectoryRefusal(
witness.directory,
directory,
);
if (directoryRefusal !== undefined) return err(directoryRefusal);
return ok({
...counted(task, filed),
witness: summaryOf(filed, null, witness.record),
@@ -21,20 +21,17 @@ const manifest: RecordingManifest = {
};
const witness = async (): Promise<EndedWitness> => {
const parsed = parseDesktopTask(
{
name: "add-eggs",
prompt: "Add eggs.",
checks: [
{
kind: "lines_present",
file: "Documents/today.txt",
lines: ["eggs"],
},
],
},
(name) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name),
);
const parsed = parseDesktopTask({
name: "add-eggs",
prompt: "Add eggs.",
checks: [
{
kind: "lines_present",
file: "Documents/today.txt",
lines: ["eggs"],
},
],
});
if (!parsed.ok) throw new Error(parsed.error);
return {
task: parsed.value,
@@ -19,7 +19,7 @@ import {
withExtraPackages,
} from "../../domain/desktop/appExportInstall.js";
import type { DesktopApplicationCapture } from "../../domain/desktop/desktopCapture.js";
import { isPortableBundleName } from "../environment/environmentBundle.js";
import { isPortableBundleName } from "../../domain/portableBundleName.js";
const OPERATION = "export_desktop_application";
const MAX_DOWNLOAD_INSTALL_BYTES = 64 * 1024;
@@ -189,3 +189,37 @@ it("does not start a second cleanup while a timed-out cleanup may still be runni
expect(calls).toBe(1);
expect(owner.cleanupResources).toHaveLength(2);
});
it.each([
{ ...success, exit_code: 4, stderr: "reader failed" },
{ ...success, stdout: "invalid reader answer" },
])(
"preserves earlier views and absences when a later runner fails after settling (%#)",
async (failed) => {
let readers = 0;
const owner = new DesktopRemoteTaskReaders(async (argv) => {
if (argv[3] === PYTHON_READER_CLEANUP) return success;
readers += 1;
if (readers === 1) return success;
if (readers === 2)
return { ...success, stdout: JSON.stringify({ absent: true }) };
return failed;
});
const failure = await owner
.read(
[reader, { ...reader, id: "absent" }, { ...reader, id: "failed" }],
new AbortController().signal,
)
.catch((error: unknown) => error);
expect(failure).toMatchObject({
operation: "check_build_task",
diagnostics: {
completed_reader_views: {
note: { format: "json", view: { value: 7 } },
},
completed_reader_absent: ["absent"],
},
});
expect(owner.cleanupResources).toEqual([]);
},
);
@@ -14,6 +14,7 @@ import {
type ReaderLease,
} from "../../domain/desktop/appTaskReaders.js";
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import {
AGENTENV_PROVIDER,
type ComputerRelease,
@@ -137,7 +138,12 @@ export class DesktopRemoteTaskReaders {
signal.aborted ? this.#cancelled(signal, views, absent) : undefined,
);
if (signal.aborted) throw this.#cancelled(signal, views, absent);
if (failure !== undefined) throw failure;
if (failure !== undefined)
throw new ProviderAdapterError(AGENTENV_PROVIDER, "check_build_task", {
cause: failure,
userMessage: `The task reader could not complete: ${failure instanceof Error ? failure.message : String(failure)}`,
diagnostics: this.#facts(views, absent),
});
}
return views;
}
+1 -5
View File
@@ -45,7 +45,6 @@ import { EvidenceFileError } from "../../domain/evidenceErrors.js";
import { err } from "../../domain/result.js";
import { describeValidationFailure } from "../../domain/validationFailure.js";
import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js";
import { isPortableBundleName } from "../environment/environmentBundle.js";
import { readJsonFile } from "../JsonFiles.js";
import { desktopOperationFailure } from "./DesktopCaptureRefusals.js";
import { refuse } from "./DesktopExportInstall.js";
@@ -663,10 +662,7 @@ export const admitDesktopTasks = async (
): Promise<AdmittedTasks | undefined> => {
const tasks: DesktopTask[] = [];
for (const path of given.paths) {
const parsed = parseDesktopTask(
await readTaskFile(path, signal),
isPortableBundleName,
);
const parsed = parseDesktopTask(await readTaskFile(path, signal));
if (!parsed.ok)
throw refuse(
`The task file ${path} is refused: ${parsed.error}`,
@@ -166,3 +166,34 @@ it.each([
expect((failure as Error).cause).toBe(reason);
},
);
it("preserves completed views and source absences when a later source read fails", async () => {
const sourceFailure = new Error("cannot read the third retained file");
const failure = await readerViewsOfBytes(
["first", "absent", "third"].map((id) => ({
id,
file: `${id}.txt`,
argv: ["reader", "{file}"],
output: "json" as const,
})),
async ({ id }) => {
if (id === "third") throw sourceFailure;
return id === "absent"
? { kind: "absent" }
: { kind: "bytes", bytes: Buffer.from("input") };
},
async () => ({
exit_code: 0,
stdout: JSON.stringify({ view: { format: "json", view: { value: 7 } } }),
stderr: "",
timed_out: false,
}),
).catch((cause: unknown) => cause);
expect(failure).toMatchObject({
cause: sourceFailure,
diagnostics: {
completed_reader_views: { first: { format: "json", view: { value: 7 } } },
completed_reader_absent: ["absent"],
},
});
});
+15 -3
View File
@@ -88,13 +88,25 @@ export const readerViewsOfBytes = async (
if (readers.length > 0) await processWorkerCleanup.retry();
const absent: string[] = [];
for (const reader of readers) {
const file = await bytesOf(reader);
if (file.kind === "absent") continue;
let file: ReaderFileBytes;
try {
file = await bytesOf(reader);
} catch (cause: unknown) {
throw readerFailure(cause, { views, absent });
}
if (file.kind === "absent") {
absent.push(reader.id);
continue;
}
if (file.kind === "error") {
views.set(reader.id, { format: null, view: null, error: file.error });
continue;
}
const folder = await mkdtemp(join(tmpdir(), "rea-reader-bytes-"));
const folder = await mkdtemp(join(tmpdir(), "rea-reader-bytes-")).catch(
(cause: unknown) => {
throw readerFailure(cause, { views, absent });
},
);
let failure: { readonly cause: unknown } | undefined;
try {
const path = join(folder, reader.file.split("/").at(-1) ?? "file");
@@ -1,3 +1,7 @@
import {
isPortableBundleName,
PORTABLE_BUNDLE_NAME_CONSTRAINT,
} from "../../domain/portableBundleName.js";
import { setImmediate } from "node:timers/promises";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
@@ -15,9 +19,7 @@ import type { EnvironmentBuildReport } from "../../domain/environmentBundleExpor
import { err, ok, type Result } from "../../domain/result.js";
import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js";
import {
BUNDLE_NAME_CONSTRAINT,
ENVIRONMENT_EXPORT_OPERATION,
isPortableBundleName,
planEnvironmentBundle,
registryPackageSource,
type EnvironmentPackageSource,
@@ -190,7 +192,7 @@ const unnamed = (path: string): AnalysisInputError =>
new AnalysisInputError(
ENVIRONMENT_BUILD_OPERATION,
{
userMessage: `The capture's origin is a loopback or IP address, and ${path === "" ? "retained Evidence has no file name to take one from" : `${path} yields no portable environment name (${BUNDLE_NAME_CONSTRAINT})`}; name the environment.`,
userMessage: `The capture's origin is a loopback or IP address, and ${path === "" ? "retained Evidence has no file name to take one from" : `${path} yields no portable environment name (${PORTABLE_BUNDLE_NAME_CONSTRAINT})`}; name the environment.`,
},
[
{
@@ -1,3 +1,4 @@
import { isPortableBundleName } from "../../domain/portableBundleName.js";
import { describe, expect, it } from "vitest";
import { timedCaptureEvidence } from "../../../tests/fixtures/analysisView.js";
@@ -15,7 +16,6 @@ import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js";
import {
BUNDLE_BASE_IMAGE,
bundleDockerfile,
isPortableBundleName,
ENVIRONMENT_EXPORT_OPERATION,
planEnvironmentBundle,
registryPackageSource,
@@ -1,4 +1,8 @@
import { normalizeHttpHostname } from "../../domain/httpHostname.js";
import {
parsePortableBundleName,
PORTABLE_BUNDLE_NAME_PATTERN,
PORTABLE_BUNDLE_NAME_MAX_LENGTH,
} from "../../domain/portableBundleName.js";
import { PRODUCT_IDENTITY } from "../../identity.js";
import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
@@ -27,18 +31,6 @@ export const BUNDLE_ALLOWED_HOSTS = "127.0.0.1,localhost";
/** Name of the packed CLI inside the env folder when a tarball is exported. */
export const BUNDLE_TARBALL_NAME = "rea-agents.tgz";
/** Folder and env id rule: a kebab-case label the agentenv framework accepts as a local id. */
const BUNDLE_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
// A compose service name is one DNS label, whose wire limit is 63 octets.
const BUNDLE_NAME_MAX_LENGTH = 63;
export const BUNDLE_NAME_CONSTRAINT = `a kebab-case label of at most ${String(BUNDLE_NAME_MAX_LENGTH)} characters, excluding Windows device names, that remains unchanged as an HTTP service hostname`;
/** Win32 reserves these device names for files and folders, so the env folder can never take one. */
const WINDOWS_RESERVED_NAME = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])$/u;
/** Whether a name can be the env folder and env id: the checks the export applies. */
export const isPortableBundleName = (name: string): boolean =>
validateBundleName(name).ok;
/** Where the Dockerfile installs the CLI from. */
export type EnvironmentPackageSource =
| { readonly kind: "registry"; readonly specifier: string }
@@ -95,13 +87,13 @@ const nameError = (name: string): AnalysisInputError =>
new AnalysisInputError(
ENVIRONMENT_EXPORT_OPERATION,
{
userMessage: `The bundle name ${JSON.stringify(name)} is not a portable kebab-case label of at most ${String(BUNDLE_NAME_MAX_LENGTH)} characters.`,
userMessage: `The bundle name ${JSON.stringify(name)} is not a portable kebab-case label of at most ${String(PORTABLE_BUNDLE_NAME_MAX_LENGTH)} characters.`,
},
[
{
path: ["name"],
reason: "invalid_format",
expected: BUNDLE_NAME.source,
expected: PORTABLE_BUNDLE_NAME_PATTERN.source,
message:
"The bundle name is the env folder and env id; use lowercase letters, digits and single hyphens, excluding Windows device names.",
},
@@ -128,27 +120,26 @@ const reservedNameError = (name: string): AnalysisInputError =>
const validateBundleName = (
name: string,
): Result<string, AnalysisInputError> => {
if (!BUNDLE_NAME.test(name) || name.length > BUNDLE_NAME_MAX_LENGTH)
return err(nameError(name));
if (WINDOWS_RESERVED_NAME.test(name)) return err(reservedNameError(name));
if (normalizeHttpHostname(name) !== name)
return err(
new AnalysisInputError(
ENVIRONMENT_EXPORT_OPERATION,
const parsed = parsePortableBundleName(name);
if (parsed.ok) return parsed;
if (parsed.error === "format") return err(nameError(name));
if (parsed.error === "reserved") return err(reservedNameError(name));
return err(
new AnalysisInputError(
ENVIRONMENT_EXPORT_OPERATION,
{
userMessage: `The bundle name ${JSON.stringify(name)} cannot be used unchanged as an HTTP host name. Choose a name such as replay-1.`,
},
[
{
userMessage: `The bundle name ${JSON.stringify(name)} cannot be used unchanged as an HTTP host name. Choose a name such as replay-1.`,
path: ["name"],
reason: "invalid_value",
message:
"The service name must remain a hostname rather than be rejected or interpreted as an IP address.",
},
[
{
path: ["name"],
reason: "invalid_value",
message:
"The service name must remain a hostname rather than be rejected or interpreted as an IP address.",
},
],
),
);
return ok(name);
],
),
);
};
const installLine = (source: EnvironmentPackageSource): string =>
+4 -4
View File
@@ -1,3 +1,4 @@
import { portableBundleNameSchema } from "../domain/portableBundleName.js";
import { z } from "zod";
import { absoluteLocalPathStringSchema } from "../domain/localPath.js";
@@ -42,10 +43,9 @@ export const MEASURE_BUILD_TASK_CONTRACT = {
bundle: absoluteLocalPathStringSchema.describe(
"The exported bundle's folder on this host.",
),
task: z
.string()
.min(1)
.describe("The task's name, as its tasks/<name>.json names it."),
task: portableBundleNameSchema.describe(
"The task's name, as its tasks/<name>.json names it.",
),
model: z
.string()
.min(1)
@@ -1,6 +1,7 @@
import { execFile } from "node:child_process";
import { constants } from "node:fs";
import {
chmod,
mkdir,
open,
readFile,
@@ -67,6 +68,75 @@ const readFacts = async (
}
};
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
"keeps disk and archive permission failures distinct from missing paths",
async () => {
const root = await createTestTempDirectory("rea-doctor-disk-permission-");
const docker = join(root, "docker");
const denied = join(root, "denied");
const target = join(denied, "docker");
const archive = join(denied, "runtime.tar.xz");
await program(docker, "exit 0");
await mkdir(target, { recursive: true });
await writeFile(archive, "unreadable archive");
const resources = new ArtifactResourceScope();
const factsAt = (directory: string, selectedArchive: string) =>
readDesktopBuildFacts({
environment: {
HOME: root,
PATH: "",
REA_DOCKER_COMMAND: docker,
REA_BUILD_COMPUTER_REA_ARCHIVE: selectedArchive,
},
platform: process.platform,
resources,
signal: new AbortController().signal,
run: async (_command, args) => {
const stdout = args[0] === "info" ? `${directory}\n` : "";
return {
stdout,
stderr: "",
stdoutBytes: Buffer.byteLength(stdout),
stderrBytes: 0,
};
},
clientIds: [],
registrations: async () => [],
});
await chmod(denied, 0);
try {
const facts = await factsAt(target, archive);
expect(facts.disk).toMatchObject({
path: target,
error: expect.stringContaining("EACCES"),
});
expect(facts.disk).not.toHaveProperty("freeBytes");
expect(facts.archive).toMatchObject({
path: archive,
status: "unknown",
detail: expect.stringContaining("EACCES"),
});
const missingArchive = join(root, "missing", "runtime.tar.xz");
const missing = await factsAt(
join(root, "missing", "docker"),
missingArchive,
);
expect(missing.disk).toMatchObject({
path: root,
freeBytes: expect.any(Number),
});
expect(missing.disk).not.toHaveProperty("error");
expect(missing.archive).toMatchObject({
path: missingArchive,
status: "missing",
});
} finally {
await chmod(denied, 0o700);
await resources.close();
}
},
);
it.skipIf(process.platform === "win32")(
"refuses a FIFO model configuration without waiting for a writer",
async () => {
+18 -7
View File
@@ -1,6 +1,6 @@
import { constants } from "node:fs";
import { access, stat, statfs } from "node:fs/promises";
import { dirname, isAbsolute, join } from "node:path";
import { dirname, isAbsolute } from "node:path";
import { z } from "zod";
@@ -31,6 +31,9 @@ import {
const DEFAULT_DOCKER_ROOT = "/var/lib/docker";
const CONFIG_BYTES = 1024 * 1024;
const missingPath = (cause: unknown): boolean =>
cause instanceof Error && "code" in cause && cause.code === "ENOENT";
/** Inspect the same tar compression formats as the build's Python extractor without extracting files. */
const ARCHIVE_VERSION = String.raw`
import json, pathlib, sys, tarfile
@@ -235,7 +238,7 @@ const freeSpace = async (path: string): Promise<DesktopBuildFacts["disk"]> => {
return { path: candidate, freeBytes: fs.bavail * fs.bsize };
} catch (cause: unknown) {
const parent = dirname(candidate);
if (parent === candidate)
if (parent === candidate || !missingPath(cause))
return {
path,
error: cause instanceof Error ? cause.message : String(cause),
@@ -265,11 +268,19 @@ const archiveFact = async (
source === "cache"
? defaultBuildComputerArchive(input.environment, input.platform)
: (selected as string);
const file = await stat(path).then(
(entry) => entry.isFile(),
() => false,
);
if (!file) return { path, source, status: "missing" };
try {
if (!(await stat(path)).isFile())
return { path, source, status: "missing" };
} catch (cause: unknown) {
return missingPath(cause)
? { path, source, status: "missing" }
: {
path,
source,
status: "unknown",
detail: `REA could not inspect ${path}: ${cause instanceof Error ? cause.message : String(cause)}`,
};
}
if (source === "cache") return { path, source, status: "current" };
const python = await pythonForArchive();
if (python === undefined)
-2
View File
@@ -1,7 +1,6 @@
import type { PartialTaskMeasurement } from "./desktop/taskDifficulty.js";
import type { AppleIpsPartialObservation } from "./native/appleIps.js";
import type { PlaywrightTrace } from "./playwrightTrace/playwrightTrace.js";
import type { ApiContract } from "./apiContract/apiContract.js";
import type { GraphqlContract } from "./apiContract/graphqlContract.js";
import type { SavedAnalysisSnapshotPartialObservation } from "./savedAnalysisSnapshot.js";
import type { AnalysisSettingLimit } from "./analysisSettings.js";
@@ -499,7 +498,6 @@ export type AnalysisPartialObservation =
}
| { readonly kind: "unretained-evidence"; readonly evidence: Evidence }
| { readonly kind: "retained-evidence"; readonly evidence_id: string }
| ApiContract
| GraphqlContract
| PlaywrightTrace
| SavedAnalysisSnapshotPartialObservation
+1 -1
View File
@@ -185,7 +185,7 @@ export const projectApiContractView = (
unknowns:
context.source_coverage.status === "partial"
? [
"This inventory is partial. Inspect limitations for coverage and query path-item-references and diagnostics for any retained source pointers.",
"Unresolved or malformed declarations limit this inventory; query path-item-references and diagnostics for exact source pointers.",
]
: [],
};
@@ -3,7 +3,6 @@ import { readFileSync } from "node:fs";
import { expect, it } from "vitest";
import { apiContractSchema, type ApiContractArtifact } from "./apiContract.js";
import { parseApiContractSteps } from "./parseApiContract.js";
import { AnalysisCancelledError } from "../analysisErrorCore.js";
const artifact: ApiContractArtifact = {
path: "/contract.json",
@@ -306,39 +305,9 @@ it("handles 3.2 QUERY and case-sensitive additional methods, diagnosing invalid
it("bounds expanded identities and source pointers before building a giant row", () => {
const result = parse(
document({
"/retained": { get: {} },
[`/${"a".repeat(1_100_000)}`]: { get: {} },
}),
document({ [`/${"a".repeat(1_100_000)}`]: { get: {} } }),
);
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected a retained-facts refusal");
expect(result.error._tag).toBe("AnalysisResourceConstraintError");
const partial = apiContractSchema.parse(result.error.partialObservation);
expect(partial.operations.map((row) => row.path)).toEqual(["/retained"]);
expect(partial.source_coverage.status).toBe("partial");
});
it("retains an interpreted path when cancellation interrupts traversal", () => {
const input = document({
"/first": { get: { operationId: "first" } },
"/second": { get: { operationId: "second" } },
});
const steps = parseApiContractSteps(input, artifact);
for (let checkpoint = 0; checkpoint < 3; checkpoint++)
expect(steps.next().done).toBe(false);
const cancellation = new AnalysisCancelledError("inspect_api_contract");
const stopped = steps.throw(cancellation);
if (!stopped.done || stopped.value.ok)
throw new Error("Expected cancellation");
expect(stopped.value.error).toBe(cancellation);
const partial = apiContractSchema.parse(cancellation.partialObservation);
const complete = report(input);
expect(partial.operations).toEqual(complete.operations.slice(0, 1));
expect(partial.summary.operation_slots).toBe(1);
expect(partial.source_coverage).toMatchObject({
status: "partial",
path_entries_examined: 1,
});
expect(complete.source_coverage.status).toBe("complete-within-scope");
if (!result.ok)
expect(result.error._tag).toBe("AnalysisResourceConstraintError");
});
+39 -72
View File
@@ -1,5 +1,4 @@
import {
AnalysisCancelledError,
AnalysisInputError,
AnalysisResourceConstraintError,
AnalysisUnsupportedTargetError,
@@ -317,17 +316,14 @@ function* pathItem(
return ok(undefined);
}
interface ContractReportContext {
readonly openapi: string;
readonly info: ApiContract["info"];
readonly pathsState: ApiContract["source_coverage"]["paths_state"];
examined: number;
}
const contractReport = (
const completedInventory = (
inventory: ContractInventory,
context: ContractReportContext,
incomplete = false,
context: {
openapi: string;
info: ApiContract["info"];
pathsState: ApiContract["source_coverage"]["paths_state"];
examined: number;
},
): ApiContract => {
const artifact = inventory.artifact;
let declared = 0;
@@ -352,9 +348,7 @@ const contractReport = (
source_coverage: {
scope: "inline-path-operations",
status:
incomplete ||
inventory.references.length > 0 ||
inventory.diagnostics.length > 0
inventory.references.length > 0 || inventory.diagnostics.length > 0
? "partial"
: "complete-within-scope",
paths_state: context.pathsState,
@@ -364,11 +358,6 @@ const contractReport = (
duplicate_json_members: "last-value-wins",
},
limitations: [
...(incomplete
? [
"Traversal stopped before the inventory was complete. Counts describe the retained rows.",
]
: []),
"Operations are document declarations, not evidence of runtime implementation, reachability or authorization.",
"Only inline operations under paths are inventoried; callbacks, webhooks, components, servers, parameters, schemas, request/response contracts and security are not analyzed.",
"No reference is resolved, including local fragments. A Path Item reference makes its effective operations unknown; adjacent inline slots are preserved without merging referenced definitions.",
@@ -379,32 +368,6 @@ const contractReport = (
};
};
function* collectPaths(
sourcePaths: Record<string, unknown>,
inventory: ContractInventory,
context: ContractReportContext,
modern: boolean,
): Generator<void, Result<void, AnalysisError>> {
for (const path in sourcePaths) {
if (!Object.hasOwn(sourcePaths, path) || path.startsWith("x-")) continue;
yield;
context.examined++;
// Reserve expanded pointer/path strings before escaping or building a row.
const admitted = inventory.reserve(512 + path.length * 16);
if (!admitted.ok) return admitted;
const pointer = `/paths/${pointerPart(path)}`;
const read = path.startsWith("/")
? yield* pathItem(sourcePaths[path], { inventory, path, pointer, modern })
: inventory.diagnostic(
"invalid-path-key",
pointer,
"A Paths entry must start with /; it was not interpreted as an endpoint.",
);
if (!read.ok) return read;
}
return ok(undefined);
}
/** Enumerate declarations only; never resolve references, servers, callbacks or runtime expressions. */
export function* parseApiContractSteps(
value: unknown,
@@ -454,32 +417,36 @@ export function* parseApiContractSteps(
);
if (!diagnosed.ok) return diagnosed;
}
const context: ContractReportContext = {
openapi: root.openapi,
info: info.value,
pathsState,
examined: 0,
};
const interrupted = (error: AnalysisError) => {
const retainedRows =
inventory.operations.length +
inventory.references.length +
inventory.diagnostics.length;
if (retainedRows > 0)
error.retainPartialObservation(contractReport(inventory, context, true));
return err(error);
};
try {
const read = yield* collectPaths(
paths ?? {},
inventory,
context,
version[1] === "2",
);
if (!read.ok) return interrupted(read.error);
} catch (cause: unknown) {
if (cause instanceof AnalysisCancelledError) return interrupted(cause);
throw cause;
let examined = 0;
const sourcePaths = paths ?? {};
for (const path in sourcePaths) {
if (!Object.hasOwn(sourcePaths, path) || path.startsWith("x-")) continue;
yield;
examined++;
// Reserve expanded pointer/path strings before escaping or building a row.
const admitted = inventory.reserve(512 + path.length * 16);
if (!admitted.ok) return admitted;
const pointer = `/paths/${pointerPart(path)}`;
const read = path.startsWith("/")
? yield* pathItem(sourcePaths[path], {
inventory,
path,
pointer,
modern: version[1] === "2",
})
: inventory.diagnostic(
"invalid-path-key",
pointer,
"A Paths entry must start with /; it was not interpreted as an endpoint.",
);
if (!read.ok) return read;
}
return ok(contractReport(inventory, context));
return ok(
completedInventory(inventory, {
openapi: root.openapi,
info: info.value,
pathsState,
examined,
}),
);
}
+2 -2
View File
@@ -1,3 +1,4 @@
import { isPortableBundleName } from "../portableBundleName.js";
import { z } from "zod";
import { jsonValueSchema, type JsonValue } from "../jsonValue.js";
@@ -403,12 +404,11 @@ const screenProblem = (
/** Parse a task file and settle its check ids, or say what is wrong with it. */
export const parseDesktopTask = (
value: unknown,
isPortableName: (name: string) => boolean,
): Result<DesktopTask, string> => {
const parsed = desktopTaskSchema.safeParse(value);
if (!parsed.success) return err(describeValidationFailure(parsed.error));
const { name, prompt, checks, readers } = parsed.data;
if (!isPortableName(name) || name === "smoke")
if (!isPortableBundleName(name) || name === "smoke")
return err(
`name ${JSON.stringify(name)} must be a portable bundle name other than smoke: lowercase letters, digits and hyphens.`,
);
+20 -31
View File
@@ -13,8 +13,6 @@ import { elementText } from "./viewPointer.js";
import { redactSecretsInJson } from "./buildSecrets.js";
const portable = (name: string) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name);
it("redacts full check values before shortening descriptions without changing the verdict", () => {
const secret = "e-Plum7-Kite-Throwaway-42";
const actual = `${"x".repeat(180)}${secret}`;
@@ -53,10 +51,7 @@ const expectOutcome = (
};
const task = (checks: readonly unknown[]) =>
parseDesktopTask(
{ name: "edit-notes", prompt: "Edit my notes.", checks },
portable,
);
parseDesktopTask({ name: "edit-notes", prompt: "Edit my notes.", checks });
/** Task changes the parser refuses, each with part of its message. */
const REFUSALS: readonly (readonly [Record<string, unknown>, string])[] = [
@@ -1071,22 +1066,19 @@ describe("evaluateTaskChecks whole text", () => {
describe("parseDesktopTask", () => {
it("settles ids and keeps the prompt trimmed", () => {
const parsed = parseDesktopTask(
{
name: "add-milk",
prompt: " Add milk.\n",
checks: [
{ kind: "file_present", file: "Documents/todo.txt" },
{
id: "milk",
kind: "lines_present",
file: "Documents/todo.txt",
lines: ["milk"],
},
],
},
portable,
);
const parsed = parseDesktopTask({
name: "add-milk",
prompt: " Add milk.\n",
checks: [
{ kind: "file_present", file: "Documents/todo.txt" },
{
id: "milk",
kind: "lines_present",
file: "Documents/todo.txt",
lines: ["milk"],
},
],
});
expect(parsed).toMatchObject({
ok: true,
@@ -1098,15 +1090,12 @@ describe("parseDesktopTask", () => {
});
it.each(REFUSALS)("refuses %j", (change, message) => {
const parsed = parseDesktopTask(
{
name: "add-milk",
prompt: "Add milk.",
checks: [{ kind: "file_present", file: "a" }],
...change,
},
portable,
);
const parsed = parseDesktopTask({
name: "add-milk",
prompt: "Add milk.",
checks: [{ kind: "file_present", file: "a" }],
...change,
});
expect(parsed.ok).toBe(false);
if (!parsed.ok) expect(parsed.error).toContain(message);
+30 -7
View File
@@ -3,6 +3,7 @@ import { isNameLikeKey } from "./attributeRank.js";
import type { DesktopStateDifference } from "./stateChanges.js";
import { elementText, isXmlElement, resolvePointer } from "./viewPointer.js";
import { selectorOf } from "./viewSelector.js";
import { zipEntryAt } from "./zipView.js";
/** A file's views at its two ends. */
interface Ends {
@@ -10,6 +11,12 @@ interface Ends {
readonly final: JsonValue | null;
}
/** XML identifiers and field mirrors belong to one document, including within an archive. */
const documentAt = (view: JsonValue | null, pointer: string) => {
const entry = zipEntryAt(view, pointer);
return typeof entry === "object" ? entry : view;
};
/** The value of the name-like attribute a pointer's last step names its element by, as `label` for `[name=elementInformation][@name=label]`. */
const fieldKeyOf = (element: string) =>
selectorOf(
@@ -38,22 +45,35 @@ export const mirrorsOf = (
return element !== pointer &&
typeof before === "string" &&
typeof after === "string"
? [{ pointer, element, before, after }]
? [
{
pointer,
element,
before,
after,
document: documentAt(ends.final, pointer),
},
]
: [];
});
// The keys of the changed texts' elements, by what each was and became.
const keys = new Map<string, { pointer: string; key: string }[]>();
for (const { pointer, element, before, after } of texts) {
const keys = new Map<
string,
{ pointer: string; key: string; document: ReturnType<typeof documentAt> }[]
>();
for (const { pointer, element, before, after, document } of texts) {
const key = fieldKeyOf(element);
const alike = JSON.stringify([before, after]);
if (key === undefined) continue;
const group = keys.get(alike) ?? [];
group.push({ pointer, key });
group.push({ pointer, key, document });
keys.set(alike, group);
}
const mirrored = new Set(
texts.flatMap((mirror) => {
const sources = keys.get(JSON.stringify([mirror.before, mirror.after]));
const sources = keys
.get(JSON.stringify([mirror.before, mirror.after]))
?.filter(({ document }) => document === mirror.document);
const record = sources?.some(({ pointer }) => pointer !== mirror.pointer)
? resolvePointer(
ends.final,
@@ -136,13 +156,16 @@ export const idReferencesOf = (
) => {
const added = change.added.flatMap((pointer) => {
const element = resolvePointer(ends.final, pointer, "xml");
return isXmlElement(element) ? [{ pointer, element }] : [];
return isXmlElement(element)
? [{ pointer, element, document: documentAt(ends.final, pointer) }]
: [];
});
const mirrors = added.flatMap(({ pointer, element }) => {
const mirrors = added.flatMap(({ pointer, element, document }) => {
const [key = ""] = Object.keys(element.attributes);
return isIdReference(element) &&
added.some(
(record) =>
record.document === document &&
record.element !== element &&
!isIdReference(record.element) &&
record.element.attributes[key] === element.attributes[key],
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import type { JsonValue } from "../jsonValue.js";
import { evaluateTaskChecks } from "./appTaskChecks.js";
import {
draftOf,
passing,
@@ -776,3 +777,109 @@ describe("draftTaskFiles when a sibling lock changed", () => {
expect(lock?.suggested.length ?? 0).toBeGreaterThan(0);
});
});
it("keeps an added ID-only member when another XML archive entry independently adds the same ID", () => {
const archive = (added: boolean) => ({
entries: [
{
name: "records.xml",
size: 1,
sha256: added ? "end" : "seed",
format: "xml",
view: xml(
added
? '<doc><record id="6"><title>New record</title></record></doc>'
: "<doc/>",
),
view_error: null,
},
{
name: "other.xml",
size: 1,
sha256: added ? "end" : "seed",
format: "xml",
view: xml(added ? '<doc><entry id="6"/></doc>' : "<doc/>"),
view_error: null,
},
],
entry_count: 2,
truncated: false,
});
const file = "Documents/project.zip";
const [drafted] = draftOf({
[file]: statePath("document", "zip", archive(false), archive(true)),
});
expect(
drafted?.values.find(({ pointer }) => pointer.includes("[name=other.xml]"))
?.note,
).toBeUndefined();
const checks = (drafted?.suggested ?? []).filter(
(check) => "pointer" in check && check.pointer.includes("[name=other.xml]"),
);
expect(
evaluateTaskChecks(
checks.map((check, index) => ({ ...check, id: `c${index}` })),
new Map([[file, { format: "zip", view: archive(false) }]]),
).map(({ passed }) => passed),
).toContain(false);
expect(
evaluateTaskChecks(
checks.map((check, index) => ({ ...check, id: `c${index}` })),
new Map([[file, { format: "zip", view: archive(true) }]]),
).every(({ passed }) => passed),
).toBe(true);
});
it("keeps changed display text when its matching label belongs to another XML archive entry", () => {
const archive = (text: string) => ({
entries: [
{
name: "source.xml",
size: 1,
sha256: text,
format: "xml",
view: xml(
`<doc><record><field name="function">Independent function</field><field name="label">${text}</field></record></doc>`,
),
view_error: null,
},
{
name: "display.xml",
size: 1,
sha256: text,
format: "xml",
view: xml(
`<doc><record><display><text>${text}</text><info_name>label</info_name></display></record></doc>`,
),
view_error: null,
},
],
entry_count: 2,
truncated: false,
});
const file = "Documents/project.zip";
const [drafted] = draftOf({
[file]: statePath("document", "zip", archive("Old"), archive("New")),
});
expect(
drafted?.values.find(({ pointer }) =>
pointer.includes("[name=display.xml]"),
)?.note,
).toBeUndefined();
const checks = (drafted?.suggested ?? []).filter(
(check) =>
"pointer" in check && check.pointer.includes("[name=display.xml]"),
);
expect(
evaluateTaskChecks(
checks.map((check, index) => ({ ...check, id: `c${index}` })),
new Map([[file, { format: "zip", view: archive("Old") }]]),
).map(({ passed }) => passed),
).toContain(false);
expect(
evaluateTaskChecks(
checks.map((check, index) => ({ ...check, id: `c${index}` })),
new Map([[file, { format: "zip", view: archive("New") }]]),
).every(({ passed }) => passed),
).toBe(true);
});
+148
View File
@@ -0,0 +1,148 @@
import { describe, expect, it } from "vitest";
import type { JsonValue } from "../jsonValue.js";
import { evaluateTaskChecks } from "./appTaskChecks.js";
import { draftOf, statePath } from "./appTaskDraft.fixture.js";
const FILE = "Documents/notes.db";
const table = (body: string, truncated = false) => ({
columns: ["id", "body"],
primary_key: ["id"],
rows: [[1, body]],
truncated,
});
const database = (tables: JsonValue, tables_truncated = false) => ({
tables,
tables_truncated,
});
const draft = (before: JsonValue, after: JsonValue) =>
draftOf({ [FILE]: statePath("document", "sqlite", before, after) })[0];
describe("SQLite draft row boundaries", () => {
it.each([
database({ notes: table("kept", true) }),
database({ notes: { error: "permission denied" } }),
database({}, true),
])(
"keeps unknown baseline rows separate from known empty tables: %j",
(before) => {
const after = database({ notes: table("kept"), other: table("new") });
const file = draft(before, after);
expect(file).toBeDefined();
expect(
file?.suggested.some(
(check) => "table" in check && check.table === "notes",
),
).toBe(false);
// The complete inventory can still establish that another table is new.
expect(
file?.suggested.some(
(check) => "table" in check && check.table === "other",
),
).toBe(before.tables_truncated === false);
},
);
it("retains checks for changes in fully read sibling tables", () => {
const before = database({
notes: table("kept", true),
other: table("old"),
});
const after = database({ notes: table("kept"), other: table("new") });
const checks = draft(before, after)?.suggested ?? [];
expect(checks).toEqual([
{
kind: "row_present",
file: FILE,
table: "other",
where: { id: 1, body: "new" },
},
]);
for (const [view, passed] of [
[before, false],
[after, true],
] as const)
expect(
evaluateTaskChecks(
checks.map((check) => ({ ...check, id: "changed" })),
new Map([[FILE, { format: "sqlite", view }]]),
).map((result) => result.passed),
).toEqual([passed]);
});
it("suggests only predicates that distinguish keyed rows after timestamp omission", () => {
const saved = (stamp: string) =>
database({
notes: {
columns: ["id", "body", "saved"],
primary_key: ["id"],
rows: [[1, "kept", stamp]],
truncated: false,
},
});
expect(
draft(saved("2026-11-04 09:00:00"), saved("2026-11-04 10:00:00"))
?.suggested ?? [],
).toEqual([]);
});
it("compares projected predicates against siblings when timestamp keys are omitted", () => {
const rows = (first: string) =>
database({
notes: {
columns: ["stamp", "body"],
primary_key: ["stamp"],
rows: [
["2026-11-04 09:00:00", first],
["2026-11-04 10:00:00", "new"],
],
truncated: false,
},
});
const before = rows("old");
const after = rows("new");
expect(draft(before, after)?.suggested ?? []).toEqual([]);
});
it("keeps a newly created SQLite file's presence check", () => {
const after = database({ notes: table("new") });
const file = draftOf({
[FILE]: statePath("document", "sqlite", null, after),
})[0];
expect(file?.suggested).toEqual([{ kind: "file_present", file: FILE }]);
});
it("compares keyed rows by column names through a table rebuild", () => {
const before = database({ notes: table("kept") });
const reordered = (body: string) =>
database({
notes: {
columns: ["body", "id"],
primary_key: ["id"],
rows: [[body, 1]],
truncated: false,
},
});
expect(draft(before, reordered("kept"))?.suggested ?? []).toEqual([]);
const after = reordered("edited");
const checks = draft(before, after)?.suggested ?? [];
expect(checks).toEqual([
{
kind: "row_present",
file: FILE,
table: "notes",
where: { body: "edited", id: 1 },
},
]);
for (const [view, passed] of [
[before, false],
[after, true],
] as const)
expect(
evaluateTaskChecks(
checks.map((check) => ({ ...check, id: "changed" })),
new Map([[FILE, { format: "sqlite", view }]]),
).map((result) => result.passed),
).toEqual([passed]);
});
});
+40 -18
View File
@@ -18,11 +18,14 @@ interface TableView {
readonly rows: readonly (readonly JsonValue[])[];
}
const tableViews = (version: Version): ReadonlyMap<string, TableView> => {
const tables = new Map<string, TableView>();
/** Keep unread tables distinct from absence in a complete table inventory. */
const tableViews = (version: Version) => {
const tables = new Map<string, TableView | undefined>();
const view = version.present ? version.view : null;
if (!isRecord(view) || !isRecord(view.tables)) return tables;
if (!isRecord(view) || !isRecord(view.tables))
return { tables, complete: false };
for (const [name, table] of Object.entries(view.tables)) {
tables.set(name, undefined);
if (!isRecord(table) || table.truncated !== false) continue;
const strings = (value: JsonValue | undefined) =>
Array.isArray(value)
@@ -34,7 +37,7 @@ const tableViews = (version: Version): ReadonlyMap<string, TableView> => {
rows: Array.isArray(table.rows) ? table.rows.filter(Array.isArray) : [],
});
}
return tables;
return { tables, complete: view.tables_truncated === false };
};
/** Column values of a row, without volatile values or the columns left out. */
@@ -110,16 +113,37 @@ export const rowSuggestions = (
): Suggestion[] => {
const before = tableViews(baseline);
const suggestions: Suggestion[] = [];
const push = (
kind: "row_present" | "row_absent",
table: string,
where: Record<string, JsonValue>,
) => {
if (Object.keys(where).length > 0)
suggestions.push({ kind, file, table, where });
};
for (const [name, table] of tableViews(final)) {
const old = before.get(name) ?? { ...table, rows: [] };
for (const [name, table] of tableViews(final).tables) {
if (table === undefined) continue;
const previousTable = before.tables.get(name);
if (
previousTable === undefined &&
(before.tables.has(name) || !before.complete)
)
continue;
const old = previousTable ?? { ...table, rows: [] };
// Judge the predicate after volatile values are omitted, using the same
// matching semantics as the evaluator rather than raw row positions.
const push = (
kind: "row_present" | "row_absent",
where: Record<string, JsonValue>,
previousRow?: readonly JsonValue[],
) => {
if (Object.keys(where).length === 0) return;
// A paired row is sufficient only while the predicate retains its
// complete key. Omitted timestamp keys can let siblings match too.
const previousRows =
previousRow !== undefined &&
old.key.every((column) => Object.hasOwn(where, column))
? [previousRow]
: old.rows;
if (
kind === "row_present"
? countTaskRows(old.columns, previousRows, where) === 0
: countTaskRows(table.columns, table.rows, where) === 0
)
suggestions.push({ kind, file, table: name, where });
};
const oldRows = keyedRows(old);
const newRows = keyedRows(table);
for (const [id, row] of newRows) {
@@ -127,13 +151,11 @@ export const rowSuggestions = (
if (previous === undefined)
push(
"row_present",
name,
rowWhere(table, row, (column) => !table.key.includes(column)),
);
else if (JSON.stringify(previous) !== JSON.stringify(row))
else
push(
"row_present",
name,
rowWhere(
table,
row,
@@ -142,13 +164,13 @@ export const rowSuggestions = (
JSON.stringify(row[table.columns.indexOf(column)]) !==
JSON.stringify(previous[old.columns.indexOf(column)]),
),
previous,
);
}
for (const [id, row] of oldRows)
if (!newRows.has(id))
push(
"row_absent",
name,
rowWhere(old, row, (column) => old.key.includes(column)),
);
suggestions.push(...unkeyedRowSuggestions(file, name, old, table));
@@ -157,7 +157,9 @@ describe("draftTaskFiles values", () => {
["removed", `${paragraph}/2`, "by_position"],
]);
});
});
describe("draftTaskFiles values budgets", () => {
it("lists at most 20 values and 8 KB a file and 24 KB a draft, saying when it left any out", () => {
const items = (count: number, value: (index: number) => string) =>
`<doc>${Array.from({ length: count }, (_, index) => `<p id="${String(index)}" v="${value(index)}"/>`).join("")}</doc>`;
@@ -194,6 +196,47 @@ describe("draftTaskFiles values", () => {
drafted[0]?.values.length ?? 0,
);
});
it.each([
{ files: 1, width: 1989, exact: true, cut: false },
{ files: 1, width: 1989, exact: false, cut: true },
{ files: 4, width: 1477, exact: true, cut: false },
{ files: 4, width: 1477, exact: false, cut: true },
])(
"includes serialized array delimiters in the values budget ($files files, exact: $exact)",
({ files, width, exact, cut }) => {
const keys = Array.from(
{ length: 4 },
(_, index) =>
`${String(index)}${"a".repeat(width - (exact && index === 0 ? 1 : 0))}`,
);
const drafted = draftOf(
Object.fromEntries(
Array.from({ length: files }, (_, index) => [
`settings-${String(index)}.json`,
statePath(
"application_state",
"json",
Object.fromEntries(keys.map((key) => [key, "x"])),
Object.fromEntries(keys.map((key) => [key, "y"])),
),
]),
),
);
const sizes = drafted.map(
(file) => new TextEncoder().encode(JSON.stringify(file.values)).length,
);
expect(sizes.every((size) => size <= 8192)).toBe(true);
expect(sizes.reduce((sum, size) => sum + size, 0)).toBeLessThanOrEqual(
24576,
);
expect(drafted.some((file) => file.values_cut)).toBe(cut);
if (exact)
expect(sizes.reduce((sum, size) => sum + size, 0)).toBe(
files === 1 ? 8192 : 24576,
);
},
);
});
it("preserves JSON objects whose keys resemble XML and archive views", () => {
+5 -4
View File
@@ -554,9 +554,9 @@ export const draftValues = (
.map((pointer) => entryOf(context, references, pointer, "removed")),
].toSorted((left, right) => rank(left) - rank(right));
const values: DraftValue[] = [];
let bytes = 0;
let bytes = 2; // The serialized values array's brackets.
for (const entry of entries) {
const size = byteLength(entry);
const size = byteLength(entry) + (values.length === 0 ? 0 : 1);
if (
values.length >= VALUE_LIMITS.entries ||
bytes + size > VALUE_LIMITS.fileBytes
@@ -606,11 +606,12 @@ export const withinDraftBudget = <
>(
files: readonly File[],
): File[] => {
let left = VALUE_LIMITS.draftBytes;
// Reserve every file's array brackets before assigning entry bytes in file order.
let left = VALUE_LIMITS.draftBytes - files.length * 2;
return files.map((file) => {
const kept: DraftValue[] = [];
for (const entry of file.values) {
const size = byteLength(entry);
const size = byteLength(entry) + (kept.length === 0 ? 0 : 1);
if (size > left) break;
kept.push(entry);
left -= size;
+3 -140
View File
@@ -492,146 +492,9 @@ async def _settled_views(url):
return views, True, False
`;
/** Where the verifier's evaluator begins. The task description is everything before it. */
const VERIFIER_EVALUATOR = "\nMISSING = object()\n";
/** The index helper this REA ships, and the one an older export wrote. The older one also accepts a leading zero. */
const CURRENT_INDEX = 'INDEX = re.compile(r"0|[1-9][0-9]*")';
const OLDER_INDEX = 'INDEX = re.compile(r"[0-9]+")';
/** A pointer segment the older index helper reads as an array index and this one does not, such as `01`. */
const LEADING_ZERO_SEGMENT = /^0\d+$/u;
/**
* The task a verifier states: the app, the checks and the settle timing,
* without the evaluator that runs them.
*/
export const taskVerifierDescription = (source: string): string | undefined => {
const at = source.indexOf(VERIFIER_EVALUATOR);
return at < 0 ? undefined : source.slice(0, at + 1);
};
/** The evaluator, from `MISSING` through `verify`, or undefined when that marker is absent. */
const verifierEvaluator = (source: string): string | undefined => {
const at = source.indexOf(VERIFIER_EVALUATOR);
return at < 0 ? undefined : source.slice(at);
};
/** Read one Python literal that `pythonLiteral` wrote, from `source` at `index`. */
const readPythonLiteral = (
source: string,
index: number,
): { readonly value: JsonValue; readonly index: number } | undefined => {
let cursor = index;
while (cursor < source.length && /\s/u.test(source[cursor] ?? ""))
cursor += 1;
const character = source[cursor];
if (source.startsWith("None", cursor))
return { value: null, index: cursor + 4 };
if (source.startsWith("True", cursor))
return { value: true, index: cursor + 4 };
if (source.startsWith("False", cursor))
return { value: false, index: cursor + 5 };
if (character === '"' || character === "'") {
let end = cursor + 1;
let escaped = false;
while (end < source.length) {
const next = source[end] ?? "";
if (escaped) escaped = false;
else if (next === "\\") escaped = true;
else if (next === character) break;
end += 1;
}
if (source[end] !== character) return undefined;
const token = source.slice(cursor, end + 1);
const value =
character === '"'
? (JSON.parse(token) as string)
: token.slice(1, -1).replaceAll("\\'", "'");
return { value, index: end + 1 };
}
if (character === "[") {
const items: JsonValue[] = [];
cursor += 1;
while (cursor < source.length) {
while (cursor < source.length && /\s/u.test(source[cursor] ?? ""))
cursor += 1;
if (source[cursor] === "]") return { value: items, index: cursor + 1 };
const item = readPythonLiteral(source, cursor);
if (item === undefined) return undefined;
items.push(item.value);
cursor = item.index;
while (cursor < source.length && /\s/u.test(source[cursor] ?? ""))
cursor += 1;
if (source[cursor] === ",") cursor += 1;
}
return undefined;
}
if (character === "{") {
const record: Record<string, JsonValue> = {};
cursor += 1;
while (cursor < source.length) {
while (cursor < source.length && /\s/u.test(source[cursor] ?? ""))
cursor += 1;
if (source[cursor] === "}") return { value: record, index: cursor + 1 };
const key = readPythonLiteral(source, cursor);
if (key === undefined || typeof key.value !== "string") return undefined;
cursor = key.index;
while (cursor < source.length && /\s/u.test(source[cursor] ?? ""))
cursor += 1;
if (source[cursor] !== ":") return undefined;
const item = readPythonLiteral(source, cursor + 1);
if (item === undefined) return undefined;
record[key.value] = item.value;
cursor = item.index;
while (cursor < source.length && /\s/u.test(source[cursor] ?? ""))
cursor += 1;
if (source[cursor] === ",") cursor += 1;
}
return undefined;
}
const number = /^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/u.exec(
source.slice(cursor),
);
if (number?.[0] === undefined) return undefined;
return { value: Number(number[0]), index: cursor + number[0].length };
};
/** Pointers of the checks a verifier states. A pointer inside a check's value is not one. */
const describedPointers = (description: string): readonly string[] => {
const at = description.indexOf("CHECKS = ");
if (at < 0) return [];
const parsed = readPythonLiteral(description, at + "CHECKS = ".length);
if (parsed === undefined || !Array.isArray(parsed.value)) return [];
return parsed.value.flatMap((check) =>
typeof check === "object" &&
check !== null &&
!Array.isArray(check) &&
typeof check.pointer === "string"
? [check.pointer]
: [],
);
};
/**
* Whether two verifiers grade the same task the same way. The description must
* match. The evaluator must be this REA's, or the older index helper alone
* when no pointer segment is a leading-zero number, which that helper would
* index and this one would not.
*/
export const sameTaskVerifier = (hosted: string, bundled: string): boolean => {
const expected = taskVerifierDescription(hosted);
const found = taskVerifierDescription(bundled);
if (expected === undefined || expected !== found) return false;
const hostedBody = verifierEvaluator(hosted);
const bundledBody = verifierEvaluator(bundled);
if (hostedBody === undefined || bundledBody === undefined) return false;
if (hostedBody === bundledBody) return true;
const indexed = describedPointers(expected).some((pointer) =>
pointer.split("/").some((segment) => LEADING_ZERO_SEGMENT.test(segment)),
);
return (
bundledBody === hostedBody.replace(CURRENT_INDEX, OLDER_INDEX) && !indexed
);
};
/** Match the current generated verifier exactly before witnessing its task. Older evaluators require a new export. */
export const sameTaskVerifier = (hosted: string, bundled: string): boolean =>
hosted.includes("\nMISSING = object()\n") && hosted === bundled;
/** The verifier artifact: a screen-judged task's judge, with the build's examples, or else reads of the app's state through its data/get once its files settle and the task's checks on them. */
export const renderTaskVerifier = (
@@ -18,120 +18,19 @@ const task: DesktopTask = {
};
describe("sameTaskVerifier", () => {
it("accepts an older evaluator of the same checks and rejects a different task", () => {
it("accepts the current generated verifier and rejects changed checks or evaluator code", () => {
const hosted = renderTaskVerifier(task, "notes");
const older = hosted.replace(
'INDEX = re.compile(r"0|[1-9][0-9]*")',
'INDEX = re.compile(r"[0-9]+")',
);
expect(older).not.toBe(hosted);
expect(sameTaskVerifier(hosted, older)).toBe(true);
expect(sameTaskVerifier(hosted, hosted.replace('"eggs"', '"milk"'))).toBe(
false,
);
expect(
sameTaskVerifier(hosted, older.replace("def _shown", "def _show")),
).toBe(false);
expect(
sameTaskVerifier(
hosted,
hosted.replace(/\nasync def verify[\s\S]*$/u, "\n"),
),
).toBe(false);
});
it("rejects the older index helper when a path segment has a leading zero", () => {
const indexed: DesktopTask = {
...task,
checks: [
{
id: "line",
kind: "value_absent",
file: "Documents/today.txt",
pointer: "/lines/01",
},
],
};
const hosted = renderTaskVerifier(indexed, "notes");
const older = hosted.replace(
'INDEX = re.compile(r"0|[1-9][0-9]*")',
'INDEX = re.compile(r"[0-9]+")',
);
expect(sameTaskVerifier(hosted, older)).toBe(false);
expect(sameTaskVerifier(hosted, hosted)).toBe(true);
});
it("accepts the older index helper when a leading zero is only in a file name or a URL", () => {
const named: DesktopTask = {
...task,
checks: [
{
id: "guide",
kind: "text_contains",
file: "Documents/01-notes.xml",
pointer: "/root",
value: "https://example.com/01-guide",
},
],
};
const hosted = renderTaskVerifier(named, "notes");
const older = hosted.replace(
'INDEX = re.compile(r"0|[1-9][0-9]*")',
'INDEX = re.compile(r"[0-9]+")',
);
expect(sameTaskVerifier(hosted, older)).toBe(true);
});
it("accepts the older index helper when a leading zero is only inside a check value", () => {
const stored: DesktopTask = {
...task,
checks: [
{
id: "settings",
kind: "value_equals",
file: "Documents/settings.json",
pointer: "/settings",
value: { pointer: "/lines/01" },
},
],
};
const hosted = renderTaskVerifier(stored, "notes");
const older = hosted.replace(
'INDEX = re.compile(r"0|[1-9][0-9]*")',
'INDEX = re.compile(r"[0-9]+")',
);
expect(sameTaskVerifier(hosted, older)).toBe(true);
});
it("rejects the older index helper when a brace inside an earlier check's text hides a leading-zero pointer", () => {
const indexed: DesktopTask = {
...task,
checks: [
{
id: "brace",
kind: "value_equals",
file: "Documents/today.txt",
pointer: "/lines/0",
value: "{",
},
{
id: "line",
kind: "value_absent",
file: "Documents/today.txt",
pointer: "/lines/01",
},
],
};
const hosted = renderTaskVerifier(indexed, "notes");
const older = hosted.replace(
'INDEX = re.compile(r"0|[1-9][0-9]*")',
'INDEX = re.compile(r"[0-9]+")',
);
expect(sameTaskVerifier(hosted, older)).toBe(false);
for (const bundled of [
hosted.replace('"eggs"', '"milk"'),
hosted.replace(
'INDEX = re.compile(r"0|[1-9][0-9]*")',
'INDEX = re.compile(r"[0-9]+")',
),
hosted.replace("def _shown", "def _show"),
hosted.replace(/\nasync def verify[\s\S]*$/u, "\n"),
])
expect(sameTaskVerifier(hosted, bundled)).toBe(false);
expect(sameTaskVerifier("not a verifier", "not a verifier")).toBe(false);
});
});
+7 -11
View File
@@ -8,7 +8,6 @@ import {
readerCommand,
} from "./appTaskReaders.js";
const portable = (name: string) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name);
const POSTER = "Documents/poster.xcf";
const LAYERS = {
@@ -27,16 +26,13 @@ const LAYER_CHECK = {
};
const parse = (change: Record<string, unknown>) =>
parseDesktopTask(
{
name: "rename-layer",
prompt: "In GIMP, rename the Headline layer to Summer Sale and save it.",
checks: [LAYER_CHECK],
readers: [LAYERS],
...change,
},
portable,
);
parseDesktopTask({
name: "rename-layer",
prompt: "In GIMP, rename the Headline layer to Summer Sale and save it.",
checks: [LAYER_CHECK],
readers: [LAYERS],
...change,
});
/** Task changes the parser refuses for its readers, each with part of its message. */
const REFUSALS: readonly (readonly [Record<string, unknown>, string])[] = [
+56 -40
View File
@@ -53,7 +53,6 @@ POLL_S = 0.05
DRAIN_S = 2.0
MAX_LINES = 5000
MAX_SAFE_INTEGER = 2**53 - 1
LINE_BREAK = re.compile(r"\r?\n")
SESSION = ("DISPLAY", "WAYLAND_DISPLAY", "XAUTHORITY")
NEVER = ("DBUS_SESSION_BUS_ADDRESS", "SESSION_MANAGER", "XDG_RUNTIME_DIR")
FRESH = (("XDG_CONFIG_HOME", ".config"), ("XDG_DATA_HOME", ".local/share"), ("XDG_CACHE_HOME", ".cache"),
@@ -167,7 +166,14 @@ def _run(arguments: dict, source: str, work: str) -> dict:
return {"error": f"the reader ran past its {timeout} s{_tail(stderr)}"}
if process.returncode != 0:
return {"error": f"the reader exited {process.returncode}{_tail(stderr)}"}
lines = [line[len(MARKER):] for line in LINE_BREAK.split(printed.decode("utf-8", "replace")) if line.startswith(MARKER)]
# Unmarked startup text is not observation data. Decode only marked rows,
# without replacing malformed bytes with characters the reader never gave.
try:
marker = MARKER.encode("ascii")
lines = [line[len(marker):].removesuffix(b"\r").decode("utf-8")
for line in printed.split(b"\n") if line.startswith(marker)]
except UnicodeDecodeError:
return {"error": "the reader's marked output is not valid UTF-8"}
if not lines:
return {"error": f"the reader printed no line starting with {json.dumps(MARKER)}{_tail(stderr)}"}
return _view(arguments["output"], lines)
@@ -181,9 +187,9 @@ def main(arguments: dict) -> int:
except (FileNotFoundError, NotADirectoryError):
return _say({"absent": True})
except OSError as error:
return _say({"error": f"could not inspect the reader's file: {error.strerror or error}"})
return _say({"error": f"cannot inspect the reader's source: {error.strerror or error}"})
if not stat.S_ISREG(mode):
return _say({"absent": True})
return _say({"error": "the reader's source is not a regular file"})
# A remote caller owns this exact folder before starting the command, and
# removes it only after the command has settled.
lease = arguments.get("lease")
@@ -397,46 +403,56 @@ async def _reader_views(env_url, card):
if protocol.find_extension(card, EXEC) is None:
raise RuntimeError(f"the computer at {env_url} offers no {EXEC}, which runs the task's readers")
await _cleanup_readers(env_url, card)
views = {}
for reader in READERS:
token = uuid.uuid4().hex
lease = {"path": f"/tmp/rea-reader-{token}", "token": token}
arguments = json.loads("".join(reader["arguments"]))
arguments["lease"] = lease
encoded = json.dumps(arguments)
params = {"argv": ["python3", "-I", "-c", READER_RUNNER,
*(encoded[i:i + 8192] for i in range(0, len(encoded), 8192))],
"request_id": f"rea-reader-{token}",
"timeout_s": reader["timeout_s"] + RUNNER_SLACK_S}
task = _owned_task(protocol.invoke_extension(
env_url, card, EXEC, params, method="run", timeout=reader["timeout_s"] + 2 * RUNNER_SLACK_S))
_PENDING_READERS[token] = {"env_url": env_url, "lease": lease, "task": task,
"request_id": params["request_id"], "settled": False, "abandoned": False}
try:
answer = await asyncio.shield(task)
except BaseException as error:
_PENDING_READERS[token]["abandoned"] = True
error.add_note(f"Reader command {params['request_id']} and folder {lease['path']} remain owned for retry.")
raise
if not isinstance(answer, dict) or answer.get("timed_out"):
_PENDING_READERS[token]["abandoned"] = True
raise RuntimeError(f"Reader {params['request_id']} may still be running; retained {lease['path']}")
_PENDING_READERS[token]["settled"] = True
try:
view = _reader_answer(reader["id"], answer)
if view is not None:
views[reader["id"]] = view
finally:
views, absent = {}, []
try:
for reader in READERS:
token = uuid.uuid4().hex
lease = {"path": f"/tmp/rea-reader-{token}", "token": token}
arguments = json.loads("".join(reader["arguments"]))
arguments["lease"] = lease
encoded = json.dumps(arguments)
params = {"argv": ["python3", "-I", "-c", READER_RUNNER,
*(encoded[i:i + 8192] for i in range(0, len(encoded), 8192))],
"request_id": f"rea-reader-{token}",
"timeout_s": reader["timeout_s"] + RUNNER_SLACK_S}
task = _owned_task(protocol.invoke_extension(
env_url, card, EXEC, params, method="run", timeout=reader["timeout_s"] + 2 * RUNNER_SLACK_S))
_PENDING_READERS[token] = {"env_url": env_url, "lease": lease, "task": task,
"request_id": params["request_id"], "settled": False, "abandoned": False}
try:
answer = await asyncio.shield(task)
except BaseException as error:
_PENDING_READERS[token]["abandoned"] = True
error.add_note(f"Reader command {params['request_id']} and folder {lease['path']} remain owned for retry.")
raise
if not isinstance(answer, dict) or answer.get("timed_out"):
_PENDING_READERS[token]["abandoned"] = True
raise RuntimeError(f"Reader {params['request_id']} may still be running; retained {lease['path']}")
_PENDING_READERS[token]["settled"] = True
failure = None
try:
view = _reader_answer(reader["id"], answer)
if view is None:
absent.append(reader["id"])
else:
views[reader["id"]] = view
except BaseException as error:
failure = error
try:
await _cleanup_readers(env_url, card, token)
except asyncio.CancelledError as error:
except BaseException as error:
_PENDING_READERS[token]["abandoned"] = True
error.add_note(f"Reader cleanup remains owned for {lease['path']}; completed reader views: {json.dumps(views)}")
if failure is not None:
failure.add_note(f"Reader cleanup remains owned for {lease['path']}: {type(error).__name__}: {error}")
raise failure from error
error.add_note(f"Reader cleanup remains owned for {lease['path']}")
raise
except Exception as error:
_PENDING_READERS[token]["abandoned"] = True
raise RuntimeError(f"{error}; completed reader views: {json.dumps(views)}") from error
return views
if failure is not None:
raise failure
return views
except BaseException as error:
error.add_note(f"completed reader views: {json.dumps(views)}; completed reader absent: {json.dumps(absent)}")
raise
def evaluate_with_readers(checks: list, views: dict, readers: dict) -> list:
+7 -11
View File
@@ -10,23 +10,19 @@ import {
screenExamplesOf,
} from "./appTaskScreen.js";
const portable = (name: string) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name);
const SHOWN = {
id: "shown",
kind: "screen_judge",
question: "Does the calculator's display show the result 8.568?",
};
const parse = (change: Record<string, unknown>) =>
parseDesktopTask(
{
name: "tip-total",
prompt:
"In Calculator, work out 18% of 47.60 and leave the result showing.",
checks: [SHOWN],
...change,
},
portable,
);
parseDesktopTask({
name: "tip-total",
prompt:
"In Calculator, work out 18% of 47.60 and leave the result showing.",
checks: [SHOWN],
...change,
});
const EXAMPLES = { fail: "c2VlZA==", pass: "ZW5k" };
describe("screen_judge checks", () => {
+12
View File
@@ -188,6 +188,18 @@ describe("build secrets in escaped text", () => {
[{ handle: "xml", value: marked }],
),
).toBe('<p title="<secret:xml>">');
const xmlValue = 'e-a&b<c>d"-Fake';
const xmlSecrets = [{ handle: "xml", value: xmlValue }];
for (const encoded of [
"e-a&amp;b&lt;c>d&quot;-Fake",
"e-a&amp;b&lt;c&gt;d&quot;-Fake",
]) {
const document = `<p title="${encoded}">`;
expect(secretsIn(Buffer.from(document), xmlSecrets)).toEqual(["xml"]);
expect(redactSecrets(document, xmlSecrets)).toBe(
'<p title="<secret:xml>">',
);
}
expect(redactSecrets("text… unrelated", secrets)).toBe("text… unrelated");
const quoted = 'e-say"hi"\\now-Fake';
expect(
+5 -2
View File
@@ -74,10 +74,13 @@ const textSpellings = (value: string): readonly string[] => {
const form = new URLSearchParams([["value", value]])
.toString()
.slice("value=".length);
const escapedXml = xmlEscaped(value);
const xml = [
value,
xmlEscaped(value),
xmlEscaped(value).replaceAll('"', "&quot;"),
...[escapedXml, escapedXml.replaceAll(">", "&gt;")].flatMap((text) => [
text,
text.replaceAll('"', "&quot;"),
]),
];
return [
...new Set([
@@ -32,6 +32,7 @@ describe("the adaptive band", () => {
["3 of 3", "too_easy", [pass, pass, pass]],
["1 of 3", "undecided", [pass, miss, miss]],
["2 of 3", "undecided", [pass, pass, miss]],
["0 of 5", "too_hard", [miss, miss, miss, miss, miss]],
["1 of 5", "in_band", [pass, miss, miss, miss, miss]],
["3 of 5", "in_band", [pass, pass, miss, pass, miss]],
["4 of 5", "too_easy", [pass, pass, miss, pass, pass]],
+5 -1
View File
@@ -57,7 +57,11 @@ const counts = (solves: readonly SolveRecord[]) => {
export const bandOf = (solves: readonly SolveRecord[]): Band => {
const { done, passes } = counts(solves);
if (done >= ALL_SOLVES)
return passes <= MOST_BAND_PASSES ? "in_band" : "too_easy";
return passes === 0
? "too_hard"
: passes <= MOST_BAND_PASSES
? "in_band"
: "too_easy";
if (done >= FIRST_SOLVES && passes === 0) return "too_hard";
if (done >= FIRST_SOLVES && passes === done) return "too_easy";
return "undecided";
+38
View File
@@ -0,0 +1,38 @@
import { z } from "zod";
import { normalizeHttpHostname } from "./httpHostname.js";
import { err, ok, type Result } from "./result.js";
/** Agentenv folder and local-id spelling, also used for authored task files. */
export const PORTABLE_BUNDLE_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
// A compose service name is one DNS label, whose wire limit is 63 octets.
export const PORTABLE_BUNDLE_NAME_MAX_LENGTH = 63;
export const PORTABLE_BUNDLE_NAME_CONSTRAINT = `a kebab-case label of at most ${String(PORTABLE_BUNDLE_NAME_MAX_LENGTH)} characters, excluding Windows device names, that remains unchanged as an HTTP service hostname`;
const WINDOWS_RESERVED_NAME = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])$/u;
export type PortableBundleNameFailure = "format" | "reserved" | "hostname";
/** Admit the exact source name without normalizing it into another identity. */
export const parsePortableBundleName = (
name: string,
): Result<string, PortableBundleNameFailure> => {
if (
!PORTABLE_BUNDLE_NAME_PATTERN.test(name) ||
name.length > PORTABLE_BUNDLE_NAME_MAX_LENGTH
)
return err("format");
if (WINDOWS_RESERVED_NAME.test(name)) return err("reserved");
if (normalizeHttpHostname(name) !== name) return err("hostname");
return ok(name);
};
/** Boolean admission for callers that do not project a refusal. */
export const isPortableBundleName = (name: string): boolean =>
parsePortableBundleName(name).ok;
/** Lookup names share portable identity; authoring reservations are separate. */
export const portableBundleNameSchema = z
.string()
.regex(PORTABLE_BUNDLE_NAME_PATTERN)
.max(PORTABLE_BUNDLE_NAME_MAX_LENGTH)
.refine(isPortableBundleName, `Expected ${PORTABLE_BUNDLE_NAME_CONSTRAINT}`);
+1 -2
View File
@@ -4,7 +4,6 @@ import { join } from "node:path";
import { expect, it } from "vitest";
import { isPortableBundleName } from "../../../src/application/environment/environmentBundle.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { parseDesktopTask } from "../../../src/domain/desktop/appTask.js";
import {
@@ -40,7 +39,7 @@ const README = "# Notes computer app\n\n## Task `set-year`\n\nThe task.\n";
/** A bundle holding TASK as the export writes it, with its README. */
const bundleWithTask = async () => {
const parsed = parseDesktopTask(TASK, isPortableBundleName);
const parsed = parseDesktopTask(TASK);
if (!parsed.ok) throw new Error(parsed.error);
const bundle = await createTestTempDirectory("measured-bundle-");
const steps = renderTaskSteps(parsed.value, "notes");
+1 -2
View File
@@ -12,7 +12,6 @@ import { gzipSync } from "node:zlib";
import type { CallToolResult } from "@modelcontextprotocol/server";
import { expect, it, vi } from "vitest";
import { isPortableBundleName } from "../../../src/application/environment/environmentBundle.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { parseDesktopTask } from "../../../src/domain/desktop/appTask.js";
import {
@@ -142,7 +141,7 @@ const lines = (...text: string[]) => ({
/** A bundle folder holding TASK as the export writes it for the app. */
const taskBundle = async (app: string) => {
const parsed = parseDesktopTask(TASK, isPortableBundleName);
const parsed = parseDesktopTask(TASK);
if (!parsed.ok) throw new Error(parsed.error);
const bundle = await bundleFolder();
const verifier = join(
+10 -7
View File
@@ -8,7 +8,6 @@ import type {
BuildSettle,
SettleClock,
} from "../../../src/application/desktop/DesktopBuildSettle.js";
import { isPortableBundleName } from "../../../src/application/environment/environmentBundle.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import type { SettleTiming } from "../../../src/domain/desktop/appSettle.js";
import { parseDesktopTask } from "../../../src/domain/desktop/appTask.js";
@@ -58,7 +57,7 @@ const lines = (...text: string[]) => ({
/** A bundle folder holding `task` as the export writes it for the app. */
const taskBundle = async (task: unknown, app: string) => {
const parsed = parseDesktopTask(task, isPortableBundleName);
const parsed = parseDesktopTask(task);
if (!parsed.ok) throw new Error(parsed.error);
const bundle = await bundleFolder();
const verifier = join(
@@ -516,11 +515,15 @@ it("ends the witness when the seed already passes, and refuses its later stages"
parsed(await check({ task: passing, stage: "seed", directory })).witness,
).toEqual(seed.witness);
const elsewhere = `${directory}-elsewhere`;
expect(
parseMcpToolError(
await check({ task: passing, stage: "seed", directory: elsewhere }),
).error.message,
).toContain(`saved to ${directory}, not ${elsewhere}`);
const refusal = parseMcpToolError(
await check({ task: passing, stage: "seed", directory: elsewhere }),
).error;
expect(refusal).toMatchObject({
code: "invalid_request",
message: expect.stringContaining(
`record belongs to ${directory}, not ${elsewhere}`,
),
});
await act("type", { text: "eggs\n" });
expect(
parseMcpToolError(await check({ task: passing, stage: "after_edit" })).error
@@ -157,11 +157,17 @@ async def run():
second = asyncio.create_task(other.verify("http://mcp", "http://computer:8080"))
results, _ = await asyncio.gather(first, second)
return results, first_error
if scenario.get("report_failure"):
try:
await module.verify("http://mcp", "http://computer:8080")
except BaseException as error:
return [], first_error or str(error) + " ".join(getattr(error, "__notes__", ()))
raise AssertionError("the controlled reader failure did not occur")
results = await module.verify("http://mcp", "http://computer:8080")
return results, first_error
results, first_error = asyncio.run(run())
print(json.dumps({"called": called, "reads": reads, "ended": clock[0], "results": results, "execs": execs, "first_error": first_error}))
print(json.dumps({"called": called, "reads": reads, "ended": clock[0], "results": results, "execs": execs, "first_error": first_error, "pending_readers": len(module._PENDING_READERS) if hasattr(module, "_PENDING_READERS") else 0}))
`;
type VerifierOutput = {
@@ -170,6 +176,7 @@ type VerifierOutput = {
reads: number[];
ended: number;
first_error: string | null;
pending_readers: number;
results: {
id: string;
result: boolean;
@@ -194,6 +201,7 @@ const runVerifier = async (
readonly reloadVerifier?: boolean;
readonly concurrent?: boolean;
readonly parallelRetry?: boolean;
readonly reportFailure?: boolean;
} = {},
): Promise<VerifierOutput> => {
const folder = await createTestTempDirectory("desktop-task-python-");
@@ -232,6 +240,7 @@ const runVerifier = async (
reload_verifier: computer.reloadVerifier,
concurrent: computer.concurrent,
parallel_retry: computer.parallelRetry,
report_failure: computer.reportFailure,
}),
encoding: "utf8",
}),
@@ -636,3 +645,60 @@ it.each([
).rejects.toThrow(message);
},
);
it.each([
{ transport: false, pending: 0 },
{ transport: true, pending: 1 },
])(
"preserves completed views and absence when a later reader fails (transport: $transport)",
async ({ transport, pending }) => {
const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], {
readers: [READERS[0]!, READERS[3]!, READERS[2]!],
answers: {
layers: ANSWERS.layers,
gone: ANSWERS.gone,
...(transport ? {} : { broken: "broken runner answer" }),
},
reportFailure: true,
});
expect(output.first_error).toContain("broken");
const facts =
/completed reader views: (.*); completed reader absent: (.*)$/u.exec(
output.first_error ?? "",
);
expect(facts).not.toBeNull();
expect(JSON.parse(facts![1]!)).toEqual({
layers: (ANSWERS.layers as { view: unknown }).view,
});
expect(JSON.parse(facts![2]!)).toEqual(["gone"]);
expect(output.pending_readers).toBe(pending);
},
);
it.each([false, true])(
"keeps a malformed reader answer primary alongside cleanup and retry (retry: %s)",
async (retry) => {
const output = await runVerifier(READER_CHECKS, [[0, CHECK_VIEWS]], [], {
readers: [READERS[2]!],
answers: { broken: "invalid runner payload" },
cleanupFailures: 1,
reportFailure: true,
retry,
});
expect(output.first_error).toContain("did not answer for broken");
expect(output.first_error).toContain("invalid runner payload");
expect(output.first_error).toContain("permission denied");
expect(output.first_error).toContain("/tmp/rea-reader-");
expect(output.pending_readers).toBe(retry ? 0 : 1);
if (retry) {
const cleanups = output.execs.filter(({ request_id }) =>
String(request_id).startsWith("rea-reader-cleanup-"),
);
expect(cleanups).toHaveLength(3);
const lease = (entry: Record<string, unknown>) =>
JSON.parse((entry.argv as string[])[4]!);
expect(lease(cleanups[1]!)).toEqual(lease(cleanups[0]!));
expect(lease(cleanups[2]!)).not.toEqual(lease(cleanups[0]!));
}
},
);
@@ -35,7 +35,7 @@ const expectUnread = async (source: string) => {
expect(view).toMatchObject({
format: null,
view: null,
error: expect.stringContaining("could not inspect the reader's file"),
error: expect.stringContaining("cannot inspect the reader's source"),
});
if (view === undefined) throw new Error("inspection failure became absence");
const [outcome] = evaluateTaskChecks(
@@ -1,5 +1,5 @@
import { execFile } from "node:child_process";
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { promisify } from "node:util";
@@ -209,6 +209,66 @@ it("answers absent for a file the home does not hold, without running the reader
await expect(access(join(home, "ran"))).rejects.toThrow();
});
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)(
"keeps an inaccessible source unknown instead of answering absent",
async () => {
const { home, app } = await computer();
const documents = join(home, "Documents");
await chmod(documents, 0);
try {
const view = readerAnswerView(await run(home, reporting(app)));
expect(view).toMatchObject({ format: null, view: null });
expect(view?.error).toContain("cannot inspect");
} finally {
await chmod(documents, 0o700);
}
},
);
it.each(["json", "lines"] as const)(
"refuses malformed UTF-8 in marked %s output while preserving literal replacement characters",
async (output) => {
const { home } = await computer();
const reader = (bytes: string): DesktopTaskReader => ({
id: "encoding",
file: NOTE,
argv: [
"python3",
"-I",
"-c",
`import sys; sys.stdout.buffer.write(${bytes})`,
"{file}",
],
output,
});
const malformed = readerAnswerView(
await run(home, reader(String.raw`b'REA {"value":"\xff"}\n'`)),
);
expect(malformed).toMatchObject({ format: null, view: null });
expect(malformed?.error).toContain("UTF-8");
const valid = readerAnswerView(
await run(
home,
reader(
String.raw`b'\xff ignored banner\nREA {"value":"\xef\xbf\xbd"}\r\n'`,
),
),
);
expect(valid).toMatchObject(
output === "json"
? { format: "json", view: { value: "\ufffd" } }
: {
format: "text",
view: {
lines: ['{"value":"\ufffd"}'],
line_count: 1,
truncated: false,
},
},
);
},
);
it.each([
[
"exits non-zero",