diff --git a/docs/testing.md b/docs/testing.md index 1e56a0348..1473bbf35 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -282,10 +282,12 @@ establish live DeepSeek API or native Windows coverage. Receipts include request artifacts, result digests and owned-process lineage. Full E2E tests invoke the production command dispatcher and real providers, -without fake launchers, runners or responses. `verify:keyed-archive` writes an -actual Foundation binary archive, runs the CLI and a separate stdio MCP -subprocess, checks parity and pagination, rejects an escaping path and checks -an XML graph golden. `verify:asset-catalog` compiles source-owned colors with +without fake launchers, runners or responses. `verify:keyed-archive` writes +Foundation binary, XML, and integer archives using one run-owned Swift module +cache. It checks CLI and separate stdio MCP parity, pagination, escaping-path +rejection, an XML graph golden, and numeric meaning for safe controls, unsafe +integers, boxed numbers, and integral-valued reals. `verify:asset-catalog` +compiles source-owned colors with `actool`, invokes real `assetutil`, then checks CLI/MCP results, exact catalog digest, every raw metadata field, pagination and malformed input rejection. Neither artifact workflow requires Hopper or launches it. Both run in macOS CI. diff --git a/scripts/verify/apple/keyed-archive-integers.mjs b/scripts/verify/apple/keyed-archive-integers.mjs new file mode 100644 index 000000000..c95a56d3a --- /dev/null +++ b/scripts/verify/apple/keyed-archive-integers.mjs @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; +import { keyedArchiveResultSchema } from "../../../dist/domain/apple/keyedArchive.js"; + +const rowSchema = z.object({ + container: z.enum(["roots", "objects"]), + root: z.string(), + objectId: z.number().int().nonnegative().optional(), + path: z.array(z.union([z.string(), z.number().int().nonnegative()])), + kind: z.enum(["integer", "real"]), + literal: z.string(), +}); +const archiveSchema = z.object({ + filename: z.string(), + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + rows: z.array(rowSchema), +}); +const oracleSchema = z.object({ xml: archiveSchema, binary: archiveSchema }); + +const formats = ["xml", "binary"]; +const requiredRows = [ + { root: "unsafeSigned", kind: "integer", literal: "9007199254740993" }, + { + root: "unsafeNegative", + kind: "integer", + literal: "-9223372036854775808", + }, + { root: "safeInteger", kind: "integer", literal: "9007199254740991" }, + { root: "safeNegative", kind: "integer", literal: "-42" }, + { root: "integralReal", kind: "real", literal: "9007199254740992" }, + { + root: "boxedUnsigned", + kind: "integer", + literal: "9007199254740993", + container: "objects", + }, + { + root: "boxedReal", + kind: "real", + literal: "9007199254740992", + container: "objects", + }, +]; + +const valueAt = (value, path) => { + for (const key of path) { + if (value === null || typeof value !== "object") return undefined; + value = value[key]; + } + return value; +}; + +const isUnsafeInteger = (row) => + row.kind === "integer" && + (BigInt(row.literal) > BigInt(Number.MAX_SAFE_INTEGER) || + BigInt(row.literal) < BigInt(Number.MIN_SAFE_INTEGER)); + +/** Verify the actual Foundation output and the production CLI/MCP result. */ +export async function verifyKeyedArchiveIntegerArchives(root) { + const oracle = oracleSchema.parse( + JSON.parse(await readFile(join(root, "oracle.json"), "utf8")), + ); + const archives = []; + + for (const format of formats) { + const archive = oracle[format]; + for (const required of requiredRows) { + assert.ok( + archive.rows.some( + (row) => + row.root === required.root && + row.kind === required.kind && + row.literal === required.literal && + (required.container === undefined || + row.container === required.container), + ), + `${format} Foundation oracle omitted ${required.root} ${required.literal}`, + ); + } + + const archivePath = join(root, archive.filename); + const digest = createHash("sha256") + .update(await readFile(archivePath)) + .digest("hex"); + assert.equal(digest, archive.sha256, `${format} archive digest`); + archives.push({ format, archive, archivePath, digest }); + } + return archives; +} + +/** Check every oracle numeric slot, preserving object-table identity lookups. */ +export function assertKeyedArchiveIntegerResult( + result, + format, + digest, + oracle, +) { + const parsed = keyedArchiveResultSchema.parse(result); + assert.equal(parsed.archive_sha256, digest); + assert.equal(parsed.target_sha256, digest); + assert.equal( + parsed.archive_format, + format === "xml" ? "xml-plist" : "binary-plist", + ); + + for (const row of oracle.rows) { + const unsafe = isUnsafeInteger(row); + const source = + row.container === "roots" + ? parsed.roots + : parsed.objects.find((object) => object.id === row.objectId)?.value; + const value = valueAt(source, row.path); + assert.notEqual( + value, + undefined, + `${row.root} numeric slot remains present`, + ); + + if (unsafe) { + const exact = + typeof value === "object" && + value !== null && + value.$plist_type === "integer" && + value.decimal === BigInt(row.literal).toString(); + const disclosed = parsed.limitations.some( + (reason) => + /integer|number/iu.test(reason) && + /precision|exact range|rounded/iu.test(reason), + ); + assert.ok( + exact || disclosed, + `${row.root} ${row.literal} must be exact or disclose numeric loss`, + ); + } else { + assert.equal( + typeof value, + "number", + `${row.root} retains its numeric type`, + ); + assert.equal(value, Number(row.literal), `${row.root} numeric value`); + } + } + return parsed; +} diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjs index f50eaacd1..471b76ef6 100644 --- a/scripts/verify/apple/keyed-archive.mjs +++ b/scripts/verify/apple/keyed-archive.mjs @@ -1,5 +1,5 @@ -import { execFile, spawnSync } from "node:child_process"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -11,6 +11,10 @@ import { artifactMcpResult, withArtifactMcp, } from "../../lib/artifact-e2e.mjs"; +import { + assertKeyedArchiveIntegerResult, + verifyKeyedArchiveIntegerArchives, +} from "./keyed-archive-integers.mjs"; if (process.platform !== "darwin") throw new Error( @@ -170,26 +174,37 @@ try { ); }); } - const numericTests = spawnSync( - "npm", - [ - "run", - "test:focused", - "--", - "tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts", - ], - { - cwd: fileURLToPath(new URL("../../../", import.meta.url)), - stdio: "inherit", - timeout: 240_000, - }, - ); - if (numericTests.error) throw numericTests.error; - assert.equal( - numericTests.status, - 0, - "Native Foundation integer cases failed", - ); + const integers = join(root, "integers"); + await mkdir(integers); + await promisify(execFile)("/usr/bin/xcrun", [ + "swift", + "-module-cache-path", + join(root, "modules"), + fileURLToPath( + new URL( + "../../../tests/conformance/native/keyed-archive-integers.swift", + import.meta.url, + ), + ), + integers, + ]); + for (const fixture of await verifyKeyedArchiveIntegerArchives(integers)) { + const cliResult = assertKeyedArchiveIntegerResult( + await artifactCli("inspect-keyed-archive", fixture.archivePath), + fixture.format, + fixture.digest, + fixture.archive, + ); + await withArtifactMcp(fixture.archivePath, async (client) => { + const mcpResult = assertKeyedArchiveIntegerResult( + await artifactMcpResult(client, "inspect_keyed_archive"), + fixture.format, + fixture.digest, + fixture.archive, + ); + assert.deepEqual(mcpResult, cliResult); + }); + } report = { ok: true, mocked: false, @@ -197,6 +212,9 @@ try { stdio_mcp: true, xml_golden: true, uid_named_root: true, + numeric_formats: ["xml", "binary"], + numeric_cli: true, + numeric_stdio_mcp: true, format: graph.archive_format, objects: graph.total_objects, references: graph.total_references, diff --git a/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts b/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts deleted file mode 100644 index 4508c05fa..000000000 --- a/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { describe, expect } from "vitest"; -import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; -import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; -import { parseEvidence } from "../../../src/domain/evidence.js"; -import { - keyedArchiveIntegerFormats, - keyedArchiveIntegerCases, - keyedArchiveIntegerFixtures, - expectKeyedArchiveIntegerCase, - type KeyedArchiveIntegerFixtures, -} from "../../fixtures/keyedArchiveIntegers.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; -import { - createTestWorkspace, - removeTestWorkspace, -} from "../../support/workspace/workspaceFixture.js"; - -const archiveTest = cliTest.extend<{ - $worker: { keyedArchiveIntegerArchives: KeyedArchiveIntegerFixtures }; -}>({ - keyedArchiveIntegerArchives: [ - // oxlint-disable-next-line no-empty-pattern -- Vitest parses fixture dependencies from object destructuring. - async ({}, use) => { - const workspace = await createTestWorkspace("rea-keyed-integers-"); - try { - await use(await keyedArchiveIntegerFixtures(workspace.root)); - } finally { - await removeTestWorkspace(workspace.root); - } - }, - { scope: "worker" }, - ], -}); - -const cases = keyedArchiveIntegerFormats.flatMap((format) => - keyedArchiveIntegerCases.map((selectedCase) => ({ format, selectedCase })), -); -// Selected by the existing required native keyed-archive verifier, not Linux skips. -describe.skipIf(process.platform !== "darwin")( - "Foundation archive integer precision", - () => { - archiveTest.for(cases)( - "filesystem preserves $format $selectedCase numeric meaning", - async ({ format, selectedCase }, { keyedArchiveIntegerArchives }) => { - const fixture = keyedArchiveIntegerArchives[format]; - const result = await inspectBundleKeyedArchive({ - bundlePath: fixture.root, - targetSha256: fixture.digest, - parameters: { path: fixture.selected.filename, limit: 100 }, - }); - expectKeyedArchiveIntegerCase( - result, - selectedCase, - fixture.selected, - fixture.digest, - ); - }, - ); - archiveTest.for(cases)( - "built CLI preserves $format $selectedCase numeric meaning", - async ( - { format, selectedCase }, - { cli, keyedArchiveIntegerArchives }, - ) => { - const fixture = keyedArchiveIntegerArchives[format]; - const output = await cli.run({ - arguments: ["inspect-keyed-archive", fixture.archivePath, "--json"], - environment: { - REA_LOG_LEVEL: "silent", - REA_ANALYSIS_PROVIDER: "auto", - }, - }); - expect(output.exitCode).toBe(0); - const result = keyedArchiveResultSchema.parse( - parseEvidence(output.json).normalized_result, - ); - expectKeyedArchiveIntegerCase( - result, - selectedCase, - fixture.selected, - fixture.digest, - ); - }, - ); - }, -); diff --git a/tests/fixtures/keyedArchiveIntegers.ts b/tests/fixtures/keyedArchiveIntegers.ts deleted file mode 100644 index 11f543751..000000000 --- a/tests/fixtures/keyedArchiveIntegers.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; -import { promisify } from "node:util"; - -import { expect } from "vitest"; -import { z } from "zod"; -import { keyedArchiveResultSchema } from "../../src/domain/apple/keyedArchive.js"; - -const rowSchema = z.object({ - container: z.enum(["roots", "objects"]), - root: z.string(), - objectId: z.number().int().nonnegative().optional(), - path: z.array(z.union([z.string(), z.number().int().nonnegative()])), - kind: z.enum(["integer", "real"]), - literal: z.string(), -}); -const archiveSchema = z.object({ - filename: z.string(), - sha256: z.string().regex(/^[a-f0-9]{64}$/u), - rows: z.array(rowSchema), -}); -const oracleSchema = z.object({ xml: archiveSchema, binary: archiveSchema }); -export const keyedArchiveIntegerFormats = ["xml", "binary"] as const; -export const keyedArchiveIntegerCases = ["controls", "unsafe"] as const; -type Case = (typeof keyedArchiveIntegerCases)[number]; -type Row = z.infer; -type SelectedFixture = { - readonly root: string; - readonly selected: z.infer; - readonly archivePath: string; - readonly digest: string; -}; -export interface KeyedArchiveIntegerFixtures { - readonly xml: SelectedFixture; - readonly binary: SelectedFixture; -} - -export const keyedArchiveIntegerFixtures = async ( - root: string, -): Promise => { - await promisify(execFile)( - "/usr/bin/xcrun", - [ - "swift", - "-module-cache-path", - join(root, "modules"), - "tests/conformance/native/keyed-archive-integers.swift", - root, - ], - { maxBuffer: 1024 * 1024 }, - ); - const oracle = oracleSchema.parse( - JSON.parse(await readFile(join(root, "oracle.json"), "utf8")), - ); - for (const entry of [oracle.xml, oracle.binary]) { - expect(entry.rows).toContainEqual( - expect.objectContaining({ - root: "unsafeSigned", - kind: "integer", - literal: "9007199254740993", - }), - ); - expect(entry.rows).toContainEqual( - expect.objectContaining({ - root: "unsafeNegative", - kind: "integer", - literal: "-9223372036854775808", - }), - ); - expect( - entry.rows.some( - (row) => - row.root === "boxedUnsigned" && - row.kind === "integer" && - row.literal === "9007199254740993", - ), - ).toBe(true); - expect( - entry.rows.some( - (row) => - row.root === "boxedReal" && - row.kind === "real" && - Number(row.literal) === 9007199254740992, - ), - ).toBe(true); - } - const selectedFixture = async ( - selected: z.infer, - ): Promise => { - const archivePath = join(root, selected.filename); - const bytes = await readFile(archivePath); - const digest = createHash("sha256").update(bytes).digest("hex"); - expect(digest).toBe(selected.sha256); - return { root, selected, archivePath, digest }; - }; - return { - xml: await selectedFixture(oracle.xml), - binary: await selectedFixture(oracle.binary), - }; -}; -const at = (value: unknown, path: Row["path"]): unknown => { - for (const key of path) { - if (value === null || typeof value !== "object") return undefined; - value = (value as Record)[key]; - } - return value; -}; -export const expectKeyedArchiveIntegerCase = ( - result: z.infer, - selectedCase: Case, - selected: z.infer, - digest: string, -) => { - expect(result.archive_sha256).toBe(digest); - const unsafe = (row: Row) => - row.kind === "integer" && - (BigInt(row.literal) > BigInt(Number.MAX_SAFE_INTEGER) || - BigInt(row.literal) < BigInt(Number.MIN_SAFE_INTEGER)); - const rows = selected.rows.filter((row) => - selectedCase === "unsafe" ? unsafe(row) : !unsafe(row), - ); - expect(rows.length).toBeGreaterThan(0); - for (const row of rows) { - const value = at( - row.container === "roots" - ? result.roots - : result.objects.find((object) => object.id === row.objectId)?.value, - row.path, - ); - expect( - value, - `${row.root} numeric slot remains present`, - ).not.toBeUndefined(); - if (unsafe(row)) { - const exact = - typeof value === "object" && - value !== null && - (value as Record).$plist_type === "integer" && - (value as Record).decimal === - BigInt(row.literal).toString(); - // The accepted alternate contract must explicitly disclose numeric precision loss. - const disclosed = result.limitations.some( - (reason) => - /integer|number/iu.test(reason) && - /precision|exact range|rounded/iu.test(reason), - ); - expect( - exact || disclosed, - `${row.root} ${row.literal} must be exact or disclose numeric loss`, - ).toBe(true); - } else { - expect(typeof value, `${row.root} retains its numeric type`).toBe( - "number", - ); - expect(value).toBe(Number(row.literal)); - } - } -};