diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index 305a14b8e1..a9af0b0fb3 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -9,6 +9,7 @@ import { } from "../native/CommandRunner.js"; import { parseLipoOutputResult, + LipoFieldRetentionError, type LipoArchitecture, } from "../native/parsers/lipo.js"; import type { MachoSlice } from "../domain/apple/dylibResolution.js"; @@ -122,9 +123,14 @@ export class MachOSliceArtifactReader implements ArtifactReader { const parsed = parseLipoOutputResult(captured.value.stdout, this.path); if (!parsed.ok) throw new ArtifactReaderFailure( - "format", + parsed.error instanceof LipoFieldRetentionError ? "limit" : "format", `lipo did not produce readable architecture records: ${parsed.error.message}`, - { cause: parsed.error }, + { + cause: parsed.error, + ...(parsed.error instanceof LipoFieldRetentionError + ? { boundary: "memory" as const } + : {}), + }, ); const lipo = parsed.value; const { architectures } = lipo; @@ -254,6 +260,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { declaredIntegrity: null, unpacked: false, limitations: [ + ...architecture.limitations, ...(structural.status === "not-mach-o" ? [ "Structural Mach-O facts were unavailable; slice range is based on lipo output.", @@ -265,7 +272,10 @@ export class MachOSliceArtifactReader implements ArtifactReader { : []), ...(architecture.file_offset === null || architecture.size === null ? [ - "Slice range was derived from structural Mach-O facts because lipo omitted it.", + architecture.fields.offset.kind === "uninterpretable" || + architecture.fields.size.kind === "uninterpretable" + ? "Slice range was derived from structural Mach-O facts because lipo's range declarations could not all be interpreted." + : "Slice range was derived from structural Mach-O facts because lipo omitted it.", ] : []), ...(architecture.cpu_type !== null && diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index c19d728e1f..ce222652b4 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -641,6 +641,9 @@ class NativeMacOSClient implements AnalysisClient { ); if (!parsed.ok) return parsed; const { architectures } = parsed.value; + const limitations = architectures.flatMap( + (architecture) => architecture.limitations, + ); const result = listArchitecturesSchema.parse({ architectures: { items: architectures, @@ -649,11 +652,11 @@ class NativeMacOSClient implements AnalysisClient { limitations: [], }, provenance, - limitations: [], + limitations, }); return ok({ result: jsonValueSchema.parse(result), - limitations: [], + limitations, locations: architectureLocations(result.architectures.items), }); } diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index d9388393f2..c981e4138a 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -149,7 +149,9 @@ export const inspectNativeMacho = async ( return err(unusableNativeCommandOutput(captured.value)); captures.push(captured.value); } - const limitations: string[] = []; + const limitations = architectures.flatMap( + (architecture) => architecture.limitations, + ); if (optional === null) limitations.push( `vtool cannot select the ${selectedArchitecture} slice; build metadata is normalized from otool only.`, diff --git a/src/native/parsers/lipo.test.ts b/src/native/parsers/lipo.test.ts index e98c39a84e..bd5a51bba1 100644 --- a/src/native/parsers/lipo.test.ts +++ b/src/native/parsers/lipo.test.ts @@ -53,6 +53,82 @@ it("retains native command capture when shared lipo parsing rejects records", () }); }); +describe("lipo numeric field observations", () => { + it.each([ + { field: "offset", source: "-1" }, + { field: "size", source: "9007199254740992" }, + { field: "align", source: "2^12 (2048)" }, + { field: "align", source: "2^53" }, + { field: "offset", source: "" }, + ] as const)( + "retains an explicit uninterpretable $field source: $source", + ({ field, source }) => { + const sourceLine = `\t${field} ${source} `; + const architecture = parseLipoOutput( + `architecture arm64\n cputype 16777228\n${sourceLine}\n`, + ).architectures[0]; + expect(architecture?.cpu_type_code).toBe(16777228); + expect(architecture?.fields[field]).toEqual({ + kind: "uninterpretable", + sourceLines: [sourceLine], + }); + expect(architecture?.limitations).toEqual([ + `lipo ${field} for arm64 could not be interpreted; source line: ${sourceLine}`, + ]); + }, + ); + + it("parses each valid numeric field once while preserving its exact source line", () => { + const architecture = parseLipoOutput( + "architecture arm64\n offset 4096 \n size 48\n align 2^12 (4096)\n", + ).architectures[0]; + expect(architecture).toMatchObject({ + file_offset: 4096, + size: 48, + alignment: 4096, + limitations: [], + fields: { + offset: { + kind: "parsed", + sourceLines: [" offset 4096 "], + value: 4096, + }, + size: { kind: "parsed", sourceLines: [" size 48"], value: 48 }, + align: { + kind: "parsed", + sourceLines: [" align 2^12 (4096)"], + value: 4096, + }, + }, + }); + }); + + it.each(["offset", "size", "align"] as const)( + "retains every repeated %s declaration and requires their parsed values to agree", + (field) => { + const first = field === "align" ? "2^12 (4096)" : "4096"; + const conflicting = field === "align" ? "2^13 (8192)" : "8192"; + for (const second of [first, conflicting, "unsupported"]) { + const sourceLines = [` ${field} ${first}`, `\t${field} ${second} `]; + const row = parseLipoOutput( + `architecture arm64\n${sourceLines.join("\n")}\n`, + ).architectures[0]; + expect(row?.fields[field]).toEqual( + second === first + ? { kind: "parsed", sourceLines, value: 4096 } + : { kind: "uninterpretable", sourceLines }, + ); + if (second === first) expect(row?.limitations).toEqual([]); + else { + expect(row?.limitations).toHaveLength(1); + for (const line of sourceLines) + expect(row?.limitations[0]).toContain(line); + } + } + }, + ); +}); + describe("lipo architecture facts", () => { it("preserves Apple's declared and hidden ARM64 architecture scopes", () => { const output = parseLipoOutput(`fat_magic 0xcafebabe @@ -263,6 +339,12 @@ describe("lipo FAT operand framing", () => { file_offset: null, size: null, alignment: null, + fields: { + offset: { kind: "absent" }, + size: { kind: "absent" }, + align: { kind: "absent" }, + }, + limitations: [], }, ], }); diff --git a/src/native/parsers/lipo.ts b/src/native/parsers/lipo.ts index c985ddfb14..953e36a405 100644 --- a/src/native/parsers/lipo.ts +++ b/src/native/parsers/lipo.ts @@ -1,7 +1,13 @@ import { z } from "zod"; -import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../../domain/result.js"; -import type { NativeCommandCapture } from "../CommandRunner.js"; +import { + NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, + type NativeCommandCapture, +} from "../CommandRunner.js"; import { machoCpuSubtypeBase } from "../../domain/apple/machoCpuIdentity.js"; const architectureSchema = z.object({ @@ -18,7 +24,35 @@ const architectureSchema = z.object({ alignment: z.int().min(0).nullable(), }); -export type LipoArchitecture = z.infer; +export type LipoFieldObservation = + | { readonly kind: "absent" } + | { + readonly kind: "parsed"; + readonly sourceLines: readonly string[]; + readonly value: number; + } + | { + readonly kind: "uninterpretable"; + readonly sourceLines: readonly string[]; + }; + +export type LipoArchitecture = z.infer & { + readonly fields: Readonly< + Record<"offset" | "size" | "align", LipoFieldObservation> + >; + readonly limitations: readonly string[]; +}; + +/** Capacity refusal before retaining expanded per-declaration source facts. */ +export class LipoFieldRetentionError extends Error { + readonly _tag = "LipoFieldRetentionError"; + constructor(readonly requiredBytes: number) { + super( + "lipo numeric field observations exceed the retained native output budget", + ); + this.name = this._tag; + } +} interface LipoOutput { readonly format: "thin" | "fat"; @@ -55,31 +89,80 @@ export const parseLipoOutput = ( declaredArchitectureCount: null, hiddenArchitectureCount: null, architectures: [ - architectureSchema.parse({ - name: thinArchitecture, - hidden: false, - cpu_type: null, - cpu_subtype: null, - cpu_type_code: null, - cpu_subtype_code: null, - cpu_subtype_base: null, - capabilities: null, - file_offset: null, - size: null, - alignment: null, - }), + { + ...architectureSchema.parse({ + name: thinArchitecture, + hidden: false, + cpu_type: null, + cpu_subtype: null, + cpu_type_code: null, + cpu_subtype_code: null, + cpu_subtype_base: null, + capabilities: null, + file_offset: null, + size: null, + alignment: null, + }), + fields: { + offset: { kind: "absent" }, + size: { kind: "absent" }, + align: { kind: "absent" }, + }, + limitations: [], + }, ], }; const architectures: LipoArchitecture[] = []; let declaredArchitectureCount: number | null = null; let hiddenArchitectureCount: number | null = null; let current: Record | undefined; + let observations: Partial< + Record< + "offset" | "size" | "align", + { readonly sourceLine: string; readonly value: number | null }[] + > + > = {}; + let retainedFieldBytes = 0; + const admitFieldBytes = (bytes: number): void => { + retainedFieldBytes += bytes; + if (retainedFieldBytes > NATIVE_COMMAND_OUTPUT_BUDGET_BYTES) + throw new LipoFieldRetentionError(retainedFieldBytes); + }; const flush = (): void => { if (current === undefined) return; const name = current.architecture ?? current["Non-fat file"]; - if (name !== undefined) - architectures.push( - architectureSchema.parse({ + if (name !== undefined) { + const observe = ( + key: "offset" | "size" | "align", + ): LipoFieldObservation => { + const observed = observations[key]; + if (observed === undefined) return { kind: "absent" }; + const sourceLines = observed.map(({ sourceLine }) => sourceLine); + const value = observed[0]?.value; + return value === null || + value === undefined || + observed.some((candidate) => candidate.value !== value) + ? { kind: "uninterpretable", sourceLines } + : { kind: "parsed", sourceLines, value }; + }; + const fields = { + offset: observe("offset"), + size: observe("size"), + align: observe("align"), + }; + const value = (field: LipoFieldObservation): number | null => + field.kind === "parsed" ? field.value : null; + const limitations = Object.entries(fields).flatMap( + ([field, observation]) => { + if (observation.kind !== "uninterpretable") return []; + admitFieldBytes(name.length * 14 + 256); + return [ + `lipo ${field} for ${name} could not be interpreted; ${observation.sourceLines.length === 1 ? "source line: " : "source lines:\n"}${observation.sourceLines.join("\n")}`, + ]; + }, + ); + architectures.push({ + ...architectureSchema.parse({ name: name.includes(" is architecture: ") ? (name.split(" is architecture: ").at(-1) ?? name) : name.replace(/ \(hidden\)$/u, ""), @@ -93,12 +176,16 @@ export const parseLipoOutput = ( ), cpu_subtype_base: cpuSubtypeBase(current.cpusubtype), capabilities: current.capabilities ?? null, - file_offset: integer(current.offset), - size: integer(current.size), - alignment: alignment(current.align), + file_offset: value(fields.offset), + size: value(fields.size), + alignment: value(fields.align), }), - ); + fields, + limitations, + }); + } current = undefined; + observations = {}; }; for (const rawLine of output.split(/\r?\n/u)) { const line = rawLine.trim(); @@ -118,10 +205,29 @@ export const parseLipoOutput = ( current = { architecture: line.slice("architecture ".length).trim() }; continue; } - const match = - /^(cputype|cpusubtype|capabilities|offset|size|align)\s+(.+)$/u.exec( - line, - ); + const numericField = /^\s*(offset|size|align)(?:\s+(.*))?$/u.exec(rawLine); + if ( + numericField?.[1] === "offset" || + numericField?.[1] === "size" || + numericField?.[1] === "align" + ) { + current ??= {}; + const field = numericField[1]; + // UTF-16 source, joined and rendered strings, worst-case escaped JSON, + // and per-observation bookkeeping fit the existing native output allowance. + // Charge before storing another observation; every source line is retained + // at most once per field and rendered at most once, never per duplicate pair. + admitFieldBytes(rawLine.length * 18 + 128); + const sourceValue = (numericField[2] ?? "").trim(); + const observed = { + sourceLine: rawLine, + value: + field === "align" ? alignment(sourceValue) : integer(sourceValue), + }; + (observations[field] ??= []).push(observed); + continue; + } + const match = /^(cputype|cpusubtype|capabilities)\s+(.+)$/u.exec(line); if (match?.[1] !== undefined && match[2] !== undefined) { current ??= {}; current[match[1]] = match[2].trim(); @@ -142,11 +248,15 @@ export const parseLipoOutput = ( export const parseLipoOutputResult = ( output: string, operandPath?: string, -): Result => { +): Result => { try { return ok(parseLipoOutput(output, operandPath)); } catch (cause: unknown) { - if (!(cause instanceof TypeError) && !(cause instanceof z.ZodError)) + if ( + !(cause instanceof TypeError) && + !(cause instanceof z.ZodError) && + !(cause instanceof LipoFieldRetentionError) + ) throw cause; return err(cause); } @@ -157,24 +267,41 @@ export const parseLipoCapture = ( capture: NativeCommandCapture, operation: "inspect_macho" | "list_architectures", operandPath: string, -): Result => { +): Result< + LipoOutput, + AnalysisOutputError | AnalysisResourceConstraintError +> => { const parsed = parseLipoOutputResult(capture.stdout, operandPath); if (parsed.ok) return parsed; + const capturedOutput = { + stdout: capture.stdout, + stderr: capture.stderr, + truncated: false, + stdout_bytes: capture.stdoutBytes, + stderr_bytes: capture.stderrBytes, + exit_code: capture.exitCode, + signal: capture.signal, + }; + if (parsed.error instanceof LipoFieldRetentionError) + return err( + new AnalysisResourceConstraintError( + operation, + "memory", + parsed.error.message, + { + maximum_retained_bytes: NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, + required_retained_bytes: parsed.error.requiredBytes, + }, + { cause: parsed.error, retryable: false, capturedOutput }, + ), + ); return err( new AnalysisOutputError( operation, "lipo did not produce readable architecture records", { cause: parsed.error, - capturedOutput: { - stdout: capture.stdout, - stderr: capture.stderr, - truncated: false, - stdout_bytes: capture.stdoutBytes, - stderr_bytes: capture.stderrBytes, - exit_code: capture.exitCode, - signal: capture.signal, - }, + capturedOutput, }, ), ); diff --git a/tests/boundary/providers/native/lipoFieldOwnership.test.ts b/tests/boundary/providers/native/lipoFieldOwnership.test.ts new file mode 100644 index 0000000000..80fe4f1595 --- /dev/null +++ b/tests/boundary/providers/native/lipoFieldOwnership.test.ts @@ -0,0 +1,225 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + CPU, + fatImage, + machoImage, +} from "../../../../src/artifacts/apple/MachoImage.fixture.js"; +import { MachOSliceArtifactReader } from "../../../../src/artifacts/MachOSliceArtifactReader.js"; +import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; +import { + listArchitecturesSchema, + inspectMachoSchema, +} from "../../../../src/domain/native/nativeInspection.js"; +import { + NativeFixtureRunner, + nativeMachoTarget, +} from "../../../fixtures/nativeCommands.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { NATIVE_COMMAND_OUTPUT_BUDGET_BYTES } from "../../../../src/native/CommandRunner.js"; +import { + LipoFieldRetentionError, + parseLipoCapture, +} from "../../../../src/native/parsers/lipo.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, +} from "../../../../src/artifacts/ArtifactReader.js"; + +it.each(["absent", "valid", "uninterpretable"] as const)( + "preserves %s lipo fields through native observations and structural slice inventory", + async (mode) => { + const root = await createTestTempDirectory("rea-lipo-field-owner-"); + const path = join(root, "fat"); + const thin = machoImage({ cpu: CPU.arm64 }); + await writeFile(path, fatImage([{ cpu: CPU.arm64, bytes: thin }])); + const badLines = [ + " offset 4096", + " offset -1 ", + " size 9007199254740992", + " align 2^12 (2048)", + ]; + const lines = + mode === "absent" + ? [] + : mode === "uninterpretable" + ? badLines + : [" offset 4096", ` size ${thin.length}`, " align 2^12 (4096)"]; + const stdout = [ + "architecture arm64", + " cputype 16777228", + " cpusubtype 0", + ...lines, + "", + ].join("\n"); + const runner = new NativeFixtureRunner({ lipo: stdout }); + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + nativeMachoTarget(path), + ); + const reader = new MachOSliceArtifactReader(path, {}, runner); + try { + const listed = await client.execute("list_architectures", {}); + if (!listed.ok) throw listed.error; + const list = listArchitecturesSchema.parse(listed.value.result); + expect(list.architectures.items).toMatchObject([ + { + name: "arm64", + cpu_type: "16777228", + cpu_subtype: "0", + file_offset: mode === "valid" ? 4096 : null, + size: mode === "valid" ? thin.length : null, + alignment: mode === "valid" ? 4096 : null, + }, + ]); + const inspected = await client.execute("inspect_macho", {}); + if (!inspected.ok) throw inspected.error; + const macho = inspectMachoSchema.parse(inspected.value.result); + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + expect(entries).toMatchObject([ + { path: "slices/arm64", byteOffset: 4096, declaredSize: thin.length }, + ]); + if (mode === "uninterpretable") { + expect(list.limitations).toHaveLength(3); + expect(list.architectures.limitations).toEqual([]); + for (const line of badLines) { + expect(list.limitations.some((note) => note.includes(line))).toBe( + true, + ); + expect(macho.architectures.limitations).toEqual([]); + expect(macho.limitations.some((note) => note.includes(line))).toBe( + true, + ); + expect( + entries[0]?.limitations.some((note) => note.includes(line)), + ).toBe(true); + } + expect(entries[0]?.limitations.join(" ")).not.toContain("lipo omitted"); + expect(entries[0]?.limitations).toContain( + "Slice range was derived from structural Mach-O facts because lipo's range declarations could not all be interpreted.", + ); + } else { + expect(list.limitations).toEqual([]); + expect(entries[0]?.limitations).toEqual( + mode === "valid" + ? [] + : [ + "Slice range was derived from structural Mach-O facts because lipo omitted it.", + ], + ); + } + } finally { + await reader.close(); + await client.close(); + } + }, +); + +it("does not replace a parsed contradictory lipo range with structural fallback", async () => { + const root = await createTestTempDirectory("rea-lipo-known-mismatch-"); + const path = join(root, "fat"); + const thin = machoImage({ cpu: CPU.arm64 }); + await writeFile(path, fatImage([{ cpu: CPU.arm64, bytes: thin }])); + const reader = new MachOSliceArtifactReader( + path, + {}, + new NativeFixtureRunner({ + lipo: `architecture arm64\n cputype 16777228\n cpusubtype 0\n offset 8192\n size ${thin.length}\n align 2^12 (4096)\n`, + }), + ); + try { + await expect( + reader.entries()[Symbol.asyncIterator]().next(), + ).rejects.toMatchObject({ + reason: "integrity", + message: expect.stringContaining("disagrees with the Mach-O FAT table"), + }); + } finally { + await reader.close(); + } +}); + +it("refuses retained raw-field expansion as capacity rather than a malformed lipo record", async () => { + const root = await createTestTempDirectory("rea-lipo-field-capacity-"); + const path = join(root, "fat"); + await writeFile( + path, + fatImage([{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }]), + ); + const source = "\u0000".repeat( + Math.floor(NATIVE_COMMAND_OUTPUT_BUDGET_BYTES / 18) + 1, + ); + const stdout = `architecture arm64\n offset ${source}\n`; + expect(Buffer.byteLength(stdout)).toBeLessThan( + NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, + ); + const runner = new NativeFixtureRunner({ lipo: stdout }); + const captured = await runner.run("lipo", ["-detailed_info", path]); + if (!captured.ok) throw captured.error; + const stderr = "retained raw lipo diagnostic\n"; + const refused = parseLipoCapture( + { + ...captured.value, + stderr, + stderrBytes: Buffer.byteLength(stderr), + exitCode: 7, + signal: "SIGTERM", + }, + "list_architectures", + path, + ); + expect(refused.ok).toBe(false); + if (refused.ok) throw new Error("Expected lipo field retention refusal"); + expect(refused.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + capturedOutput: { + stdout, + stderr, + stdout_bytes: Buffer.byteLength(stdout), + stderr_bytes: Buffer.byteLength(stderr), + exit_code: 7, + signal: "SIGTERM", + truncated: false, + }, + }); + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + nativeMachoTarget(path), + ); + const reader = new MachOSliceArtifactReader(path, {}, runner); + try { + const result = await client.execute("list_architectures", {}); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); + const published: ArtifactEntry[] = []; + const failure: unknown = await (async () => { + for await (const entry of reader.entries()) published.push(entry); + })().catch((cause: unknown) => cause); + expect(failure).toMatchObject({ reason: "limit", boundary: "memory" }); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + if (!(failure instanceof ArtifactReaderFailure)) + throw new Error("Expected artifact capacity refusal"); + expect(failure.cause).toBeInstanceOf(LipoFieldRetentionError); + if (!(failure.cause instanceof LipoFieldRetentionError)) + throw new Error("Expected exact parser capacity cause"); + expect(failure.cause.requiredBytes).toBeGreaterThan( + NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, + ); + expect(published).toEqual([]); + expect(reader.provenance()).toEqual([ + { + tool: "lipo", + arguments: ["-detailed_info", "$ARTIFACT"], + tool_version: captured.value.toolVersion, + executable_sha256: captured.value.executableSha256, + exit_code: captured.value.exitCode, + effects: ["read"], + }, + ]); + } finally { + await reader.close(); + await client.close(); + } +});