Compare commits

..
Author SHA1 Message Date
morluto 2eb48add14 fix(python): validate shared pyc header facts 2026-11-10 21:41:05 -05:00
18 changed files with 225 additions and 302 deletions

No files matched your search

@@ -66,6 +66,28 @@ const report: PythonPyc = {
],
};
it.each([
{
flags: 1,
hash_based: true,
source_hash_hex: "",
source_mtime: null,
source_size: null,
},
{ flags: 1 },
])("refuses inconsistent provider header facts %j", async (header) => {
const result = await new PythonPycService({
close: async () => {},
identity,
inspect: async () =>
ok({ ...report, header: { ...report.header, ...header } }),
}).inspect({ path: report.artifact.path });
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisOutputError" },
});
});
it("rejects a relative path and extra arguments before reading", async () => {
let calls = 0;
const service = new PythonPycService({
+20 -38
View File
@@ -1,9 +1,5 @@
import {
resolveMappedFileOffset,
type FileMapping,
type ZeroFillMapping,
} from "./virtualFileMapping.js";
import type {
BinaryArchitecture,
ElfFamilyMetadata,
HeaderArchitecture,
MipsElfMetadata,
@@ -275,14 +271,9 @@ const parseFatMachO = (
// family declared more than once, such as arm64 and arm64e, has no
// provider-neutral choice and is not selected.
const preferred = fatHostArchitecture(host);
const hostSlices = declarations.filter(
const hostSlice = declarations.find(
({ architecture }) => architecture === preferred,
);
if (hostSlices.length > 1)
return err(
`FAT binary declares several ${preferred} slices; no slice was selected`,
);
const hostSlice = hostSlices[0];
const unambiguous = declarations.filter(
({ architecture }) =>
declarations.filter((other) => other.architecture === architecture)
@@ -613,37 +604,28 @@ export const peRvaToFileOffset = (
sizeOfHeaders: number,
size = 1,
): number | undefined => {
const mappings: FileMapping[] = [
{ address: 0n, offset: 0, size: sizeOfHeaders },
];
const zeroFills: ZeroFillMapping[] = [];
for (let index = 0; index < numberOfSections; index += 1) {
const offset = index * 40;
if (rva < sizeOfHeaders && size <= sizeOfHeaders - rva) return rva;
for (let i = 0; i < numberOfSections; i++) {
const offset = i * 40;
const parsed = readPeSectionHeader(
sectionTable.subarray(offset, offset + 40),
);
if (!parsed.ok) return undefined;
const section = parsed.value;
const address = BigInt(section.virtual_address);
if (section.raw_size > 0)
mappings.push({
address,
offset: section.offset,
size: section.raw_size,
});
if (section.virtual_size > section.raw_size)
zeroFills.push({
address: address + BigInt(section.raw_size),
size: BigInt(section.virtual_size - section.raw_size),
});
if (!parsed.ok) break;
const {
virtual_address: virtualAddress,
raw_size: sizeOfRawData,
offset: pointerToRawData,
} = parsed.value;
if (
pointerToRawData > 0 &&
rva >= virtualAddress &&
rva - virtualAddress < sizeOfRawData &&
size <= sizeOfRawData - (rva - virtualAddress)
) {
return pointerToRawData + (rva - virtualAddress);
}
}
const mapped = resolveMappedFileOffset(
mappings,
BigInt(rva),
size,
zeroFills,
);
return mapped.ok ? mapped.value : undefined;
return undefined;
};
/** CodeView RVA admission uses the same bounded prefix as target file reads. */
+40
View File
@@ -3,9 +3,49 @@ import {
inspectPythonPycInputSchema,
pythonPycMarshalProfile,
PYTHON_PYC_MARSHAL_PROFILES,
pythonPycHeaderSchema,
} from "./pythonPyc.js";
import { pythonBytecodeIdentity } from "./pythonBytecodeIdentity.js";
const hashHeader = {
magic: 3531,
python_version: "3.12",
code_layout: "py311",
flags: 3,
hash_based: true,
check_source: true,
source_hash_hex: "0123456789abcdef",
source_mtime: null,
source_size: null,
};
it.each(["", "ab", "g".repeat(16)])(
"rejects an invalid source hash %j",
(source_hash_hex) => {
expect(
pythonPycHeaderSchema.safeParse({ ...hashHeader, source_hash_hex })
.success,
).toBe(false);
},
);
it.each([
{ hash_based: false },
{ check_source: false },
{ source_hash_hex: null },
{ source_mtime: 0 },
{ source_size: 0 },
])("keeps header facts consistent with the recorded flags: %j", (changed) => {
expect(
pythonPycHeaderSchema.safeParse({ ...hashHeader, ...changed }).success,
).toBe(false);
});
it("preserves unknown raw header flags and valid hash bytes", () => {
const header = { ...hashHeader, flags: 0x80000003 };
expect(pythonPycHeaderSchema.parse(header)).toEqual(header);
});
it("keeps pinned CPython identity separate from standalone marshal coverage", () => {
expect(pythonPycMarshalProfile(3394)).toEqual({
code_layout: "py37",
+47 -34
View File
@@ -205,17 +205,53 @@ const pythonCodeObjectSchema = z.strictObject({
});
export type PythonPycCodeObject = z.infer<typeof pythonCodeObjectSchema>;
export const pythonPycHeaderSchema = z.strictObject({
magic: z.number().int().min(0).max(0xffff),
python_version: z.string().min(1),
code_layout: z.enum(PYTHON_PYC_CODE_LAYOUTS),
flags: uint,
hash_based: z.boolean(),
check_source: z.boolean(),
source_hash_hex: z.string().nullable(),
source_mtime: uint.nullable(),
source_size: uint.nullable(),
});
export const pythonPycHeaderSchema = z
.strictObject({
magic: z.number().int().min(0).max(0xffff),
python_version: z.string().min(1),
code_layout: z.enum(PYTHON_PYC_CODE_LAYOUTS),
flags: uint,
hash_based: z.boolean(),
check_source: z.boolean(),
source_hash_hex: z
.string()
.regex(/^[0-9a-f]{16}$/u)
.nullable(),
source_mtime: uint.nullable(),
source_size: uint.nullable(),
})
.superRefine((header, context) => {
for (const [field, flag] of [
["hash_based", PYTHON_PYC_HASH_BASED],
["check_source", PYTHON_PYC_CHECK_SOURCE],
] as const)
if (header[field] !== ((header.flags & flag) !== 0))
context.addIssue({
code: "custom",
path: [field],
message: `${field} disagrees with its recorded header flag.`,
});
if (header.hash_based === (header.source_hash_hex === null))
context.addIssue({
code: "custom",
path: ["source_hash_hex"],
message:
"Hash-based pycs require an 8-byte source hash; others must omit it.",
});
if (header.hash_based === (header.source_mtime !== null))
context.addIssue({
code: "custom",
path: ["source_mtime"],
message: "Timestamp pycs require mtime; hash-based pycs must omit it.",
});
if (header.hash_based === (header.source_size !== null))
context.addIssue({
code: "custom",
path: ["source_size"],
message:
"Timestamp pycs require source_size; hash-based pycs must omit it.",
});
});
/** Bounded CPython 3.7–3.13 pyc observation; marshal is walked, bytecode is not decompiled. */
export const pythonPycSchema = z
@@ -253,29 +289,6 @@ export const pythonPycSchema = z
message: "Marshal byte count disagrees with the header-stripped file.",
});
}
if (report.header.hash_based === (report.header.source_hash_hex === null)) {
context.addIssue({
code: "custom",
path: ["header", "source_hash_hex"],
message:
"Hash-based pycs require an 8-byte source hash; others must omit it.",
});
}
if (report.header.hash_based === (report.header.source_mtime !== null)) {
context.addIssue({
code: "custom",
path: ["header", "source_mtime"],
message: "Timestamp pycs require mtime; hash-based pycs must omit it.",
});
}
if (report.header.hash_based === (report.header.source_size !== null)) {
context.addIssue({
code: "custom",
path: ["header", "source_size"],
message:
"Timestamp pycs require source_size; hash-based pycs must omit it.",
});
}
if (report.coverage.code_objects !== report.code_objects.length) {
context.addIssue({
code: "custom",
-101
View File
@@ -1,101 +0,0 @@
import { err, ok, type Result } from "./result.js";
/** One file-backed virtual range; addresses stay exact until mapped to file offsets. */
export interface FileMapping {
readonly address: bigint;
readonly offset: number;
readonly size: number;
}
/** Declared zero-initialized virtual memory with no corresponding file bytes. */
export interface ZeroFillMapping {
readonly address: bigint;
readonly size: bigint;
}
/** Why a declared virtual span does not establish one contiguous file span. */
export interface VirtualFileMappingFailure {
readonly reason: "invalid-size" | "unmapped" | "ambiguous";
readonly message: string;
}
/** Map a physically contiguous virtual span across consistent adjacent ranges. */
export const resolveMappedFileOffset = (
mappings: readonly FileMapping[],
address: bigint,
size: number,
zeroFills: readonly ZeroFillMapping[] = [],
): Result<number, VirtualFileMappingFailure> => {
if (!Number.isSafeInteger(size) || size < 0)
return err({
reason: "invalid-size",
message: "Virtual span size is invalid",
});
const end = address + BigInt(size);
let found: number | undefined;
const covered: { start: bigint; end: bigint }[] = [];
let whollyCovered = size === 0;
for (const mapping of mappings) {
const mappingEnd = mapping.address + BigInt(mapping.size);
const start = address > mapping.address ? address : mapping.address;
const overlapEnd = end < mappingEnd ? end : mappingEnd;
if (
size === 0
? address < mapping.address || address > mappingEnd
: start >= overlapEnd
)
continue;
const origin = mapping.offset + Number(address - mapping.address);
if (found !== undefined && found !== origin)
return err({
reason: "ambiguous",
message: "Virtual span has ambiguous or noncontiguous file mappings",
});
found = origin;
if (start === address && overlapEnd === end) whollyCovered = true;
else if (!whollyCovered) covered.push({ start, end: overlapEnd });
}
if (found === undefined)
return err({
reason: "unmapped",
message: "Virtual address or string range is not file mapped",
});
if (!whollyCovered) {
covered.sort((left, right) =>
left.start < right.start ? -1 : left.start > right.start ? 1 : 0,
);
let cursor = address;
for (const span of covered) {
if (span.start > cursor)
return err({
reason: "unmapped",
message: "Virtual address or string range is not file mapped",
});
if (span.end > cursor) cursor = span.end;
}
if (cursor < end)
return err({
reason: "unmapped",
message: "Virtual address or string range is not file mapped",
});
}
for (const zeroFill of zeroFills) {
if (spansOverlap(address, BigInt(size), zeroFill.address, zeroFill.size))
return err({
reason: "ambiguous",
message: "Virtual address has ambiguous file and zero-fill mappings",
});
}
return ok(found);
};
const spansOverlap = (
address: bigint,
size: bigint,
otherAddress: bigint,
otherSize: bigint,
): boolean =>
size > 0n &&
otherSize > 0n &&
address < otherAddress + otherSize &&
otherAddress < address + size;
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../domain/virtualFileMapping.js";
import {
BinaryFormatFailure,
BinaryResourceFailure,
@@ -10,6 +6,8 @@ import {
imageFileMappings,
isZeroFilledSpan,
type BinaryContainer,
type FileMapping,
type ZeroFillMapping,
} from "../native/metadata/BinaryContainer.js";
import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js";
import {
+1 -1
View File
@@ -1,9 +1,9 @@
import type { FileMapping } from "../domain/virtualFileMapping.js";
import {
BinaryFormatFailure,
BinaryReader,
mappedFileOffset,
type BinaryContainer,
type FileMapping,
} from "../native/metadata/BinaryContainer.js";
export interface GoBuildInfoSearchRange extends FileMapping {
+1 -1
View File
@@ -1,8 +1,8 @@
import type { FileMapping } from "../domain/virtualFileMapping.js";
import { expect, it } from "vitest";
import { GO_ANALYSIS_BUDGET_DEFAULTS } from "../domain/go/goBinary.js";
import type {
BinaryContainer,
FileMapping,
ImageSection,
LoadRegion,
} from "../native/metadata/BinaryContainer.js";
+1 -1
View File
@@ -1,4 +1,3 @@
import type { FileMapping } from "../domain/virtualFileMapping.js";
import { isUtf8 } from "node:buffer";
import {
GO_ANALYSIS_BUDGET_DEFAULTS,
@@ -12,6 +11,7 @@ import {
createFileMappingIndex,
imageFileMappings,
type BinaryContainer,
type FileMapping,
type FileMappingIndex,
type LoadRegion,
} from "../native/metadata/BinaryContainer.js";
+5 -5
View File
@@ -1,12 +1,12 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../domain/virtualFileMapping.js";
import {
readElfRelocationWritesSteps,
ElfPointerRelocationFailure,
} from "../native/metadata/ElfPointerRelocations.js";
import type { BinaryContainer } from "../native/metadata/BinaryContainer.js";
import type {
BinaryContainer,
FileMapping,
ZeroFillMapping,
} from "../native/metadata/BinaryContainer.js";
import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js";
/** Materialize the shared loader facts only for IL2CPP's existing private reader. */
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../domain/virtualFileMapping.js";
import {
machoDependencyCommand,
readMachoDylibCommandFacts,
@@ -19,6 +15,8 @@ import { createHash } from "node:crypto";
import {
BinaryFormatFailure,
createFileMappingIndex,
type ZeroFillMapping,
type FileMapping,
} from "./metadata/BinaryContainer.js";
import type { FixupCommands } from "./AppleMachoFixups.js";
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../../domain/virtualFileMapping.js";
import { readBinaryContainer } from "./readBinaryContainer.js";
import { expect, it } from "vitest";
import {
@@ -12,6 +8,8 @@ import {
mappedFileOffset,
readMappedCString,
type BinaryContainer,
type FileMapping,
type ZeroFillMapping,
} from "./BinaryContainer.js";
it("distinguishes a mapped string read allowance from malformed bytes", () => {
+72 -12
View File
@@ -1,8 +1,3 @@
import {
resolveMappedFileOffset,
type FileMapping,
type ZeroFillMapping,
} from "../../domain/virtualFileMapping.js";
import type { PeDataDirectory } from "../../domain/peOptionalHeader.js";
/** Distinguish corrupt recognized binaries from containers outside parser coverage. */
@@ -50,6 +45,19 @@ export class BinaryStringReadLimitFailure extends BinaryFormatFailure {
}
}
/** One file-backed virtual range; addresses stay exact until mapped to file offsets. */
export interface FileMapping {
readonly address: bigint;
readonly offset: number;
readonly size: number;
}
/** Declared zero-initialized virtual memory with no corresponding file bytes. */
export interface ZeroFillMapping {
readonly address: bigint;
readonly size: bigint;
}
/** Native loader region facts; language-specific fallback selection belongs to consumers. */
export interface LoadRegion {
readonly mapping: FileMapping;
@@ -256,19 +264,71 @@ export const declaredFileRange = (
};
};
/** Translate the domain mapping result into native parser failure classes. */
/** Map a physically contiguous virtual span across consistent adjacent ranges. */
export const mappedFileOffset = (
mappings: readonly FileMapping[],
address: bigint,
size: number,
zeroFills: readonly ZeroFillMapping[] = [],
): number => {
const mapped = resolveMappedFileOffset(mappings, address, size, zeroFills);
if (mapped.ok) return mapped.value;
const { reason, message } = mapped.error;
if (reason === "invalid-size")
throw new BinaryFormatFailure("malformed", message);
throw new BinaryMappingFailure(reason, message);
if (!Number.isSafeInteger(size) || size < 0)
throw new BinaryFormatFailure("malformed", "Virtual span size is invalid");
const end = address + BigInt(size);
let found: number | undefined;
const covered: { start: bigint; end: bigint }[] = [];
let whollyCovered = size === 0;
for (const mapping of mappings) {
const mappingEnd = mapping.address + BigInt(mapping.size);
const start = address > mapping.address ? address : mapping.address;
const overlapEnd = end < mappingEnd ? end : mappingEnd;
if (
size === 0
? address < mapping.address || address > mappingEnd
: start >= overlapEnd
)
continue;
const origin = mapping.offset + Number(address - mapping.address);
if (found !== undefined && found !== origin)
throw new BinaryMappingFailure(
"ambiguous",
"Virtual span has ambiguous or noncontiguous file mappings",
);
found = origin;
if (start === address && overlapEnd === end) whollyCovered = true;
else if (!whollyCovered) covered.push({ start, end: overlapEnd });
}
if (found === undefined)
throw new BinaryMappingFailure(
"unmapped",
"Virtual address or string range is not file mapped",
);
if (!whollyCovered) {
covered.sort((left, right) =>
left.start < right.start ? -1 : left.start > right.start ? 1 : 0,
);
let cursor = address;
for (const span of covered) {
if (span.start > cursor)
throw new BinaryMappingFailure(
"unmapped",
"Virtual address or string range is not file mapped",
);
if (span.end > cursor) cursor = span.end;
}
if (cursor < end)
throw new BinaryMappingFailure(
"unmapped",
"Virtual address or string range is not file mapped",
);
}
for (const zeroFill of zeroFills) {
if (spansOverlap(address, BigInt(size), zeroFill.address, zeroFill.size))
throw new BinaryMappingFailure(
"ambiguous",
"Virtual address has ambiguous file and zero-fill mappings",
);
}
return found;
};
/** Index of the last sorted value at or below `value`, or -1. */
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../../domain/virtualFileMapping.js";
import { admitElfLoadSegment } from "../../domain/elfProgramHeader.js";
import {
declaredFileRange,
@@ -9,8 +5,10 @@ import {
BinaryReader,
type BinaryContainer,
type ElfDynamicSegment,
type FileMapping,
type ImageSection,
type LoadRegion,
type ZeroFillMapping,
} from "./BinaryContainer.js";
import { elfArchitecture } from "../../domain/elfFixedHeader.js";
import { readElfDirectoryFromBuffer } from "./NativeSectionDirectory.js";
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../../domain/virtualFileMapping.js";
import { readElfDynamicEntry } from "../../domain/elfDynamicEntry.js";
import { readElfRelocationEntry } from "../../domain/elfRelocation.js";
import {
@@ -9,6 +5,8 @@ import {
createFileMappingIndex,
isZeroFilledSpan,
type ElfDynamicSegment,
type FileMapping,
type ZeroFillMapping,
} from "./BinaryContainer.js";
/** Shared loader interpretation; language adapters choose their public errors. */
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../../domain/virtualFileMapping.js";
import { decodeMachoFixedName } from "../AppleMachoNames.js";
import {
isMachoZeroFillSection,
@@ -17,8 +13,10 @@ import {
BinaryFormatFailure,
BinaryReader,
type BinaryContainer,
type FileMapping,
type ImageSection,
type LoadRegion,
type ZeroFillMapping,
} from "./BinaryContainer.js";
import {
MachoLoadCommandCursor,
+2 -4
View File
@@ -1,7 +1,3 @@
import type {
FileMapping,
ZeroFillMapping,
} from "../../domain/virtualFileMapping.js";
import { readPeCoffHeader } from "../../domain/peCoffHeader.js";
import { readPeOptionalHeader } from "../../domain/peMachine.js";
import { readPeSectionHeader } from "../../domain/peSectionHeader.js";
@@ -14,7 +10,9 @@ import {
BinaryFormatFailure,
BinaryReader,
type BinaryContainer,
type FileMapping,
type ImageSection,
type ZeroFillMapping,
} from "./BinaryContainer.js";
const peHeaderOffset = (reader: BinaryReader): number => {
+2 -81
View File
@@ -178,24 +178,6 @@ it.each([{ cpus: [0x0100000c] }, { cpus: [0x01000007, 0x0100000c] }])(
},
);
it("refuses ambiguous host subtypes even when a foreign family is selectable", async () => {
const directory = await createTestTempDirectory("rea-ambiguous-host-fat-");
const path = join(directory, "image");
const bytes = fat([0x0100000c, 0x0100000c, 0x01000007]);
bytes.writeUInt32BE(2, 8 + 20 + 4);
bytes.writeUInt32BE(2, bytes.readUInt32BE(8 + 20 + 8) + 8);
await writeFile(path, bytes);
expect(
await parseBinaryTarget(path, { hostArchitecture: "arm64" }),
).toMatchObject({
ok: false,
error: {
_tag: "BinaryTargetError",
reason: expect.stringContaining("several arm64 slices"),
},
});
});
it("reads a sparse FAT slice header beyond the bounded prefix", async () => {
const directory = await createTestTempDirectory("rea-sparse-fat-target-");
const path = join(directory, "image");
@@ -235,15 +217,11 @@ it("reads the complete bounded 128-entry FAT64 table beyond the prefix", async (
const table = Buffer.alloc(tableEnd);
table.writeUInt32BE(0xcafebabf, 0);
table.writeUInt32BE(entryCount, 4);
const thinHeader = machoImage({ cpu: CPU.arm64 }).subarray(0, 12);
for (let index = 0; index < entryCount; index += 1) {
const cpuType = index === 0 ? CPU.arm64.type : 0x100 + index;
const thinHeader = Buffer.from(
machoImage({ cpu: CPU.arm64 }).subarray(0, 12),
);
thinHeader.writeUInt32LE(cpuType, 4);
const entry = 8 + index * 32;
const sliceOffset = tableEnd + index * sliceSize;
table.writeUInt32BE(cpuType, entry);
table.writeUInt32BE(CPU.arm64.type, entry);
table.writeUInt32BE(CPU.arm64.subtype, entry + 4);
table.writeBigUInt64BE(BigInt(sliceOffset), entry + 8);
table.writeBigUInt64BE(BigInt(sliceSize), entry + 16);
@@ -1742,63 +1720,6 @@ it.each([
},
);
it.each([
{
label: "identical file aliases",
virtualAddress: 0x1000,
rawOffset: 512,
rawSize: 512,
expected: "first.pdb",
},
{
label: "conflicting file aliases",
virtualAddress: 0x1000,
rawOffset: 768,
rawSize: 512,
expected: undefined,
},
{
label: "independent sections",
virtualAddress: 0x2000,
rawOffset: 768,
rawSize: 512,
expected: "first.pdb",
},
{
label: "zero-fill aliases",
virtualAddress: 0x1000,
rawOffset: 0,
rawSize: 0,
expected: undefined,
},
])(
"preserves CodeView mapping certainty for $label",
async ({ virtualAddress, rawOffset, rawSize, expected }) => {
const directory = await createTestTempDirectory("rea-codeview-mapping-");
const path = join(directory, "symbols.exe");
const bytes = Buffer.alloc(1280);
pe(0x8664, 64, 0x0002, false, { pdbPath: "first.pdb" }).copy(bytes);
bytes.writeUInt16LE(2, 64 + 6);
const sectionTable = 64 + 24 + 240;
bytes.copy(bytes, sectionTable + 40, sectionTable, sectionTable + 40);
bytes.writeUInt32LE(virtualAddress, sectionTable + 40 + 12);
bytes.writeUInt32LE(rawOffset, sectionTable + 40 + 20);
bytes.writeUInt32LE(rawSize, sectionTable + 40 + 16);
bytes.copy(bytes, 768, 512, 600);
bytes.writeUInt32LE(796, 768 + 24);
bytes.write("other.pdb", 796 + 24);
await writeFile(path, bytes);
const resolved = await parseBinaryTarget(path);
if (!resolved.ok) throw resolved.error;
const buffered = parseExecutableHeader(bytes, "x64");
if (!buffered.ok) throw new Error(buffered.error);
for (const metadata of [resolved.value, buffered.value])
expect(metadata.format === "pe" ? metadata.codeView?.pdbPath : null).toBe(
expected,
);
},
);
describe("binary target I/O: PE CodeView header ranges", () => {
it.each(["RSDS", "NB10"] as const)(
"reads bounded %s header debug records and rejects unfinished names",