Compare commits

..
11 changed files with 182 additions and 299 deletions

No files matched your search

@@ -20,6 +20,11 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
]);
});
it("reads marshal one-byte string tags as Latin-1", () => {
for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
});
it("decodes CPython marshal Unicode with surrogatepass", () => {
expect(
readMarshalSubset(
+2 -8
View File
@@ -168,14 +168,8 @@ const readAsciiBytes = (
): Result<string, PyInstallerFailure> => {
const bytes = readBytes(cursor, length);
if (!bytes.ok) return bytes;
if (bytes.value.some((byte) => byte > 0x7f))
return err(
pyInstallerFailure(
"format",
"PYZ marshal ASCII string contains a non-ASCII byte",
),
);
return ok(bytes.value.toString("ascii"));
// These tags use CPython's one-byte Unicode constructor, not an ASCII check.
return ok(bytes.value.toString("latin1"));
};
const readUtf8 = (
@@ -202,9 +202,8 @@ describe("v1 signature block (PKCS#7 SignedData)", () => {
let nested: Uint8Array = Uint8Array.from([0x05, 0x00]);
for (let i = 0; i < 64; i += 1) nested = indefinite(0x30, nested);
expect(errorOf(nested)).toEqual({
kind: "budget-exceeded",
resource: "asn1-depth",
limit: 32,
kind: "malformed-pkcs7",
detail: "ASN.1 nesting exceeds 32 levels.",
});
});
});
@@ -412,42 +411,3 @@ it.each([0, 1])(
});
},
);
it("keeps a valid constructed CMS content depth refusal distinct from malformed input", () => {
let content = tlv(0x04, Uint8Array.of(1));
const block = () =>
signedDataBlock({
certificates: [LEAF_CERTIFICATE],
signerInfos: [signerInfo({ names: LEAF_CERTIFICATE })],
encapsulatedContent: content,
});
expect(decodeV1SignatureBlock(block()).ok).toBe(true);
for (let depth = 0; depth < 40; depth += 1)
content = indefinite(0x24, content);
expect(errorOf(block())).toEqual({
kind: "budget-exceeded",
resource: "asn1-depth",
limit: 32,
});
});
it("preserves the canonical OID byte allowance for a valid unknown digest algorithm", () => {
const algorithm = tlv(
0x06,
Uint8Array.from([0x2a, ...Array<number>(64).fill(0)]),
);
expect(
errorOf(
signedDataBlock({
certificates: [LEAF_CERTIFICATE],
signerInfos: [
signerInfo({
names: LEAF_CERTIFICATE,
digestAlgorithmOid: algorithm,
}),
],
digestAlgorithmOid: algorithm,
}),
),
).toEqual({ kind: "budget-exceeded", resource: "oid-bytes", limit: 64 });
});
+8 -26
View File
@@ -27,10 +27,7 @@ export type ApkV1BlockError =
readonly resource:
| "v1-block-certificates"
| "v1-signer-infos"
| "asn1-elements"
| "asn1-depth"
| "oid-bytes"
| "oid-arc-bytes";
| "asn1-elements";
readonly limit: number;
};
@@ -101,14 +98,11 @@ class ApkBerReader {
return this.#reader.read(at, limit, depth);
} catch (cause: unknown) {
if (!(cause instanceof Asn1BerFailure)) throw cause;
if (cause.constraint !== null)
if (cause.kind === "budget-exceeded")
throw new BlockFailure({
kind: "budget-exceeded",
resource:
cause.constraint.resource === "nodes"
? "asn1-elements"
: "asn1-depth",
limit: cause.constraint.limit,
resource: "asn1-elements",
limit: MAX_ASN1_ELEMENTS,
});
throw malformed(cause.message);
}
@@ -161,13 +155,6 @@ const expectOctetString = (
const oidText = (content: Uint8Array): string => {
const decoded = decodeAsn1Oid(content, { bytes: 64, arcBytes: 64 });
if (decoded.ok) return decoded.value;
if (decoded.error.kind === "byte-limit" || decoded.error.kind === "arc-limit")
throw new BlockFailure({
kind: "budget-exceeded",
resource:
decoded.error.kind === "byte-limit" ? "oid-bytes" : "oid-arc-bytes",
limit: decoded.error.limit,
});
if (decoded.error.kind === "truncated")
throw malformed("object identifier is truncated");
throw malformed(`object identifier of ${String(content.length)} bytes`);
@@ -201,13 +188,6 @@ const integerIdentityContents = (bytes: Uint8Array): Uint8Array => {
return start === 0 ? bytes : bytes.subarray(start);
};
/** Optional identity interpretation may remain unknown while raw certificate identity survives. */
const unavailableIdentityInterpretation = (cause: unknown): boolean =>
cause instanceof BlockFailure &&
(cause.error.kind === "malformed-pkcs7" ||
cause.error.resource === "oid-bytes" ||
cause.error.resource === "oid-arc-bytes");
/**
* RFC 5280 §7.1: RDN order matters; attributes in one SET do not. This
* comparison profile admits ASCII PrintableString/UTF8String for common
@@ -257,7 +237,8 @@ const directoryNameKey = (name: Asn1BerNode): string | null => {
}
return JSON.stringify(rdns);
} catch (cause) {
if (unavailableIdentityInterpretation(cause)) return null;
if (cause instanceof BlockFailure && cause.error.kind === "malformed-pkcs7")
return null;
throw cause;
}
};
@@ -317,7 +298,8 @@ const certificateIdentity = (
subjectKeyIdentifier,
};
} catch (cause: unknown) {
if (unavailableIdentityInterpretation(cause)) return null;
if (cause instanceof BlockFailure && cause.error.kind === "malformed-pkcs7")
return null;
throw cause;
}
};
+8 -14
View File
@@ -11,16 +11,11 @@ export interface Asn1BerNode {
readonly children: readonly Asn1BerNode[];
}
/** The fixed parser allowance that prevented examining more BER structure. */
export interface Asn1BerConstraint {
readonly resource: "nodes" | "depth";
readonly limit: number;
}
export class Asn1BerFailure extends Error {
constructor(
message: string,
readonly constraint: Asn1BerConstraint | null = null,
readonly kind: "malformed" | "budget-exceeded" = "malformed",
readonly limit: number | null = null,
) {
super(message);
this.name = "Asn1BerFailure";
@@ -33,9 +28,10 @@ interface Budget {
const fail = (
message: string,
constraint: Asn1BerConstraint | null = null,
kind: Asn1BerFailure["kind"] = "malformed",
limit: number | null = null,
): never => {
throw new Asn1BerFailure(message, constraint);
throw new Asn1BerFailure(message, kind, limit);
};
/**
@@ -55,14 +51,12 @@ export class Asn1BerReader {
read(offset = 0, end = this.bytes.length, depth = 0): Asn1BerNode {
if (depth > this.maxDepth)
return fail(`ASN.1 nesting exceeds ${String(this.maxDepth)} levels.`, {
resource: "depth",
limit: this.maxDepth,
});
return fail(`ASN.1 nesting exceeds ${String(this.maxDepth)} levels.`);
if (this.#budget.remaining <= 0)
return fail(
`ASN.1 value count exceeds the ${String(this.maxNodes)}-node budget.`,
{ resource: "nodes", limit: this.maxNodes },
"budget-exceeded",
this.maxNodes,
);
this.#budget.remaining -= 1;
if (offset >= end || offset + 2 > end)
+6 -19
View File
@@ -28,12 +28,7 @@ const OID_NAMES: Readonly<Record<string, string>> = {
/** A fixed per-payload decoder allowance and its limit. */
export interface PeCertificateDecodeAllowance {
readonly allowance:
| "asn1-nodes"
| "asn1-depth"
| "decoded-text"
| "oid-bytes"
| "oid-arc-bytes";
readonly allowance: "asn1-nodes" | "decoded-text";
readonly limit: number;
}
@@ -67,15 +62,9 @@ export const readAsn1 = (bytes: Buffer): Asn1BerNode => {
if (cause instanceof Asn1BerFailure)
throw new PeCertificateAsn1Failure(
cause.message,
cause.constraint === null
? null
: {
allowance:
cause.constraint.resource === "nodes"
? "asn1-nodes"
: "asn1-depth",
limit: cause.constraint.limit,
},
cause.kind === "budget-exceeded"
? { allowance: "asn1-nodes", limit: cause.limit ?? ASN1_NODE_BUDGET }
: null,
);
throw cause;
}
@@ -123,14 +112,12 @@ export const decodeOid = (content: Buffer): string => {
case "empty":
return fail("Empty OBJECT IDENTIFIER.");
case "byte-limit":
throw new PeCertificateAsn1Failure(
return fail(
`OBJECT IDENTIFIER exceeds ${String(decoded.error.limit)} bytes.`,
{ allowance: "oid-bytes", limit: decoded.error.limit },
);
case "arc-limit":
throw new PeCertificateAsn1Failure(
return fail(
`OBJECT IDENTIFIER arc exceeds ${String(decoded.error.limit)} bytes.`,
{ allowance: "oid-arc-bytes", limit: decoded.error.limit },
);
case "non-minimal":
return fail("Non-minimal OBJECT IDENTIFIER component.");
@@ -1,132 +0,0 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { repeatedCertificateSignedData } from "../../../tests/fixtures/peCertificateAllowance.js";
import {
indefinite,
tlv,
signedDataBlock,
signerInfo,
LEAF_CERTIFICATE,
} from "../../../tests/fixtures/apkSignature/v1SignatureBlock.js";
import { inspectPeCertificatesInputSchema } from "../../domain/native/peCertificates.js";
import {
AnalysisInputError,
AnalysisResourceConstraintError,
} from "../../domain/analysisErrorCore.js";
import { peCertificateFixture } from "./PeCertificates.fixture.js";
import { PeCertificatesProvider } from "./PeCertificatesProvider.js";
const encapsulated = (depth: number): Buffer => {
let content = tlv(0x04, Uint8Array.of(1));
for (let level = 0; level < depth; level += 1)
content = indefinite(0x24, content);
return Buffer.from(
signedDataBlock({
certificates: [LEAF_CERTIFICATE],
signerInfos: [signerInfo({ names: LEAF_CERTIFICATE })],
encapsulatedContent: content,
}),
);
};
const inspect = async (payload: Buffer) => {
const path = join(
await createTestTempDirectory("rea-pe-decoder-allowance-"),
"inert.exe",
);
await writeFile(path, peCertificateFixture(false, payload));
const provider = new PeCertificatesProvider();
try {
return await provider.inspect(
inspectPeCertificatesInputSchema.parse({ path }),
);
} finally {
await provider.close();
}
};
it.each([
{
name: "depth",
payload: () => encapsulated(40),
limits: { max_asn1_depth: 32 },
description: "32 nesting levels",
},
{
name: "nodes",
payload: () => repeatedCertificateSignedData(600),
limits: { max_asn1_values: 16_384 },
description: "16384 values",
},
{
name: "text",
payload: () => repeatedCertificateSignedData(350),
limits: { max_decoded_text_bytes: 65_536 },
description: "65536 bytes",
},
])(
"reports the fixed $name allowance with its own resource and recovery",
async ({ payload, limits, description }) => {
const result = await inspect(payload());
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected decoder refusal");
expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError);
if (!(result.error instanceof AnalysisResourceConstraintError))
throw result.error;
expect(result.error.reportedLimits).toEqual(limits);
expect(result.error.retryable).toBe(false);
expect(result.error.fixedAllowance?.name).toContain(description);
expect(result.error.fixedAllowance?.name).toContain("no runtime override");
},
);
it("accepts shallower valid BER and keeps malformed BER an input failure", async () => {
expect((await inspect(encapsulated(1))).ok).toBe(true);
const malformed = await inspect(Buffer.from([0x30, 0x03, 0x05]));
expect(malformed.ok).toBe(false);
if (malformed.ok) throw new Error("Expected malformed input refusal");
expect(malformed.error).toBeInstanceOf(AnalysisInputError);
});
it.each([
{
name: "bytes",
content: Buffer.from([0x2a, ...Array<number>(256).fill(0)]),
limits: { max_oid_bytes: 256 },
},
{
name: "arc bytes",
content: Buffer.from([0x2a, 0x81, ...Array<number>(31).fill(0x80), 0]),
limits: { max_oid_arc_bytes: 32 },
},
])(
"preserves the canonical OID $name allowance in a valid CMS algorithm",
async ({ content, limits }) => {
const algorithm = tlv(0x06, content);
const result = await inspect(
Buffer.from(
signedDataBlock({
certificates: [LEAF_CERTIFICATE],
signerInfos: [
signerInfo({
names: LEAF_CERTIFICATE,
digestAlgorithmOid: algorithm,
}),
],
digestAlgorithmOid: algorithm,
}),
),
);
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected OID allowance refusal");
expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError);
if (!(result.error instanceof AnalysisResourceConstraintError))
throw result.error;
expect(result.error.reportedLimits).toEqual(limits);
expect(result.error.retryable).toBe(false);
expect(result.error.fixedAllowance?.name).toContain("OBJECT IDENTIFIER");
expect(result.error.fixedAllowance?.name).toContain("no runtime override");
},
);
+11 -27
View File
@@ -105,40 +105,24 @@ const inputBudgetRemediation = (
const decoderAllowanceFailure = (
cause: PeCertificateAsn1Failure,
{ allowance, limit }: PeCertificateDecodeAllowance,
) => {
const selected = {
"asn1-nodes": {
limits: { max_asn1_values: limit },
description: `The PE certificate ASN.1 allowance of ${String(limit)} values per PKCS#7 payload (this release has no runtime override)`,
},
"asn1-depth": {
limits: { max_asn1_depth: limit },
description: `The PE certificate ASN.1 allowance of ${String(limit)} nesting levels per PKCS#7 payload (this release has no runtime override)`,
},
"decoded-text": {
limits: { max_decoded_text_bytes: limit },
description: `The PE certificate decoded-text allowance of ${String(limit)} bytes per PKCS#7 payload (this release has no runtime override)`,
},
"oid-bytes": {
limits: { max_oid_bytes: limit },
description: `The PE certificate OBJECT IDENTIFIER allowance of ${String(limit)} bytes per OID (this release has no runtime override)`,
},
"oid-arc-bytes": {
limits: { max_oid_arc_bytes: limit },
description: `The PE certificate OBJECT IDENTIFIER allowance of ${String(limit)} bytes per base-128 arc (this release has no runtime override)`,
},
}[allowance];
return new AnalysisResourceConstraintError(
) =>
new AnalysisResourceConstraintError(
OPERATION,
"memory",
`PKCS#7 signedData exceeds a fixed decoder allowance: ${cause.message}`,
selected.limits,
allowance === "asn1-nodes"
? { max_asn1_values: limit }
: { max_decoded_text_bytes: limit },
{
cause,
...fixedAllowanceOptions(selected.description, "parser"),
...fixedAllowanceOptions(
allowance === "asn1-nodes"
? `The PE certificate ASN.1 allowance of ${String(limit)} values per PKCS#7 payload (this release has no runtime override)`
: `The PE certificate decoded-text allowance of ${String(limit)} bytes per PKCS#7 payload (this release has no runtime override)`,
"parser",
),
},
);
};
const peCertificatesFailurePrimary = (
facts: ArtifactFilePrimaryFacts,
+54
View File
@@ -407,6 +407,30 @@ it("charges long digits to the string budget and keeps large previews bounded",
);
});
it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
const encoded = Buffer.concat([
Buffer.from([tag]),
tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
Buffer.from([0xe9]),
]);
const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
const set = Buffer.concat([
Buffer.from([0x3c]),
u32(2),
encoded,
sameUnicode,
]);
const report = root(parse(withConsts([encoded, set])));
expect(report.consts[0]).toMatchObject({
kind: "str",
value: "é",
bytes: 1,
});
expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
}
});
it("decodes marshal strings with CPython surrogatepass rules", () => {
const report = parse(
withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
@@ -1534,6 +1558,36 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
expect(pythonPycSchema.safeParse(report).success).toBe(true);
});
it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
const links = 3000;
const codes = Array.from({ length: links }, (_, index) =>
marshalCode("py311", {
flagged: true,
name: "f",
consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
}),
);
const bytes = Buffer.concat([
pythonPycHeader(3531),
marshalCode("py311", {
constsObject: Buffer.concat([
Buffer.from([0x28]),
u32(links + 1),
...codes,
Buffer.from([0x3c]),
u32(1),
marshalRef(links - 1),
]),
}),
]);
const report = parseWith(bytes, { max_objects: 65_536 });
expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
expect(report.coverage).toMatchObject({
status: "complete",
code_objects: links + 1,
});
});
it("rejects code CPython treats as corrupt instead of complete coverage", () => {
const file = (
layout: "py37" | "py38" | "py311",
+83 -17
View File
@@ -332,7 +332,7 @@ class MarshalReader {
readonly #codeKeys = new WeakMap<CodeNode, string>();
readonly #codeIdentities = new WeakMap<CodeNode, number>();
readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
/** Python hashability per code object. False while that check is in progress. */
/** Completed Python hashability checks per code object. */
readonly #codeHashable = new WeakMap<CodeNode, boolean>();
/** Tuples whose items are still being read; CPython cannot hash them yet. */
readonly #readingTuples = new WeakSet<MarshalValue>();
@@ -436,13 +436,19 @@ class MarshalReader {
};
case TYPE_INTERNED:
case TYPE_UNICODE:
return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
return this.string(
this.nonNegativeInt32("Marshal string length"),
"utf8",
);
case TYPE_ASCII:
case TYPE_ASCII_INTERNED:
return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
return this.string(
this.nonNegativeInt32("Marshal string length"),
"latin1",
);
case TYPE_SHORT_ASCII:
case TYPE_SHORT_ASCII_INTERNED:
return this.utf8(this.u8(), true);
return this.string(this.u8(), "latin1");
case TYPE_TUPLE:
return this.sequenceValue(
"tuple",
@@ -1309,13 +1315,70 @@ class MarshalReader {
#isCodeHashable(node: CodeNode): boolean {
const known = this.#codeHashable.get(node);
if (known !== undefined) return known;
this.#codeHashable.set(node, false);
const material =
this.#codeEquality.get(node) ??
malformed("Code object is missing equality material.");
const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
this.#codeHashable.set(node, hashable);
return hashable;
const frames: { value: MarshalValue; finish: boolean }[] = [
{ value: { kind: "code", value: node }, finish: false },
];
const gray = new Set<MarshalValue | CodeNode>();
const complete = new Set<MarshalValue>();
const reject = (): false => {
this.#codeHashable.set(node, false);
return false;
};
while (frames.length > 0) {
this.#takeKeyStep();
const frame = frames.pop()!;
const value = frame.value;
const identity = value.kind === "code" ? value.value : value;
if (frame.finish) {
gray.delete(identity);
if (value.kind === "code") {
const material =
this.#codeEquality.get(value.value) ??
malformed("Code object is missing equality material.");
// Child code checks are now complete, so the existing tuple key
// walker preserves its equality and expansion budgets without recursion.
const hashable =
this.#equalityKey(material.constsTuple, "hash") !== null;
this.#codeHashable.set(value.value, hashable);
if (!hashable) return reject();
} else complete.add(value);
continue;
}
if (value.kind === "code") {
const cached = this.#codeHashable.get(value.value);
if (cached === false) return reject();
if (cached === true) continue;
} else {
const cached = this.#keyCache.get(value);
if (cached === null) return reject();
if (cached !== undefined || complete.has(value)) continue;
}
if (
gray.has(identity) ||
this.#readingTuples.has(value) ||
value.kind === "list" ||
value.kind === "set" ||
value.kind === "dict" ||
value.kind === "null"
)
return reject();
const children =
value.kind === "code"
? [
(
this.#codeEquality.get(value.value) ??
malformed("Code object is missing equality material.")
).constsTuple,
]
: value.kind === "tuple" || value.kind === "frozenset"
? value.items
: [];
gray.add(identity);
frames.push({ value, finish: true });
for (let index = children.length - 1; index >= 0; index -= 1)
frames.push({ value: children[index]!, finish: false });
}
return true;
}
#codeIdentity(node: CodeNode): number {
@@ -1492,17 +1555,20 @@ class MarshalReader {
);
}
utf8(length: number, ascii: boolean): MarshalValue {
string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
const bytes = this.blob(length);
if (ascii) {
if (bytes.some((byte) => byte > 0x7f))
malformed("Marshal ASCII string contains a non-ASCII byte.");
if (encoding === "latin1") {
// CPython reads the ASCII-tagged forms as one-byte Unicode, even when
// high bytes appear. Identity must agree with the UTF-8 string form.
const value = bytes.toString("latin1");
return {
kind: "str",
value: bytes.toString("latin1"),
value,
previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
bytes: length,
raw: bytes,
raw: bytes.some((byte) => byte > 0x7f)
? Buffer.from(value, "utf8")
: bytes,
};
}
const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);
+3 -14
View File
@@ -230,15 +230,12 @@ export interface SignerInfoSpec {
* from the certificate named by `names`.
*/
readonly serial?: Uint8Array;
/** Complete digest algorithm OID TLV, including bounded-parser fixtures. */
readonly digestAlgorithmOid?: Uint8Array;
}
export const signerInfo = ({
names,
signature,
serial,
digestAlgorithmOid,
}: SignerInfoSpec): Uint8Array =>
tlv(
0x30,
@@ -247,7 +244,7 @@ export const signerInfo = ({
names,
serial === undefined ? undefined : tlv(0x02, serial),
),
tlv(0x30, digestAlgorithmOid ?? oid(OID_SHA256)),
algorithm(OID_SHA256),
algorithm(OID_ECDSA_SHA256),
signature ?? tlv(0x04, Uint8Array.from([1, 2, 3])),
);
@@ -257,8 +254,6 @@ export const signedDataBlock = (options: {
readonly certificates: readonly Uint8Array[];
readonly signerInfos: readonly Uint8Array[];
readonly contentType?: string;
readonly encapsulatedContent?: Uint8Array;
readonly digestAlgorithmOid?: Uint8Array;
}): Uint8Array =>
tlv(
0x30,
@@ -268,14 +263,8 @@ export const signedDataBlock = (options: {
tlv(
0x30,
integer(1),
tlv(0x31, tlv(0x30, options.digestAlgorithmOid ?? oid(OID_SHA256))),
tlv(
0x30,
oid("1.2.840.113549.1.7.1"),
...(options.encapsulatedContent === undefined
? []
: [tlv(0xa0, options.encapsulatedContent)]),
),
tlv(0x31, algorithm(OID_SHA256)),
tlv(0x30, oid("1.2.840.113549.1.7.1")),
tlv(0xa0, ...options.certificates),
tlv(0x31, ...options.signerInfos),
),