mirror of
https://github.com/morluto/rea.git
synced 2026-11-10 17:30:32 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4091a40797 |
No files matched your search
@@ -20,11 +20,6 @@ it("reads a Python 3.9 reference-flagged PYZ table", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads marshal one-byte string tags as Latin-1", () => {
|
||||
for (const hex of ["6101000000e9", "4101000000e9", "7a01e9", "5a01e9"])
|
||||
expect(readMarshalSubset(bytes(hex))).toEqual({ ok: true, value: "é" });
|
||||
});
|
||||
|
||||
it("decodes CPython marshal Unicode with surrogatepass", () => {
|
||||
expect(
|
||||
readMarshalSubset(
|
||||
|
||||
@@ -168,8 +168,14 @@ const readAsciiBytes = (
|
||||
): Result<string, PyInstallerFailure> => {
|
||||
const bytes = readBytes(cursor, length);
|
||||
if (!bytes.ok) return bytes;
|
||||
// These tags use CPython's one-byte Unicode constructor, not an ASCII check.
|
||||
return ok(bytes.value.toString("latin1"));
|
||||
if (bytes.value.some((byte) => byte > 0x7f))
|
||||
return err(
|
||||
pyInstallerFailure(
|
||||
"format",
|
||||
"PYZ marshal ASCII string contains a non-ASCII byte",
|
||||
),
|
||||
);
|
||||
return ok(bytes.value.toString("ascii"));
|
||||
};
|
||||
|
||||
const readUtf8 = (
|
||||
|
||||
@@ -8,8 +8,7 @@ export const objcClassSymbol = (
|
||||
readonly metaclass: boolean;
|
||||
}
|
||||
| undefined => {
|
||||
const namespace = name.lastIndexOf("::");
|
||||
const leaf = namespace < 0 ? name : name.slice(namespace + 2);
|
||||
const leaf = objcSymbolLeaf(name);
|
||||
const match = /^_?(OBJC_(META)?CLASS_\$_([^\s]+))$/u.exec(leaf);
|
||||
if (match?.[1] === undefined || match[3] === undefined) return undefined;
|
||||
return {
|
||||
@@ -18,3 +17,9 @@ export const objcClassSymbol = (
|
||||
metaclass: match[2] !== undefined,
|
||||
};
|
||||
};
|
||||
|
||||
/** Preserve the exact runtime symbol leaf after a provider namespace. */
|
||||
export const objcSymbolLeaf = (name: string): string => {
|
||||
const namespace = name.lastIndexOf("::");
|
||||
return namespace < 0 ? name : name.slice(namespace + 2);
|
||||
};
|
||||
@@ -282,3 +282,23 @@ it("distinguishes protocol identity from protocol member contexts", () => {
|
||||
unclassified: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("selects exact Objective-C protocol symbol leaves", () => {
|
||||
const declarations = [
|
||||
"OBJC_PROTOCOL_$_Runnable",
|
||||
"_OBJC_PROTOCOL_$_Runnable",
|
||||
"__OBJC_PROTOCOL_$_Runnable",
|
||||
"Framework::__OBJC_PROTOCOL_$_Runnable",
|
||||
].map((name, index) => ({ address: `0x${index}`, name }));
|
||||
const otherNames = [
|
||||
"s__OBJC_PROTOCOL_$_Runnable_1000",
|
||||
"PTR__OBJC_PROTOCOL_$_Runnable_1000",
|
||||
"-[Example useOBJC_PROTOCOL_$_Runnable]",
|
||||
"__OBJC_PROTOCOL_REFERENCE_$_Runnable",
|
||||
"__OBJC_PROTOCOL_$_",
|
||||
].map((name, index) => ({ address: `0x${index + 4}`, name }));
|
||||
expect(discoverObjcProtocols([...otherNames, ...declarations])).toEqual({
|
||||
count: declarations.length,
|
||||
protocols: declarations,
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { objcClassSymbol } from "./objcClassSymbol.js";
|
||||
import { objcClassSymbol, objcSymbolLeaf } from "./objcClassSymbol.js";
|
||||
import type { JsonValue } from "./jsonValue.js";
|
||||
import type { AddressedName } from "./native/nativeInventory.js";
|
||||
import type {
|
||||
@@ -14,11 +14,8 @@ import {
|
||||
/** Local class records emitted by Apple's fragile Objective-C ABI. */
|
||||
const LEGACY_OBJC_CLASS_PREFIX = "l_OBJC_CLASS_" as const;
|
||||
|
||||
/**
|
||||
* Objective-C protocol declaration marker. `PROTOCOL_REFERENCE` slots
|
||||
* share the substring but are uses, not definitions.
|
||||
*/
|
||||
const OBJC_PROTOCOL_MARKER = "OBJC_PROTOCOL_$_" as const;
|
||||
/** Objective-C protocol declaration labels with their object-file prefixes. */
|
||||
const OBJC_PROTOCOL_SYMBOL = /^_{0,2}OBJC_PROTOCOL_\$_[^\s]+$/u;
|
||||
|
||||
export interface ObjcClassImportBindingScan {
|
||||
readonly bindings: readonly MachoImportBinding[];
|
||||
@@ -66,7 +63,7 @@ export const discoverObjcProtocols = (
|
||||
const protocols = uniqueByName(
|
||||
names.filter(
|
||||
({ name }) =>
|
||||
name.includes(OBJC_PROTOCOL_MARKER) ||
|
||||
OBJC_PROTOCOL_SYMBOL.test(objcSymbolLeaf(name)) ||
|
||||
classifySwiftSymbol(name)?.protocolIdentity === true,
|
||||
),
|
||||
);
|
||||
|
||||
@@ -407,30 +407,6 @@ it("charges long digits to the string budget and keeps large previews bounded",
|
||||
);
|
||||
});
|
||||
|
||||
it("reads marshal one-byte string tags as Latin-1 with Unicode identity", () => {
|
||||
for (const tag of [0x61, 0x41, 0x7a, 0x5a]) {
|
||||
const encoded = Buffer.concat([
|
||||
Buffer.from([tag]),
|
||||
tag === 0x61 || tag === 0x41 ? u32(1) : Buffer.from([1]),
|
||||
Buffer.from([0xe9]),
|
||||
]);
|
||||
const sameUnicode = marshalUnicode(Buffer.from("é", "utf8"));
|
||||
const set = Buffer.concat([
|
||||
Buffer.from([0x3c]),
|
||||
u32(2),
|
||||
encoded,
|
||||
sameUnicode,
|
||||
]);
|
||||
const report = root(parse(withConsts([encoded, set])));
|
||||
expect(report.consts[0]).toMatchObject({
|
||||
kind: "str",
|
||||
value: "é",
|
||||
bytes: 1,
|
||||
});
|
||||
expect(report.consts[1]).toEqual({ kind: "set", length: 1 });
|
||||
}
|
||||
});
|
||||
|
||||
it("decodes marshal strings with CPython surrogatepass rules", () => {
|
||||
const report = parse(
|
||||
withConsts([marshalUnicode(Buffer.from("efbbbf73eda080efbbbf", "hex"))]),
|
||||
@@ -1558,36 +1534,6 @@ it("hashes a long TYPE_REF tuple chain without recursing", () => {
|
||||
expect(pythonPycSchema.safeParse(report).success).toBe(true);
|
||||
});
|
||||
|
||||
it("hashes shallow TYPE_REF code chains without using the JavaScript stack", () => {
|
||||
const links = 3000;
|
||||
const codes = Array.from({ length: links }, (_, index) =>
|
||||
marshalCode("py311", {
|
||||
flagged: true,
|
||||
name: "f",
|
||||
consts: index === 0 ? [Buffer.from([0x4e])] : [marshalRef(index - 1)],
|
||||
}),
|
||||
);
|
||||
const bytes = Buffer.concat([
|
||||
pythonPycHeader(3531),
|
||||
marshalCode("py311", {
|
||||
constsObject: Buffer.concat([
|
||||
Buffer.from([0x28]),
|
||||
u32(links + 1),
|
||||
...codes,
|
||||
Buffer.from([0x3c]),
|
||||
u32(1),
|
||||
marshalRef(links - 1),
|
||||
]),
|
||||
}),
|
||||
]);
|
||||
const report = parseWith(bytes, { max_objects: 65_536 });
|
||||
expect(root(report).consts.at(-1)).toEqual({ kind: "set", length: 1 });
|
||||
expect(report.coverage).toMatchObject({
|
||||
status: "complete",
|
||||
code_objects: links + 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects code CPython treats as corrupt instead of complete coverage", () => {
|
||||
const file = (
|
||||
layout: "py37" | "py38" | "py311",
|
||||
|
||||
@@ -332,7 +332,7 @@ class MarshalReader {
|
||||
readonly #codeKeys = new WeakMap<CodeNode, string>();
|
||||
readonly #codeIdentities = new WeakMap<CodeNode, number>();
|
||||
readonly #codeEquality = new WeakMap<CodeNode, CodeEquality>();
|
||||
/** Completed Python hashability checks per code object. */
|
||||
/** Python hashability per code object. False while that check is in progress. */
|
||||
readonly #codeHashable = new WeakMap<CodeNode, boolean>();
|
||||
/** Tuples whose items are still being read; CPython cannot hash them yet. */
|
||||
readonly #readingTuples = new WeakSet<MarshalValue>();
|
||||
@@ -436,19 +436,13 @@ class MarshalReader {
|
||||
};
|
||||
case TYPE_INTERNED:
|
||||
case TYPE_UNICODE:
|
||||
return this.string(
|
||||
this.nonNegativeInt32("Marshal string length"),
|
||||
"utf8",
|
||||
);
|
||||
return this.utf8(this.nonNegativeInt32("Marshal string length"), false);
|
||||
case TYPE_ASCII:
|
||||
case TYPE_ASCII_INTERNED:
|
||||
return this.string(
|
||||
this.nonNegativeInt32("Marshal string length"),
|
||||
"latin1",
|
||||
);
|
||||
return this.utf8(this.nonNegativeInt32("Marshal string length"), true);
|
||||
case TYPE_SHORT_ASCII:
|
||||
case TYPE_SHORT_ASCII_INTERNED:
|
||||
return this.string(this.u8(), "latin1");
|
||||
return this.utf8(this.u8(), true);
|
||||
case TYPE_TUPLE:
|
||||
return this.sequenceValue(
|
||||
"tuple",
|
||||
@@ -1315,70 +1309,13 @@ class MarshalReader {
|
||||
#isCodeHashable(node: CodeNode): boolean {
|
||||
const known = this.#codeHashable.get(node);
|
||||
if (known !== undefined) return known;
|
||||
const frames: { value: MarshalValue; finish: boolean }[] = [
|
||||
{ value: { kind: "code", value: node }, finish: false },
|
||||
];
|
||||
const gray = new Set<MarshalValue | CodeNode>();
|
||||
const complete = new Set<MarshalValue>();
|
||||
const reject = (): false => {
|
||||
this.#codeHashable.set(node, false);
|
||||
return false;
|
||||
};
|
||||
while (frames.length > 0) {
|
||||
this.#takeKeyStep();
|
||||
const frame = frames.pop()!;
|
||||
const value = frame.value;
|
||||
const identity = value.kind === "code" ? value.value : value;
|
||||
if (frame.finish) {
|
||||
gray.delete(identity);
|
||||
if (value.kind === "code") {
|
||||
const material =
|
||||
this.#codeEquality.get(value.value) ??
|
||||
malformed("Code object is missing equality material.");
|
||||
// Child code checks are now complete, so the existing tuple key
|
||||
// walker preserves its equality and expansion budgets without recursion.
|
||||
const hashable =
|
||||
this.#equalityKey(material.constsTuple, "hash") !== null;
|
||||
this.#codeHashable.set(value.value, hashable);
|
||||
if (!hashable) return reject();
|
||||
} else complete.add(value);
|
||||
continue;
|
||||
}
|
||||
if (value.kind === "code") {
|
||||
const cached = this.#codeHashable.get(value.value);
|
||||
if (cached === false) return reject();
|
||||
if (cached === true) continue;
|
||||
} else {
|
||||
const cached = this.#keyCache.get(value);
|
||||
if (cached === null) return reject();
|
||||
if (cached !== undefined || complete.has(value)) continue;
|
||||
}
|
||||
if (
|
||||
gray.has(identity) ||
|
||||
this.#readingTuples.has(value) ||
|
||||
value.kind === "list" ||
|
||||
value.kind === "set" ||
|
||||
value.kind === "dict" ||
|
||||
value.kind === "null"
|
||||
)
|
||||
return reject();
|
||||
const children =
|
||||
value.kind === "code"
|
||||
? [
|
||||
(
|
||||
this.#codeEquality.get(value.value) ??
|
||||
malformed("Code object is missing equality material.")
|
||||
).constsTuple,
|
||||
]
|
||||
: value.kind === "tuple" || value.kind === "frozenset"
|
||||
? value.items
|
||||
: [];
|
||||
gray.add(identity);
|
||||
frames.push({ value, finish: true });
|
||||
for (let index = children.length - 1; index >= 0; index -= 1)
|
||||
frames.push({ value: children[index]!, finish: false });
|
||||
}
|
||||
return true;
|
||||
this.#codeHashable.set(node, false);
|
||||
const material =
|
||||
this.#codeEquality.get(node) ??
|
||||
malformed("Code object is missing equality material.");
|
||||
const hashable = this.#equalityKey(material.constsTuple, "hash") !== null;
|
||||
this.#codeHashable.set(node, hashable);
|
||||
return hashable;
|
||||
}
|
||||
|
||||
#codeIdentity(node: CodeNode): number {
|
||||
@@ -1555,20 +1492,17 @@ class MarshalReader {
|
||||
);
|
||||
}
|
||||
|
||||
string(length: number, encoding: "latin1" | "utf8"): MarshalValue {
|
||||
utf8(length: number, ascii: boolean): MarshalValue {
|
||||
const bytes = this.blob(length);
|
||||
if (encoding === "latin1") {
|
||||
// CPython reads the ASCII-tagged forms as one-byte Unicode, even when
|
||||
// high bytes appear. Identity must agree with the UTF-8 string form.
|
||||
const value = bytes.toString("latin1");
|
||||
if (ascii) {
|
||||
if (bytes.some((byte) => byte > 0x7f))
|
||||
malformed("Marshal ASCII string contains a non-ASCII byte.");
|
||||
return {
|
||||
kind: "str",
|
||||
value,
|
||||
value: bytes.toString("latin1"),
|
||||
previewEnd: Math.min(length, PYTHON_PYC_CONST_PREVIEW_LIMIT),
|
||||
bytes: length,
|
||||
raw: bytes.some((byte) => byte > 0x7f)
|
||||
? Buffer.from(value, "utf8")
|
||||
: bytes,
|
||||
raw: bytes,
|
||||
};
|
||||
}
|
||||
const decoded = decodeMarshalUtf8(bytes, PYTHON_PYC_CONST_PREVIEW_LIMIT);
|
||||
|
||||
Reference in new issue
Block a user