diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts index 7f742ff500..c2c15e1f05 100644 --- a/src/artifacts/PyInstallerArtifactReader.ts +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -422,13 +422,12 @@ export class PyInstallerArtifactReader implements ArtifactReader { this.#pyzStore.reserveImages( toc.reduce( (count, entry) => - count + - (entry.typeCode === "z" && entry.compressionFlag === 1 ? 1 : 0), + count + (entry.role === "pyz" && entry.compressionFlag === 1 ? 1 : 0), 0, ), ); for (const entry of toc) { - if (entry.typeCode !== "z") continue; + if (entry.role !== "pyz") continue; abortIfNeeded(signal); const spooled = entry.compressionFlag === 0 @@ -495,7 +494,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { for (const entry of toc) { if (members.length % 256 === 0) await setImmediate(); abortIfNeeded(signal); - const image = entry.typeCode === "z" ? images.get(entry) : undefined; + const image = entry.role === "pyz" ? images.get(entry) : undefined; if (image === undefined) { const bytecode = await this.#pyzStore.bytecodeHeader( handle, diff --git a/src/artifacts/pyinstaller/CArchiveLayout.ts b/src/artifacts/pyinstaller/CArchiveLayout.ts index a6ac959cb2..b1549d44c9 100644 --- a/src/artifacts/pyinstaller/CArchiveLayout.ts +++ b/src/artifacts/pyinstaller/CArchiveLayout.ts @@ -39,24 +39,49 @@ export interface PyInstallerCookie { readonly cookieOffset: number; } -/** One CArchive table entry. */ +type CArchiveEntryRole = + | "script" + | "module" + | "package" + | "runtime-option" + | "dependency" + | "symlink" + | "binary" + | "pyz" + | "other"; + +const CARCHIVE_ENTRY_ROLES: ReadonlyMap = new Map([ + ["s", "script"], + ["m", "module"], + ["M", "package"], + ["o", "runtime-option"], + ["d", "dependency"], + ["n", "symlink"], + ["b", "binary"], + ["z", "pyz"], +]); + +/** One CArchive table entry, retaining its source code separately from the parsed role. */ export interface CArchiveEntry { readonly index: number; readonly tocRecordOffset: number; readonly tocRecordLength: number; readonly name: string; readonly typeCode: string; + readonly role: CArchiveEntryRole; readonly offset: number; readonly compressedSize: number; readonly uncompressedSize: number; readonly compressionFlag: number; } -/** Runtime options and external dependencies are declarations, never local blobs. */ -export const isCArchiveDeclaration = ( - entry: CArchiveEntry, -): entry is CArchiveEntry & { readonly typeCode: "o" | "d" } => - entry.typeCode === "o" || entry.typeCode === "d"; +/** Role-defined declarations never own local payload bytes. */ +export const isCArchiveDeclaration = (entry: CArchiveEntry): boolean => + entry.role === "runtime-option" || entry.role === "dependency"; + +/** Importable Python modules and packages share headerless-code and marker semantics. */ +export const isCArchivePythonModule = (entry: CArchiveEntry): boolean => + entry.role === "module" || entry.role === "package"; /** True when a native executable has a structurally valid CArchive cookie. */ export const containsPyInstallerArchive = async ( @@ -173,12 +198,15 @@ export const parseCArchiveToc = ( const typeCode = bytes[offset + 17] ?? 0; if (typeCode < 0x20 || typeCode > 0x7e) return malformedEntry("CArchive type code is not a printable character"); + const sourceTypeCode = String.fromCharCode(typeCode); entries.push({ index: entries.length, tocRecordOffset: offset, tocRecordLength: entryLength, name: name.value, - typeCode: String.fromCharCode(typeCode), + typeCode: sourceTypeCode, + // Other printable codes keep generic-file behavior; compression is independent. + role: CARCHIVE_ENTRY_ROLES.get(sourceTypeCode) ?? "other", offset: bytes.readUInt32BE(offset + 4), compressedSize: bytes.readUInt32BE(offset + 8), uncompressedSize: bytes.readUInt32BE(offset + 12), diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts index 2e99508a83..9c5d653bb7 100644 --- a/src/artifacts/pyinstaller/CArchiveMembers.ts +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -1,6 +1,10 @@ import type { FileHandle } from "node:fs/promises"; import { copyArtifactEntry, type ArtifactEntry } from "../ArtifactReader.js"; -import { isCArchiveDeclaration, type CArchiveEntry } from "./CArchiveLayout.js"; +import { + isCArchiveDeclaration, + isCArchivePythonModule, + type CArchiveEntry, +} from "./CArchiveLayout.js"; import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; import { RECONSTRUCTED_PYTHON_HEADER_LIMITATION, @@ -54,7 +58,7 @@ export const hasEncryptionMarker = ( entries.some( (entry) => entry.name === CRYPTO_KEY_NAME && - (entry.typeCode === "m" || entry.typeCode === "M") && + isCArchivePythonModule(entry) && entry.uncompressedSize > 0, ); @@ -67,13 +71,13 @@ export const projectStoredMember = ( if (isCArchiveDeclaration(entry)) return { entry: projectEntry({ - path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`, + path: `__pyinstaller__/${entry.role === "runtime-option" ? "runtime-options" : "dependencies"}/${String(entry.index)}`, kind: "metadata", declaredSize: entry.uncompressedSize, compressedSize: entry.compressedSize, byteOffset: null, limitations: - entry.typeCode === "d" + entry.role === "dependency" ? [ "Multipackage dependency bytes belong to another executable and remain unresolved.", ] @@ -85,7 +89,7 @@ export const projectStoredMember = ( }; const { header } = bytecode; const path = canonicalMemberPath(entry.name, pathDialect); - const symlink = entry.typeCode === "n"; + const symlink = entry.role === "symlink"; const limitations: string[] = []; if (symlink) limitations.push("PyInstaller symlink target was not read."); if (header !== undefined) @@ -98,7 +102,7 @@ export const projectStoredMember = ( kind: symlink ? "symlink" : "file", declaredSize, compressedSize: entry.compressedSize, - executable: entry.typeCode === "b", + executable: entry.role === "binary", byteOffset: sourceByteOffset(entry, archiveStart), limitations, adapterKey: `carchive:${String(entry.index)}:${entry.name}`, diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 63b178d66c..352b54c72d 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -1627,3 +1627,129 @@ it.each([ ).toBe("contains"); }, ); + +const mixedCArchiveRoleFixture = () => { + const body = Buffer.from("not a marshal code prefix"); + const pyz = pyzArchive({ + magic: PYTHON_39_MAGIC, + modules: [{ name: "child", typeCode: 0, body }], + }); + const members: CArchiveFixtureMember[] = [ + { + name: "runtime value", + typeCode: "o", + payload: Buffer.alloc(0), + compress: false, + }, + { + name: "other-app:assets/data.bin", + typeCode: "d", + payload: Buffer.alloc(0), + compress: false, + }, + { name: "app/script", typeCode: "s", payload: body, compress: true }, + { name: "app/module", typeCode: "m", payload: body, compress: true }, + { name: "app/package", typeCode: "M", payload: body, compress: false }, + { name: "bin/runner", typeCode: "b", payload: body, compress: false }, + { + name: "link", + typeCode: "n", + payload: Buffer.from("app/module"), + compress: false, + }, + { name: "nested/PYZ.pyz", typeCode: "z", payload: pyz, compress: false }, + { name: "stored.pyz", typeCode: "Z", payload: pyz, compress: true }, + ]; + return { + body, + pyz, + members, + archive: cArchiveExecutable({ layout: "2.1", pythonVersion: 309, members }), + }; +}; + +it("keeps mixed CArchive roles distinct across declarations, bytecode, and stored PYZ lookalikes", async () => { + const fixture = mixedCArchiveRoleFixture(); + const path = await writeArchive("mixed-roles", fixture.archive); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + expect(reader.formatMetadata()).toMatchObject({ + entry_count: 9, + runtime_option_count: 1, + external_dependency_count: 1, + }); + const expected = [ + { + path: "__pyinstaller__/runtime-options/0", + type: "o", + kind: "metadata", + bytecode: "not-bytecode", + }, + { + path: "__pyinstaller__/dependencies/1", + type: "d", + kind: "metadata", + bytecode: "not-bytecode", + }, + { path: "app/script", type: "s", kind: "file", bytecode: "derived" }, + { path: "app/module", type: "m", kind: "file", bytecode: "unknown" }, + { path: "app/package", type: "M", kind: "file", bytecode: "unknown" }, + { path: "bin/runner", type: "b", kind: "file", bytecode: "not-bytecode" }, + { path: "link", type: "n", kind: "symlink", bytecode: "not-bytecode" }, + { + path: "nested/PYZ.pyz", + type: "z", + kind: "directory", + bytecode: "not-bytecode", + }, + { path: "stored.pyz", type: "Z", kind: "file", bytecode: "not-bytecode" }, + ]; + for (const [index, row] of expected.entries()) { + const entry = entries.find((entry) => entry.path === row.path); + if (entry === undefined) + throw new Error(`Missing role fixture ${row.type}`); + expect(entry).toMatchObject({ + kind: row.kind, + executable: row.type === "b", + sourcePath: fixture.members[index]?.name, + formatMetadata: { type_code: row.type, bytecode_header: row.bytecode }, + }); + if (entry.kind === "file") { + const original = row.type === "Z" ? fixture.pyz : fixture.body; + expect( + (await buffer(await reader.open(entry))).equals( + row.type === "s" ? derived(original) : original, + ), + ).toBe(true); + } + } + const child = entries.find( + (entry) => entry.path === "nested/PYZ.pyz/child.pyc", + ); + if (child === undefined) + throw new Error("Missing actual nested PYZ module"); + expect( + (await buffer(await reader.open(child))).equals(derived(fixture.body)), + ).toBe(true); + expect( + entries.filter((entry) => entry.path.startsWith("stored.pyz/")), + ).toEqual([]); + expect( + entries.find((entry) => entry.path === "stored.pyz")?.formatMetadata, + ).toMatchObject({ + pyz_status: "not-pyz", + type_code: "Z", + }); + expect( + entries.find((entry) => entry.path === "nested/PYZ.pyz")?.formatMetadata, + ).toMatchObject({ + pyz_status: "decoded", + pyz_python_version: "3.9", + pyz_entry_count: 1, + }); + } finally { + await reader.close(); + } +}); diff --git a/src/artifacts/pyinstaller/PyInstallerMetadata.ts b/src/artifacts/pyinstaller/PyInstallerMetadata.ts index f452fe185c..e612120ea6 100644 --- a/src/artifacts/pyinstaller/PyInstallerMetadata.ts +++ b/src/artifacts/pyinstaller/PyInstallerMetadata.ts @@ -2,7 +2,11 @@ import type { PyInstallerMetadata, PyInstallerEntryMetadata, } from "../../domain/artifactFormatMetadata.js"; -import type { CArchiveEntry, PyInstallerCookie } from "./CArchiveLayout.js"; +import { + isCArchivePythonModule, + type CArchiveEntry, + type PyInstallerCookie, +} from "./CArchiveLayout.js"; import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; const cookiePythonVersion = (value: number) => @@ -27,10 +31,12 @@ export const cookieMetadata = ( toc_offset: cookie.tocOffset, toc_size: cookie.tocLength, entry_count: entries.length, - runtime_option_count: entries.filter((entry) => entry.typeCode === "o") - .length, - external_dependency_count: entries.filter((entry) => entry.typeCode === "d") - .length, + runtime_option_count: entries.filter( + (entry) => entry.role === "runtime-option", + ).length, + external_dependency_count: entries.filter( + (entry) => entry.role === "dependency", + ).length, python_version_raw: cookie.pythonVersion, python_major: version?.major ?? null, python_minor: version?.minor ?? null, @@ -108,10 +114,10 @@ export const cArchiveEntryMetadata = ( relative_offset: entry.offset, compression_flag: entry.compressionFlag === 1 ? 1 : 0, }, - ["s", "m", "M"].includes(entry.typeCode), + entry.role === "script" || isCArchivePythonModule(entry), header, ), - ...(entry.typeCode === "d" ? dependencyMetadata(entry.name) : {}), + ...(entry.role === "dependency" ? dependencyMetadata(entry.name) : {}), pyz_status: archive?.kind === "entries" ? "decoded" diff --git a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts index cfc22b2425..60ac6d65de 100644 --- a/src/artifacts/pyinstaller/PyInstallerPyzStore.ts +++ b/src/artifacts/pyinstaller/PyInstallerPyzStore.ts @@ -22,7 +22,10 @@ import { pyInstallerMemberStream, readPyInstallerMemberPrefix, } from "./PyInstallerMemberStream.js"; -import type { CArchiveEntry } from "./CArchiveLayout.js"; +import { + isCArchivePythonModule, + type CArchiveEntry, +} from "./CArchiveLayout.js"; import type { PyInstallerMember } from "./CArchiveMembers.js"; import { artifactFileAdmissionFailure } from "../ArtifactFileAdmission.js"; import { @@ -74,9 +77,9 @@ export class PyInstallerPyzStore { ): Promise { const unchanged = { header: undefined, limitation: undefined }; if (magic === undefined) return unchanged; - if (entry.typeCode === "s") + if (entry.role === "script") return { header: reconstructedPycHeader(magic), limitation: undefined }; - if (entry.typeCode !== "m" && entry.typeCode !== "M") return unchanged; + if (!isCArchivePythonModule(entry)) return unchanged; const prefix = await readPyInstallerMemberPrefix( async (active) => (