From 4b985e5021f578e6ef0bc2a412abe9cba6bdaca1 Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Sat, 10 Oct 2026 06:17:40 -0700 Subject: [PATCH] fix(artifacts): take nested container families from bytes (#1553) Inventory gave nested members a ZIP or ASAR family from their path suffix. Byte classification replaced it only for ZIP magic, so text named `.apk` or `.asar` stayed a container, and encrypted entries and symlinks kept the suffix guess. A filesystem `.asar` without an ASAR header failed the whole inventory as a malformed archive. Nested ZIP families now follow ZIP magic and ASAR follows its header pickle, shared with root classification. An unread member named like a container has an unknown format and a limitation naming the suffix. ASAR expansion requires the header. The Android projection adds a limitation when no root AndroidManifest.xml was observed, and its rejection names the observed root format. Fixes #1491 Co-authored-by: Claude Opus 5.5 --- docs/android-analysis.md | 4 + .../inventory/ArtifactGraphConstruction.ts | 114 +++++++++++++----- src/artifacts/inventory/classify.ts | 31 +---- src/artifacts/inventory/scanReader.ts | 11 +- src/domain/android/androidApplication.ts | 9 +- .../asarInventoryContainment.test.ts | 43 ++++++- .../boundary/filesystem/zipDetection.test.ts | 79 +++++++++++- tests/composition/android/application.test.ts | 39 +++++- 8 files changed, 263 insertions(+), 67 deletions(-) diff --git a/docs/android-analysis.md b/docs/android-analysis.md index 849aeb2b1..1ef5e7f40 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -129,6 +129,10 @@ signature validation, full resource-table semantics, native-library analysis or Android runtime capture. An `.aab` suffix is a ZIP archive, not an APK, so Android tools and the application graph do not treat it as a standalone APK. Existing artifact inventory/extraction tools can supply archive evidence. +Inventory records an APK root only for ZIP bytes whose name ends in `.apk`. +The projection adds a limitation when no root `AndroidManifest.xml` was +observed, because such a ZIP may not be an APK. Nested members take a ZIP or +ASAR family from their bytes; an unread member keeps an unknown format. Inventory keeps ZIP names that differ only in case and records that a case-insensitive destination cannot store both spellings. Extraction fails only when that destination directory cannot keep both names. Project that diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index 2b85cd3a3..4868b9fab 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -15,6 +15,7 @@ import type { import { hasZipSignature, zipPackageFormatForPath, + type ZipPackageFormat, } from "../../domain/zipPackageFormat.js"; import { mzWindowsHeaderOffset, parseDosMzHeader } from "../../domain/dosMz.js"; @@ -47,37 +48,66 @@ export const indexOccurrencesByPath = ( return byPath; }; +/** + * Create an occurrence before its bytes are read. A container suffix is not a + * container family until the bytes are classified, so a member named like a + * container starts with an unknown format. + */ export const createOccurrence = ( entry: ArtifactEntry, path: string, parent: string | null, -): MutableOccurrence => ({ - occurrence_id: `occ_${digestCanonicalValue({ path, kind: entry.kind }, "Artifact")}`, - artifact_id: null, - parent_occurrence_id: parent, - logical_path: path, - entry_kind: entry.kind, - artifact_kind: +): MutableOccurrence => { + const role = entry.kind === "directory" - ? path.toLowerCase().endsWith(".framework") - ? "framework" - : "container" - : classifyArtifactPath(path).kind, - artifact_format: - entry.kind === "directory" - ? "directory" - : classifyArtifactPath(path).format, - declared_size: entry.declaredSize, - compressed_size: entry.compressedSize, - executable: entry.executable, - encrypted: entry.encrypted, - hash_status: entry.encrypted ? "unavailable" : "not-hashed", - source_location: - entry.byteOffset === null || entry.declaredSize === null - ? null - : { offset: entry.byteOffset, length: entry.declaredSize }, - limitations: [...entry.limitations], -}); + ? { + kind: path.toLowerCase().endsWith(".framework") + ? ("framework" as const) + : ("container" as const), + format: "directory" as const, + } + : containerSuffixFamily(path) === undefined + ? classifyArtifactPath(path) + : ({ kind: "unknown", format: "unknown" } as const); + return { + occurrence_id: `occ_${digestCanonicalValue({ path, kind: entry.kind }, "Artifact")}`, + artifact_id: null, + parent_occurrence_id: parent, + logical_path: path, + entry_kind: entry.kind, + artifact_kind: role.kind, + artifact_format: role.format, + declared_size: entry.declaredSize, + compressed_size: entry.compressedSize, + executable: entry.executable, + encrypted: entry.encrypted, + hash_status: entry.encrypted ? "unavailable" : "not-hashed", + source_location: + entry.byteOffset === null || entry.declaredSize === null + ? null + : { offset: entry.byteOffset, length: entry.declaredSize }, + limitations: [...entry.limitations], + }; +}; + +/** Record that an unread member's container suffix was not checked against bytes. */ +export const noteUnreadContainerSuffix = ( + occurrence: MutableOccurrence, +): void => { + if (occurrence.entry_kind === "directory") return; + const family = containerSuffixFamily(occurrence.logical_path); + if (family === undefined) return; + occurrence.limitations.push( + `The path suffix suggests ${family.toUpperCase()}, but these bytes were not read, so the container family is unknown.`, + ); +}; + +const containerSuffixFamily = ( + path: string, +): "asar" | ZipPackageFormat | undefined => { + const lower = path.toLowerCase(); + return lower.endsWith(".asar") ? "asar" : zipPackageFormatForPath(lower); +}; /** Compare exact child names independently of host locale and traversal order. */ export const materializeDirectoryNodes = ( @@ -284,6 +314,10 @@ export const nearestParent = ( return undefined; }; +/** + * Assign a path role hint. Container families are not path roles: ZIP and + * ASAR families are chosen only after their bytes have been classified. + */ export const classifyArtifactPath = ( path: string, ): { @@ -295,10 +329,6 @@ export const classifyArtifactPath = ( return { kind: "source-map", format: "source-map" }; if (/\.(?:m?js|cjs)$/u.test(lower)) return { kind: "javascript", format: "javascript-bundle" }; - if (lower.endsWith(".asar")) return { kind: "container", format: "asar" }; - const archiveFormat = zipPackageFormatForPath(lower); - if (archiveFormat !== undefined) - return { kind: "container", format: archiveFormat }; if (/\.framework(?:\/|$)/u.test(lower)) return { kind: "framework", format: "file" }; if (/\.(?:node|dylib|so)$/u.test(lower)) @@ -321,7 +351,29 @@ export const classifyArtifactContent = ( readonly kind: ArtifactOccurrence["artifact_kind"]; readonly format: ArtifactOccurrence["artifact_format"]; } => { - return artifactRoleForFormat(path, classifyArtifactBytes(prefix, fileSize)); + const format = classifyArtifactBytes(prefix, fileSize); + if ( + format === "file" && + path.toLowerCase().endsWith(".asar") && + hasAsarHeader(prefix) + ) + return { kind: "container", format: "asar" }; + return artifactRoleForFormat(path, format); +}; + +/** + * Electron ASAR starts with a size pickle whose payload is one uint32, then + * a header pickle whose payload begins with a JSON object. + */ +export const hasAsarHeader = (prefix: Buffer): boolean => { + if (prefix.length < 17 || prefix.readUInt32LE(0) !== 4) return false; + const headerSize = prefix.readUInt32LE(4); + const payloadSize = prefix.readUInt32LE(8); + if (headerSize < 16 || payloadSize !== headerSize - 4) return false; + const stringLength = prefix.readUInt32LE(12); + return ( + stringLength >= 2 && stringLength <= payloadSize - 4 && prefix[16] === 0x7b + ); }; const artifactRoleForFormat = ( diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index d0c8dfef8..fe29c6057 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -6,10 +6,7 @@ import { import type { FileHandle } from "node:fs/promises"; import type { Stats } from "node:fs"; -import { - classifyArtifactBytes, - classifyArtifactContent, -} from "./ArtifactGraphConstruction.js"; +import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js"; import type { ArtifactOccurrence } from "../../domain/artifactGraph.js"; import { @@ -78,35 +75,15 @@ const classifyRootFormat = async ( const prefix = magic.subarray(0, observed.bytesRead); const lower = path.toLowerCase(); if (hasZipSignature(prefix)) return zipPackageFormatForPath(lower) ?? "zip"; - if (lower.endsWith(".asar") && hasAsarHeader(prefix)) return "asar"; if ( lower.endsWith(".pkg") && prefix.subarray(0, 4).toString("ascii") === "xar!" ) return "pkg"; if (lower.endsWith(".dmg") && (await hasKolyTrailer(handle))) return "dmg"; - const size = (await handle.stat()).size; - const format = classifyArtifactContent(path, prefix, size).format; - // Path suffixes stay role hints for nested members. A root whose container - // magic did not match is classified from the bytes that were read. - if (format === "asar" || format === "pkg" || format === "dmg") - return classifyArtifactBytes(prefix, size); - return format; -}; - -/** - * Electron ASAR starts with a size pickle whose payload is one uint32, then - * a header pickle whose payload begins with a JSON object. - */ -const hasAsarHeader = (prefix: Buffer): boolean => { - if (prefix.length < 16 || prefix.readUInt32LE(0) !== 4) return false; - const headerSize = prefix.readUInt32LE(4); - const payloadSize = prefix.readUInt32LE(8); - if (headerSize < 16 || payloadSize !== headerSize - 4) return false; - const stringLength = prefix.readUInt32LE(12); - return ( - stringLength >= 2 && stringLength <= payloadSize - 4 && prefix[16] === 0x7b - ); + // ASAR is chosen from its header pickle; any other suffix is a role hint. + return classifyArtifactContent(path, prefix, (await handle.stat()).size) + .format; }; const hasKolyTrailer = async (handle: FileHandle): Promise => { diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index 89b1b628e..a68f4030e 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -17,6 +17,7 @@ import { classifyArtifactBytes, createArtifactNode, createOccurrence, + noteUnreadContainerSuffix, nearestParent, type MutableOccurrence, } from "./ArtifactGraphConstruction.js"; @@ -161,7 +162,8 @@ const visitArtifactEntries = async ( occurrence.hash_status = "unavailable"; occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION); } - if (digested !== undefined) { + if (digested === undefined) noteUnreadContainerSuffix(occurrence); + else { context.nodes.set(digested.node.artifact_id, digested.node); occurrence.artifact_id = digested.node.artifact_id; occurrence.artifact_kind = digested.classification.kind; @@ -176,7 +178,12 @@ const visitArtifactEntries = async ( } context.occurrences.push(occurrence); context.occurrenceByPath.set(logicalPath, occurrence); - if (expandableAsar && digested !== undefined && !digested.mismatched) { + if ( + expandableAsar && + digested !== undefined && + digested.classification.format === "asar" && + !digested.mismatched + ) { const nested = new AsarArtifactReader(entry.adapterKey); context.ownedReaders.push(nested); await nested.prepareContainer(digested.node.sha256, context.signal); diff --git a/src/domain/android/androidApplication.ts b/src/domain/android/androidApplication.ts index 67b58499f..5d6e73469 100644 --- a/src/domain/android/androidApplication.ts +++ b/src/domain/android/androidApplication.ts @@ -91,7 +91,9 @@ export const projectAndroidApplication = ( parsed.inventory_evidence, ); if (inventory.manifest.root_format !== "apk") - throw new TypeError("Android application projection requires APK Evidence"); + throw new TypeError( + `Android application projection requires APK inventory Evidence; the inventory root format is ${inventory.manifest.root_format}. APK is chosen for a ZIP archive whose name ends in .apk.`, + ); const nodes = new Map( inventory.nodes.map((node) => [node.artifact_id, node]), ); @@ -128,6 +130,11 @@ export const projectAndroidApplication = ( "Bridge candidates are path-based hypotheses, not decoded JNI declarations or observed runtime calls.", "A bridge basis is inferred from the native path and is repeated for every managed component.", "ZIP inventory cannot see the APK Signing Block, so an empty signing array does not mean the APK is unsigned.", + ...(all.some(({ path }) => path === "AndroidManifest.xml") + ? [] + : [ + "No root AndroidManifest.xml occurrence was observed. The APK family comes from the .apk suffix of a ZIP archive, so these bytes may not be an APK.", + ]), ...(bridgeProjection.coverage.status === "partial" ? [ `Bridge candidate pairs exceeded the projection safety budget; ${bridgeProjection.coverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, diff --git a/tests/boundary/filesystem/asarInventoryContainment.test.ts b/tests/boundary/filesystem/asarInventoryContainment.test.ts index 0f4b6bab9..5251a30e9 100644 --- a/tests/boundary/filesystem/asarInventoryContainment.test.ts +++ b/tests/boundary/filesystem/asarInventoryContainment.test.ts @@ -154,9 +154,23 @@ describe("unexpanded ASAR inventory controls", () => { "resources/encrypted.asar": "resources", "resources/ordinary.asar": "resources", }); + expect(artifactOccurrenceAt(observed, "resources/app.asar")).toMatchObject({ + artifact_kind: "container", + artifact_format: "asar", + }); expect( - artifactOccurrenceAt(observed, "resources/encrypted.asar").hash_status, - ).toBe("unavailable"); + artifactOccurrenceAt(observed, "resources/ordinary.asar"), + ).toMatchObject({ artifact_kind: "resource", artifact_format: "file" }); + expect( + artifactOccurrenceAt(observed, "resources/encrypted.asar"), + ).toMatchObject({ + artifact_kind: "unknown", + artifact_format: "unknown", + hash_status: "unavailable", + limitations: [ + "The path suffix suggests ASAR, but these bytes were not read, so the container family is unknown.", + ], + }); }); it("keeps ASAR-named directories as directory parents", async () => { @@ -195,9 +209,30 @@ describe("unexpanded ASAR inventory controls", () => { }, ); - it("keeps malformed filesystem archives rejected", async () => { - const root = await createTestTempDirectory("rea-malformed-asar-"); + it("records filesystem files without an ASAR header as ordinary bytes", async () => { + const root = await createTestTempDirectory("rea-ordinary-asar-"); await writeFile(join(root, "ordinary.asar"), "ordinary bytes"); + const observed = await inventoryArtifact(root); + expect(artifactParentPaths(observed)).toEqual({ + ".": null, + "ordinary.asar": ".", + }); + expect(artifactOccurrenceAt(observed, "ordinary.asar")).toMatchObject({ + artifact_kind: "resource", + artifact_format: "file", + hash_status: "verified", + }); + }); + + it("keeps filesystem archives with a malformed ASAR header rejected", async () => { + const root = await createTestTempDirectory("rea-malformed-asar-"); + const json = Buffer.from("{not json"); + const header = Buffer.alloc(16); + header.writeUInt32LE(4, 0); + header.writeUInt32LE(json.length + 12, 4); + header.writeUInt32LE(json.length + 8, 8); + header.writeUInt32LE(json.length, 12); + await writeFile(join(root, "broken.asar"), Buffer.concat([header, json])); await expect(inventoryArtifact(root)).rejects.toMatchObject({ reason: "format", }); diff --git a/tests/boundary/filesystem/zipDetection.test.ts b/tests/boundary/filesystem/zipDetection.test.ts index 07336e61a..3464afa59 100644 --- a/tests/boundary/filesystem/zipDetection.test.ts +++ b/tests/boundary/filesystem/zipDetection.test.ts @@ -1,10 +1,18 @@ -import { lstat, writeFile } from "node:fs/promises"; +import { lstat, mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { + TextReader, + Uint8ArrayReader, + Uint8ArrayWriter, + ZipWriter, +} from "@zip.js/zip.js"; import { describe, expect, it } from "vitest"; +import { inventoryArtifact } from "../../../src/artifacts/inventory/ArtifactInventory.js"; import { classifyAndHashRoot } from "../../../src/artifacts/inventory/classify.js"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; +import { artifactOccurrenceAt } from "../../fixtures/artifactEntryOrder.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("ZIP signature detection", () => { @@ -120,3 +128,72 @@ describe("ZIP signature detection", () => { }, ); }); + +describe("nested container family", () => { + const plainZip = async (): Promise => { + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("readme.txt", new TextReader("hello\n")); + return writer.close(); + }; + + it("takes ZIP families from member bytes, not member suffixes", async () => { + const root = await createTestTempDirectory("rea-nested-zip-family-"); + const path = join(root, "outer.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("inner-text.apk", new TextReader("just some text\n")); + await writer.add("inner-real.apk", new Uint8ArrayReader(await plainZip())); + await writer.add("renamed.bin", new Uint8ArrayReader(await plainZip())); + await writer.add("locked.ipa", new TextReader("encrypted bytes"), { + password: "fixture-only", + }); + await writeFile(path, await writer.close()); + const observed = await inventoryArtifact(path); + expect(artifactOccurrenceAt(observed, "inner-text.apk")).toMatchObject({ + artifact_kind: "resource", + artifact_format: "file", + limitations: [], + }); + expect(artifactOccurrenceAt(observed, "inner-real.apk")).toMatchObject({ + artifact_kind: "container", + artifact_format: "apk", + }); + expect(artifactOccurrenceAt(observed, "renamed.bin")).toMatchObject({ + artifact_kind: "container", + artifact_format: "zip", + }); + expect(artifactOccurrenceAt(observed, "locked.ipa")).toMatchObject({ + artifact_kind: "unknown", + artifact_format: "unknown", + hash_status: "unavailable", + limitations: [ + "The path suffix suggests IPA, but these bytes were not read, so the container family is unknown.", + ], + }); + }); + + it.skipIf(process.platform === "win32")( + "does not name a container family for a symlink it did not follow", + async () => { + const root = await createTestTempDirectory("rea-nested-symlink-family-"); + const directory = join(root, "application"); + await mkdir(directory); + await writeFile(join(directory, "real.apk"), await plainZip()); + await symlink("real.apk", join(directory, "link.apk")); + const observed = await inventoryArtifact(directory); + expect(artifactOccurrenceAt(observed, "link.apk")).toMatchObject({ + entry_kind: "symlink", + artifact_kind: "unknown", + artifact_format: "unknown", + hash_status: "not-hashed", + limitations: [ + "Symlink target was not followed or disclosed.", + "The path suffix suggests APK, but these bytes were not read, so the container family is unknown.", + ], + }); + expect(artifactOccurrenceAt(observed, "real.apk")).toMatchObject({ + artifact_kind: "container", + artifact_format: "apk", + }); + }, + ); +}); diff --git a/tests/composition/android/application.test.ts b/tests/composition/android/application.test.ts index 67e2e1c6f..b4260a170 100644 --- a/tests/composition/android/application.test.ts +++ b/tests/composition/android/application.test.ts @@ -119,7 +119,14 @@ describe("Android application projection", () => { projectAndroidApplicationEvidence({ inventory_evidence: [inventory] }), ).toMatchObject({ ok: false, - error: { _tag: "AnalysisInputError" }, + error: { + _tag: "AnalysisInputError", + issues: [ + expect.objectContaining({ + message: expect.stringContaining("root format is zip"), + }), + ], + }, }); }); @@ -168,6 +175,36 @@ describe("Android application projection", () => { }); }); +describe("APK family without a root manifest", () => { + const missingManifest = + "No root AndroidManifest.xml occurrence was observed. The APK family comes from the .apk suffix of a ZIP archive, so these bytes may not be an APK."; + const projectApk = async (entries: Record) => { + const root = await createTestTempDirectory("rea-android-manifest-"); + const path = join(root, "Plain.apk"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + for (const [entry, text] of Object.entries(entries)) + await writer.add(entry, new TextReader(text)); + await writeFile(path, await writer.close()); + const inventory = parseEvidence( + await runProviderAnalysis(path, "inventory_artifact", {}), + ); + return androidApplicationProjectionResultSchema.parse( + requireSuccessfulProjection( + projectAndroidApplicationEvidence({ inventory_evidence: [inventory] }), + ).normalized_result, + ); + }; + + it("says when the APK family rests only on the ZIP filename suffix", async () => { + expect((await projectApk({ "readme.txt": "hello" })).limitations).toContain( + missingManifest, + ); + expect( + (await projectApk({ "AndroidManifest.xml": "manifest" })).limitations, + ).not.toContain(missingManifest); + }); +}); + describe("case-distinct APK resource names", () => { it("inventories and projects an APK whose resource names differ only in case", async () => { const root = await createTestTempDirectory("rea-android-case-");